From 03a5c3d4e22c5d0e0a5fba769cef338fac7c5029 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 26 Nov 2025 19:11:02 +0800 Subject: [PATCH 01/49] tmp save --- .../storageengine/dataregion/DataRegion.java | 43 ++++++- .../dataregion/tsfile/TsFileResource.java | 4 + .../tsfile/evolution/ColumnRename.java | 74 ++++++++++++ .../tsfile/evolution/EvolvedSchema.java | 80 +++++++++++++ .../tsfile/evolution/SchemaEvolution.java | 66 +++++++++++ .../tsfile/evolution/SchemaEvolutionFile.java | 92 +++++++++++++++ .../tsfile/evolution/TableRename.java | 72 ++++++++++++ .../dataregion/tsfile/fileset/TsFileSet.java | 109 ++++++++++++++++++ 8 files changed, 537 insertions(+), 3 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolution.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 0dbe6b8870f9c..734576d9f0dc9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -126,6 +126,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.fileset.TsFileSet; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex; @@ -359,6 +360,8 @@ public class DataRegion implements IDataRegionForQuery { private ILoadDiskSelector ordinaryLoadDiskSelector; private ILoadDiskSelector pipeAndIoTV2LoadDiskSelector; + private Map lastTsFileSetMap = new ConcurrentHashMap<>(); + /** * Construct a database processor. * @@ -644,6 +647,9 @@ private void recover() throws DataRegionException { throw new RuntimeException(e); } } + // ensure that seq and unseq files in the same partition have the same TsFileSet + Map> recoveredTsFileSetMap = new HashMap<>(); + for (Entry> partitionFiles : partitionTmpSeqTsFiles.entrySet()) { Callable asyncRecoverTask = recoverFilesInPartition( @@ -651,7 +657,8 @@ private void recover() throws DataRegionException { dataRegionRecoveryContext, partitionFiles.getValue(), fileTimeIndexMap, - true); + true, + recoveredTsFileSetMap); if (asyncRecoverTask != null) { asyncTsFileResourceRecoverTaskList.add(asyncRecoverTask); } @@ -664,7 +671,8 @@ private void recover() throws DataRegionException { dataRegionRecoveryContext, partitionFiles.getValue(), fileTimeIndexMap, - false); + false, + recoveredTsFileSetMap); if (asyncRecoverTask != null) { asyncTsFileResourceRecoverTaskList.add(asyncRecoverTask); } @@ -987,11 +995,40 @@ private Callable recoverFilesInPartition( DataRegionRecoveryContext context, List resourceList, Map fileTimeIndexMap, - boolean isSeq) { + boolean isSeq, + Map> tsFileSetMap) { + List resourceListForAsyncRecover = new ArrayList<>(); List resourceListForSyncRecover = new ArrayList<>(); Callable asyncRecoverTask = null; for (TsFileResource tsFileResource : resourceList) { + List tsFileSets = tsFileSetMap.computeIfAbsent(partitionId, + pid -> { + File fileSetDir = new File(dataRegionSysDir + File.separator + partitionId + File.separator + TsFileSet.FILE_SET_DIR_NAME); + File[] fileSets = fileSetDir.listFiles(); + if (fileSets == null || fileSets.length == 0) { + return Collections.emptyList(); + } else { + List results = new ArrayList<>(); + for (File fileSet : fileSets) { + TsFileSet tsFileSet; + try { + tsFileSet = new TsFileSet(Long.parseLong(fileSet.getName()), + fileSetDir.getAbsolutePath(), true); + } catch (NumberFormatException e) { + continue; + } + results.add(tsFileSet); + } + return results; + } + }); + if (!tsFileSets.isEmpty()) { + tsFileSets.sort(null); + } + + + tsFileManager.add(tsFileResource, isSeq); if (fileTimeIndexMap.containsKey(tsFileResource.getTsFileID()) && tsFileResource.resourceFileExists()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index 3169cf85ccb4d..13c00febe3cee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -42,6 +42,7 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion; import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Modification; import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.fileset.TsFileSet; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex; @@ -209,6 +210,9 @@ public class TsFileResource implements PersistentResource, Cloneable { private Map>> lastValues; + // TsFileSets this TsFile belongs to + private List tsFileSets; + @TestOnly public TsFileResource() { this.tsFileID = new TsFileID(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java new file mode 100644 index 0000000000000..bc8b5946f794f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import org.apache.tsfile.file.metadata.TsFileMetadata; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.ReadWriteForEncodingUtils; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +/** + * A schema evolution operation that renames a column in a table schema. + */ +public class ColumnRename implements SchemaEvolution { + + private String tableName; + private String nameBefore; + private String nameAfter; + + // for deserialization + public ColumnRename() { + } + + public ColumnRename(String tableName, String nameBefore, String nameAfter) { + this.tableName = tableName.toLowerCase(); + this.nameBefore = nameBefore.toLowerCase(); + this.nameAfter = nameAfter.toLowerCase(); + } + + @Override + public SchemaEvolutionType getEvolutionType() { + return SchemaEvolutionType.COLUMN_RENAME; + } + + @Override + public void applyTo(EvolvedSchema evolvedSchema) { + evolvedSchema.renameColumn(tableName, nameBefore, nameAfter); + } + + @Override + public long serialize(OutputStream stream) throws IOException { + int size = ReadWriteForEncodingUtils.writeVarInt(getEvolutionType().ordinal(), stream); + size += ReadWriteIOUtils.writeVar(tableName, stream); + size += ReadWriteIOUtils.writeVar(nameBefore, stream); + size += ReadWriteIOUtils.writeVar(nameAfter, stream); + return size; + } + + @Override + public void deserialize(InputStream stream) throws IOException { + tableName = ReadWriteIOUtils.readVarIntString(stream); + nameBefore = ReadWriteIOUtils.readVarIntString(stream); + nameAfter = ReadWriteIOUtils.readVarIntString(stream); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java new file mode 100644 index 0000000000000..165c36fc99b10 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import org.apache.tsfile.file.metadata.TableSchema; + +public class EvolvedSchema { + // the evolved table names after applying all schema evolution operations + private final Map originalTableNames = new HashMap<>(); + /** + * the first key is the evolved table name, the second key is the evolved column name, + * and the value is the original column name before any schema evolution. + */ + private final Map> originalColumnNames = new HashMap<>(); + + public void renameTable(String oldTableName, String newTableName) { + if (!originalTableNames.containsKey(oldTableName)) { + originalTableNames.put(newTableName, oldTableName); + // mark the old table name as non-exists + originalTableNames.put(oldTableName, ""); + } else { + // mark the old table name as non-exists + String originalName = originalTableNames.put(oldTableName, ""); + originalTableNames.put(newTableName, originalName); + } + + if (originalColumnNames.containsKey(oldTableName)) { + Map columnMap = originalColumnNames.remove(oldTableName); + originalColumnNames.put(newTableName, columnMap); + } + } + + public void renameColumn(String tableName, String oldColumnName, String newColumnName) { + Map columnNameMap = originalColumnNames.computeIfAbsent(tableName, + t -> new LinkedHashMap<>()); + if (!columnNameMap.containsKey(oldColumnName)) { + columnNameMap.put(newColumnName, oldColumnName); + // mark the old column name as non-exists + columnNameMap.put(oldColumnName, ""); + } else { + String originalName = columnNameMap.put(oldColumnName, ""); + columnNameMap.put(newColumnName, originalName); + } + } + + public String getOriginalTableName(String evolvedTableName) { + return originalTableNames.getOrDefault(evolvedTableName, evolvedTableName); + } + + public String getOriginalColumnName(String tableName, String evolvedColumnName) { + Map columnNameMap = originalColumnNames.get(tableName); + if (columnNameMap == null) { + return evolvedColumnName; + } + return columnNameMap.getOrDefault(evolvedColumnName, evolvedColumnName); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolution.java new file mode 100644 index 0000000000000..7b3b8b8e50d0a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolution.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; + +import java.io.IOException; +import java.io.InputStream; +import org.apache.iotdb.db.utils.io.StreamSerializable; +import org.apache.tsfile.utils.ReadWriteForEncodingUtils; + +/** + * A schema evolution operation that can be applied to a TableSchemaMap. + */ +public interface SchemaEvolution extends StreamSerializable { + + /** + * Apply this schema evolution operation to the given metadata. + * + * @param schema the schema to apply the operation to + */ + void applyTo(EvolvedSchema schema); + + SchemaEvolutionType getEvolutionType(); + + enum SchemaEvolutionType { + TABLE_RENAME, + COLUMN_RENAME + } + + static SchemaEvolution createFrom(InputStream stream) throws IOException { + int type = ReadWriteForEncodingUtils.readVarInt(stream); + if (type < 0 || type > SchemaEvolutionType.values().length) { + throw new IOException("Invalid evolution type: " + type); + } + SchemaEvolution evolution = null; + SchemaEvolutionType evolutionType = SchemaEvolutionType.values()[type]; + switch (evolutionType) { + case TABLE_RENAME: + evolution = new TableRename(); + break; + case COLUMN_RENAME: + evolution = new ColumnRename(); + break; + default: + throw new IOException("Invalid evolution type: " + evolutionType); + } + evolution.deserialize(stream); + return evolution; + } +} \ No newline at end of file diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java new file mode 100644 index 0000000000000..1f10683c7bf85 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.util.Collection; +import org.apache.iotdb.commons.utils.FileUtils; + +/** + * SchemaEvolutionFile manages schema evolutions related to a TsFileSet. + */ +public class SchemaEvolutionFile { + public static final String FILE_SUFFIX = ".sevo"; + + private String filePath; + + public SchemaEvolutionFile(String filePath) { + this.filePath = filePath; + } + + private void recoverFile() throws IOException { + File file = new File(filePath); + if (!file.exists() || file.length() == 0) { + return; + } + + long length = file.length(); + String fileName = file.getName(); + long validLength = Long.parseLong(fileName.substring(fileName.lastIndexOf('.'))); + if (length > validLength) { + try (FileInputStream fis = new FileInputStream(file); + FileChannel fileChannel = fis.getChannel()) { + fileChannel.truncate(validLength); + } + } + } + + + public void append(Collection schemaEvolutions) throws IOException { + recoverFile(); + + try (FileOutputStream fos = new FileOutputStream(filePath, true); + BufferedOutputStream bos = new BufferedOutputStream(fos)) { + for (SchemaEvolution schemaEvolution : schemaEvolutions) { + schemaEvolution.serialize(bos); + } + } + + File originFile = new File(filePath); + long newLength = originFile.length(); + File newFile = new File(originFile.getParentFile(), newLength + FILE_SUFFIX); + FileUtils.moveFileSafe(originFile, newFile); + filePath = newFile.getAbsolutePath(); + } + + public EvolvedSchema readAsSchema() throws IOException { + recoverFile(); + + EvolvedSchema evolvedSchema = new EvolvedSchema(); + try (FileInputStream fis = new FileInputStream(filePath); + BufferedInputStream bis = new BufferedInputStream(fis)) { + while (bis.available() > 0) { + SchemaEvolution evolution = SchemaEvolution.createFrom(bis); + evolution.applyTo(evolvedSchema); + } + } + return evolvedSchema; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java new file mode 100644 index 0000000000000..aac78060b6166 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import org.apache.tsfile.file.metadata.MetadataIndexNode; +import org.apache.tsfile.file.metadata.TsFileMetadata; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.ReadWriteForEncodingUtils; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +/** + * A schema evolution operation that renames a table in a schema map. + */ +public class TableRename implements SchemaEvolution { + + private String nameBefore; + private String nameAfter; + + // for deserialization + public TableRename() { + } + + public TableRename(String nameBefore, String nameAfter) { + this.nameBefore = nameBefore.toLowerCase(); + this.nameAfter = nameAfter.toLowerCase(); + } + + + @Override + public void applyTo(EvolvedSchema evolvedSchema) { + evolvedSchema.renameTable(nameBefore, nameAfter); + } + + @Override + public SchemaEvolutionType getEvolutionType() { + return SchemaEvolutionType.TABLE_RENAME; + } + + @Override + public long serialize(OutputStream stream) throws IOException { + long size = ReadWriteForEncodingUtils.writeVarInt(getEvolutionType().ordinal(), stream); + size += ReadWriteIOUtils.writeVar(nameBefore, stream); + size += ReadWriteIOUtils.writeVar(nameAfter, stream); + return size; + } + + @Override + public void deserialize(InputStream stream) throws IOException { + nameBefore = ReadWriteIOUtils.readVarIntString(stream); + nameAfter = ReadWriteIOUtils.readVarIntString(stream); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java new file mode 100644 index 0000000000000..19ee2a7ed663a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.tsfile.fileset; + + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolutionFile; + +/** + * TsFileSet represents a set of TsFiles in a time partition whose version <= endVersion. + */ +public class TsFileSet implements Comparable { + + public static final String FILE_SET_DIR_NAME = "filesets"; + + private final long endVersion; + private final File fileSetDir; + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private SchemaEvolutionFile schemaEvolutionFile; + + public TsFileSet(long endVersion, String fileSetsDir, boolean recover) { + this.endVersion = endVersion; + this.fileSetDir = new File(fileSetsDir + File.separator + endVersion); + + if (recover) { + recover(); + } else { + //noinspection ResultOfMethodCallIgnored + fileSetDir.mkdirs(); + } + + if (schemaEvolutionFile == null) { + schemaEvolutionFile = new SchemaEvolutionFile(0 + SchemaEvolutionFile.FILE_SUFFIX); + } + } + + private void recover() { + File[] files = fileSetDir.listFiles(); + if (files != null) { + for (File file : files) { + if (file.getName().endsWith(SchemaEvolutionFile.FILE_SUFFIX)) { + schemaEvolutionFile = new SchemaEvolutionFile(file.getAbsolutePath()); + } + } + } + } + + public void appendSchemaEvolution(Collection schemaEvolutions) + throws IOException { + writeLock(); + try { + schemaEvolutionFile.append(schemaEvolutions); + } finally { + writeUnlock(); + } + } + + public EvolvedSchema readEvolvedSchema() throws IOException { + readLock(); + try { + return schemaEvolutionFile.readAsSchema(); + } finally { + readUnlock(); + } + } + + @Override + public int compareTo(TsFileSet o) { + return Long.compare(endVersion, o.endVersion); + } + + public void writeLock() { + lock.writeLock().lock(); + } + + public void readLock() { + lock.readLock().lock(); + } + + public void writeUnlock() { + lock.writeLock().unlock(); + } + + public void readUnlock() { + lock.readLock().unlock(); + } +} From f348433e8036a3a30d173383b06e6f81560cd107 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 27 Nov 2025 14:57:47 +0800 Subject: [PATCH 02/49] Add SchemaEvolutionFIle --- .../storageengine/dataregion/DataRegion.java | 53 ++++---- .../tsfile/evolution/ColumnRename.java | 14 +- .../tsfile/evolution/EvolvedSchema.java | 13 +- .../tsfile/evolution/SchemaEvolution.java | 12 +- .../tsfile/evolution/SchemaEvolutionFile.java | 18 +-- .../tsfile/evolution/TableRename.java | 16 +-- .../dataregion/tsfile/fileset/TsFileSet.java | 10 +- .../evolution/SchemaEvolutionFileTest.java | 123 ++++++++++++++++++ 8 files changed, 189 insertions(+), 70 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFileTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 734576d9f0dc9..cb6803a270af0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -1002,33 +1002,42 @@ private Callable recoverFilesInPartition( List resourceListForSyncRecover = new ArrayList<>(); Callable asyncRecoverTask = null; for (TsFileResource tsFileResource : resourceList) { - List tsFileSets = tsFileSetMap.computeIfAbsent(partitionId, - pid -> { - File fileSetDir = new File(dataRegionSysDir + File.separator + partitionId + File.separator + TsFileSet.FILE_SET_DIR_NAME); - File[] fileSets = fileSetDir.listFiles(); - if (fileSets == null || fileSets.length == 0) { - return Collections.emptyList(); - } else { - List results = new ArrayList<>(); - for (File fileSet : fileSets) { - TsFileSet tsFileSet; - try { - tsFileSet = new TsFileSet(Long.parseLong(fileSet.getName()), - fileSetDir.getAbsolutePath(), true); - } catch (NumberFormatException e) { - continue; + List tsFileSets = + tsFileSetMap.computeIfAbsent( + partitionId, + pid -> { + File fileSetDir = + new File( + dataRegionSysDir + + File.separator + + partitionId + + File.separator + + TsFileSet.FILE_SET_DIR_NAME); + File[] fileSets = fileSetDir.listFiles(); + if (fileSets == null || fileSets.length == 0) { + return Collections.emptyList(); + } else { + List results = new ArrayList<>(); + for (File fileSet : fileSets) { + TsFileSet tsFileSet; + try { + tsFileSet = + new TsFileSet( + Long.parseLong(fileSet.getName()), + fileSetDir.getAbsolutePath(), + true); + } catch (NumberFormatException e) { + continue; + } + results.add(tsFileSet); + } + return results; } - results.add(tsFileSet); - } - return results; - } - }); + }); if (!tsFileSets.isEmpty()) { tsFileSets.sort(null); } - - tsFileManager.add(tsFileResource, isSeq); if (fileTimeIndexMap.containsKey(tsFileResource.getTsFileID()) && tsFileResource.resourceFileExists()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java index bc8b5946f794f..fc363247b13b0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java @@ -19,17 +19,14 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; +import org.apache.tsfile.utils.ReadWriteForEncodingUtils; +import org.apache.tsfile.utils.ReadWriteIOUtils; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import org.apache.tsfile.file.metadata.TsFileMetadata; -import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.utils.ReadWriteForEncodingUtils; -import org.apache.tsfile.utils.ReadWriteIOUtils; -/** - * A schema evolution operation that renames a column in a table schema. - */ +/** A schema evolution operation that renames a column in a table schema. */ public class ColumnRename implements SchemaEvolution { private String tableName; @@ -37,8 +34,7 @@ public class ColumnRename implements SchemaEvolution { private String nameAfter; // for deserialization - public ColumnRename() { - } + public ColumnRename() {} public ColumnRename(String tableName, String nameBefore, String nameAfter) { this.tableName = tableName.toLowerCase(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java index 165c36fc99b10..452f79b720b87 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java @@ -20,19 +20,16 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import org.apache.tsfile.file.metadata.TableSchema; public class EvolvedSchema { // the evolved table names after applying all schema evolution operations private final Map originalTableNames = new HashMap<>(); + /** - * the first key is the evolved table name, the second key is the evolved column name, - * and the value is the original column name before any schema evolution. + * the first key is the evolved table name, the second key is the evolved column name, and the + * value is the original column name before any schema evolution. */ private final Map> originalColumnNames = new HashMap<>(); @@ -54,8 +51,8 @@ public void renameTable(String oldTableName, String newTableName) { } public void renameColumn(String tableName, String oldColumnName, String newColumnName) { - Map columnNameMap = originalColumnNames.computeIfAbsent(tableName, - t -> new LinkedHashMap<>()); + Map columnNameMap = + originalColumnNames.computeIfAbsent(tableName, t -> new LinkedHashMap<>()); if (!columnNameMap.containsKey(oldColumnName)) { columnNameMap.put(newColumnName, oldColumnName); // mark the old column name as non-exists diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolution.java index 7b3b8b8e50d0a..5a670879bebe8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolution.java @@ -19,14 +19,14 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; -import java.io.IOException; -import java.io.InputStream; import org.apache.iotdb.db.utils.io.StreamSerializable; + import org.apache.tsfile.utils.ReadWriteForEncodingUtils; -/** - * A schema evolution operation that can be applied to a TableSchemaMap. - */ +import java.io.IOException; +import java.io.InputStream; + +/** A schema evolution operation that can be applied to a TableSchemaMap. */ public interface SchemaEvolution extends StreamSerializable { /** @@ -63,4 +63,4 @@ static SchemaEvolution createFrom(InputStream stream) throws IOException { evolution.deserialize(stream); return evolution; } -} \ No newline at end of file +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java index 1f10683c7bf85..ac9e94bdc5ff0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; +import org.apache.iotdb.commons.utils.FileUtils; + import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; @@ -27,11 +29,8 @@ import java.io.IOException; import java.nio.channels.FileChannel; import java.util.Collection; -import org.apache.iotdb.commons.utils.FileUtils; -/** - * SchemaEvolutionFile manages schema evolutions related to a TsFileSet. - */ +/** SchemaEvolutionFile manages schema evolutions related to a TsFileSet. */ public class SchemaEvolutionFile { public static final String FILE_SUFFIX = ".sevo"; @@ -49,21 +48,24 @@ private void recoverFile() throws IOException { long length = file.length(); String fileName = file.getName(); - long validLength = Long.parseLong(fileName.substring(fileName.lastIndexOf('.'))); + long validLength = parseValidLength(fileName); if (length > validLength) { - try (FileInputStream fis = new FileInputStream(file); + try (FileOutputStream fis = new FileOutputStream(file, true); FileChannel fileChannel = fis.getChannel()) { fileChannel.truncate(validLength); } } } + public static long parseValidLength(String fileName) { + return Long.parseLong(fileName.substring(0, fileName.lastIndexOf('.'))); + } public void append(Collection schemaEvolutions) throws IOException { recoverFile(); try (FileOutputStream fos = new FileOutputStream(filePath, true); - BufferedOutputStream bos = new BufferedOutputStream(fos)) { + BufferedOutputStream bos = new BufferedOutputStream(fos)) { for (SchemaEvolution schemaEvolution : schemaEvolutions) { schemaEvolution.serialize(bos); } @@ -81,7 +83,7 @@ public EvolvedSchema readAsSchema() throws IOException { EvolvedSchema evolvedSchema = new EvolvedSchema(); try (FileInputStream fis = new FileInputStream(filePath); - BufferedInputStream bis = new BufferedInputStream(fis)) { + BufferedInputStream bis = new BufferedInputStream(fis)) { while (bis.available() > 0) { SchemaEvolution evolution = SchemaEvolution.createFrom(bis); evolution.applyTo(evolvedSchema); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java index aac78060b6166..02f7f0cfd3e79 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java @@ -19,33 +19,27 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; +import org.apache.tsfile.utils.ReadWriteForEncodingUtils; +import org.apache.tsfile.utils.ReadWriteIOUtils; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import org.apache.tsfile.file.metadata.MetadataIndexNode; -import org.apache.tsfile.file.metadata.TsFileMetadata; -import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.utils.ReadWriteForEncodingUtils; -import org.apache.tsfile.utils.ReadWriteIOUtils; -/** - * A schema evolution operation that renames a table in a schema map. - */ +/** A schema evolution operation that renames a table in a schema map. */ public class TableRename implements SchemaEvolution { private String nameBefore; private String nameAfter; // for deserialization - public TableRename() { - } + public TableRename() {} public TableRename(String nameBefore, String nameAfter) { this.nameBefore = nameBefore.toLowerCase(); this.nameAfter = nameAfter.toLowerCase(); } - @Override public void applyTo(EvolvedSchema evolvedSchema) { evolvedSchema.renameTable(nameBefore, nameAfter); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java index 19ee2a7ed663a..9d75031d30f43 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java @@ -19,18 +19,16 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.fileset; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolutionFile; import java.io.File; import java.io.IOException; import java.util.Collection; import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolutionFile; -/** - * TsFileSet represents a set of TsFiles in a time partition whose version <= endVersion. - */ +/** TsFileSet represents a set of TsFiles in a time partition whose version <= endVersion. */ public class TsFileSet implements Comparable { public static final String FILE_SET_DIR_NAME = "filesets"; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFileTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFileTest.java new file mode 100644 index 0000000000000..d4386dd8172f7 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFileTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; + +import java.io.FileOutputStream; +import org.apache.iotdb.db.utils.constant.TestConstant; + +import org.junit.After; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +@SuppressWarnings("ResultOfMethodCallIgnored") +public class SchemaEvolutionFileTest { + + @After + public void tearDown() throws Exception { + clearSchemaEvolutionFile(); + } + + @Test + public void testSchemaEvolutionFile() throws IOException { + String filePath = TestConstant.BASE_OUTPUT_PATH + File.separator + "0.sevo"; + + SchemaEvolutionFile schemaEvolutionFile = new SchemaEvolutionFile(filePath); + + // t1 -> t2, t2.s1 -> t2.s2, t3 -> t1 + List schemaEvolutionList = + Arrays.asList( + new TableRename("t1", "t2"), + new ColumnRename("t2", "s1", "s2"), + new TableRename("t3", "t1")); + schemaEvolutionFile.append(schemaEvolutionList); + + EvolvedSchema evolvedSchema = schemaEvolutionFile.readAsSchema(); + assertEquals("t1", evolvedSchema.getOriginalTableName("t2")); + assertEquals("s1", evolvedSchema.getOriginalColumnName("t2", "s2")); + assertEquals("t3", evolvedSchema.getOriginalTableName("t1")); + // not evolved, should remain the same + assertEquals("t4", evolvedSchema.getOriginalTableName("t4")); + assertEquals("s3", evolvedSchema.getOriginalColumnName("t2", "s3")); + + // t1 -> t2 -> t3, t2.s1 -> t2.s2 -> t3.s1, t3 -> t1 -> t2 + schemaEvolutionList = + Arrays.asList( + new TableRename("t2", "t3"), + new ColumnRename("t3", "s2", "s1"), + new TableRename("t1", "t2")); + schemaEvolutionFile.append(schemaEvolutionList); + evolvedSchema = schemaEvolutionFile.readAsSchema(); + assertEquals("t1", evolvedSchema.getOriginalTableName("t3")); + assertEquals("s1", evolvedSchema.getOriginalColumnName("t3", "s1")); + assertEquals("t3", evolvedSchema.getOriginalTableName("t2")); + // not evolved, should remain the same + assertEquals("t4", evolvedSchema.getOriginalTableName("t4")); + assertEquals("s3", evolvedSchema.getOriginalColumnName("t2", "s3")); + } + + private void clearSchemaEvolutionFile() { + File dir = new File(TestConstant.BASE_OUTPUT_PATH); + File[] files = dir.listFiles(f -> f.getName().endsWith(SchemaEvolutionFile.FILE_SUFFIX)); + if (files != null) { + for (File file : files) { + file.delete(); + } + } + } + + @Test + public void testRecover() throws IOException { + String filePath = TestConstant.BASE_OUTPUT_PATH + File.separator + "0.sevo"; + + SchemaEvolutionFile schemaEvolutionFile = new SchemaEvolutionFile(filePath); + List schemaEvolutionList = + Arrays.asList( + new TableRename("t1", "t2"), + new ColumnRename("t2", "s1", "s2"), + new TableRename("t3", "t1")); + schemaEvolutionFile.append(schemaEvolutionList); + + File dir = new File(TestConstant.BASE_OUTPUT_PATH); + File[] files = dir.listFiles(f -> f.getName().endsWith(SchemaEvolutionFile.FILE_SUFFIX)); + assertNotNull(files); + assertEquals(1, files.length); + assertEquals(24, SchemaEvolutionFile.parseValidLength(files[0].getName())); + + try (FileOutputStream fileOutputStream = new FileOutputStream(files[0], true)) { + fileOutputStream.write(new byte[100]); + } + + schemaEvolutionFile = new SchemaEvolutionFile(files[0].getAbsolutePath()); + EvolvedSchema evolvedSchema = schemaEvolutionFile.readAsSchema(); + assertEquals("t1", evolvedSchema.getOriginalTableName("t2")); + assertEquals("s1", evolvedSchema.getOriginalColumnName("t2", "s2")); + assertEquals("t3", evolvedSchema.getOriginalTableName("t1")); + // not evolved, should remain the same + assertEquals("t4", evolvedSchema.getOriginalTableName("t4")); + assertEquals("s3", evolvedSchema.getOriginalColumnName("t2", "s3")); + } +} From eb1df4a5f9df0cc52608931f44e844299802b1f9 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 27 Nov 2025 16:26:35 +0800 Subject: [PATCH 03/49] EvolvedSchema may merge --- .../storageengine/dataregion/DataRegion.java | 138 +++++++++++++----- .../dataregion/tsfile/TsFileManager.java | 22 +++ .../dataregion/tsfile/TsFileResource.java | 8 + .../tsfile/evolution/EvolvedSchema.java | 69 ++++++++- .../dataregion/tsfile/fileset/TsFileSet.java | 18 ++- .../tsfile/evolution/EvolvedSchemaTest.java | 57 ++++++++ 6 files changed, 265 insertions(+), 47 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchemaTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index cb6803a270af0..b1520c56ec9c0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -126,6 +126,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; import org.apache.iotdb.db.storageengine.dataregion.tsfile.fileset.TsFileSet; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; @@ -648,7 +649,7 @@ private void recover() throws DataRegionException { } } // ensure that seq and unseq files in the same partition have the same TsFileSet - Map> recoveredTsFileSetMap = new HashMap<>(); + Map> recoveredPartitionTsFileSetMap = new HashMap<>(); for (Entry> partitionFiles : partitionTmpSeqTsFiles.entrySet()) { Callable asyncRecoverTask = @@ -658,7 +659,7 @@ private void recover() throws DataRegionException { partitionFiles.getValue(), fileTimeIndexMap, true, - recoveredTsFileSetMap); + recoveredPartitionTsFileSetMap); if (asyncRecoverTask != null) { asyncTsFileResourceRecoverTaskList.add(asyncRecoverTask); } @@ -672,7 +673,7 @@ private void recover() throws DataRegionException { partitionFiles.getValue(), fileTimeIndexMap, false, - recoveredTsFileSetMap); + recoveredPartitionTsFileSetMap); if (asyncRecoverTask != null) { asyncTsFileResourceRecoverTaskList.add(asyncRecoverTask); } @@ -990,52 +991,74 @@ private void recoverSealedTsFiles( } } + private String getFileSetsDir(long partitionId) { + return dataRegionSysDir + + File.separator + + partitionId + + File.separator + + TsFileSet.FILE_SET_DIR_NAME; + } + + private List recoverTsFileSets( + long partitionId, + Map> tsFileSetMap + ) { + List tsFileSets = + tsFileSetMap.computeIfAbsent( + partitionId, + pid -> { + File fileSetDir = + new File(getFileSetsDir(partitionId)); + File[] fileSets = fileSetDir.listFiles(); + if (fileSets == null || fileSets.length == 0) { + return Collections.emptyList(); + } else { + List results = new ArrayList<>(); + for (File fileSet : fileSets) { + TsFileSet tsFileSet; + try { + tsFileSet = + new TsFileSet( + Long.parseLong(fileSet.getName()), + fileSetDir.getAbsolutePath(), + true); + } catch (NumberFormatException e) { + continue; + } + results.add(tsFileSet); + } + return results; + } + }); + if (!tsFileSets.isEmpty()) { + tsFileSets.sort(null); + lastTsFileSetMap.put(partitionId, tsFileSets.get(tsFileSets.size() - 1)); + } + return tsFileSets; + } + + private Callable recoverFilesInPartition( long partitionId, DataRegionRecoveryContext context, List resourceList, Map fileTimeIndexMap, boolean isSeq, - Map> tsFileSetMap) { + Map> partitionTsFileSetMap) { List resourceListForAsyncRecover = new ArrayList<>(); List resourceListForSyncRecover = new ArrayList<>(); Callable asyncRecoverTask = null; for (TsFileResource tsFileResource : resourceList) { - List tsFileSets = - tsFileSetMap.computeIfAbsent( - partitionId, - pid -> { - File fileSetDir = - new File( - dataRegionSysDir - + File.separator - + partitionId - + File.separator - + TsFileSet.FILE_SET_DIR_NAME); - File[] fileSets = fileSetDir.listFiles(); - if (fileSets == null || fileSets.length == 0) { - return Collections.emptyList(); - } else { - List results = new ArrayList<>(); - for (File fileSet : fileSets) { - TsFileSet tsFileSet; - try { - tsFileSet = - new TsFileSet( - Long.parseLong(fileSet.getName()), - fileSetDir.getAbsolutePath(), - true); - } catch (NumberFormatException e) { - continue; - } - results.add(tsFileSet); - } - return results; - } - }); - if (!tsFileSets.isEmpty()) { - tsFileSets.sort(null); + List tsFileSets = recoverTsFileSets(partitionId, partitionTsFileSetMap); + long fileVersion = tsFileResource.getTsFileID().fileVersion; + int i = Collections.binarySearch(tsFileSets, TsFileSet.comparatorKey(fileVersion)); + if (i < 0) { + i = -i; + } + if (i < tsFileSets.size()) { + List containedSets = tsFileSets.subList(i, tsFileSets.size()); + containedSets.forEach(tsFileResource::addFileSet); } tsFileManager.add(tsFileResource, isSeq); @@ -1158,6 +1181,45 @@ private int compareFileName(File o1, File o2) { } } + private TsFileSet createNewFileSet(long maxVersion, long partitionId) { + TsFileSet newSet = new TsFileSet(maxVersion, getFileSetsDir(partitionId), false); + tsFileManager.addTsFileSet(newSet, partitionId); + return newSet; + } + + public void applySchemaEvolution(List schemaEvolutions) { + long startTime = System.nanoTime(); + writeLock("InsertRow"); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleLockCost(System.nanoTime() - startTime); + try { + if (deleted) { + return; + } + + syncCloseAllWorkingTsFileProcessors(); + + for (Entry partitionVersionEntry : partitionMaxFileVersions.entrySet()) { + long partitionId = partitionVersionEntry.getKey(); + long maxVersion = partitionVersionEntry.getValue(); + lastTsFileSetMap.compute(partitionId, (pid, lastSet) -> { + if (lastSet == null) { + lastSet = createNewFileSet(maxVersion, partitionId); + } else if (lastSet.getEndVersion() < maxVersion) { + lastSet = createNewFileSet(maxVersion, partitionId); + } + try { + lastSet.appendSchemaEvolution(schemaEvolutions); + } catch (IOException e) { + logger.error("Cannot append schema evolutions to fileSets in partition {}-{}", dataRegionId, partitionId, e); + } + return lastSet; + }); + } + } finally { + writeUnlock(); + } + } + /** * insert one row of data. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java index b7c1ba2c14fb4..4466668ad5fa3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.storageengine.dataregion.modification.ModFileManagement; import org.apache.iotdb.db.storageengine.dataregion.modification.PartitionLevelModFileManager; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.fileset.TsFileSet; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndexCacheRecorder; import org.apache.iotdb.db.storageengine.rescon.memory.TsFileResourceManager; @@ -507,4 +508,25 @@ public long getMaxFileTimestampOfUnSequenceFile() { } return maxFileTimestamp; } + + public void addTsFileSet(TsFileSet newSet, long partitionId) { + writeLock("addTsFileSet"); + try { + TsFileResourceList tsFileResources = sequenceFiles.get(partitionId); + if (tsFileResources != null) { + for (TsFileResource tsFileResource : tsFileResources) { + tsFileResource.addFileSet(newSet); + } + } + + tsFileResources = unsequenceFiles.get(partitionId); + if (tsFileResources != null) { + for (TsFileResource tsFileResource : tsFileResources) { + tsFileResource.addFileSet(newSet); + } + } + } finally { + writeUnlock(); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index 13c00febe3cee..53b07166b9416 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -1630,4 +1630,12 @@ public TsFileResource shallowClone() { public TsFileResource shallowCloneForNative() throws CloneNotSupportedException { return (TsFileResource) clone(); } + + public void addFileSet(TsFileSet tsFileSet) { + tsFileSets.add(tsFileSet); + } + + public List getTsFileSets() { + return tsFileSets; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java index 452f79b720b87..f56a30bd1e300 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java @@ -22,25 +22,25 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; public class EvolvedSchema { // the evolved table names after applying all schema evolution operations - private final Map originalTableNames = new HashMap<>(); + private Map originalTableNames = new LinkedHashMap<>(); /** * the first key is the evolved table name, the second key is the evolved column name, and the * value is the original column name before any schema evolution. */ - private final Map> originalColumnNames = new HashMap<>(); + private Map> originalColumnNames = new LinkedHashMap<>(); public void renameTable(String oldTableName, String newTableName) { if (!originalTableNames.containsKey(oldTableName)) { originalTableNames.put(newTableName, oldTableName); - // mark the old table name as non-exists - originalTableNames.put(oldTableName, ""); } else { // mark the old table name as non-exists - String originalName = originalTableNames.put(oldTableName, ""); + String originalName = originalTableNames.remove(oldTableName); originalTableNames.put(newTableName, originalName); } @@ -55,10 +55,8 @@ public void renameColumn(String tableName, String oldColumnName, String newColum originalColumnNames.computeIfAbsent(tableName, t -> new LinkedHashMap<>()); if (!columnNameMap.containsKey(oldColumnName)) { columnNameMap.put(newColumnName, oldColumnName); - // mark the old column name as non-exists - columnNameMap.put(oldColumnName, ""); } else { - String originalName = columnNameMap.put(oldColumnName, ""); + String originalName = columnNameMap.remove(oldColumnName); columnNameMap.put(newColumnName, originalName); } } @@ -74,4 +72,59 @@ public String getOriginalColumnName(String tableName, String evolvedColumnName) } return columnNameMap.getOrDefault(evolvedColumnName, evolvedColumnName); } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + EvolvedSchema that = (EvolvedSchema) o; + return Objects.equals(originalTableNames, that.originalTableNames) + && Objects.equals(originalColumnNames, that.originalColumnNames); + } + + @Override + public int hashCode() { + return Objects.hash(originalTableNames, originalColumnNames); + } + + @Override + public String toString() { + return "EvolvedSchema{" + + "originalTableNames=" + originalTableNames + + ", originalColumnNames=" + originalColumnNames + + '}'; + } + + public static EvolvedSchema deepCopy(EvolvedSchema evolvedSchema) { + EvolvedSchema newEvolvedSchema = new EvolvedSchema(); + newEvolvedSchema.originalTableNames = new HashMap<>(evolvedSchema.originalTableNames); + newEvolvedSchema.originalColumnNames = new HashMap<>(evolvedSchema.originalColumnNames); + return newEvolvedSchema; + } + + public static EvolvedSchema merge(EvolvedSchema oldSchema, EvolvedSchema newSchema) { + if (oldSchema == null) { + return newSchema; + } + if (newSchema == null) { + return oldSchema; + } + + EvolvedSchema mergedSchema = deepCopy(oldSchema); + for (Entry finalOriginalTableName : newSchema.originalTableNames.entrySet()) { + mergedSchema.renameTable(finalOriginalTableName.getValue(), finalOriginalTableName.getKey()); + } + for (Entry> finalTableNameColumnNameMapEntry : newSchema.originalColumnNames.entrySet()) { + for (Entry finalColNameOriginalColNameEntry : finalTableNameColumnNameMapEntry.getValue() + .entrySet()) { + String finalTableName = finalTableNameColumnNameMapEntry.getKey(); + String finalColName = finalColNameOriginalColNameEntry.getKey(); + String originalColName = finalColNameOriginalColNameEntry.getValue(); + mergedSchema.renameColumn(finalTableName, originalColName, finalColName); + } + } + + return mergedSchema; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java index 9d75031d30f43..d3271d98b45e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java @@ -35,12 +35,24 @@ public class TsFileSet implements Comparable { private final long endVersion; private final File fileSetDir; - private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private final ReentrantReadWriteLock lock; private SchemaEvolutionFile schemaEvolutionFile; + // only As comparator key + private TsFileSet(long endVersion) { + this.endVersion = endVersion; + this.fileSetDir = null; + this.lock = null; + } + + public static TsFileSet comparatorKey(long endVersion) { + return new TsFileSet(endVersion); + } + public TsFileSet(long endVersion, String fileSetsDir, boolean recover) { this.endVersion = endVersion; this.fileSetDir = new File(fileSetsDir + File.separator + endVersion); + this.lock = new ReentrantReadWriteLock(); if (recover) { recover(); @@ -104,4 +116,8 @@ public void writeUnlock() { public void readUnlock() { lock.readLock().unlock(); } + + public long getEndVersion() { + return endVersion; + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchemaTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchemaTest.java new file mode 100644 index 0000000000000..d9c76be4710fa --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchemaTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; + +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; +import java.util.List; +import org.junit.Test; + +public class EvolvedSchemaTest { + + @Test + public void testMerge() { + // t1 -> t2, t2.s1 -> t2.s2, t3 -> t1 + List schemaEvolutionList = + Arrays.asList( + new TableRename("t1", "t2"), + new ColumnRename("t2", "s1", "s2"), + new TableRename("t3", "t1")); + EvolvedSchema oldSchema = new EvolvedSchema(); + EvolvedSchema allSchema = new EvolvedSchema(); + schemaEvolutionList.forEach(schemaEvolution -> schemaEvolution.applyTo(oldSchema)); + schemaEvolutionList.forEach(schemaEvolution -> schemaEvolution.applyTo(allSchema)); + + // t1 -> t2 -> t3, t2.s1 -> t2.s2 -> t3.s1, t3 -> t1 -> t2 + schemaEvolutionList = + Arrays.asList( + new TableRename("t2", "t3"), + new ColumnRename("t3", "s2", "s1"), + new TableRename("t1", "t2")); + EvolvedSchema newSchema = new EvolvedSchema(); + schemaEvolutionList.forEach(schemaEvolution -> schemaEvolution.applyTo(newSchema)); + schemaEvolutionList.forEach(schemaEvolution -> schemaEvolution.applyTo(allSchema)); + + EvolvedSchema mergedShema = EvolvedSchema.merge(oldSchema, newSchema); + + assertEquals(allSchema, mergedShema); + } +} \ No newline at end of file From f4d9110622ef24069ed263b945b675a748c060ab Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 27 Nov 2025 17:15:17 +0800 Subject: [PATCH 04/49] add mergedSchema --- .../storageengine/dataregion/DataRegion.java | 4 +- .../dataregion/tsfile/TsFileResource.java | 12 +++++ .../tsfile/evolution/EvolvedSchema.java | 53 +++++++++++-------- .../tsfile/evolution/SchemaEvolutionFile.java | 15 ++++-- 4 files changed, 58 insertions(+), 26 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index b1520c56ec9c0..4a28034c84ee0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -1189,13 +1189,13 @@ private TsFileSet createNewFileSet(long maxVersion, long partitionId) { public void applySchemaEvolution(List schemaEvolutions) { long startTime = System.nanoTime(); - writeLock("InsertRow"); + writeLock("applySchemaEvolution"); PERFORMANCE_OVERVIEW_METRICS.recordScheduleLockCost(System.nanoTime() - startTime); try { if (deleted) { return; } - + DataNodeTableCache.getInstance().invalid(databaseName); syncCloseAllWorkingTsFileProcessors(); for (Entry partitionVersionEntry : partitionMaxFileVersions.entrySet()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index 53b07166b9416..d26172122a69a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile; +import java.util.stream.Collectors; import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.consensus.index.ProgressIndexType; import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; @@ -42,6 +43,7 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion; import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Modification; import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; import org.apache.iotdb.db.storageengine.dataregion.tsfile.fileset.TsFileSet; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; @@ -1638,4 +1640,14 @@ public void addFileSet(TsFileSet tsFileSet) { public List getTsFileSets() { return tsFileSets; } + + public EvolvedSchema getMergedSchema() throws IOException { + List list = new ArrayList<>(); + for (TsFileSet fileSet : getTsFileSets()) { + EvolvedSchema readEvolvedSchema = fileSet.readEvolvedSchema(); + list.add(readEvolvedSchema); + } + + return EvolvedSchema.merge(list.toArray(new EvolvedSchema[0])); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java index f56a30bd1e300..13d6bc6fdeaa5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java @@ -98,33 +98,44 @@ public String toString() { public static EvolvedSchema deepCopy(EvolvedSchema evolvedSchema) { EvolvedSchema newEvolvedSchema = new EvolvedSchema(); - newEvolvedSchema.originalTableNames = new HashMap<>(evolvedSchema.originalTableNames); - newEvolvedSchema.originalColumnNames = new HashMap<>(evolvedSchema.originalColumnNames); + newEvolvedSchema.originalTableNames = new LinkedHashMap<>(evolvedSchema.originalTableNames); + newEvolvedSchema.originalColumnNames = new LinkedHashMap<>(evolvedSchema.originalColumnNames); return newEvolvedSchema; } - public static EvolvedSchema merge(EvolvedSchema oldSchema, EvolvedSchema newSchema) { - if (oldSchema == null) { - return newSchema; - } - if (newSchema == null) { - return oldSchema; - } + public static EvolvedSchema merge(EvolvedSchema... schemas) { + EvolvedSchema firstNotNullSchema = null; + int i = 0; + for (; i < schemas.length; i++) { + if (schemas[i] != null) { + firstNotNullSchema = schemas[i]; + i++; + break; + } + } - EvolvedSchema mergedSchema = deepCopy(oldSchema); - for (Entry finalOriginalTableName : newSchema.originalTableNames.entrySet()) { - mergedSchema.renameTable(finalOriginalTableName.getValue(), finalOriginalTableName.getKey()); - } - for (Entry> finalTableNameColumnNameMapEntry : newSchema.originalColumnNames.entrySet()) { - for (Entry finalColNameOriginalColNameEntry : finalTableNameColumnNameMapEntry.getValue() - .entrySet()) { - String finalTableName = finalTableNameColumnNameMapEntry.getKey(); - String finalColName = finalColNameOriginalColNameEntry.getKey(); - String originalColName = finalColNameOriginalColNameEntry.getValue(); - mergedSchema.renameColumn(finalTableName, originalColName, finalColName); + if (firstNotNullSchema == null) { + return null; + } + EvolvedSchema mergedSchema = deepCopy(firstNotNullSchema); + + for (; i < schemas.length; i++) { + if (schemas[i] != null) { + EvolvedSchema newSchema = schemas[i]; + for (Entry finalOriginalTableName : newSchema.originalTableNames.entrySet()) { + mergedSchema.renameTable(finalOriginalTableName.getValue(), finalOriginalTableName.getKey()); + } + for (Entry> finalTableNameColumnNameMapEntry : newSchema.originalColumnNames.entrySet()) { + for (Entry finalColNameOriginalColNameEntry : finalTableNameColumnNameMapEntry.getValue() + .entrySet()) { + String finalTableName = finalTableNameColumnNameMapEntry.getKey(); + String finalColName = finalColNameOriginalColNameEntry.getKey(); + String originalColName = finalColNameOriginalColNameEntry.getValue(); + mergedSchema.renameColumn(finalTableName, originalColName, finalColName); + } + } } } - return mergedSchema; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java index ac9e94bdc5ff0..e90c1ac32cac6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java @@ -40,10 +40,15 @@ public SchemaEvolutionFile(String filePath) { this.filePath = filePath; } - private void recoverFile() throws IOException { + /** + * Recover the SchemaEvolutionFile if it is broken. + * @return true if the file exists false otherwise + * @throws IOException if the file cannot be recovered + */ + private boolean recoverFile() throws IOException { File file = new File(filePath); if (!file.exists() || file.length() == 0) { - return; + return false; } long length = file.length(); @@ -55,6 +60,7 @@ private void recoverFile() throws IOException { fileChannel.truncate(validLength); } } + return true; } public static long parseValidLength(String fileName) { @@ -79,7 +85,10 @@ public void append(Collection schemaEvolutions) throws IOExcept } public EvolvedSchema readAsSchema() throws IOException { - recoverFile(); + boolean exists = recoverFile(); + if (!exists) { + return null; + } EvolvedSchema evolvedSchema = new EvolvedSchema(); try (FileInputStream fis = new FileInputStream(filePath); From 06a848fd846e923c66de3fb1280864c222913c66 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 27 Nov 2025 18:43:08 +0800 Subject: [PATCH 05/49] rewrite lastFlushTimeMap --- .gitignore | 1 + .../storageengine/dataregion/DataRegion.java | 18 ++- .../dataregion/DeviceLastFlushTime.java | 13 ++ .../dataregion/HashLastFlushTimeMap.java | 30 +++- .../dataregion/ILastFlushTime.java | 3 + .../dataregion/ILastFlushTimeMap.java | 3 + .../dataregion/PartitionLastFlushTime.java | 6 + .../dataregion/tsfile/TsFileResource.java | 5 +- .../tsfile/evolution/EvolvedSchema.java | 42 +++++- .../tsfile/evolution/TableRename.java | 38 +++++ .../dataregion/DataRegionTest.java | 136 +++++++++++++----- 11 files changed, 242 insertions(+), 53 deletions(-) diff --git a/.gitignore b/.gitignore index a489dd048da5f..e4c7c98239322 100644 --- a/.gitignore +++ b/.gitignore @@ -124,3 +124,4 @@ iotdb-core/tsfile/src/main/antlr4/org/apache/tsfile/parser/gen/ .mvn/.gradle-enterprise/ .mvn/.develocity/ .run/ +*.sevo diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 4a28034c84ee0..f288e81f82c2e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -126,6 +126,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; import org.apache.iotdb.db.storageengine.dataregion.tsfile.fileset.TsFileSet; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; @@ -167,6 +168,7 @@ import org.apache.tsfile.external.commons.io.FileUtils; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.fileSystem.FSFactoryProducer; import org.apache.tsfile.fileSystem.FSType; import org.apache.tsfile.fileSystem.fsFactory.FSFactory; @@ -1196,6 +1198,8 @@ public void applySchemaEvolution(List schemaEvolutions) { return; } DataNodeTableCache.getInstance().invalid(databaseName); + schemaEvolutions.forEach(lastFlushTimeMap::accept); + syncCloseAllWorkingTsFileProcessors(); for (Entry partitionVersionEntry : partitionMaxFileVersions.entrySet()) { @@ -2607,6 +2611,7 @@ public void writeUnlock() { * @param tsFileResources includes sealed and unsealed tsfile resources * @return fill unsealed tsfile resources with memory data and ChunkMetadataList of data in disk */ + @SuppressWarnings("SuspiciousSystemArraycopy") private List getFileResourceListForQuery( Collection tsFileResources, List pathList, @@ -2619,7 +2624,18 @@ private List getFileResourceListForQuery( List tsfileResourcesForQuery = new ArrayList<>(); for (TsFileResource tsFileResource : tsFileResources) { - if (!tsFileResource.isSatisfied(singleDeviceId, globalTimeFilter, isSeq, context.isDebug())) { + EvolvedSchema evolvedSchema; + try { + evolvedSchema = tsFileResource.getMergedEvolvedSchema(); + } catch (IOException e) { + throw new MetadataException(e); + } + IDeviceID deviceIdBackThen = singleDeviceId; + if (evolvedSchema != null) { + deviceIdBackThen = evolvedSchema.rewriteDeviceId(singleDeviceId); + } + + if (!tsFileResource.isSatisfied(deviceIdBackThen, globalTimeFilter, isSeq, context.isDebug())) { continue; } try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DeviceLastFlushTime.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DeviceLastFlushTime.java index f02044b041472..e7c13f149a9fd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DeviceLastFlushTime.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DeviceLastFlushTime.java @@ -19,6 +19,10 @@ package org.apache.iotdb.db.storageengine.dataregion; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.TableRename; import org.apache.tsfile.file.metadata.IDeviceID; import java.util.HashMap; @@ -53,4 +57,13 @@ public ILastFlushTime degradeLastFlushTime() { Map getDeviceLastFlushTimeMap() { return deviceLastFlushTimeMap; } + + @Override + public void accept(SchemaEvolution schemaEvolution) { + if (!(schemaEvolution instanceof TableRename)) { + return; + } + TableRename tableRename = (TableRename) schemaEvolution; + tableRename.rewriteMap(deviceLastFlushTimeMap); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/HashLastFlushTimeMap.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/HashLastFlushTimeMap.java index 3f0abfd3e2481..cf8badd6d089c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/HashLastFlushTimeMap.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/HashLastFlushTimeMap.java @@ -19,9 +19,16 @@ package org.apache.iotdb.db.storageengine.dataregion; +import java.util.List; +import java.util.Map.Entry; +import java.util.stream.Collectors; import org.apache.iotdb.db.storageengine.StorageEngine; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.TableRename; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,7 +77,7 @@ public void updateMultiDeviceFlushedTime( timePartitionId, id -> new DeviceLastFlushTime()); long memIncr = 0L; - for (Map.Entry entry : flushedTimeMap.entrySet()) { + for (Entry entry : flushedTimeMap.entrySet()) { if (flushTimeMapForPartition.getLastFlushTime(entry.getKey()) == Long.MIN_VALUE) { memIncr += HASHMAP_NODE_BASIC_SIZE + entry.getKey().ramBytesUsed(); } @@ -93,7 +100,7 @@ public void upgradeAndUpdateMultiDeviceFlushedTime( long maxFlushTime = flushTimeMapForPartition.getLastFlushTime(null); ILastFlushTime newDeviceLastFlushTime = new DeviceLastFlushTime(); long memIncr = 0; - for (Map.Entry entry : flushedTimeMap.entrySet()) { + for (Entry entry : flushedTimeMap.entrySet()) { memIncr += HASHMAP_NODE_BASIC_SIZE + entry.getKey().ramBytesUsed(); newDeviceLastFlushTime.updateLastFlushTime(entry.getKey(), entry.getValue()); maxFlushTime = Math.max(maxFlushTime, entry.getValue()); @@ -104,7 +111,7 @@ public void upgradeAndUpdateMultiDeviceFlushedTime( } else { // go here when DeviceLastFlushTime was recovered by wal recovery long memIncr = 0; - for (Map.Entry entry : flushedTimeMap.entrySet()) { + for (Entry entry : flushedTimeMap.entrySet()) { if (flushTimeMapForPartition.getLastFlushTime(entry.getKey()) == Long.MIN_VALUE) { memIncr += HASHMAP_NODE_BASIC_SIZE + entry.getKey().ramBytesUsed(); } @@ -131,7 +138,7 @@ public void updatePartitionFlushedTime(long timePartitionId, long maxFlushedTime // go here when DeviceLastFlushTime was recovered by wal recovery DeviceLastFlushTime deviceLastFlushTime = (DeviceLastFlushTime) flushTimeMapForPartition; Map flushedTimeMap = deviceLastFlushTime.getDeviceLastFlushTimeMap(); - for (Map.Entry entry : flushedTimeMap.entrySet()) { + for (Entry entry : flushedTimeMap.entrySet()) { flushTimeMapForPartition.updateLastFlushTime(entry.getKey(), entry.getValue()); } } @@ -139,7 +146,7 @@ public void updatePartitionFlushedTime(long timePartitionId, long maxFlushedTime @Override public void updateMultiDeviceGlobalFlushedTime(Map globalFlushedTimeMap) { - for (Map.Entry entry : globalFlushedTimeMap.entrySet()) { + for (Entry entry : globalFlushedTimeMap.entrySet()) { globalLatestFlushedTimeForEachDevice.merge(entry.getKey(), entry.getValue(), Math::max); } } @@ -161,7 +168,7 @@ public boolean checkAndCreateFlushedTimePartition( // For insert @Override public void updateLatestFlushTime(long partitionId, Map updateMap) { - for (Map.Entry entry : updateMap.entrySet()) { + for (Entry entry : updateMap.entrySet()) { partitionLatestFlushedTime .computeIfAbsent(partitionId, id -> new DeviceLastFlushTime()) .updateLastFlushTime(entry.getKey(), entry.getValue()); @@ -212,4 +219,15 @@ public long getMemSize(long partitionId) { } return 0; } + + @Override + public void accept(SchemaEvolution schemaEvolution) { + if (!(schemaEvolution instanceof TableRename)) { + return; + } + + TableRename tableRename = (TableRename) schemaEvolution; + tableRename.rewriteMap(globalLatestFlushedTimeForEachDevice); + partitionLatestFlushedTime.values().forEach(t -> t.accept(schemaEvolution)); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/ILastFlushTime.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/ILastFlushTime.java index be68369a42b87..f78b6822e9038 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/ILastFlushTime.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/ILastFlushTime.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; import org.apache.tsfile.file.metadata.IDeviceID; public interface ILastFlushTime { @@ -28,4 +29,6 @@ public interface ILastFlushTime { void updateLastFlushTime(IDeviceID device, long time); ILastFlushTime degradeLastFlushTime(); + + void accept(SchemaEvolution schemaEvolution); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/ILastFlushTimeMap.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/ILastFlushTimeMap.java index 7bdd141bf6b5e..40186cef57357 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/ILastFlushTimeMap.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/ILastFlushTimeMap.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; import org.apache.tsfile.file.metadata.IDeviceID; import java.util.Map; @@ -63,4 +64,6 @@ void upgradeAndUpdateMultiDeviceFlushedTime( void degradeLastFlushTime(long partitionId); long getMemSize(long partitionId); + + void accept(SchemaEvolution schemaEvolution); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PartitionLastFlushTime.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PartitionLastFlushTime.java index a5976861441e7..100d023861872 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PartitionLastFlushTime.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PartitionLastFlushTime.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; import org.apache.tsfile.file.metadata.IDeviceID; public class PartitionLastFlushTime implements ILastFlushTime { @@ -43,4 +44,9 @@ public void updateLastFlushTime(IDeviceID device, long time) { public ILastFlushTime degradeLastFlushTime() { return this; } + + @Override + public void accept(SchemaEvolution schemaEvolution) { + // no-op + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index d26172122a69a..80ab28eec9b2c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile; -import java.util.stream.Collectors; import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.consensus.index.ProgressIndexType; import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; @@ -213,7 +212,7 @@ public class TsFileResource implements PersistentResource, Cloneable { private Map>> lastValues; // TsFileSets this TsFile belongs to - private List tsFileSets; + private final List tsFileSets = new ArrayList<>(); @TestOnly public TsFileResource() { @@ -1641,7 +1640,7 @@ public List getTsFileSets() { return tsFileSets; } - public EvolvedSchema getMergedSchema() throws IOException { + public EvolvedSchema getMergedEvolvedSchema() throws IOException { List list = new ArrayList<>(); for (TsFileSet fileSet : getTsFileSets()) { EvolvedSchema readEvolvedSchema = fileSet.readEvolvedSchema(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java index 13d6bc6fdeaa5..cd899ccc97ca8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java @@ -24,6 +24,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; public class EvolvedSchema { // the evolved table names after applying all schema evolution operations @@ -38,9 +40,10 @@ public class EvolvedSchema { public void renameTable(String oldTableName, String newTableName) { if (!originalTableNames.containsKey(oldTableName)) { originalTableNames.put(newTableName, oldTableName); + originalTableNames.put(oldTableName, ""); } else { // mark the old table name as non-exists - String originalName = originalTableNames.remove(oldTableName); + String originalName = originalTableNames.put(oldTableName, ""); originalTableNames.put(newTableName, originalName); } @@ -55,8 +58,10 @@ public void renameColumn(String tableName, String oldColumnName, String newColum originalColumnNames.computeIfAbsent(tableName, t -> new LinkedHashMap<>()); if (!columnNameMap.containsKey(oldColumnName)) { columnNameMap.put(newColumnName, oldColumnName); + columnNameMap.put(oldColumnName, ""); } else { - String originalName = columnNameMap.remove(oldColumnName); + // mark the old column name as non-exists + String originalName = columnNameMap.put(oldColumnName, ""); columnNameMap.put(newColumnName, originalName); } } @@ -96,6 +101,25 @@ public String toString() { '}'; } + public IDeviceID rewriteDeviceId(IDeviceID deviceID) { + String tableName = deviceID.getTableName(); + String originalTableName = getOriginalTableName(tableName); + return rewriteDeviceId(deviceID, originalTableName); + } + + @SuppressWarnings("SuspiciousSystemArraycopy") + public static IDeviceID rewriteDeviceId(IDeviceID deviceID, String originalTableName) { + String tableName = deviceID.getTableName(); + if (!tableName.equals(originalTableName)) { + Object[] segments = deviceID.getSegments(); + String[] newSegments = new String[segments.length]; + newSegments[0] = originalTableName; + System.arraycopy(segments, 1, newSegments, 1, segments.length - 1); + return Factory.DEFAULT_FACTORY.create(newSegments); + } + return deviceID; + } + public static EvolvedSchema deepCopy(EvolvedSchema evolvedSchema) { EvolvedSchema newEvolvedSchema = new EvolvedSchema(); newEvolvedSchema.originalTableNames = new LinkedHashMap<>(evolvedSchema.originalTableNames); @@ -123,15 +147,19 @@ public static EvolvedSchema merge(EvolvedSchema... schemas) { if (schemas[i] != null) { EvolvedSchema newSchema = schemas[i]; for (Entry finalOriginalTableName : newSchema.originalTableNames.entrySet()) { - mergedSchema.renameTable(finalOriginalTableName.getValue(), finalOriginalTableName.getKey()); + if (!finalOriginalTableName.getValue().isEmpty()) { + mergedSchema.renameTable(finalOriginalTableName.getValue(), finalOriginalTableName.getKey()); + } } for (Entry> finalTableNameColumnNameMapEntry : newSchema.originalColumnNames.entrySet()) { for (Entry finalColNameOriginalColNameEntry : finalTableNameColumnNameMapEntry.getValue() .entrySet()) { - String finalTableName = finalTableNameColumnNameMapEntry.getKey(); - String finalColName = finalColNameOriginalColNameEntry.getKey(); - String originalColName = finalColNameOriginalColNameEntry.getValue(); - mergedSchema.renameColumn(finalTableName, originalColName, finalColName); + if (!finalColNameOriginalColNameEntry.getValue().isEmpty()) { + String finalTableName = finalTableNameColumnNameMapEntry.getKey(); + String finalColName = finalColNameOriginalColNameEntry.getKey(); + String originalColName = finalColNameOriginalColNameEntry.getValue(); + mergedSchema.renameColumn(finalTableName, originalColName, finalColName); + } } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java index 02f7f0cfd3e79..02c9eaf0c267d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java @@ -19,6 +19,12 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.iotdb.db.queryengine.execution.schedule.queue.ID; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -63,4 +69,36 @@ public void deserialize(InputStream stream) throws IOException { nameBefore = ReadWriteIOUtils.readVarIntString(stream); nameAfter = ReadWriteIOUtils.readVarIntString(stream); } + + public String getNameBefore() { + return nameBefore; + } + + public String getNameAfter() { + return nameAfter; + } + + @SuppressWarnings("SuspiciousSystemArraycopy") + public IDeviceID rewriteDeviceId(IDeviceID deviceId) { + if (!deviceId.getTableName().equals(nameBefore)) { + return deviceId; + } + + Object[] segments = deviceId.getSegments(); + String[] newSegments = new String[segments.length]; + newSegments[0] = nameAfter; + System.arraycopy(segments, 1, newSegments, 1, segments.length - 1); + return Factory.DEFAULT_FACTORY.create(newSegments); + } + + public void rewriteMap(Map map) { + List affectedDeviceId = map.keySet().stream() + .filter(k -> k.getTableName().equals(getNameBefore())).collect( + Collectors.toList()); + for (IDeviceID deviceID : affectedDeviceId) { + IDeviceID newDeviceId = rewriteDeviceId(deviceID); + T removed = map.remove(deviceID); + map.put(newDeviceId, removed); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index 921fe860b8c88..5b692e7057115 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion; +import java.nio.charset.StandardCharsets; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.consensus.DataRegionId; @@ -30,6 +31,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.NonAlignedFullPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.DataRegionException; @@ -63,6 +65,7 @@ import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.TableRename; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; import org.apache.iotdb.db.storageengine.rescon.memory.MemTableManager; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; @@ -71,10 +74,12 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.reader.IPointReader; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -96,6 +101,8 @@ import static org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils.genInsertRowNode; import static org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils.genInsertTabletNode; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class DataRegionTest { private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); @@ -266,7 +273,7 @@ device, new MeasurementSchema(measurementId, TSDataType.INT32))), null); Assert.assertEquals(10, queryDataSource.getSeqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } } @@ -314,7 +321,7 @@ public void testRelationalTabletWriteAndSyncClose() Assert.assertEquals(1, queryDataSource.getSeqResources().size()); Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } } @@ -362,7 +369,7 @@ public void testRelationRowWriteAndSyncClose() Assert.assertEquals(1, queryDataSource.getSeqResources().size()); Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } } @@ -451,7 +458,7 @@ public void testIoTDBTabletWriteAndSyncClose() Assert.assertEquals(2, queryDataSource.getSeqResources().size()); Assert.assertEquals(1, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } } @@ -518,7 +525,7 @@ public void testIoTDBTabletWriteAndDeleteDataRegion() times.length); dataRegion.insertTablet(insertTabletNode2); - Assert.assertTrue(SystemInfo.getInstance().getTotalMemTableSize() > 0); + assertTrue(SystemInfo.getInstance().getTotalMemTableSize() > 0); dataRegion.syncDeleteDataFiles(); Assert.assertEquals(0, SystemInfo.getInstance().getTotalMemTableSize()); @@ -603,7 +610,7 @@ public void testEmptyTabletWriteAndSyncClose() Assert.assertEquals(0, queryDataSource.getSeqResources().size()); Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } } @@ -679,7 +686,7 @@ public void testAllMeasurementsFailedTabletWriteAndSyncClose() Assert.assertEquals(0, queryDataSource.getSeqResources().size()); Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } } @@ -706,10 +713,10 @@ public void testSeqAndUnSeqSyncClose() Assert.assertEquals(10, queryDataSource.getSeqResources().size()); Assert.assertEquals(10, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } for (TsFileResource resource : queryDataSource.getUnseqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } } @@ -740,10 +747,10 @@ public void testAllMeasurementsFailedRecordSeqAndUnSeqSyncClose() Assert.assertEquals(0, queryDataSource.getSeqResources().size()); Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } for (TsFileResource resource : queryDataSource.getUnseqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } } @@ -773,10 +780,10 @@ public void testDisableSeparateDataForInsertRowPlan() Assert.assertEquals(0, queryDataSource.getSeqResources().size()); Assert.assertEquals(20, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } for (TsFileResource resource : queryDataSource.getUnseqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } config.setEnableSeparateData(defaultValue); @@ -855,7 +862,7 @@ public void testDisableSeparateDataForInsertTablet1() Assert.assertEquals(0, queryDataSource.getSeqResources().size()); Assert.assertEquals(2, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } config.setEnableSeparateData(defaultEnableDiscard); @@ -935,7 +942,7 @@ public void testDisableSeparateDataForInsertTablet2() Assert.assertEquals(0, queryDataSource.getSeqResources().size()); Assert.assertEquals(2, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } config.setEnableSeparateData(defaultEnableDiscard); @@ -1015,7 +1022,7 @@ public void testDisableSeparateDataForInsertTablet3() Assert.assertEquals(0, queryDataSource.getSeqResources().size()); Assert.assertEquals(2, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } config.setEnableSeparateData(defaultEnableDiscard); @@ -1055,7 +1062,7 @@ tmpDeviceId, new MeasurementSchema(measurementId, TSDataType.INT32))), Assert.assertEquals(1, queryDataSource.getSeqResources().size()); Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } dataRegion1.syncDeleteDataFiles(); } @@ -1092,10 +1099,10 @@ tmpDeviceId, new MeasurementSchema(measurementId, TSDataType.INT32))), Assert.assertEquals(10, queryDataSource.getSeqResources().size()); Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } for (TsFileResource resource : queryDataSource.getUnseqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } dataRegion1.syncDeleteDataFiles(); @@ -1160,10 +1167,10 @@ public void testMerge() Collections.singletonList(nonAlignedFullPath), device, context, null, null); Assert.assertEquals(2, queryDataSource.getSeqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } for (TsFileResource resource : queryDataSource.getUnseqResources()) { - Assert.assertTrue(resource.isClosed()); + assertTrue(resource.isClosed()); } IoTDBDescriptor.getInstance() .getConfig() @@ -1232,7 +1239,7 @@ public void testDeleteStorageGroupWhenCompacting() throws Exception { + CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX); Assert.assertFalse(logFile.exists()); Assert.assertFalse(CommonDescriptor.getInstance().getConfig().isReadOnly()); - Assert.assertTrue(dataRegion.getTsFileManager().isAllowCompaction()); + assertTrue(dataRegion.getTsFileManager().isAllowCompaction()); } finally { new CompactionConfigRestorer().restoreCompactionConfig(); } @@ -1392,10 +1399,10 @@ public void testDeleteDataNotInFile() for (int i = 0; i < dataRegion.getSequenceFileList().size(); i++) { TsFileResource resource = dataRegion.getSequenceFileList().get(i); if (i == 1) { - Assert.assertTrue(resource.anyModFileExists()); + assertTrue(resource.anyModFileExists()); Assert.assertEquals(2, resource.getAllModEntries().size()); } else if (i == 3) { - Assert.assertTrue(resource.anyModFileExists()); + assertTrue(resource.anyModFileExists()); Assert.assertEquals(1, resource.getAllModEntries().size()); } else { Assert.assertFalse(resource.anyModFileExists()); @@ -1489,7 +1496,7 @@ public void testDeleteDataInSeqFlushingMemtable() dataRegion.deleteByDevice(new MeasurementPath("root.vehicle.d0.s0"), deleteDataNode4); dataRegion.syncCloseAllWorkingTsFileProcessors(); - Assert.assertTrue(tsFileResource.anyModFileExists()); + assertTrue(tsFileResource.anyModFileExists()); Assert.assertEquals(3, tsFileResource.getAllModEntries().size()); } @@ -1584,7 +1591,7 @@ public void testDeleteDataInUnSeqFlushingMemtable() dataRegion.deleteByDevice(new MeasurementPath("root.vehicle.d0.s0"), deleteDataNode12); dataRegion.syncCloseAllWorkingTsFileProcessors(); - Assert.assertTrue(tsFileResource.anyModFileExists()); + assertTrue(tsFileResource.anyModFileExists()); Assert.assertEquals(3, tsFileResource.getAllModEntries().size()); } @@ -1686,7 +1693,7 @@ public void testDeleteDataDirectlySeqWriteModsOrDeleteFiles() new DeleteDataNode(new PlanNodeId("1"), Collections.singletonList(path), 50, 100); deleteDataNode1.setSearchIndex(0); dataRegion.deleteDataDirectly(new MeasurementPath("root.vehicle.d0.**"), deleteDataNode1); - Assert.assertTrue(tsFileResource.getTsFile().exists()); + assertTrue(tsFileResource.getTsFile().exists()); Assert.assertFalse(tsFileResource.anyModFileExists()); dataRegion.syncCloseAllWorkingTsFileProcessors(); @@ -1696,8 +1703,8 @@ public void testDeleteDataDirectlySeqWriteModsOrDeleteFiles() new DeleteDataNode(new PlanNodeId("2"), Collections.singletonList(path), 100, 120); deleteDataNode2.setSearchIndex(0); dataRegion.deleteDataDirectly(new MeasurementPath("root.vehicle.d0.**"), deleteDataNode2); - Assert.assertTrue(tsFileResource.getTsFile().exists()); - Assert.assertTrue(tsFileResource.anyModFileExists()); + assertTrue(tsFileResource.getTsFile().exists()); + assertTrue(tsFileResource.anyModFileExists()); // delete data in closed file, and time all match DeleteDataNode deleteDataNode3 = @@ -1727,8 +1734,8 @@ public void testDeleteDataDirectlyUnseqWriteModsOrDeleteFiles() dataRegion.syncCloseWorkingTsFileProcessors(true); TsFileResource tsFileResourceUnSeq = dataRegion.getTsFileManager().getTsFileList(false).get(0); - Assert.assertTrue(tsFileResourceSeq.getTsFile().exists()); - Assert.assertTrue(tsFileResourceUnSeq.getTsFile().exists()); + assertTrue(tsFileResourceSeq.getTsFile().exists()); + assertTrue(tsFileResourceUnSeq.getTsFile().exists()); // already closed, will have a mods file. MeasurementPath path = new MeasurementPath("root.vehicle.d0.**"); @@ -1743,9 +1750,9 @@ public void testDeleteDataDirectlyUnseqWriteModsOrDeleteFiles() dataRegion.deleteDataDirectly(new MeasurementPath("root.vehicle.d0.**"), deleteDataNode2); // delete data in mem table, there is no mods - Assert.assertTrue(tsFileResourceSeq.getTsFile().exists()); - Assert.assertTrue(tsFileResourceUnSeq.getTsFile().exists()); - Assert.assertTrue(tsFileResourceSeq.anyModFileExists()); + assertTrue(tsFileResourceSeq.getTsFile().exists()); + assertTrue(tsFileResourceUnSeq.getTsFile().exists()); + assertTrue(tsFileResourceSeq.anyModFileExists()); Assert.assertFalse(tsFileResourceUnSeq.anyModFileExists()); dataRegion.syncCloseAllWorkingTsFileProcessors(); @@ -1753,8 +1760,8 @@ public void testDeleteDataDirectlyUnseqWriteModsOrDeleteFiles() new DeleteDataNode(new PlanNodeId("3"), Collections.singletonList(path), 40, 80); deleteDataNode3.setSearchIndex(0); dataRegion.deleteDataDirectly(new MeasurementPath("root.vehicle.d0.**"), deleteDataNode3); - Assert.assertTrue(tsFileResourceUnSeq.getTsFile().exists()); - Assert.assertTrue(tsFileResourceUnSeq.anyModFileExists()); + assertTrue(tsFileResourceUnSeq.getTsFile().exists()); + assertTrue(tsFileResourceUnSeq.anyModFileExists()); // seq file and unseq file have data file and mod file now, // this deletion will remove data file and mod file. @@ -1772,4 +1779,61 @@ public void testDeleteDataDirectlyUnseqWriteModsOrDeleteFiles() Assert.assertFalse(tsFileResourceSeq.anyModFileExists()); Assert.assertFalse(tsFileResourceUnSeq.anyModFileExists()); } + + @Test + public void testSchemaEvolution() + throws IllegalPathException, WriteProcessException, QueryProcessException { + String[] measurements = {"tag1", "s1", "s2"}; + MeasurementSchema[] measurementSchemas = { + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.INT64), + new MeasurementSchema("s2", TSDataType.DOUBLE) + }; + RelationalInsertRowNode insertRowNode = new RelationalInsertRowNode(new PlanNodeId(""), + new PartialPath(new String[] {"table1"}), + true, + measurements, + new TSDataType[]{TSDataType.STRING, TSDataType.INT64, TSDataType.DOUBLE}, + measurementSchemas, + 10, + new Object[]{new Binary("tag1".getBytes(StandardCharsets.UTF_8)), 1L, 1.0}, + false, + new TsTableColumnCategory[]{TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD}); + dataRegion.insert(insertRowNode); + + // table1 -> table2 + dataRegion.applySchemaEvolution(Collections.singletonList(new TableRename("table1", "table2"))); + + // cannot query with the old name + IDeviceID deviceID1 = Factory.DEFAULT_FACTORY.create(new String[]{"table1", "tag1"}); + List fullPaths = Arrays.asList( + new AlignedFullPath(deviceID1, Arrays.asList(measurements), Arrays.asList(measurementSchemas)) + ); + QueryDataSource dataSource = dataRegion.query(fullPaths, deviceID1, new QueryContext(), null, + Collections.singletonList(0L), Long.MAX_VALUE); + assertTrue(dataSource.getSeqResources().isEmpty()); + + // can query with the new name + IDeviceID deviceID2 = Factory.DEFAULT_FACTORY.create(new String[]{"table2", "tag1"}); + fullPaths = Arrays.asList( + new AlignedFullPath(deviceID2, Arrays.asList(measurements), Arrays.asList(measurementSchemas)) + ); + dataSource = dataRegion.query(fullPaths, deviceID2, new QueryContext(), null, + Collections.singletonList(0L), Long.MAX_VALUE); + assertEquals(1, dataSource.getSeqResources().size()); + + // write again with table1 + insertRowNode = new RelationalInsertRowNode(new PlanNodeId(""), + new PartialPath(new String[] {"table1"}), + true, + measurements, + new TSDataType[]{TSDataType.STRING, TSDataType.INT64, TSDataType.DOUBLE}, + measurementSchemas, + 10, + new Object[]{new Binary("tag1".getBytes(StandardCharsets.UTF_8)), 1L, 1.0}, + false, + new TsTableColumnCategory[]{TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD}); + dataRegion.insert(insertRowNode); + + } } From 4ced268f8b1c54de8b36c25e3b245f0f7419e9e7 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 27 Nov 2025 19:12:56 +0800 Subject: [PATCH 06/49] supplement DataRegionTest --- .../plan/statement/StatementTestUtils.java | 12 ++++++-- .../dataregion/DataRegionTest.java | 28 +++++++++++++++++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/StatementTestUtils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/StatementTestUtils.java index a80095458a942..2b064a77721b1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/StatementTestUtils.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/StatementTestUtils.java @@ -55,7 +55,11 @@ private StatementTestUtils() { } public static String tableName() { - return "table1"; + return tableName(1); + } + + public static String tableName(int i) { + return "table" + i; } public static String[] genColumnNames() { @@ -232,7 +236,11 @@ public static InsertRowStatement genInsertRowStatement(boolean writeToTable) { } public static TsTable genTsTable() { - final TsTable tsTable = new TsTable(tableName()); + return genTsTable(1); + } + + public static TsTable genTsTable(int tableId) { + final TsTable tsTable = new TsTable(tableName(tableId)); String[] measurements = genColumnNames(); TSDataType[] dataTypes = genDataTypes(); TsTableColumnCategory[] columnCategories = genColumnCategories(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index 5b692e7057115..b525b4860e7b3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -141,9 +141,13 @@ public void setUp() throws Exception { config.setInnerUnsequenceCompactionSelector( InnerUnsequenceCompactionSelector.SIZE_TIERED_SINGLE_TARGET); DataNodeTableCache.getInstance() - .preUpdateTable(dataRegion.getDatabaseName(), StatementTestUtils.genTsTable(), null); + .preUpdateTable(dataRegion.getDatabaseName(), StatementTestUtils.genTsTable(1), null); DataNodeTableCache.getInstance() - .commitUpdateTable(dataRegion.getDatabaseName(), StatementTestUtils.tableName(), null); + .commitUpdateTable(dataRegion.getDatabaseName(), StatementTestUtils.tableName(1), null); + DataNodeTableCache.getInstance() + .preUpdateTable(dataRegion.getDatabaseName(), StatementTestUtils.genTsTable(2), null); + DataNodeTableCache.getInstance() + .commitUpdateTable(dataRegion.getDatabaseName(), StatementTestUtils.tableName(2), null); } @After @@ -1822,6 +1826,11 @@ public void testSchemaEvolution() Collections.singletonList(0L), Long.MAX_VALUE); assertEquals(1, dataSource.getSeqResources().size()); + DataNodeTableCache.getInstance() + .preUpdateTable(dataRegion.getDatabaseName(), StatementTestUtils.genTsTable(1), null); + DataNodeTableCache.getInstance() + .commitUpdateTable(dataRegion.getDatabaseName(), StatementTestUtils.tableName(1), null); + // write again with table1 insertRowNode = new RelationalInsertRowNode(new PlanNodeId(""), new PartialPath(new String[] {"table1"}), @@ -1835,5 +1844,20 @@ public void testSchemaEvolution() new TsTableColumnCategory[]{TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD}); dataRegion.insert(insertRowNode); + // can query with table1 + fullPaths = Arrays.asList( + new AlignedFullPath(deviceID1, Arrays.asList(measurements), Arrays.asList(measurementSchemas)) + ); + dataSource = dataRegion.query(fullPaths, deviceID1, new QueryContext(), null, + Collections.singletonList(0L), Long.MAX_VALUE); + assertEquals(1, dataSource.getUnseqResources().size()); + + // can query with table2 + fullPaths = Arrays.asList( + new AlignedFullPath(deviceID2, Arrays.asList(measurements), Arrays.asList(measurementSchemas)) + ); + dataSource = dataRegion.query(fullPaths, deviceID2, new QueryContext(), null, + Collections.singletonList(0L), Long.MAX_VALUE); + assertEquals(1, dataSource.getSeqResources().size()); } } From f71b7156cae45d762d6849239c9199c032749bd0 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 10 Dec 2025 16:27:57 +0800 Subject: [PATCH 07/49] Add DataRegionTask --- .../storageengine/dataregion/DataRegion.java | 76 ++++++++---- .../dataregion/task/DataRegionTask.java | 61 ++++++++++ .../task/DataRegionTaskManager.java | 108 ++++++++++++++++++ .../dataregion/task/SchemaEvolutionTask.java | 86 ++++++++++++++ .../dataregion/tsfile/TsFileResource.java | 10 +- .../tsfile/evolution/ColumnRename.java | 17 +++ .../dataregion/tsfile/fileset/TsFileSet.java | 8 ++ .../dataregion/DataRegionTest.java | 2 +- .../apache/iotdb/commons/utils/FileUtils.java | 81 +++++++++++++ 9 files changed, 420 insertions(+), 29 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTask.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTaskManager.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/SchemaEvolutionTask.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index f288e81f82c2e..b2a340d14bd8c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -122,6 +122,8 @@ import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.IFileScanHandle; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.impl.ClosedFileScanHandleImpl; +import org.apache.iotdb.db.storageengine.dataregion.task.DataRegionTaskManager; +import org.apache.iotdb.db.storageengine.dataregion.task.SchemaEvolutionTask; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -365,6 +367,8 @@ public class DataRegion implements IDataRegionForQuery { private Map lastTsFileSetMap = new ConcurrentHashMap<>(); + private DataRegionTaskManager dataRegionTaskManager; + /** * Construct a database processor. * @@ -727,6 +731,9 @@ private void recover() throws DataRegionException { throw new DataRegionException(e); } + dataRegionTaskManager = new DataRegionTaskManager(this); + dataRegionTaskManager.recover(); + if (asyncTsFileResourceRecoverTaskList.isEmpty()) { initCompactionSchedule(); } @@ -756,9 +763,13 @@ private void updatePartitionLastFlushTime(TsFileResource resource) { protected void updateDeviceLastFlushTime(TsFileResource resource) { long timePartitionId = resource.getTimePartition(); Map endTimeMap = new HashMap<>(); + EvolvedSchema mergedEvolvedSchema = resource.getMergedEvolvedSchema(); for (IDeviceID deviceId : resource.getDevices()) { @SuppressWarnings("OptionalGetWithoutIsPresent") // checked above long endTime = resource.getEndTime(deviceId).get(); + if (mergedEvolvedSchema != null) { + deviceId = mergedEvolvedSchema.rewriteDeviceId(deviceId); + } endTimeMap.put(deviceId, endTime); } if (config.isEnableSeparateData()) { @@ -773,10 +784,14 @@ protected void upgradeAndUpdateDeviceLastFlushTime( long timePartitionId, List resources) { Map endTimeMap = new HashMap<>(); for (TsFileResource resource : resources) { + EvolvedSchema mergedEvolvedSchema = resource.getMergedEvolvedSchema(); for (IDeviceID deviceId : resource.getDevices()) { // checked above //noinspection OptionalGetWithoutIsPresent long endTime = resource.getEndTime(deviceId).get(); + if (mergedEvolvedSchema != null) { + deviceId = mergedEvolvedSchema.rewriteDeviceId(deviceId); + } endTimeMap.put(deviceId, endTime); } } @@ -1001,6 +1016,10 @@ private String getFileSetsDir(long partitionId) { + TsFileSet.FILE_SET_DIR_NAME; } + public File getDataRegionSysDir() { + return dataRegionSysDir; + } + private List recoverTsFileSets( long partitionId, Map> tsFileSetMap @@ -1189,7 +1208,7 @@ private TsFileSet createNewFileSet(long maxVersion, long partitionId) { return newSet; } - public void applySchemaEvolution(List schemaEvolutions) { + public void applySchemaEvolution(List schemaEvolutions) throws IOException { long startTime = System.nanoTime(); writeLock("applySchemaEvolution"); PERFORMANCE_OVERVIEW_METRICS.recordScheduleLockCost(System.nanoTime() - startTime); @@ -1198,32 +1217,44 @@ public void applySchemaEvolution(List schemaEvolutions) { return; } DataNodeTableCache.getInstance().invalid(databaseName); - schemaEvolutions.forEach(lastFlushTimeMap::accept); syncCloseAllWorkingTsFileProcessors(); - for (Entry partitionVersionEntry : partitionMaxFileVersions.entrySet()) { - long partitionId = partitionVersionEntry.getKey(); - long maxVersion = partitionVersionEntry.getValue(); - lastTsFileSetMap.compute(partitionId, (pid, lastSet) -> { - if (lastSet == null) { - lastSet = createNewFileSet(maxVersion, partitionId); - } else if (lastSet.getEndVersion() < maxVersion) { - lastSet = createNewFileSet(maxVersion, partitionId); - } - try { - lastSet.appendSchemaEvolution(schemaEvolutions); - } catch (IOException e) { - logger.error("Cannot append schema evolutions to fileSets in partition {}-{}", dataRegionId, partitionId, e); - } - return lastSet; - }); - } + // may update table names in deviceIds + schemaEvolutions.forEach(lastFlushTimeMap::accept); + + SchemaEvolutionTask evolutionTask = new SchemaEvolutionTask(schemaEvolutions, this); + dataRegionTaskManager.submitAndRun(evolutionTask); } finally { writeUnlock(); } } + public void recordSchemaEvolution(List schemaEvolutions) { + for (Entry partitionVersionEntry : partitionMaxFileVersions.entrySet()) { + long partitionId = partitionVersionEntry.getKey(); + long maxVersion = partitionVersionEntry.getValue(); + lastTsFileSetMap.compute(partitionId, (pid, lastSet) -> { + if (lastSet == null) { + lastSet = createNewFileSet(maxVersion, partitionId); + } else if (lastSet.getEndVersion() < maxVersion) { + lastSet = createNewFileSet(maxVersion, partitionId); + } + try { + lastSet.appendSchemaEvolution(schemaEvolutions); + } catch (IOException e) { + logger.error("Cannot append schema evolutions to fileSets in partition {}-{}", dataRegionId, partitionId, e); + } + return lastSet; + }); + } + } + + public void applySchemaEvolutionToObjects(List schemaEvolutions) { + // TODO-SchemaEvolution + throw new UnsupportedOperationException(); + } + /** * insert one row of data. * @@ -2624,12 +2655,7 @@ private List getFileResourceListForQuery( List tsfileResourcesForQuery = new ArrayList<>(); for (TsFileResource tsFileResource : tsFileResources) { - EvolvedSchema evolvedSchema; - try { - evolvedSchema = tsFileResource.getMergedEvolvedSchema(); - } catch (IOException e) { - throw new MetadataException(e); - } + EvolvedSchema evolvedSchema = tsFileResource.getMergedEvolvedSchema(); IDeviceID deviceIdBackThen = singleDeviceId; if (evolvedSchema != null) { deviceIdBackThen = evolvedSchema.rewriteDeviceId(singleDeviceId); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTask.java new file mode 100644 index 0000000000000..f171003aa693f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTask.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.task; + +import java.io.IOException; +import java.io.InputStream; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; +import org.apache.iotdb.db.utils.io.StreamSerializable; +import org.apache.tsfile.utils.ReadWriteForEncodingUtils; + +public interface DataRegionTask extends Runnable, StreamSerializable { + + long getTaskId(); + + void setTaskId(long taskId); + + TaskType getTaskType(); + + enum TaskType { + SchemaEvolutionTask + } + + @SuppressWarnings("SwitchStatementWithTooFewBranches") + static DataRegionTask createFrom(InputStream stream, long taskId, DataRegion dataRegion) throws IOException { + int typeOrdinal = ReadWriteForEncodingUtils.readVarInt(stream); + if (typeOrdinal < 0 || typeOrdinal >= TaskType.values().length) { + throw new IOException("Invalid task type: " + typeOrdinal); + } + + TaskType taskType = TaskType.values()[typeOrdinal]; + + DataRegionTask task = null; + switch (taskType) { + case SchemaEvolutionTask: + task = new SchemaEvolutionTask(dataRegion); + break; + default: + throw new IOException("Invalid task type: " + taskType); + } + task.deserialize(stream); + task.setTaskId(taskId); + return task; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTaskManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTaskManager.java new file mode 100644 index 0000000000000..5441b9b19c7bf --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTaskManager.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.task; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings("ResultOfMethodCallIgnored") +public class DataRegionTaskManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(DataRegionTaskManager.class); + private static final String TASKS_DIR_NAME = "tasks"; + private static final String TASK_FILE_SUFFIX = ".tsk"; + + private final DataRegion dataRegion; + private final AtomicLong lastestTaskId = new AtomicLong(0); + private final File tasksDir; + + public DataRegionTaskManager(DataRegion dataRegion) { + this.dataRegion = dataRegion; + this.tasksDir = new File(dataRegion.getDataRegionSysDir() + File.separator + TASKS_DIR_NAME); + } + + public void recover() { + tasksDir.mkdirs(); + File[] files = tasksDir.listFiles((File dir, String name) -> name.endsWith(TASK_FILE_SUFFIX)); + if (files == null) { + return; + } + + Arrays.sort(files, (f1, f2) -> { + String fileName1 = f1.getName(); + int suffixIndex1 = fileName1.indexOf("."); + long taskId1 = Long.parseLong(fileName1.substring(0, suffixIndex1)); + + String fileName2 = f2.getName(); + int suffixIndex2 = fileName2.indexOf("."); + long taskId2 = Long.parseLong(fileName1.substring(0, suffixIndex2)); + + return Long.compare(taskId1, taskId2); + }); + + for (File file : files) { + String fileName = file.getName(); + int suffixIndex = fileName.indexOf("."); + long taskId = Long.parseLong(fileName.substring(0, suffixIndex)); + lastestTaskId.getAndUpdate(l -> Math.max(l, taskId)); + + try (FileInputStream fis = new FileInputStream(file); + BufferedInputStream bufferedInputStream = new BufferedInputStream(fis)) { + DataRegionTask task = DataRegionTask.createFrom(bufferedInputStream, taskId, dataRegion); + task.run(); + } catch (IOException e) { + if (LOGGER.isWarnEnabled()) { + LOGGER.warn("Cannot recover task from file {}", file.getAbsolutePath(), e); + } + } finally { + file.delete(); + } + } + } + + private void persistTask(DataRegionTask task) throws IOException { + File taskFile = new File(tasksDir, task.getTaskId() + ".tsk"); + try (FileOutputStream fos = new FileOutputStream(taskFile); + BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fos)) { + task.serialize(bufferedOutputStream); + } + } + + private void removeTask(DataRegionTask task) throws IOException { + File taskFile = new File(tasksDir, task.getTaskId() + ".tsk"); + taskFile.delete(); + } + + public void submitAndRun(DataRegionTask dataRegionTask) throws IOException { + dataRegionTask.setTaskId(lastestTaskId.getAndIncrement()); + persistTask(dataRegionTask); + dataRegionTask.run(); + removeTask(dataRegionTask); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/SchemaEvolutionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/SchemaEvolutionTask.java new file mode 100644 index 0000000000000..23a84fbbd720c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/SchemaEvolutionTask.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.task; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; +import org.apache.tsfile.utils.ReadWriteForEncodingUtils; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +public class SchemaEvolutionTask implements DataRegionTask { + + private List schemaEvolutions; + private final DataRegion dataRegion; + private long taskId; + + @Override + public void run() { + dataRegion.recordSchemaEvolution(schemaEvolutions); + dataRegion.applySchemaEvolutionToObjects(schemaEvolutions); + } + + public SchemaEvolutionTask(DataRegion dataRegion) { + this.dataRegion = dataRegion; + } + + public SchemaEvolutionTask(List schemaEvolutions, DataRegion dataRegion) { + this.schemaEvolutions = schemaEvolutions; + this.dataRegion = dataRegion; + } + + @Override + public long serialize(OutputStream stream) throws IOException { + long size = ReadWriteForEncodingUtils.writeVarInt(getTaskType().ordinal(), stream); + size += ReadWriteForEncodingUtils.writeVarInt(schemaEvolutions.size(), stream); + for (SchemaEvolution schemaEvolution : schemaEvolutions) { + size += schemaEvolution.serialize(stream); + } + return size; + } + + @Override + public void deserialize(InputStream stream) throws IOException { + int size = ReadWriteForEncodingUtils.readVarInt(stream); + schemaEvolutions = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + schemaEvolutions.add(SchemaEvolution.createFrom(stream)); + } + } + + @Override + public long getTaskId() { + return taskId; + } + + @Override + public void setTaskId(long taskId) { + this.taskId = taskId; + } + + @Override + public TaskType getTaskType() { + return TaskType.SchemaEvolutionTask; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index 80ab28eec9b2c..060c16fea7d3f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -1640,11 +1640,15 @@ public List getTsFileSets() { return tsFileSets; } - public EvolvedSchema getMergedEvolvedSchema() throws IOException { + public EvolvedSchema getMergedEvolvedSchema() { List list = new ArrayList<>(); for (TsFileSet fileSet : getTsFileSets()) { - EvolvedSchema readEvolvedSchema = fileSet.readEvolvedSchema(); - list.add(readEvolvedSchema); + try { + EvolvedSchema readEvolvedSchema = fileSet.readEvolvedSchema(); + list.add(readEvolvedSchema); + } catch (IOException e) { + LOGGER.warn("Cannot read evolved schema from {}, skipping it", fileSet); + } } return EvolvedSchema.merge(list.toArray(new EvolvedSchema[0])); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java index fc363247b13b0..23c18cea9f4b9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; +import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -32,6 +34,7 @@ public class ColumnRename implements SchemaEvolution { private String tableName; private String nameBefore; private String nameAfter; + private TSDataType dataType; // for deserialization public ColumnRename() {} @@ -58,6 +61,7 @@ public long serialize(OutputStream stream) throws IOException { size += ReadWriteIOUtils.writeVar(tableName, stream); size += ReadWriteIOUtils.writeVar(nameBefore, stream); size += ReadWriteIOUtils.writeVar(nameAfter, stream); + size += ReadWriteIOUtils.write(dataType != null ? (byte) dataType.ordinal() : -1, stream); return size; } @@ -66,5 +70,18 @@ public void deserialize(InputStream stream) throws IOException { tableName = ReadWriteIOUtils.readVarIntString(stream); nameBefore = ReadWriteIOUtils.readVarIntString(stream); nameAfter = ReadWriteIOUtils.readVarIntString(stream); + byte category = ReadWriteIOUtils.readByte(stream); + if (category != -1) { + dataType = TSDataType.values()[category]; + } + } + + public TSDataType getDataType() { + return dataType; + } + + public void setDataType( + TSDataType dataType) { + this.dataType = dataType; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java index d3271d98b45e4..5c43418fa64f7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java @@ -120,4 +120,12 @@ public void readUnlock() { public long getEndVersion() { return endVersion; } + + @Override + public String toString() { + return "TsFileSet{" + + "endVersion=" + endVersion + + ", fileSetDir=" + fileSetDir + + '}'; + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index b525b4860e7b3..e896a599c4335 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -1786,7 +1786,7 @@ public void testDeleteDataDirectlyUnseqWriteModsOrDeleteFiles() @Test public void testSchemaEvolution() - throws IllegalPathException, WriteProcessException, QueryProcessException { + throws IllegalPathException, WriteProcessException, QueryProcessException, IOException { String[] measurements = {"tag1", "s1", "s2"}; MeasurementSchema[] measurementSchemas = { new MeasurementSchema("tag1", TSDataType.STRING), diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java index baed8bcd537e7..a2510454e7379 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java @@ -19,6 +19,12 @@ package org.apache.iotdb.commons.utils; +import java.util.HashSet; +import java.util.Objects; +import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.tsfile.external.commons.codec.digest.DigestUtils; @@ -334,6 +340,81 @@ public static boolean moveFileSafe(File source, File target) { return true; } + public static List applyReversedIndexesOnListV2( + final List filteredIndexes, final List originalList) { + // filteredIndexes.sort(null); if necessary + List filteredList = new ArrayList<>(originalList.size() - filteredIndexes.size()); + int filteredIndexPos = 0; + int processingIndex = 0; + for (; processingIndex < originalList.size(); processingIndex++) { + if (filteredIndexPos >= filteredIndexes.size()) { + // all filteredIndexes processed, add remaining to the filteredList + filteredList.addAll(originalList.subList(processingIndex, originalList.size())); + break; + } else { + int filteredIndex = filteredIndexes.get(filteredIndexPos); + if (filteredIndex == processingIndex) { + // the index is filtered, move to the next filtered pos + filteredIndexPos ++; + } else { + // the index is not filtered, add to the filteredList + filteredList.add(originalList.get(processingIndex)); + } + } + } + return filteredList; + } + + public static List applyReversedIndexesOnListV1( + final List filteredIndexes, final List originalList) { + final Set indexes = new HashSet<>(filteredIndexes); + return Objects.nonNull(originalList) + ? IntStream.range(0, originalList.size()) + .filter(index -> !indexes.contains(index)) // 保留不在排除列表中的下标 + .mapToObj(originalList::get) + .collect(Collectors.toList()) + : null; + } + + public static void main(String[] args) { + int elementNum = 10_000_000; + int filteredNum = elementNum / 10; + Random random = new Random(); + List originalList = IntStream.range(0, elementNum).boxed().collect(Collectors.toList()); + List filteredIndexes = new ArrayList<>(filteredNum); + for (int i = 0; i < filteredNum; i++) { + filteredIndexes.add(random.nextInt(elementNum)); + } + filteredIndexes = filteredIndexes.stream().sorted().distinct().collect(Collectors.toList()); + + long start = System.currentTimeMillis(); + List appliedList = applyReversedIndexesOnListV1(filteredIndexes, originalList); + System.out.println(System.currentTimeMillis() - start); + Set appliedSet = new HashSet<>(appliedList); + for (Integer filteredIndex : filteredIndexes) { + if (appliedSet.contains(filteredIndex)) { + System.out.println("Incorrect implementation"); + System.exit(-1); + } + } + + + start = System.currentTimeMillis(); + appliedList = WapplyReversedIndexesOnListV2(filteredIndexes, originalList); + System.out.println(System.currentTimeMillis() - start); + appliedSet = new HashSet<>(appliedList); + if (appliedList.size() != originalList.size() - filteredIndexes.size()) { + System.out.println("Incorrect implementation"); + System.exit(-1); + } + for (Integer filteredIndex : filteredIndexes) { + if (appliedSet.contains(filteredIndex)) { + System.out.println("Incorrect implementation"); + System.exit(-1); + } + } + } + public static File createHardLink(File sourceFile, File hardlink) throws IOException { if (!hardlink.getParentFile().exists() && !hardlink.getParentFile().mkdirs()) { synchronized (FileUtils.class) { From c0af94b6fd1896e80c4748999eab84b8b9420456 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 10 Dec 2025 16:29:56 +0800 Subject: [PATCH 08/49] Add DataRegionTask --- .../iotdb/db/storageengine/dataregion/task/DataRegionTask.java | 2 +- .../db/storageengine/dataregion/task/SchemaEvolutionTask.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTask.java index f171003aa693f..f727e2846e752 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTask.java @@ -46,7 +46,7 @@ static DataRegionTask createFrom(InputStream stream, long taskId, DataRegion dat TaskType taskType = TaskType.values()[typeOrdinal]; - DataRegionTask task = null; + DataRegionTask task; switch (taskType) { case SchemaEvolutionTask: task = new SchemaEvolutionTask(dataRegion); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/SchemaEvolutionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/SchemaEvolutionTask.java index 23a84fbbd720c..e3bae4c571c02 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/SchemaEvolutionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/SchemaEvolutionTask.java @@ -27,7 +27,6 @@ import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; -import org.apache.tsfile.utils.ReadWriteIOUtils; public class SchemaEvolutionTask implements DataRegionTask { From acd76238a93ca801830ab8c8ad6fab2562ecef57 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 10 Dec 2025 17:31:13 +0800 Subject: [PATCH 09/49] add EvolceSchemaNode --- .../plan/planner/plan/node/PlanNodeType.java | 8 + .../plan/node/write/EvolveSchemaNode.java | 179 ++++++++++++++++++ .../tsfile/evolution/ColumnRename.java | 22 +++ .../tsfile/evolution/SchemaEvolution.java | 26 ++- .../tsfile/evolution/TableRename.java | 15 ++ .../org/apache/iotdb/db/utils/io/IOUtils.java | 52 +++++ 6 files changed, 296 insertions(+), 6 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/EvolveSchemaNode.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/io/IOUtils.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 1100793ca8842..5a0bdcc3cafdd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -107,6 +107,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.TimeseriesRegionScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ContinuousSameSearchIndexSeparatorNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.EvolveSchemaNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertMultiTabletsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; @@ -319,6 +320,7 @@ public enum PlanNodeType { RELATIONAL_INSERT_ROW((short) 2001), RELATIONAL_INSERT_ROWS((short) 2002), RELATIONAL_DELETE_DATA((short) 2003), + EVOLVE_SCHEMA((short) 2004), ; public static final int BYTES = Short.BYTES; @@ -362,6 +364,8 @@ public static PlanNode deserializeFromWAL(DataInputStream stream) throws IOExcep return RelationalInsertRowsNode.deserializeFromWAL(stream); case 2003: return RelationalDeleteDataNode.deserializeFromWAL(stream); + case 2004: + return EvolveSchemaNode.deserializeFromWAL(stream); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } @@ -388,6 +392,8 @@ public static PlanNode deserializeFromWAL(ByteBuffer buffer) { return RelationalInsertRowsNode.deserializeFromWAL(buffer); case 2003: return RelationalDeleteDataNode.deserializeFromWAL(buffer); + case 2004: + return EvolveSchemaNode.deserializeFromWAL(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } @@ -713,6 +719,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { return RelationalInsertRowsNode.deserialize(buffer); case 2003: return RelationalDeleteDataNode.deserialize(buffer); + case 2004: + return EvolveSchemaNode.deserialize(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/EvolveSchemaNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/EvolveSchemaNode.java new file mode 100644 index 0000000000000..f85ccbfe17af5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/EvolveSchemaNode.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.write; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.consensus.index.ProgressIndex; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; +import org.apache.iotdb.db.utils.io.IOUtils; +import org.apache.tsfile.utils.ReadWriteForEncodingUtils; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EvolveSchemaNode extends SearchNode implements WALEntryValue { + + private static final Logger LOGGER = + LoggerFactory.getLogger(EvolveSchemaNode.class); + + protected TRegionReplicaSet regionReplicaSet; + protected ProgressIndex progressIndex; + private final List schemaEvolutions; + + public EvolveSchemaNode(PlanNodeId id, + List schemaEvolutions) { + super(id); + this.schemaEvolutions = schemaEvolutions; + } + + public static PlanNode deserializeFromWAL(DataInputStream stream) throws IOException { + long searchIndex = stream.readLong(); + int size = ReadWriteForEncodingUtils.readVarInt(stream); + List evolutions = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + evolutions.add(SchemaEvolution.createFrom(stream)); + } + + EvolveSchemaNode evolveSchemaNode = new EvolveSchemaNode(new PlanNodeId(""), evolutions); + evolveSchemaNode.setSearchIndex(searchIndex); + + return evolveSchemaNode; + } + + public static PlanNode deserializeFromWAL(ByteBuffer buffer) { + long searchIndex = buffer.getLong(); + int size = ReadWriteForEncodingUtils.readVarInt(buffer); + List evolutions = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + evolutions.add(SchemaEvolution.createFrom(buffer)); + } + + EvolveSchemaNode evolveSchemaNode = new EvolveSchemaNode(new PlanNodeId(""), evolutions); + evolveSchemaNode.setSearchIndex(searchIndex); + + return evolveSchemaNode; + } + + public static PlanNode deserialize(ByteBuffer buffer) { + int size = ReadWriteForEncodingUtils.readVarInt(buffer); + List evolutions = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + evolutions.add(SchemaEvolution.createFrom(buffer)); + } + + PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); + + // EvolveSchemaNode has no child + int ignoredChildrenSize = ReadWriteIOUtils.readInt(buffer); + return new EvolveSchemaNode(planNodeId, evolutions); + } + + @Override + public SearchNode merge(List searchNodes) { + return this; + } + + @Override + public ProgressIndex getProgressIndex() { + return progressIndex; + } + + @Override + public void setProgressIndex(ProgressIndex progressIndex) { + this.progressIndex = progressIndex; + } + + @Override + public List splitByPartition(IAnalysis analysis) { + return Collections.singletonList(this); + } + + @Override + public TRegionReplicaSet getRegionReplicaSet() { + return regionReplicaSet; + } + + @Override + public List getChildren() { + return Collections.emptyList(); + } + + @Override + public void addChild(PlanNode child) { + throw new UnsupportedOperationException(); + } + + @Override + public PlanNode clone() { + return new EvolveSchemaNode(id, schemaEvolutions); + } + + @Override + public int allowedChildCount() { + return 0; + } + + @Override + public List getOutputColumnNames() { + return Collections.emptyList(); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.EVOLVE_SCHEMA.serialize(byteBuffer); + IOUtils.writeList(schemaEvolutions, byteBuffer); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.EVOLVE_SCHEMA.serialize(stream); + IOUtils.writeList(schemaEvolutions, stream); + } + + @Override + public void serializeToWAL(IWALByteBufferView buffer) { + buffer.putShort(PlanNodeType.EVOLVE_SCHEMA.getNodeType()); + buffer.putLong(searchIndex); + try { + IOUtils.writeList(schemaEvolutions, buffer); + } catch (IOException e) { + LOGGER.warn("Error writing schema evolutions to WAL", e); + } + } + + @Override + public int serializedSize() { + return 0; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java index 23c18cea9f4b9..4986107a32d60 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; +import java.nio.ByteBuffer; import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; @@ -76,6 +77,27 @@ public void deserialize(InputStream stream) throws IOException { } } + @Override + public long serialize(ByteBuffer buffer) { + int size = ReadWriteForEncodingUtils.writeVarInt(getEvolutionType().ordinal(), buffer); + size += ReadWriteIOUtils.writeVar(tableName, buffer); + size += ReadWriteIOUtils.writeVar(nameBefore, buffer); + size += ReadWriteIOUtils.writeVar(nameAfter, buffer); + size += ReadWriteIOUtils.write(dataType != null ? (byte) dataType.ordinal() : -1, buffer); + return size; + } + + @Override + public void deserialize(ByteBuffer buffer) { + tableName = ReadWriteIOUtils.readVarIntString(buffer); + nameBefore = ReadWriteIOUtils.readVarIntString(buffer); + nameAfter = ReadWriteIOUtils.readVarIntString(buffer); + byte category = ReadWriteIOUtils.readByte(buffer); + if (category != -1) { + dataType = TSDataType.values()[category]; + } + } + public TSDataType getDataType() { return dataType; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolution.java index 5a670879bebe8..b017f6509b2c5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolution.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; +import java.nio.ByteBuffer; +import org.apache.iotdb.db.utils.io.BufferSerializable; import org.apache.iotdb.db.utils.io.StreamSerializable; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; @@ -27,7 +29,7 @@ import java.io.InputStream; /** A schema evolution operation that can be applied to a TableSchemaMap. */ -public interface SchemaEvolution extends StreamSerializable { +public interface SchemaEvolution extends StreamSerializable, BufferSerializable { /** * Apply this schema evolution operation to the given metadata. @@ -43,12 +45,11 @@ enum SchemaEvolutionType { COLUMN_RENAME } - static SchemaEvolution createFrom(InputStream stream) throws IOException { - int type = ReadWriteForEncodingUtils.readVarInt(stream); + static SchemaEvolution createFrom(int type) { if (type < 0 || type > SchemaEvolutionType.values().length) { - throw new IOException("Invalid evolution type: " + type); + throw new IllegalArgumentException("Invalid evolution type: " + type); } - SchemaEvolution evolution = null; + SchemaEvolution evolution; SchemaEvolutionType evolutionType = SchemaEvolutionType.values()[type]; switch (evolutionType) { case TABLE_RENAME: @@ -58,9 +59,22 @@ static SchemaEvolution createFrom(InputStream stream) throws IOException { evolution = new ColumnRename(); break; default: - throw new IOException("Invalid evolution type: " + evolutionType); + throw new IllegalArgumentException("Invalid evolution type: " + evolutionType); } + return evolution; + } + + static SchemaEvolution createFrom(InputStream stream) throws IOException { + int type = ReadWriteForEncodingUtils.readVarInt(stream); + SchemaEvolution evolution = createFrom(type); evolution.deserialize(stream); return evolution; } + + static SchemaEvolution createFrom(ByteBuffer buffer) { + int type = ReadWriteForEncodingUtils.readVarInt(buffer); + SchemaEvolution evolution = createFrom(type); + evolution.deserialize(buffer); + return evolution; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java index 02c9eaf0c267d..dfc80974e85fe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; +import java.nio.ByteBuffer; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -70,6 +71,20 @@ public void deserialize(InputStream stream) throws IOException { nameAfter = ReadWriteIOUtils.readVarIntString(stream); } + @Override + public long serialize(ByteBuffer buffer) { + long size = ReadWriteForEncodingUtils.writeVarInt(getEvolutionType().ordinal(), buffer); + size += ReadWriteIOUtils.writeVar(nameBefore, buffer); + size += ReadWriteIOUtils.writeVar(nameAfter, buffer); + return size; + } + + @Override + public void deserialize(ByteBuffer buffer) { + nameBefore = ReadWriteIOUtils.readVarIntString(buffer); + nameAfter = ReadWriteIOUtils.readVarIntString(buffer); + } + public String getNameBefore() { return nameBefore; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/io/IOUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/io/IOUtils.java new file mode 100644 index 0000000000000..f41ae6de8def0 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/io/IOUtils.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils.io; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import org.apache.tsfile.utils.ReadWriteForEncodingUtils; + +// TODO: move to TsFile +public class IOUtils { + + private IOUtils() { + // util class + } + + public static long writeList(List list, ByteBuffer byteBuffer) { + long size = ReadWriteForEncodingUtils.writeVarInt(list.size(), byteBuffer); + for (BufferSerializable item : list) { + size += item.serialize(byteBuffer); + } + return size; + } + + public static long writeList(List list, OutputStream stream) + throws IOException { + long size = ReadWriteForEncodingUtils.writeVarInt(list.size(), stream); + for (StreamSerializable item : list) { + size += item.serialize(stream); + } + return size; + } +} From a7dd10210a0413d66a5c2fdff81d8e263e5c162b Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 10 Dec 2025 18:42:28 +0800 Subject: [PATCH 10/49] Add execution of EvolveSchemaNode --- .../dataregion/DataExecutionVisitor.java | 12 ++ .../impl/DataNodeInternalRPCServiceImpl.java | 23 +++ .../plan/planner/plan/node/PlanNodeType.java | 8 + .../plan/planner/plan/node/PlanVisitor.java | 10 + .../pipe/PipeEnrichedEvolveSchemaNode.java | 194 ++++++++++++++++++ .../plan/node/write/EvolveSchemaNode.java | 32 ++- .../storageengine/dataregion/DataRegion.java | 88 +++++--- .../dataregion/DeviceLastFlushTime.java | 3 +- .../dataregion/HashLastFlushTimeMap.java | 8 +- .../dataregion/ILastFlushTime.java | 1 + .../dataregion/ILastFlushTimeMap.java | 1 + .../dataregion/PartitionLastFlushTime.java | 1 + .../dataregion/task/DataRegionTask.java | 17 +- .../task/DataRegionTaskManager.java | 28 +-- .../dataregion/task/SchemaEvolutionTask.java | 8 +- .../tsfile/evolution/ColumnRename.java | 25 ++- .../tsfile/evolution/EvolvedSchema.java | 53 ++--- .../tsfile/evolution/SchemaEvolution.java | 13 +- .../tsfile/evolution/SchemaEvolutionFile.java | 1 + .../tsfile/evolution/TableRename.java | 16 +- .../dataregion/tsfile/fileset/TsFileSet.java | 5 +- .../org/apache/iotdb/db/utils/io/IOUtils.java | 4 +- .../dataregion/DataRegionTest.java | 128 +++++++----- .../tsfile/evolution/EvolvedSchemaTest.java | 12 +- .../evolution/SchemaEvolutionFileTest.java | 11 +- .../apache/iotdb/commons/utils/FileUtils.java | 81 -------- .../src/main/thrift/datanode.thrift | 8 + 27 files changed, 535 insertions(+), 256 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedEvolveSchemaNode.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java index cdce05b46c0b7..c3141b33c815e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java @@ -34,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedInsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.EvolveSchemaNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertMultiTabletsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; @@ -286,6 +287,17 @@ public TSStatus visitDeleteData( } } + @Override + public TSStatus visitEvolveSchemaNode(EvolveSchemaNode node, DataRegion dataRegion) { + try { + dataRegion.applySchemaEvolution(node.getSchemaEvolutions()); + return StatusUtils.OK; + } catch (final IOException e) { + LOGGER.error("Error in executing plan node: {}", node, e); + return new TSStatus(TSStatusCode.WRITE_PROCESS_ERROR.getStatusCode()); + } + } + @Override public TSStatus visitPipeEnrichedDeleteDataNode( final PipeEnrichedDeleteDataNode node, final DataRegion context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index dcc62fcd3c8f1..b1fb833a38ec6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -158,8 +158,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.view.DeleteLogicalViewNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.view.RollbackLogicalViewBlackListNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedEvolveSchemaNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedNonWritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.EvolveSchemaNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; @@ -195,6 +197,7 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeSpaceQuotaManager; import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeThrottleQuotaManager; import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; @@ -233,6 +236,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TCreateTriggerInstanceReq; import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatReq; import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatResp; +import org.apache.iotdb.mpp.rpc.thrift.TDataRegionEvolveSchemaReq; import org.apache.iotdb.mpp.rpc.thrift.TDeactivateTemplateReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteColumnDataReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataForDeleteSchemaReq; @@ -787,6 +791,25 @@ public TSStatus deleteDataForDeleteSchema(final TDeleteDataForDeleteSchemaReq re .getStatus()); } + @Override + public TSStatus evolveSchemaInDataRegion(final TDataRegionEvolveSchemaReq req) { + final List schemaEvolutions = + SchemaEvolution.createListFrom(req.schemaEvolutions); + return executeInternalSchemaTask( + req.getDataRegionIdList(), + consensusGroupId -> + new RegionWriteExecutor() + .execute( + new DataRegionId(consensusGroupId.getId()), + // Now the deletion plan may be re-collected here by pipe, resulting multiple + // transfer to delete time series plan. Now just ignore. + req.isSetIsGeneratedByPipe() && req.isIsGeneratedByPipe() + ? new PipeEnrichedEvolveSchemaNode( + new EvolveSchemaNode(new PlanNodeId(""), schemaEvolutions)) + : new EvolveSchemaNode(new PlanNodeId(""), schemaEvolutions)) + .getStatus()); + } + @Override public TSStatus deleteTimeSeries(final TDeleteTimeSeriesReq req) throws TException { final PathPatternTree patternTree = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 5a0bdcc3cafdd..95a121e61b421 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -60,6 +60,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.view.DeleteLogicalViewNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.view.RollbackLogicalViewBlackListNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedEvolveSchemaNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedInsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedNonWritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedWritePlanNode; @@ -321,6 +322,7 @@ public enum PlanNodeType { RELATIONAL_INSERT_ROWS((short) 2002), RELATIONAL_DELETE_DATA((short) 2003), EVOLVE_SCHEMA((short) 2004), + PIPE_ENRICHED_EVOLVE_SCHEMA((short) 2005), ; public static final int BYTES = Short.BYTES; @@ -366,6 +368,8 @@ public static PlanNode deserializeFromWAL(DataInputStream stream) throws IOExcep return RelationalDeleteDataNode.deserializeFromWAL(stream); case 2004: return EvolveSchemaNode.deserializeFromWAL(stream); + case 2005: + return PipeEnrichedEvolveSchemaNode.deserializeFromWAL(stream); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } @@ -394,6 +398,8 @@ public static PlanNode deserializeFromWAL(ByteBuffer buffer) { return RelationalDeleteDataNode.deserializeFromWAL(buffer); case 2004: return EvolveSchemaNode.deserializeFromWAL(buffer); + case 2005: + return PipeEnrichedEvolveSchemaNode.deserializeFromWAL(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } @@ -721,6 +727,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { return RelationalDeleteDataNode.deserialize(buffer); case 2004: return EvolveSchemaNode.deserialize(buffer); + case 2005: + return PipeEnrichedEvolveSchemaNode.deserialize(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 9e8e183424758..d9218b3477430 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -57,6 +57,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.view.DeleteLogicalViewNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.view.RollbackLogicalViewBlackListNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedEvolveSchemaNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedInsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedNonWritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedWritePlanNode; @@ -111,6 +112,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.SourceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.TimeseriesRegionScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.EvolveSchemaNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertMultiTabletsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; @@ -632,6 +634,10 @@ public R visitDeleteData(RelationalDeleteDataNode node, C context) { return visitPlan(node, context); } + public R visitEvolveSchemaNode(EvolveSchemaNode node, C context) { + return visitPlan(node, context); + } + ///////////////////////////////////////////////////////////////////////////////////////////////// // Pipe Related Node ///////////////////////////////////////////////////////////////////////////////////////////////// @@ -644,6 +650,10 @@ public R visitPipeEnrichedDeleteDataNode(PipeEnrichedDeleteDataNode node, C cont return visitPlan(node, context); } + public R visitPipeEnrichedEvolveSchemaNode(PipeEnrichedEvolveSchemaNode node, C context) { + return visitPlan(node, context); + } + public R visitPipeEnrichedWritePlanNode(PipeEnrichedWritePlanNode node, C context) { return visitPlan(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedEvolveSchemaNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedEvolveSchemaNode.java new file mode 100644 index 0000000000000..794601eeb4e5e --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedEvolveSchemaNode.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe; + +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.consensus.index.ProgressIndex; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.EvolveSchemaNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.SearchNode; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.stream.Collectors; + +public class PipeEnrichedEvolveSchemaNode extends EvolveSchemaNode { + + private final EvolveSchemaNode evolveSchemaNode; + + public PipeEnrichedEvolveSchemaNode(final EvolveSchemaNode evolveSchemaNode) { + super(evolveSchemaNode.getPlanNodeId(), evolveSchemaNode.getSchemaEvolutions()); + this.evolveSchemaNode = evolveSchemaNode; + } + + public PlanNode EvolveSchemaNode() { + return evolveSchemaNode; + } + + @Override + public boolean isGeneratedByPipe() { + return evolveSchemaNode.isGeneratedByPipe(); + } + + @Override + public void markAsGeneratedByPipe() { + evolveSchemaNode.markAsGeneratedByPipe(); + } + + @Override + public PlanNodeId getPlanNodeId() { + return evolveSchemaNode.getPlanNodeId(); + } + + @Override + public void setPlanNodeId(final PlanNodeId id) { + evolveSchemaNode.setPlanNodeId(id); + } + + @Override + public ProgressIndex getProgressIndex() { + return evolveSchemaNode.getProgressIndex(); + } + + @Override + public void setProgressIndex(ProgressIndex progressIndex) { + evolveSchemaNode.setProgressIndex(progressIndex); + } + + @Override + public List getChildren() { + return evolveSchemaNode.getChildren(); + } + + @Override + public void addChild(final PlanNode child) { + evolveSchemaNode.addChild(child); + } + + @Override + public PlanNodeType getType() { + return PlanNodeType.PIPE_ENRICHED_EVOLVE_SCHEMA; + } + + @Override + public PlanNode clone() { + return new PipeEnrichedEvolveSchemaNode((EvolveSchemaNode) evolveSchemaNode.clone()); + } + + @Override + public PlanNode createSubNode(final int subNodeId, final int startIndex, final int endIndex) { + return new PipeEnrichedEvolveSchemaNode( + (EvolveSchemaNode) evolveSchemaNode.createSubNode(subNodeId, startIndex, endIndex)); + } + + @Override + public PlanNode cloneWithChildren(final List children) { + return new PipeEnrichedEvolveSchemaNode( + (EvolveSchemaNode) evolveSchemaNode.cloneWithChildren(children)); + } + + @Override + public int allowedChildCount() { + return evolveSchemaNode.allowedChildCount(); + } + + @Override + public List getOutputColumnNames() { + return evolveSchemaNode.getOutputColumnNames(); + } + + @Override + public R accept(final PlanVisitor visitor, final C context) { + return visitor.visitPipeEnrichedEvolveSchemaNode(this, context); + } + + @Override + protected void serializeAttributes(final ByteBuffer byteBuffer) { + PlanNodeType.PIPE_ENRICHED_EVOLVE_SCHEMA.serialize(byteBuffer); + evolveSchemaNode.serialize(byteBuffer); + } + + @Override + protected void serializeAttributes(final DataOutputStream stream) throws IOException { + PlanNodeType.PIPE_ENRICHED_EVOLVE_SCHEMA.serialize(stream); + evolveSchemaNode.serialize(stream); + } + + public static PipeEnrichedEvolveSchemaNode deserialize(final ByteBuffer buffer) { + return new PipeEnrichedEvolveSchemaNode((EvolveSchemaNode) PlanNodeType.deserialize(buffer)); + } + + @Override + public boolean equals(final Object o) { + return o instanceof PipeEnrichedEvolveSchemaNode + && evolveSchemaNode.equals(((PipeEnrichedEvolveSchemaNode) o).evolveSchemaNode); + } + + @Override + public int hashCode() { + return evolveSchemaNode.hashCode(); + } + + @Override + public TRegionReplicaSet getRegionReplicaSet() { + return evolveSchemaNode.getRegionReplicaSet(); + } + + @Override + public List splitByPartition(final IAnalysis analysis) { + return evolveSchemaNode.splitByPartition(analysis).stream() + .map( + plan -> + plan instanceof PipeEnrichedEvolveSchemaNode + ? plan + : new PipeEnrichedEvolveSchemaNode((EvolveSchemaNode) plan)) + .collect(Collectors.toList()); + } + + @Override + public void serializeToWAL(final IWALByteBufferView buffer) { + evolveSchemaNode.serializeToWAL(buffer); + } + + @Override + public int serializedSize() { + return evolveSchemaNode.serializedSize(); + } + + @Override + public SearchNode merge(List searchNodes) { + List unrichedNodes = + searchNodes.stream() + .map( + searchNode -> + (SearchNode) ((PipeEnrichedEvolveSchemaNode) searchNode).EvolveSchemaNode()) + .collect(Collectors.toList()); + return new PipeEnrichedEvolveSchemaNode( + (EvolveSchemaNode) evolveSchemaNode.merge(unrichedNodes)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/EvolveSchemaNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/EvolveSchemaNode.java index f85ccbfe17af5..b4fc82e0710ef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/EvolveSchemaNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/EvolveSchemaNode.java @@ -19,40 +19,41 @@ package org.apache.iotdb.db.queryengine.plan.planner.plan.node.write; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; import org.apache.iotdb.db.utils.io.IOUtils; + import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + public class EvolveSchemaNode extends SearchNode implements WALEntryValue { - private static final Logger LOGGER = - LoggerFactory.getLogger(EvolveSchemaNode.class); + private static final Logger LOGGER = LoggerFactory.getLogger(EvolveSchemaNode.class); protected TRegionReplicaSet regionReplicaSet; protected ProgressIndex progressIndex; private final List schemaEvolutions; - public EvolveSchemaNode(PlanNodeId id, - List schemaEvolutions) { + public EvolveSchemaNode(PlanNodeId id, List schemaEvolutions) { super(id); this.schemaEvolutions = schemaEvolutions; } @@ -176,4 +177,13 @@ public void serializeToWAL(IWALByteBufferView buffer) { public int serializedSize() { return 0; } + + public List getSchemaEvolutions() { + return schemaEvolutions; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitEvolveSchemaNode(this, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index b2a340d14bd8c..c37b84d22b4d7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -112,6 +112,7 @@ import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessorInfo; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry.ModType; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; @@ -128,8 +129,10 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.ColumnRename; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.TableRename; import org.apache.iotdb.db.storageengine.dataregion.tsfile.fileset.TsFileSet; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; @@ -148,6 +151,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.WALFlushListener; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.WALRecoverListener; import org.apache.iotdb.db.storageengine.load.disk.ILoadDiskSelector; +import org.apache.iotdb.db.storageengine.load.disk.ILoadDiskSelector.DiskDirectorySelector; import org.apache.iotdb.db.storageengine.load.limiter.LoadTsFileRateLimiter; import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; @@ -167,10 +171,10 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.external.commons.io.FileUtils; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.fileSystem.FSFactoryProducer; import org.apache.tsfile.fileSystem.FSType; import org.apache.tsfile.fileSystem.fsFactory.FSFactory; @@ -452,7 +456,7 @@ public DataRegion(String databaseName, String id) { } private void initDiskSelector() { - final ILoadDiskSelector.DiskDirectorySelector selector = + final DiskDirectorySelector selector = (sourceDirectory, fileName, tierLevel) -> { try { return TierManager.getInstance() @@ -1021,15 +1025,12 @@ public File getDataRegionSysDir() { } private List recoverTsFileSets( - long partitionId, - Map> tsFileSetMap - ) { + long partitionId, Map> tsFileSetMap) { List tsFileSets = tsFileSetMap.computeIfAbsent( partitionId, pid -> { - File fileSetDir = - new File(getFileSetsDir(partitionId)); + File fileSetDir = new File(getFileSetsDir(partitionId)); File[] fileSets = fileSetDir.listFiles(); if (fileSets == null || fileSets.length == 0) { return Collections.emptyList(); @@ -1040,9 +1041,7 @@ private List recoverTsFileSets( try { tsFileSet = new TsFileSet( - Long.parseLong(fileSet.getName()), - fileSetDir.getAbsolutePath(), - true); + Long.parseLong(fileSet.getName()), fileSetDir.getAbsolutePath(), true); } catch (NumberFormatException e) { continue; } @@ -1058,7 +1057,6 @@ private List recoverTsFileSets( return tsFileSets; } - private Callable recoverFilesInPartition( long partitionId, DataRegionRecoveryContext context, @@ -1234,23 +1232,48 @@ public void recordSchemaEvolution(List schemaEvolutions) { for (Entry partitionVersionEntry : partitionMaxFileVersions.entrySet()) { long partitionId = partitionVersionEntry.getKey(); long maxVersion = partitionVersionEntry.getValue(); - lastTsFileSetMap.compute(partitionId, (pid, lastSet) -> { - if (lastSet == null) { - lastSet = createNewFileSet(maxVersion, partitionId); - } else if (lastSet.getEndVersion() < maxVersion) { - lastSet = createNewFileSet(maxVersion, partitionId); - } - try { - lastSet.appendSchemaEvolution(schemaEvolutions); - } catch (IOException e) { - logger.error("Cannot append schema evolutions to fileSets in partition {}-{}", dataRegionId, partitionId, e); - } - return lastSet; - }); + lastTsFileSetMap.compute( + partitionId, + (pid, lastSet) -> { + if (lastSet == null) { + lastSet = createNewFileSet(maxVersion, partitionId); + } else if (lastSet.getEndVersion() < maxVersion) { + lastSet = createNewFileSet(maxVersion, partitionId); + } + try { + lastSet.appendSchemaEvolution(schemaEvolutions); + } catch (IOException e) { + logger.error( + "Cannot append schema evolutions to fileSets in partition {}-{}", + dataRegionId, + partitionId, + e); + } + return lastSet; + }); } } public void applySchemaEvolutionToObjects(List schemaEvolutions) { + for (SchemaEvolution schemaEvolution : schemaEvolutions) { + if (schemaEvolution instanceof TableRename) { + TableRename tableRename = (TableRename) schemaEvolution; + renameTableForObjects(tableRename.getNameBefore(), tableRename.getNameAfter()); + } else if (schemaEvolution instanceof ColumnRename) { + ColumnRename columnRename = (ColumnRename) schemaEvolution; + if (columnRename.getDataType() == TSDataType.OBJECT) { + renameMeasurementForObjects(columnRename.getTableName(), columnRename.getNameBefore(), columnRename.getNameAfter()); + } + } + } + } + + private void renameTableForObjects(String nameBefore, String nameAfter) { + // TODO-SchemaEvolution + throw new UnsupportedOperationException(); + } + + private void renameMeasurementForObjects(String tableName, String nameBefore, String nameAfter) { // TODO-SchemaEvolution throw new UnsupportedOperationException(); } @@ -1686,7 +1709,7 @@ private List insertToTsFileProcessors( } List executedInsertRowNodeList = new ArrayList<>(); - for (Map.Entry entry : tsFileProcessorMap.entrySet()) { + for (Entry entry : tsFileProcessorMap.entrySet()) { TsFileProcessor tsFileProcessor = entry.getKey(); InsertRowsNode subInsertRowsNode = entry.getValue(); try { @@ -2661,7 +2684,8 @@ private List getFileResourceListForQuery( deviceIdBackThen = evolvedSchema.rewriteDeviceId(singleDeviceId); } - if (!tsFileResource.isSatisfied(deviceIdBackThen, globalTimeFilter, isSeq, context.isDebug())) { + if (!tsFileResource.isSatisfied( + deviceIdBackThen, globalTimeFilter, isSeq, context.isDebug())) { continue; } try { @@ -2885,12 +2909,12 @@ private List logDeletionInWAL(RelationalDeleteDataNode deleteD for (TableDeletionEntry modEntry : deleteDataNode.getModEntries()) { long startTime = modEntry.getStartTime(); long endTime = modEntry.getEndTime(); - for (Map.Entry entry : workSequenceTsFileProcessors.entrySet()) { + for (Entry entry : workSequenceTsFileProcessors.entrySet()) { if (TimePartitionUtils.satisfyPartitionId(startTime, endTime, entry.getKey())) { involvedProcessors.add(entry.getValue()); } } - for (Map.Entry entry : workUnsequenceTsFileProcessors.entrySet()) { + for (Entry entry : workUnsequenceTsFileProcessors.entrySet()) { if (TimePartitionUtils.satisfyPartitionId(startTime, endTime, entry.getKey())) { involvedProcessors.add(entry.getValue()); } @@ -2926,13 +2950,13 @@ private List logDeletionInWAL( DeleteDataNode deleteDataNode = new DeleteDataNode(new PlanNodeId(""), Collections.singletonList(path), startTime, endTime); deleteDataNode.setSearchIndex(searchIndex); - for (Map.Entry entry : workSequenceTsFileProcessors.entrySet()) { + for (Entry entry : workSequenceTsFileProcessors.entrySet()) { if (TimePartitionUtils.satisfyPartitionId(startTime, endTime, entry.getKey())) { WALFlushListener walFlushListener = entry.getValue().logDeleteDataNodeInWAL(deleteDataNode); walFlushListeners.add(walFlushListener); } } - for (Map.Entry entry : workUnsequenceTsFileProcessors.entrySet()) { + for (Entry entry : workUnsequenceTsFileProcessors.entrySet()) { if (TimePartitionUtils.satisfyPartitionId(startTime, endTime, entry.getKey())) { WALFlushListener walFlushListener = entry.getValue().logDeleteDataNodeInWAL(deleteDataNode); walFlushListeners.add(walFlushListener); @@ -3060,7 +3084,7 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M ITimeIndex timeIndex = sealedTsFile.getTimeIndex(); if ((timeIndex instanceof ArrayDeviceTimeIndex) - && (deletion.getType() == ModEntry.ModType.TABLE_DELETION)) { + && (deletion.getType() == ModType.TABLE_DELETION)) { ArrayDeviceTimeIndex deviceTimeIndex = (ArrayDeviceTimeIndex) timeIndex; Set devicesInFile = deviceTimeIndex.getDevices(); boolean onlyOneTable = false; @@ -4225,7 +4249,7 @@ public void insert(InsertRowsOfOneDeviceNode insertRowsOfOneDeviceNode) // infoForMetrics[2]: ScheduleWalTimeCost // infoForMetrics[3]: ScheduleMemTableTimeCost // infoForMetrics[4]: InsertedPointsNumber - for (Map.Entry entry : tsFileProcessorMap.entrySet()) { + for (Entry entry : tsFileProcessorMap.entrySet()) { TsFileProcessor tsFileProcessor = entry.getKey(); InsertRowsNode subInsertRowsNode = entry.getValue(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DeviceLastFlushTime.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DeviceLastFlushTime.java index e7c13f149a9fd..3e72acaa34dbc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DeviceLastFlushTime.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DeviceLastFlushTime.java @@ -19,10 +19,9 @@ package org.apache.iotdb.db.storageengine.dataregion; -import java.util.List; -import java.util.stream.Collectors; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.TableRename; + import org.apache.tsfile.file.metadata.IDeviceID; import java.util.HashMap; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/HashLastFlushTimeMap.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/HashLastFlushTimeMap.java index cf8badd6d089c..d02835015eb58 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/HashLastFlushTimeMap.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/HashLastFlushTimeMap.java @@ -19,20 +19,16 @@ package org.apache.iotdb.db.storageengine.dataregion; -import java.util.List; -import java.util.Map.Entry; -import java.util.stream.Collectors; import org.apache.iotdb.db.storageengine.StorageEngine; - -import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.TableRename; + import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; public class HashLastFlushTimeMap implements ILastFlushTimeMap { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/ILastFlushTime.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/ILastFlushTime.java index f78b6822e9038..9b685407326f0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/ILastFlushTime.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/ILastFlushTime.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.storageengine.dataregion; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; + import org.apache.tsfile.file.metadata.IDeviceID; public interface ILastFlushTime { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/ILastFlushTimeMap.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/ILastFlushTimeMap.java index 40186cef57357..ca3a5e37ced69 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/ILastFlushTimeMap.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/ILastFlushTimeMap.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.storageengine.dataregion; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; + import org.apache.tsfile.file.metadata.IDeviceID; import java.util.Map; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PartitionLastFlushTime.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PartitionLastFlushTime.java index 100d023861872..e37ce43b930a0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PartitionLastFlushTime.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PartitionLastFlushTime.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.storageengine.dataregion; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; + import org.apache.tsfile.file.metadata.IDeviceID; public class PartitionLastFlushTime implements ILastFlushTime { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTask.java index f727e2846e752..297518538dd08 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTask.java @@ -19,12 +19,14 @@ package org.apache.iotdb.db.storageengine.dataregion.task; -import java.io.IOException; -import java.io.InputStream; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.utils.io.StreamSerializable; + import org.apache.tsfile.utils.ReadWriteForEncodingUtils; +import java.io.IOException; +import java.io.InputStream; + public interface DataRegionTask extends Runnable, StreamSerializable { long getTaskId(); @@ -38,7 +40,8 @@ enum TaskType { } @SuppressWarnings("SwitchStatementWithTooFewBranches") - static DataRegionTask createFrom(InputStream stream, long taskId, DataRegion dataRegion) throws IOException { + static DataRegionTask createFrom(InputStream stream, long taskId, DataRegion dataRegion) + throws IOException { int typeOrdinal = ReadWriteForEncodingUtils.readVarInt(stream); if (typeOrdinal < 0 || typeOrdinal >= TaskType.values().length) { throw new IOException("Invalid task type: " + typeOrdinal); @@ -49,10 +52,10 @@ static DataRegionTask createFrom(InputStream stream, long taskId, DataRegion dat DataRegionTask task; switch (taskType) { case SchemaEvolutionTask: - task = new SchemaEvolutionTask(dataRegion); - break; - default: - throw new IOException("Invalid task type: " + taskType); + task = new SchemaEvolutionTask(dataRegion); + break; + default: + throw new IOException("Invalid task type: " + taskType); } task.deserialize(stream); task.setTaskId(taskId); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTaskManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTaskManager.java index 5441b9b19c7bf..59339aed00795 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTaskManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/DataRegionTaskManager.java @@ -19,6 +19,11 @@ package org.apache.iotdb.db.storageengine.dataregion.task; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; @@ -27,9 +32,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.concurrent.atomic.AtomicLong; -import org.apache.iotdb.db.storageengine.dataregion.DataRegion; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; @SuppressWarnings("ResultOfMethodCallIgnored") public class DataRegionTaskManager { @@ -54,17 +56,19 @@ public void recover() { return; } - Arrays.sort(files, (f1, f2) -> { - String fileName1 = f1.getName(); - int suffixIndex1 = fileName1.indexOf("."); - long taskId1 = Long.parseLong(fileName1.substring(0, suffixIndex1)); + Arrays.sort( + files, + (f1, f2) -> { + String fileName1 = f1.getName(); + int suffixIndex1 = fileName1.indexOf("."); + long taskId1 = Long.parseLong(fileName1.substring(0, suffixIndex1)); - String fileName2 = f2.getName(); - int suffixIndex2 = fileName2.indexOf("."); - long taskId2 = Long.parseLong(fileName1.substring(0, suffixIndex2)); + String fileName2 = f2.getName(); + int suffixIndex2 = fileName2.indexOf("."); + long taskId2 = Long.parseLong(fileName1.substring(0, suffixIndex2)); - return Long.compare(taskId1, taskId2); - }); + return Long.compare(taskId1, taskId2); + }); for (File file : files) { String fileName = file.getName(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/SchemaEvolutionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/SchemaEvolutionTask.java index e3bae4c571c02..9a361ca4700e9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/SchemaEvolutionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/task/SchemaEvolutionTask.java @@ -19,14 +19,16 @@ package org.apache.iotdb.db.storageengine.dataregion.task; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; + +import org.apache.tsfile.utils.ReadWriteForEncodingUtils; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.List; -import org.apache.iotdb.db.storageengine.dataregion.DataRegion; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; -import org.apache.tsfile.utils.ReadWriteForEncodingUtils; public class SchemaEvolutionTask implements DataRegionTask { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java index 4986107a32d60..80ff4b0f3ba41 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java @@ -19,8 +19,6 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; -import java.nio.ByteBuffer; -import org.apache.iotdb.session.subscription.payload.SubscriptionSessionDataSet.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -28,6 +26,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; /** A schema evolution operation that renames a column in a table schema. */ public class ColumnRename implements SchemaEvolution { @@ -40,10 +39,11 @@ public class ColumnRename implements SchemaEvolution { // for deserialization public ColumnRename() {} - public ColumnRename(String tableName, String nameBefore, String nameAfter) { + public ColumnRename(String tableName, String nameBefore, String nameAfter, TSDataType dataType) { this.tableName = tableName.toLowerCase(); this.nameBefore = nameBefore.toLowerCase(); this.nameAfter = nameAfter.toLowerCase(); + this.dataType = dataType; } @Override @@ -72,7 +72,7 @@ public void deserialize(InputStream stream) throws IOException { nameBefore = ReadWriteIOUtils.readVarIntString(stream); nameAfter = ReadWriteIOUtils.readVarIntString(stream); byte category = ReadWriteIOUtils.readByte(stream); - if (category != -1) { + if (category != -1) { dataType = TSDataType.values()[category]; } } @@ -93,7 +93,7 @@ public void deserialize(ByteBuffer buffer) { nameBefore = ReadWriteIOUtils.readVarIntString(buffer); nameAfter = ReadWriteIOUtils.readVarIntString(buffer); byte category = ReadWriteIOUtils.readByte(buffer); - if (category != -1) { + if (category != -1) { dataType = TSDataType.values()[category]; } } @@ -102,8 +102,19 @@ public TSDataType getDataType() { return dataType; } - public void setDataType( - TSDataType dataType) { + public void setDataType(TSDataType dataType) { this.dataType = dataType; } + + public String getTableName() { + return tableName; + } + + public String getNameBefore() { + return nameBefore; + } + + public String getNameAfter() { + return nameAfter; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java index cd899ccc97ca8..8913d795eb37e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java @@ -19,13 +19,13 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; -import java.util.HashMap; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; + import java.util.LinkedHashMap; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; -import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.IDeviceID.Factory; public class EvolvedSchema { // the evolved table names after applying all schema evolution operations @@ -95,10 +95,12 @@ public int hashCode() { @Override public String toString() { - return "EvolvedSchema{" + - "originalTableNames=" + originalTableNames + - ", originalColumnNames=" + originalColumnNames + - '}'; + return "EvolvedSchema{" + + "originalTableNames=" + + originalTableNames + + ", originalColumnNames=" + + originalColumnNames + + '}'; } public IDeviceID rewriteDeviceId(IDeviceID deviceID) { @@ -128,32 +130,35 @@ public static EvolvedSchema deepCopy(EvolvedSchema evolvedSchema) { } public static EvolvedSchema merge(EvolvedSchema... schemas) { - EvolvedSchema firstNotNullSchema = null; - int i = 0; - for (; i < schemas.length; i++) { - if (schemas[i] != null) { - firstNotNullSchema = schemas[i]; - i++; - break; - } + EvolvedSchema firstNotNullSchema = null; + int i = 0; + for (; i < schemas.length; i++) { + if (schemas[i] != null) { + firstNotNullSchema = schemas[i]; + i++; + break; } + } - if (firstNotNullSchema == null) { - return null; - } - EvolvedSchema mergedSchema = deepCopy(firstNotNullSchema); + if (firstNotNullSchema == null) { + return null; + } + EvolvedSchema mergedSchema = deepCopy(firstNotNullSchema); for (; i < schemas.length; i++) { if (schemas[i] != null) { EvolvedSchema newSchema = schemas[i]; - for (Entry finalOriginalTableName : newSchema.originalTableNames.entrySet()) { + for (Entry finalOriginalTableName : + newSchema.originalTableNames.entrySet()) { if (!finalOriginalTableName.getValue().isEmpty()) { - mergedSchema.renameTable(finalOriginalTableName.getValue(), finalOriginalTableName.getKey()); + mergedSchema.renameTable( + finalOriginalTableName.getValue(), finalOriginalTableName.getKey()); } } - for (Entry> finalTableNameColumnNameMapEntry : newSchema.originalColumnNames.entrySet()) { - for (Entry finalColNameOriginalColNameEntry : finalTableNameColumnNameMapEntry.getValue() - .entrySet()) { + for (Entry> finalTableNameColumnNameMapEntry : + newSchema.originalColumnNames.entrySet()) { + for (Entry finalColNameOriginalColNameEntry : + finalTableNameColumnNameMapEntry.getValue().entrySet()) { if (!finalColNameOriginalColNameEntry.getValue().isEmpty()) { String finalTableName = finalTableNameColumnNameMapEntry.getKey(); String finalColName = finalColNameOriginalColNameEntry.getKey(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolution.java index b017f6509b2c5..f1ac8edbcfa41 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolution.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; -import java.nio.ByteBuffer; import org.apache.iotdb.db.utils.io.BufferSerializable; import org.apache.iotdb.db.utils.io.StreamSerializable; @@ -27,6 +26,9 @@ import java.io.IOException; import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; /** A schema evolution operation that can be applied to a TableSchemaMap. */ public interface SchemaEvolution extends StreamSerializable, BufferSerializable { @@ -77,4 +79,13 @@ static SchemaEvolution createFrom(ByteBuffer buffer) { evolution.deserialize(buffer); return evolution; } + + static List createListFrom(ByteBuffer buffer) { + int size = ReadWriteForEncodingUtils.readVarInt(buffer); + List list = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + list.add(createFrom(buffer)); + } + return list; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java index e90c1ac32cac6..1c4343cd1544a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java @@ -42,6 +42,7 @@ public SchemaEvolutionFile(String filePath) { /** * Recover the SchemaEvolutionFile if it is broken. + * * @return true if the file exists false otherwise * @throws IOException if the file cannot be recovered */ diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java index dfc80974e85fe..a37557f45aa88 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/TableRename.java @@ -19,11 +19,6 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.apache.iotdb.db.queryengine.execution.schedule.queue.ID; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; @@ -32,6 +27,10 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; /** A schema evolution operation that renames a table in a schema map. */ public class TableRename implements SchemaEvolution { @@ -107,9 +106,10 @@ public IDeviceID rewriteDeviceId(IDeviceID deviceId) { } public void rewriteMap(Map map) { - List affectedDeviceId = map.keySet().stream() - .filter(k -> k.getTableName().equals(getNameBefore())).collect( - Collectors.toList()); + List affectedDeviceId = + map.keySet().stream() + .filter(k -> k.getTableName().equals(getNameBefore())) + .collect(Collectors.toList()); for (IDeviceID deviceID : affectedDeviceId) { IDeviceID newDeviceId = rewriteDeviceId(deviceID); T removed = map.remove(deviceID); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java index 5c43418fa64f7..aaa7bb195ca28 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java @@ -123,9 +123,6 @@ public long getEndVersion() { @Override public String toString() { - return "TsFileSet{" + - "endVersion=" + endVersion + - ", fileSetDir=" + fileSetDir + - '}'; + return "TsFileSet{" + "endVersion=" + endVersion + ", fileSetDir=" + fileSetDir + '}'; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/io/IOUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/io/IOUtils.java index f41ae6de8def0..3a1841190cb2b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/io/IOUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/io/IOUtils.java @@ -19,12 +19,12 @@ package org.apache.iotdb.db.utils.io; +import org.apache.tsfile.utils.ReadWriteForEncodingUtils; + import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.List; -import org.apache.tsfile.utils.ReadWriteForEncodingUtils; // TODO: move to TsFile public class IOUtils { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index e896a599c4335..014874e589321 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.storageengine.dataregion; -import java.nio.charset.StandardCharsets; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.consensus.DataRegionId; @@ -94,6 +93,7 @@ import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -1789,41 +1789,59 @@ public void testSchemaEvolution() throws IllegalPathException, WriteProcessException, QueryProcessException, IOException { String[] measurements = {"tag1", "s1", "s2"}; MeasurementSchema[] measurementSchemas = { - new MeasurementSchema("tag1", TSDataType.STRING), - new MeasurementSchema("s1", TSDataType.INT64), - new MeasurementSchema("s2", TSDataType.DOUBLE) + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.INT64), + new MeasurementSchema("s2", TSDataType.DOUBLE) }; - RelationalInsertRowNode insertRowNode = new RelationalInsertRowNode(new PlanNodeId(""), - new PartialPath(new String[] {"table1"}), - true, - measurements, - new TSDataType[]{TSDataType.STRING, TSDataType.INT64, TSDataType.DOUBLE}, - measurementSchemas, - 10, - new Object[]{new Binary("tag1".getBytes(StandardCharsets.UTF_8)), 1L, 1.0}, - false, - new TsTableColumnCategory[]{TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD}); + RelationalInsertRowNode insertRowNode = + new RelationalInsertRowNode( + new PlanNodeId(""), + new PartialPath(new String[] {"table1"}), + true, + measurements, + new TSDataType[] {TSDataType.STRING, TSDataType.INT64, TSDataType.DOUBLE}, + measurementSchemas, + 10, + new Object[] {new Binary("tag1".getBytes(StandardCharsets.UTF_8)), 1L, 1.0}, + false, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD + }); dataRegion.insert(insertRowNode); // table1 -> table2 dataRegion.applySchemaEvolution(Collections.singletonList(new TableRename("table1", "table2"))); // cannot query with the old name - IDeviceID deviceID1 = Factory.DEFAULT_FACTORY.create(new String[]{"table1", "tag1"}); - List fullPaths = Arrays.asList( - new AlignedFullPath(deviceID1, Arrays.asList(measurements), Arrays.asList(measurementSchemas)) - ); - QueryDataSource dataSource = dataRegion.query(fullPaths, deviceID1, new QueryContext(), null, - Collections.singletonList(0L), Long.MAX_VALUE); + IDeviceID deviceID1 = Factory.DEFAULT_FACTORY.create(new String[] {"table1", "tag1"}); + List fullPaths = + Arrays.asList( + new AlignedFullPath( + deviceID1, Arrays.asList(measurements), Arrays.asList(measurementSchemas))); + QueryDataSource dataSource = + dataRegion.query( + fullPaths, + deviceID1, + new QueryContext(), + null, + Collections.singletonList(0L), + Long.MAX_VALUE); assertTrue(dataSource.getSeqResources().isEmpty()); // can query with the new name - IDeviceID deviceID2 = Factory.DEFAULT_FACTORY.create(new String[]{"table2", "tag1"}); - fullPaths = Arrays.asList( - new AlignedFullPath(deviceID2, Arrays.asList(measurements), Arrays.asList(measurementSchemas)) - ); - dataSource = dataRegion.query(fullPaths, deviceID2, new QueryContext(), null, - Collections.singletonList(0L), Long.MAX_VALUE); + IDeviceID deviceID2 = Factory.DEFAULT_FACTORY.create(new String[] {"table2", "tag1"}); + fullPaths = + Arrays.asList( + new AlignedFullPath( + deviceID2, Arrays.asList(measurements), Arrays.asList(measurementSchemas))); + dataSource = + dataRegion.query( + fullPaths, + deviceID2, + new QueryContext(), + null, + Collections.singletonList(0L), + Long.MAX_VALUE); assertEquals(1, dataSource.getSeqResources().size()); DataNodeTableCache.getInstance() @@ -1832,32 +1850,50 @@ public void testSchemaEvolution() .commitUpdateTable(dataRegion.getDatabaseName(), StatementTestUtils.tableName(1), null); // write again with table1 - insertRowNode = new RelationalInsertRowNode(new PlanNodeId(""), - new PartialPath(new String[] {"table1"}), - true, - measurements, - new TSDataType[]{TSDataType.STRING, TSDataType.INT64, TSDataType.DOUBLE}, - measurementSchemas, - 10, - new Object[]{new Binary("tag1".getBytes(StandardCharsets.UTF_8)), 1L, 1.0}, - false, - new TsTableColumnCategory[]{TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD}); + insertRowNode = + new RelationalInsertRowNode( + new PlanNodeId(""), + new PartialPath(new String[] {"table1"}), + true, + measurements, + new TSDataType[] {TSDataType.STRING, TSDataType.INT64, TSDataType.DOUBLE}, + measurementSchemas, + 10, + new Object[] {new Binary("tag1".getBytes(StandardCharsets.UTF_8)), 1L, 1.0}, + false, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD + }); dataRegion.insert(insertRowNode); // can query with table1 - fullPaths = Arrays.asList( - new AlignedFullPath(deviceID1, Arrays.asList(measurements), Arrays.asList(measurementSchemas)) - ); - dataSource = dataRegion.query(fullPaths, deviceID1, new QueryContext(), null, - Collections.singletonList(0L), Long.MAX_VALUE); + fullPaths = + Arrays.asList( + new AlignedFullPath( + deviceID1, Arrays.asList(measurements), Arrays.asList(measurementSchemas))); + dataSource = + dataRegion.query( + fullPaths, + deviceID1, + new QueryContext(), + null, + Collections.singletonList(0L), + Long.MAX_VALUE); assertEquals(1, dataSource.getUnseqResources().size()); // can query with table2 - fullPaths = Arrays.asList( - new AlignedFullPath(deviceID2, Arrays.asList(measurements), Arrays.asList(measurementSchemas)) - ); - dataSource = dataRegion.query(fullPaths, deviceID2, new QueryContext(), null, - Collections.singletonList(0L), Long.MAX_VALUE); + fullPaths = + Arrays.asList( + new AlignedFullPath( + deviceID2, Arrays.asList(measurements), Arrays.asList(measurementSchemas))); + dataSource = + dataRegion.query( + fullPaths, + deviceID2, + new QueryContext(), + null, + Collections.singletonList(0L), + Long.MAX_VALUE); assertEquals(1, dataSource.getSeqResources().size()); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchemaTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchemaTest.java index d9c76be4710fa..5f4f7e8280d32 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchemaTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchemaTest.java @@ -19,11 +19,13 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; -import static org.junit.Assert.assertEquals; +import org.apache.tsfile.enums.TSDataType; +import org.junit.Test; import java.util.Arrays; import java.util.List; -import org.junit.Test; + +import static org.junit.Assert.assertEquals; public class EvolvedSchemaTest { @@ -33,7 +35,7 @@ public void testMerge() { List schemaEvolutionList = Arrays.asList( new TableRename("t1", "t2"), - new ColumnRename("t2", "s1", "s2"), + new ColumnRename("t2", "s1", "s2", TSDataType.INT32), new TableRename("t3", "t1")); EvolvedSchema oldSchema = new EvolvedSchema(); EvolvedSchema allSchema = new EvolvedSchema(); @@ -44,7 +46,7 @@ public void testMerge() { schemaEvolutionList = Arrays.asList( new TableRename("t2", "t3"), - new ColumnRename("t3", "s2", "s1"), + new ColumnRename("t3", "s2", "s1", TSDataType.INT32), new TableRename("t1", "t2")); EvolvedSchema newSchema = new EvolvedSchema(); schemaEvolutionList.forEach(schemaEvolution -> schemaEvolution.applyTo(newSchema)); @@ -54,4 +56,4 @@ public void testMerge() { assertEquals(allSchema, mergedShema); } -} \ No newline at end of file +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFileTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFileTest.java index d4386dd8172f7..10348a92c17db 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFileTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFileTest.java @@ -19,13 +19,14 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; -import java.io.FileOutputStream; import org.apache.iotdb.db.utils.constant.TestConstant; +import org.apache.tsfile.enums.TSDataType; import org.junit.After; import org.junit.Test; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.util.Arrays; import java.util.List; @@ -51,7 +52,7 @@ public void testSchemaEvolutionFile() throws IOException { List schemaEvolutionList = Arrays.asList( new TableRename("t1", "t2"), - new ColumnRename("t2", "s1", "s2"), + new ColumnRename("t2", "s1", "s2", TSDataType.INT32), new TableRename("t3", "t1")); schemaEvolutionFile.append(schemaEvolutionList); @@ -67,7 +68,7 @@ public void testSchemaEvolutionFile() throws IOException { schemaEvolutionList = Arrays.asList( new TableRename("t2", "t3"), - new ColumnRename("t3", "s2", "s1"), + new ColumnRename("t3", "s2", "s1", TSDataType.INT32), new TableRename("t1", "t2")); schemaEvolutionFile.append(schemaEvolutionList); evolvedSchema = schemaEvolutionFile.readAsSchema(); @@ -97,7 +98,7 @@ public void testRecover() throws IOException { List schemaEvolutionList = Arrays.asList( new TableRename("t1", "t2"), - new ColumnRename("t2", "s1", "s2"), + new ColumnRename("t2", "s1", "s2", TSDataType.INT32), new TableRename("t3", "t1")); schemaEvolutionFile.append(schemaEvolutionList); @@ -111,7 +112,7 @@ public void testRecover() throws IOException { fileOutputStream.write(new byte[100]); } - schemaEvolutionFile = new SchemaEvolutionFile(files[0].getAbsolutePath()); + schemaEvolutionFile = new SchemaEvolutionFile(files[0].getAbsolutePath()); EvolvedSchema evolvedSchema = schemaEvolutionFile.readAsSchema(); assertEquals("t1", evolvedSchema.getOriginalTableName("t2")); assertEquals("s1", evolvedSchema.getOriginalColumnName("t2", "s2")); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java index a2510454e7379..baed8bcd537e7 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java @@ -19,12 +19,6 @@ package org.apache.iotdb.commons.utils; -import java.util.HashSet; -import java.util.Objects; -import java.util.Random; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.IntStream; import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.tsfile.external.commons.codec.digest.DigestUtils; @@ -340,81 +334,6 @@ public static boolean moveFileSafe(File source, File target) { return true; } - public static List applyReversedIndexesOnListV2( - final List filteredIndexes, final List originalList) { - // filteredIndexes.sort(null); if necessary - List filteredList = new ArrayList<>(originalList.size() - filteredIndexes.size()); - int filteredIndexPos = 0; - int processingIndex = 0; - for (; processingIndex < originalList.size(); processingIndex++) { - if (filteredIndexPos >= filteredIndexes.size()) { - // all filteredIndexes processed, add remaining to the filteredList - filteredList.addAll(originalList.subList(processingIndex, originalList.size())); - break; - } else { - int filteredIndex = filteredIndexes.get(filteredIndexPos); - if (filteredIndex == processingIndex) { - // the index is filtered, move to the next filtered pos - filteredIndexPos ++; - } else { - // the index is not filtered, add to the filteredList - filteredList.add(originalList.get(processingIndex)); - } - } - } - return filteredList; - } - - public static List applyReversedIndexesOnListV1( - final List filteredIndexes, final List originalList) { - final Set indexes = new HashSet<>(filteredIndexes); - return Objects.nonNull(originalList) - ? IntStream.range(0, originalList.size()) - .filter(index -> !indexes.contains(index)) // 保留不在排除列表中的下标 - .mapToObj(originalList::get) - .collect(Collectors.toList()) - : null; - } - - public static void main(String[] args) { - int elementNum = 10_000_000; - int filteredNum = elementNum / 10; - Random random = new Random(); - List originalList = IntStream.range(0, elementNum).boxed().collect(Collectors.toList()); - List filteredIndexes = new ArrayList<>(filteredNum); - for (int i = 0; i < filteredNum; i++) { - filteredIndexes.add(random.nextInt(elementNum)); - } - filteredIndexes = filteredIndexes.stream().sorted().distinct().collect(Collectors.toList()); - - long start = System.currentTimeMillis(); - List appliedList = applyReversedIndexesOnListV1(filteredIndexes, originalList); - System.out.println(System.currentTimeMillis() - start); - Set appliedSet = new HashSet<>(appliedList); - for (Integer filteredIndex : filteredIndexes) { - if (appliedSet.contains(filteredIndex)) { - System.out.println("Incorrect implementation"); - System.exit(-1); - } - } - - - start = System.currentTimeMillis(); - appliedList = WapplyReversedIndexesOnListV2(filteredIndexes, originalList); - System.out.println(System.currentTimeMillis() - start); - appliedSet = new HashSet<>(appliedList); - if (appliedList.size() != originalList.size() - filteredIndexes.size()) { - System.out.println("Incorrect implementation"); - System.exit(-1); - } - for (Integer filteredIndex : filteredIndexes) { - if (appliedSet.contains(filteredIndex)) { - System.out.println("Incorrect implementation"); - System.exit(-1); - } - } - } - public static File createHardLink(File sourceFile, File hardlink) throws IOException { if (!hardlink.getParentFile().exists() && !hardlink.getParentFile().mkdirs()) { synchronized (FileUtils.class) { diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index caaf44c16a770..fd4f15c1e1a37 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -459,6 +459,12 @@ struct TDeleteDataForDeleteSchemaReq { 3: optional bool isGeneratedByPipe } +struct TDataRegionEvolveSchemaReq { + 1: required list dataRegionIdList + 2: required binary schemaEvolutions + 3: optional bool isGeneratedByPipe +} + struct TDeleteTimeSeriesReq { 1: required list schemaRegionIdList 2: required binary pathPatternTree @@ -1076,6 +1082,8 @@ service IDataNodeRPCService { */ common.TSStatus deleteDataForDeleteSchema(TDeleteDataForDeleteSchemaReq req) + common.TSStatus evolveSchemaInDataRegion(TDataRegionEvolveSchemaReq req) + /** * Delete matched timeseries and remove according schema black list in target schemRegion */ From 5e680e087d0660faaa8a80c0caa4af92a73a1410 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 24 Dec 2025 14:55:59 +0800 Subject: [PATCH 11/49] Evolve schema in FileLoaderUtils --- .../execution/operator/source/FileLoaderUtils.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 35d8a1705ab00..cf128e33f18cb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; +import java.util.concurrent.LinkedBlockingDeque; import org.apache.iotdb.commons.path.AlignedFullPath; import org.apache.iotdb.commons.path.NonAlignedFullPath; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; @@ -34,6 +35,7 @@ import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.metadata.MemAlignedChunkMetadataLoader; import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.metadata.MemChunkMetadataLoader; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.utils.ModificationUtils; @@ -93,14 +95,18 @@ public static TimeseriesMetadata loadTimeSeriesMetadata( if (resource.isClosed()) { // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex // we should not ignore the non-exist of device in TsFileMetadata + EvolvedSchema evolvedSchema = resource.getMergedEvolvedSchema(); + IDeviceID deviceId = seriesPath.getDeviceId(); + String measurement = seriesPath.getMeasurement(); + timeSeriesMetadata = TimeSeriesMetadataCache.getInstance() .get( resource.getTsFilePath(), new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey( resource.getTsFileID(), - seriesPath.getDeviceId(), - seriesPath.getMeasurement()), + deviceId, + measurement), allSensors, context.ignoreNotExistsDevice() || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, @@ -110,7 +116,7 @@ public static TimeseriesMetadata loadTimeSeriesMetadata( long t2 = System.nanoTime(); List pathModifications = context.getPathModifications( - resource, seriesPath.getDeviceId(), seriesPath.getMeasurement()); + resource, deviceId, measurement); timeSeriesMetadata.setModified(!pathModifications.isEmpty()); timeSeriesMetadata.setChunkMetadataLoader( new DiskChunkMetadataLoader(resource, context, globalTimeFilter, pathModifications)); From 67bc84578aefef56db6abac881170f27bad53bc3 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 24 Dec 2025 18:57:40 +0800 Subject: [PATCH 12/49] add EvolvedSchema.toSchemaEvolutions --- .../tsfile/evolution/ColumnRename.java | 1 + .../tsfile/evolution/EvolvedSchema.java | 19 +++++++++++++++++++ .../tsfile/evolution/EvolvedSchemaTest.java | 18 ++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java index 80ff4b0f3ba41..9d13ce4f7e645 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java @@ -34,6 +34,7 @@ public class ColumnRename implements SchemaEvolution { private String tableName; private String nameBefore; private String nameAfter; + // to judge if the Object directories should be renamed private TSDataType dataType; // for deserialization diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java index 8913d795eb37e..9a8d2cda88a9a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; +import java.util.ArrayList; +import java.util.List; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; @@ -103,6 +105,23 @@ public String toString() { + '}'; } + public List toSchemaEvolutions() { + List schemaEvolutions = new ArrayList<>(); + originalTableNames.forEach((finalTableName, originalTableName) -> { + if (!originalTableName.isEmpty()) { + schemaEvolutions.add(new TableRename(originalTableName, finalTableName)); + } + }); + originalColumnNames.forEach((finalTableName, originalColumnNameMap) -> { + originalColumnNameMap.forEach((finalColumnName, originalColumnName) -> { + if (!originalColumnName.isEmpty()) { + schemaEvolutions.add(new ColumnRename(finalTableName, originalColumnName, finalColumnName, null)); + } + }); + }); + return schemaEvolutions; + } + public IDeviceID rewriteDeviceId(IDeviceID deviceID) { String tableName = deviceID.getTableName(); String originalTableName = getOriginalTableName(tableName); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchemaTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchemaTest.java index 5f4f7e8280d32..488971cc8c934 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchemaTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchemaTest.java @@ -56,4 +56,22 @@ public void testMerge() { assertEquals(allSchema, mergedShema); } + + @Test + public void testCovert() { + // t1 -> t2, t2.s1 -> t2.s2, t3 -> t1 + List schemaEvolutionList = + Arrays.asList( + new TableRename("t1", "t2"), + new ColumnRename("t2", "s1", "s2", TSDataType.INT32), + new TableRename("t3", "t1")); + EvolvedSchema oldSchema = new EvolvedSchema(); + schemaEvolutionList.forEach(schemaEvolution -> schemaEvolution.applyTo(oldSchema)); + + List convertedSchemaEvolutions = oldSchema.toSchemaEvolutions(); + EvolvedSchema newSchema = new EvolvedSchema(); + convertedSchemaEvolutions.forEach(schemaEvolution -> schemaEvolution.applyTo(newSchema)); + + assertEquals(oldSchema, newSchema); + } } From 5f64c454499aadceb6e3c9050c36017cf6c704d8 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 25 Dec 2025 17:45:46 +0800 Subject: [PATCH 13/49] support delete with sevo --- .../impl/DataNodeInternalRPCServiceImpl.java | 4 +- .../plan/analyze/AnalyzeUtils.java | 38 +-- .../storageengine/dataregion/DataRegion.java | 172 ++++++------- .../execute/utils/CompactionUtils.java | 2 +- .../modification/DeletionPredicate.java | 40 +-- .../{IDPredicate.java => TagPredicate.java} | 46 ++-- .../tsfile/evolution/EvolvedSchema.java | 38 ++- .../dataregion/tsfile/fileset/TsFileSet.java | 2 +- .../file/UnsealedTsFileRecoverPerformer.java | 2 +- .../db/metadata/path/PatternTreeMapTest.java | 2 +- .../pipe/consensus/DeletionRecoverTest.java | 2 +- .../pipe/consensus/DeletionResourceTest.java | 2 +- .../PipePlanTablePatternParseVisitorTest.java | 8 +- .../write/RelationalDeleteDataNodeTest.java | 8 +- .../dataregion/DataRegionTest.java | 243 +++++++++++++----- .../CompactionWithAllNullRowsTest.java | 12 +- .../modification/ModificationFileTest.java | 6 +- .../modification/TableDeletionEntryTest.java | 8 +- 18 files changed, 398 insertions(+), 237 deletions(-) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/{IDPredicate.java => TagPredicate.java} (89%) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index b1fb833a38ec6..84e3e17a4983d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -195,7 +195,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.settle.SettleRequestHandler; import org.apache.iotdb.db.storageengine.dataregion.flush.CompressionRatio; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeSpaceQuotaManager; @@ -1989,7 +1989,7 @@ public TSStatus deleteColumnData(final TDeleteColumnDataReq req) { new TableDeletionEntry( new DeletionPredicate( req.getTableName(), - new IDPredicate.NOP(), + new TagPredicate.NOP(), Collections.singletonList(req.getColumnName())), new TimeRange(Long.MIN_VALUE, Long.MAX_VALUE)), // the request is only sent to associated region diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java index 2e0b0b52fdc09..331c18d54c994 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java @@ -53,9 +53,9 @@ import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.table.DataNodeTreeViewSchemaUtils; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.And; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.SegmentExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.And; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.SegmentExactMatch; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -436,23 +436,23 @@ private static TableDeletionEntry parsePredicate(Expression expression, TsTable Queue expressionQueue = new LinkedList<>(); expressionQueue.add(expression); DeletionPredicate predicate = new DeletionPredicate(table.getTableName()); - IDPredicate idPredicate = null; + TagPredicate tagPredicate = null; TimeRange timeRange = new TimeRange(Long.MIN_VALUE, Long.MAX_VALUE, true); while (!expressionQueue.isEmpty()) { Expression currExp = expressionQueue.remove(); if (currExp instanceof LogicalExpression) { parseAndPredicate(((LogicalExpression) currExp), expressionQueue); } else if (currExp instanceof ComparisonExpression) { - idPredicate = - parseComparison(((ComparisonExpression) currExp), timeRange, idPredicate, table); + tagPredicate = + parseComparison(((ComparisonExpression) currExp), timeRange, tagPredicate, table); } else if (currExp instanceof IsNullPredicate) { - idPredicate = parseIsNull((IsNullPredicate) currExp, idPredicate, table); + tagPredicate = parseIsNull((IsNullPredicate) currExp, tagPredicate, table); } else { throw new SemanticException("Unsupported expression: " + currExp + " in " + expression); } } - if (idPredicate != null) { - predicate.setIdPredicate(idPredicate); + if (tagPredicate != null) { + predicate.setIdPredicate(tagPredicate); } if (timeRange.getStartTime() > timeRange.getEndTime()) { throw new SemanticException( @@ -472,8 +472,8 @@ private static void parseAndPredicate( expressionQueue.addAll(expression.getTerms()); } - private static IDPredicate parseIsNull( - IsNullPredicate isNullPredicate, IDPredicate oldPredicate, TsTable table) { + private static TagPredicate parseIsNull( + IsNullPredicate isNullPredicate, TagPredicate oldPredicate, TsTable table) { Expression leftHandExp = isNullPredicate.getValue(); if (!(leftHandExp instanceof Identifier)) { throw new SemanticException("Left hand expression is not an identifier: " + leftHandExp); @@ -486,25 +486,25 @@ private static IDPredicate parseIsNull( } // the first segment is the table name, so + 1 - IDPredicate newPredicate = new SegmentExactMatch(null, idColumnOrdinal + 1); + TagPredicate newPredicate = new SegmentExactMatch(null, idColumnOrdinal + 1); return combinePredicates(oldPredicate, newPredicate); } - private static IDPredicate combinePredicates(IDPredicate oldPredicate, IDPredicate newPredicate) { + private static TagPredicate combinePredicates(TagPredicate oldPredicate, TagPredicate newPredicate) { if (oldPredicate == null) { return newPredicate; } - if (oldPredicate instanceof IDPredicate.And) { + if (oldPredicate instanceof TagPredicate.And) { ((And) oldPredicate).add(newPredicate); return oldPredicate; } - return new IDPredicate.And(oldPredicate, newPredicate); + return new TagPredicate.And(oldPredicate, newPredicate); } - private static IDPredicate parseComparison( + private static TagPredicate parseComparison( ComparisonExpression comparisonExpression, TimeRange timeRange, - IDPredicate oldPredicate, + TagPredicate oldPredicate, TsTable table) { Expression left = comparisonExpression.getLeft(); Expression right = comparisonExpression.getRight(); @@ -556,11 +556,11 @@ private static IDPredicate parseComparison( "The column '" + columnName + "' does not exist or is not a tag column"); } - IDPredicate newPredicate = getIdPredicate(comparisonExpression, right, idColumnOrdinal); + TagPredicate newPredicate = getIdPredicate(comparisonExpression, right, idColumnOrdinal); return combinePredicates(oldPredicate, newPredicate); } - private static IDPredicate getIdPredicate( + private static TagPredicate getIdPredicate( ComparisonExpression comparisonExpression, Expression right, int idColumnOrdinal) { if (comparisonExpression.getOperator() != ComparisonExpression.Operator.EQUAL) { throw new SemanticException("The operator of tag predicate must be '=' for " + right); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index c37b84d22b4d7..835a0977fa520 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -772,7 +772,7 @@ protected void updateDeviceLastFlushTime(TsFileResource resource) { @SuppressWarnings("OptionalGetWithoutIsPresent") // checked above long endTime = resource.getEndTime(deviceId).get(); if (mergedEvolvedSchema != null) { - deviceId = mergedEvolvedSchema.rewriteDeviceId(deviceId); + deviceId = mergedEvolvedSchema.rewriteToOriginal(deviceId); } endTimeMap.put(deviceId, endTime); } @@ -794,7 +794,7 @@ protected void upgradeAndUpdateDeviceLastFlushTime( //noinspection OptionalGetWithoutIsPresent long endTime = resource.getEndTime(deviceId).get(); if (mergedEvolvedSchema != null) { - deviceId = mergedEvolvedSchema.rewriteDeviceId(deviceId); + deviceId = mergedEvolvedSchema.rewriteToOriginal(deviceId); } endTimeMap.put(deviceId, endTime); } @@ -1270,12 +1270,12 @@ public void applySchemaEvolutionToObjects(List schemaEvolutions private void renameTableForObjects(String nameBefore, String nameAfter) { // TODO-SchemaEvolution - throw new UnsupportedOperationException(); + // throw new UnsupportedOperationException(); } private void renameMeasurementForObjects(String tableName, String nameBefore, String nameAfter) { // TODO-SchemaEvolution - throw new UnsupportedOperationException(); + // throw new UnsupportedOperationException(); } /** @@ -2681,7 +2681,7 @@ private List getFileResourceListForQuery( EvolvedSchema evolvedSchema = tsFileResource.getMergedEvolvedSchema(); IDeviceID deviceIdBackThen = singleDeviceId; if (evolvedSchema != null) { - deviceIdBackThen = evolvedSchema.rewriteDeviceId(singleDeviceId); + deviceIdBackThen = evolvedSchema.rewriteToOriginal(singleDeviceId); } if (!tsFileResource.isSatisfied( @@ -3020,6 +3020,11 @@ private boolean canSkipDelete(TsFileResource tsFileResource, ModEntry deletion) return false; } + EvolvedSchema evolvedSchema = tsFileResource.getMergedEvolvedSchema(); + if (evolvedSchema != null) { + deletion = evolvedSchema.rewriteToOriginal(deletion); + } + for (IDeviceID device : tsFileResource.getDevices()) { // we are iterating the time index so the times are definitely present long startTime = tsFileResource.getTimeIndex().getStartTime(device).get(); @@ -3071,10 +3076,70 @@ private void deleteDataInUnsealedFiles( } } + private boolean canBeFullyDeleted(ArrayDeviceTimeIndex deviceTimeIndex, TableDeletionEntry tableDeletionEntry) { + Set devicesInFile = deviceTimeIndex.getDevices(); + String tableName = tableDeletionEntry.getTableName(); + long matchSize = + devicesInFile.stream() + .filter( + device -> { + if (logger.isDebugEnabled()) { + logger.debug( + "device is {}, deviceTable is {}, tableDeletionEntry.getPredicate().matches(device) is {}", + device, + device.getTableName(), + tableDeletionEntry.getPredicate().matches(device)); + } + return tableName.equals(device.getTableName()) + && tableDeletionEntry.getPredicate().matches(device); + }) + .count(); + boolean onlyOneTable = matchSize == devicesInFile.size(); + if (logger.isDebugEnabled()) { + logger.debug( + "tableName is {}, matchSize is {}, onlyOneTable is {}", + tableName, + matchSize, + onlyOneTable); + } + + if (onlyOneTable) { + matchSize = 0; + for (IDeviceID device : devicesInFile) { + Optional optStart = deviceTimeIndex.getStartTime(device); + Optional optEnd = deviceTimeIndex.getEndTime(device); + if (!optStart.isPresent() || !optEnd.isPresent()) { + continue; + } + + long fileStartTime = optStart.get(); + long fileEndTime = optEnd.get(); + + if (logger.isDebugEnabled()) { + logger.debug( + "tableName is {}, device is {}, deletionStartTime is {}, deletionEndTime is {}, fileStartTime is {}, fileEndTime is {}", + device.getTableName(), + device, + tableDeletionEntry.getStartTime(), + tableDeletionEntry.getEndTime(), + fileStartTime, + fileEndTime); + } + if (isFileFullyMatchedByTime(tableDeletionEntry, fileStartTime, fileEndTime)) { + ++matchSize; + } else { + return false; + } + } + return matchSize == devicesInFile.size(); + } else { + return false; + } + } + private void deleteDataInSealedFiles(Collection sealedTsFiles, ModEntry deletion) throws IOException { - Set involvedModificationFiles = new HashSet<>(); - List deletedByMods = new ArrayList<>(); + Set> involvedModificationFiles = new HashSet<>(); List deletedByFiles = new ArrayList<>(); for (TsFileResource sealedTsFile : sealedTsFiles) { if (canSkipDelete(sealedTsFile, deletion)) { @@ -3082,96 +3147,23 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M } ITimeIndex timeIndex = sealedTsFile.getTimeIndex(); + EvolvedSchema evolvedSchema = sealedTsFile.getMergedEvolvedSchema(); if ((timeIndex instanceof ArrayDeviceTimeIndex) && (deletion.getType() == ModType.TABLE_DELETION)) { ArrayDeviceTimeIndex deviceTimeIndex = (ArrayDeviceTimeIndex) timeIndex; - Set devicesInFile = deviceTimeIndex.getDevices(); - boolean onlyOneTable = false; - - if (deletion instanceof TableDeletionEntry) { - TableDeletionEntry tableDeletionEntry = (TableDeletionEntry) deletion; - String tableName = tableDeletionEntry.getTableName(); - long matchSize = - devicesInFile.stream() - .filter( - device -> { - if (logger.isDebugEnabled()) { - logger.debug( - "device is {}, deviceTable is {}, tableDeletionEntry.getPredicate().matches(device) is {}", - device, - device.getTableName(), - tableDeletionEntry.getPredicate().matches(device)); - } - return tableName.equals(device.getTableName()) - && tableDeletionEntry.getPredicate().matches(device); - }) - .count(); - onlyOneTable = matchSize == devicesInFile.size(); - if (logger.isDebugEnabled()) { - logger.debug( - "tableName is {}, matchSize is {}, onlyOneTable is {}", - tableName, - matchSize, - onlyOneTable); - } - } - - if (onlyOneTable) { - int matchSize = 0; - for (IDeviceID device : devicesInFile) { - Optional optStart = deviceTimeIndex.getStartTime(device); - Optional optEnd = deviceTimeIndex.getEndTime(device); - if (!optStart.isPresent() || !optEnd.isPresent()) { - continue; - } - - long fileStartTime = optStart.get(); - long fileEndTime = optEnd.get(); - - if (logger.isDebugEnabled()) { - logger.debug( - "tableName is {}, device is {}, deletionStartTime is {}, deletionEndTime is {}, fileStartTime is {}, fileEndTime is {}", - device.getTableName(), - device, - deletion.getStartTime(), - deletion.getEndTime(), - fileStartTime, - fileEndTime); - } - if (isFileFullyMatchedByTime(deletion, fileStartTime, fileEndTime)) { - ++matchSize; - } else { - deletedByMods.add(sealedTsFile); - break; - } - } - if (matchSize == devicesInFile.size()) { - deletedByFiles.add(sealedTsFile); - } - - if (logger.isDebugEnabled()) { - logger.debug("expect is {}, actual is {}", devicesInFile.size(), matchSize); - for (TsFileResource tsFileResource : deletedByFiles) { - logger.debug( - "delete tsFileResource is {}", tsFileResource.getTsFile().getAbsolutePath()); - } - } + TableDeletionEntry tableDeletionEntry = (TableDeletionEntry) deletion; + tableDeletionEntry = evolvedSchema != null? evolvedSchema.rewriteToOriginal(tableDeletionEntry) : tableDeletionEntry; + if (canBeFullyDeleted(deviceTimeIndex, tableDeletionEntry)) { + deletedByFiles.add(sealedTsFile); } else { - involvedModificationFiles.add(sealedTsFile.getModFileForWrite()); + involvedModificationFiles.add(new Pair<>(sealedTsFile.getModFileForWrite(), tableDeletionEntry)); } } else { - involvedModificationFiles.add(sealedTsFile.getModFileForWrite()); + involvedModificationFiles.add(new Pair<>(sealedTsFile.getModFileForWrite(), evolvedSchema != null? evolvedSchema.rewriteToOriginal(deletion) : deletion)); } } - for (TsFileResource tsFileResource : deletedByMods) { - if (tsFileResource.isClosed() - || !tsFileResource.getProcessor().deleteDataInMemory(deletion)) { - involvedModificationFiles.add(tsFileResource.getModFileForWrite()); - } // else do nothing - } - if (!deletedByFiles.isEmpty()) { deleteTsFileCompletely(deletedByFiles); if (logger.isDebugEnabled()) { @@ -3188,10 +3180,10 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M List exceptions = involvedModificationFiles.parallelStream() .map( - modFile -> { + modFileEntryPair -> { try { - modFile.write(deletion); - modFile.close(); + modFileEntryPair.getLeft().write(modFileEntryPair.getRight()); + modFileEntryPair.getLeft().close(); } catch (Exception e) { return e; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java index 4f58e8fc1323e..e6ec34e529b0b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java @@ -33,7 +33,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.FullExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.ModFileManagement; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java index 7e79e8f580dc5..da9617a7e6730 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.storageengine.dataregion.modification; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.NOP; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.iotdb.db.utils.io.BufferSerializable; import org.apache.iotdb.db.utils.io.StreamSerializable; @@ -42,7 +42,7 @@ public class DeletionPredicate implements StreamSerializable, BufferSerializable public static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(DeletionPredicate.class); private String tableName; - private IDPredicate idPredicate = new NOP(); + private TagPredicate tagPredicate = new NOP(); // an empty list means affecting all columns private List measurementNames = Collections.emptyList(); @@ -52,24 +52,24 @@ public DeletionPredicate(String tableName) { this.tableName = tableName; } - public DeletionPredicate(String tableName, IDPredicate idPredicate) { + public DeletionPredicate(String tableName, TagPredicate tagPredicate) { this.tableName = tableName; - this.idPredicate = idPredicate; + this.tagPredicate = tagPredicate; } public DeletionPredicate( - String tableName, IDPredicate idPredicate, List measurementNames) { + String tableName, TagPredicate tagPredicate, List measurementNames) { this.tableName = tableName; - this.idPredicate = idPredicate; + this.tagPredicate = tagPredicate; this.measurementNames = measurementNames; } public boolean matches(IDeviceID deviceID) { - return tableName.equals(deviceID.getTableName()) && idPredicate.matches(deviceID); + return tableName.equals(deviceID.getTableName()) && tagPredicate.matches(deviceID); } - public void setIdPredicate(IDPredicate idPredicate) { - this.idPredicate = idPredicate; + public void setIdPredicate(TagPredicate tagPredicate) { + this.tagPredicate = tagPredicate; } public String getTableName() { @@ -80,6 +80,10 @@ public List getMeasurementNames() { return measurementNames; } + public TagPredicate getTagPredicate() { + return tagPredicate; + } + public boolean affects(String measurementName) { return measurementNames.isEmpty() || measurementNames.contains(measurementName); } @@ -87,7 +91,7 @@ public boolean affects(String measurementName) { @Override public long serialize(OutputStream stream) throws IOException { long size = ReadWriteIOUtils.writeVar(tableName, stream); - size += idPredicate.serialize(stream); + size += tagPredicate.serialize(stream); size += ReadWriteForEncodingUtils.writeVarInt(measurementNames.size(), stream); for (String measurementName : measurementNames) { size += ReadWriteIOUtils.writeVar(measurementName, stream); @@ -98,7 +102,7 @@ public long serialize(OutputStream stream) throws IOException { @Override public long serialize(ByteBuffer buffer) { long size = ReadWriteIOUtils.writeVar(tableName, buffer); - size += idPredicate.serialize(buffer); + size += tagPredicate.serialize(buffer); size += ReadWriteForEncodingUtils.writeVarInt(measurementNames.size(), buffer); for (String measurementName : measurementNames) { size += ReadWriteIOUtils.writeVar(measurementName, buffer); @@ -109,7 +113,7 @@ public long serialize(ByteBuffer buffer) { @Override public void deserialize(InputStream stream) throws IOException { tableName = ReadWriteIOUtils.readVarIntString(stream); - idPredicate = IDPredicate.createFrom(stream); + tagPredicate = TagPredicate.createFrom(stream); int measurementLength = ReadWriteForEncodingUtils.readVarInt(stream); if (measurementLength > 0) { @@ -125,7 +129,7 @@ public void deserialize(InputStream stream) throws IOException { @Override public void deserialize(ByteBuffer buffer) { tableName = ReadWriteIOUtils.readVarIntString(buffer); - idPredicate = IDPredicate.createFrom(buffer); + tagPredicate = TagPredicate.createFrom(buffer); int measurementLength = ReadWriteForEncodingUtils.readVarInt(buffer); if (measurementLength > 0) { @@ -143,7 +147,7 @@ public int serializedSize() { int size = ReadWriteForEncodingUtils.varIntSize(tableName.length()) + tableName.length() * Character.BYTES - + idPredicate.serializedSize() + + tagPredicate.serializedSize() + ReadWriteForEncodingUtils.varIntSize(measurementNames.size()); for (String measurementName : measurementNames) { size += @@ -163,13 +167,13 @@ public boolean equals(Object o) { } DeletionPredicate that = (DeletionPredicate) o; return Objects.equals(tableName, that.tableName) - && Objects.equals(idPredicate, that.idPredicate) + && Objects.equals(tagPredicate, that.tagPredicate) && Objects.equals(measurementNames, that.measurementNames); } @Override public int hashCode() { - return Objects.hash(tableName, idPredicate, measurementNames); + return Objects.hash(tableName, tagPredicate, measurementNames); } @Override @@ -179,7 +183,7 @@ public String toString() { + tableName + '\'' + ", idPredicate=" - + idPredicate + + tagPredicate + ", measurementNames=" + measurementNames + '}'; @@ -189,7 +193,7 @@ public String toString() { public long ramBytesUsed() { return SHALLOW_SIZE + RamUsageEstimator.sizeOf(tableName) - + RamUsageEstimator.sizeOfObject(idPredicate) + + RamUsageEstimator.sizeOfObject(tagPredicate) + RamUsageEstimator.sizeOfArrayList(measurementNames); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/IDPredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TagPredicate.java similarity index 89% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/IDPredicate.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TagPredicate.java index 44741f9e67940..8aca6ed5fe77e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/IDPredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TagPredicate.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.storageengine.dataregion.modification; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; import org.apache.iotdb.db.utils.io.BufferSerializable; import org.apache.iotdb.db.utils.io.StreamSerializable; @@ -38,7 +39,7 @@ import java.util.List; import java.util.Objects; -public abstract class IDPredicate implements StreamSerializable, BufferSerializable, Accountable { +public abstract class TagPredicate implements StreamSerializable, BufferSerializable, Accountable { public int serializedSize() { // type @@ -73,12 +74,16 @@ public static IDPredicateType deserialize(ByteBuffer buffer) { protected final IDPredicateType type; - protected IDPredicate(IDPredicateType type) { + protected TagPredicate(IDPredicateType type) { this.type = type; } public abstract boolean matches(IDeviceID deviceID); + public TagPredicate rewriteToOriginal(EvolvedSchema evolvedSchema) { + return this; + } + @Override public long serialize(OutputStream stream) throws IOException { return type.serialize(stream); @@ -89,9 +94,9 @@ public long serialize(ByteBuffer buffer) { return type.serialize(buffer); } - public static IDPredicate createFrom(ByteBuffer buffer) { + public static TagPredicate createFrom(ByteBuffer buffer) { IDPredicateType type = IDPredicateType.deserialize(buffer); - IDPredicate predicate; + TagPredicate predicate; if (Objects.requireNonNull(type) == IDPredicateType.NOP) { predicate = new NOP(); } else if (Objects.requireNonNull(type) == IDPredicateType.FULL_EXACT_MATCH) { @@ -107,9 +112,9 @@ public static IDPredicate createFrom(ByteBuffer buffer) { return predicate; } - public static IDPredicate createFrom(InputStream stream) throws IOException { + public static TagPredicate createFrom(InputStream stream) throws IOException { IDPredicateType type = IDPredicateType.deserialize(stream); - IDPredicate predicate; + TagPredicate predicate; if (Objects.requireNonNull(type) == IDPredicateType.NOP) { predicate = new NOP(); } else if (Objects.requireNonNull(type) == IDPredicateType.FULL_EXACT_MATCH) { @@ -125,7 +130,7 @@ public static IDPredicate createFrom(InputStream stream) throws IOException { return predicate; } - public static class NOP extends IDPredicate { + public static class NOP extends TagPredicate { public static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(NOP.class); public NOP() { @@ -168,7 +173,7 @@ public long ramBytesUsed() { } } - public static class FullExactMatch extends IDPredicate { + public static class FullExactMatch extends TagPredicate { public static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(FullExactMatch.class); @@ -243,9 +248,14 @@ public String toString() { public long ramBytesUsed() { return SHALLOW_SIZE + RamUsageEstimator.sizeOfObject(deviceID); } + + @Override + public TagPredicate rewriteToOriginal(EvolvedSchema evolvedSchema) { + return new FullExactMatch(evolvedSchema.rewriteToOriginal(deviceID)); + } } - public static class SegmentExactMatch extends IDPredicate { + public static class SegmentExactMatch extends TagPredicate { public static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(SegmentExactMatch.class); @@ -342,17 +352,17 @@ public long ramBytesUsed() { } } - public static class And extends IDPredicate { + public static class And extends TagPredicate { public static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(And.class); - private final List predicates = new ArrayList<>(); + private final List predicates = new ArrayList<>(); - public And(IDPredicate... predicates) { + public And(TagPredicate... predicates) { super(IDPredicateType.AND); Collections.addAll(this.predicates, predicates); } - public void add(IDPredicate predicate) { + public void add(TagPredicate predicate) { predicates.add(predicate); } @@ -360,7 +370,7 @@ public void add(IDPredicate predicate) { public int serializedSize() { int serializedSize = super.serializedSize(); serializedSize += ReadWriteForEncodingUtils.varIntSize(predicates.size()); - for (IDPredicate predicate : predicates) { + for (TagPredicate predicate : predicates) { serializedSize += predicate.serializedSize(); } return serializedSize; @@ -370,7 +380,7 @@ public int serializedSize() { public long serialize(OutputStream stream) throws IOException { long size = super.serialize(stream); size += ReadWriteForEncodingUtils.writeVarInt(predicates.size(), stream); - for (IDPredicate predicate : predicates) { + for (TagPredicate predicate : predicates) { size += predicate.serialize(stream); } return size; @@ -380,7 +390,7 @@ public long serialize(OutputStream stream) throws IOException { public long serialize(ByteBuffer buffer) { long size = super.serialize(buffer); size += ReadWriteForEncodingUtils.writeVarInt(predicates.size(), buffer); - for (IDPredicate predicate : predicates) { + for (TagPredicate predicate : predicates) { size += predicate.serialize(buffer); } return size; @@ -390,7 +400,7 @@ public long serialize(ByteBuffer buffer) { public void deserialize(InputStream stream) throws IOException { int size = ReadWriteForEncodingUtils.readVarInt(stream); for (int i = 0; i < size; i++) { - predicates.add(IDPredicate.createFrom(stream)); + predicates.add(TagPredicate.createFrom(stream)); } } @@ -398,7 +408,7 @@ public void deserialize(InputStream stream) throws IOException { public void deserialize(ByteBuffer buffer) { int size = ReadWriteForEncodingUtils.readVarInt(buffer); for (int i = 0; i < size; i++) { - predicates.add(IDPredicate.createFrom(buffer)); + predicates.add(TagPredicate.createFrom(buffer)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java index 9a8d2cda88a9a..6e50bf5f0c1d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java @@ -21,6 +21,12 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; +import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry.ModType; +import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; @@ -40,7 +46,7 @@ public class EvolvedSchema { private Map> originalColumnNames = new LinkedHashMap<>(); public void renameTable(String oldTableName, String newTableName) { - if (!originalTableNames.containsKey(oldTableName)) { + if (!originalTableNames.containsKey(oldTableName) || originalTableNames.get(oldTableName).isEmpty()) { originalTableNames.put(newTableName, oldTableName); originalTableNames.put(oldTableName, ""); } else { @@ -58,7 +64,7 @@ public void renameTable(String oldTableName, String newTableName) { public void renameColumn(String tableName, String oldColumnName, String newColumnName) { Map columnNameMap = originalColumnNames.computeIfAbsent(tableName, t -> new LinkedHashMap<>()); - if (!columnNameMap.containsKey(oldColumnName)) { + if (!columnNameMap.containsKey(oldColumnName) || columnNameMap.get(oldColumnName).isEmpty()) { columnNameMap.put(newColumnName, oldColumnName); columnNameMap.put(oldColumnName, ""); } else { @@ -122,14 +128,36 @@ public List toSchemaEvolutions() { return schemaEvolutions; } - public IDeviceID rewriteDeviceId(IDeviceID deviceID) { + public ModEntry rewriteToOriginal(ModEntry entry) { + if (entry.getType() == ModType.TABLE_DELETION) { + return rewriteToOriginal(((TableDeletionEntry) entry)); + } + return entry; + } + + public TableDeletionEntry rewriteToOriginal(TableDeletionEntry entry) { + DeletionPredicate deletionPredicate = rewriteToOriginal(entry.getPredicate()); + return new TableDeletionEntry(deletionPredicate, entry.getTimeRange()); + } + + private DeletionPredicate rewriteToOriginal(DeletionPredicate predicate) { + String originalTableName = getOriginalTableName(predicate.getTableName()); + TagPredicate tagPredicate = predicate.getTagPredicate(); + tagPredicate = tagPredicate.rewriteToOriginal(this); + List newMeasurements = + predicate.getMeasurementNames().stream().map(m -> getOriginalColumnName(predicate.getTableName(), m)).collect( + Collectors.toList()); + return new DeletionPredicate(originalTableName, tagPredicate, newMeasurements); + } + + public IDeviceID rewriteToOriginal(IDeviceID deviceID) { String tableName = deviceID.getTableName(); String originalTableName = getOriginalTableName(tableName); - return rewriteDeviceId(deviceID, originalTableName); + return rewriteToOriginal(deviceID, originalTableName); } @SuppressWarnings("SuspiciousSystemArraycopy") - public static IDeviceID rewriteDeviceId(IDeviceID deviceID, String originalTableName) { + public static IDeviceID rewriteToOriginal(IDeviceID deviceID, String originalTableName) { String tableName = deviceID.getTableName(); if (!tableName.equals(originalTableName)) { Object[] segments = deviceID.getSegments(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java index aaa7bb195ca28..b9fd86316b8f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java @@ -62,7 +62,7 @@ public TsFileSet(long endVersion, String fileSetsDir, boolean recover) { } if (schemaEvolutionFile == null) { - schemaEvolutionFile = new SchemaEvolutionFile(0 + SchemaEvolutionFile.FILE_SUFFIX); + schemaEvolutionFile = new SchemaEvolutionFile(fileSetsDir + File.separator + 0 + SchemaEvolutionFile.FILE_SUFFIX); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java index f90cb0353bb8f..6569a7c6eb3e5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java @@ -34,7 +34,7 @@ import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunk; import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunkGroup; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.FullExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/path/PatternTreeMapTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/path/PatternTreeMapTest.java index 7a7d71bef5127..1db3890d9fe41 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/path/PatternTreeMapTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/path/PatternTreeMapTest.java @@ -24,7 +24,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.NOP; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionRecoverTest.java index 06f823c0e23fb..6eeeda629eb69 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionRecoverTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionRecoverTest.java @@ -30,7 +30,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.NOP; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.tsfile.read.common.TimeRange; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionResourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionResourceTest.java index f94d909f94bd1..bc6dc9e625ab8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionResourceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionResourceTest.java @@ -43,7 +43,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.NOP; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTablePatternParseVisitorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTablePatternParseVisitorTest.java index d01351ce60adc..63909828e6a8a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTablePatternParseVisitorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTablePatternParseVisitorTest.java @@ -29,7 +29,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.storageengine.dataregion.memtable.DeviceIDFactory; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.tsfile.read.common.TimeRange; @@ -115,12 +115,12 @@ public void testDeleteData() { new TableDeletionEntry( new DeletionPredicate( "ac", - new IDPredicate.And( - new IDPredicate.FullExactMatch( + new TagPredicate.And( + new TagPredicate.FullExactMatch( DeviceIDFactory.getInstance() .getDeviceID( new PartialPath(new String[] {"ac", "device1"}))), - new IDPredicate.SegmentExactMatch("device2", 1))), + new TagPredicate.SegmentExactMatch("device2", 1))), new TimeRange(0, 1))), "db1"), tablePattern) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java index e51a8b99db91d..a6b6bf1bf6747 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java @@ -24,10 +24,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.And; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.FullExactMatch; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.NOP; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.SegmentExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.And; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.SegmentExactMatch; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index 014874e589321..ec9a4f5eae21a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -45,6 +45,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils; @@ -60,10 +61,18 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.storageengine.dataregion.flush.FlushManager; import org.apache.iotdb.db.storageengine.dataregion.flush.TsFileFlushPolicy; +import org.apache.iotdb.db.storageengine.dataregion.flush.TsFileFlushPolicy.DirectFlushPolicy; import org.apache.iotdb.db.storageengine.dataregion.memtable.ReadOnlyMemChunk; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; +import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource.ModIterator; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.ColumnRename; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.TableRename; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; import org.apache.iotdb.db.storageengine.rescon.memory.MemTableManager; @@ -77,6 +86,7 @@ import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; @@ -102,6 +112,7 @@ import static org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils.genInsertRowNode; import static org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils.genInsertTabletNode; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class DataRegionTest { @@ -113,7 +124,7 @@ public class DataRegionTest { private String systemDir = TestConstant.OUTPUT_DATA_DIR.concat("info"); private String deviceId = "root.vehicle.d0"; - private IDeviceID device = IDeviceID.Factory.DEFAULT_FACTORY.create(deviceId); + private IDeviceID device = Factory.DEFAULT_FACTORY.create(deviceId); private String measurementId = "s0"; private NonAlignedFullPath nonAlignedFullPath = @@ -242,7 +253,7 @@ record = new TSRecord(deviceId, j); null); } - Assert.assertEquals(1, tsfileResourcesForQuery.size()); + assertEquals(1, tsfileResourcesForQuery.size()); List memChunks = tsfileResourcesForQuery.get(0).getReadOnlyMemChunk(IFullPath.convertToIFullPath(fullPath)); long time = 16; @@ -250,7 +261,7 @@ record = new TSRecord(deviceId, j); IPointReader iterator = memChunk.getPointReader(); while (iterator.hasNextTimeValuePair()) { TimeValuePair timeValuePair = iterator.nextTimeValuePair(); - Assert.assertEquals(time++, timeValuePair.getTimestamp()); + assertEquals(time++, timeValuePair.getTimestamp()); } } } @@ -265,7 +276,7 @@ public void testSequenceSyncClose() dataRegion.syncCloseAllWorkingTsFileProcessors(); } - IDeviceID device = IDeviceID.Factory.DEFAULT_FACTORY.create(deviceId); + IDeviceID device = Factory.DEFAULT_FACTORY.create(deviceId); QueryDataSource queryDataSource = dataRegion.query( Collections.singletonList( @@ -275,7 +286,7 @@ device, new MeasurementSchema(measurementId, TSDataType.INT32))), context, null, null); - Assert.assertEquals(10, queryDataSource.getSeqResources().size()); + assertEquals(10, queryDataSource.getSeqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -308,8 +319,8 @@ public void testRelationalTabletWriteAndSyncClose() context, null, null); - Assert.assertEquals(1, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(1, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); queryDataSource = dataRegion.query( @@ -322,8 +333,8 @@ public void testRelationalTabletWriteAndSyncClose() context, null, null); - Assert.assertEquals(1, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(1, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -356,8 +367,8 @@ public void testRelationRowWriteAndSyncClose() context, null, null); - Assert.assertEquals(1, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(1, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); queryDataSource = dataRegion.query( @@ -370,8 +381,8 @@ public void testRelationRowWriteAndSyncClose() context, null, null); - Assert.assertEquals(1, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(1, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -429,8 +440,8 @@ public void testIoTDBTabletWriteAndSyncClose() int hashCode2 = Arrays.hashCode((long[]) columns[1]); dataRegion.insertTablet(insertTabletNode1); // the hashCode should not be changed when insert - Assert.assertEquals(hashCode1, Arrays.hashCode((int[]) columns[0])); - Assert.assertEquals(hashCode2, Arrays.hashCode((long[]) columns[1])); + assertEquals(hashCode1, Arrays.hashCode((int[]) columns[0])); + assertEquals(hashCode2, Arrays.hashCode((long[]) columns[1])); dataRegion.syncCloseAllWorkingTsFileProcessors(); for (int r = 50; r < 149; r++) { @@ -459,8 +470,8 @@ public void testIoTDBTabletWriteAndSyncClose() dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(2, queryDataSource.getSeqResources().size()); - Assert.assertEquals(1, queryDataSource.getUnseqResources().size()); + assertEquals(2, queryDataSource.getSeqResources().size()); + assertEquals(1, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -531,14 +542,14 @@ public void testIoTDBTabletWriteAndDeleteDataRegion() dataRegion.insertTablet(insertTabletNode2); assertTrue(SystemInfo.getInstance().getTotalMemTableSize() > 0); dataRegion.syncDeleteDataFiles(); - Assert.assertEquals(0, SystemInfo.getInstance().getTotalMemTableSize()); + assertEquals(0, SystemInfo.getInstance().getTotalMemTableSize()); QueryDataSource queryDataSource = dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(0, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(0, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); } @Test @@ -611,8 +622,8 @@ public void testEmptyTabletWriteAndSyncClose() dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(0, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(0, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -687,8 +698,8 @@ public void testAllMeasurementsFailedTabletWriteAndSyncClose() dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(0, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(0, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -714,8 +725,8 @@ public void testSeqAndUnSeqSyncClose() QueryDataSource queryDataSource = dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(10, queryDataSource.getSeqResources().size()); - Assert.assertEquals(10, queryDataSource.getUnseqResources().size()); + assertEquals(10, queryDataSource.getSeqResources().size()); + assertEquals(10, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -748,8 +759,8 @@ public void testAllMeasurementsFailedRecordSeqAndUnSeqSyncClose() QueryDataSource queryDataSource = dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(0, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(0, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -781,8 +792,8 @@ public void testDisableSeparateDataForInsertRowPlan() QueryDataSource queryDataSource = dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(0, queryDataSource.getSeqResources().size()); - Assert.assertEquals(20, queryDataSource.getUnseqResources().size()); + assertEquals(0, queryDataSource.getSeqResources().size()); + assertEquals(20, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -863,8 +874,8 @@ public void testDisableSeparateDataForInsertTablet1() dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(0, queryDataSource.getSeqResources().size()); - Assert.assertEquals(2, queryDataSource.getUnseqResources().size()); + assertEquals(0, queryDataSource.getSeqResources().size()); + assertEquals(2, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -943,8 +954,8 @@ public void testDisableSeparateDataForInsertTablet2() dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(0, queryDataSource.getSeqResources().size()); - Assert.assertEquals(2, queryDataSource.getUnseqResources().size()); + assertEquals(0, queryDataSource.getSeqResources().size()); + assertEquals(2, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -1023,8 +1034,8 @@ public void testDisableSeparateDataForInsertTablet3() dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(0, queryDataSource.getSeqResources().size()); - Assert.assertEquals(2, queryDataSource.getUnseqResources().size()); + assertEquals(0, queryDataSource.getSeqResources().size()); + assertEquals(2, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -1053,7 +1064,7 @@ public void testInsertUnSequenceRows() InsertRowsNode insertRowsNode = new InsertRowsNode(new PlanNodeId(""), indexList, nodes); dataRegion1.insert(insertRowsNode); dataRegion1.syncCloseAllWorkingTsFileProcessors(); - IDeviceID tmpDeviceId = IDeviceID.Factory.DEFAULT_FACTORY.create("root.Rows"); + IDeviceID tmpDeviceId = Factory.DEFAULT_FACTORY.create("root.Rows"); QueryDataSource queryDataSource = dataRegion1.query( Collections.singletonList( @@ -1063,8 +1074,8 @@ tmpDeviceId, new MeasurementSchema(measurementId, TSDataType.INT32))), context, null, null); - Assert.assertEquals(1, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(1, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -1090,7 +1101,7 @@ public void testSmallReportProportionInsertRow() dataRegion1.syncCloseAllWorkingTsFileProcessors(); } dataRegion1.syncCloseAllWorkingTsFileProcessors(); - IDeviceID tmpDeviceId = IDeviceID.Factory.DEFAULT_FACTORY.create("root.ln22"); + IDeviceID tmpDeviceId = Factory.DEFAULT_FACTORY.create("root.ln22"); QueryDataSource queryDataSource = dataRegion1.query( Collections.singletonList( @@ -1100,8 +1111,8 @@ tmpDeviceId, new MeasurementSchema(measurementId, TSDataType.INT32))), context, null, null); - Assert.assertEquals(10, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(10, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -1169,7 +1180,7 @@ public void testMerge() QueryDataSource queryDataSource = dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(2, queryDataSource.getSeqResources().size()); + assertEquals(2, queryDataSource.getSeqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -1256,7 +1267,7 @@ public void testTimedFlushSeqMemTable() TSRecord record = new TSRecord(deviceId, 10000); record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(1000))); dataRegion.insert(buildInsertRowNodeByTSRecord(record)); - Assert.assertEquals(1, MemTableManager.getInstance().getCurrentMemtableNumber()); + assertEquals(1, MemTableManager.getInstance().getCurrentMemtableNumber()); // change config & reboot timed service boolean prevEnableTimedFlushSeqMemtable = config.isEnableTimedFlushSeqMemtable(); @@ -1267,7 +1278,7 @@ public void testTimedFlushSeqMemTable() Thread.sleep(500); - Assert.assertEquals(1, dataRegion.getWorkSequenceTsFileProcessors().size()); + assertEquals(1, dataRegion.getWorkSequenceTsFileProcessors().size()); TsFileProcessor tsFileProcessor = dataRegion.getWorkSequenceTsFileProcessors().iterator().next(); FlushManager flushManager = FlushManager.getInstance(); @@ -1292,7 +1303,7 @@ public void testTimedFlushSeqMemTable() } } - Assert.assertEquals(0, MemTableManager.getInstance().getCurrentMemtableNumber()); + assertEquals(0, MemTableManager.getInstance().getCurrentMemtableNumber()); config.setEnableTimedFlushSeqMemtable(prevEnableTimedFlushSeqMemtable); config.setSeqMemtableFlushInterval(preFLushInterval); @@ -1305,15 +1316,15 @@ public void testTimedFlushUnseqMemTable() TSRecord record = new TSRecord(deviceId, 10000); record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(1000))); dataRegion.insert(buildInsertRowNodeByTSRecord(record)); - Assert.assertEquals(1, MemTableManager.getInstance().getCurrentMemtableNumber()); + assertEquals(1, MemTableManager.getInstance().getCurrentMemtableNumber()); dataRegion.syncCloseAllWorkingTsFileProcessors(); - Assert.assertEquals(0, MemTableManager.getInstance().getCurrentMemtableNumber()); + assertEquals(0, MemTableManager.getInstance().getCurrentMemtableNumber()); // create one unsequence memtable record = new TSRecord(deviceId, 1); record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(1000))); dataRegion.insert(buildInsertRowNodeByTSRecord(record)); - Assert.assertEquals(1, MemTableManager.getInstance().getCurrentMemtableNumber()); + assertEquals(1, MemTableManager.getInstance().getCurrentMemtableNumber()); // change config & reboot timed service boolean prevEnableTimedFlushUnseqMemtable = config.isEnableTimedFlushUnseqMemtable(); @@ -1324,7 +1335,7 @@ record = new TSRecord(deviceId, 1); Thread.sleep(500); - Assert.assertEquals(1, dataRegion.getWorkUnsequenceTsFileProcessors().size()); + assertEquals(1, dataRegion.getWorkUnsequenceTsFileProcessors().size()); TsFileProcessor tsFileProcessor = dataRegion.getWorkUnsequenceTsFileProcessors().iterator().next(); FlushManager flushManager = FlushManager.getInstance(); @@ -1349,7 +1360,7 @@ record = new TSRecord(deviceId, 1); } } - Assert.assertEquals(0, MemTableManager.getInstance().getCurrentMemtableNumber()); + assertEquals(0, MemTableManager.getInstance().getCurrentMemtableNumber()); config.setEnableTimedFlushUnseqMemtable(prevEnableTimedFlushUnseqMemtable); config.setUnseqMemtableFlushInterval(preFLushInterval); @@ -1404,10 +1415,10 @@ public void testDeleteDataNotInFile() TsFileResource resource = dataRegion.getSequenceFileList().get(i); if (i == 1) { assertTrue(resource.anyModFileExists()); - Assert.assertEquals(2, resource.getAllModEntries().size()); + assertEquals(2, resource.getAllModEntries().size()); } else if (i == 3) { assertTrue(resource.anyModFileExists()); - Assert.assertEquals(1, resource.getAllModEntries().size()); + assertEquals(1, resource.getAllModEntries().size()); } else { Assert.assertFalse(resource.anyModFileExists()); } @@ -1501,7 +1512,7 @@ public void testDeleteDataInSeqFlushingMemtable() dataRegion.syncCloseAllWorkingTsFileProcessors(); assertTrue(tsFileResource.anyModFileExists()); - Assert.assertEquals(3, tsFileResource.getAllModEntries().size()); + assertEquals(3, tsFileResource.getAllModEntries().size()); } @Test @@ -1596,7 +1607,7 @@ public void testDeleteDataInUnSeqFlushingMemtable() dataRegion.syncCloseAllWorkingTsFileProcessors(); assertTrue(tsFileResource.anyModFileExists()); - Assert.assertEquals(3, tsFileResource.getAllModEntries().size()); + assertEquals(3, tsFileResource.getAllModEntries().size()); } @Test @@ -1638,7 +1649,7 @@ public void testDeleteDataInSeqWorkingMemtable() Assert.assertFalse( tsFileResource .getDevices() - .contains(IDeviceID.Factory.DEFAULT_FACTORY.create("root.vehicle.d199"))); + .contains(Factory.DEFAULT_FACTORY.create("root.vehicle.d199"))); } @Test @@ -1670,7 +1681,7 @@ public static class DummyDataRegion extends DataRegion { public DummyDataRegion(String systemInfoDir, String storageGroupName) throws DataRegionException { - super(systemInfoDir, "0", new TsFileFlushPolicy.DirectFlushPolicy(), storageGroupName); + super(systemInfoDir, "0", new DirectFlushPolicy(), storageGroupName); } } @@ -1786,7 +1797,7 @@ public void testDeleteDataDirectlyUnseqWriteModsOrDeleteFiles() @Test public void testSchemaEvolution() - throws IllegalPathException, WriteProcessException, QueryProcessException, IOException { + throws WriteProcessException, QueryProcessException, IOException { String[] measurements = {"tag1", "s1", "s2"}; MeasurementSchema[] measurementSchemas = { new MeasurementSchema("tag1", TSDataType.STRING), @@ -1896,4 +1907,120 @@ public void testSchemaEvolution() Long.MAX_VALUE); assertEquals(1, dataSource.getSeqResources().size()); } + + @Test + public void testSchemaEvolutionWithPartialDeletion() + throws WriteProcessException, IOException { + String[] measurements = {"tag1", "s1", "s2"}; + MeasurementSchema[] measurementSchemas = { + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.INT64), + new MeasurementSchema("s2", TSDataType.DOUBLE) + }; + RelationalInsertRowNode insertRowNode = + new RelationalInsertRowNode( + new PlanNodeId(""), + new PartialPath(new String[] {"table1"}), + true, + measurements, + new TSDataType[] {TSDataType.STRING, TSDataType.INT64, TSDataType.DOUBLE}, + measurementSchemas, + 10, + new Object[] {new Binary("tag1".getBytes(StandardCharsets.UTF_8)), 1L, 1.0}, + false, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD + }); + dataRegion.insert(insertRowNode); + insertRowNode.setTime(20); + dataRegion.insert(insertRowNode); + + // table1 -> table2 + dataRegion.applySchemaEvolution(Collections.singletonList(new TableRename("table1", "table2"))); + // s1 -> s3 + dataRegion.applySchemaEvolution(Collections.singletonList(new ColumnRename("table2", "s1", "s3", null))); + + // delete with table2 + TableDeletionEntry tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table2"), new TimeRange(0, 15)); + RelationalDeleteDataNode relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + dataRegion.deleteByTable(relationalDeleteDataNode); + // delete with s3 + tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table2", new NOP(), Collections.singletonList("s3")), new TimeRange(0, 15)); + relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + dataRegion.deleteByTable(relationalDeleteDataNode); + // delete with table1 + tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table1"), new TimeRange(0, 15)); + relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + dataRegion.deleteByTable(relationalDeleteDataNode); + // delete with s1 + tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table2", new NOP(), Collections.singletonList("s1")), new TimeRange(0, 15)); + relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + dataRegion.deleteByTable(relationalDeleteDataNode); + + List sequenceFileList = dataRegion.getSequenceFileList(); + assertEquals(1, sequenceFileList.size()); + ModIterator modEntryIterator = sequenceFileList.get(0).getModEntryIterator(); + ModEntry next = modEntryIterator.next(); + // the table2 modification should be rewritten to table1 + assertEquals("table1", ((TableDeletionEntry) next).getTableName()); + next = modEntryIterator.next(); + // the s3 modification should be rewritten to s1 + assertEquals(Collections.singletonList("s1"), ((TableDeletionEntry) next).getPredicate().getMeasurementNames()); + next = modEntryIterator.next(); + // the table1 modification should be skipped + // the s1 modification should be rewritten to empty + assertEquals(Collections.singletonList(""), ((TableDeletionEntry) next).getPredicate().getMeasurementNames()); + assertFalse(modEntryIterator.hasNext()); + } + + @Test + public void testSchemaEvolutionWithFullDeletion() + throws WriteProcessException, IOException { + String[] measurements = {"tag1", "s1", "s2"}; + MeasurementSchema[] measurementSchemas = { + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.INT64), + new MeasurementSchema("s2", TSDataType.DOUBLE) + }; + RelationalInsertRowNode insertRowNode = + new RelationalInsertRowNode( + new PlanNodeId(""), + new PartialPath(new String[] {"table1"}), + true, + measurements, + new TSDataType[] {TSDataType.STRING, TSDataType.INT64, TSDataType.DOUBLE}, + measurementSchemas, + 10, + new Object[] {new Binary("tag1".getBytes(StandardCharsets.UTF_8)), 1L, 1.0}, + false, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD + }); + dataRegion.insert(insertRowNode); + insertRowNode.setTime(20); + dataRegion.insert(insertRowNode); + + // table1 -> table2 + dataRegion.applySchemaEvolution(Collections.singletonList(new TableRename("table1", "table2"))); + // s1 -> s3 + dataRegion.applySchemaEvolution(Collections.singletonList(new ColumnRename("table2", "s1", "s3", null))); + + // delete with table1 + TableDeletionEntry tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table1"), new TimeRange(0, 30)); + RelationalDeleteDataNode relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + dataRegion.deleteByTable(relationalDeleteDataNode); + // nothing should be deleted + List sequenceFileList = dataRegion.getSequenceFileList(); + assertEquals(1, sequenceFileList.size()); + ModIterator modEntryIterator = sequenceFileList.get(0).getModEntryIterator(); + assertFalse(modEntryIterator.hasNext()); + + // delete with table2 + tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table2"), new TimeRange(0, 30)); + relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + dataRegion.deleteByTable(relationalDeleteDataNode); + // the file should be deleted + sequenceFileList = dataRegion.getSequenceFileList(); + assertEquals(0, sequenceFileList.size()); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java index 9ff8a401150b8..a83fa920853aa 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java @@ -35,8 +35,8 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.FullExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -333,7 +333,7 @@ public void testCompactionWithAllValueColumnDeletion() throws IOException, Illeg new TableDeletionEntry( new DeletionPredicate( "t1", - new IDPredicate.FullExactMatch(deviceID), + new TagPredicate.FullExactMatch(deviceID), Collections.singletonList("s0")), new TimeRange(Long.MIN_VALUE, 11))); resource1 @@ -342,7 +342,7 @@ public void testCompactionWithAllValueColumnDeletion() throws IOException, Illeg new TableDeletionEntry( new DeletionPredicate( "t1", - new IDPredicate.FullExactMatch(deviceID), + new TagPredicate.FullExactMatch(deviceID), Collections.singletonList("s1")), new TimeRange(Long.MIN_VALUE, 11))); resource1 @@ -351,7 +351,7 @@ public void testCompactionWithAllValueColumnDeletion() throws IOException, Illeg new TableDeletionEntry( new DeletionPredicate( "t1", - new IDPredicate.FullExactMatch(deviceID), + new TagPredicate.FullExactMatch(deviceID), Collections.singletonList("s2")), new TimeRange(Long.MIN_VALUE, 11))); resource1 @@ -360,7 +360,7 @@ public void testCompactionWithAllValueColumnDeletion() throws IOException, Illeg new TableDeletionEntry( new DeletionPredicate( "t1", - new IDPredicate.FullExactMatch(deviceID), + new TagPredicate.FullExactMatch(deviceID), Collections.singletonList("s3")), new TimeRange(Long.MIN_VALUE, 11))); resource1.getModFileForWrite().close(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFileTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFileTest.java index a0a9885ecf08e..29609cc9c2740 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFileTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFileTest.java @@ -22,9 +22,9 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.recover.CompactionRecoverManager; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.FullExactMatch; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.NOP; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.SegmentExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.SegmentExactMatch; import org.apache.iotdb.db.utils.constant.TestConstant; import org.apache.tsfile.file.metadata.IDeviceID.Factory; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntryTest.java index 5c2979a90755c..ee9a4dfa405e4 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntryTest.java @@ -18,10 +18,10 @@ */ package org.apache.iotdb.db.storageengine.dataregion.modification; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.And; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.FullExactMatch; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.NOP; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.SegmentExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.And; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.SegmentExactMatch; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; From 10a555360a46be9629cf9a6112b29739d340e600 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 29 Dec 2025 18:19:19 +0800 Subject: [PATCH 14/49] support load with sevo --- .../it/db/it/IoTDBLoadTsFileIT.java | 73 ++++++- .../iot/client/DispatchLogHandler.java | 2 +- .../impl/DataNodeInternalRPCServiceImpl.java | 2 +- .../operator/source/FileLoaderUtils.java | 10 +- .../plan/analyze/AnalyzeUtils.java | 5 +- .../plan/analyze/load/LoadTsFileAnalyzer.java | 32 ++- .../plan/planner/LogicalPlanVisitor.java | 3 +- .../plan/node/load/LoadSingleTsFileNode.java | 15 +- .../plan/node/load/LoadTsFileNode.java | 15 +- .../relational/planner/RelationPlanner.java | 6 +- .../plan/relational/sql/ast/LoadTsFile.java | 6 + .../scheduler/load/LoadTsFileScheduler.java | 4 +- .../statement/crud/LoadTsFileStatement.java | 6 + .../storageengine/dataregion/DataRegion.java | 21 +- .../execute/utils/CompactionUtils.java | 2 +- .../dataregion/modification/TagPredicate.java | 9 + .../tsfile/evolution/EvolvedSchema.java | 202 ++++++++++++++---- .../tsfile/evolution/SchemaEvolutionFile.java | 4 + .../dataregion/tsfile/fileset/TsFileSet.java | 4 +- .../file/UnsealedTsFileRecoverPerformer.java | 2 +- .../load/config/LoadTsFileConfigurator.java | 18 ++ .../load/splitter/AlignedChunkData.java | 13 +- .../load/splitter/DeletionData.java | 8 +- .../load/splitter/NonAlignedChunkData.java | 11 +- .../load/splitter/TsFileData.java | 3 + .../load/splitter/TsFileSplitter.java | 50 ++++- .../db/metadata/path/PatternTreeMapTest.java | 2 +- .../pipe/consensus/DeletionRecoverTest.java | 2 +- .../pipe/consensus/DeletionResourceTest.java | 2 +- .../PipePlanTablePatternParseVisitorTest.java | 2 +- .../planner/node/load/LoadTsFileNodeTest.java | 2 +- .../write/RelationalDeleteDataNodeTest.java | 2 +- .../dataregion/DataRegionTest.java | 88 +++++--- ...tchedCompactionWithTsFileSplitterTest.java | 3 +- .../CompactionWithAllNullRowsTest.java | 2 +- 35 files changed, 510 insertions(+), 121 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java index 5f88d50c9d90c..46cc2e63adb35 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java @@ -19,6 +19,10 @@ package org.apache.iotdb.relational.it.db.it; +import java.sql.SQLException; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolutionFile; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.TableRename; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.it.utils.TsFileTableGenerator; @@ -47,10 +51,15 @@ import java.sql.ResultSet; import java.sql.Statement; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) public class IoTDBLoadTsFileIT { @@ -255,7 +264,69 @@ public void testLoadWithTableMod() throws Exception { try (final ResultSet resultSet = statement.executeQuery("show tables")) { Assert.assertTrue(resultSet.next()); - Assert.assertFalse(resultSet.next()); + assertFalse(resultSet.next()); + } + } + } + + @Test + public void testLoadWithSevoFile() throws Exception { + final int lineCount = 10000; + + List> measurementSchemas = + generateMeasurementSchemas(); + List columnCategories = + generateTabletColumnCategory(0, measurementSchemas.size()); + + final File file = new File(tmpDir, "1-0-0-0.tsfile"); + + List schemaList1 = + measurementSchemas.stream().map(pair -> pair.left).collect(Collectors.toList()); + + try (final TsFileTableGenerator generator = new TsFileTableGenerator(file)) { + generator.registerTable(SchemaConfig.TABLE_0, new ArrayList<>(schemaList1), columnCategories); + generator.generateData(SchemaConfig.TABLE_0, lineCount, PARTITION_INTERVAL / 10_000); + } + + // rename table0 to table1 + File sevoFile = new File(tmpDir, "0.sevo"); + SchemaEvolutionFile schemaEvolutionFile = new SchemaEvolutionFile(sevoFile.getAbsolutePath()); + SchemaEvolution schemaEvolution = new TableRename(SchemaConfig.TABLE_0, SchemaConfig.TABLE_1); + schemaEvolutionFile.append(Collections.singletonList(schemaEvolution)); + // rename INT322INT32 + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(String.format("create database if not exists %s", SchemaConfig.DATABASE_0)); + statement.execute(String.format("use %s", SchemaConfig.DATABASE_0)); + statement.execute( + String.format( + "load '%s' with ('database'='%s', 'sevo-file-path'='%s')", + file.getAbsolutePath(), SchemaConfig.DATABASE_0, schemaEvolutionFile.getFilePath())); + + // cannot query using table0 + try (final ResultSet resultSet = + statement.executeQuery(String.format("select count(*) from %s", SchemaConfig.TABLE_0))) { + fail(); + } catch (SQLException e) { + assertEquals("550: Table 'root.test' does not exist.", e.getMessage()); + } + + // can query with table1 + try (final ResultSet resultSet = + statement.executeQuery(String.format("select count(*) from %s", SchemaConfig.TABLE_1))) { + if (resultSet.next()) { + Assert.assertEquals(lineCount, resultSet.getLong(1)); + } else { + Assert.fail("This ResultSet is empty."); + } + } + + try (final ResultSet resultSet = statement.executeQuery("show tables")) { + Assert.assertTrue(resultSet.next()); + assertEquals(SchemaConfig.TABLE_1, resultSet.getString(1)); + assertFalse(resultSet.next()); } } } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java index bb0326d7473e7..9bfd48cea9b72 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/client/DispatchLogHandler.java @@ -106,7 +106,7 @@ public void onError(Exception exception) { ++retryCount; Throwable rootCause = ExceptionUtils.getRootCause(exception); logger.warn( - "Can not send {} to peer for {} times {} because {}", + "v {} to peer for {} times {} because {}", batch, thread.getPeer(), retryCount, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 84e3e17a4983d..dc6256ca13a96 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -195,8 +195,8 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.settle.SettleRequestHandler; import org.apache.iotdb.db.storageengine.dataregion.flush.CompressionRatio; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeSpaceQuotaManager; import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeThrottleQuotaManager; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index cf128e33f18cb..da8fd358e979f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; -import java.util.concurrent.LinkedBlockingDeque; import org.apache.iotdb.commons.path.AlignedFullPath; import org.apache.iotdb.commons.path.NonAlignedFullPath; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; @@ -98,15 +97,13 @@ public static TimeseriesMetadata loadTimeSeriesMetadata( EvolvedSchema evolvedSchema = resource.getMergedEvolvedSchema(); IDeviceID deviceId = seriesPath.getDeviceId(); String measurement = seriesPath.getMeasurement(); - + timeSeriesMetadata = TimeSeriesMetadataCache.getInstance() .get( resource.getTsFilePath(), new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey( - resource.getTsFileID(), - deviceId, - measurement), + resource.getTsFileID(), deviceId, measurement), allSensors, context.ignoreNotExistsDevice() || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, @@ -115,8 +112,7 @@ public static TimeseriesMetadata loadTimeSeriesMetadata( if (timeSeriesMetadata != null) { long t2 = System.nanoTime(); List pathModifications = - context.getPathModifications( - resource, deviceId, measurement); + context.getPathModifications(resource, deviceId, measurement); timeSeriesMetadata.setModified(!pathModifications.isEmpty()); timeSeriesMetadata.setChunkMetadataLoader( new DiskChunkMetadataLoader(resource, context, globalTimeFilter, pathModifications)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java index 331c18d54c994..41df1928112b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java @@ -53,10 +53,10 @@ import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.table.DataNodeTreeViewSchemaUtils; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.And; import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.SegmentExactMatch; -import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -490,7 +490,8 @@ private static TagPredicate parseIsNull( return combinePredicates(oldPredicate, newPredicate); } - private static TagPredicate combinePredicates(TagPredicate oldPredicate, TagPredicate newPredicate) { + private static TagPredicate combinePredicates( + TagPredicate oldPredicate, TagPredicate newPredicate) { if (oldPredicate == null) { return newPredicate; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java index 8202de5c49947..9b2bf168f6668 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java @@ -41,6 +41,8 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolutionFile; import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils; import org.apache.iotdb.db.storageengine.load.active.ActiveLoadPathHelper; import org.apache.iotdb.db.storageengine.load.converter.LoadTsFileDataTypeConverter; @@ -73,6 +75,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; import static org.apache.iotdb.db.queryengine.plan.execution.config.TableConfigTaskVisitor.DATABASE_NOT_SPECIFIED; import static org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet.ANALYSIS; @@ -106,6 +109,8 @@ public class LoadTsFileAnalyzer implements AutoCloseable { private boolean isMiniTsFileConverted = false; private final List isTableModelTsFile; private int isTableModelTsFileReliableIndex = -1; + private final File schemaEvolutionFile; + private EvolvedSchema evolvedSchema; // User specified configs private final int databaseLevel; @@ -134,6 +139,7 @@ public LoadTsFileAnalyzer( this.tsFiles = loadTsFileStatement.getTsFiles(); this.isMiniTsFile = new ArrayList<>(Collections.nCopies(this.tsFiles.size(), false)); this.isTableModelTsFile = new ArrayList<>(Collections.nCopies(this.tsFiles.size(), false)); + this.schemaEvolutionFile = loadTsFileStatement.getSchemaEvolutionFile(); this.databaseLevel = loadTsFileStatement.getDatabaseLevel(); this.databaseForTableData = loadTsFileStatement.getDatabase(); @@ -158,6 +164,7 @@ public LoadTsFileAnalyzer( this.tsFiles = loadTsFileTableStatement.getTsFiles(); this.isMiniTsFile = new ArrayList<>(Collections.nCopies(this.tsFiles.size(), false)); this.isTableModelTsFile = new ArrayList<>(Collections.nCopies(this.tsFiles.size(), false)); + this.schemaEvolutionFile = loadTsFileTableStatement.getSchemaEvolutionFile(); this.databaseLevel = loadTsFileTableStatement.getDatabaseLevel(); this.databaseForTableData = loadTsFileTableStatement.getDatabase(); @@ -200,6 +207,12 @@ public IAnalysis analyzeFileByFile(IAnalysis analysis) { } try { + if (schemaEvolutionFile != null && schemaEvolutionFile.exists()) { + SchemaEvolutionFile sevoFile = + new SchemaEvolutionFile(schemaEvolutionFile.getAbsolutePath()); + evolvedSchema = sevoFile.readAsSchema(); + } + if (!doAnalyzeFileByFile(analysis)) { return analysis; } @@ -526,7 +539,7 @@ private void doAnalyzeSingleTableFile( final File tsFile, final TsFileSequenceReader reader, final TsFileSequenceReaderTimeseriesMetadataIterator timeseriesMetadataIterator, - final Map tableSchemaMap) + Map tableSchemaMap) throws IOException, LoadAnalyzeException { // construct tsfile resource final TsFileResource tsFileResource = constructTsFileResource(reader, tsFile); @@ -550,13 +563,28 @@ private void doAnalyzeSingleTableFile( } getOrCreateTableSchemaCache().setDatabase(databaseForTableData); + if (evolvedSchema != null) { + tableSchemaMap = evolvedSchema.rewriteToFinal(tableSchemaMap); + } getOrCreateTableSchemaCache().setTableSchemaMap(tableSchemaMap); getOrCreateTableSchemaCache().setCurrentModificationsAndTimeIndex(tsFileResource, reader); while (timeseriesMetadataIterator.hasNext()) { - final Map> device2TimeseriesMetadata = + Map> device2TimeseriesMetadata = timeseriesMetadataIterator.next(); + if (evolvedSchema != null) { + device2TimeseriesMetadata = + device2TimeseriesMetadata.entrySet().stream() + .collect( + Collectors.toMap( + e -> evolvedSchema.rewriteToFinal(e.getKey()), + e -> { + evolvedSchema.rewriteToFinal(e.getKey().getTableName(), e.getValue()); + return e.getValue(); + })); + } + // Update time index no matter if resource file exists or not, because resource file may be // untrusted TsFileResourceUtils.updateTsFileResource( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index fa39b3ba7dcef..a084cba06242a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@ -531,7 +531,8 @@ public PlanNode visitLoadFile( context.getQueryId().genPlanNodeId(), loadTsFileStatement.getResources(), isTableModel, - loadTsFileStatement.getDatabase()); + loadTsFileStatement.getDatabase(), + loadTsFileStatement.getSchemaEvolutionFile()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java index 604fda6e1e8d5..c0d5b582fbee8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -62,6 +62,7 @@ public class LoadSingleTsFileNode extends WritePlanNode { private final boolean deleteAfterLoad; private final long writePointCount; private boolean needDecodeTsFile; + private final File schemaEvolutionFile; private TRegionReplicaSet localRegionReplicaSet; @@ -71,7 +72,8 @@ public LoadSingleTsFileNode( boolean isTableModel, String database, boolean deleteAfterLoad, - long writePointCount) { + long writePointCount, + File schemaEvolutionFile) { super(id); this.tsFile = resource.getTsFile(); this.resource = resource; @@ -79,6 +81,7 @@ public LoadSingleTsFileNode( this.database = database; this.deleteAfterLoad = deleteAfterLoad; this.writePointCount = writePointCount; + this.schemaEvolutionFile = schemaEvolutionFile; } public boolean isTsFileEmpty() { @@ -89,6 +92,12 @@ public boolean isTsFileEmpty() { public boolean needDecodeTsFile( Function>, List> partitionFetcher) { + if (schemaEvolutionFile != null) { + // with schema evolution, must split + needDecodeTsFile = true; + return needDecodeTsFile; + } + List> slotList = new ArrayList<>(); resource .getDevices() @@ -152,6 +161,10 @@ public long getWritePointCount() { return writePointCount; } + public File getSchemaEvolutionFile() { + return schemaEvolutionFile; + } + /** * only used for load locally. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java index 3588b6ddbb052..41ae3b9b90b09 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java @@ -34,6 +34,7 @@ import org.apache.tsfile.exception.NotImplementedException; import java.io.DataOutputStream; +import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -46,13 +47,19 @@ public class LoadTsFileNode extends WritePlanNode { private final List resources; private final List isTableModel; private final String database; + private final File schemaEvolutionFile; public LoadTsFileNode( - PlanNodeId id, List resources, List isTableModel, String database) { + PlanNodeId id, + List resources, + List isTableModel, + String database, + File schemaEvolutionFile) { super(id); this.resources = resources; this.isTableModel = isTableModel; this.database = database; + this.schemaEvolutionFile = schemaEvolutionFile; } @Override @@ -121,7 +128,8 @@ private List splitByPartitionForTreeModel(Analysis analysis) { isTableModel.get(i), database, statement.isDeleteAfterLoad(), - statement.getWritePointCount(i))); + statement.getWritePointCount(i), + schemaEvolutionFile)); } return res; } @@ -143,7 +151,8 @@ private List splitByPartitionForTableModel( isTableModel.get(i), database, statement.isDeleteAfterLoad(), - statement.getWritePointCount(i))); + statement.getWritePointCount(i), + schemaEvolutionFile)); } } return res; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index ce573ce1e2d7c..c872fe0f27435 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -1299,7 +1299,11 @@ protected RelationPlan visitLoadTsFile(final LoadTsFile node, final Void context } return new RelationPlan( new LoadTsFileNode( - idAllocator.genPlanNodeId(), node.getResources(), isTableModel, node.getDatabase()), + idAllocator.genPlanNodeId(), + node.getResources(), + isTableModel, + node.getDatabase(), + node.getSchemaEvolutionFile()), analysis.getRootScope(), Collections.emptyList(), outerContext); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java index 93fc8c7b58330..9e551cb2b2047 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java @@ -56,6 +56,7 @@ public class LoadTsFile extends Statement { private List resources; private List writePointCountList; private List isTableModel; + private File schemaEvolutionFile; public LoadTsFile(NodeLocation location, String filePath, Map loadAttributes) { super(location); @@ -179,6 +180,10 @@ public long getWritePointCount(int resourceIndex) { return writePointCountList.get(resourceIndex); } + public File getSchemaEvolutionFile() { + return schemaEvolutionFile; + } + private void initAttributes() { this.databaseLevel = LoadTsFileConfigurator.parseOrGetDefaultDatabaseLevel(loadAttributes); this.database = LoadTsFileConfigurator.parseDatabaseName(loadAttributes); @@ -189,6 +194,7 @@ private void initAttributes() { LoadTsFileConfigurator.parseOrGetDefaultTabletConversionThresholdBytes(loadAttributes); this.verify = LoadTsFileConfigurator.parseOrGetDefaultVerify(loadAttributes); this.isAsyncLoad = LoadTsFileConfigurator.parseOrGetDefaultAsyncLoad(loadAttributes); + this.schemaEvolutionFile = LoadTsFileConfigurator.parseSevoFile(loadAttributes); } public boolean reconstructStatementIfMiniFileConverted(final List isMiniTsFile) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java index 7709e20b55705..d6bd9a398d14a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java @@ -315,7 +315,9 @@ private boolean firstPhase(LoadSingleTsFileNode node) { final TsFileDataManager tsFileDataManager = new TsFileDataManager(this, node, block); try { new TsFileSplitter( - node.getTsFileResource().getTsFile(), tsFileDataManager::addOrSendTsFileData) + node.getTsFileResource().getTsFile(), + tsFileDataManager::addOrSendTsFileData, + node.getSchemaEvolutionFile()) .splitTsFileByDataPartition(); if (!tsFileDataManager.sendAllTsFileData()) { return false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java index d1dff1bb9cf25..3bd7c62a7ff7a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java @@ -68,6 +68,7 @@ public class LoadTsFileStatement extends Statement { private List isTableModel; private List resources; private List writePointCountList; + private File schemaEvolutionFile; public LoadTsFileStatement(String filePath) throws FileNotFoundException { this.file = new File(filePath).getAbsoluteFile(); @@ -247,6 +248,10 @@ public void setLoadAttributes(final Map loadAttributes) { initAttributes(loadAttributes); } + public File getSchemaEvolutionFile() { + return schemaEvolutionFile; + } + public boolean isAsyncLoad() { return isAsyncLoad; } @@ -264,6 +269,7 @@ private void initAttributes(final Map loadAttributes) { if (LoadTsFileConfigurator.parseOrGetDefaultPipeGenerated(loadAttributes)) { markIsGeneratedByPipe(); } + this.schemaEvolutionFile = LoadTsFileConfigurator.parseSevoFile(loadAttributes); } public boolean reconstructStatementIfMiniFileConverted(final List isMiniTsFile) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 835a0977fa520..3e3a4b53b9719 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -1262,7 +1262,10 @@ public void applySchemaEvolutionToObjects(List schemaEvolutions } else if (schemaEvolution instanceof ColumnRename) { ColumnRename columnRename = (ColumnRename) schemaEvolution; if (columnRename.getDataType() == TSDataType.OBJECT) { - renameMeasurementForObjects(columnRename.getTableName(), columnRename.getNameBefore(), columnRename.getNameAfter()); + renameMeasurementForObjects( + columnRename.getTableName(), + columnRename.getNameBefore(), + columnRename.getNameAfter()); } } } @@ -3076,7 +3079,8 @@ private void deleteDataInUnsealedFiles( } } - private boolean canBeFullyDeleted(ArrayDeviceTimeIndex deviceTimeIndex, TableDeletionEntry tableDeletionEntry) { + private boolean canBeFullyDeleted( + ArrayDeviceTimeIndex deviceTimeIndex, TableDeletionEntry tableDeletionEntry) { Set devicesInFile = deviceTimeIndex.getDevices(); String tableName = tableDeletionEntry.getTableName(); long matchSize = @@ -3153,14 +3157,21 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M && (deletion.getType() == ModType.TABLE_DELETION)) { ArrayDeviceTimeIndex deviceTimeIndex = (ArrayDeviceTimeIndex) timeIndex; TableDeletionEntry tableDeletionEntry = (TableDeletionEntry) deletion; - tableDeletionEntry = evolvedSchema != null? evolvedSchema.rewriteToOriginal(tableDeletionEntry) : tableDeletionEntry; + tableDeletionEntry = + evolvedSchema != null + ? evolvedSchema.rewriteToOriginal(tableDeletionEntry) + : tableDeletionEntry; if (canBeFullyDeleted(deviceTimeIndex, tableDeletionEntry)) { deletedByFiles.add(sealedTsFile); } else { - involvedModificationFiles.add(new Pair<>(sealedTsFile.getModFileForWrite(), tableDeletionEntry)); + involvedModificationFiles.add( + new Pair<>(sealedTsFile.getModFileForWrite(), tableDeletionEntry)); } } else { - involvedModificationFiles.add(new Pair<>(sealedTsFile.getModFileForWrite(), evolvedSchema != null? evolvedSchema.rewriteToOriginal(deletion) : deletion)); + involvedModificationFiles.add( + new Pair<>( + sealedTsFile.getModFileForWrite(), + evolvedSchema != null ? evolvedSchema.rewriteToOriginal(deletion) : deletion)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java index e6ec34e529b0b..5f426452a63ed 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java @@ -33,11 +33,11 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.ModFileManagement; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TagPredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TagPredicate.java index 8aca6ed5fe77e..a0624c8555d6d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TagPredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TagPredicate.java @@ -84,6 +84,10 @@ public TagPredicate rewriteToOriginal(EvolvedSchema evolvedSchema) { return this; } + public TagPredicate rewriteToFinal(EvolvedSchema evolvedSchema) { + return this; + } + @Override public long serialize(OutputStream stream) throws IOException { return type.serialize(stream); @@ -253,6 +257,11 @@ public long ramBytesUsed() { public TagPredicate rewriteToOriginal(EvolvedSchema evolvedSchema) { return new FullExactMatch(evolvedSchema.rewriteToOriginal(deviceID)); } + + @Override + public TagPredicate rewriteToFinal(EvolvedSchema evolvedSchema) { + return new FullExactMatch(evolvedSchema.rewriteToFinal(deviceID)); + } } public static class SegmentExactMatch extends TagPredicate { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java index 6e50bf5f0c1d8..7d04bb1aa3e40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java @@ -19,113 +19,151 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry.ModType; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; + +import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.stream.Collectors; public class EvolvedSchema { // the evolved table names after applying all schema evolution operations - private Map originalTableNames = new LinkedHashMap<>(); + private Map finalToOriginalTableNames = new LinkedHashMap<>(); /** * the first key is the evolved table name, the second key is the evolved column name, and the * value is the original column name before any schema evolution. */ - private Map> originalColumnNames = new LinkedHashMap<>(); + private Map> finalToOriginalColumnNames = new LinkedHashMap<>(); + + // the reversed version of finalToOriginalTableNames + private Map originalToFinalTableNames = new LinkedHashMap<>(); + + // the reversed version of finalToOriginalColumnNames + private Map> originalToFinalColumnNames = new LinkedHashMap<>(); public void renameTable(String oldTableName, String newTableName) { - if (!originalTableNames.containsKey(oldTableName) || originalTableNames.get(oldTableName).isEmpty()) { - originalTableNames.put(newTableName, oldTableName); - originalTableNames.put(oldTableName, ""); + if (!finalToOriginalTableNames.containsKey(oldTableName) + || finalToOriginalTableNames.get(oldTableName).isEmpty()) { + finalToOriginalTableNames.put(newTableName, oldTableName); + finalToOriginalTableNames.put(oldTableName, ""); + originalToFinalTableNames.put(oldTableName, newTableName); } else { - // mark the old table name as non-exists - String originalName = originalTableNames.put(oldTableName, ""); - originalTableNames.put(newTableName, originalName); + // mark the old table name as non-exists (empty) + String originalName = finalToOriginalTableNames.put(oldTableName, ""); + finalToOriginalTableNames.put(newTableName, originalName); + originalToFinalTableNames.put(originalName, newTableName); } - if (originalColumnNames.containsKey(oldTableName)) { - Map columnMap = originalColumnNames.remove(oldTableName); - originalColumnNames.put(newTableName, columnMap); + if (finalToOriginalColumnNames.containsKey(oldTableName)) { + Map columnMap = finalToOriginalColumnNames.remove(oldTableName); + finalToOriginalColumnNames.put(newTableName, columnMap); } } - public void renameColumn(String tableName, String oldColumnName, String newColumnName) { + public void renameColumn(String newTableName, String oldColumnName, String newColumnName) { Map columnNameMap = - originalColumnNames.computeIfAbsent(tableName, t -> new LinkedHashMap<>()); + finalToOriginalColumnNames.computeIfAbsent(newTableName, t -> new LinkedHashMap<>()); + String originalTableName = getOriginalTableName(newTableName); if (!columnNameMap.containsKey(oldColumnName) || columnNameMap.get(oldColumnName).isEmpty()) { columnNameMap.put(newColumnName, oldColumnName); columnNameMap.put(oldColumnName, ""); + originalToFinalColumnNames + .computeIfAbsent(originalTableName, t -> new LinkedHashMap<>()) + .put(oldColumnName, newColumnName); } else { // mark the old column name as non-exists String originalName = columnNameMap.put(oldColumnName, ""); columnNameMap.put(newColumnName, originalName); + originalToFinalColumnNames + .computeIfAbsent(originalTableName, t -> new LinkedHashMap<>()) + .put(originalName, newColumnName); } } - public String getOriginalTableName(String evolvedTableName) { - return originalTableNames.getOrDefault(evolvedTableName, evolvedTableName); + public String getOriginalTableName(String finalTableName) { + return finalToOriginalTableNames.getOrDefault(finalTableName, finalTableName); + } + + private String getFinalTableName(String originalTableName) { + return originalToFinalTableNames.getOrDefault(originalTableName, originalTableName); } public String getOriginalColumnName(String tableName, String evolvedColumnName) { - Map columnNameMap = originalColumnNames.get(tableName); + Map columnNameMap = finalToOriginalColumnNames.get(tableName); if (columnNameMap == null) { return evolvedColumnName; } return columnNameMap.getOrDefault(evolvedColumnName, evolvedColumnName); } + public String getFinalColumnName(String originalTableName, String originalColumnName) { + return originalToFinalColumnNames + .getOrDefault(originalTableName, Collections.emptyMap()) + .getOrDefault(originalColumnName, originalColumnName); + } + @Override public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } EvolvedSchema that = (EvolvedSchema) o; - return Objects.equals(originalTableNames, that.originalTableNames) - && Objects.equals(originalColumnNames, that.originalColumnNames); + return Objects.equals(finalToOriginalTableNames, that.finalToOriginalTableNames) + && Objects.equals(finalToOriginalColumnNames, that.finalToOriginalColumnNames); } @Override public int hashCode() { - return Objects.hash(originalTableNames, originalColumnNames); + return Objects.hash(finalToOriginalTableNames, finalToOriginalColumnNames); } @Override public String toString() { return "EvolvedSchema{" + "originalTableNames=" - + originalTableNames + + finalToOriginalTableNames + ", originalColumnNames=" - + originalColumnNames + + finalToOriginalColumnNames + '}'; } public List toSchemaEvolutions() { - List schemaEvolutions = new ArrayList<>(); - originalTableNames.forEach((finalTableName, originalTableName) -> { - if (!originalTableName.isEmpty()) { - schemaEvolutions.add(new TableRename(originalTableName, finalTableName)); - } - }); - originalColumnNames.forEach((finalTableName, originalColumnNameMap) -> { - originalColumnNameMap.forEach((finalColumnName, originalColumnName) -> { - if (!originalColumnName.isEmpty()) { - schemaEvolutions.add(new ColumnRename(finalTableName, originalColumnName, finalColumnName, null)); + List schemaEvolutions = new ArrayList<>(); + finalToOriginalTableNames.forEach( + (finalTableName, originalTableName) -> { + if (!originalTableName.isEmpty()) { + schemaEvolutions.add(new TableRename(originalTableName, finalTableName)); } }); - }); - return schemaEvolutions; + finalToOriginalColumnNames.forEach( + (finalTableName, originalColumnNameMap) -> { + originalColumnNameMap.forEach( + (finalColumnName, originalColumnName) -> { + if (!originalColumnName.isEmpty()) { + schemaEvolutions.add( + new ColumnRename(finalTableName, originalColumnName, finalColumnName, null)); + } + }); + }); + return schemaEvolutions; } public ModEntry rewriteToOriginal(ModEntry entry) { @@ -135,34 +173,104 @@ public ModEntry rewriteToOriginal(ModEntry entry) { return entry; } + public ModEntry rewriteToFinal(ModEntry entry) { + if (entry.getType() == ModType.TABLE_DELETION) { + return rewriteToFinal(((TableDeletionEntry) entry)); + } + return entry; + } + public TableDeletionEntry rewriteToOriginal(TableDeletionEntry entry) { DeletionPredicate deletionPredicate = rewriteToOriginal(entry.getPredicate()); return new TableDeletionEntry(deletionPredicate, entry.getTimeRange()); } + public TableDeletionEntry rewriteToFinal(TableDeletionEntry entry) { + DeletionPredicate deletionPredicate = rewriteToFinal(entry.getPredicate()); + return new TableDeletionEntry(deletionPredicate, entry.getTimeRange()); + } + + private DeletionPredicate rewriteToFinal(DeletionPredicate predicate) { + String finalTableName = getFinalTableName(predicate.getTableName()); + TagPredicate tagPredicate = predicate.getTagPredicate(); + tagPredicate = tagPredicate.rewriteToOriginal(this); + List newMeasurements = + predicate.getMeasurementNames().stream() + .map(m -> getFinalColumnName(predicate.getTableName(), m)) + .collect(Collectors.toList()); + return new DeletionPredicate(finalTableName, tagPredicate, newMeasurements); + } + private DeletionPredicate rewriteToOriginal(DeletionPredicate predicate) { String originalTableName = getOriginalTableName(predicate.getTableName()); TagPredicate tagPredicate = predicate.getTagPredicate(); tagPredicate = tagPredicate.rewriteToOriginal(this); List newMeasurements = - predicate.getMeasurementNames().stream().map(m -> getOriginalColumnName(predicate.getTableName(), m)).collect( - Collectors.toList()); + predicate.getMeasurementNames().stream() + .map(m -> getOriginalColumnName(predicate.getTableName(), m)) + .collect(Collectors.toList()); return new DeletionPredicate(originalTableName, tagPredicate, newMeasurements); } public IDeviceID rewriteToOriginal(IDeviceID deviceID) { String tableName = deviceID.getTableName(); String originalTableName = getOriginalTableName(tableName); - return rewriteToOriginal(deviceID, originalTableName); + return rewriteTableName(deviceID, originalTableName); + } + + public IDeviceID rewriteToFinal(IDeviceID deviceID) { + String tableName = deviceID.getTableName(); + String finalTableName = getFinalTableName(tableName); + return rewriteTableName(deviceID, finalTableName); + } + + public void rewriteToFinal( + String originalTableName, List timeseriesMetadataList) { + timeseriesMetadataList.forEach( + timeseriesMetadata -> { + timeseriesMetadata.setMeasurementId( + getFinalColumnName(originalTableName, timeseriesMetadata.getMeasurementId())); + }); + } + + public Map rewriteToFinal(Map tableSchemas) { + Map finalTableSchemas = new HashMap<>(tableSchemas.size()); + for (Map.Entry entry : tableSchemas.entrySet()) { + TableSchema tableSchema = entry.getValue(); + tableSchema = rewriteToFinal(tableSchema); + finalTableSchemas.put(tableSchema.getTableName(), tableSchema); + } + return finalTableSchemas; + } + + public TableSchema rewriteToFinal(TableSchema tableSchema) { + String finalTableName = getFinalTableName(tableSchema.getTableName()); + + List measurementSchemas = + new ArrayList<>(tableSchema.getColumnSchemas().size()); + List columnCategories = new ArrayList<>(tableSchema.getColumnTypes().size()); + List columnSchemas = tableSchema.getColumnSchemas(); + for (int i = 0, columnSchemasSize = columnSchemas.size(); i < columnSchemasSize; i++) { + IMeasurementSchema measurementSchema = columnSchemas.get(i); + measurementSchemas.add( + new MeasurementSchema( + getFinalColumnName( + tableSchema.getTableName(), measurementSchema.getMeasurementName()), + measurementSchema.getType(), + measurementSchema.getEncodingType(), measurementSchema.getCompressor())); + columnCategories.add(tableSchema.getColumnTypes().get(i)); + } + + return new TableSchema(finalTableName, measurementSchemas, columnCategories); } @SuppressWarnings("SuspiciousSystemArraycopy") - public static IDeviceID rewriteToOriginal(IDeviceID deviceID, String originalTableName) { + public static IDeviceID rewriteTableName(IDeviceID deviceID, String newTableName) { String tableName = deviceID.getTableName(); - if (!tableName.equals(originalTableName)) { + if (!tableName.equals(newTableName)) { Object[] segments = deviceID.getSegments(); String[] newSegments = new String[segments.length]; - newSegments[0] = originalTableName; + newSegments[0] = newTableName; System.arraycopy(segments, 1, newSegments, 1, segments.length - 1); return Factory.DEFAULT_FACTORY.create(newSegments); } @@ -171,8 +279,10 @@ public static IDeviceID rewriteToOriginal(IDeviceID deviceID, String originalTab public static EvolvedSchema deepCopy(EvolvedSchema evolvedSchema) { EvolvedSchema newEvolvedSchema = new EvolvedSchema(); - newEvolvedSchema.originalTableNames = new LinkedHashMap<>(evolvedSchema.originalTableNames); - newEvolvedSchema.originalColumnNames = new LinkedHashMap<>(evolvedSchema.originalColumnNames); + newEvolvedSchema.finalToOriginalTableNames = + new LinkedHashMap<>(evolvedSchema.finalToOriginalTableNames); + newEvolvedSchema.finalToOriginalColumnNames = + new LinkedHashMap<>(evolvedSchema.finalToOriginalColumnNames); return newEvolvedSchema; } @@ -196,14 +306,14 @@ public static EvolvedSchema merge(EvolvedSchema... schemas) { if (schemas[i] != null) { EvolvedSchema newSchema = schemas[i]; for (Entry finalOriginalTableName : - newSchema.originalTableNames.entrySet()) { + newSchema.finalToOriginalTableNames.entrySet()) { if (!finalOriginalTableName.getValue().isEmpty()) { mergedSchema.renameTable( finalOriginalTableName.getValue(), finalOriginalTableName.getKey()); } } for (Entry> finalTableNameColumnNameMapEntry : - newSchema.originalColumnNames.entrySet()) { + newSchema.finalToOriginalColumnNames.entrySet()) { for (Entry finalColNameOriginalColNameEntry : finalTableNameColumnNameMapEntry.getValue().entrySet()) { if (!finalColNameOriginalColNameEntry.getValue().isEmpty()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java index 1c4343cd1544a..e7dd3326913bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/SchemaEvolutionFile.java @@ -101,4 +101,8 @@ public EvolvedSchema readAsSchema() throws IOException { } return evolvedSchema; } + + public String getFilePath() { + return filePath; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java index b9fd86316b8f3..52ec3ab6b8bce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java @@ -62,7 +62,9 @@ public TsFileSet(long endVersion, String fileSetsDir, boolean recover) { } if (schemaEvolutionFile == null) { - schemaEvolutionFile = new SchemaEvolutionFile(fileSetsDir + File.separator + 0 + SchemaEvolutionFile.FILE_SUFFIX); + schemaEvolutionFile = + new SchemaEvolutionFile( + fileSetsDir + File.separator + 0 + SchemaEvolutionFile.FILE_SUFFIX); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java index 6569a7c6eb3e5..d699ca433c15b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java @@ -34,9 +34,9 @@ import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunk; import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunkGroup; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndexCacheRecorder; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/config/LoadTsFileConfigurator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/config/LoadTsFileConfigurator.java index 8478486781be5..9bffbfffce46a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/config/LoadTsFileConfigurator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/config/LoadTsFileConfigurator.java @@ -26,6 +26,7 @@ import javax.annotation.Nullable; +import java.io.File; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -60,11 +61,21 @@ public static void validateParameters(final String key, final String value) { case ASYNC_LOAD_KEY: validateAsyncLoadParam(value); break; + case SEVO_FILE_PATH_KEY: + validateSevoFilePathParam(value); + break; default: throw new SemanticException("Invalid parameter '" + key + "' for LOAD TSFILE command."); } } + private static void validateSevoFilePathParam(String value) { + File file = new File(value); + if (!file.exists()) { + throw new SemanticException("The sevo file " + value + " does not exist."); + } + } + public static void validateSynonymParameters(final Map parameters) { if (parameters.containsKey(DATABASE_KEY) && parameters.containsKey(DATABASE_NAME_KEY)) { throw new SemanticException( @@ -115,6 +126,13 @@ public static int parseOrGetDefaultDatabaseLevel(final Map loadA return Objects.nonNull(databaseName) ? databaseName.toLowerCase(Locale.ENGLISH) : null; } + public static final String SEVO_FILE_PATH_KEY = "sevo-file-path"; + + public static @Nullable File parseSevoFile(final Map loadAttributes) { + String sevoFilePath = loadAttributes.get(SEVO_FILE_PATH_KEY); + return sevoFilePath != null ? new File(sevoFilePath) : null; + } + public static final String ON_SUCCESS_KEY = "on-success"; public static final String ON_SUCCESS_DELETE_VALUE = "delete"; public static final String ON_SUCCESS_NONE_VALUE = "none"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java index 72268168258ee..216a22eaa6f57 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.PageException; @@ -66,7 +67,7 @@ public class AlignedChunkData implements ChunkData { protected static final Binary DEFAULT_BINARY = null; protected final TTimePartitionSlot timePartitionSlot; - protected final IDeviceID device; + protected IDeviceID device; protected List chunkHeaderList; protected PublicBAOS byteStream; @@ -508,4 +509,14 @@ public String toString() { + needDecodeChunk + '}'; } + + @Override + public void rewriteToFinal(EvolvedSchema evolvedSchema) { + IDeviceID newDevice = evolvedSchema.rewriteToFinal(device); + chunkHeaderList.forEach( + h -> + h.setMeasurementID( + evolvedSchema.getFinalColumnName(device.getTableName(), h.getMeasurementID()))); + device = newDevice; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/DeletionData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/DeletionData.java index 0695c7a84def9..c140b79bc1226 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/DeletionData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/DeletionData.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -31,7 +32,7 @@ import java.io.InputStream; public class DeletionData implements TsFileData { - private final ModEntry deletion; + private ModEntry deletion; public DeletionData(ModEntry deletion) { this.deletion = deletion; @@ -51,6 +52,11 @@ public TsFileDataType getType() { return TsFileDataType.DELETION; } + @Override + public void rewriteToFinal(EvolvedSchema evolvedSchema) { + deletion = evolvedSchema.rewriteToFinal(deletion); + } + @Override public void serialize(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(getType().ordinal(), stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java index 2310b9cb95c3e..5ad970c38de65 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.ChunkHeader; @@ -52,7 +53,7 @@ public class NonAlignedChunkData implements ChunkData { private final TTimePartitionSlot timePartitionSlot; - private final IDeviceID device; + private IDeviceID device; private final ChunkHeader chunkHeader; private final PublicBAOS byteStream; @@ -316,6 +317,14 @@ private void close() throws IOException { stream.close(); } + @Override + public void rewriteToFinal(EvolvedSchema evolvedSchema) { + IDeviceID newDevice = evolvedSchema.rewriteToFinal(device); + chunkHeader.setMeasurementID( + evolvedSchema.getFinalColumnName(device.getTableName(), chunkHeader.getMeasurementID())); + device = newDevice; + } + @Override public String toString() { return "NonAlignedChunkData{" diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileData.java index f24eb45c01bc6..d3c5d150b37e7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileData.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.storageengine.load.splitter; import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -35,6 +36,8 @@ public interface TsFileData { void serialize(DataOutputStream stream) throws IOException; + void rewriteToFinal(EvolvedSchema evolvedSchema); + static TsFileData deserialize(InputStream stream) throws IOException, PageException, IllegalPathException { final TsFileDataType type = TsFileDataType.values()[ReadWriteIOUtils.readInt(stream)]; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java index 5a75f4fb8e085..f67c0d96ed00a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java @@ -26,6 +26,8 @@ import org.apache.iotdb.db.exception.load.LoadFileException; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolutionFile; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; @@ -67,7 +69,7 @@ public class TsFileSplitter { private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); private final File tsFile; - private final TsFileDataConsumer consumer; + private TsFileDataConsumer consumer; private Map offset2ChunkMetadata = new HashMap<>(); private List deletions = new ArrayList<>(); private Map> pageIndex2ChunkData = new HashMap<>(); @@ -77,6 +79,7 @@ public class TsFileSplitter { private boolean isAligned; private int timeChunkIndexOfCurrentValueColumn = 0; private Set timePartitionSlots = new HashSet<>(); + private EvolvedSchema evolvedSchema; // Maintain the number of times the chunk of each measurement appears. private Map valueColumn2TimeChunkIndex = new HashMap<>(); @@ -87,9 +90,18 @@ public class TsFileSplitter { private List> pageIndex2TimesList = null; private List isTimeChunkNeedDecodeList = new ArrayList<>(); - public TsFileSplitter(File tsFile, TsFileDataConsumer consumer) { + public TsFileSplitter(File tsFile, TsFileDataConsumer consumer, File schemaEvolutionFile) { this.tsFile = tsFile; this.consumer = consumer; + if (schemaEvolutionFile != null && schemaEvolutionFile.exists()) { + SchemaEvolutionFile sevoFile = new SchemaEvolutionFile(schemaEvolutionFile.getAbsolutePath()); + try { + this.evolvedSchema = sevoFile.readAsSchema(); + this.consumer = new SchemaEvolutionTsFileDataConsumer(this.consumer, evolvedSchema); + } catch (IOException e) { + logger.error("Cannot read schema evolution file, ignoring it.", e); + } + } } @SuppressWarnings({"squid:S3776", "squid:S6541"}) @@ -588,4 +600,38 @@ private TsPrimitiveType[] decodeValuePage( public interface TsFileDataConsumer { boolean apply(TsFileData tsFileData) throws LoadFileException; } + + public abstract class WrappedTsFileDataConsumer implements TsFileDataConsumer { + + private TsFileDataConsumer delegate; + + public WrappedTsFileDataConsumer(TsFileDataConsumer delegate) { + this.delegate = delegate; + } + + protected abstract TsFileData rewrite(TsFileData tsFileData); + + @Override + public boolean apply(TsFileData tsFileData) throws LoadFileException { + tsFileData = rewrite(tsFileData); + return delegate.apply(tsFileData); + } + } + + private class SchemaEvolutionTsFileDataConsumer extends WrappedTsFileDataConsumer { + + private EvolvedSchema evolvedSchema; + + public SchemaEvolutionTsFileDataConsumer( + TsFileDataConsumer delegate, EvolvedSchema evolvedSchema) { + super(delegate); + this.evolvedSchema = evolvedSchema; + } + + @Override + protected TsFileData rewrite(TsFileData tsFileData) { + tsFileData.rewriteToFinal(evolvedSchema); + return tsFileData; + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/path/PatternTreeMapTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/path/PatternTreeMapTest.java index 1db3890d9fe41..6f14ac8638467 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/path/PatternTreeMapTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/path/PatternTreeMapTest.java @@ -24,9 +24,9 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory.ModsSerializer; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionRecoverTest.java index 6eeeda629eb69..fcb2ffdb79294 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionRecoverTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionRecoverTest.java @@ -30,8 +30,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.tsfile.read.common.TimeRange; import org.junit.After; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionResourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionResourceTest.java index bc6dc9e625ab8..05c1f9361c66b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionResourceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionResourceTest.java @@ -43,8 +43,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.tsfile.read.common.TimeRange; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTablePatternParseVisitorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTablePatternParseVisitorTest.java index 63909828e6a8a..82aee3b1fba33 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTablePatternParseVisitorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTablePatternParseVisitorTest.java @@ -29,8 +29,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.storageengine.dataregion.memtable.DeviceIDFactory; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; import org.apache.tsfile.read.common.TimeRange; import org.junit.Assert; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java index e425c709815c3..6b4eec7f63f41 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java @@ -41,7 +41,7 @@ public void testLoadSingleTsFileNode() { TsFileResource resource = new TsFileResource(new File("1")); String database = "root.db"; LoadSingleTsFileNode node = - new LoadSingleTsFileNode(new PlanNodeId(""), resource, false, database, true, 0L); + new LoadSingleTsFileNode(new PlanNodeId(""), resource, false, database, true, 0L, null); Assert.assertTrue(node.isDeleteAfterLoad()); Assert.assertEquals(resource, node.getTsFileResource()); Assert.assertEquals(database, node.getDatabase()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java index a6b6bf1bf6747..a58b4ae7c00da 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java @@ -24,11 +24,11 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.And; import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.SegmentExactMatch; -import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; import org.apache.tsfile.file.metadata.IDeviceID.Factory; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index ec9a4f5eae21a..35e2ed84a6cb9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -60,14 +60,12 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.constant.InnerUnsequenceCompactionSelector; import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.storageengine.dataregion.flush.FlushManager; -import org.apache.iotdb.db.storageengine.dataregion.flush.TsFileFlushPolicy; import org.apache.iotdb.db.storageengine.dataregion.flush.TsFileFlushPolicy.DirectFlushPolicy; import org.apache.iotdb.db.storageengine.dataregion.memtable.ReadOnlyMemChunk; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; -import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -1647,9 +1645,7 @@ public void testDeleteDataInSeqWorkingMemtable() dataRegion.syncCloseAllWorkingTsFileProcessors(); Assert.assertFalse(tsFileResource.anyModFileExists()); Assert.assertFalse( - tsFileResource - .getDevices() - .contains(Factory.DEFAULT_FACTORY.create("root.vehicle.d199"))); + tsFileResource.getDevices().contains(Factory.DEFAULT_FACTORY.create("root.vehicle.d199"))); } @Test @@ -1909,13 +1905,12 @@ public void testSchemaEvolution() } @Test - public void testSchemaEvolutionWithPartialDeletion() - throws WriteProcessException, IOException { + public void testSchemaEvolutionWithPartialDeletion() throws WriteProcessException, IOException { String[] measurements = {"tag1", "s1", "s2"}; MeasurementSchema[] measurementSchemas = { - new MeasurementSchema("tag1", TSDataType.STRING), - new MeasurementSchema("s1", TSDataType.INT64), - new MeasurementSchema("s2", TSDataType.DOUBLE) + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.INT64), + new MeasurementSchema("s2", TSDataType.DOUBLE) }; RelationalInsertRowNode insertRowNode = new RelationalInsertRowNode( @@ -1929,7 +1924,7 @@ public void testSchemaEvolutionWithPartialDeletion() new Object[] {new Binary("tag1".getBytes(StandardCharsets.UTF_8)), 1L, 1.0}, false, new TsTableColumnCategory[] { - TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD + TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD }); dataRegion.insert(insertRowNode); insertRowNode.setTime(20); @@ -1938,23 +1933,40 @@ public void testSchemaEvolutionWithPartialDeletion() // table1 -> table2 dataRegion.applySchemaEvolution(Collections.singletonList(new TableRename("table1", "table2"))); // s1 -> s3 - dataRegion.applySchemaEvolution(Collections.singletonList(new ColumnRename("table2", "s1", "s3", null))); + dataRegion.applySchemaEvolution( + Collections.singletonList(new ColumnRename("table2", "s1", "s3", null))); // delete with table2 - TableDeletionEntry tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table2"), new TimeRange(0, 15)); - RelationalDeleteDataNode relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + TableDeletionEntry tableDeletionEntry = + new TableDeletionEntry(new DeletionPredicate("table2"), new TimeRange(0, 15)); + RelationalDeleteDataNode relationalDeleteDataNode = + new RelationalDeleteDataNode( + new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); dataRegion.deleteByTable(relationalDeleteDataNode); // delete with s3 - tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table2", new NOP(), Collections.singletonList("s3")), new TimeRange(0, 15)); - relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + tableDeletionEntry = + new TableDeletionEntry( + new DeletionPredicate("table2", new NOP(), Collections.singletonList("s3")), + new TimeRange(0, 15)); + relationalDeleteDataNode = + new RelationalDeleteDataNode( + new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); dataRegion.deleteByTable(relationalDeleteDataNode); // delete with table1 - tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table1"), new TimeRange(0, 15)); - relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + tableDeletionEntry = + new TableDeletionEntry(new DeletionPredicate("table1"), new TimeRange(0, 15)); + relationalDeleteDataNode = + new RelationalDeleteDataNode( + new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); dataRegion.deleteByTable(relationalDeleteDataNode); // delete with s1 - tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table2", new NOP(), Collections.singletonList("s1")), new TimeRange(0, 15)); - relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + tableDeletionEntry = + new TableDeletionEntry( + new DeletionPredicate("table2", new NOP(), Collections.singletonList("s1")), + new TimeRange(0, 15)); + relationalDeleteDataNode = + new RelationalDeleteDataNode( + new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); dataRegion.deleteByTable(relationalDeleteDataNode); List sequenceFileList = dataRegion.getSequenceFileList(); @@ -1965,22 +1977,25 @@ public void testSchemaEvolutionWithPartialDeletion() assertEquals("table1", ((TableDeletionEntry) next).getTableName()); next = modEntryIterator.next(); // the s3 modification should be rewritten to s1 - assertEquals(Collections.singletonList("s1"), ((TableDeletionEntry) next).getPredicate().getMeasurementNames()); + assertEquals( + Collections.singletonList("s1"), + ((TableDeletionEntry) next).getPredicate().getMeasurementNames()); next = modEntryIterator.next(); // the table1 modification should be skipped // the s1 modification should be rewritten to empty - assertEquals(Collections.singletonList(""), ((TableDeletionEntry) next).getPredicate().getMeasurementNames()); + assertEquals( + Collections.singletonList(""), + ((TableDeletionEntry) next).getPredicate().getMeasurementNames()); assertFalse(modEntryIterator.hasNext()); } @Test - public void testSchemaEvolutionWithFullDeletion() - throws WriteProcessException, IOException { + public void testSchemaEvolutionWithFullDeletion() throws WriteProcessException, IOException { String[] measurements = {"tag1", "s1", "s2"}; MeasurementSchema[] measurementSchemas = { - new MeasurementSchema("tag1", TSDataType.STRING), - new MeasurementSchema("s1", TSDataType.INT64), - new MeasurementSchema("s2", TSDataType.DOUBLE) + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.INT64), + new MeasurementSchema("s2", TSDataType.DOUBLE) }; RelationalInsertRowNode insertRowNode = new RelationalInsertRowNode( @@ -1994,7 +2009,7 @@ public void testSchemaEvolutionWithFullDeletion() new Object[] {new Binary("tag1".getBytes(StandardCharsets.UTF_8)), 1L, 1.0}, false, new TsTableColumnCategory[] { - TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD + TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD }); dataRegion.insert(insertRowNode); insertRowNode.setTime(20); @@ -2003,11 +2018,15 @@ public void testSchemaEvolutionWithFullDeletion() // table1 -> table2 dataRegion.applySchemaEvolution(Collections.singletonList(new TableRename("table1", "table2"))); // s1 -> s3 - dataRegion.applySchemaEvolution(Collections.singletonList(new ColumnRename("table2", "s1", "s3", null))); + dataRegion.applySchemaEvolution( + Collections.singletonList(new ColumnRename("table2", "s1", "s3", null))); // delete with table1 - TableDeletionEntry tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table1"), new TimeRange(0, 30)); - RelationalDeleteDataNode relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + TableDeletionEntry tableDeletionEntry = + new TableDeletionEntry(new DeletionPredicate("table1"), new TimeRange(0, 30)); + RelationalDeleteDataNode relationalDeleteDataNode = + new RelationalDeleteDataNode( + new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); dataRegion.deleteByTable(relationalDeleteDataNode); // nothing should be deleted List sequenceFileList = dataRegion.getSequenceFileList(); @@ -2016,8 +2035,11 @@ public void testSchemaEvolutionWithFullDeletion() assertFalse(modEntryIterator.hasNext()); // delete with table2 - tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table2"), new TimeRange(0, 30)); - relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + tableDeletionEntry = + new TableDeletionEntry(new DeletionPredicate("table2"), new TimeRange(0, 30)); + relationalDeleteDataNode = + new RelationalDeleteDataNode( + new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); dataRegion.deleteByTable(relationalDeleteDataNode); // the file should be deleted sequenceFileList = dataRegion.getSequenceFileList(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/BatchedCompactionWithTsFileSplitterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/BatchedCompactionWithTsFileSplitterTest.java index 272d9e6ae5ca7..b64e67a572546 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/BatchedCompactionWithTsFileSplitterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/BatchedCompactionWithTsFileSplitterTest.java @@ -301,7 +301,8 @@ private void consumeChunkDataAndValidate(TsFileResource resource) throw new RuntimeException(e); } return true; - }); + }, + null); splitter.splitTsFileByDataPartition(); List splitResources = new ArrayList<>(); for (Map.Entry entry : writerMap.entrySet()) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java index a83fa920853aa..69e83e769a3ce 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java @@ -35,9 +35,9 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; -import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.tsfile.exception.write.WriteProcessException; From 3d2c88095c1e1da0ff7cf44fb54f7b2db18d4a20 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 30 Dec 2025 18:45:58 +0800 Subject: [PATCH 15/49] remove empty file sets in start up --- .../it/db/it/IoTDBLoadTsFileIT.java | 23 +++++++++++++- .../storageengine/dataregion/DataRegion.java | 30 ++++++++++++++++--- .../tsfile/evolution/ColumnRename.java | 6 +++- .../dataregion/tsfile/fileset/TsFileSet.java | 5 ++++ 4 files changed, 58 insertions(+), 6 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java index 46cc2e63adb35..b9241b66182ef 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java @@ -20,6 +20,7 @@ package org.apache.iotdb.relational.it.db.it; import java.sql.SQLException; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.ColumnRename; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolutionFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.TableRename; @@ -293,7 +294,9 @@ public void testLoadWithSevoFile() throws Exception { SchemaEvolutionFile schemaEvolutionFile = new SchemaEvolutionFile(sevoFile.getAbsolutePath()); SchemaEvolution schemaEvolution = new TableRename(SchemaConfig.TABLE_0, SchemaConfig.TABLE_1); schemaEvolutionFile.append(Collections.singletonList(schemaEvolution)); - // rename INT322INT32 + // rename INT322INT32 to INT322INT32_NEW + schemaEvolution = new ColumnRename(SchemaConfig.TABLE_1, "INT322INT32", "INT322INT32_NEW"); + schemaEvolutionFile.append(Collections.singletonList(schemaEvolution)); try (final Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -323,6 +326,24 @@ public void testLoadWithSevoFile() throws Exception { } } + // cannot query using INT322INT32 + try (final ResultSet resultSet = + statement.executeQuery(String.format("select count(%s) from %s", "INT322INT32", SchemaConfig.TABLE_1))) { + fail(); + } catch (SQLException e) { + assertEquals("616: Column 'int322int32' cannot be resolved", e.getMessage()); + } + + // can query with INT322INT32_NEW + try (final ResultSet resultSet = + statement.executeQuery(String.format("select count(%s) from %s", "INT322INT32_NEW", SchemaConfig.TABLE_1))) { + if (resultSet.next()) { + Assert.assertEquals(lineCount, resultSet.getLong(1)); + } else { + Assert.fail("This ResultSet is empty."); + } + } + try (final ResultSet resultSet = statement.executeQuery("show tables")) { Assert.assertTrue(resultSet.next()); assertEquals(SchemaConfig.TABLE_1, resultSet.getString(1)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 3e3a4b53b9719..e64350500a333 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -660,6 +660,7 @@ private void recover() throws DataRegionException { } // ensure that seq and unseq files in the same partition have the same TsFileSet Map> recoveredPartitionTsFileSetMap = new HashMap<>(); + Map partitionMinimalVersion = new HashMap<>(); for (Entry> partitionFiles : partitionTmpSeqTsFiles.entrySet()) { Callable asyncRecoverTask = @@ -669,7 +670,8 @@ private void recover() throws DataRegionException { partitionFiles.getValue(), fileTimeIndexMap, true, - recoveredPartitionTsFileSetMap); + recoveredPartitionTsFileSetMap, + partitionMinimalVersion); if (asyncRecoverTask != null) { asyncTsFileResourceRecoverTaskList.add(asyncRecoverTask); } @@ -683,7 +685,7 @@ private void recover() throws DataRegionException { partitionFiles.getValue(), fileTimeIndexMap, false, - recoveredPartitionTsFileSetMap); + recoveredPartitionTsFileSetMap, partitionMinimalVersion); if (asyncRecoverTask != null) { asyncTsFileResourceRecoverTaskList.add(asyncRecoverTask); } @@ -698,6 +700,18 @@ private void recover() throws DataRegionException { Long.MAX_VALUE, lastFlushTimeMap.getMemSize(latestPartitionId))); } + + // remove empty file sets + for (Entry> entry : recoveredPartitionTsFileSetMap.entrySet()) { + long partitionId = entry.getKey(); + // if no file in the partition, all filesets should be cleared + long minimumFileVersion = partitionMinimalVersion.getOrDefault(partitionId, Long.MAX_VALUE); + for (TsFileSet tsFileSet : entry.getValue()) { + if (tsFileSet.getEndVersion() < minimumFileVersion) { + tsFileSet.remove(); + } + } + } } // wait until all unsealed TsFiles have been recovered for (WALRecoverListener recoverListener : recoverListeners) { @@ -1063,16 +1077,24 @@ private Callable recoverFilesInPartition( List resourceList, Map fileTimeIndexMap, boolean isSeq, - Map> partitionTsFileSetMap) { + Map> partitionTsFileSetMap, Map partitionMinimalVersion) { List resourceListForAsyncRecover = new ArrayList<>(); List resourceListForSyncRecover = new ArrayList<>(); Callable asyncRecoverTask = null; + List tsFileSets = recoverTsFileSets(partitionId, partitionTsFileSetMap); for (TsFileResource tsFileResource : resourceList) { - List tsFileSets = recoverTsFileSets(partitionId, partitionTsFileSetMap); long fileVersion = tsFileResource.getTsFileID().fileVersion; + partitionMinimalVersion.compute(partitionId, (pid, oldVersion) -> { + if (oldVersion == null) { + return fileVersion; + } + return Math.min(oldVersion, fileVersion); + }); + int i = Collections.binarySearch(tsFileSets, TsFileSet.comparatorKey(fileVersion)); if (i < 0) { + // if the binary search does not find an exact match, -i indicates the closest one i = -i; } if (i < tsFileSets.size()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java index 9d13ce4f7e645..adb6e7e935911 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/ColumnRename.java @@ -40,10 +40,14 @@ public class ColumnRename implements SchemaEvolution { // for deserialization public ColumnRename() {} - public ColumnRename(String tableName, String nameBefore, String nameAfter, TSDataType dataType) { + public ColumnRename(String tableName, String nameBefore, String nameAfter) { this.tableName = tableName.toLowerCase(); this.nameBefore = nameBefore.toLowerCase(); this.nameAfter = nameAfter.toLowerCase(); + } + + public ColumnRename(String tableName, String nameBefore, String nameAfter, TSDataType dataType) { + this(tableName, nameBefore, nameAfter); this.dataType = dataType; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java index 52ec3ab6b8bce..c2364b20b616a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.util.Collection; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.tsfile.external.commons.io.FileUtils; /** TsFileSet represents a set of TsFiles in a time partition whose version <= endVersion. */ public class TsFileSet implements Comparable { @@ -127,4 +128,8 @@ public long getEndVersion() { public String toString() { return "TsFileSet{" + "endVersion=" + endVersion + ", fileSetDir=" + fileSetDir + '}'; } + + public void remove() { + FileUtils.deleteQuietly(fileSetDir); + } } From 12b959ae2439a633fce5a54895d2727c07455d58 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 31 Dec 2025 16:35:07 +0800 Subject: [PATCH 16/49] temp --- .../impl/FastCompactionPerformer.java | 14 +++- .../impl/ReadChunkCompactionPerformer.java | 2 +- .../impl/ReadPointCompactionPerformer.java | 13 +++- .../execute/utils/CompactionTableSchema.java | 9 +++ .../utils/CompactionTableSchemaCollector.java | 20 +++++- .../utils/MultiTsFileDeviceIterator.java | 18 ++++- .../utils/ReorderedTsFileDeviceIterator.java | 65 +++++++++++++++++++ .../TransformedTsFileDeviceIterator.java | 51 +++++++++++++++ .../writer/AbstractCompactionWriter.java | 4 +- .../writer/AbstractCrossCompactionWriter.java | 25 ++++++- .../writer/AbstractInnerCompactionWriter.java | 16 ++++- .../compaction/io/CompactionTsFileWriter.java | 13 +++- .../dataregion/tsfile/TsFileManager.java | 33 ++++++---- .../dataregion/tsfile/TsFileResource.java | 49 ++++++++++++-- .../tsfile/evolution/EvolvedSchema.java | 41 ++++++++++++ 15 files changed, 335 insertions(+), 38 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/ReorderedTsFileDeviceIterator.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/TransformedTsFileDeviceIterator.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java index 91184aaec82cb..bc950a2711478 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PatternTreeMap; @@ -162,10 +164,18 @@ public void perform() throws Exception { ? new FastCrossCompactionWriter( targetFiles, seqFiles, readerCacheMap, encryptParameter) : new FastInnerCompactionWriter(targetFiles, encryptParameter)) { + + List allSourceFiles = Stream.concat(seqFiles.stream(), unseqFiles.stream()) + .sorted(TsFileResource::compareFileName) + .collect(Collectors.toList()); + Pair maxTsFileSetEndVersionAndMinResource = TsFileResource.getMaxTsFileSetEndVersionAndMinResource( + allSourceFiles); List schemas = CompactionTableSchemaCollector.collectSchema( - seqFiles, unseqFiles, readerCacheMap, deviceIterator.getDeprecatedTableSchemaMap()); - compactionWriter.setSchemaForAllTargetFile(schemas); + seqFiles, unseqFiles, readerCacheMap, deviceIterator.getDeprecatedTableSchemaMap(), + maxTsFileSetEndVersionAndMinResource); + + compactionWriter.setSchemaForAllTargetFile(schemas, maxTsFileSetEndVersionAndMinResource); readModification(seqFiles); readModification(unseqFiles); while (deviceIterator.hasNextDevice()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java index eaed8a7f6ebde..2fefbba9ab1de 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java @@ -205,7 +205,7 @@ private void rollCompactionFileWriter() throws IOException { private void useNewWriter() throws IOException { currentWriter = new CompactionTsFileWriter( - targetResources.get(currentTargetFileIndex).getTsFile(), + targetResources.get(currentTargetFileIndex), memoryBudgetForFileWriter, CompactionType.INNER_SEQ_COMPACTION, firstEncryptParameter); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java index c58870357d915..824bb7e5196db 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl; +import java.util.stream.Stream; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.path.AlignedFullPath; @@ -153,13 +154,21 @@ public void perform() throws Exception { // Do not close device iterator, because tsfile reader is managed by FileReaderManager. MultiTsFileDeviceIterator deviceIterator = new MultiTsFileDeviceIterator(seqFiles, unseqFiles); + List allSourceFiles = Stream.concat(seqFiles.stream(), unseqFiles.stream()) + .sorted(TsFileResource::compareFileName) + .collect(Collectors.toList()); + Pair maxTsFileSetEndVersionAndMinResource = TsFileResource.getMaxTsFileSetEndVersionAndMinResource( + allSourceFiles); + List schemas = CompactionTableSchemaCollector.collectSchema( seqFiles, unseqFiles, deviceIterator.getReaderMap(), - deviceIterator.getDeprecatedTableSchemaMap()); - compactionWriter.setSchemaForAllTargetFile(schemas); + deviceIterator.getDeprecatedTableSchemaMap(), + maxTsFileSetEndVersionAndMinResource); + + compactionWriter.setSchemaForAllTargetFile(schemas, maxTsFileSetEndVersionAndMinResource); while (deviceIterator.hasNextDevice()) { checkThreadInterrupted(); Pair deviceInfo = deviceIterator.nextDevice(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionTableSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionTableSchema.java index 3f6e83cbe96c0..99c4286c7595f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionTableSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionTableSchema.java @@ -33,6 +33,15 @@ public CompactionTableSchema(String tableName) { super(tableName); } + public CompactionTableSchema(TableSchema tableSchema) { + this(tableSchema.getTableName(), tableSchema.getColumnSchemas(), tableSchema.getColumnTypes()); + } + + public CompactionTableSchema(String tableName, List columnSchemas, + List columnCategories) { + super(tableName, columnSchemas, columnCategories); + } + public boolean merge(TableSchema tableSchema) { if (tableSchema == null) { return true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionTableSchemaCollector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionTableSchemaCollector.java index 55640c3fcfa4d..d5eb8d08d98fa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionTableSchemaCollector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionTableSchemaCollector.java @@ -21,8 +21,10 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.schema.Schema; import java.io.IOException; @@ -42,7 +44,8 @@ public static List collectSchema( List seqFiles, List unseqFiles, Map readerMap, - Map> deprecatedTableSchemaMap) + Map> deprecatedTableSchemaMap, + Pair maxTsFileSetEndVersionAndAssociatedResource) throws IOException { List targetSchemas = new ArrayList<>(seqFiles.size()); Schema schema = @@ -51,7 +54,8 @@ public static List collectSchema( .sorted(TsFileResource::compareFileName) .collect(Collectors.toList()), readerMap, - deprecatedTableSchemaMap); + deprecatedTableSchemaMap, + maxTsFileSetEndVersionAndAssociatedResource); targetSchemas.add(schema); for (int i = 1; i < seqFiles.size(); i++) { @@ -72,10 +76,12 @@ public static Schema copySchema(Schema source) { public static Schema collectSchema( List sourceFiles, Map readerMap, - Map> deprecatedTableSchemaMap) + Map> deprecatedTableSchemaMap, + Pair maxTsFileSetEndVersionAndAssociatedResource) throws IOException { Schema targetSchema = new Schema(); Map targetTableSchemaMap = new HashMap<>(); + for (int i = 0; i < sourceFiles.size(); i++) { TsFileResource resource = sourceFiles.get(i); TsFileSequenceReader reader = readerMap.get(resource); @@ -84,12 +90,20 @@ public static Schema collectSchema( // v3 tsfile continue; } + + EvolvedSchema evolvedSchema = resource.getMergedEvolvedSchema( + maxTsFileSetEndVersionAndAssociatedResource.getLeft()); + for (Map.Entry entry : tableSchemaMap.entrySet()) { String tableName = entry.getKey(); TableSchema currentTableSchema = entry.getValue(); if (isTreeModel(currentTableSchema)) { continue; } + if (evolvedSchema != null) { + currentTableSchema = evolvedSchema.rewriteToFinal(currentTableSchema); + } + // merge all id columns, measurement schema will be generated automatically when end chunk // group CompactionTableSchema collectedTableSchema = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java index 1889182a2db4a..aef15ad3dd42d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java @@ -32,6 +32,8 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.fileset.TsFileSet; import org.apache.iotdb.db.utils.EncryptDBUtils; import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; @@ -95,6 +97,12 @@ public MultiTsFileDeviceIterator(List tsFileResources) throws IO // sort the files from the newest to the oldest Collections.sort( this.tsFileResourcesSortedByDesc, TsFileResource::compareFileCreationOrderByDesc); + long maxTsFileSetEndVersion = this.tsFileResourcesSortedByDesc.stream().mapToLong( + // max endVersion of all filesets of a TsFile + resource -> resource.getTsFileSets().stream().mapToLong(TsFileSet::getEndVersion).max() + .orElse(Long.MAX_VALUE)) + // overall max endVersion + .max().orElse(Long.MAX_VALUE); try { for (TsFileResource tsFileResource : this.tsFileResourcesSortedByDesc) { CompactionTsFileReader reader = @@ -103,7 +111,15 @@ public MultiTsFileDeviceIterator(List tsFileResources) throws IO CompactionType.INNER_SEQ_COMPACTION, EncryptDBUtils.getFirstEncryptParamFromTSFilePath(tsFileResource.getTsFilePath())); readerMap.put(tsFileResource, reader); - deviceIteratorMap.put(tsFileResource, reader.getAllDevicesIteratorWithIsAligned()); + TsFileDeviceIterator tsFileDeviceIterator; + EvolvedSchema evolvedSchema = tsFileResource.getMergedEvolvedSchema(maxTsFileSetEndVersion); + if (evolvedSchema != null) { + tsFileDeviceIterator = new ReorderedTsFileDeviceIterator(reader, + evolvedSchema::rewriteToFinal); + } else { + tsFileDeviceIterator = reader.getAllDevicesIteratorWithIsAligned(); + } + deviceIteratorMap.put(tsFileResource, tsFileDeviceIterator); } } catch (Exception e) { // if there is any exception occurs diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/ReorderedTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/ReorderedTsFileDeviceIterator.java new file mode 100644 index 0000000000000..14d2db8a191d1 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/ReorderedTsFileDeviceIterator.java @@ -0,0 +1,65 @@ +package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.function.Function; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.MetadataIndexNode; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.utils.Pair; + +public class ReorderedTsFileDeviceIterator extends TransformedTsFileDeviceIterator { + + private final List, MetadataIndexNode>> deviceIDAndFirstMeasurementNodeList = new ArrayList<>(); + private Iterator, MetadataIndexNode>> deviceIDListIterator; + private Pair, MetadataIndexNode> current; + + public ReorderedTsFileDeviceIterator(TsFileSequenceReader reader, + Function transformer) + throws IOException { + super(reader, transformer); + collectAndSort(); + } + + public ReorderedTsFileDeviceIterator(TsFileSequenceReader reader, String tableName, + Function transformer) throws IOException { + super(reader, tableName, transformer); + collectAndSort(); + } + + private void collectAndSort() { + while (super.hasNext()) { + Pair next = super.next(); + next.left = transformer.apply(next.left); + deviceIDAndFirstMeasurementNodeList.add(new Pair<>(next, super.getFirstMeasurementNodeOfCurrentDevice())); + } + deviceIDAndFirstMeasurementNodeList.sort(Comparator.comparing(p -> p.getLeft().getLeft())); + deviceIDListIterator = deviceIDAndFirstMeasurementNodeList.iterator(); + } + + @Override + public boolean hasNext() { + return deviceIDListIterator.hasNext(); + } + + @Override + public Pair next() { + Pair, MetadataIndexNode> next = deviceIDListIterator.next(); + current = next; + return next.left; + } + + @Override + public Pair current() { + return current.left; + } + + @Override + public MetadataIndexNode getFirstMeasurementNodeOfCurrentDevice() { + // the devices have been reordered, cannot use the measurementNode + return current.right; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/TransformedTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/TransformedTsFileDeviceIterator.java new file mode 100644 index 0000000000000..f1af028226d84 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/TransformedTsFileDeviceIterator.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils; + +import java.io.IOException; +import java.util.function.Function; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.read.TsFileDeviceIterator; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.utils.Pair; + +public class TransformedTsFileDeviceIterator extends TsFileDeviceIterator { + + protected Function transformer; + + public TransformedTsFileDeviceIterator(TsFileSequenceReader reader, Function transformer) + throws IOException { + super(reader); + this.transformer = transformer; + } + + public TransformedTsFileDeviceIterator(TsFileSequenceReader reader, String tableName, Function transformer) + throws IOException { + super(reader, tableName); + this.transformer = transformer; + } + + @Override + public Pair next() { + Pair next = super.next(); + next.left = transformer.apply(next.left); + return next; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java index 3458f9870d466..623f2b3287d64 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFileWriter; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.PageHeader; @@ -38,6 +39,7 @@ import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; import org.apache.tsfile.write.chunk.ChunkWriterImpl; @@ -338,5 +340,5 @@ protected void checkPreviousTimestamp(long currentWritingTimestamp, int subTaskI } } - public abstract void setSchemaForAllTargetFile(List schemas); + public abstract void setSchemaForAllTargetFile(List schemas, Pair maxTsFileSetEndVersionAndAssociatedResource); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java index f970ad65e56c3..6584513f60660 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java @@ -19,12 +19,17 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer; +import java.util.stream.Collectors; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionTableSchema; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFileWriter; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.fileset.TsFileSet; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; import org.apache.iotdb.db.utils.EncryptDBUtils; @@ -35,6 +40,7 @@ import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.schema.Schema; @@ -99,7 +105,7 @@ protected AbstractCrossCompactionWriter( for (int i = 0; i < targetResources.size(); i++) { this.targetFileWriters.add( new CompactionTsFileWriter( - targetResources.get(i).getTsFile(), + targetResources.get(i), memorySizeForEachWriter, CompactionType.CROSS_COMPACTION, this.encryptParameter)); @@ -266,9 +272,22 @@ public long getWriterSize() throws IOException { } @Override - public void setSchemaForAllTargetFile(List schemas) { + public void setSchemaForAllTargetFile(List schemas, Pair maxTsFileSetEndVersionAndMinResource) { for (int i = 0; i < targetFileWriters.size(); i++) { - targetFileWriters.get(i).setSchema(schemas.get(i)); + CompactionTsFileWriter compactionTsFileWriter = targetFileWriters.get(i); + Schema schema = schemas.get(i); + TsFileResource targetResource = compactionTsFileWriter.getTsFileResource(); + if (maxTsFileSetEndVersionAndMinResource.right != null) { + long maxTsFileSetEndVersion = maxTsFileSetEndVersionAndMinResource.left; + TsFileResource minVersionResource = maxTsFileSetEndVersionAndMinResource.getRight(); + targetResource.setTsFileManager(minVersionResource.getTsFileManager()); + EvolvedSchema evolvedSchema = targetResource.getMergedEvolvedSchema(maxTsFileSetEndVersion); + + schema = evolvedSchema.rewriteToOriginal(schema, CompactionTableSchema::new); + compactionTsFileWriter.setSchema(schema); + } else { + compactionTsFileWriter.setSchema(schema); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java index 6573bb7e96e86..d7d7a44c00b3e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java @@ -21,11 +21,13 @@ import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionTableSchema; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionTableSchemaCollector; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFileWriter; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; import org.apache.iotdb.db.utils.EncryptDBUtils; @@ -33,6 +35,7 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.schema.Schema; import java.io.IOException; @@ -46,6 +49,7 @@ public abstract class AbstractInnerCompactionWriter extends AbstractCompactionWr protected long endedFileSize = 0; protected List schemas; protected EncryptParameter encryptParameter; + protected Pair maxTsFileSetEndVersionAndMinResource; protected final long memoryBudgetForFileWriter = (long) @@ -115,13 +119,18 @@ private void rollCompactionFileWriter() throws IOException { private void useNewWriter() throws IOException { fileWriter = new CompactionTsFileWriter( - targetResources.get(currentFileIndex).getTsFile(), + targetResources.get(currentFileIndex), memoryBudgetForFileWriter, targetResources.get(currentFileIndex).isSeq() ? CompactionType.INNER_SEQ_COMPACTION : CompactionType.INNER_UNSEQ_COMPACTION, encryptParameter); - fileWriter.setSchema(CompactionTableSchemaCollector.copySchema(schemas.get(0))); + Schema schema = CompactionTableSchemaCollector.copySchema(schemas.get(0)); + long maxTsFileSetEndVersion = maxTsFileSetEndVersionAndMinResource.left; + TsFileResource minVersionResource = maxTsFileSetEndVersionAndMinResource.getRight(); + fileWriter.getTsFileResource().setTsFileManager(minVersionResource.getTsFileManager()); + EvolvedSchema evolvedSchema = fileWriter.getTsFileResource().getMergedEvolvedSchema(maxTsFileSetEndVersion); + fileWriter.setSchema(evolvedSchema.rewriteToOriginal(schema, CompactionTableSchema::new)); } @Override @@ -174,8 +183,9 @@ public void checkAndMayFlushChunkMetadata() throws IOException { } @Override - public void setSchemaForAllTargetFile(List schemas) { + public void setSchemaForAllTargetFile(List schemas, Pair maxTsFileSetEndVersionAndMinResource) { this.schemas = schemas; + this.maxTsFileSetEndVersionAndMinResource = maxTsFileSetEndVersionAndMinResource; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java index 1f822e3b9345d..7d7bf5cba8249 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionIoDataType; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.utils.EncryptDBUtils; import org.apache.tsfile.encrypt.EncryptParameter; @@ -54,19 +55,21 @@ public class CompactionTsFileWriter extends TsFileIOWriter { private volatile boolean isWritingAligned = false; private boolean isEmptyTargetFile = true; private IDeviceID currentDeviceId; + private TsFileResource tsFileResource; private EncryptParameter firstEncryptParameter; @TestOnly public CompactionTsFileWriter(File file, long maxMetadataSize, CompactionType type) throws IOException { - this(file, maxMetadataSize, type, EncryptDBUtils.getDefaultFirstEncryptParam()); + this(new TsFileResource(file), maxMetadataSize, type, EncryptDBUtils.getDefaultFirstEncryptParam()); } public CompactionTsFileWriter( - File file, long maxMetadataSize, CompactionType type, EncryptParameter encryptParameter) + TsFileResource tsFile, long maxMetadataSize, CompactionType type, EncryptParameter encryptParameter) throws IOException { - super(file, maxMetadataSize, encryptParameter); + super(tsFile.getTsFile(), maxMetadataSize, encryptParameter); + this.tsFileResource = tsFile; this.firstEncryptParameter = encryptParameter; this.type = type; super.out = @@ -192,4 +195,8 @@ private void removeUnusedTableSchema() { iterator.remove(); } } + + public TsFileResource getTsFileResource() { + return tsFileResource; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java index 4466668ad5fa3..11dcde29b37ae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile; +import java.util.stream.Collectors; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.storageengine.dataregion.modification.ModFileManagement; @@ -56,6 +57,7 @@ public class TsFileManager { private final TreeMap sequenceFiles = new TreeMap<>(); private final TreeMap unsequenceFiles = new TreeMap<>(); private final TreeMap modFileManagementMap = new TreeMap<>(); + private final TreeMap> tsfileSets = new TreeMap<>(); private volatile boolean allowCompaction = true; private final AtomicLong currentCompactionTaskSerialId = new AtomicLong(0); @@ -237,6 +239,7 @@ public void insertToPartitionFileList( modFileManagementMap.computeIfAbsent( timePartition, t -> new PartitionLevelModFileManager())); } + tsFileResource.setTsFileManager(this); } finally { writeUnlock(); } @@ -255,6 +258,7 @@ public void add(TsFileResource tsFileResource, boolean sequence) { modFileManagementMap.computeIfAbsent( tsFileResource.getTimePartition(), t -> new PartitionLevelModFileManager())); } + tsFileResource.setTsFileManager(this); } finally { writeUnlock(); } @@ -273,6 +277,7 @@ public void keepOrderInsert(TsFileResource tsFileResource, boolean sequence) thr modFileManagementMap.computeIfAbsent( tsFileResource.getTimePartition(), t -> new PartitionLevelModFileManager())); } + tsFileResource.setTsFileManager(this); } finally { writeUnlock(); } @@ -333,6 +338,7 @@ public void replace( modFileManagementMap.computeIfAbsent( resource.getTimePartition(), t -> new PartitionLevelModFileManager())); } + resource.setTsFileManager(this); } } } finally { @@ -512,21 +518,22 @@ public long getMaxFileTimestampOfUnSequenceFile() { public void addTsFileSet(TsFileSet newSet, long partitionId) { writeLock("addTsFileSet"); try { - TsFileResourceList tsFileResources = sequenceFiles.get(partitionId); - if (tsFileResources != null) { - for (TsFileResource tsFileResource : tsFileResources) { - tsFileResource.addFileSet(newSet); - } - } - - tsFileResources = unsequenceFiles.get(partitionId); - if (tsFileResources != null) { - for (TsFileResource tsFileResource : tsFileResources) { - tsFileResource.addFileSet(newSet); - } - } + List tsFileSetList = tsfileSets.computeIfAbsent(partitionId, + p -> new ArrayList<>()); + tsFileSetList.add(newSet); } finally { writeUnlock(); } } + + public List getTsFileSet(long partitionId, long minFileVersionIncluded, long maxFileVersionExcluded) { + readLock(); + try { + List tsFileSetList = tsfileSets.get(partitionId); + return tsFileSetList.stream().filter(s -> s.getEndVersion() < maxFileVersionExcluded && s.getEndVersion() >= minFileVersionIncluded).collect( + Collectors.toList()); + } finally { + readUnlock(); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index 060c16fea7d3f..a387db509c698 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -211,8 +211,7 @@ public class TsFileResource implements PersistentResource, Cloneable { private Map>> lastValues; - // TsFileSets this TsFile belongs to - private final List tsFileSets = new ArrayList<>(); + private TsFileManager tsFileManager = null; @TestOnly public TsFileResource() { @@ -1632,17 +1631,25 @@ public TsFileResource shallowCloneForNative() throws CloneNotSupportedException return (TsFileResource) clone(); } - public void addFileSet(TsFileSet tsFileSet) { - tsFileSets.add(tsFileSet); + public List getTsFileSets() { + return tsFileManager.getTsFileSet(tsFileID.timePartitionId, tsFileID.fileVersion, Long.MAX_VALUE); } - public List getTsFileSets() { - return tsFileSets; + public List getTsFileSets(long maxEndVersionExcluded) { + return tsFileManager.getTsFileSet(tsFileID.timePartitionId, tsFileID.fileVersion, maxEndVersionExcluded); } public EvolvedSchema getMergedEvolvedSchema() { + return getMergedEvolvedSchema(Long.MAX_VALUE); + } + + public EvolvedSchema getMergedEvolvedSchema(long excludedMaxFileVersion) { List list = new ArrayList<>(); for (TsFileSet fileSet : getTsFileSets()) { + if (fileSet.getEndVersion() >= excludedMaxFileVersion) { + continue; + } + try { EvolvedSchema readEvolvedSchema = fileSet.readEvolvedSchema(); list.add(readEvolvedSchema); @@ -1653,4 +1660,34 @@ public EvolvedSchema getMergedEvolvedSchema() { return EvolvedSchema.merge(list.toArray(new EvolvedSchema[0])); } + + public static Pair getMaxTsFileSetEndVersionAndMinResource(List tsFileResources) { + long maxTsFileSetEndVersion = Long.MIN_VALUE; + long minResourceVersion = Long.MAX_VALUE; + TsFileResource minTsFileResource = null; + for (TsFileResource tsFileResource : tsFileResources) { + List tsFileSets = tsFileResource.getTsFileSets(); + if (tsFileSets.isEmpty()) { + continue; + } + TsFileSet lastTsFileSet = tsFileSets.get(tsFileSets.size() - 1); + if (lastTsFileSet.getEndVersion() > maxTsFileSetEndVersion) { + maxTsFileSetEndVersion = lastTsFileSet.getEndVersion(); + } + if (tsFileResource.getTsFileID().fileVersion < minResourceVersion) { + minTsFileResource = tsFileResource; + minResourceVersion = tsFileResource.getTsFileID().fileVersion; + } + } + return new Pair<>(maxTsFileSetEndVersion, minTsFileResource); + } + + public void setTsFileManager( + TsFileManager tsFileManager) { + this.tsFileManager = tsFileManager; + } + + public TsFileManager getTsFileManager() { + return tsFileManager; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java index 7d04bb1aa3e40..e447a1fc630c7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; +import java.util.function.Function; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionTableSchema; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry.ModType; @@ -42,6 +44,7 @@ import java.util.Map.Entry; import java.util.Objects; import java.util.stream.Collectors; +import org.apache.tsfile.write.schema.Schema; public class EvolvedSchema { // the evolved table names after applying all schema evolution operations @@ -243,6 +246,27 @@ public Map rewriteToFinal(Map tableSch return finalTableSchemas; } + private TableSchema rewriteToOriginal(TableSchema tableSchema) { + String originalTableName = getOriginalTableName(tableSchema.getTableName()); + + List measurementSchemas = + new ArrayList<>(tableSchema.getColumnSchemas().size()); + List columnCategories = new ArrayList<>(tableSchema.getColumnTypes().size()); + List columnSchemas = tableSchema.getColumnSchemas(); + for (int i = 0, columnSchemasSize = columnSchemas.size(); i < columnSchemasSize; i++) { + IMeasurementSchema measurementSchema = columnSchemas.get(i); + measurementSchemas.add( + new MeasurementSchema( + getOriginalColumnName( + tableSchema.getTableName(), measurementSchema.getMeasurementName()), + measurementSchema.getType(), + measurementSchema.getEncodingType(), measurementSchema.getCompressor())); + columnCategories.add(tableSchema.getColumnTypes().get(i)); + } + + return new TableSchema(originalTableName, measurementSchemas, columnCategories); + } + public TableSchema rewriteToFinal(TableSchema tableSchema) { String finalTableName = getFinalTableName(tableSchema.getTableName()); @@ -328,4 +352,21 @@ public static EvolvedSchema merge(EvolvedSchema... schemas) { } return mergedSchema; } + + public Schema rewriteToOriginal(Schema schema) { + return rewriteToOriginal(schema, null); + } + public Schema rewriteToOriginal(Schema schema, Function tableSchemaTransformer) { + Schema copySchema = new Schema(); + for (TableSchema tableSchema : schema.getTableSchemaMap().values()) { + TableSchema originalSchema = rewriteToOriginal(tableSchema); + if (tableSchemaTransformer != null) { + originalSchema = tableSchemaTransformer.apply(originalSchema); + } + copySchema.registerTableSchema(originalSchema); + } + return copySchema; + } + + } From c54b065574e67509880c9ef45392e244c847ee62 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Sun, 4 Jan 2026 09:30:30 +0800 Subject: [PATCH 17/49] temp --- .../constant/InnerSeqCompactionPerformer.java | 8 +++--- .../impl/FastCompactionPerformer.java | 9 +++++-- .../impl/ReadChunkCompactionPerformer.java | 26 +++++++++++++------ .../impl/ReadPointCompactionPerformer.java | 19 +++++++------- .../task/InnerSpaceCompactionTask.java | 1 + .../RepairUnsortedFileCompactionTask.java | 8 +++--- .../utils/MultiTsFileDeviceIterator.java | 16 ++++++++++-- .../writer/AbstractCrossCompactionWriter.java | 10 +++---- .../writer/AbstractInnerCompactionWriter.java | 5 ++-- .../compaction/io/CompactionTsFileWriter.java | 14 +++++++--- .../tsfile/evolution/EvolvedSchema.java | 5 +++- .../tsfile/generator/TsFileNameGenerator.java | 1 + .../compaction/AbstractCompactionTest.java | 3 ++- .../utils/CompactionCheckerUtils.java | 3 ++- 14 files changed, 86 insertions(+), 42 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/constant/InnerSeqCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/constant/InnerSeqCompactionPerformer.java index f64597e25d40f..0404ec65e7591 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/constant/InnerSeqCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/constant/InnerSeqCompactionPerformer.java @@ -25,7 +25,9 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.utils.Pair; public enum InnerSeqCompactionPerformer { READ_CHUNK, @@ -54,12 +56,12 @@ public ISeqCompactionPerformer createInstance() { } } - public ISeqCompactionPerformer createInstance(EncryptParameter encryptParameter) { + public ISeqCompactionPerformer createInstance(EncryptParameter encryptParameter, Pair maxTsFileSetEndVersionAndMinResource) { switch (this) { case READ_CHUNK: - return new ReadChunkCompactionPerformer(encryptParameter); + return new ReadChunkCompactionPerformer(encryptParameter, maxTsFileSetEndVersionAndMinResource); case FAST: - return new FastCompactionPerformer(false, encryptParameter); + return new FastCompactionPerformer(false, encryptParameter, maxTsFileSetEndVersionAndMinResource); default: throw new IllegalCompactionPerformerException( "Illegal compaction performer for seq inner compaction " + this); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java index bc950a2711478..a45d0deb5b2bc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java @@ -102,12 +102,13 @@ public class FastCompactionPerformer private final boolean isCrossCompaction; private EncryptParameter encryptParameter; + private final Pair maxTsFileSetEndVersionAndMinResource; @TestOnly public FastCompactionPerformer( List seqFiles, List unseqFiles, - List targetFiles) { + List targetFiles, Pair maxTsFileSetEndVersionAndMinResource) { this.seqFiles = seqFiles; this.unseqFiles = unseqFiles; this.targetFiles = targetFiles; @@ -121,13 +122,14 @@ public FastCompactionPerformer( new EncryptParameter( TSFileDescriptor.getInstance().getConfig().getEncryptType(), TSFileDescriptor.getInstance().getConfig().getEncryptKey()); + this.maxTsFileSetEndVersionAndMinResource = maxTsFileSetEndVersionAndMinResource; } public FastCompactionPerformer( List seqFiles, List unseqFiles, List targetFiles, - EncryptParameter encryptParameter) { + EncryptParameter encryptParameter, Pair maxTsFileSetEndVersionAndMinResource) { this.seqFiles = seqFiles; this.unseqFiles = unseqFiles; this.targetFiles = targetFiles; @@ -138,6 +140,7 @@ public FastCompactionPerformer( isCrossCompaction = true; } this.encryptParameter = encryptParameter; + this.maxTsFileSetEndVersionAndMinResource = maxTsFileSetEndVersionAndMinResource; } @TestOnly @@ -147,11 +150,13 @@ public FastCompactionPerformer(boolean isCrossCompaction) { new EncryptParameter( TSFileDescriptor.getInstance().getConfig().getEncryptType(), TSFileDescriptor.getInstance().getConfig().getEncryptKey()); + this.maxTsFileSetEndVersionAndMinResource = new Pair<>(Long.MIN_VALUE, null); } public FastCompactionPerformer(boolean isCrossCompaction, EncryptParameter encryptParameter) { this.isCrossCompaction = isCrossCompaction; this.encryptParameter = encryptParameter; + this.maxTsFileSetEndVersionAndMinResource = new Pair<>(Long.MIN_VALUE, null); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java index 2fefbba9ab1de..4280c13b24fbc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java @@ -70,7 +70,8 @@ public class ReadChunkCompactionPerformer implements ISeqCompactionPerformer { * IoTDBDescriptor.getInstance().getConfig().getChunkMetadataSizeProportion()); private Schema schema = null; - private EncryptParameter firstEncryptParameter; + private final EncryptParameter firstEncryptParameter; + protected final Pair maxTsFileSetEndVersionAndMinResource; @TestOnly public ReadChunkCompactionPerformer(List sourceFiles, TsFileResource targetFile) { @@ -80,8 +81,9 @@ public ReadChunkCompactionPerformer(List sourceFiles, TsFileReso public ReadChunkCompactionPerformer( List sourceFiles, TsFileResource targetFile, - EncryptParameter encryptParameter) { - this(sourceFiles, Collections.singletonList(targetFile), encryptParameter); + EncryptParameter encryptParameter, + Pair maxTsFileSetEndVersionAndMinResource) { + this(sourceFiles, Collections.singletonList(targetFile), encryptParameter, maxTsFileSetEndVersionAndMinResource); } @TestOnly @@ -90,27 +92,32 @@ public ReadChunkCompactionPerformer( setSourceFiles(sourceFiles); setTargetFiles(targetFiles); this.firstEncryptParameter = EncryptDBUtils.getDefaultFirstEncryptParam(); + this.maxTsFileSetEndVersionAndMinResource = new Pair<>(Long.MIN_VALUE, null); } public ReadChunkCompactionPerformer( List sourceFiles, List targetFiles, - EncryptParameter encryptParameter) { + EncryptParameter encryptParameter, + Pair maxTsFileSetEndVersionAndMinResource) { setSourceFiles(sourceFiles); setTargetFiles(targetFiles); this.firstEncryptParameter = encryptParameter; + this.maxTsFileSetEndVersionAndMinResource = maxTsFileSetEndVersionAndMinResource; } @TestOnly public ReadChunkCompactionPerformer(List sourceFiles) { setSourceFiles(sourceFiles); this.firstEncryptParameter = EncryptDBUtils.getDefaultFirstEncryptParam(); + this.maxTsFileSetEndVersionAndMinResource = new Pair<>(Long.MIN_VALUE, null); } public ReadChunkCompactionPerformer( - List sourceFiles, EncryptParameter encryptParameter) { + List sourceFiles, EncryptParameter encryptParameter, Pair maxTsFileSetEndVersionAndMinResource) { setSourceFiles(sourceFiles); this.firstEncryptParameter = encryptParameter; + this.maxTsFileSetEndVersionAndMinResource = maxTsFileSetEndVersionAndMinResource; } @TestOnly @@ -119,10 +126,12 @@ public ReadChunkCompactionPerformer() { new EncryptParameter( TSFileDescriptor.getInstance().getConfig().getEncryptType(), TSFileDescriptor.getInstance().getConfig().getEncryptKey()); + this.maxTsFileSetEndVersionAndMinResource = new Pair<>(Long.MIN_VALUE, null); } - public ReadChunkCompactionPerformer(EncryptParameter encryptParameter) { + public ReadChunkCompactionPerformer(EncryptParameter encryptParameter, Pair maxTsFileSetEndVersionAndMinResource) { this.firstEncryptParameter = encryptParameter; + this.maxTsFileSetEndVersionAndMinResource = maxTsFileSetEndVersionAndMinResource; } @Override @@ -137,7 +146,8 @@ public void perform() CompactionTableSchemaCollector.collectSchema( seqFiles, deviceIterator.getReaderMap(), - deviceIterator.getDeprecatedTableSchemaMap()); + deviceIterator.getDeprecatedTableSchemaMap(), + maxTsFileSetEndVersionAndMinResource); while (deviceIterator.hasNextDevice()) { currentWriter = getAvailableCompactionWriter(); Pair deviceInfo = deviceIterator.nextDevice(); @@ -208,7 +218,7 @@ private void useNewWriter() throws IOException { targetResources.get(currentTargetFileIndex), memoryBudgetForFileWriter, CompactionType.INNER_SEQ_COMPACTION, - firstEncryptParameter); + firstEncryptParameter, maxTsFileSetEndVersionAndMinResource.getLeft()); currentWriter.setSchema(CompactionTableSchemaCollector.copySchema(schema)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java index 824bb7e5196db..9b18cee6839b9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java @@ -178,10 +178,10 @@ public void perform() throws Exception { if (isAligned) { compactAlignedSeries( - device, deviceIterator, compactionWriter, fragmentInstanceContext, queryDataSource); + device, deviceIterator, compactionWriter, fragmentInstanceContext, queryDataSource, maxTsFileSetEndVersionAndMinResource); } else { compactNonAlignedSeries( - device, deviceIterator, compactionWriter, fragmentInstanceContext, queryDataSource); + device, deviceIterator, compactionWriter, fragmentInstanceContext, queryDataSource, maxTsFileSetEndVersionAndMinResource); } summary.setTemporaryFileSize(compactionWriter.getWriterSize()); } @@ -217,9 +217,10 @@ private void compactAlignedSeries( MultiTsFileDeviceIterator deviceIterator, AbstractCompactionWriter compactionWriter, FragmentInstanceContext fragmentInstanceContext, - QueryDataSource queryDataSource) + QueryDataSource queryDataSource, + Pair maxTsFileSetEndVersionAndMinResource) throws IOException, MetadataException { - Map schemaMap = deviceIterator.getAllSchemasOfCurrentDevice(); + Map schemaMap = deviceIterator.getAllSchemasOfCurrentDevice(maxTsFileSetEndVersionAndMinResource); IMeasurementSchema timeSchema = schemaMap.remove(TsFileConstant.TIME_COLUMN_ID); List measurementSchemas = new ArrayList<>(schemaMap.values()); if (measurementSchemas.isEmpty()) { @@ -242,13 +243,11 @@ private void compactAlignedSeries( true); if (dataBlockReader.hasNextBatch()) { - // chunkgroup is serialized only when at least one timeseries under this device has data compactionWriter.startChunkGroup(device, true); - measurementSchemas.add(0, timeSchema); compactionWriter.startMeasurement( TsFileConstant.TIME_COLUMN_ID, new AlignedChunkWriterImpl( - measurementSchemas.remove(0), + timeSchema, measurementSchemas, EncryptUtils.getEncryptParameter(getEncryptParameter())), 0); @@ -265,9 +264,11 @@ private void compactNonAlignedSeries( MultiTsFileDeviceIterator deviceIterator, AbstractCompactionWriter compactionWriter, FragmentInstanceContext fragmentInstanceContext, - QueryDataSource queryDataSource) + QueryDataSource queryDataSource, + Pair maxTsFileSetEndVersionAndMinResource) throws IOException, InterruptedException, ExecutionException { - Map schemaMap = deviceIterator.getAllSchemasOfCurrentDevice(); + Map schemaMap = deviceIterator.getAllSchemasOfCurrentDevice( + maxTsFileSetEndVersionAndMinResource); List allMeasurements = new ArrayList<>(schemaMap.keySet()); allMeasurements.sort((String::compareTo)); int subTaskNums = Math.min(allMeasurements.size(), SUB_TASK_NUM); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java index ef4d1d850fdbb..86ea3ea6989e0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java @@ -378,6 +378,7 @@ private void calculateRenamedTargetFiles(boolean needAdjustSourceFilePosition) new File(skippedSourceFile.getParentFile().getPath() + File.separator + newFileName), TsFileResourceStatus.COMPACTING); filesView.renamedTargetFiles.add(renamedTargetFile); + renamedTargetFile.setTsFileManager(tsFileManager); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java index fe1957975cf63..dba8b5e4f2aa1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java @@ -121,9 +121,11 @@ protected void prepare() throws IOException { @Override protected void calculateSourceFilesAndTargetFiles() throws IOException { filesView.sourceFilesInLog = filesView.sourceFilesInCompactionPerformer; + TsFileResource targetResource = new TsFileResource(generateTargetFile(), + TsFileResourceStatus.COMPACTING); + targetResource.setTsFileManager(tsFileManager); filesView.targetFilesInLog = - Collections.singletonList( - new TsFileResource(generateTargetFile(), TsFileResourceStatus.COMPACTING)); + Collections.singletonList(targetResource); filesView.targetFilesInPerformer = filesView.targetFilesInLog; } @@ -137,7 +139,7 @@ private File generateTargetFile() throws IOException { sourceFile.isSeq() ? lastAllocatedFileTimestamp.incrementAndGet() : sourceFileName.getTime(), - sourceFile.isSeq() ? 0 : sourceFileName.getVersion(), + sourceFileName.getVersion(), sourceFileName.getInnerCompactionCnt() + 1, sourceFileName.getCrossCompactionCnt()); // if source file is sequence, the sequence data targetFileDir should be replaced to unsequence diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java index aef15ad3dd42d..a6136d7bd2f0b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java @@ -275,7 +275,8 @@ public long getTimeLowerBoundForCurrentDevice() { * * @throws IOException if io errors occurred */ - public Map getAllSchemasOfCurrentDevice() throws IOException { + public Map getAllSchemasOfCurrentDevice( + Pair maxTsFileSetEndVersionAndMinResource) throws IOException { Map schemaMap = new ConcurrentHashMap<>(); // get schemas from the newest file to the oldest file for (TsFileResource resource : tsFileResourcesSortedByDesc) { @@ -292,12 +293,23 @@ public Map getAllSchemasOfCurrentDevice() throws IOEx deviceIteratorMap.get(resource).getFirstMeasurementNodeOfCurrentDevice(), schemaMap.keySet(), true); + EvolvedSchema evolvedSchema = resource.getMergedEvolvedSchema( + maxTsFileSetEndVersionAndMinResource.left); + if (evolvedSchema != null) { + // the device has been rewritten, should get the original name for rewriting + evolvedSchema.rewriteToFinal(evolvedSchema.getOriginalTableName(currentDevice.left.getTableName()), timeseriesMetadataList); + } + for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) { if (!schemaMap.containsKey(timeseriesMetadata.getMeasurementId()) && !timeseriesMetadata.getChunkMetadataList().isEmpty()) { + MeasurementSchema measurementSchema = reader.getMeasurementSchema( + timeseriesMetadata.getChunkMetadataList()); + // the column may be renamed + measurementSchema.setMeasurementName(timeseriesMetadata.getMeasurementId()); schemaMap.put( timeseriesMetadata.getMeasurementId(), - reader.getMeasurementSchema(timeseriesMetadata.getChunkMetadataList())); + measurementSchema); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java index 6584513f60660..cfa377c3133aa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java @@ -19,17 +19,14 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer; -import java.util.stream.Collectors; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionTableSchema; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFileWriter; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.fileset.TsFileSet; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; import org.apache.iotdb.db.utils.EncryptDBUtils; @@ -83,13 +80,14 @@ public abstract class AbstractCrossCompactionWriter extends AbstractCompactionWr protected AbstractCrossCompactionWriter( List targetResources, List seqFileResources) throws IOException { - this(targetResources, seqFileResources, EncryptDBUtils.getDefaultFirstEncryptParam()); + this(targetResources, seqFileResources, EncryptDBUtils.getDefaultFirstEncryptParam(), Long.MIN_VALUE); } protected AbstractCrossCompactionWriter( List targetResources, List seqFileResources, - EncryptParameter encryptParameter) + EncryptParameter encryptParameter, + long maxTsFileSetEndVersion) throws IOException { currentDeviceEndTime = new long[seqFileResources.size()]; isCurrentDeviceExistedInSourceSeqFiles = new boolean[seqFileResources.size()]; @@ -108,7 +106,7 @@ protected AbstractCrossCompactionWriter( targetResources.get(i), memorySizeForEachWriter, CompactionType.CROSS_COMPACTION, - this.encryptParameter)); + this.encryptParameter, maxTsFileSetEndVersion)); isEmptyFile[i] = true; } this.seqTsFileResources = seqFileResources; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java index d7d7a44c00b3e..28ca734f32dbd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java @@ -117,6 +117,7 @@ private void rollCompactionFileWriter() throws IOException { } private void useNewWriter() throws IOException { + long maxTsFileSetEndVersion = maxTsFileSetEndVersionAndMinResource.left; fileWriter = new CompactionTsFileWriter( targetResources.get(currentFileIndex), @@ -124,9 +125,9 @@ private void useNewWriter() throws IOException { targetResources.get(currentFileIndex).isSeq() ? CompactionType.INNER_SEQ_COMPACTION : CompactionType.INNER_UNSEQ_COMPACTION, - encryptParameter); + encryptParameter, + maxTsFileSetEndVersion); Schema schema = CompactionTableSchemaCollector.copySchema(schemas.get(0)); - long maxTsFileSetEndVersion = maxTsFileSetEndVersionAndMinResource.left; TsFileResource minVersionResource = maxTsFileSetEndVersionAndMinResource.getRight(); fileWriter.getTsFileResource().setTsFileManager(minVersionResource.getTsFileManager()); EvolvedSchema evolvedSchema = fileWriter.getTsFileResource().getMergedEvolvedSchema(maxTsFileSetEndVersion); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java index 7d7bf5cba8249..c8c24fca8e438 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionIoDataType; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; import org.apache.iotdb.db.utils.EncryptDBUtils; import org.apache.tsfile.encrypt.EncryptParameter; @@ -55,18 +56,22 @@ public class CompactionTsFileWriter extends TsFileIOWriter { private volatile boolean isWritingAligned = false; private boolean isEmptyTargetFile = true; private IDeviceID currentDeviceId; - private TsFileResource tsFileResource; - private EncryptParameter firstEncryptParameter; + private final TsFileResource tsFileResource; + private final EvolvedSchema evolvedSchema; + + private final EncryptParameter firstEncryptParameter; @TestOnly public CompactionTsFileWriter(File file, long maxMetadataSize, CompactionType type) throws IOException { - this(new TsFileResource(file), maxMetadataSize, type, EncryptDBUtils.getDefaultFirstEncryptParam()); + this(new TsFileResource(file), maxMetadataSize, type, EncryptDBUtils.getDefaultFirstEncryptParam(), + Long.MIN_VALUE); } public CompactionTsFileWriter( - TsFileResource tsFile, long maxMetadataSize, CompactionType type, EncryptParameter encryptParameter) + TsFileResource tsFile, long maxMetadataSize, CompactionType type, EncryptParameter encryptParameter, + long maxTsFileSetEndVersion) throws IOException { super(tsFile.getTsFile(), maxMetadataSize, encryptParameter); this.tsFileResource = tsFile; @@ -75,6 +80,7 @@ public CompactionTsFileWriter( super.out = new CompactionTsFileOutput( super.out, CompactionTaskManager.getInstance().getMergeWriteRateLimiter()); + evolvedSchema = tsFileResource.getMergedEvolvedSchema(maxTsFileSetEndVersion); } public EncryptParameter getEncryptParameter() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java index e447a1fc630c7..a88e76d43278b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.TableSchema; @@ -231,8 +232,10 @@ public void rewriteToFinal( String originalTableName, List timeseriesMetadataList) { timeseriesMetadataList.forEach( timeseriesMetadata -> { + String finalColumnName = getFinalColumnName(originalTableName, + timeseriesMetadata.getMeasurementId()); timeseriesMetadata.setMeasurementId( - getFinalColumnName(originalTableName, timeseriesMetadata.getMeasurementId())); + finalColumnName); }); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java index 16be82188e9ca..698b4b95be681 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java @@ -363,6 +363,7 @@ public static List getNewInnerCompactionTargetFileResources( TsFileResourceStatus.COMPACTING); targetResource.setSeq(sequence); targetResources.add(targetResource); + targetResource.setTsFileManager(resource.getTsFileManager()); } return targetResources; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java index a5503bb9e645c..31ed98eb0e109 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java @@ -850,7 +850,8 @@ protected List getPaths(List resources) Pair iDeviceIDBooleanPair = deviceIterator.nextDevice(); IDeviceID deviceID = iDeviceIDBooleanPair.getLeft(); boolean isAlign = iDeviceIDBooleanPair.getRight(); - Map schemaMap = deviceIterator.getAllSchemasOfCurrentDevice(); + Map schemaMap = deviceIterator.getAllSchemasOfCurrentDevice( + new Pair<>(Long.MIN_VALUE, null)); IMeasurementSchema timeSchema = schemaMap.remove(TsFileConstant.TIME_COLUMN_ID); List measurementSchemas = new ArrayList<>(schemaMap.values()); if (measurementSchemas.isEmpty()) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java index 6c3e692212a85..d3851e4e48ef2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java @@ -538,7 +538,8 @@ public static List getAllPathsOfResources(List resour Pair iDeviceIDBooleanPair = deviceIterator.nextDevice(); IDeviceID deviceID = iDeviceIDBooleanPair.getLeft(); boolean isAlign = iDeviceIDBooleanPair.getRight(); - Map schemaMap = deviceIterator.getAllSchemasOfCurrentDevice(); + Map schemaMap = deviceIterator.getAllSchemasOfCurrentDevice( + new Pair<>(Long.MIN_VALUE, null)); IMeasurementSchema timeSchema = schemaMap.remove(TsFileConstant.TIME_COLUMN_ID); List measurementSchemas = new ArrayList<>(schemaMap.values()); if (measurementSchemas.isEmpty()) { From 12bcb3a06cb23ab058c787e634a9aa6623e30026 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 7 Jan 2026 19:11:47 +0800 Subject: [PATCH 18/49] support ReadPointCompactionPerformer with sevo --- .../it/db/it/IoTDBLoadTsFileIT.java | 8 +- .../org/apache/iotdb/session/Session.java | 7 +- .../source/AlignedSeriesScanUtil.java | 24 +- .../operator/source/FileLoaderUtils.java | 38 ++- .../operator/source/SeriesScanUtil.java | 39 ++- .../storageengine/dataregion/DataRegion.java | 36 +- .../constant/InnerSeqCompactionPerformer.java | 8 +- .../impl/FastCompactionPerformer.java | 38 ++- .../impl/ReadChunkCompactionPerformer.java | 26 +- .../impl/ReadPointCompactionPerformer.java | 59 ++-- .../RepairUnsortedFileCompactionTask.java | 7 +- .../subtask/ReadPointPerformerSubTask.java | 7 +- .../execute/utils/CompactionTableSchema.java | 5 +- .../utils/CompactionTableSchemaCollector.java | 7 +- .../utils/MultiTsFileDeviceIterator.java | 94 ++++-- .../utils/ReorderedTsFileDeviceIterator.java | 31 +- .../TransformedTsFileDeviceIterator.java | 14 +- .../utils/reader/SeriesDataBlockReader.java | 9 +- .../writer/AbstractCompactionWriter.java | 5 +- .../writer/AbstractCrossCompactionWriter.java | 12 +- .../writer/AbstractInnerCompactionWriter.java | 8 +- .../writer/FastCrossCompactionWriter.java | 11 +- .../ReadPointCrossCompactionWriter.java | 11 +- .../compaction/io/CompactionTsFileWriter.java | 33 +- .../repair/RepairDataFileScanUtil.java | 2 +- .../dataregion/read/QueryDataSource.java | 16 +- .../dataregion/tsfile/TsFileManager.java | 17 +- .../dataregion/tsfile/TsFileResource.java | 41 ++- .../tsfile/evolution/EvolvedSchema.java | 41 ++- .../dataregion/tsfile/fileset/TsFileSet.java | 5 +- .../compaction/AbstractCompactionTest.java | 4 +- .../ReadPointCompactionPerformerTest.java | 307 ++++++++++++++++++ .../utils/CompactionCheckerUtils.java | 4 +- 33 files changed, 757 insertions(+), 217 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java index b9241b66182ef..5a754c6afe808 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java @@ -19,7 +19,6 @@ package org.apache.iotdb.relational.it.db.it; -import java.sql.SQLException; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.ColumnRename; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolutionFile; @@ -50,6 +49,7 @@ import java.nio.file.Files; import java.sql.Connection; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Collections; @@ -328,7 +328,8 @@ public void testLoadWithSevoFile() throws Exception { // cannot query using INT322INT32 try (final ResultSet resultSet = - statement.executeQuery(String.format("select count(%s) from %s", "INT322INT32", SchemaConfig.TABLE_1))) { + statement.executeQuery( + String.format("select count(%s) from %s", "INT322INT32", SchemaConfig.TABLE_1))) { fail(); } catch (SQLException e) { assertEquals("616: Column 'int322int32' cannot be resolved", e.getMessage()); @@ -336,7 +337,8 @@ public void testLoadWithSevoFile() throws Exception { // can query with INT322INT32_NEW try (final ResultSet resultSet = - statement.executeQuery(String.format("select count(%s) from %s", "INT322INT32_NEW", SchemaConfig.TABLE_1))) { + statement.executeQuery( + String.format("select count(%s) from %s", "INT322INT32_NEW", SchemaConfig.TABLE_1))) { if (resultSet.next()) { Assert.assertEquals(lineCount, resultSet.getLong(1)); } else { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java index 8844f36c247d2..ea7c7c85cd7ef 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java @@ -3074,10 +3074,9 @@ private TSInsertTabletReq genTSInsertTabletReq(Tablet tablet, boolean sorted, bo this.columnEncodersMap .getOrDefault( measurementSchema.getType(), - TSEncoding.valueOf( - TSFileDescriptor.getInstance() - .getConfig() - .getValueEncoder(measurementSchema.getType()))) + TSFileDescriptor.getInstance() + .getConfig() + .getValueEncoder(measurementSchema.getType())) .serialize()); } } else { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java index f7ddaee472ea9..c9d7dfeabbc7b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java @@ -62,6 +62,15 @@ public AlignedSeriesScanUtil( this(seriesPath, scanOrder, scanOptions, context, false, null); } + public AlignedSeriesScanUtil( + AlignedFullPath seriesPath, + Ordering scanOrder, + SeriesScanOptions scanOptions, + FragmentInstanceContext context, + long maxTsFileSetEndVersion) { + this(seriesPath, scanOrder, scanOptions, context, false, null, maxTsFileSetEndVersion); + } + public AlignedSeriesScanUtil( AlignedFullPath seriesPath, Ordering scanOrder, @@ -69,7 +78,18 @@ public AlignedSeriesScanUtil( FragmentInstanceContext context, boolean queryAllSensors, List givenDataTypes) { - super(seriesPath, scanOrder, scanOptions, context); + this(seriesPath, scanOrder, scanOptions, context, queryAllSensors, givenDataTypes, Long.MAX_VALUE); + } + + public AlignedSeriesScanUtil( + AlignedFullPath seriesPath, + Ordering scanOrder, + SeriesScanOptions scanOptions, + FragmentInstanceContext context, + boolean queryAllSensors, + List givenDataTypes, + long maxTsFileSetEndVersion) { + super(seriesPath, scanOrder, scanOptions, context, maxTsFileSetEndVersion); isAligned = true; this.dataTypes = givenDataTypes != null @@ -100,7 +120,7 @@ protected AbstractAlignedTimeSeriesMetadata loadTimeSeriesMetadata( context, scanOptions.getGlobalTimeFilter(), isSeq, - ignoreAllNullRows); + ignoreAllNullRows, maxTsFileSetEndVersion); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index da8fd358e979f..ffaeb60bab060 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; +import java.util.stream.Collectors; import org.apache.iotdb.commons.path.AlignedFullPath; import org.apache.iotdb.commons.path.NonAlignedFullPath; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; @@ -55,6 +56,7 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import org.apache.tsfile.write.schema.IMeasurementSchema; import static com.google.common.base.Preconditions.checkArgument; @@ -83,7 +85,8 @@ public static TimeseriesMetadata loadTimeSeriesMetadata( FragmentInstanceContext context, Filter globalTimeFilter, Set allSensors, - boolean isSeq) + boolean isSeq, + long maxTsFileSetEndVersion) throws IOException { long t1 = System.nanoTime(); boolean loadFromMem = false; @@ -94,9 +97,13 @@ public static TimeseriesMetadata loadTimeSeriesMetadata( if (resource.isClosed()) { // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex // we should not ignore the non-exist of device in TsFileMetadata - EvolvedSchema evolvedSchema = resource.getMergedEvolvedSchema(); + EvolvedSchema evolvedSchema = resource.getMergedEvolvedSchema(maxTsFileSetEndVersion); IDeviceID deviceId = seriesPath.getDeviceId(); String measurement = seriesPath.getMeasurement(); + if (evolvedSchema != null) { + measurement = evolvedSchema.getOriginalColumnName(deviceId.getTableName(), measurement); + deviceId = evolvedSchema.rewriteToOriginal(deviceId); + } timeSeriesMetadata = TimeSeriesMetadataCache.getInstance() @@ -186,7 +193,8 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata( FragmentInstanceContext context, Filter globalTimeFilter, boolean isSeq, - boolean ignoreAllNullRows) + boolean ignoreAllNullRows, + long maxTsFileSetEndVersion) throws IOException { final long t1 = System.nanoTime(); boolean loadFromMem = false; @@ -196,7 +204,7 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata( if (resource.isClosed()) { alignedTimeSeriesMetadata = loadAlignedTimeSeriesMetadataFromDisk( - resource, alignedPath, context, globalTimeFilter, ignoreAllNullRows); + resource, alignedPath, context, globalTimeFilter, ignoreAllNullRows, maxTsFileSetEndVersion); } else { // if the tsfile is unclosed, we just get it directly from TsFileResource loadFromMem = true; alignedTimeSeriesMetadata = @@ -262,7 +270,8 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr AlignedFullPath alignedPath, FragmentInstanceContext context, Filter globalTimeFilter, - boolean ignoreAllNullRows) + boolean ignoreAllNullRows, + long maxTsFileSetEndVersion) throws IOException { AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null; // load all the TimeseriesMetadata of vector, the first one is for time column and the @@ -275,6 +284,15 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr String filePath = resource.getTsFilePath(); IDeviceID deviceId = alignedPath.getDeviceId(); + EvolvedSchema evolvedSchema = resource.getMergedEvolvedSchema(maxTsFileSetEndVersion); + if (evolvedSchema != null) { + IDeviceID finalDeviceId = deviceId; + valueMeasurementList = valueMeasurementList.stream().map(m -> evolvedSchema.getOriginalColumnName(finalDeviceId.getTableName(), m)).collect( + Collectors.toList()); + allSensors = allSensors.stream().map(m -> evolvedSchema.getOriginalColumnName(finalDeviceId.getTableName(), m)).collect(Collectors.toSet()); + deviceId = evolvedSchema.rewriteToOriginal(deviceId); + } + // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex // we should not ignore the non-exist of device in TsFileMetadata TimeseriesMetadata timeColumn = @@ -296,7 +314,7 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr resource, timeColumn, Collections.emptyList(), - alignedPath, + deviceId, context, globalTimeFilter, false); @@ -325,7 +343,7 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr resource, timeColumn, valueTimeSeriesMetadataList, - alignedPath, + deviceId, context, globalTimeFilter, ignoreAllNullRows); @@ -339,7 +357,7 @@ private static AbstractAlignedTimeSeriesMetadata setModifications( TsFileResource resource, TimeseriesMetadata timeColumnMetadata, List valueColumnMetadataList, - AlignedFullPath alignedPath, + IDeviceID deviceID, QueryContext context, Filter globalTimeFilter, boolean ignoreAllNullRows) { @@ -348,7 +366,7 @@ private static AbstractAlignedTimeSeriesMetadata setModifications( // deal with time column List timeModifications = context.getPathModifications( - resource, alignedPath.getDeviceId(), timeColumnMetadata.getMeasurementId()); + resource, deviceID, timeColumnMetadata.getMeasurementId()); // all rows are deleted, just return null to skip device data in this file if (ModificationUtils.isAllDeletedByMods( timeModifications, @@ -371,7 +389,7 @@ private static AbstractAlignedTimeSeriesMetadata setModifications( if (valueColumnMetadata != null) { List modifications = context.getPathModifications( - resource, alignedPath.getDeviceId(), valueColumnMetadata.getMeasurementId()); + resource, deviceID, valueColumnMetadata.getMeasurementId()); valueColumnMetadata.setModified(!modifications.isEmpty()); valueColumnsModifications.add(modifications); modified = (modified || !modifications.isEmpty()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java index 233d24bf097cc..b232b1377d6b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java @@ -130,6 +130,9 @@ public class SeriesScanUtil implements Accountable { protected final int MAX_NUMBER_OF_POINTS_IN_PAGE = TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage(); + // to restrict the scope of sevo files for compaction + protected final long maxTsFileSetEndVersion; + private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(SeriesScanUtil.class) + RamUsageEstimator.shallowSizeOfInstance(IDeviceID.class) @@ -145,6 +148,15 @@ public SeriesScanUtil( Ordering scanOrder, SeriesScanOptions scanOptions, FragmentInstanceContext context) { + this(seriesPath, scanOrder, scanOptions, context, Long.MAX_VALUE); + } + + public SeriesScanUtil( + IFullPath seriesPath, + Ordering scanOrder, + SeriesScanOptions scanOptions, + FragmentInstanceContext context, + long maxTsFileSetEndVersion) { this.seriesPath = seriesPath; this.deviceID = seriesPath.getDeviceId(); this.dataType = seriesPath.getSeriesType(); @@ -182,6 +194,8 @@ public SeriesScanUtil( new PriorityQueue<>( orderUtils.comparingLong( versionPageReader -> orderUtils.getOrderTime(versionPageReader.getStatistics()))); + + this.maxTsFileSetEndVersion = maxTsFileSetEndVersion; } /** @@ -190,7 +204,7 @@ public SeriesScanUtil( * @param dataSource the query data source */ public void initQueryDataSource(QueryDataSource dataSource) { - dataSource.fillOrderIndexes(deviceID, orderUtils.getAscending()); + dataSource.fillOrderIndexes(deviceID, orderUtils.getAscending(), maxTsFileSetEndVersion); this.dataSource = dataSource; // updated filter concerning TTL @@ -1339,7 +1353,8 @@ protected ITimeSeriesMetadata loadTimeSeriesMetadata(TsFileResource resource, bo context, scanOptions.getGlobalTimeFilter(), scanOptions.getAllSensors(), - isSeq); + isSeq, + maxTsFileSetEndVersion); } public List getTsDataTypeList() { @@ -1753,26 +1768,26 @@ public Ordering getScanOrder() { @Override public boolean hasNextSeqResource() { - while (dataSource.hasNextSeqResource(curSeqFileIndex, false, deviceID)) { + while (dataSource.hasNextSeqResource(curSeqFileIndex, false, deviceID, maxTsFileSetEndVersion)) { if (dataSource.isSeqSatisfied( - deviceID, curSeqFileIndex, scanOptions.getGlobalTimeFilter(), false)) { + deviceID, curSeqFileIndex, scanOptions.getGlobalTimeFilter(), false, maxTsFileSetEndVersion)) { break; } curSeqFileIndex--; } - return dataSource.hasNextSeqResource(curSeqFileIndex, false, deviceID); + return dataSource.hasNextSeqResource(curSeqFileIndex, false, deviceID, maxTsFileSetEndVersion); } @Override public boolean hasNextUnseqResource() { - while (dataSource.hasNextUnseqResource(curUnseqFileIndex, false, deviceID)) { + while (dataSource.hasNextUnseqResource(curUnseqFileIndex, false, deviceID, maxTsFileSetEndVersion)) { if (dataSource.isUnSeqSatisfied( deviceID, curUnseqFileIndex, scanOptions.getGlobalTimeFilter(), false)) { break; } curUnseqFileIndex++; } - return dataSource.hasNextUnseqResource(curUnseqFileIndex, false, deviceID); + return dataSource.hasNextUnseqResource(curUnseqFileIndex, false, deviceID, maxTsFileSetEndVersion); } @Override @@ -1882,26 +1897,26 @@ public Ordering getScanOrder() { @Override public boolean hasNextSeqResource() { - while (dataSource.hasNextSeqResource(curSeqFileIndex, true, deviceID)) { + while (dataSource.hasNextSeqResource(curSeqFileIndex, true, deviceID, maxTsFileSetEndVersion)) { if (dataSource.isSeqSatisfied( - deviceID, curSeqFileIndex, scanOptions.getGlobalTimeFilter(), false)) { + deviceID, curSeqFileIndex, scanOptions.getGlobalTimeFilter(), false, maxTsFileSetEndVersion)) { break; } curSeqFileIndex++; } - return dataSource.hasNextSeqResource(curSeqFileIndex, true, deviceID); + return dataSource.hasNextSeqResource(curSeqFileIndex, true, deviceID, maxTsFileSetEndVersion); } @Override public boolean hasNextUnseqResource() { - while (dataSource.hasNextUnseqResource(curUnseqFileIndex, true, deviceID)) { + while (dataSource.hasNextUnseqResource(curUnseqFileIndex, true, deviceID, maxTsFileSetEndVersion)) { if (dataSource.isUnSeqSatisfied( deviceID, curUnseqFileIndex, scanOptions.getGlobalTimeFilter(), false)) { break; } curUnseqFileIndex++; } - return dataSource.hasNextUnseqResource(curUnseqFileIndex, true, deviceID); + return dataSource.hasNextUnseqResource(curUnseqFileIndex, true, deviceID, maxTsFileSetEndVersion); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index e64350500a333..201007dcdb342 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -685,7 +685,8 @@ private void recover() throws DataRegionException { partitionFiles.getValue(), fileTimeIndexMap, false, - recoveredPartitionTsFileSetMap, partitionMinimalVersion); + recoveredPartitionTsFileSetMap, + partitionMinimalVersion); if (asyncRecoverTask != null) { asyncTsFileResourceRecoverTaskList.add(asyncRecoverTask); } @@ -705,7 +706,8 @@ private void recover() throws DataRegionException { for (Entry> entry : recoveredPartitionTsFileSetMap.entrySet()) { long partitionId = entry.getKey(); // if no file in the partition, all filesets should be cleared - long minimumFileVersion = partitionMinimalVersion.getOrDefault(partitionId, Long.MAX_VALUE); + long minimumFileVersion = + partitionMinimalVersion.getOrDefault(partitionId, Long.MAX_VALUE); for (TsFileSet tsFileSet : entry.getValue()) { if (tsFileSet.getEndVersion() < minimumFileVersion) { tsFileSet.remove(); @@ -1056,6 +1058,7 @@ private List recoverTsFileSets( tsFileSet = new TsFileSet( Long.parseLong(fileSet.getName()), fileSetDir.getAbsolutePath(), true); + tsFileManager.addTsFileSet(tsFileSet, partitionId); } catch (NumberFormatException e) { continue; } @@ -1077,30 +1080,23 @@ private Callable recoverFilesInPartition( List resourceList, Map fileTimeIndexMap, boolean isSeq, - Map> partitionTsFileSetMap, Map partitionMinimalVersion) { + Map> partitionTsFileSetMap, + Map partitionMinimalVersion) { List resourceListForAsyncRecover = new ArrayList<>(); List resourceListForSyncRecover = new ArrayList<>(); Callable asyncRecoverTask = null; - List tsFileSets = recoverTsFileSets(partitionId, partitionTsFileSetMap); + recoverTsFileSets(partitionId, partitionTsFileSetMap); for (TsFileResource tsFileResource : resourceList) { long fileVersion = tsFileResource.getTsFileID().fileVersion; - partitionMinimalVersion.compute(partitionId, (pid, oldVersion) -> { - if (oldVersion == null) { - return fileVersion; - } - return Math.min(oldVersion, fileVersion); - }); - - int i = Collections.binarySearch(tsFileSets, TsFileSet.comparatorKey(fileVersion)); - if (i < 0) { - // if the binary search does not find an exact match, -i indicates the closest one - i = -i; - } - if (i < tsFileSets.size()) { - List containedSets = tsFileSets.subList(i, tsFileSets.size()); - containedSets.forEach(tsFileResource::addFileSet); - } + partitionMinimalVersion.compute( + partitionId, + (pid, oldVersion) -> { + if (oldVersion == null) { + return fileVersion; + } + return Math.min(oldVersion, fileVersion); + }); tsFileManager.add(tsFileResource, isSeq); if (fileTimeIndexMap.containsKey(tsFileResource.getTsFileID()) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/constant/InnerSeqCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/constant/InnerSeqCompactionPerformer.java index 0404ec65e7591..f64597e25d40f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/constant/InnerSeqCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/constant/InnerSeqCompactionPerformer.java @@ -25,9 +25,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.tsfile.encrypt.EncryptParameter; -import org.apache.tsfile.utils.Pair; public enum InnerSeqCompactionPerformer { READ_CHUNK, @@ -56,12 +54,12 @@ public ISeqCompactionPerformer createInstance() { } } - public ISeqCompactionPerformer createInstance(EncryptParameter encryptParameter, Pair maxTsFileSetEndVersionAndMinResource) { + public ISeqCompactionPerformer createInstance(EncryptParameter encryptParameter) { switch (this) { case READ_CHUNK: - return new ReadChunkCompactionPerformer(encryptParameter, maxTsFileSetEndVersionAndMinResource); + return new ReadChunkCompactionPerformer(encryptParameter); case FAST: - return new FastCompactionPerformer(false, encryptParameter, maxTsFileSetEndVersionAndMinResource); + return new FastCompactionPerformer(false, encryptParameter); default: throw new IllegalCompactionPerformerException( "Illegal compaction performer for seq inner compaction " + this); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java index a45d0deb5b2bc..568db7081bcf2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java @@ -19,8 +19,6 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PatternTreeMap; @@ -74,6 +72,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class FastCompactionPerformer implements ICrossCompactionPerformer, ISeqCompactionPerformer, IUnseqCompactionPerformer { @@ -108,7 +108,7 @@ public class FastCompactionPerformer public FastCompactionPerformer( List seqFiles, List unseqFiles, - List targetFiles, Pair maxTsFileSetEndVersionAndMinResource) { + List targetFiles) { this.seqFiles = seqFiles; this.unseqFiles = unseqFiles; this.targetFiles = targetFiles; @@ -122,14 +122,14 @@ public FastCompactionPerformer( new EncryptParameter( TSFileDescriptor.getInstance().getConfig().getEncryptType(), TSFileDescriptor.getInstance().getConfig().getEncryptKey()); - this.maxTsFileSetEndVersionAndMinResource = maxTsFileSetEndVersionAndMinResource; + this.maxTsFileSetEndVersionAndMinResource = new Pair<>(Long.MIN_VALUE, null); } public FastCompactionPerformer( List seqFiles, List unseqFiles, List targetFiles, - EncryptParameter encryptParameter, Pair maxTsFileSetEndVersionAndMinResource) { + EncryptParameter encryptParameter) { this.seqFiles = seqFiles; this.unseqFiles = unseqFiles; this.targetFiles = targetFiles; @@ -140,7 +140,9 @@ public FastCompactionPerformer( isCrossCompaction = true; } this.encryptParameter = encryptParameter; - this.maxTsFileSetEndVersionAndMinResource = maxTsFileSetEndVersionAndMinResource; + this.maxTsFileSetEndVersionAndMinResource = + TsFileResource.getMaxTsFileSetEndVersionAndMinResource( + Stream.concat(seqFiles.stream(), unseqFiles.stream()).collect(Collectors.toList())); } @TestOnly @@ -162,22 +164,30 @@ public FastCompactionPerformer(boolean isCrossCompaction, EncryptParameter encry @Override public void perform() throws Exception { this.subTaskSummary.setTemporalFileNum(targetFiles.size()); + List allSourceFiles = + Stream.concat(seqFiles.stream(), unseqFiles.stream()) + .sorted(TsFileResource::compareFileName) + .collect(Collectors.toList()); + Pair maxTsFileSetEndVersionAndMinResource = + TsFileResource.getMaxTsFileSetEndVersionAndMinResource(allSourceFiles); + try (MultiTsFileDeviceIterator deviceIterator = new MultiTsFileDeviceIterator(seqFiles, unseqFiles, readerCacheMap); AbstractCompactionWriter compactionWriter = isCrossCompaction ? new FastCrossCompactionWriter( - targetFiles, seqFiles, readerCacheMap, encryptParameter) + targetFiles, + seqFiles, + readerCacheMap, + encryptParameter, + maxTsFileSetEndVersionAndMinResource.left) : new FastInnerCompactionWriter(targetFiles, encryptParameter)) { - - List allSourceFiles = Stream.concat(seqFiles.stream(), unseqFiles.stream()) - .sorted(TsFileResource::compareFileName) - .collect(Collectors.toList()); - Pair maxTsFileSetEndVersionAndMinResource = TsFileResource.getMaxTsFileSetEndVersionAndMinResource( - allSourceFiles); List schemas = CompactionTableSchemaCollector.collectSchema( - seqFiles, unseqFiles, readerCacheMap, deviceIterator.getDeprecatedTableSchemaMap(), + seqFiles, + unseqFiles, + readerCacheMap, + deviceIterator.getDeprecatedTableSchemaMap(), maxTsFileSetEndVersionAndMinResource); compactionWriter.setSchemaForAllTargetFile(schemas, maxTsFileSetEndVersionAndMinResource); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java index 4280c13b24fbc..904b17567d952 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java @@ -71,7 +71,7 @@ public class ReadChunkCompactionPerformer implements ISeqCompactionPerformer { private Schema schema = null; private final EncryptParameter firstEncryptParameter; - protected final Pair maxTsFileSetEndVersionAndMinResource; + protected Pair maxTsFileSetEndVersionAndMinResource; @TestOnly public ReadChunkCompactionPerformer(List sourceFiles, TsFileResource targetFile) { @@ -81,9 +81,8 @@ public ReadChunkCompactionPerformer(List sourceFiles, TsFileReso public ReadChunkCompactionPerformer( List sourceFiles, TsFileResource targetFile, - EncryptParameter encryptParameter, - Pair maxTsFileSetEndVersionAndMinResource) { - this(sourceFiles, Collections.singletonList(targetFile), encryptParameter, maxTsFileSetEndVersionAndMinResource); + EncryptParameter encryptParameter) { + this(sourceFiles, Collections.singletonList(targetFile), encryptParameter); } @TestOnly @@ -98,12 +97,12 @@ public ReadChunkCompactionPerformer( public ReadChunkCompactionPerformer( List sourceFiles, List targetFiles, - EncryptParameter encryptParameter, - Pair maxTsFileSetEndVersionAndMinResource) { + EncryptParameter encryptParameter) { setSourceFiles(sourceFiles); setTargetFiles(targetFiles); this.firstEncryptParameter = encryptParameter; - this.maxTsFileSetEndVersionAndMinResource = maxTsFileSetEndVersionAndMinResource; + this.maxTsFileSetEndVersionAndMinResource = + TsFileResource.getMaxTsFileSetEndVersionAndMinResource(sourceFiles); } @TestOnly @@ -114,10 +113,11 @@ public ReadChunkCompactionPerformer(List sourceFiles) { } public ReadChunkCompactionPerformer( - List sourceFiles, EncryptParameter encryptParameter, Pair maxTsFileSetEndVersionAndMinResource) { + List sourceFiles, EncryptParameter encryptParameter) { setSourceFiles(sourceFiles); this.firstEncryptParameter = encryptParameter; - this.maxTsFileSetEndVersionAndMinResource = maxTsFileSetEndVersionAndMinResource; + this.maxTsFileSetEndVersionAndMinResource = + TsFileResource.getMaxTsFileSetEndVersionAndMinResource(sourceFiles); } @TestOnly @@ -129,9 +129,8 @@ public ReadChunkCompactionPerformer() { this.maxTsFileSetEndVersionAndMinResource = new Pair<>(Long.MIN_VALUE, null); } - public ReadChunkCompactionPerformer(EncryptParameter encryptParameter, Pair maxTsFileSetEndVersionAndMinResource) { + public ReadChunkCompactionPerformer(EncryptParameter encryptParameter) { this.firstEncryptParameter = encryptParameter; - this.maxTsFileSetEndVersionAndMinResource = maxTsFileSetEndVersionAndMinResource; } @Override @@ -218,7 +217,8 @@ private void useNewWriter() throws IOException { targetResources.get(currentTargetFileIndex), memoryBudgetForFileWriter, CompactionType.INNER_SEQ_COMPACTION, - firstEncryptParameter, maxTsFileSetEndVersionAndMinResource.getLeft()); + firstEncryptParameter, + maxTsFileSetEndVersionAndMinResource.getLeft()); currentWriter.setSchema(CompactionTableSchemaCollector.copySchema(schema)); } @@ -357,6 +357,8 @@ private void compactNotAlignedSeries( @Override public void setSourceFiles(List seqFiles) { this.seqFiles = seqFiles; + this.maxTsFileSetEndVersionAndMinResource = + TsFileResource.getMaxTsFileSetEndVersionAndMinResource(seqFiles); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java index 9b18cee6839b9..b7c143808c66d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java @@ -18,9 +18,7 @@ */ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl; -import java.util.stream.Stream; import org.apache.iotdb.commons.conf.IoTDBConstant; -import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.path.AlignedFullPath; import org.apache.iotdb.commons.path.IFullPath; import org.apache.iotdb.commons.path.NonAlignedFullPath; @@ -72,6 +70,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.stream.Collectors; +import java.util.stream.Stream; public class ReadPointCompactionPerformer implements ICrossCompactionPerformer, IUnseqCompactionPerformer { @@ -154,11 +153,12 @@ public void perform() throws Exception { // Do not close device iterator, because tsfile reader is managed by FileReaderManager. MultiTsFileDeviceIterator deviceIterator = new MultiTsFileDeviceIterator(seqFiles, unseqFiles); - List allSourceFiles = Stream.concat(seqFiles.stream(), unseqFiles.stream()) - .sorted(TsFileResource::compareFileName) - .collect(Collectors.toList()); - Pair maxTsFileSetEndVersionAndMinResource = TsFileResource.getMaxTsFileSetEndVersionAndMinResource( - allSourceFiles); + List allSourceFiles = + Stream.concat(seqFiles.stream(), unseqFiles.stream()) + .sorted(TsFileResource::compareFileName) + .collect(Collectors.toList()); + Pair maxTsFileSetEndVersionAndMinResource = + TsFileResource.getMaxTsFileSetEndVersionAndMinResource(allSourceFiles); List schemas = CompactionTableSchemaCollector.collectSchema( @@ -174,14 +174,24 @@ public void perform() throws Exception { Pair deviceInfo = deviceIterator.nextDevice(); IDeviceID device = deviceInfo.left; boolean isAligned = deviceInfo.right; - queryDataSource.fillOrderIndexes(device, true); + queryDataSource.fillOrderIndexes(device, true, maxTsFileSetEndVersionAndMinResource.left); if (isAligned) { compactAlignedSeries( - device, deviceIterator, compactionWriter, fragmentInstanceContext, queryDataSource, maxTsFileSetEndVersionAndMinResource); + device, + deviceIterator, + compactionWriter, + fragmentInstanceContext, + queryDataSource, + maxTsFileSetEndVersionAndMinResource); } else { compactNonAlignedSeries( - device, deviceIterator, compactionWriter, fragmentInstanceContext, queryDataSource, maxTsFileSetEndVersionAndMinResource); + device, + deviceIterator, + compactionWriter, + fragmentInstanceContext, + queryDataSource, + maxTsFileSetEndVersionAndMinResource); } summary.setTemporaryFileSize(compactionWriter.getWriterSize()); } @@ -219,8 +229,9 @@ private void compactAlignedSeries( FragmentInstanceContext fragmentInstanceContext, QueryDataSource queryDataSource, Pair maxTsFileSetEndVersionAndMinResource) - throws IOException, MetadataException { - Map schemaMap = deviceIterator.getAllSchemasOfCurrentDevice(maxTsFileSetEndVersionAndMinResource); + throws IOException { + Map schemaMap = + deviceIterator.getAllSchemasOfCurrentDevice(maxTsFileSetEndVersionAndMinResource); IMeasurementSchema timeSchema = schemaMap.remove(TsFileConstant.TIME_COLUMN_ID); List measurementSchemas = new ArrayList<>(schemaMap.values()); if (measurementSchemas.isEmpty()) { @@ -240,7 +251,8 @@ private void compactAlignedSeries( new ArrayList<>(schemaMap.keySet()), fragmentInstanceContext, queryDataSource, - true); + true, + maxTsFileSetEndVersionAndMinResource.left); if (dataBlockReader.hasNextBatch()) { compactionWriter.startChunkGroup(device, true); @@ -267,8 +279,8 @@ private void compactNonAlignedSeries( QueryDataSource queryDataSource, Pair maxTsFileSetEndVersionAndMinResource) throws IOException, InterruptedException, ExecutionException { - Map schemaMap = deviceIterator.getAllSchemasOfCurrentDevice( - maxTsFileSetEndVersionAndMinResource); + Map schemaMap = + deviceIterator.getAllSchemasOfCurrentDevice(maxTsFileSetEndVersionAndMinResource); List allMeasurements = new ArrayList<>(schemaMap.keySet()); allMeasurements.sort((String::compareTo)); int subTaskNums = Math.min(allMeasurements.size(), SUB_TASK_NUM); @@ -297,7 +309,7 @@ private void compactNonAlignedSeries( new QueryDataSource(queryDataSource), compactionWriter, schemaMap, - i))); + i, maxTsFileSetEndVersionAndMinResource.left))); } for (Future future : futures) { future.get(); @@ -321,7 +333,8 @@ public static IDataBlockReader constructReader( List allSensors, FragmentInstanceContext fragmentInstanceContext, QueryDataSource queryDataSource, - boolean isAlign) { + boolean isAlign, + long maxTsFileSetEndVersion) { IFullPath seriesPath; if (isAlign) { seriesPath = new AlignedFullPath(deviceId, measurementIds, measurementSchemas); @@ -330,7 +343,7 @@ public static IDataBlockReader constructReader( } return new SeriesDataBlockReader( - seriesPath, new HashSet<>(allSensors), fragmentInstanceContext, queryDataSource, true); + seriesPath, new HashSet<>(allSensors), fragmentInstanceContext, queryDataSource, true, maxTsFileSetEndVersion); } @SuppressWarnings("squid:S1172") @@ -361,8 +374,16 @@ protected AbstractCompactionWriter getCompactionWriter( throws IOException { if (!seqFileResources.isEmpty() && !unseqFileResources.isEmpty()) { // cross space + List allSourceFiles = + Stream.concat(seqFileResources.stream(), unseqFileResources.stream()) + .collect(Collectors.toList()); + Pair maxTsFileSetEndVersionAndMinResource = + TsFileResource.getMaxTsFileSetEndVersionAndMinResource(allSourceFiles); return new ReadPointCrossCompactionWriter( - targetFileResources, seqFileResources, encryptParameter); + targetFileResources, + seqFileResources, + encryptParameter, + maxTsFileSetEndVersionAndMinResource.left); } else { // inner space return new ReadPointInnerCompactionWriter(targetFileResources, encryptParameter); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java index dba8b5e4f2aa1..d027957c6e656 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java @@ -121,11 +121,10 @@ protected void prepare() throws IOException { @Override protected void calculateSourceFilesAndTargetFiles() throws IOException { filesView.sourceFilesInLog = filesView.sourceFilesInCompactionPerformer; - TsFileResource targetResource = new TsFileResource(generateTargetFile(), - TsFileResourceStatus.COMPACTING); + TsFileResource targetResource = + new TsFileResource(generateTargetFile(), TsFileResourceStatus.COMPACTING); targetResource.setTsFileManager(tsFileManager); - filesView.targetFilesInLog = - Collections.singletonList(targetResource); + filesView.targetFilesInLog = Collections.singletonList(targetResource); filesView.targetFilesInPerformer = filesView.targetFilesInLog; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/subtask/ReadPointPerformerSubTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/subtask/ReadPointPerformerSubTask.java index 74f7259074508..ed967030b52c5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/subtask/ReadPointPerformerSubTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/subtask/ReadPointPerformerSubTask.java @@ -57,6 +57,7 @@ public class ReadPointPerformerSubTask implements Callable { private final AbstractCompactionWriter compactionWriter; private final Map schemaMap; private final int taskId; + private final long maxTsFileSetEndVersion; public ReadPointPerformerSubTask( IDeviceID device, @@ -65,7 +66,7 @@ public ReadPointPerformerSubTask( QueryDataSource queryDataSource, AbstractCompactionWriter compactionWriter, Map schemaMap, - int taskId) { + int taskId, long maxTsFileSetEndVersion) { this.device = device; this.measurementList = measurementList; this.fragmentInstanceContext = fragmentInstanceContext; @@ -73,6 +74,7 @@ public ReadPointPerformerSubTask( this.compactionWriter = compactionWriter; this.schemaMap = schemaMap; this.taskId = taskId; + this.maxTsFileSetEndVersion = maxTsFileSetEndVersion; } @Override @@ -88,7 +90,8 @@ public Void call() throws Exception { new ArrayList<>(schemaMap.keySet()), fragmentInstanceContext, queryDataSource, - false); + false, + maxTsFileSetEndVersion); if (dataBlockReader.hasNextBatch()) { compactionWriter.startMeasurement( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionTableSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionTableSchema.java index 99c4286c7595f..8d43305d94e19 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionTableSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionTableSchema.java @@ -35,9 +35,12 @@ public CompactionTableSchema(String tableName) { public CompactionTableSchema(TableSchema tableSchema) { this(tableSchema.getTableName(), tableSchema.getColumnSchemas(), tableSchema.getColumnTypes()); + this.updatable = tableSchema.isUpdatable(); } - public CompactionTableSchema(String tableName, List columnSchemas, + public CompactionTableSchema( + String tableName, + List columnSchemas, List columnCategories) { super(tableName, columnSchemas, columnCategories); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionTableSchemaCollector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionTableSchemaCollector.java index d5eb8d08d98fa..2c2f34e0fca1f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionTableSchemaCollector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionTableSchemaCollector.java @@ -20,8 +20,8 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; - import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; + import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.utils.Pair; @@ -91,8 +91,8 @@ public static Schema collectSchema( continue; } - EvolvedSchema evolvedSchema = resource.getMergedEvolvedSchema( - maxTsFileSetEndVersionAndAssociatedResource.getLeft()); + EvolvedSchema evolvedSchema = + resource.getMergedEvolvedSchema(maxTsFileSetEndVersionAndAssociatedResource.getLeft()); for (Map.Entry entry : tableSchemaMap.entrySet()) { String tableName = entry.getKey(); @@ -102,6 +102,7 @@ public static Schema collectSchema( } if (evolvedSchema != null) { currentTableSchema = evolvedSchema.rewriteToFinal(currentTableSchema); + tableName = currentTableSchema.getTableName(); } // merge all id columns, measurement schema will be generated automatically when end chunk diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java index a6136d7bd2f0b..067cb3602a58f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java @@ -97,12 +97,18 @@ public MultiTsFileDeviceIterator(List tsFileResources) throws IO // sort the files from the newest to the oldest Collections.sort( this.tsFileResourcesSortedByDesc, TsFileResource::compareFileCreationOrderByDesc); - long maxTsFileSetEndVersion = this.tsFileResourcesSortedByDesc.stream().mapToLong( - // max endVersion of all filesets of a TsFile - resource -> resource.getTsFileSets().stream().mapToLong(TsFileSet::getEndVersion).max() - .orElse(Long.MAX_VALUE)) - // overall max endVersion - .max().orElse(Long.MAX_VALUE); + long maxTsFileSetEndVersion = + this.tsFileResourcesSortedByDesc.stream() + .mapToLong( + // max endVersion of all filesets of a TsFile + resource -> + resource.getTsFileSets().stream() + .mapToLong(TsFileSet::getEndVersion) + .max() + .orElse(Long.MAX_VALUE)) + // overall max endVersion + .max() + .orElse(Long.MAX_VALUE); try { for (TsFileResource tsFileResource : this.tsFileResourcesSortedByDesc) { CompactionTsFileReader reader = @@ -114,8 +120,8 @@ public MultiTsFileDeviceIterator(List tsFileResources) throws IO TsFileDeviceIterator tsFileDeviceIterator; EvolvedSchema evolvedSchema = tsFileResource.getMergedEvolvedSchema(maxTsFileSetEndVersion); if (evolvedSchema != null) { - tsFileDeviceIterator = new ReorderedTsFileDeviceIterator(reader, - evolvedSchema::rewriteToFinal); + tsFileDeviceIterator = + new ReorderedTsFileDeviceIterator(reader, evolvedSchema::rewriteToFinal); } else { tsFileDeviceIterator = reader.getAllDevicesIteratorWithIsAligned(); } @@ -144,12 +150,35 @@ public MultiTsFileDeviceIterator( // sort the files from the newest to the oldest Collections.sort( this.tsFileResourcesSortedByDesc, TsFileResource::compareFileCreationOrderByDesc); + + long maxTsFileSetEndVersion = + this.tsFileResourcesSortedByDesc.stream() + .mapToLong( + // max endVersion of all filesets of a TsFile + resource -> + resource.getTsFileSets().stream() + .mapToLong(TsFileSet::getEndVersion) + .max() + .orElse(Long.MAX_VALUE)) + // overall max endVersion + .max() + .orElse(Long.MAX_VALUE); + for (TsFileResource tsFileResource : tsFileResourcesSortedByDesc) { TsFileSequenceReader reader = FileReaderManager.getInstance() .get(tsFileResource.getTsFilePath(), tsFileResource.getTsFileID(), true); readerMap.put(tsFileResource, reader); - deviceIteratorMap.put(tsFileResource, reader.getAllDevicesIteratorWithIsAligned()); + + TsFileDeviceIterator tsFileDeviceIterator; + EvolvedSchema evolvedSchema = tsFileResource.getMergedEvolvedSchema(maxTsFileSetEndVersion); + if (evolvedSchema != null) { + tsFileDeviceIterator = + new ReorderedTsFileDeviceIterator(reader, evolvedSchema::rewriteToFinal); + } else { + tsFileDeviceIterator = reader.getAllDevicesIteratorWithIsAligned(); + } + deviceIteratorMap.put(tsFileResource, tsFileDeviceIterator); } } @@ -171,6 +200,19 @@ public MultiTsFileDeviceIterator( this.tsFileResourcesSortedByDesc, TsFileResource::compareFileCreationOrderByDesc); this.readerMap = readerMap; + long maxTsFileSetEndVersion = + this.tsFileResourcesSortedByDesc.stream() + .mapToLong( + // max endVersion of all filesets of a TsFile + resource -> + resource.getTsFileSets().stream() + .mapToLong(TsFileSet::getEndVersion) + .max() + .orElse(Long.MAX_VALUE)) + // overall max endVersion + .max() + .orElse(Long.MAX_VALUE); + CompactionType type = null; if (!seqResources.isEmpty() && !unseqResources.isEmpty()) { type = CompactionType.CROSS_COMPACTION; @@ -187,11 +229,20 @@ public MultiTsFileDeviceIterator( type, EncryptDBUtils.getFirstEncryptParamFromTSFilePath(tsFileResource.getTsFilePath())); readerMap.put(tsFileResource, reader); - deviceIteratorMap.put(tsFileResource, reader.getAllDevicesIteratorWithIsAligned()); + + TsFileDeviceIterator tsFileDeviceIterator; + EvolvedSchema evolvedSchema = tsFileResource.getMergedEvolvedSchema(maxTsFileSetEndVersion); + if (evolvedSchema != null) { + tsFileDeviceIterator = + new ReorderedTsFileDeviceIterator(reader, evolvedSchema::rewriteToFinal); + } else { + tsFileDeviceIterator = reader.getAllDevicesIteratorWithIsAligned(); + } + deviceIteratorMap.put(tsFileResource, tsFileDeviceIterator); } } - public boolean hasNextDevice() { + public boolean hasNextDevice() throws IOException { boolean hasNext = false; for (TsFileDeviceIterator iterator : deviceIteratorMap.values()) { hasNext = @@ -212,7 +263,7 @@ public boolean hasNextDevice() { * @return Pair of device full path and whether this device is aligned */ @SuppressWarnings({"squid:S135", "java:S2259"}) - public Pair nextDevice() throws IllegalPathException { + public Pair nextDevice() throws IllegalPathException, IOException { List toBeRemovedResources = new LinkedList<>(); Pair minDevice = null; // get the device from source files sorted from the newest to the oldest by version @@ -292,24 +343,25 @@ public Map getAllSchemasOfCurrentDevice( timeseriesMetadataList, deviceIteratorMap.get(resource).getFirstMeasurementNodeOfCurrentDevice(), schemaMap.keySet(), - true); - EvolvedSchema evolvedSchema = resource.getMergedEvolvedSchema( - maxTsFileSetEndVersionAndMinResource.left); + true, + null); + EvolvedSchema evolvedSchema = + resource.getMergedEvolvedSchema(maxTsFileSetEndVersionAndMinResource.left); if (evolvedSchema != null) { // the device has been rewritten, should get the original name for rewriting - evolvedSchema.rewriteToFinal(evolvedSchema.getOriginalTableName(currentDevice.left.getTableName()), timeseriesMetadataList); + evolvedSchema.rewriteToFinal( + evolvedSchema.getOriginalTableName(currentDevice.left.getTableName()), + timeseriesMetadataList); } for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) { if (!schemaMap.containsKey(timeseriesMetadata.getMeasurementId()) && !timeseriesMetadata.getChunkMetadataList().isEmpty()) { - MeasurementSchema measurementSchema = reader.getMeasurementSchema( - timeseriesMetadata.getChunkMetadataList()); + MeasurementSchema measurementSchema = + reader.getMeasurementSchema(timeseriesMetadata.getChunkMetadataList()); // the column may be renamed measurementSchema.setMeasurementName(timeseriesMetadata.getMeasurementId()); - schemaMap.put( - timeseriesMetadata.getMeasurementId(), - measurementSchema); + schemaMap.put(timeseriesMetadata.getMeasurementId(), measurementSchema); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/ReorderedTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/ReorderedTsFileDeviceIterator.java index 14d2db8a191d1..3b9bf6ab8da8f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/ReorderedTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/ReorderedTsFileDeviceIterator.java @@ -1,40 +1,43 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.MetadataIndexNode; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.utils.Pair; + import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.function.Function; -import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.MetadataIndexNode; -import org.apache.tsfile.read.TsFileSequenceReader; -import org.apache.tsfile.utils.Pair; public class ReorderedTsFileDeviceIterator extends TransformedTsFileDeviceIterator { - private final List, MetadataIndexNode>> deviceIDAndFirstMeasurementNodeList = new ArrayList<>(); + private final List, MetadataIndexNode>> + deviceIDAndFirstMeasurementNodeList = new ArrayList<>(); private Iterator, MetadataIndexNode>> deviceIDListIterator; private Pair, MetadataIndexNode> current; - public ReorderedTsFileDeviceIterator(TsFileSequenceReader reader, - Function transformer) - throws IOException { + public ReorderedTsFileDeviceIterator( + TsFileSequenceReader reader, Function transformer) throws IOException { super(reader, transformer); collectAndSort(); } - public ReorderedTsFileDeviceIterator(TsFileSequenceReader reader, String tableName, - Function transformer) throws IOException { + public ReorderedTsFileDeviceIterator( + TsFileSequenceReader reader, String tableName, Function transformer) + throws IOException { super(reader, tableName, transformer); collectAndSort(); } - private void collectAndSort() { + private void collectAndSort() throws IOException { while (super.hasNext()) { Pair next = super.next(); next.left = transformer.apply(next.left); - deviceIDAndFirstMeasurementNodeList.add(new Pair<>(next, super.getFirstMeasurementNodeOfCurrentDevice())); + deviceIDAndFirstMeasurementNodeList.add( + new Pair<>(next, super.getFirstMeasurementNodeOfCurrentDevice())); } deviceIDAndFirstMeasurementNodeList.sort(Comparator.comparing(p -> p.getLeft().getLeft())); deviceIDListIterator = deviceIDAndFirstMeasurementNodeList.iterator(); @@ -54,12 +57,12 @@ public Pair next() { @Override public Pair current() { - return current.left; + return current == null ? null : current.left; } @Override public MetadataIndexNode getFirstMeasurementNodeOfCurrentDevice() { // the devices have been reordered, cannot use the measurementNode - return current.right; + return current == null ? null : current.right; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/TransformedTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/TransformedTsFileDeviceIterator.java index f1af028226d84..a361adb18e611 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/TransformedTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/TransformedTsFileDeviceIterator.java @@ -19,26 +19,28 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils; -import java.io.IOException; -import java.util.function.Function; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TsFileDeviceIterator; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.utils.Pair; +import java.io.IOException; +import java.util.function.Function; + public class TransformedTsFileDeviceIterator extends TsFileDeviceIterator { protected Function transformer; - public TransformedTsFileDeviceIterator(TsFileSequenceReader reader, Function transformer) - throws IOException { + public TransformedTsFileDeviceIterator( + TsFileSequenceReader reader, Function transformer) throws IOException { super(reader); this.transformer = transformer; } - public TransformedTsFileDeviceIterator(TsFileSequenceReader reader, String tableName, Function transformer) + public TransformedTsFileDeviceIterator( + TsFileSequenceReader reader, String tableName, Function transformer) throws IOException { - super(reader, tableName); + super(reader, tableName, null); this.transformer = transformer; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/reader/SeriesDataBlockReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/reader/SeriesDataBlockReader.java index a49b97c8f5222..9c78d866852bb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/reader/SeriesDataBlockReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/reader/SeriesDataBlockReader.java @@ -53,7 +53,8 @@ public SeriesDataBlockReader( Set allSensors, FragmentInstanceContext context, QueryDataSource dataSource, - boolean ascending) { + boolean ascending, + long maxTsFileSetEndVersion) { SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder(); scanOptionsBuilder.withAllSensors(allSensors); @@ -63,14 +64,16 @@ public SeriesDataBlockReader( (AlignedFullPath) seriesPath, ascending ? Ordering.ASC : Ordering.DESC, scanOptionsBuilder.build(), - context); + context, + maxTsFileSetEndVersion); } else if (seriesPath instanceof NonAlignedFullPath) { this.seriesScanUtil = new SeriesScanUtil( seriesPath, ascending ? Ordering.ASC : Ordering.DESC, scanOptionsBuilder.build(), - context); + context, + maxTsFileSetEndVersion); } else { throw new IllegalArgumentException("Should call exact sub class!"); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java index 623f2b3287d64..63f148e5a58d1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java @@ -27,8 +27,8 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.flushcontroller.AbstractCompactionFlushController; import org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFileWriter; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; - import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; + import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.PageHeader; @@ -340,5 +340,6 @@ protected void checkPreviousTimestamp(long currentWritingTimestamp, int subTaskI } } - public abstract void setSchemaForAllTargetFile(List schemas, Pair maxTsFileSetEndVersionAndAssociatedResource); + public abstract void setSchemaForAllTargetFile( + List schemas, Pair maxTsFileSetEndVersionAndAssociatedResource); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java index cfa377c3133aa..5c4a56ce54af6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java @@ -80,7 +80,11 @@ public abstract class AbstractCrossCompactionWriter extends AbstractCompactionWr protected AbstractCrossCompactionWriter( List targetResources, List seqFileResources) throws IOException { - this(targetResources, seqFileResources, EncryptDBUtils.getDefaultFirstEncryptParam(), Long.MIN_VALUE); + this( + targetResources, + seqFileResources, + EncryptDBUtils.getDefaultFirstEncryptParam(), + Long.MIN_VALUE); } protected AbstractCrossCompactionWriter( @@ -106,7 +110,8 @@ protected AbstractCrossCompactionWriter( targetResources.get(i), memorySizeForEachWriter, CompactionType.CROSS_COMPACTION, - this.encryptParameter, maxTsFileSetEndVersion)); + this.encryptParameter, + maxTsFileSetEndVersion)); isEmptyFile[i] = true; } this.seqTsFileResources = seqFileResources; @@ -270,7 +275,8 @@ public long getWriterSize() throws IOException { } @Override - public void setSchemaForAllTargetFile(List schemas, Pair maxTsFileSetEndVersionAndMinResource) { + public void setSchemaForAllTargetFile( + List schemas, Pair maxTsFileSetEndVersionAndMinResource) { for (int i = 0; i < targetFileWriters.size(); i++) { CompactionTsFileWriter compactionTsFileWriter = targetFileWriters.get(i); Schema schema = schemas.get(i); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java index 28ca734f32dbd..42338a8ad1fbb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java @@ -130,8 +130,9 @@ private void useNewWriter() throws IOException { Schema schema = CompactionTableSchemaCollector.copySchema(schemas.get(0)); TsFileResource minVersionResource = maxTsFileSetEndVersionAndMinResource.getRight(); fileWriter.getTsFileResource().setTsFileManager(minVersionResource.getTsFileManager()); - EvolvedSchema evolvedSchema = fileWriter.getTsFileResource().getMergedEvolvedSchema(maxTsFileSetEndVersion); - fileWriter.setSchema(evolvedSchema.rewriteToOriginal(schema, CompactionTableSchema::new)); + EvolvedSchema evolvedSchema = + fileWriter.getTsFileResource().getMergedEvolvedSchema(maxTsFileSetEndVersion); + fileWriter.setSchema(evolvedSchema != null ? evolvedSchema.rewriteToOriginal(schema, CompactionTableSchema::new) : schema); } @Override @@ -184,7 +185,8 @@ public void checkAndMayFlushChunkMetadata() throws IOException { } @Override - public void setSchemaForAllTargetFile(List schemas, Pair maxTsFileSetEndVersionAndMinResource) { + public void setSchemaForAllTargetFile( + List schemas, Pair maxTsFileSetEndVersionAndMinResource) { this.schemas = schemas; this.maxTsFileSetEndVersionAndMinResource = maxTsFileSetEndVersionAndMinResource; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java index 59a87b4211c46..f379d02704add 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java @@ -53,7 +53,11 @@ public FastCrossCompactionWriter( List seqSourceResources, Map readerMap) throws IOException { - super(targetResources, seqSourceResources, EncryptDBUtils.getDefaultFirstEncryptParam()); + super( + targetResources, + seqSourceResources, + EncryptDBUtils.getDefaultFirstEncryptParam(), + Long.MIN_VALUE); this.readerMap = readerMap; } @@ -61,9 +65,10 @@ public FastCrossCompactionWriter( List targetResources, List seqSourceResources, Map readerMap, - EncryptParameter encryptParameter) + EncryptParameter encryptParameter, + long maxTsFileSetEndVersion) throws IOException { - super(targetResources, seqSourceResources, encryptParameter); + super(targetResources, seqSourceResources, encryptParameter, maxTsFileSetEndVersion); this.readerMap = readerMap; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/ReadPointCrossCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/ReadPointCrossCompactionWriter.java index 6810df4d1a3b6..b2799c0dfe5e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/ReadPointCrossCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/ReadPointCrossCompactionWriter.java @@ -47,15 +47,20 @@ public class ReadPointCrossCompactionWriter extends AbstractCrossCompactionWrite public ReadPointCrossCompactionWriter( List targetResources, List seqFileResources) throws IOException { - super(targetResources, seqFileResources, EncryptDBUtils.getDefaultFirstEncryptParam()); + super( + targetResources, + seqFileResources, + EncryptDBUtils.getDefaultFirstEncryptParam(), + Long.MIN_VALUE); } public ReadPointCrossCompactionWriter( List targetResources, List seqFileResources, - EncryptParameter encryptParameter) + EncryptParameter encryptParameter, + long maxTsFileSetEndVersion) throws IOException { - super(targetResources, seqFileResources, encryptParameter); + super(targetResources, seqFileResources, encryptParameter, maxTsFileSetEndVersion); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java index c8c24fca8e438..efe33d228f6a4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java @@ -65,12 +65,19 @@ public class CompactionTsFileWriter extends TsFileIOWriter { @TestOnly public CompactionTsFileWriter(File file, long maxMetadataSize, CompactionType type) throws IOException { - this(new TsFileResource(file), maxMetadataSize, type, EncryptDBUtils.getDefaultFirstEncryptParam(), + this( + new TsFileResource(file), + maxMetadataSize, + type, + EncryptDBUtils.getDefaultFirstEncryptParam(), Long.MIN_VALUE); } public CompactionTsFileWriter( - TsFileResource tsFile, long maxMetadataSize, CompactionType type, EncryptParameter encryptParameter, + TsFileResource tsFile, + long maxMetadataSize, + CompactionType type, + EncryptParameter encryptParameter, long maxTsFileSetEndVersion) throws IOException { super(tsFile.getTsFile(), maxMetadataSize, encryptParameter); @@ -101,7 +108,13 @@ public void writeChunk(IChunkWriter chunkWriter) throws IOException { if (!chunkWriter.isEmpty()) { isEmptyTargetFile = false; } - chunkWriter.writeToFileWriter(this); + chunkWriter.writeToFileWriter( + this, + evolvedSchema == null + ? null + : measurementName -> + evolvedSchema.getOriginalColumnName( + evolvedSchema.getFinalTableName(currentDeviceId.getTableName()), measurementName)); long writtenDataSize = this.getPos() - beforeOffset; CompactionMetrics.getInstance() .recordWriteInfo( @@ -116,6 +129,13 @@ public void writeChunk(Chunk chunk, ChunkMetadata chunkMetadata) throws IOExcept if (chunkMetadata.getNumOfPoints() != 0) { isEmptyTargetFile = false; } + if (evolvedSchema != null) { + chunk + .getHeader() + .setMeasurementID( + evolvedSchema.getOriginalColumnName( + currentDeviceId.getTableName(), chunk.getHeader().getMeasurementID())); + } super.writeChunk(chunk, chunkMetadata); long writtenDataSize = this.getPos() - beforeOffset; CompactionMetrics.getInstance() @@ -133,6 +153,10 @@ public void writeEmptyValueChunk( TSEncoding encodingType, Statistics statistics) throws IOException { + if (evolvedSchema != null) { + measurementId = + evolvedSchema.getOriginalColumnName(currentDeviceId.getTableName(), measurementId); + } long beforeOffset = this.getPos(); super.writeEmptyValueChunk( measurementId, compressionType, tsDataType, encodingType, statistics); @@ -150,6 +174,9 @@ public int checkMetadataSizeAndMayFlush() throws IOException { @Override public int startChunkGroup(IDeviceID deviceId) throws IOException { + if (evolvedSchema != null) { + deviceId = evolvedSchema.rewriteToOriginal(deviceId); + } currentDeviceId = deviceId; return super.startChunkGroup(deviceId); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java index d515a7fa3a908..d0f3426423ce9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java @@ -283,7 +283,7 @@ private void checkNonAlignedDeviceSeries( throws IOException { List timeseriesMetadataList = new ArrayList<>(); reader.getDeviceTimeseriesMetadata( - timeseriesMetadataList, metadataIndexNode, Collections.emptySet(), true); + timeseriesMetadataList, metadataIndexNode, Collections.emptySet(), true, null); long actualDeviceStartTime = Long.MAX_VALUE; long actualDeviceEndTime = Long.MIN_VALUE; for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSource.java index 2816493401671..b659585d9fe58 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSource.java @@ -113,18 +113,18 @@ public IQueryDataSource clone() { return queryDataSource; } - public boolean hasNextSeqResource(int curIndex, boolean ascending, IDeviceID deviceID) { + public boolean hasNextSeqResource(int curIndex, boolean ascending, IDeviceID deviceID, long maxTsFileSetEndVersion) { boolean res = ascending ? curIndex < seqResources.size() : curIndex >= 0; if (res && curIndex != this.curSeqIndex) { this.curSeqIndex = curIndex; - this.curSeqOrderTime = seqResources.get(curIndex).getOrderTimeForSeq(deviceID, ascending); + this.curSeqOrderTime = seqResources.get(curIndex).getOrderTimeForSeq(deviceID, ascending, maxTsFileSetEndVersion); this.curSeqSatisfied = null; } return res; } public boolean isSeqSatisfied( - IDeviceID deviceID, int curIndex, Filter timeFilter, boolean debug) { + IDeviceID deviceID, int curIndex, Filter timeFilter, boolean debug, long maxTsFileSetEndVersion) { if (curIndex != this.curSeqIndex) { throw new IllegalArgumentException( String.format("curIndex %d is not equal to curSeqIndex %d", curIndex, this.curSeqIndex)); @@ -133,7 +133,7 @@ public boolean isSeqSatisfied( TsFileResource tsFileResource = seqResources.get(curSeqIndex); curSeqSatisfied = tsFileResource != null - && (isSingleDevice || tsFileResource.isSatisfied(deviceID, timeFilter, true, debug)); + && (isSingleDevice || tsFileResource.isSatisfied(deviceID, timeFilter, true, debug, maxTsFileSetEndVersion)); } return curSeqSatisfied; @@ -154,14 +154,14 @@ public TsFileResource getSeqResourceByIndex(int curIndex) { return null; } - public boolean hasNextUnseqResource(int curIndex, boolean ascending, IDeviceID deviceID) { + public boolean hasNextUnseqResource(int curIndex, boolean ascending, IDeviceID deviceID, long maxTsFileSetEndVersion) { boolean res = curIndex < unseqResources.size(); if (res && curIndex != this.curUnSeqIndex) { this.curUnSeqIndex = curIndex; this.curUnSeqOrderTime = unseqResources .get(unSeqFileOrderIndex[curIndex]) - .getOrderTimeForUnseq(deviceID, ascending); + .getOrderTimeForUnseq(deviceID, ascending, maxTsFileSetEndVersion); this.curUnSeqSatisfied = null; } return res; @@ -209,7 +209,7 @@ public int getUnseqResourcesSize() { return unseqResources.size(); } - public void fillOrderIndexes(IDeviceID deviceId, boolean ascending) { + public void fillOrderIndexes(IDeviceID deviceId, boolean ascending, long maxTsFileSetEndVersion) { if (unseqResources == null || unseqResources.isEmpty()) { return; } @@ -219,7 +219,7 @@ public void fillOrderIndexes(IDeviceID deviceId, boolean ascending) { for (TsFileResource resource : unseqResources) { orderTimeToIndexMap .computeIfAbsent( - resource.getOrderTimeForUnseq(deviceId, ascending), key -> new ArrayList<>()) + resource.getOrderTimeForUnseq(deviceId, ascending, maxTsFileSetEndVersion), key -> new ArrayList<>()) .add(index++); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java index 11dcde29b37ae..610365cbfb795 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile; -import java.util.stream.Collectors; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.storageengine.dataregion.modification.ModFileManagement; @@ -43,6 +42,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; public class TsFileManager { private final String storageGroupName; @@ -518,20 +518,25 @@ public long getMaxFileTimestampOfUnSequenceFile() { public void addTsFileSet(TsFileSet newSet, long partitionId) { writeLock("addTsFileSet"); try { - List tsFileSetList = tsfileSets.computeIfAbsent(partitionId, - p -> new ArrayList<>()); + List tsFileSetList = + tsfileSets.computeIfAbsent(partitionId, p -> new ArrayList<>()); tsFileSetList.add(newSet); } finally { writeUnlock(); } } - public List getTsFileSet(long partitionId, long minFileVersionIncluded, long maxFileVersionExcluded) { + public List getTsFileSet( + long partitionId, long minFileVersionIncluded, long maxFileVersionExcluded) { readLock(); try { List tsFileSetList = tsfileSets.get(partitionId); - return tsFileSetList.stream().filter(s -> s.getEndVersion() < maxFileVersionExcluded && s.getEndVersion() >= minFileVersionIncluded).collect( - Collectors.toList()); + return tsFileSetList.stream() + .filter( + s -> + s.getEndVersion() < maxFileVersionExcluded + && s.getEndVersion() >= minFileVersionIncluded) + .collect(Collectors.toList()); } finally { readUnlock(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index a387db509c698..0bc12e3f6f23b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -641,8 +641,12 @@ public Optional getEndTime(IDeviceID deviceId) { } // cannot use FileTimeIndex - public long getOrderTimeForSeq(IDeviceID deviceId, boolean ascending) { + public long getOrderTimeForSeq(IDeviceID deviceId, boolean ascending, long maxTsFileSetEndVersion) { if (timeIndex instanceof ArrayDeviceTimeIndex) { + EvolvedSchema evolvedSchema = getMergedEvolvedSchema(maxTsFileSetEndVersion); + if (evolvedSchema != null) { + deviceId = evolvedSchema.rewriteToOriginal(deviceId); + } return ascending ? timeIndex.getStartTime(deviceId).orElse(Long.MIN_VALUE) : timeIndex.getEndTime(deviceId).orElse(Long.MAX_VALUE); @@ -652,8 +656,12 @@ public long getOrderTimeForSeq(IDeviceID deviceId, boolean ascending) { } // can use FileTimeIndex - public long getOrderTimeForUnseq(IDeviceID deviceId, boolean ascending) { + public long getOrderTimeForUnseq(IDeviceID deviceId, boolean ascending, long maxTsFileSetEndVersion) { if (timeIndex instanceof ArrayDeviceTimeIndex) { + EvolvedSchema evolvedSchema = getMergedEvolvedSchema(maxTsFileSetEndVersion); + if (evolvedSchema != null) { + deviceId = evolvedSchema.rewriteToOriginal(deviceId); + } if (ascending) { return timeIndex.getStartTime(deviceId).orElse(Long.MIN_VALUE); } else { @@ -998,14 +1006,26 @@ public boolean stillLives(long timeLowerBound) { } public boolean isDeviceIdExist(IDeviceID deviceId) { + EvolvedSchema evolvedSchema = getMergedEvolvedSchema(); + if (evolvedSchema != null) { + deviceId = evolvedSchema.rewriteToOriginal(deviceId); + } return timeIndex.checkDeviceIdExist(deviceId); } + public boolean isSatisfied(IDeviceID deviceId, Filter timeFilter, boolean isSeq, boolean debug) { + return isSatisfied(deviceId, timeFilter, isSeq, debug, Long.MAX_VALUE); + } + /** * @return true if the device is contained in the TsFile */ @SuppressWarnings("OptionalGetWithoutIsPresent") - public boolean isSatisfied(IDeviceID deviceId, Filter timeFilter, boolean isSeq, boolean debug) { + public boolean isSatisfied(IDeviceID deviceId, Filter timeFilter, boolean isSeq, boolean debug, long maxTsFileSetEndVersion) { + EvolvedSchema evolvedSchema = getMergedEvolvedSchema(maxTsFileSetEndVersion); + if (evolvedSchema != null) { + deviceId = evolvedSchema.rewriteToOriginal(deviceId); + } if (deviceId != null && definitelyNotContains(deviceId)) { if (debug) { DEBUG_LOGGER.info( @@ -1632,11 +1652,13 @@ public TsFileResource shallowCloneForNative() throws CloneNotSupportedException } public List getTsFileSets() { - return tsFileManager.getTsFileSet(tsFileID.timePartitionId, tsFileID.fileVersion, Long.MAX_VALUE); + return tsFileManager.getTsFileSet( + tsFileID.timePartitionId, tsFileID.fileVersion, Long.MAX_VALUE); } public List getTsFileSets(long maxEndVersionExcluded) { - return tsFileManager.getTsFileSet(tsFileID.timePartitionId, tsFileID.fileVersion, maxEndVersionExcluded); + return tsFileManager.getTsFileSet( + tsFileID.timePartitionId, tsFileID.fileVersion, maxEndVersionExcluded); } public EvolvedSchema getMergedEvolvedSchema() { @@ -1645,7 +1667,8 @@ public EvolvedSchema getMergedEvolvedSchema() { public EvolvedSchema getMergedEvolvedSchema(long excludedMaxFileVersion) { List list = new ArrayList<>(); - for (TsFileSet fileSet : getTsFileSets()) { + List tsFileSets = getTsFileSets(); + for (TsFileSet fileSet : tsFileSets) { if (fileSet.getEndVersion() >= excludedMaxFileVersion) { continue; } @@ -1661,7 +1684,8 @@ public EvolvedSchema getMergedEvolvedSchema(long excludedMaxFileVersion) { return EvolvedSchema.merge(list.toArray(new EvolvedSchema[0])); } - public static Pair getMaxTsFileSetEndVersionAndMinResource(List tsFileResources) { + public static Pair getMaxTsFileSetEndVersionAndMinResource( + List tsFileResources) { long maxTsFileSetEndVersion = Long.MIN_VALUE; long minResourceVersion = Long.MAX_VALUE; TsFileResource minTsFileResource = null; @@ -1682,8 +1706,7 @@ public static Pair getMaxTsFileSetEndVersionAndMinResource return new Pair<>(maxTsFileSetEndVersion, minTsFileResource); } - public void setTsFileManager( - TsFileManager tsFileManager) { + public void setTsFileManager(TsFileManager tsFileManager) { this.tsFileManager = tsFileManager; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java index a88e76d43278b..3dca08b13d710 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java @@ -19,8 +19,6 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; -import java.util.function.Function; -import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionTableSchema; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry.ModType; @@ -28,13 +26,13 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; import org.apache.tsfile.enums.ColumnCategory; -import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.schema.Schema; import java.util.ArrayList; import java.util.Collections; @@ -44,8 +42,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.tsfile.write.schema.Schema; public class EvolvedSchema { // the evolved table names after applying all schema evolution operations @@ -106,7 +104,7 @@ public String getOriginalTableName(String finalTableName) { return finalToOriginalTableNames.getOrDefault(finalTableName, finalTableName); } - private String getFinalTableName(String originalTableName) { + public String getFinalTableName(String originalTableName) { return originalToFinalTableNames.getOrDefault(originalTableName, originalTableName); } @@ -232,10 +230,9 @@ public void rewriteToFinal( String originalTableName, List timeseriesMetadataList) { timeseriesMetadataList.forEach( timeseriesMetadata -> { - String finalColumnName = getFinalColumnName(originalTableName, - timeseriesMetadata.getMeasurementId()); - timeseriesMetadata.setMeasurementId( - finalColumnName); + String finalColumnName = + getFinalColumnName(originalTableName, timeseriesMetadata.getMeasurementId()); + timeseriesMetadata.setMeasurementId(finalColumnName); }); } @@ -263,11 +260,15 @@ private TableSchema rewriteToOriginal(TableSchema tableSchema) { getOriginalColumnName( tableSchema.getTableName(), measurementSchema.getMeasurementName()), measurementSchema.getType(), - measurementSchema.getEncodingType(), measurementSchema.getCompressor())); + measurementSchema.getEncodingType(), + measurementSchema.getCompressor())); columnCategories.add(tableSchema.getColumnTypes().get(i)); } - return new TableSchema(originalTableName, measurementSchemas, columnCategories); + TableSchema schema = new TableSchema(originalTableName, measurementSchemas, + columnCategories); + schema.setUpdatable(tableSchema.isUpdatable()); + return schema; } public TableSchema rewriteToFinal(TableSchema tableSchema) { @@ -288,7 +289,10 @@ public TableSchema rewriteToFinal(TableSchema tableSchema) { columnCategories.add(tableSchema.getColumnTypes().get(i)); } - return new TableSchema(finalTableName, measurementSchemas, columnCategories); + TableSchema schema = new TableSchema(finalTableName, measurementSchemas, + columnCategories); + schema.setUpdatable(tableSchema.isUpdatable()); + return schema; } @SuppressWarnings("SuspiciousSystemArraycopy") @@ -310,6 +314,10 @@ public static EvolvedSchema deepCopy(EvolvedSchema evolvedSchema) { new LinkedHashMap<>(evolvedSchema.finalToOriginalTableNames); newEvolvedSchema.finalToOriginalColumnNames = new LinkedHashMap<>(evolvedSchema.finalToOriginalColumnNames); + newEvolvedSchema.originalToFinalTableNames = + new LinkedHashMap<>(evolvedSchema.originalToFinalTableNames); + newEvolvedSchema.originalToFinalColumnNames = + new LinkedHashMap<>(evolvedSchema.originalToFinalColumnNames); return newEvolvedSchema; } @@ -323,6 +331,9 @@ public static EvolvedSchema merge(EvolvedSchema... schemas) { break; } } + if (i == schemas.length) { + return firstNotNullSchema; + } if (firstNotNullSchema == null) { return null; @@ -359,7 +370,9 @@ public static EvolvedSchema merge(EvolvedSchema... schemas) { public Schema rewriteToOriginal(Schema schema) { return rewriteToOriginal(schema, null); } - public Schema rewriteToOriginal(Schema schema, Function tableSchemaTransformer) { + + public Schema rewriteToOriginal( + Schema schema, Function tableSchemaTransformer) { Schema copySchema = new Schema(); for (TableSchema tableSchema : schema.getTableSchemaMap().values()) { TableSchema originalSchema = rewriteToOriginal(tableSchema); @@ -370,6 +383,4 @@ public Schema rewriteToOriginal(Schema schema, Function { @@ -65,7 +66,7 @@ public TsFileSet(long endVersion, String fileSetsDir, boolean recover) { if (schemaEvolutionFile == null) { schemaEvolutionFile = new SchemaEvolutionFile( - fileSetsDir + File.separator + 0 + SchemaEvolutionFile.FILE_SUFFIX); + fileSetDir + File.separator + 0 + SchemaEvolutionFile.FILE_SUFFIX); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java index 31ed98eb0e109..7a90f6b16d61a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java @@ -850,8 +850,8 @@ protected List getPaths(List resources) Pair iDeviceIDBooleanPair = deviceIterator.nextDevice(); IDeviceID deviceID = iDeviceIDBooleanPair.getLeft(); boolean isAlign = iDeviceIDBooleanPair.getRight(); - Map schemaMap = deviceIterator.getAllSchemasOfCurrentDevice( - new Pair<>(Long.MIN_VALUE, null)); + Map schemaMap = + deviceIterator.getAllSchemasOfCurrentDevice(new Pair<>(Long.MIN_VALUE, null)); IMeasurementSchema timeSchema = schemaMap.remove(TsFileConstant.TIME_COLUMN_ID); List measurementSchemas = new ArrayList<>(schemaMap.values()); if (measurementSchemas.isEmpty()) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointCompactionPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointCompactionPerformerTest.java index f21571ce4f87d..f926506e88903 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointCompactionPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointCompactionPerformerTest.java @@ -37,18 +37,32 @@ import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.ColumnRename; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.TableRename; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.fileset.TsFileSet; import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.db.utils.constant.TestConstant; import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.NoMeasurementException; +import org.apache.tsfile.exception.write.NoTableException; import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.ColumnSchemaBuilder; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; +import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.read.common.IBatchDataIterator; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.query.dataset.ResultSet; +import org.apache.tsfile.read.v4.ITsFileReader; +import org.apache.tsfile.read.v4.TsFileReaderBuilder; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.TsFileGeneratorUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; @@ -56,8 +70,10 @@ import org.junit.Before; import org.junit.Test; +import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -66,6 +82,8 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; @SuppressWarnings("OptionalGetWithoutIsPresent") public class ReadPointCompactionPerformerTest extends AbstractCompactionTest { @@ -6993,4 +7011,293 @@ public void testCrossSpaceCompactionWithDeviceMaxTimeLaterInUnseqFile() Assert.fail(); } } + + @Test + public void testWithSevoFile() throws Exception { + String fileSetDir = + TestConstant.BASE_OUTPUT_PATH + File.separator + TsFileSet.FILE_SET_DIR_NAME; + // file1: + // table1[s1, s2, s3] + // table2[s1, s2, s3] + File f1 = new File(SEQ_DIRS, "0-1-0-0.tsfile"); + TableSchema tableSchema1_1 = + new TableSchema( + "table1", + Arrays.asList( + new ColumnSchemaBuilder() + .name("s1") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build(), + new ColumnSchemaBuilder() + .name("s2") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build(), + new ColumnSchemaBuilder() + .name("s3") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build())); + TableSchema tableSchema1_2 = + new TableSchema( + "table2", + Arrays.asList( + new ColumnSchemaBuilder() + .name("s1") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build(), + new ColumnSchemaBuilder() + .name("s2") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build(), + new ColumnSchemaBuilder() + .name("s3") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build())); + try (TsFileWriter tsFileWriter = new TsFileWriter(f1)) { + tsFileWriter.registerTableSchema(tableSchema1_1); + tsFileWriter.registerTableSchema(tableSchema1_2); + + Tablet tablet1 = new Tablet(tableSchema1_1.getTableName(), tableSchema1_1.getColumnSchemas()); + tablet1.addTimestamp(0, 0); + tablet1.addValue(0, 0, 1); + tablet1.addValue(0, 1, 2); + tablet1.addValue(0, 2, 3); + + Tablet tablet2 = new Tablet(tableSchema1_2.getTableName(), tableSchema1_2.getColumnSchemas()); + tablet2.addTimestamp(0, 0); + tablet2.addValue(0, 0, 101); + tablet2.addValue(0, 1, 102); + tablet2.addValue(0, 2, 103); + + tsFileWriter.writeTable(tablet1); + tsFileWriter.writeTable(tablet2); + } + TsFileResource resource1 = new TsFileResource(f1); + resource1.setTsFileManager(tsFileManager); + resource1.updateStartTime(Factory.DEFAULT_FACTORY.create(new String[]{"table1"}), 0); + resource1.updateEndTime(Factory.DEFAULT_FACTORY.create(new String[]{"table1"}), 0); + resource1.updateStartTime(Factory.DEFAULT_FACTORY.create(new String[]{"table2"}), 0); + resource1.updateEndTime(Factory.DEFAULT_FACTORY.create(new String[]{"table2"}), 0); + resource1.close(); + + // rename table1 -> table0 + TsFileSet tsFileSet1 = new TsFileSet(1, fileSetDir, false); + tsFileSet1.appendSchemaEvolution( + Collections.singletonList(new TableRename("table1", "table0"))); + tsFileManager.addTsFileSet(tsFileSet1, 0); + + // file2: + // table0[s1, s2, s3] + // table2[s1, s2, s3] + File f2 = new File(SEQ_DIRS, "0-2-0-0.tsfile"); + TableSchema tableSchema2_1 = + new TableSchema( + "table0", + Arrays.asList( + new ColumnSchemaBuilder() + .name("s1") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build(), + new ColumnSchemaBuilder() + .name("s2") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build(), + new ColumnSchemaBuilder() + .name("s3") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build())); + TableSchema tableSchema2_2 = + new TableSchema( + "table2", + Arrays.asList( + new ColumnSchemaBuilder() + .name("s1") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build(), + new ColumnSchemaBuilder() + .name("s2") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build(), + new ColumnSchemaBuilder() + .name("s3") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build())); + try (TsFileWriter tsFileWriter = new TsFileWriter(f2)) { + tsFileWriter.registerTableSchema(tableSchema2_1); + tsFileWriter.registerTableSchema(tableSchema2_2); + + Tablet tablet1 = new Tablet(tableSchema2_1.getTableName(), tableSchema2_1.getColumnSchemas()); + tablet1.addTimestamp(0, 1); + tablet1.addValue(0, 0, 11); + tablet1.addValue(0, 1, 12); + tablet1.addValue(0, 2, 13); + + Tablet tablet2 = new Tablet(tableSchema2_2.getTableName(), tableSchema2_2.getColumnSchemas()); + tablet2.addTimestamp(0, 1); + tablet2.addValue(0, 0, 111); + tablet2.addValue(0, 1, 112); + tablet2.addValue(0, 2, 113); + + tsFileWriter.writeTable(tablet1); + tsFileWriter.writeTable(tablet2); + } + TsFileResource resource2 = new TsFileResource(f2); + resource2.setTsFileManager(tsFileManager); + resource2.updateStartTime(Factory.DEFAULT_FACTORY.create(new String[]{"table0"}), 1); + resource2.updateEndTime(Factory.DEFAULT_FACTORY.create(new String[]{"table0"}), 1); + resource2.updateStartTime(Factory.DEFAULT_FACTORY.create(new String[]{"table2"}), 1); + resource2.updateEndTime(Factory.DEFAULT_FACTORY.create(new String[]{"table2"}), 1); + resource2.close(); + + + // rename table0.s1 -> table0.s0 + TsFileSet tsFileSet2 = new TsFileSet(2, fileSetDir, false); + tsFileSet2.appendSchemaEvolution( + Collections.singletonList(new ColumnRename("table0", "s1", "s0"))); + tsFileManager.addTsFileSet(tsFileSet2, 0); + + // file3: + // table0[s0, s2, s3] + // table2[s1, s2, s3] + File f3 = new File(SEQ_DIRS, "0-3-0-0.tsfile"); + TableSchema tableSchema3_1 = + new TableSchema( + "table0", + Arrays.asList( + new ColumnSchemaBuilder() + .name("s0") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build(), + new ColumnSchemaBuilder() + .name("s2") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build(), + new ColumnSchemaBuilder() + .name("s3") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build())); + TableSchema tableSchema3_2 = + new TableSchema( + "table2", + Arrays.asList( + new ColumnSchemaBuilder() + .name("s1") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build(), + new ColumnSchemaBuilder() + .name("s2") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build(), + new ColumnSchemaBuilder() + .name("s3") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build())); + try (TsFileWriter tsFileWriter = new TsFileWriter(f3)) { + tsFileWriter.registerTableSchema(tableSchema3_1); + tsFileWriter.registerTableSchema(tableSchema3_2); + + Tablet tablet1 = new Tablet(tableSchema3_1.getTableName(), tableSchema3_1.getColumnSchemas()); + tablet1.addTimestamp(0, 2); + tablet1.addValue(0, 0, 21); + tablet1.addValue(0, 1, 22); + tablet1.addValue(0, 2, 23); + + Tablet tablet2 = new Tablet(tableSchema3_2.getTableName(), tableSchema3_2.getColumnSchemas()); + tablet2.addTimestamp(0, 2); + tablet2.addValue(0, 0, 121); + tablet2.addValue(0, 1, 122); + tablet2.addValue(0, 2, 123); + + tsFileWriter.writeTable(tablet1); + tsFileWriter.writeTable(tablet2); + } + TsFileResource resource3 = new TsFileResource(f3); + resource3.setTsFileManager(tsFileManager); + resource3.updateStartTime(Factory.DEFAULT_FACTORY.create(new String[]{"table0"}), 2); + resource3.updateEndTime(Factory.DEFAULT_FACTORY.create(new String[]{"table0"}), 2); + resource3.updateStartTime(Factory.DEFAULT_FACTORY.create(new String[]{"table2"}), 2); + resource3.updateEndTime(Factory.DEFAULT_FACTORY.create(new String[]{"table2"}), 2); + resource3.close(); + + // rename table2 -> table1 + TsFileSet tsFileSet3 = new TsFileSet(3, fileSetDir, false); + tsFileSet3.appendSchemaEvolution( + Collections.singletonList(new TableRename("table2", "table1"))); + tsFileManager.addTsFileSet(tsFileSet3, 0); + + // perform compaction + seqResources.add(resource1); + seqResources.add(resource2); + seqResources.add(resource3); + + List targetResources = + CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources(seqResources, true); + targetResources.forEach(s -> s.setTsFileManager(tsFileManager)); + + ICompactionPerformer performer = + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources); + performer.setSummary(new CompactionTaskSummary()); + performer.perform(); + + // target(version=1): + // table1[s1, s2, s3] + // table2[s1, s2, s3] + try (ITsFileReader tsFileReader = + new TsFileReaderBuilder().file(targetResources.get(0).getTsFile()).build()) { + // table1 should not exist + try { + tsFileReader.query("table0", Collections.singletonList("s2"), Long.MIN_VALUE, Long.MAX_VALUE); + fail("table0 should not exist"); + } catch (NoTableException e) { + assertEquals("Table table0 not found", e.getMessage()); + } + + // table1.s0 should not exist + try { + tsFileReader.query("table1", Collections.singletonList("s0"), Long.MIN_VALUE, Long.MAX_VALUE); + fail("table1.s0 should not exist"); + } catch (NoMeasurementException e) { + assertEquals("No measurement for s0", e.getMessage()); + } + + // check data of table1 + ResultSet resultSet = tsFileReader.query("table1", Arrays.asList("s1", "s2", "s3"), + Long.MIN_VALUE, Long.MAX_VALUE); + for (int i = 0; i < 3; i++) { + assertTrue(resultSet.next()); + assertEquals(i, resultSet.getLong(1)); + for (int j = 0; j < 3; j++) { + assertEquals(i * 10 + j + 1, resultSet.getLong(j + 2)); + } + } + + // check data of table2 + resultSet = tsFileReader.query("table2", Arrays.asList("s1", "s2", "s3"), + Long.MIN_VALUE, Long.MAX_VALUE); + for (int i = 0; i < 3; i++) { + assertTrue(resultSet.next()); + assertEquals(i, resultSet.getLong(1)); + for (int j = 0; j < 3; j++) { + assertEquals(100 + i * 10 + j + 1, resultSet.getLong(j + 2)); + } + } + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java index d3851e4e48ef2..8aca64683447f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java @@ -538,8 +538,8 @@ public static List getAllPathsOfResources(List resour Pair iDeviceIDBooleanPair = deviceIterator.nextDevice(); IDeviceID deviceID = iDeviceIDBooleanPair.getLeft(); boolean isAlign = iDeviceIDBooleanPair.getRight(); - Map schemaMap = deviceIterator.getAllSchemasOfCurrentDevice( - new Pair<>(Long.MIN_VALUE, null)); + Map schemaMap = + deviceIterator.getAllSchemasOfCurrentDevice(new Pair<>(Long.MIN_VALUE, null)); IMeasurementSchema timeSchema = schemaMap.remove(TsFileConstant.TIME_COLUMN_ID); List measurementSchemas = new ArrayList<>(schemaMap.values()); if (measurementSchemas.isEmpty()) { From d3d21df26e4222477bbba2a33e1af701e0736be8 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 12 Jan 2026 09:39:31 +0800 Subject: [PATCH 19/49] bump tsfile version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 71e0f017c5df2..73b7774f5ddf5 100644 --- a/pom.xml +++ b/pom.xml @@ -173,7 +173,7 @@ 0.14.1 1.9 1.5.6-3 - 2.2.0-251113-SNAPSHOT + 2.2.1-sevo-SNAPSHOT