diff --git a/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileTableGenerator.java b/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileTableGenerator.java index 7c2bd1623deb8..613ea8bd15984 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileTableGenerator.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileTableGenerator.java @@ -37,6 +37,7 @@ import java.io.File; import java.io.IOException; import java.time.LocalDate; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -81,15 +82,30 @@ public void registerTable( public void generateData(final String tableName, final int number, final long timeGap) throws IOException, WriteProcessException { final List schemas = table2MeasurementSchema.get(tableName); + final List columnCategoryList = table2ColumnCategory.get(tableName); + int timeIndex = -1; + for (int i = 0; i < columnCategoryList.size(); ++i) { + if (columnCategoryList.get(i) == ColumnCategory.TIME) { + timeIndex = i; + break; + } + } + final List schemaWithoutTime = new ArrayList<>(schemas); + final List columnCategoriesWithoutTime = new ArrayList<>(columnCategoryList); + if (timeIndex > -1) { + schemaWithoutTime.remove(timeIndex); + columnCategoriesWithoutTime.remove(timeIndex); + } final List columnNameList = - schemas.stream().map(IMeasurementSchema::getMeasurementName).collect(Collectors.toList()); + schemaWithoutTime.stream() + .map(IMeasurementSchema::getMeasurementName) + .collect(Collectors.toList()); final List dataTypeList = - schemas.stream().map(IMeasurementSchema::getType).collect(Collectors.toList()); - final List columnCategoryList = table2ColumnCategory.get(tableName); + schemaWithoutTime.stream().map(IMeasurementSchema::getType).collect(Collectors.toList()); final TreeSet timeSet = table2TimeSet.get(tableName); - final Tablet tablet = new Tablet(tableName, columnNameList, dataTypeList, columnCategoryList); - final Object[] values = tablet.getValues(); - final long sensorNum = schemas.size(); + final Tablet tablet = + new Tablet(tableName, columnNameList, dataTypeList, columnCategoriesWithoutTime); + final long sensorNum = schemaWithoutTime.size(); long startTime = timeSet.isEmpty() ? 0L : timeSet.last(); for (long r = 0; r < number; r++) { @@ -98,7 +114,7 @@ public void generateData(final String tableName, final int number, final long ti tablet.addTimestamp(row, startTime); timeSet.add(startTime); for (int i = 0; i < sensorNum; i++) { - generateDataPoint(tablet, i, row, schemas.get(i)); + generateDataPoint(tablet, i, row, schemaWithoutTime.get(i)); } // write if (tablet.getRowSize() == tablet.getMaxRowNumber()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstByIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstByIT.java index 490f8f52262cf..2ae5c20b56fcc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstByIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstByIT.java @@ -18,10 +18,12 @@ */ package org.apache.iotdb.db.it.query; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.TableClusterIT; import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -29,8 +31,14 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) @@ -103,11 +111,65 @@ public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + @Test + public void TestPriority() { + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + String[] retArray = {"1,1,1.0,1.0,true,1s,"}; + tableResultSetEqualTest( + "select " + + "first_by(s_int, y_criteria), " + + "first_by(s_long, y_criteria), " + + "first_by(s_float, y_criteria), " + + "first_by(s_double, y_criteria), " + + "first_by(s_bool, y_criteria), " + + "first_by(s_string, y_criteria) " + + "from table_a " + + "where table_a.device='d1'", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testNoTimeStamp() { + + String sql = + "select " + + "first_by(s_int, y_criteria)" + + " from (" + + " select " + + " s_int," + + " s_long," + + " s_float," + + " y_criteria" + + " from table_a" + + ") AS t"; + try (Connection connection = + EnvFactory.getEnv() + .getConnection( + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + BaseEnv.TABLE_SQL_DIALECT)) { + connection.setClientInfo("time_zone", "+00:00"); + try (Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + statement.executeQuery(sql); + } + fail("Missing valid time column, the query should fail"); + } catch (SQLException e) { + assertEquals( + "701: Missing valid time column. The table must contain either a column with the TIME category or at least one TIMESTAMP column.", + e.getMessage()); + } + } + @Test public void testFirstBy_d1_NoNulls() { String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; String[] retArray = {"5,5,5.0,5.0,false,5s,"}; runTest("d1", expectedHeader, retArray); + runTest2("d1", expectedHeader, retArray); + runTest3("d1", expectedHeader, retArray); } @Test @@ -115,6 +177,8 @@ public void testFirstBy_d2_ForwardTracking() { String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; String[] retArray = {"10,10,10.0,10.0,true,10s,"}; runTest("d2", expectedHeader, retArray); + runTest2("d2", expectedHeader, retArray); + runTest3("d2", expectedHeader, retArray); } @Test @@ -122,6 +186,8 @@ public void testFirstBy_d3_TargetNull() { String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; String[] retArray = {"5,5,null,null,null,null,"}; runTest("d3", expectedHeader, retArray); + runTest2("d3", expectedHeader, retArray); + runTest3("d3", expectedHeader, retArray); } @Test @@ -130,6 +196,8 @@ public void testFirstBy_d4_AllNullCriteria() { // Expected: No valid s2 found. String[] retArray = {"null,null,null,null,null,null,"}; runTest("d4", expectedHeader, retArray); + runTest2("d4", expectedHeader, retArray); + runTest3("d4", expectedHeader, retArray); } @Test @@ -138,6 +206,8 @@ public void testFirstBy_d5_AllTimeNull() { // Expected: The row with y_criteria=NULL is skipped. The row with y_criteria=50 is picked. String[] retArray = {"50,50,50.0,50.0,false,50s,"}; runTest("d5", expectedHeader, retArray); + runTest2("d5", expectedHeader, retArray); + runTest3("d5", expectedHeader, retArray); } private void runTest(String deviceId, String[] expectedHeader, String[] retArray) { @@ -159,4 +229,43 @@ private void runTest(String deviceId, String[] expectedHeader, String[] retArray retArray, DATABASE_NAME); } + + private void runTest2(String deviceId, String[] expectedHeader, String[] retArray) { + tableResultSetEqualTest( + "select " + + "first_by(s_int, y_criteria), " + + "first_by(s_long, y_criteria), " + + "first_by(s_float, y_criteria), " + + "first_by(s_double, y_criteria), " + + "first_by(s_bool, y_criteria), " + + "first_by(s_string, y_criteria) " + + "from " + + "(select s_int, s_long, s_float, s_double, s_bool, s_string, y_criteria, time_type " + + "from table_a left join table_b on table_a.time=table_b.time " + + "where table_a.device='" + + deviceId + + "') ", + expectedHeader, + retArray, + DATABASE_NAME); + } + + /** Test 3: Raw table query with explicit time_type column, 3 arguments */ + private void runTest3(String deviceId, String[] expectedHeader, String[] retArray) { + tableResultSetEqualTest( + "select " + + "first_by(s_int, y_criteria, time_type), " + + "first_by(s_long, y_criteria, time_type), " + + "first_by(s_float, y_criteria, time_type), " + + "first_by(s_double, y_criteria, time_type), " + + "first_by(s_bool, y_criteria, time_type), " + + "first_by(s_string, y_criteria, time_type) " + + "from table_a " + + "where table_a.device='" + + deviceId + + "'", + expectedHeader, + retArray, + DATABASE_NAME); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstByInGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstByInGroupIT.java index 1045d76eba8fa..f2f3496cd8a71 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstByInGroupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstByInGroupIT.java @@ -117,6 +117,7 @@ public void testGroupedFirstByAggregation() { "p5,50,50,50.0,50.0,false,50s," }; + // 1. eliminate the identity of time column tableResultSetEqualTest( "select " + "partition, " @@ -135,5 +136,42 @@ public void testGroupedFirstByAggregation() { expectedHeader, retArray, DATABASE_NAME); + + // 2. lack of the third argument, supply with timestamp column + tableResultSetEqualTest( + "select " + + "partition, " + + "first_by(s_int, y_criteria), " + + "first_by(s_long, y_criteria), " + + "first_by(s_float, y_criteria), " + + "first_by(s_double, y_criteria), " + + "first_by(s_bool, y_criteria), " + + "first_by(s_string, y_criteria) " + + "from " + // SubQuery: Rename time_type to 'ts' to avoid ambiguity with physical 'time' + + "(select s_int, s_long, s_float, s_double, s_bool, s_string, y_criteria, partition, time_type " + + "from table_a left join table_b on table_a.time=table_b.time) " + + "group by partition " + + "order by partition", + expectedHeader, + retArray, + DATABASE_NAME); + + // 3. base table query with column that with timestamp datatype + tableResultSetEqualTest( + "select " + + "partition, " + + "first_by(s_int, y_criteria, time_type), " + + "first_by(s_long, y_criteria, time_type), " + + "first_by(s_float, y_criteria, time_type), " + + "first_by(s_double, y_criteria, time_type), " + + "first_by(s_bool, y_criteria, time_type), " + + "first_by(s_string, y_criteria, time_type) " + + "from table_a " + + "group by partition " + + "order by partition", + expectedHeader, + retArray, + DATABASE_NAME); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstIT.java index 92fc15127b34c..1ce994a282349 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstIT.java @@ -19,10 +19,12 @@ package org.apache.iotdb.db.it.query; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.TableClusterIT; import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -30,8 +32,14 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) @@ -90,6 +98,59 @@ public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + // time column prior to field column with timestamp column + @Test + public void testPriority() { + + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + String[] retArray = {"-50,-50,-50.0,-50.0,false,-50s,"}; + tableResultSetEqualTest( + "select " + + "first(s_int), " + + "first(s_long), " + + "first(s_float), " + + "first(s_double), " + + "first(s_bool), " + + "first(s_string) " + + "from table_a " + + "where table_a.device='d1' ", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testNoTimeStamp() { + + String sql = + "select " + + "first(s_int)" + + " from (" + + " select " + + " s_int," + + " s_long," + + " s_float" + + " from table_a" + + ") AS t"; + try (Connection connection = + EnvFactory.getEnv() + .getConnection( + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + BaseEnv.TABLE_SQL_DIALECT)) { + connection.setClientInfo("time_zone", "+00:00"); + try (Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + statement.executeQuery(sql); + } + fail("Missing valid time column, the query should fail"); + } catch (SQLException e) { + assertEquals( + "701: Missing valid time column. The table must contain either a column with the TIME category or at least one TIMESTAMP column.", + e.getMessage()); + } + } + @Test public void testAggregation() { @@ -102,6 +163,7 @@ public void testAggregation() { // Corresponding values at time_type=5: s_int=5, s_long=5, ..., s_bool=false, s_string='5s' String[] retArray = {"5,5,5.0,5.0,false,5s,"}; + // 1. through join, eliminate the identity of time column tableResultSetEqualTest( "select " + "first(s_int, time), " @@ -121,6 +183,41 @@ public void testAggregation() { expectedHeader, retArray, DATABASE_NAME); + + // 2. lack of the second the argument + tableResultSetEqualTest( + "select " + + "first(s_int), " + + "first(s_long), " + + "first(s_float), " + + "first(s_double), " + + "first(s_bool), " + + "first(s_string) " + + "from " + + "(select " + + " time_type, " + + " s_int, s_long, s_float, s_double, s_bool, s_string " + + "from table_a " + + "left join table_b on table_a.time=table_b.time " + + "where table_a.device='d1') ", + expectedHeader, + retArray, + DATABASE_NAME); + + // 3. through field column that with timestamp datatype + tableResultSetEqualTest( + "select " + + "first(s_int, time_type), " + + "first(s_long, time_type), " + + "first(s_float, time_type), " + + "first(s_double, time_type), " + + "first(s_bool, time_type), " + + "first(s_string, time_type) " + + "from table_a " + + "where table_a.device='d1' ", + expectedHeader, + retArray, + DATABASE_NAME); } @Test @@ -154,6 +251,40 @@ public void testAggregationWithNullValue() { expectedHeader, retArray, DATABASE_NAME); + + tableResultSetEqualTest( + "select " + + "first(s_int), " + + "first(s_long), " + + "first(s_float), " + + "first(s_double), " + + "first(s_bool), " + + "first(s_string) " + + "from " + // subQuery: project all the column needed and rename the time_type to the time + + "(select " + + " time_type, " + + " s_int, s_long, s_float, s_double, s_bool, s_string " + + "from table_a " + + "left join table_b on table_a.time=table_b.time " + + "where table_a.device='d2') ", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select " + + "first(s_int, time_type), " + + "first(s_long, time_type), " + + "first(s_float, time_type), " + + "first(s_double, time_type), " + + "first(s_bool, time_type), " + + "first(s_string, time_type) " + + "from table_a " + + "where table_a.device='d2'", + expectedHeader, + retArray, + DATABASE_NAME); } @Test @@ -181,5 +312,38 @@ public void testAggregationWithAllNull() { expectedHeader, retArray, DATABASE_NAME); + + tableResultSetEqualTest( + "select " + + "first(s_int), " + + "first(s_long), " + + "first(s_float), " + + "first(s_double), " + + "first(s_bool), " + + "first(s_string) " + + "from " + + "(select " + + " time_type, " + + " s_int, s_long, s_float, s_double, s_bool, s_string " + + "from table_a " + + "left join table_b on table_a.time=table_b.time " + + "where table_a.device='d3') ", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select " + + "first(s_int, time_type), " + + "first(s_long, time_type), " + + "first(s_float, time_type), " + + "first(s_double, time_type), " + + "first(s_bool, time_type), " + + "first(s_string, time_type) " + + "from table_a " + + "where table_a.device='d3'", + expectedHeader, + retArray, + DATABASE_NAME); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstInGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstInGroupIT.java index 48decb65b8253..c6fa351023018 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstInGroupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstInGroupIT.java @@ -130,5 +130,43 @@ public void testGroupedFirstAggregation() { expectedHeader, retArray, DATABASE_NAME); + + tableResultSetEqualTest( + "select " + + "partition, " + + "first(s_int), " + + "first(s_long), " + + "first(s_float), " + + "first(s_double), " + + "first(s_bool), " + + "first(s_string) " + + "from " + + "(select " + + " time_type, " + + " partition, " + + " s_int, s_long, s_float, s_double, s_bool, s_string " + + "from table_a " + + "left join table_b on table_a.time=table_b.time) " + + "group by partition " + + "order by partition", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select " + + "partition, " + + "first(s_int, time_type), " + + "first(s_long, time_type), " + + "first(s_float, time_type), " + + "first(s_double, time_type), " + + "first(s_bool, time_type), " + + "first(s_string, time_type) " + + "from table_a " + + "group by partition " + + "order by partition", + expectedHeader, + retArray, + DATABASE_NAME); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastByIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastByIT.java index 42bef69eb44f4..1f30cbf30e249 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastByIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastByIT.java @@ -19,10 +19,12 @@ package org.apache.iotdb.db.it.query; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.TableClusterIT; import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -30,8 +32,14 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) @@ -105,11 +113,65 @@ public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + @Test + public void TestPriority() { + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + String[] retArray = {"99,99,99.0,99.0,true,99s,"}; + tableResultSetEqualTest( + "select " + + "last_by(s_int, y_criteria), " + + "last_by(s_long, y_criteria), " + + "last_by(s_float, y_criteria), " + + "last_by(s_double, y_criteria), " + + "last_by(s_bool, y_criteria), " + + "last_by(s_string, y_criteria) " + + "from table_a " + + "where table_a.device='d1'", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testNoTimeStamp() { + + String sql = + "select " + + "last_by(s_int, y_criteria)" + + " from (" + + " select " + + " s_int," + + " s_long," + + " s_float," + + " y_criteria" + + " from table_a" + + ") AS t"; + try (Connection connection = + EnvFactory.getEnv() + .getConnection( + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + BaseEnv.TABLE_SQL_DIALECT)) { + connection.setClientInfo("time_zone", "+00:00"); + try (Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + statement.executeQuery(sql); + } + fail("Missing valid time column, the query should fail"); + } catch (SQLException e) { + assertEquals( + "701: Missing valid time column. The table must contain either a column with the TIME category or at least one TIMESTAMP column.", + e.getMessage()); + } + } + @Test public void testLastBy_d1_NoNulls() { String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; String[] retArray = {"-5,-5,-5.0,-5.0,false,-5s,"}; runTest("d1", expectedHeader, retArray); + runTest2("d1", expectedHeader, retArray); + runTest3("d1", expectedHeader, retArray); } @Test @@ -117,6 +179,8 @@ public void testLastBy_d2_Backtracking() { String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; String[] retArray = {"-10,-10,-10.0,-10.0,true,-10s,"}; runTest("d2", expectedHeader, retArray); + runTest2("d2", expectedHeader, retArray); + runTest3("d2", expectedHeader, retArray); } @Test @@ -124,6 +188,8 @@ public void testLastBy_d3_TargetNull() { String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; String[] retArray = {"-5,-5,null,null,null,null,"}; runTest("d3", expectedHeader, retArray); + runTest2("d3", expectedHeader, retArray); + runTest3("d3", expectedHeader, retArray); } @Test @@ -132,6 +198,8 @@ public void testLastBy_d4_AllNullCriteria() { // Expected: No valid s2 found. String[] retArray = {"null,null,null,null,null,null,"}; runTest("d4", expectedHeader, retArray); + runTest2("d4", expectedHeader, retArray); + runTest3("d4", expectedHeader, retArray); } @Test @@ -140,6 +208,8 @@ public void testLastBy_d5_AllTimeNull() { // Expected: The row with y_criteria=NULL is skipped. The row with y_criteria=50 is picked. String[] retArray = {"50,50,50.0,50.0,false,50s,"}; runTest("d5", expectedHeader, retArray); + runTest2("d5", expectedHeader, retArray); + runTest3("d5", expectedHeader, retArray); } private void runTest(String deviceId, String[] expectedHeader, String[] retArray) { @@ -161,4 +231,42 @@ private void runTest(String deviceId, String[] expectedHeader, String[] retArray retArray, DATABASE_NAME); } + + private void runTest2(String deviceId, String[] expectedHeader, String[] retArray) { + tableResultSetEqualTest( + "select " + + "last_by(s_int, y_criteria), " + + "last_by(s_long, y_criteria), " + + "last_by(s_float, y_criteria), " + + "last_by(s_double, y_criteria), " + + "last_by(s_bool, y_criteria), " + + "last_by(s_string, y_criteria) " + + "from " + + "(select s_int, s_long, s_float, s_double, s_bool, s_string, y_criteria, time_type " + + "from table_a left join table_b on table_a.time=table_b.time " + + "where table_a.device='" + + deviceId + + "') ", + expectedHeader, + retArray, + DATABASE_NAME); + } + + private void runTest3(String deviceId, String[] expectedHeader, String[] retArray) { + tableResultSetEqualTest( + "select " + + "last_by(s_int, y_criteria, time_type), " + + "last_by(s_long, y_criteria, time_type), " + + "last_by(s_float, y_criteria, time_type), " + + "last_by(s_double, y_criteria, time_type), " + + "last_by(s_bool, y_criteria, time_type), " + + "last_by(s_string, y_criteria, time_type) " + + "from table_a " + + "where table_a.device='" + + deviceId + + "'", + expectedHeader, + retArray, + DATABASE_NAME); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastByInGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastByInGroupIT.java index 5b59e0875ed62..570e16535de6f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastByInGroupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastByInGroupIT.java @@ -132,5 +132,39 @@ public void testGroupedLastByAggregation() { expectedHeader, retArray, DATABASE_NAME); + + tableResultSetEqualTest( + "select " + + "partition, " + + "last_by(s_int, y_criteria), " + + "last_by(s_long, y_criteria), " + + "last_by(s_float, y_criteria), " + + "last_by(s_double, y_criteria), " + + "last_by(s_bool, y_criteria), " + + "last_by(s_string, y_criteria) " + + "from " + + "(select time_type, s_int, s_long, s_float, s_double, s_bool, s_string, y_criteria, partition " + + "from table_a left join table_b on table_a.time=table_b.time) " + + "group by partition " + + "order by partition", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select " + + "partition, " + + "last_by(s_int, y_criteria, time_type), " + + "last_by(s_long, y_criteria, time_type), " + + "last_by(s_float, y_criteria, time_type), " + + "last_by(s_double, y_criteria, time_type), " + + "last_by(s_bool, y_criteria, time_type), " + + "last_by(s_string, y_criteria, time_type) " + + "from table_a " + + "group by partition " + + "order by partition", + expectedHeader, + retArray, + DATABASE_NAME); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastIT.java index e9e860010f51a..3e132ab084558 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastIT.java @@ -19,10 +19,12 @@ package org.apache.iotdb.db.it.query; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.TableClusterIT; import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -30,8 +32,14 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) @@ -87,6 +95,57 @@ public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + @Test + public void TestPriority() { + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + String[] retArray = {"50,50,50.0,50.0,false,50s,"}; + tableResultSetEqualTest( + "select " + + "last(s_int), " + + "last(s_long), " + + "last(s_float), " + + "last(s_double), " + + "last(s_bool), " + + "last(s_string) " + + "from table_a " + + "where table_a.device='d1' ", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testNoTimeStamp() { + + String sql = + "select " + + "last(s_int)" + + " from (" + + " select " + + " s_int," + + " s_long," + + " s_float" + + " from table_a" + + ") AS t"; + try (Connection connection = + EnvFactory.getEnv() + .getConnection( + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + BaseEnv.TABLE_SQL_DIALECT)) { + connection.setClientInfo("time_zone", "+00:00"); + try (Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + statement.executeQuery(sql); + } + fail("Missing valid time column, the query should fail"); + } catch (SQLException e) { + assertEquals( + "701: Missing valid time column. The table must contain either a column with the TIME category or at least one TIMESTAMP column.", + e.getMessage()); + } + } + @Test public void testAggregation() { @@ -114,6 +173,38 @@ public void testAggregation() { expectedHeader, retArray, DATABASE_NAME); + + tableResultSetEqualTest( + "select " + + "last(s_int), " + + "last(s_long), " + + "last(s_float), " + + "last(s_double), " + + "last(s_bool), " + + "last(s_string) " + + "from " + + "(select " + + " s_int, s_long, s_float, s_double, s_bool, s_string, time_type " + + "from table_a " + + "left join table_b on table_a.time=table_b.time " + + "where table_a.device='d1') ", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select " + + "last(s_int, time_type), " + + "last(s_long, time_type), " + + "last(s_float, time_type), " + + "last(s_double, time_type), " + + "last(s_bool, time_type), " + + "last(s_string, time_type) " + + "from table_a " + + "where table_a.device='d1' ", + expectedHeader, + retArray, + DATABASE_NAME); } @Test @@ -143,6 +234,38 @@ public void testAggregationWithNullValue() { expectedHeader, retArray, DATABASE_NAME); + + tableResultSetEqualTest( + "select " + + "last(s_int), " + + "last(s_long), " + + "last(s_float), " + + "last(s_double), " + + "last(s_bool), " + + "last(s_string) " + + "from " + + "(select " + + " s_int, s_long, s_float, s_double, s_bool, s_string, time_type " + + "from table_a " + + "left join table_b on table_a.time=table_b.time " + + "where table_a.device='d2') ", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select " + + "last(s_int, time_type), " + + "last(s_long, time_type), " + + "last(s_float, time_type), " + + "last(s_double, time_type), " + + "last(s_bool, time_type), " + + "last(s_string, time_type) " + + "from table_a " + + "where table_a.device='d2'", + expectedHeader, + retArray, + DATABASE_NAME); } @Test @@ -172,5 +295,39 @@ public void testAggregationWithAllNull() { expectedHeader, retArray, DATABASE_NAME); + + tableResultSetEqualTest( + "select " + + "last(s_int), " + + "last(s_long), " + + "last(s_float), " + + "last(s_double), " + + "last(s_bool), " + + "last(s_string) " + + "from " + + "(select " + + " time_type, " + + " s_int, s_long, s_float, s_double, s_bool, s_string " + + "from table_a " + + "left join table_b on table_a.time=table_b.time " + + "where table_a.device='d3') ", + expectedHeader, + retArray, + DATABASE_NAME); + + // 3. through field column + tableResultSetEqualTest( + "select " + + "last(s_int, time_type), " + + "last(s_long, time_type), " + + "last(s_float, time_type), " + + "last(s_double, time_type), " + + "last(s_bool, time_type), " + + "last(s_string, time_type) " + + "from table_a " + + "where table_a.device='d3'", + expectedHeader, + retArray, + DATABASE_NAME); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastInGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastInGroupIT.java index 1f6b2c680ff13..b065200021db6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastInGroupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastInGroupIT.java @@ -124,5 +124,43 @@ public void testGroupedLastAggregation() { expectedHeader, retArray, DATABASE_NAME); + + tableResultSetEqualTest( + "select " + + "partition, " + + "last(s_int), " + + "last(s_long), " + + "last(s_float), " + + "last(s_double), " + + "last(s_bool), " + + "last(s_string) " + + "from " + + "(select " + + " time_type, " + + " partition, " + + " s_int, s_long, s_float, s_double, s_bool, s_string " + + "from table_a " + + "left join table_b on table_a.time=table_b.time) " + + "group by partition " + + "order by partition", + expectedHeader, + retArray, + DATABASE_NAME); + + tableResultSetEqualTest( + "select " + + "partition, " + + "last(s_int, time_type), " + + "last(s_long, time_type), " + + "last(s_float, time_type), " + + "last(s_double, time_type), " + + "last(s_bool, time_type), " + + "last(s_string, time_type) " + + "from table_a " + + "group by partition " + + "order by partition", + expectedHeader, + retArray, + DATABASE_NAME); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java index b3a601c83dbe6..b079d022e7946 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java @@ -535,6 +535,7 @@ public void testInsertMultiRowWithWrongTimestampPrecision() throws SQLException Statement st1 = connection.createStatement()) { try { st1.execute("use \"test\""); + st1.execute("CREATE TABLE wf16 (tag1 string tag, status boolean field)"); st1.execute( "insert into wf16(tag1, time, status) values('wt01', 1618283005586000, true), ('wt01', 1618283005586001, false)"); fail(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java index 5f88d50c9d90c..a58e0633b7425 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java @@ -47,7 +47,9 @@ import java.sql.ResultSet; import java.sql.Statement; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -116,6 +118,40 @@ private List> generateMeasurementSche return pairs; } + private List> generateMeasurementSchemasWithTime( + final int timeColumnIndex, final String timeColumnName) { + List dataTypes = + new ArrayList<>( + Arrays.asList( + TSDataType.STRING, + TSDataType.TEXT, + TSDataType.BLOB, + TSDataType.TIMESTAMP, + TSDataType.BOOLEAN, + TSDataType.DATE, + TSDataType.DOUBLE, + TSDataType.FLOAT, + TSDataType.INT32, + TSDataType.INT64)); + List> pairs = new ArrayList<>(); + for (TSDataType type : dataTypes) { + for (TSDataType dataType : dataTypes) { + String id = String.format("%s2%s", type.name(), dataType.name()); + pairs.add(new Pair<>(new MeasurementSchema(id, type), new MeasurementSchema(id, dataType))); + } + } + + if (timeColumnIndex >= 0) { + pairs.add( + timeColumnIndex, + new Pair<>( + new MeasurementSchema(timeColumnName, TSDataType.TIMESTAMP), + new MeasurementSchema(timeColumnName, TSDataType.TIMESTAMP))); + } + + return pairs; + } + @Test public void testLoadWithEmptyDatabaseForTableModel() throws Exception { final int lineCount = 10000; @@ -123,7 +159,7 @@ public void testLoadWithEmptyDatabaseForTableModel() throws Exception { final List> measurementSchemas = generateMeasurementSchemas(); final List columnCategories = - generateTabletColumnCategory(0, measurementSchemas.size()); + generateTabletColumnCategory(measurementSchemas.size()); final File file = new File(tmpDir, "1-0-0-0.tsfile"); @@ -177,8 +213,7 @@ public void testLoadWithConvertOnTypeMismatchForTableModel() throws Exception { List> measurementSchemas = generateMeasurementSchemas(); - List columnCategories = - generateTabletColumnCategory(0, measurementSchemas.size()); + List columnCategories = generateTabletColumnCategory(measurementSchemas.size()); final File file = new File(tmpDir, "1-0-0-0.tsfile"); @@ -219,8 +254,7 @@ public void testLoadWithTableMod() throws Exception { List> measurementSchemas = generateMeasurementSchemas(); - List columnCategories = - generateTabletColumnCategory(0, measurementSchemas.size()); + List columnCategories = generateTabletColumnCategory(measurementSchemas.size()); final File file = new File(tmpDir, "1-0-0-0.tsfile"); @@ -260,14 +294,175 @@ public void testLoadWithTableMod() throws Exception { } } - private List generateTabletColumnCategory(int tagNum, int filedNum) { - List columnTypes = new ArrayList<>(tagNum + filedNum); + @Test + public void testLoadWithTimeColumn() throws Exception { + final int lineCount = 10000; + + // from: 1 time + List> measurementSchemas = + generateMeasurementSchemasWithTime(1, "time"); + List columnCategories = + generateTabletColumnCategory(0, measurementSchemas.size(), 1); + + File file = new File(tmpDir, "1-0-0-0.tsfile"); + + List schemaList1 = + measurementSchemas.stream().map(pair -> pair.left).collect(Collectors.toList()); + + try (final TsFileTableGenerator generator = new TsFileTableGenerator(file)) { + generator.registerTable(SchemaConfig.TABLE_0, new ArrayList<>(schemaList1), columnCategories); + generator.generateData(SchemaConfig.TABLE_0, lineCount, PARTITION_INTERVAL / 10_000); + } + + // to: 1 time + testWithTimeColumn(lineCount, schemaList1, columnCategories, file); + // to: empty + testWithTimeColumn(lineCount, null, null, file); + + measurementSchemas = generateMeasurementSchemasWithTime(2, "time"); + columnCategories = generateTabletColumnCategory(0, measurementSchemas.size(), 2); + schemaList1 = measurementSchemas.stream().map(pair -> pair.left).collect(Collectors.toList()); + + // to: 2 time + testWithTimeColumn(lineCount, schemaList1, columnCategories, file); + + // to: 0 time + measurementSchemas = generateMeasurementSchemasWithTime(-1, "time"); + columnCategories = generateTabletColumnCategory(0, measurementSchemas.size(), -1); + schemaList1 = measurementSchemas.stream().map(pair -> pair.left).collect(Collectors.toList()); + testWithTimeColumn(lineCount, schemaList1, columnCategories, file); + + // to: 2 time1 + measurementSchemas = generateMeasurementSchemasWithTime(2, "time1"); + columnCategories = generateTabletColumnCategory(0, measurementSchemas.size(), 2); + schemaList1 = measurementSchemas.stream().map(pair -> pair.left).collect(Collectors.toList()); + testWithTimeColumn(lineCount, schemaList1, columnCategories, file); + + // from: 2 time1 + file = new File(tmpDir, "2-0-0-0.tsfile"); + try (final TsFileTableGenerator generator = new TsFileTableGenerator(file)) { + generator.registerTable(SchemaConfig.TABLE_0, new ArrayList<>(schemaList1), columnCategories); + generator.generateData(SchemaConfig.TABLE_0, lineCount, PARTITION_INTERVAL / 10_000); + } + + // to: 2 time + measurementSchemas = generateMeasurementSchemasWithTime(2, "time"); + columnCategories = generateTabletColumnCategory(0, measurementSchemas.size(), 2); + schemaList1 = measurementSchemas.stream().map(pair -> pair.left).collect(Collectors.toList()); + testWithTimeColumn(lineCount, schemaList1, columnCategories, file); + + // to: 1 time + measurementSchemas = generateMeasurementSchemasWithTime(1, "time1"); + columnCategories = generateTabletColumnCategory(0, measurementSchemas.size(), 1); + schemaList1 = measurementSchemas.stream().map(pair -> pair.left).collect(Collectors.toList()); + testWithTimeColumn(lineCount, schemaList1, columnCategories, file); + + // to: empty + testWithTimeColumn(lineCount, null, null, file); + + // to: 0 time + measurementSchemas = generateMeasurementSchemasWithTime(-1, "time"); + columnCategories = generateTabletColumnCategory(0, measurementSchemas.size(), -1); + schemaList1 = measurementSchemas.stream().map(pair -> pair.left).collect(Collectors.toList()); + testWithTimeColumn(lineCount, schemaList1, columnCategories, file); + + // from: 0 time + file = new File(tmpDir, "3-0-0-0.tsfile"); + try (final TsFileTableGenerator generator = new TsFileTableGenerator(file)) { + generator.registerTable(SchemaConfig.TABLE_0, new ArrayList<>(schemaList1), columnCategories); + generator.generateData(SchemaConfig.TABLE_0, lineCount, PARTITION_INTERVAL / 10_000); + } + + // to: 2 time + measurementSchemas = generateMeasurementSchemasWithTime(2, "time"); + columnCategories = generateTabletColumnCategory(0, measurementSchemas.size(), 2); + schemaList1 = measurementSchemas.stream().map(pair -> pair.left).collect(Collectors.toList()); + testWithTimeColumn(lineCount, schemaList1, columnCategories, file); + + // to: 1 time1 + measurementSchemas = generateMeasurementSchemasWithTime(1, "time1"); + columnCategories = generateTabletColumnCategory(0, measurementSchemas.size(), 1); + schemaList1 = measurementSchemas.stream().map(pair -> pair.left).collect(Collectors.toList()); + testWithTimeColumn(lineCount, schemaList1, columnCategories, file); + + // to: 0 time + measurementSchemas = generateMeasurementSchemasWithTime(-1, "time"); + columnCategories = generateTabletColumnCategory(0, measurementSchemas.size(), -1); + schemaList1 = measurementSchemas.stream().map(pair -> pair.left).collect(Collectors.toList()); + testWithTimeColumn(lineCount, schemaList1, columnCategories, file); + + // to: empty + testWithTimeColumn(lineCount, null, null, file); + } + + private void testWithTimeColumn( + final long lineCount, + final List schemaList1, + final List columnCategories, + final File file) + throws Exception { + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(String.format("create database if not exists %s", SchemaConfig.DATABASE_0)); + statement.execute(String.format("use %s", SchemaConfig.DATABASE_0)); + ResultSet resultSetOld = null; + if (Objects.nonNull(schemaList1)) { + statement.execute(convert2TableSQL(SchemaConfig.TABLE_0, schemaList1, columnCategories)); + resultSetOld = statement.executeQuery("desc " + SchemaConfig.TABLE_0); + } + statement.execute( + String.format( + "load '%s' with ('database'='%s')", file.getAbsolutePath(), SchemaConfig.DATABASE_0)); + try (final ResultSet resultSet = + statement.executeQuery(String.format("select count(*) from %s", SchemaConfig.TABLE_0))) { + if (resultSet.next()) { + Assert.assertEquals(lineCount, resultSet.getLong(1)); + } else { + Assert.fail("This ResultSet is empty."); + } + } + + try (final ResultSet resultSet = statement.executeQuery("show tables")) { + Assert.assertTrue(resultSet.next()); + Assert.assertFalse(resultSet.next()); + } + + // Time column's difference shall not affect the old column + if (Objects.nonNull(resultSetOld)) { + try (final ResultSet resultSet = statement.executeQuery("desc " + SchemaConfig.TABLE_0)) { + while (resultSet.next() && resultSetOld.next()) { + Assert.assertEquals(resultSet.getString(1), resultSetOld.getString(1)); + Assert.assertEquals(resultSet.getString(2), resultSetOld.getString(2)); + Assert.assertEquals(resultSet.getString(3), resultSetOld.getString(3)); + } + if (resultSet.next() || resultSetOld.next()) { + Assert.fail("The table schema has changed after load."); + } + } + } + + statement.execute(String.format("drop database %s", SchemaConfig.DATABASE_0)); + } + } + + private List generateTabletColumnCategory(final int fieldNum) { + return generateTabletColumnCategory(0, fieldNum, -1); + } + + private List generateTabletColumnCategory( + final int tagNum, final int fieldNum, final int timeIndex) { + List columnTypes = + new ArrayList<>(tagNum + fieldNum + (timeIndex >= 0 ? 1 : 0)); for (int i = 0; i < tagNum; i++) { columnTypes.add(ColumnCategory.TAG); } - for (int i = 0; i < filedNum; i++) { + for (int i = 0; i < fieldNum; i++) { columnTypes.add(ColumnCategory.FIELD); } + if (timeIndex >= 0) { + columnTypes.add(timeIndex, ColumnCategory.TIME); + } return columnTypes; } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/IoTDBRestServiceInsertValuesIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/IoTDBRestServiceInsertValuesIT.java index d4c95063bab8e..1e2a82ad1f3fb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/IoTDBRestServiceInsertValuesIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/rest/it/IoTDBRestServiceInsertValuesIT.java @@ -284,7 +284,7 @@ public void testInsertWithWrongMeasurementNum1() { public void testInsertWithWrongMeasurementNum2() { nonQuery( sqlHandler( - "t1", "create table wf04 (tag1 string tag, status int32, temperature int32 field)")); + "t1", "create table wf05 (tag1 string tag, status int32, temperature int32 field)")); JsonObject jsonObject = nonQuery( sqlHandler( diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java index 9e322b840ae82..9730c909ba2be 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java @@ -369,10 +369,13 @@ public void testInformationSchema() throws SQLException { Arrays.asList( "create database information_schema", "drop database information_schema", - "create table information_schema.tableA ()", + // table in information_schema do not have the time column, add time column just + // for simulate the base table + "create table information_schema.tableA (time timestamp time)", "alter table information_schema.tableA add column a id", "alter table information_schema.tableA set properties ttl=default", - "insert into information_schema.tables (database) values('db')", + // given that create table in information_schema is not allowed, skip insert + // "insert into information_schema.tables (database) values('db')", "update information_schema.tables set status='RUNNING'")); for (final String writeSQL : writeSQLs) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java index 0923877d9771b..57d482dd1b057 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java @@ -610,31 +610,9 @@ public void testManageTable() { } // Test time column + // More time column tests are included in other IT statement.execute("create table test100 (time time)"); statement.execute("create table test101 (time timestamp time)"); - - try { - statement.execute("create table test102 (time timestamp tag)"); - fail(); - } catch (final SQLException e) { - assertEquals( - "701: The time column category shall be bounded with column name 'time'.", - e.getMessage()); - } - - try { - statement.execute("create table test102 (time tag)"); - fail(); - } catch (final SQLException e) { - assertEquals("701: The time column's type shall be 'timestamp'.", e.getMessage()); - } - - try { - statement.execute("create table test102 (time time, time time)"); - fail(); - } catch (final SQLException e) { - assertEquals("701: Columns in table shall not share the same name time.", e.getMessage()); - } } catch (final SQLException e) { e.printStackTrace(); fail(e.getMessage()); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableWithDefinedTimeIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableWithDefinedTimeIT.java new file mode 100644 index 0000000000000..0d37085ad484b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableWithDefinedTimeIT.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.schema; + +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBTableWithDefinedTimeIT { + + private static final String TABLE_DATABASE = "user_defined_time"; + private static final String VIEW_DATABASE = "user_defined_time_for_view"; + private static final String[] SQLS = + new String[] {"CREATE DATABASE " + TABLE_DATABASE, "CREATE DATABASE " + VIEW_DATABASE}; + private final String header = "ColumnName,DataType,Category,"; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testCreateTable() { + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use " + TABLE_DATABASE); + + // create table and do not assign the time column name + try { + statement.execute( + "create table default_not_assign_time(device string tag, s1 int32 field)"); + TestUtils.assertResultSetEqual( + statement.executeQuery("describe default_not_assign_time"), + header, + Arrays.asList("time,TIMESTAMP,TIME,", "device,STRING,TAG,", "s1,INT32,FIELD,")); + } catch (SQLException e) { + fail("create table without time info fails, the specific message: " + e.getMessage()); + } + + // create table and assign the time column name + try { + statement.execute( + "create table time_in_first(date_time timestamp time, device string tag, s1 int32 field)"); + TestUtils.assertResultSetEqual( + statement.executeQuery("describe time_in_first"), + header, + Arrays.asList("date_time,TIMESTAMP,TIME,", "device,STRING,TAG,", "s1,INT32,FIELD,")); + } catch (SQLException e) { + fail("assign the name of time column fails, the specific message: " + e.getMessage()); + } + + // create table which of the time column not at the first column + try { + statement.execute( + "create table time_not_in_first(device string tag, date_time timestamp time, s1 int32 field)"); + TestUtils.assertResultSetEqual( + statement.executeQuery("describe time_not_in_first"), + header, + Arrays.asList("device,STRING,TAG,", "date_time,TIMESTAMP,TIME,", "s1,INT32,FIELD,")); + } catch (SQLException e) { + fail("assign the name of time column fails, the specific message: " + e.getMessage()); + } + + // create table with multi time-column + try { + statement.execute( + "create table with_multi_time(time_type timestamp time, device string tag, date_time timestamp time, s1 int32 field)"); + fail("Creating table is not be allowed to assign two time columns"); + } catch (SQLException e) { + assertEquals("701: A table cannot have more than one time column", e.getMessage()); + } + + // create table with time column that is not timestamp data type + try { + statement.execute( + "create table time_other_type(device string tag, date_time int64 time, s1 int32 field)"); + fail("The time column has to be assigned a timestamp data type when creating table"); + } catch (SQLException e) { + assertEquals("701: The time column's type shall be 'timestamp'.", e.getMessage()); + } + + // Columns in table shall not share the same name time when creating table + try { + statement.execute( + "create table shared_time_name(device string tag, time int64 field, s1 int32 field)"); + fail("Columns in table shall not share the same name time when creating table"); + } catch (SQLException e) { + assertEquals( + "701: Columns in table shall not share the same name: 'time'.", e.getMessage()); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private void prepareTreeData() { + try (final Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("create timeseries root.tt.device.s1 with datatype=int32"); + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testCreateView() { + prepareTreeData(); + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use " + VIEW_DATABASE); + + // create view and do not assign the time column name + try { + statement.execute( + "create view default_not_assign_time(device string tag, s1 int32 field) as root.tt.**"); + TestUtils.assertResultSetEqual( + statement.executeQuery("describe default_not_assign_time"), + header, + Arrays.asList("time,TIMESTAMP,TIME,", "device,STRING,TAG,", "s1,INT32,FIELD,")); + } catch (SQLException e) { + fail("create table without time info fails, the specific message: " + e.getMessage()); + } + + // create view which of the time column at the first column + try { + statement.execute( + "create view time_in_first(date_time timestamp time, device string tag, s1 int32 field) as root.tt.**"); + TestUtils.assertResultSetEqual( + statement.executeQuery("describe time_in_first"), + header, + Arrays.asList("date_time,TIMESTAMP,TIME,", "device,STRING,TAG,", "s1,INT32,FIELD,")); + } catch (SQLException e) { + fail("assign the name of time column fails, the specific message: " + e.getMessage()); + } + + // create view which of the time column not at the first column + try { + statement.execute( + "create view time_not_in_first(device string tag, date_time timestamp time, s1 int32 field) as root.tt.**"); + TestUtils.assertResultSetEqual( + statement.executeQuery("describe time_not_in_first"), + header, + Arrays.asList("device,STRING,TAG,", "date_time,TIMESTAMP,TIME,", "s1,INT32,FIELD,")); + } catch (SQLException e) { + fail("assign the name of time column fails, the specific message: " + e.getMessage()); + } + + // create view with multi time-column + try { + statement.execute( + "create view with_multi_time(time_type timestamp time, device string tag, date_time timestamp time, s1 int32 field) as root.tt.**"); + fail("Creating view is not be allowed to assign two time columns"); + } catch (SQLException e) { + assertEquals("701: A table cannot have more than one time column", e.getMessage()); + } + + // create table with time column that is not timestamp data type + try { + statement.execute( + "create view time_other_type(device string tag, date_time int64 time, s1 int32 field) as root.tt.**"); + fail("The time column has to be assigned a timestamp data type when creating view"); + } catch (SQLException e) { + assertEquals("701: The time column's type shall be 'timestamp'.", e.getMessage()); + } + + // Columns in table shall not share the same name time when creating table + try { + statement.execute( + "create view shared_time_time(device string tag, time int64 field, s1 int32 field) as root.tt.**"); + fail("Columns in view shall not share the same name time when creating table"); + } catch (SQLException e) { + assertEquals( + "701: Columns in table shall not share the same name: 'time'.", e.getMessage()); + } + + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index 2725053b84398..6b47773813e31 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -35,7 +35,6 @@ import org.apache.iotdb.commons.schema.table.TableNodeStatus; import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.TsTable; -import org.apache.iotdb.commons.schema.table.column.TimeColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.utils.MetadataUtils; @@ -98,7 +97,6 @@ import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; import static org.apache.iotdb.commons.schema.SchemaConstant.ROOT; import static org.apache.iotdb.commons.schema.SchemaConstant.TABLE_MNODE_TYPE; -import static org.apache.iotdb.commons.schema.table.TsTable.TIME_COLUMN_NAME; // Since the ConfigMTree is all stored in memory, thus it is not restricted to manage MNode through // MTreeStore. @@ -803,20 +801,14 @@ public void setTableColumnComment( throws MetadataException { final TsTable table = getTable(database, tableName); - final TsTableColumnSchema columnSchema = - !columnName.equals(TIME_COLUMN_NAME) || Objects.isNull(comment) - ? table.getColumnSchema(columnName) - : new TimeColumnSchema(TIME_COLUMN_NAME, TSDataType.TIMESTAMP); + final TsTableColumnSchema columnSchema = table.getColumnSchema(columnName); + if (Objects.isNull(columnSchema)) { throw new ColumnNotExistsException( PathUtils.unQualifyDatabaseName(database.getFullPath()), tableName, columnName); } if (Objects.nonNull(comment)) { columnSchema.getProps().put(TsTable.COMMENT_KEY, comment); - if (columnName.equals("time")) { - // Replace the original time column - table.addColumnSchema(columnSchema); - } } else { columnSchema.getProps().remove(TsTable.COMMENT_KEY); } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTreeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTreeTest.java index bf7d0c91ba6f0..6f2df1e129ae5 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTreeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTreeTest.java @@ -401,7 +401,8 @@ public void testTableSerialization() throws Exception { final TsTable table = tables.get(0); assertEquals("table" + i, table.getTableName()); assertEquals(1, table.getTagNum()); - assertEquals(4, table.getColumnNum()); + // currently, only construct the TsTable would not carry the time column + assertEquals(3, table.getColumnNum()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TableInsertTabletStatementGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TableInsertTabletStatementGenerator.java index a8987d652fc57..aed6cd3a4c714 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TableInsertTabletStatementGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TableInsertTabletStatementGenerator.java @@ -35,8 +35,6 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicLong; -import static org.apache.iotdb.commons.schema.table.TsTable.TIME_COLUMN_NAME; - public class TableInsertTabletStatementGenerator extends InsertTabletStatementGenerator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(TableInsertTabletStatementGenerator.class); @@ -54,13 +52,14 @@ public TableInsertTabletStatementGenerator( List inputColumnTypes, List tsTableColumnCategories, boolean isAligned, - int rowLimit) { + int rowLimit, + String timeColumnName) { super( targetTable, measurementToDataTypeMap.keySet().toArray(new String[0]), measurementToDataTypeMap.values().toArray(new TSDataType[0]), measurementToInputLocationMap.entrySet().stream() - .filter(entry -> !entry.getKey().equalsIgnoreCase(TIME_COLUMN_NAME)) + .filter(entry -> !entry.getKey().equalsIgnoreCase(timeColumnName)) .map(Map.Entry::getValue) .toArray(InputLocation[]::new), inputColumnTypes.stream().map(TypeFactory::getType).toArray(Type[]::new), @@ -69,8 +68,7 @@ public TableInsertTabletStatementGenerator( this.databaseName = databaseName; this.writtenCounter = new AtomicLong(0); this.columnCategories = tsTableColumnCategories.toArray(new TsTableColumnCategory[0]); - this.timeColumnIndex = - measurementToInputLocationMap.get(TIME_COLUMN_NAME).getValueColumnIndex(); + this.timeColumnIndex = measurementToInputLocationMap.get(timeColumnName).getValueColumnIndex(); this.reset(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TableIntoOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TableIntoOperator.java index 40b1781b69d4b..bf6f9c5bc94af 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TableIntoOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TableIntoOperator.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.Operator; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.InputLocation; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import com.google.common.util.concurrent.Futures; @@ -67,7 +68,8 @@ public TableIntoOperator( Map measurementToDataTypeMap, boolean isAligned, ExecutorService intoOperationExecutor, - long statementSizePerLine) { + long statementSizePerLine, + ColumnSchema timeColumnOfTargetTable) { super(operatorContext, child, inputColumnTypes, intoOperationExecutor, statementSizePerLine); this.maxReturnSize = MAX_RETURN_SIZE; insertTabletStatementGenerator = @@ -79,7 +81,8 @@ public TableIntoOperator( inputColumnTypes, inputColumnCategories, isAligned, - maxRowNumberInStatement); + maxRowNumberInStatement, + timeColumnOfTargetTable.getName()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java index 45617de31d1c2..cef6c365c915f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java @@ -78,6 +78,7 @@ import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinAggregationFunction.FIRST; import static org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinAggregationFunction.FIRST_BY; import static org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinAggregationFunction.LAST; import static org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinAggregationFunction.LAST_BY; @@ -100,6 +101,8 @@ public static TableAccumulator createAccumulator( boolean distinct) { TableAccumulator result; + // Input expression size of 1 indicates aggregation split has occurred and this is a final + // aggregation if (aggregationType == TAggregationType.UDAF) { // If UDAF accumulator receives raw input, it needs to check input's attribute result = createUDAFAccumulator(functionName, inputDataTypes, inputAttributes); @@ -108,11 +111,14 @@ public static TableAccumulator createAccumulator( && inputExpressions.size() > 1) { boolean xIsTimeColumn = isTimeColumn(inputExpressions.get(0), timeColumnName); boolean yIsTimeColumn = isTimeColumn(inputExpressions.get(1), timeColumnName); - // When used in AggTableScanOperator, we can finish calculation of + boolean orderKeyIsTimeColumn = isTimeColumn(inputExpressions.get(2), timeColumnName); + + // When used in AggTableScanOperator and the order column is time column, we can finish + // calculation of // LastDesc/LastByDesc/First/First_by after the result has been initialized if (LAST_BY.getFunctionName().equals(functionName)) { result = - ascending + ascending || !orderKeyIsTimeColumn ? new LastByAccumulator( inputDataTypes.get(0), inputDataTypes.get(1), xIsTimeColumn, yIsTimeColumn) : new LastByDescAccumulator( @@ -125,7 +131,7 @@ public static TableAccumulator createAccumulator( isAggTableScan); } else { result = - ascending + ascending && orderKeyIsTimeColumn ? new FirstByAccumulator( inputDataTypes.get(0), inputDataTypes.get(1), @@ -135,23 +141,24 @@ public static TableAccumulator createAccumulator( : new FirstByDescAccumulator( inputDataTypes.get(0), inputDataTypes.get(1), xIsTimeColumn, yIsTimeColumn); } - } else if (LAST.getFunctionName().equals(functionName)) { - return ascending - ? new LastAccumulator(inputDataTypes.get(0)) - : new LastDescAccumulator( - inputDataTypes.get(0), - isTimeColumn(inputExpressions.get(0), timeColumnName), - isMeasurementColumn(inputExpressions.get(0), measurementColumnNames), - isAggTableScan); - } else { + } else if (LAST.getFunctionName().equals(functionName) && inputExpressions.size() > 1) { + boolean orderKeyIsTimeColumn = isTimeColumn(inputExpressions.get(1), timeColumnName); + result = + ascending || !orderKeyIsTimeColumn + ? new LastAccumulator(inputDataTypes.get(0)) + : new LastDescAccumulator( + inputDataTypes.get(0), + isTimeColumn(inputExpressions.get(0), timeColumnName), + isMeasurementColumn(inputExpressions.get(0), measurementColumnNames), + isAggTableScan); + } else if (FIRST.getFunctionName().equals(functionName) && inputExpressions.size() > 1) { + boolean orderKeyIsTimeColumn = isTimeColumn(inputExpressions.get(1), timeColumnName); result = - createBuiltinAccumulator( - aggregationType, - inputDataTypes, - inputExpressions, - inputAttributes, - ascending, - isAggTableScan); + ascending && orderKeyIsTimeColumn + ? new FirstAccumulator(inputDataTypes.get(0), isAggTableScan) + : new FirstDescAccumulator(inputDataTypes.get(0)); + } else { + result = createBuiltinAccumulator(aggregationType, inputDataTypes); } if (distinct) { @@ -289,12 +296,7 @@ private static GroupedAccumulator createBuiltinGroupedAccumulator( } public static TableAccumulator createBuiltinAccumulator( - TAggregationType aggregationType, - List inputDataTypes, - List inputExpressions, - Map inputAttributes, - boolean ascending, - boolean isAggTableScan) { + TAggregationType aggregationType, List inputDataTypes) { switch (aggregationType) { case COUNT: return new CountAccumulator(); @@ -307,34 +309,18 @@ public static TableAccumulator createBuiltinAccumulator( case SUM: return new SumAccumulator(inputDataTypes.get(0)); case LAST: - return ascending - ? new LastAccumulator(inputDataTypes.get(0)) - : new LastDescAccumulator(inputDataTypes.get(0), false, false, isAggTableScan); + return new LastAccumulator(inputDataTypes.get(0)); case FIRST: - return ascending - ? new FirstAccumulator(inputDataTypes.get(0), isAggTableScan) - : new FirstDescAccumulator(inputDataTypes.get(0)); + return new FirstDescAccumulator(inputDataTypes.get(0)); case MAX: return new MaxAccumulator(inputDataTypes.get(0)); case MIN: return new MinAccumulator(inputDataTypes.get(0)); case LAST_BY: - return ascending - ? new LastByAccumulator(inputDataTypes.get(0), inputDataTypes.get(1), false, false) - : new LastByDescAccumulator( - inputDataTypes.get(0), - inputDataTypes.get(1), - false, - false, - false, - false, - isAggTableScan); + return new LastByAccumulator(inputDataTypes.get(0), inputDataTypes.get(1), false, false); case FIRST_BY: - return ascending - ? new FirstByAccumulator( - inputDataTypes.get(0), inputDataTypes.get(1), false, false, isAggTableScan) - : new FirstByDescAccumulator( - inputDataTypes.get(0), inputDataTypes.get(1), false, false); + return new FirstByDescAccumulator( + inputDataTypes.get(0), inputDataTypes.get(1), false, false); case MAX_BY: return new TableMaxByAccumulator(inputDataTypes.get(0), inputDataTypes.get(1)); case MIN_BY: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java index 8202de5c49947..1bcb6a26426ec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java @@ -573,6 +573,9 @@ private void doAnalyzeSingleTableFile( } getOrCreateTableSchemaCache().flush(); + if (getOrCreateTableSchemaCache().isNeedDecode4DifferentTimeColumn()) { + loadTsFileTableStatement.enableNeedDecode4TimeColumn(); + } getOrCreateTableSchemaCache().clearTagColumnMapper(); TimestampPrecisionUtils.checkTimestampPrecision(tsFileResource.getFileEndTime()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java index 690a18dbd46ec..79ede0f459e58 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java @@ -68,6 +68,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import static org.apache.iotdb.commons.schema.MemUsageUtil.computeStringMemUsage; @@ -111,6 +112,7 @@ public class LoadTsFileTableSchemaCache { private long currentTimeIndexMemoryUsageSizeInBytes = 0; private int currentBatchDevicesCount = 0; + private final AtomicBoolean needDecode4DifferentTimeColumn = new AtomicBoolean(false); public LoadTsFileTableSchemaCache( final Metadata metadata, final MPPQueryContext context, final boolean needToCreateDatabase) @@ -298,7 +300,10 @@ public void createTableAndDatabaseIfNecessary(final String tableName) org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema.fromTsFileTableSchema( tableName, schema); final TableSchema realSchema = - metadata.validateTableHeaderSchema(database, fileSchema, context, true, true).orElse(null); + metadata + .validateTableHeaderSchema4TsFile( + database, fileSchema, context, true, true, needDecode4DifferentTimeColumn) + .orElse(null); if (Objects.isNull(realSchema)) { throw new LoadAnalyzeException( String.format( @@ -308,6 +313,10 @@ public void createTableAndDatabaseIfNecessary(final String tableName) verifyTableDataTypeAndGenerateTagColumnMapper(fileSchema, realSchema); } + public boolean isNeedDecode4DifferentTimeColumn() { + return needDecode4DifferentTimeColumn.get(); + } + private void autoCreateTableDatabaseIfAbsent(final String database) throws LoadAnalyzeException { validateDatabaseName(database); if (DataNodeTableCache.getInstance().isDatabaseExist(database)) { @@ -449,6 +458,7 @@ public void close() { currentBatchTable2Devices = null; tableTagColumnMapper = null; + needDecode4DifferentTimeColumn.set(false); } private void clearDevices() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index a712e916d9e50..aa1f43d95ea42 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -580,8 +580,22 @@ private Pair parseTable4CreateTableOrView( table.addProp(TsTable.COMMENT_KEY, node.getComment()); } + // check if the time column has been specified + long timeColumnCount = + node.getElements().stream() + .filter( + columnDefinition -> + columnDefinition.getColumnCategory() == TsTableColumnCategory.TIME) + .count(); + if (timeColumnCount > 1) { + throw new SemanticException("A table cannot have more than one time column"); + } + if (timeColumnCount == 0) { + // append the time column with default name "time" if user do not specify the time column + table.addColumnSchema(new TimeColumnSchema(TIME_COLUMN_NAME, TSDataType.TIMESTAMP)); + } + // TODO: Place the check at statement analyzer - boolean hasTimeColumn = false; final Set sourceNameSet = new HashSet<>(); boolean hasObject = false; for (final ColumnDefinition columnDefinition : node.getElements()) { @@ -590,15 +604,18 @@ private Pair parseTable4CreateTableOrView( final TSDataType dataType = getDataType(columnDefinition.getType()); hasObject |= dataType == TSDataType.OBJECT; final String comment = columnDefinition.getComment(); - if (checkTimeColumnIdempotent(category, columnName, dataType, comment, table) - && !hasTimeColumn) { - hasTimeColumn = true; - continue; - } + if (table.getColumnSchema(columnName) != null) { throw new SemanticException( - String.format("Columns in table shall not share the same name %s.", columnName)); + String.format("Columns in table shall not share the same name: '%s'.", columnName)); + } + + // allow the user create time column + if (category == TsTableColumnCategory.TIME) { + validateAndGenerateTimeColumn(columnName, dataType, comment, table); + continue; } + final TsTableColumnSchema schema = TableHeaderSchemaValidator.generateColumnSchema( category, @@ -627,6 +644,25 @@ private Pair parseTable4CreateTableOrView( return new Pair<>(database, table); } + private void validateAndGenerateTimeColumn( + final String columnName, + final TSDataType dataType, + final String comment, + final TsTable table) { + + if (dataType == TSDataType.TIMESTAMP) { + final TsTableColumnSchema timeColumnSchema = + new TimeColumnSchema(columnName, TSDataType.TIMESTAMP); + if (Objects.nonNull(comment)) { + timeColumnSchema.getProps().put(TsTable.COMMENT_KEY, comment); + } + table.addColumnSchema(timeColumnSchema); + + } else { + throw new SemanticException("The time column's type shall be 'timestamp'."); + } + } + @Override protected IConfigTask visitAlterColumnDataType( AlterColumnDataType node, MPPQueryContext context) { @@ -651,34 +687,6 @@ protected IConfigTask visitAlterColumnDataType( node.isView()); } - private boolean checkTimeColumnIdempotent( - final TsTableColumnCategory category, - final String columnName, - final TSDataType dataType, - final String comment, - final TsTable table) { - if (category == TsTableColumnCategory.TIME || columnName.equals(TIME_COLUMN_NAME)) { - if (category == TsTableColumnCategory.TIME - && columnName.equals(TIME_COLUMN_NAME) - && dataType == TSDataType.TIMESTAMP) { - if (Objects.nonNull(comment)) { - final TsTableColumnSchema columnSchema = - new TimeColumnSchema(TIME_COLUMN_NAME, TSDataType.TIMESTAMP); - columnSchema.getProps().put(TsTable.COMMENT_KEY, comment); - table.addColumnSchema(columnSchema); - } - return true; - } else if (dataType == TSDataType.TIMESTAMP) { - throw new SemanticException( - "The time column category shall be bounded with column name 'time'."); - } else { - throw new SemanticException("The time column's type shall be 'timestamp'."); - } - } - - return false; - } - @Override protected IConfigTask visitRenameTable(final RenameTable node, final MPPQueryContext context) { context.setQueryType(QueryType.WRITE); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index 2274869c35543..a36564878df23 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@ -548,7 +548,8 @@ public PlanNode visitLoadFile( context.getQueryId().genPlanNodeId(), loadTsFileStatement.getResources(), isTableModel, - loadTsFileStatement.getDatabase()); + loadTsFileStatement.getDatabase(), + false); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 62e46d4598e86..1bc788251a764 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -240,6 +240,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; @@ -3462,13 +3463,7 @@ private PatternAggregator buildPatternAggregator( .collect(Collectors.toList()); TableAccumulator accumulator = - createBuiltinAccumulator( - getAggregationTypeByFuncName(functionName), - originalArgumentTypes, - arguments.stream().map(Map.Entry::getKey).collect(Collectors.toList()), - Collections.emptyMap(), - true, - false); + createBuiltinAccumulator(getAggregationTypeByFuncName(functionName), originalArgumentTypes); BoundSignature signature = resolvedFunction.getSignature(); @@ -3809,13 +3804,18 @@ public Operator visitInto(IntoNode node, LocalExecutionPlanContext context) { int index = map.get(originInputColumnNames.get(i)); inputColumns.set(index, originColumns.get(i)); } - + ColumnSchema timeColumnOfTargetTable = null; for (int i = 0; i < inputColumns.size(); i++) { String columnName = inputColumns.get(i).getName(); inputLocationMap.put(columnName, new InputLocation(0, i)); TsTableColumnCategory columnCategory = inputColumns.get(i).getColumnCategory(); if (columnCategory == TIME) { + if (timeColumnOfTargetTable == null) { + timeColumnOfTargetTable = inputColumns.get(i); + } else { + throw new SemanticException("Multiple columns with TIME category found"); + } continue; } @@ -3824,6 +3824,9 @@ public Operator visitInto(IntoNode node, LocalExecutionPlanContext context) { inputColumnTypes.add(columnType); inputColumnCategories.add(columnCategory); } + if (timeColumnOfTargetTable == null) { + throw new SemanticException("Missing TIME category column"); + } long statementSizePerLine = OperatorGeneratorUtil.calculateStatementSizePerLine(inputColumnTypes); @@ -3839,7 +3842,8 @@ public Operator visitInto(IntoNode node, LocalExecutionPlanContext context) { tsDataTypeMap, true, FragmentInstanceManager.getInstance().getIntoOperationExecutor(), - statementSizePerLine); + statementSizePerLine, + timeColumnOfTargetTable); } private boolean[] checkStatisticAndScanOrder( @@ -3884,6 +3888,22 @@ private boolean[] checkStatisticAndScanOrder( break; } + // first and last, the second argument has to be the time column + if (FIRST_AGGREGATION.equals(funcName) || LAST_AGGREGATION.equals(funcName)) { + if (!isTimeColumn(aggregation.getArguments().get(1), timeColumnName)) { + canUseStatistic = false; + break; + } + } + + // first_by and last_by, the second argument has to be the time column + if (FIRST_BY_AGGREGATION.equals(funcName) || LAST_BY_AGGREGATION.equals(funcName)) { + if (!isTimeColumn(aggregation.getArguments().get(2), timeColumnName)) { + canUseStatistic = false; + break; + } + } + // only last_by(time, x) or last_by(x,time) can use statistic if ((LAST_BY_AGGREGATION.equals(funcName) || FIRST_BY_AGGREGATION.equals(funcName)) && !isTimeColumn(aggregation.getArguments().get(0), timeColumnName) @@ -3929,6 +3949,12 @@ private OptimizeType canUseLastCacheOptimize( return OptimizeType.NOOP; } + // if the timeColumnName is null, the param of function is just a timestamp column other than + // the time column + if (timeColumnName == null || !checkOrderColumnIsTime(node.getAggregations(), timeColumnName)) { + return OptimizeType.NOOP; + } + if (canUseLastRowOptimize(aggregators)) { return OptimizeType.LAST_ROW; } @@ -3940,6 +3966,33 @@ private OptimizeType canUseLastCacheOptimize( return OptimizeType.NOOP; } + /** + * Checks if the ordering column in aggregations matches the time column. only check for + * FIRST/LAST/FIRST_BY/LAST_BY + */ + private boolean checkOrderColumnIsTime( + Map aggregations, String timeColumnName) { + + for (Map.Entry entry : aggregations.entrySet()) { + String functionName = + entry.getValue().getResolvedFunction().getSignature().getName().toLowerCase(); + List arguments = entry.getValue().getArguments(); + Expression lastParam = entry.getValue().getArguments().get(arguments.size() - 1); + + switch (functionName) { + case FIRST_AGGREGATION: + case LAST_AGGREGATION: + case FIRST_BY_AGGREGATION: + case LAST_BY_AGGREGATION: + if (!((SymbolReference) lastParam).getName().equalsIgnoreCase(timeColumnName)) { + return false; + } + break; + } + } + return true; + } + private boolean canUseLastRowOptimize(List aggregators) { for (TableAggregator aggregator : aggregators) { if (aggregator.getAccumulator() instanceof LastDescAccumulator) { @@ -4160,13 +4213,7 @@ private WindowAggregator buildWindowAggregator( .map(InternalTypeManager::getTSDataType) .collect(Collectors.toList()); TableAccumulator accumulator = - createBuiltinAccumulator( - getAggregationTypeByFuncName(functionName), - originalArgumentTypes, - function.getArguments(), - Collections.emptyMap(), - true, - false); + createBuiltinAccumulator(getAggregationTypeByFuncName(functionName), originalArgumentTypes); // Create aggregator by accumulator return new WindowAggregator( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java index 604fda6e1e8d5..c8170a4880a08 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -66,12 +66,13 @@ public class LoadSingleTsFileNode extends WritePlanNode { private TRegionReplicaSet localRegionReplicaSet; public LoadSingleTsFileNode( - PlanNodeId id, - TsFileResource resource, - boolean isTableModel, - String database, - boolean deleteAfterLoad, - long writePointCount) { + final PlanNodeId id, + final TsFileResource resource, + final boolean isTableModel, + final String database, + final boolean deleteAfterLoad, + final long writePointCount, + final boolean needDecodeTsFile) { super(id); this.tsFile = resource.getTsFile(); this.resource = resource; @@ -79,6 +80,7 @@ public LoadSingleTsFileNode( this.database = database; this.deleteAfterLoad = deleteAfterLoad; this.writePointCount = writePointCount; + this.needDecodeTsFile = needDecodeTsFile; } public boolean isTsFileEmpty() { @@ -89,6 +91,10 @@ public boolean isTsFileEmpty() { public boolean needDecodeTsFile( Function>, List> partitionFetcher) { + if (needDecodeTsFile) { + return true; + } + List> slotList = new ArrayList<>(); resource .getDevices() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java index 3588b6ddbb052..25ad9f3a5c7fb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java @@ -46,13 +46,19 @@ public class LoadTsFileNode extends WritePlanNode { private final List resources; private final List isTableModel; private final String database; + private final boolean needDecode4TimeColumn; public LoadTsFileNode( - PlanNodeId id, List resources, List isTableModel, String database) { + final PlanNodeId id, + final List resources, + final List isTableModel, + final String database, + final boolean needDecode4TimeColumn) { super(id); this.resources = resources; this.isTableModel = isTableModel; this.database = database; + this.needDecode4TimeColumn = needDecode4TimeColumn; } @Override @@ -121,7 +127,8 @@ private List splitByPartitionForTreeModel(Analysis analysis) { isTableModel.get(i), database, statement.isDeleteAfterLoad(), - statement.getWritePointCount(i))); + statement.getWritePointCount(i), + false)); } return res; } @@ -143,7 +150,8 @@ private List splitByPartitionForTableModel( isTableModel.get(i), database, statement.isDeleteAfterLoad(), - statement.getWritePointCount(i))); + statement.getWritePointCount(i), + needDecode4TimeColumn)); } } return res; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index 663c92ad83e97..a2be5ec968f9e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.SessionInfo; @@ -154,12 +155,17 @@ import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame.Type.ROWS; import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toTypeSignature; import static org.apache.iotdb.db.queryengine.plan.relational.utils.NodeUtils.getSortItemsFromOrderBy; +import static org.apache.iotdb.db.utils.constant.SqlConstant.FIRST_AGGREGATION; +import static org.apache.iotdb.db.utils.constant.SqlConstant.FIRST_BY_AGGREGATION; +import static org.apache.iotdb.db.utils.constant.SqlConstant.LAST_AGGREGATION; +import static org.apache.iotdb.db.utils.constant.SqlConstant.LAST_BY_AGGREGATION; import static org.apache.tsfile.read.common.type.BlobType.BLOB; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; import static org.apache.tsfile.read.common.type.IntType.INT32; import static org.apache.tsfile.read.common.type.LongType.INT64; import static org.apache.tsfile.read.common.type.StringType.STRING; +import static org.apache.tsfile.read.common.type.TimestampType.TIMESTAMP; import static org.apache.tsfile.read.common.type.UnknownType.UNKNOWN; public class ExpressionAnalyzer { @@ -1051,6 +1057,38 @@ protected Type visitFunctionCall( throw new SemanticException("DISTINCT is not supported for non-aggregation functions"); } + int argumentsNum = node.getArguments().size(); + RelationType relationType = context.getContext().getScope().getRelationType(); + // Syntactic sugar: first(s1) => first(s1,time), first_by(s1,s2) => first_by(s1,s2,time) + // So do last and last_by. + switch (functionName.toLowerCase()) { + case FIRST_AGGREGATION: + case LAST_AGGREGATION: + if (argumentsNum == 1) { + addTimeArgument(node.getArguments(), getActualTimeFieldName(relationType)); + } else if (argumentsNum == 2) { + if (!checkArgumentIsTimestamp( + node.getArguments().get(1), (List) relationType.getVisibleFields())) { + throw new SemanticException( + String.format( + "The second argument of %s function must be actual time name", functionName)); + } + } + break; + case FIRST_BY_AGGREGATION: + case LAST_BY_AGGREGATION: + if (argumentsNum == 2) { + addTimeArgument(node.getArguments(), getActualTimeFieldName(relationType)); + } else if (argumentsNum == 3) { + if (!checkArgumentIsTimestamp( + node.getArguments().get(2), (List) relationType.getVisibleFields())) { + throw new SemanticException( + String.format( + "The third argument of %s function must be actual time name", functionName)); + } + } + } + List argumentTypes = getCallArgumentTypes(node.getArguments(), context); if (node.getArguments().size() > 127) { @@ -1132,6 +1170,65 @@ public List getCallArgumentTypes( return argumentTypesBuilder.build(); } + private void addTimeArgument(List arguments, String actualTimeField) { + + if (arguments.get(0) instanceof DereferenceExpression) { + arguments.add( + new DereferenceExpression( + ((DereferenceExpression) arguments.get(0)).getBase(), + new Identifier(actualTimeField.toLowerCase(Locale.ENGLISH)))); + } else { + arguments.add(new Identifier(actualTimeField.toLowerCase(Locale.ENGLISH))); + } + } + + private boolean checkArgumentIsTimestamp(Expression argument, List visibleFields) { + + String argumentName = + (argument instanceof DereferenceExpression) + ? ((DereferenceExpression) argument) + .getField() + .orElseThrow(() -> new SemanticException("the input field does not exist")) + .toString() + : argument.toString(); + + for (Field field : visibleFields) { + if (field + .getName() + .orElseThrow(() -> new SemanticException("the field in table does not have a name")) + .equalsIgnoreCase(argumentName)) { + return field.getType() == TIMESTAMP; + } + } + // should never reach here + throw new SemanticException("the input argument does not exist"); + } + + /** Retrieves the effective time column name from the relation's visible fields. */ + private String getActualTimeFieldName(RelationType relation) { + + // Priority 1: Try to find a column explicitly marked as TIME category + Optional timeColumn = + relation.getVisibleFields().stream() + .filter(field -> field.getColumnCategory() == TsTableColumnCategory.TIME) + .findFirst() + .flatMap(Field::getName); + + if (timeColumn.isPresent()) { + return timeColumn.get(); + } + + // Priority 2: Fallback to the first TIMESTAMP column (e.g., for system schema compatibility) + return relation.getVisibleFields().stream() + .filter(field -> field.getType() == TIMESTAMP) + .findFirst() + .flatMap(Field::getName) + .orElseThrow( + () -> + new SemanticException( + "Missing valid time column. The table must contain either a column with the TIME category or at least one TIMESTAMP column.")); + } + private Type analyzeMatchNumber( FunctionCall node, StackableAstVisitorContext context) { if (!node.getArguments().isEmpty()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index e9df62193c28e..bc6d54c37e7fa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -269,7 +269,6 @@ import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; import static org.apache.iotdb.commons.schema.table.TsTable.TABLE_ALLOWED_PROPERTIES; -import static org.apache.iotdb.commons.schema.table.TsTable.TIME_COLUMN_NAME; import static org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinScalarFunction.DATE_BIN; import static org.apache.iotdb.db.queryengine.execution.warnings.StandardWarningCode.REDUNDANT_ORDER_BY; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.AggregationAnalyzer.verifyOrderByAggregations; @@ -684,10 +683,23 @@ protected Scope visitInsert(Insert insert, Optional scope) { .collect(toImmutableList()); analysis.registerTable(insert.getTable(), tableSchema, targetTable); - LinkedHashSet tableColumns = - columns.stream() - .map(ColumnSchema::getName) - .collect(Collectors.toCollection(LinkedHashSet::new)); + LinkedHashSet tableColumns = new LinkedHashSet<>(); + String actualTimeColumnName = null; + for (ColumnSchema column : columns) { + tableColumns.add(column.getName().toLowerCase(ENGLISH)); + + if (column.getColumnCategory() == TsTableColumnCategory.TIME) { + if (actualTimeColumnName != null) { + throw new SemanticException( + "Multiple columns found with TIME category in table schema"); + } + actualTimeColumnName = column.getName(); + } + } + if (actualTimeColumnName == null) { + throw new SemanticException("Target table schema misses a TIME category column"); + } + LinkedHashSet insertColumns; if (insert.getColumns().isPresent()) { insertColumns = @@ -712,7 +724,7 @@ protected Scope visitInsert(Insert insert, Optional scope) { } // insert columns should contain time - if (!insertColumns.contains(TIME_COLUMN_NAME)) { + if (!insertColumns.contains(actualTimeColumnName)) { throw new SemanticException("time column can not be null"); } // insert columns should contain at least one field column @@ -1865,12 +1877,12 @@ protected List visitFunctionCall(FunctionCall node, Scope context) { (childrenResultList.get(i).size() == maxSize) ? baseIndex : new AtomicInteger(0); } for (int i = 0; i < maxSize; i++) { - ImmutableList.Builder operandListBuilder = new ImmutableList.Builder<>(); + List operandListBuilder = new ArrayList<>(childrenIndexes.length); for (int j = 0; j < childrenIndexes.length; j++) { int operandIndexInResult = childrenIndexes[j].get(); operandListBuilder.add(childrenResultList.get(j).get(operandIndexInResult)); } - resultBuilder.add(new FunctionCall(node.getName(), operandListBuilder.build())); + resultBuilder.add(new FunctionCall(node.getName(), operandListBuilder)); baseIndex.getAndIncrement(); } return resultBuilder.build(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java index db706d4980cba..82ee332361409 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java @@ -43,6 +43,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; // All the input databases shall not contain "root" public interface Metadata { @@ -102,28 +103,31 @@ Map> indexScan( *

When table or column is missing, this method will execute auto creation if the user have * corresponding authority. * - *

When using SQL, the columnSchemaList could be null and there won't be any validation. + *

When using SQL, the columnSchemaList could be {@code null} and there won't be any + * validation. * - *

When the input dataType or category of one column is null, the column won't be auto created. + *

When the input dataType or category of one column is {@code null}, the column won't be auto + * created. * *

The caller need to recheck the dataType of measurement columns to decide whether to do * partial insert * - * @param isStrictIdColumn if true, when the table already exists, the id columns in the existing - * table should be the prefix of those in the input tableSchema, or input id columns be the - * prefix of existing id columns. + * @param isStrictTagColumn if {@code true}, when the table already exists, the tag columns in the + * existing table should be the prefix of those in the input tableSchema, or input tag columns + * be the prefix of existing tag columns. * @return If table doesn't exist and the user have no authority to create table, Optional.empty() * will be returned. The returned table may not include all the columns * in @param{tableSchema}, if the user have no authority to alter table. - * @throws SemanticException if column category mismatch or data types of id or attribute column - * are not STRING or Category, Type of any missing ColumnSchema is null + * @throws SemanticException if column category mismatch or data types of tag or attribute column + * are not STRING or Category, Type of any missing ColumnSchema is {@code null} */ - Optional validateTableHeaderSchema( + Optional validateTableHeaderSchema4TsFile( final String database, final TableSchema tableSchema, final MPPQueryContext context, final boolean allowCreateTable, - final boolean isStrictIdColumn) + final boolean isStrictTagColumn, + final AtomicBoolean needDecode4DifferentTimeColumn) throws LoadAnalyzeTableColumnDisorderException; void validateInsertNodeMeasurements( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 5df442d4d321f..ed0be5b10d02c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -74,6 +74,7 @@ import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import static org.apache.iotdb.db.queryengine.transformation.dag.column.FailFunctionColumnTransformer.FAIL_FUNCTION_NAME; @@ -1396,16 +1397,22 @@ public Map> indexScan( } @Override - public Optional validateTableHeaderSchema( - String database, - TableSchema tableSchema, - MPPQueryContext context, - boolean allowCreateTable, - boolean isStrictTagColumn) + public Optional validateTableHeaderSchema4TsFile( + final String database, + final TableSchema tableSchema, + final MPPQueryContext context, + final boolean allowCreateTable, + final boolean isStrictTagColumn, + final AtomicBoolean needDecode4DifferentTimeColumn) throws LoadAnalyzeTableColumnDisorderException { return TableHeaderSchemaValidator.getInstance() - .validateTableHeaderSchema( - database, tableSchema, context, allowCreateTable, isStrictTagColumn); + .validateTableHeaderSchema4TsFile( + database, + tableSchema, + context, + allowCreateTable, + isStrictTagColumn, + needDecode4DifferentTimeColumn); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java index 99da3d005f721..7f578cef4ae57 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java @@ -28,6 +28,7 @@ import org.apache.iotdb.commons.schema.table.column.AttributeColumnSchema; import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TimeColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.db.auth.AuthorityChecker; @@ -59,16 +60,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import static org.apache.iotdb.commons.schema.table.TsTable.TIME_COLUMN_NAME; import static org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager.getTSDataType; import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; @@ -92,12 +97,13 @@ public static TableHeaderSchemaValidator getInstance() { return TableHeaderSchemaValidatorHolder.INSTANCE; } - public Optional validateTableHeaderSchema( + public Optional validateTableHeaderSchema4TsFile( final String database, final TableSchema tableSchema, final MPPQueryContext context, final boolean allowCreateTable, - final boolean isStrictTagColumn) + final boolean isStrictTagColumn, + final @Nonnull AtomicBoolean needDecode4DifferentTimeColumn) throws LoadAnalyzeTableColumnDisorderException { // The schema cache R/W and fetch operation must be locked together thus the cache clean // operation executed by delete timeSeries will be effective. @@ -135,8 +141,7 @@ public Optional validateTableHeaderSchema( } else { DataNodeTreeViewSchemaUtils.checkTableInWrite(database, table); // If table with this name already exists and isStrictTagColumn is true, make sure the - // existing - // id columns are the prefix of the incoming tag columns, or vice versa + // existing tag columns are a prefix of the incoming tag columns, or vice versa if (isStrictTagColumn) { final List realTagColumns = table.getTagColumnSchemaList(); final List incomingTagColumns = tableSchema.getTagColumns(); @@ -172,6 +177,32 @@ public Optional validateTableHeaderSchema( } } } + long realTimeIndex = 0; + boolean realWithoutTimeColumn = true; + + for (final TsTableColumnSchema schema : table.getColumnList()) { + if (schema.getColumnCategory() == TsTableColumnCategory.TIME) { + realWithoutTimeColumn = false; + break; + } + if (schema.getColumnCategory() != TsTableColumnCategory.ATTRIBUTE) { + ++realTimeIndex; + } + } + + long inputTimeIndex = 0; + boolean inputWithoutTimeColumn = true; + for (final ColumnSchema schema : tableSchema.getColumns()) { + if (schema.getColumnCategory() == TsTableColumnCategory.TIME) { + inputWithoutTimeColumn = false; + break; + } + ++inputTimeIndex; + } + if (inputWithoutTimeColumn != realWithoutTimeColumn + || !inputWithoutTimeColumn && inputTimeIndex != realTimeIndex) { + needDecode4DifferentTimeColumn.set(true); + } } boolean refreshed = false; @@ -211,7 +242,7 @@ public Optional validateTableHeaderSchema( noField = false; } } else { - // leave measurement columns' dataType checking to the caller, then the caller can decide + // leave field columns' dataType checking to the caller, then the caller can decide // whether to do partial insert // only check column category @@ -233,7 +264,7 @@ public Optional validateTableHeaderSchema( final List resultColumnList = new ArrayList<>(); if (!missingColumnList.isEmpty() && isAutoCreateSchemaEnabled) { // TODO table metadata: authority check for table alter - // check id or attribute column data type in this method + // check tag or attribute column data type in this method autoCreateColumn(database, tableSchema.getTableName(), missingColumnList, context); table = DataNodeTableCache.getInstance().getTable(database, tableSchema.getTableName()); } else if (!missingColumnList.isEmpty() @@ -571,12 +602,10 @@ private void autoCreateColumnsFromMeasurements( final ListenableFuture future = task.execute(configTaskExecutor); final ConfigTaskResult result = future.get(); if (result.getStatusCode().getStatusCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - throw new RuntimeException( - new IoTDBException( - String.format( - "Auto add table column failed: %s.%s", - database, measurementInfo.getTableName()), - result.getStatusCode().getStatusCode())); + throw new IoTDBRuntimeException( + String.format( + "Auto add table column failed: %s.%s", database, measurementInfo.getTableName()), + result.getStatusCode().getStatusCode()); } DataNodeSchemaLockManager.getInstance() .takeReadLock(context, SchemaLockType.VALIDATE_VS_DELETION_TABLE); @@ -631,6 +660,18 @@ public TsTable toTsTable(InsertNodeMeasurementInfo measurementInfo) { return tsTable; } + long timeCandidateNums = + Arrays.stream(measurementInfo.getColumnCategories()) + .filter(columnDefinition -> TsTableColumnCategory.TIME == columnDefinition) + .count(); + if (timeCandidateNums > 1) { + throw new SemanticException("A table cannot have more than one time column"); + } + if (timeCandidateNums == 0) { + // append the time column with default name "time" if user do not specify the time column + tsTable.addColumnSchema(new TimeColumnSchema(TIME_COLUMN_NAME, TSDataType.TIMESTAMP)); + } + boolean hasObject = false; for (int i = 0; i < measurements.length; i++) { if (measurements[i] == null) { @@ -654,12 +695,16 @@ public TsTable toTsTable(InsertNodeMeasurementInfo measurementInfo) { throw new SemanticException( String.format("Columns in table shall not share the same name %s.", columnName)); } - TSDataType dataType = measurementInfo.getType(i); if (dataType == null && (dataType = measurementInfo.getTypeForFirstValue(i)) == null) { throw new ColumnCreationFailException( "Cannot create column " + columnName + " datatype is not provided"); } + + if (category == TsTableColumnCategory.TIME && dataType != TSDataType.TIMESTAMP) { + throw new SemanticException("The time column's type shall be 'timestamp'."); + } + hasObject |= dataType == TSDataType.OBJECT; tsTable.addColumnSchema(generateColumnSchema(category, columnName, dataType, null, null)); } @@ -675,6 +720,21 @@ public TsTable toTsTable(InsertNodeMeasurementInfo measurementInfo) { } private void addColumnSchema(final List columnSchemas, final TsTable tsTable) { + // check if the time column has been specified + long timeColumnCount = + columnSchemas.stream() + .filter( + columnDefinition -> + columnDefinition.getColumnCategory() == TsTableColumnCategory.TIME) + .count(); + if (timeColumnCount > 1) { + throw new SemanticException("A table cannot have more than one time column"); + } + if (timeColumnCount == 0) { + // append the time column with default name "time" if user do not specify the time column + tsTable.addColumnSchema(new TimeColumnSchema(TIME_COLUMN_NAME, TSDataType.TIMESTAMP)); + } + for (final ColumnSchema columnSchema : columnSchemas) { TsTableColumnCategory category = columnSchema.getColumnCategory(); if (category == null) { @@ -718,8 +778,8 @@ public static TsTableColumnSchema generateColumnSchema( schema = new AttributeColumnSchema(columnName, dataType); break; case TIME: - throw new SemanticException( - "Create table or add column statement shall not specify column category TIME"); + schema = new TimeColumnSchema(columnName, dataType); + break; case FIELD: schema = dataType != TSDataType.UNKNOWN @@ -810,10 +870,8 @@ private List parseInputColumnSchema( TSFileDescriptor.getInstance().getConfig().getCompressor(dataType))); break; case TIME: - throw new SemanticException( - "Adding column for column category " - + inputColumn.getColumnCategory() - + " is not supported"); + // Do nothing, cause the time column shall never be appended to the existing table + break; default: throw new IllegalStateException( "Unknown ColumnCategory for adding column: " + inputColumn.getColumnCategory()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index d442d51a21418..ded699588c1c2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -1388,7 +1388,11 @@ protected RelationPlan visitLoadTsFile(final LoadTsFile node, final Void context } return new RelationPlan( new LoadTsFileNode( - idAllocator.genPlanNodeId(), node.getResources(), isTableModel, node.getDatabase()), + idAllocator.genPlanNodeId(), + node.getResources(), + isTableModel, + node.getDatabase(), + node.isNeedDecode4TimeColumn()), analysis.getRootScope(), Collections.emptyList(), outerContext); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java index 9b7cd372ee940..8deb97c2e5af6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java @@ -58,12 +58,13 @@ public class LoadTsFile extends Statement { private boolean isGeneratedByPipe = false; - private Map loadAttributes; + private final Map loadAttributes; private List tsFiles; private List resources; private List writePointCountList; private List isTableModel; + private boolean needDecode4TimeColumn; public LoadTsFile(NodeLocation location, String filePath, Map loadAttributes) { super(location); @@ -167,6 +168,14 @@ public void setIsTableModel(List isTableModel) { this.isTableModel = isTableModel; } + public boolean isNeedDecode4TimeColumn() { + return needDecode4TimeColumn; + } + + public void enableNeedDecode4TimeColumn() { + this.needDecode4TimeColumn = true; + } + public List getTsFiles() { return tsFiles; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 5d529818a75e6..1ad212cf560af 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -34,7 +34,6 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.protocol.session.IClientSession; -import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimestampOperand; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AddColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AliasedRelation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllColumns; @@ -291,7 +290,6 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -305,7 +303,6 @@ import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.DATA_NODE_ID_TABLE_MODEL; -import static org.apache.iotdb.commons.schema.table.TsTable.TIME_COLUMN_NAME; import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.ATTRIBUTE; import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.FIELD; import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.TAG; @@ -338,10 +335,6 @@ import static org.apache.iotdb.db.utils.constant.SqlConstant.APPROX_COUNT_DISTINCT; import static org.apache.iotdb.db.utils.constant.SqlConstant.APPROX_MOST_FREQUENT; import static org.apache.iotdb.db.utils.constant.SqlConstant.APPROX_PERCENTILE; -import static org.apache.iotdb.db.utils.constant.SqlConstant.FIRST_AGGREGATION; -import static org.apache.iotdb.db.utils.constant.SqlConstant.FIRST_BY_AGGREGATION; -import static org.apache.iotdb.db.utils.constant.SqlConstant.LAST_AGGREGATION; -import static org.apache.iotdb.db.utils.constant.SqlConstant.LAST_BY_AGGREGATION; public class AstBuilder extends RelationalSqlBaseVisitor { @@ -855,15 +848,28 @@ private Node visitInsertValues( final List columnNames = identifiers.stream().map(Identifier::getValue).collect(toList()); int timeColumnIndex = -1; - for (int i = 0; i < columnNames.size(); i++) { - if (TIME_COLUMN_NAME.equalsIgnoreCase(columnNames.get(i))) { - if (timeColumnIndex == -1) { - timeColumnIndex = i; - } else { - throw new SemanticException("One row should only have one time value"); + + // retrieve the table schema to identify the actual time column + TsTable table = DataNodeTableCache.getInstance().getTable(databaseName, tableName); + List timeColumnCandidates = + table.getColumnList().stream() + .filter(col -> col.getColumnCategory() == TIME) + .collect(toList()); + if (timeColumnCandidates.size() != 1) { + throw new SemanticException("the table should only have one column found with TIME category"); + } else { + // locate the time column index in the input identifiers if time column exists in the schema + for (int i = 0; i < columnNames.size(); i++) { + if (timeColumnCandidates.get(0).getColumnName().equalsIgnoreCase(columnNames.get(i))) { + if (timeColumnIndex == -1) { + timeColumnIndex = i; + } else { + throw new SemanticException("One row should only have one time value"); + } } } } + if (timeColumnIndex != -1) { columnNames.remove(timeColumnIndex); } @@ -3417,29 +3423,7 @@ public Node visitFunctionCall(RelationalSqlParser.FunctionCallContext ctx) { new DereferenceExpression(getLocation(ctx.label), (Identifier) visit(ctx.label))); } - // Syntactic sugar: first(s1) => first(s1,time), first_by(s1,s2) => first_by(s1,s2,time) - // So do last and last_by. - if (name.toString().equalsIgnoreCase(FIRST_AGGREGATION) - || name.toString().equalsIgnoreCase(LAST_AGGREGATION)) { - if (arguments.size() == 1) { - appendTimeArgument(arguments); - } else if (arguments.size() == 2) { - check( - checkArgumentIsTime(arguments.get(1)), - "The second argument of 'first' or 'last' function must be 'time'", - ctx); - } - } else if (name.toString().equalsIgnoreCase(FIRST_BY_AGGREGATION) - || name.toString().equalsIgnoreCase(LAST_BY_AGGREGATION)) { - if (arguments.size() == 2) { - appendTimeArgument(arguments); - } else if (arguments.size() == 3) { - check( - checkArgumentIsTime(arguments.get(2)), - "The third argument of 'first_by' or 'last_by' function must be 'time'", - ctx); - } - } else if (name.toString().equalsIgnoreCase(APPROX_COUNT_DISTINCT)) { + if (name.toString().equalsIgnoreCase(APPROX_COUNT_DISTINCT)) { if (arguments.size() == 2 && !(arguments.get(1) instanceof DoubleLiteral || arguments.get(1) instanceof LongLiteral @@ -3467,30 +3451,6 @@ public Node visitFunctionCall(RelationalSqlParser.FunctionCallContext ctx) { return new FunctionCall(getLocation(ctx), name, window, nulls, distinct, mode, arguments); } - private void appendTimeArgument(List arguments) { - if (arguments.get(0) instanceof DereferenceExpression) { - arguments.add( - new DereferenceExpression( - ((DereferenceExpression) arguments.get(0)).getBase(), - new Identifier( - TimestampOperand.TIMESTAMP_EXPRESSION_STRING.toLowerCase(Locale.ENGLISH)))); - } else { - arguments.add( - new Identifier(TimestampOperand.TIMESTAMP_EXPRESSION_STRING.toLowerCase(Locale.ENGLISH))); - } - } - - private boolean checkArgumentIsTime(Expression argument) { - if (argument instanceof DereferenceExpression) { - return ((DereferenceExpression) argument) - .getField() - .get() - .toString() - .equalsIgnoreCase(TimestampOperand.TIMESTAMP_EXPRESSION_STRING); - } - return argument.toString().equalsIgnoreCase(TimestampOperand.TIMESTAMP_EXPRESSION_STRING); - } - @Override public Node visitDateBinGapFill(RelationalSqlParser.DateBinGapFillContext ctx) { TimeDuration timeDuration = DateTimeUtils.constructTimeDuration(ctx.timeDuration().getText()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/FunctionTestUtils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/FunctionTestUtils.java index 47f9063d899e5..bad56bba9b85d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/FunctionTestUtils.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/FunctionTestUtils.java @@ -32,7 +32,6 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; public class FunctionTestUtils { @@ -101,12 +100,7 @@ public static AggregationWindowFunction createAggregationWindowFunction( // inputExpressions and inputAttributes are not used in this method TableAccumulator accumulator = AccumulatorFactory.createBuiltinAccumulator( - aggregationType, - Collections.singletonList(inputDataType), - new ArrayList<>(), - new HashMap<>(), - ascending, - false); + aggregationType, Collections.singletonList(inputDataType)); WindowAggregator aggregator = new WindowAggregator(accumulator, outputDataType, Collections.singletonList(0)); return new AggregationWindowFunction(aggregator); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/AggregationTableScanTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/AggregationTableScanTest.java new file mode 100644 index 0000000000000..f78cae18f37d5 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/AggregationTableScanTest.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.distribution; + +import org.apache.iotdb.common.rpc.thrift.TAggregationType; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.AccumulatorFactory; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.FirstAccumulator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.FirstByAccumulator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.FirstByDescAccumulator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.FirstDescAccumulator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.LastAccumulator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.LastByAccumulator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.LastByDescAccumulator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.LastDescAccumulator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.TableAccumulator; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; + +import org.apache.tsfile.enums.TSDataType; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.Assert.assertTrue; + +public class AggregationTableScanTest { + + private static final String TIME_COL = "time"; + private static final String S1_COL = "s1"; + private static final String S2_COL = "s2"; + + @Test + public void testFirstAccumulator() { + // Case 1: Ascending scan ordered by time -> Use optimized FirstAccumulator + verifyAccumulator("first", TAggregationType.FIRST, true, true, FirstAccumulator.class); + + // Case 2: Descending scan ordered by time -> Use FirstDescAccumulator + verifyAccumulator("first", TAggregationType.FIRST, false, true, FirstDescAccumulator.class); + + // Case 3: Ordered by non-time column -> Fallback to FirstDescAccumulator + verifyAccumulator("first", TAggregationType.FIRST, true, false, FirstDescAccumulator.class); + } + + @Test + public void testLastAccumulator() { + // Case 1: Ascending scan ordered by time -> Use LastAccumulator + verifyAccumulator("last", TAggregationType.LAST, true, true, LastAccumulator.class); + + // Case 2: Descending scan ordered by time -> Use LastDescAccumulator + verifyAccumulator("last", TAggregationType.LAST, false, true, LastDescAccumulator.class); + + // Case 3: Ordered by non-time column -> Use LastAccumulator + verifyAccumulator("last", TAggregationType.LAST, false, false, LastAccumulator.class); + } + + @Test + public void testFirstByAccumulator() { + // Case 1: Ascending scan ordered by time -> Use optimized FirstByAccumulator + verifyByAccumulator( + "first_by", TAggregationType.FIRST_BY, true, true, FirstByAccumulator.class); + + // Case 2: Descending scan ordered by time -> Use FirstByDescAccumulator + verifyByAccumulator( + "first_by", TAggregationType.FIRST_BY, false, true, FirstByDescAccumulator.class); + + // Case 3: Ordered by non-time column -> Fallback to FirstByDescAccumulator + verifyByAccumulator( + "first_by", TAggregationType.FIRST_BY, true, false, FirstByDescAccumulator.class); + } + + @Test + public void testLastByAccumulator() { + // Case 1: Ascending scan ordered by time -> Use LastByAccumulator + verifyByAccumulator("last_by", TAggregationType.LAST_BY, true, true, LastByAccumulator.class); + + // Case 2: Descending scan ordered by time -> Use LastByDescAccumulator + verifyByAccumulator( + "last_by", TAggregationType.LAST_BY, false, true, LastByDescAccumulator.class); + + // Case 3: Ordered by non-time column -> Use LastByAccumulator + verifyByAccumulator("last_by", TAggregationType.LAST_BY, false, false, LastByAccumulator.class); + } + + private void verifyAccumulator( + String funcName, + TAggregationType aggType, + boolean ascending, + boolean isTimeOrder, + Class expectedClass) { + List inputExpressions = + Arrays.asList( + new SymbolReference(S1_COL), new SymbolReference(isTimeOrder ? TIME_COL : S2_COL)); + List inputDataTypes = Collections.singletonList(TSDataType.INT32); + + doCreateAndAssert( + funcName, aggType, inputDataTypes, inputExpressions, ascending, expectedClass); + } + + private void verifyByAccumulator( + String funcName, + TAggregationType aggType, + boolean ascending, + boolean isTimeOrder, + Class expectedClass) { + + List inputExpressions = + Arrays.asList( + new SymbolReference(S1_COL), + new SymbolReference(S2_COL), + new SymbolReference(isTimeOrder ? TIME_COL : S1_COL)); + List inputDataTypes = Arrays.asList(TSDataType.INT32, TSDataType.INT64); + + doCreateAndAssert( + funcName, aggType, inputDataTypes, inputExpressions, ascending, expectedClass); + } + + private void doCreateAndAssert( + String funcName, + TAggregationType aggType, + List types, + List expressions, + boolean ascending, + Class expectedClass) { + + Map inputAttribute = new HashMap<>(); + boolean isAggTableScan = true; + Set measurementColumnNames = new HashSet<>(Collections.singletonList(S1_COL)); + boolean distinct = false; + + TableAccumulator accumulator = + AccumulatorFactory.createAccumulator( + funcName, + aggType, + types, + expressions, + inputAttribute, + ascending, + isAggTableScan, + TIME_COL, + measurementColumnNames, + distinct); + + String msg = + String.format( + "Func: %s, Asc: %s, Expressions: %s. Expected: %s, Actual: %s", + funcName, + ascending, + expressions, + expectedClass.getSimpleName(), + accumulator.getClass().getSimpleName()); + + assertTrue(msg, expectedClass.isInstance(accumulator)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java index e425c709815c3..a22ab8f6739ae 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java @@ -41,7 +41,7 @@ public void testLoadSingleTsFileNode() { TsFileResource resource = new TsFileResource(new File("1")); String database = "root.db"; LoadSingleTsFileNode node = - new LoadSingleTsFileNode(new PlanNodeId(""), resource, false, database, true, 0L); + new LoadSingleTsFileNode(new PlanNodeId(""), resource, false, database, true, 0L, false); Assert.assertTrue(node.isDeleteAfterLoad()); Assert.assertEquals(resource, node.getTsFileResource()); Assert.assertEquals(database, node.getDatabase()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java index 6666c180c5e04..36bed6932a877 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java @@ -96,6 +96,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -1058,12 +1059,13 @@ private Metadata mockMetadataForInsertion() { DataNodeTableCache.getInstance().commitUpdateTable(database, table, null); return new TestMetadata() { @Override - public Optional validateTableHeaderSchema( + public Optional validateTableHeaderSchema4TsFile( String database, TableSchema schema, MPPQueryContext context, boolean allowCreateTable, - boolean isStrictIdColumn) { + boolean isStrictTagColumn, + final AtomicBoolean needDecode4DifferentTimeColumn) { TableSchema tableSchema = StatementTestUtils.genTableSchema(); assertEquals(tableSchema, schema); return Optional.of(tableSchema); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSMetadata.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSMetadata.java index 79c031560973a..49f23e0358c59 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSMetadata.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSMetadata.java @@ -64,6 +64,7 @@ import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.MockTSBSDataPartition.T1_DEVICE_1; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.MockTSBSDataPartition.T1_DEVICE_2; @@ -342,12 +343,13 @@ public Map> indexScan( } @Override - public Optional validateTableHeaderSchema( + public Optional validateTableHeaderSchema4TsFile( String database, TableSchema tableSchema, MPPQueryContext context, boolean allowCreateTable, - boolean isStrictIdColumn) { + boolean isStrictTagColumn, + final AtomicBoolean needDecode4DifferentTimeColumn) { throw new UnsupportedOperationException(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMetadata.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMetadata.java index 4b1d18944b732..722212b78b78a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMetadata.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMetadata.java @@ -80,6 +80,7 @@ import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import static org.apache.iotdb.commons.schema.table.InformationSchema.INFORMATION_DATABASE; @@ -481,12 +482,13 @@ private boolean compareNotEqualsMatch( } @Override - public Optional validateTableHeaderSchema( + public Optional validateTableHeaderSchema4TsFile( final String database, final TableSchema tableSchema, final MPPQueryContext context, final boolean allowCreateTable, - final boolean isStrictIdColumn) { + final boolean isStrictTagColumn, + final AtomicBoolean needDecode4DifferentTimeColumn) { throw new UnsupportedOperationException(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java index 81a9d88d3d3c4..dbbadd1be91fb 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java @@ -51,6 +51,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertThrows; @@ -100,12 +101,13 @@ public void setUp() throws Exception { DataNodeTableCache.getInstance().preUpdateTable("test", tsTable, null); DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null); - when(metadata.validateTableHeaderSchema( + when(metadata.validateTableHeaderSchema4TsFile( any(String.class), any(TableSchema.class), any(MPPQueryContext.class), any(Boolean.class), - any(Boolean.class))) + any(Boolean.class), + any(AtomicBoolean.class))) .thenReturn(Optional.of(tableSchema)); doAnswer( @@ -252,12 +254,13 @@ public void testConflictCategory() { DataNodeTableCache.getInstance().preUpdateTable("test", tsTable, null); DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null); - when(metadata.validateTableHeaderSchema( + when(metadata.validateTableHeaderSchema4TsFile( any(String.class), any(TableSchema.class), any(MPPQueryContext.class), any(Boolean.class), - any(Boolean.class))) + any(Boolean.class), + any(AtomicBoolean.class))) .thenReturn(Optional.of(tableSchema)); assertThrows( @@ -287,12 +290,13 @@ public void testMissingIdColumn() { DataNodeTableCache.getInstance().preUpdateTable("test", tsTable, null); DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null); - when(metadata.validateTableHeaderSchema( + when(metadata.validateTableHeaderSchema4TsFile( any(String.class), any(TableSchema.class), any(MPPQueryContext.class), any(Boolean.class), - any(Boolean.class))) + any(Boolean.class), + any(AtomicBoolean.class))) .thenReturn(Optional.of(tableSchema)); assertThrows( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java index 3c15b2eaa281d..ac6c8ebab24f2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java @@ -69,7 +69,6 @@ public class InformationSchema { new AttributeColumnSchema(ColumnHeaderConstant.STATEMENT_TABLE_MODEL, TSDataType.STRING)); queriesTable.addColumnSchema( new AttributeColumnSchema(ColumnHeaderConstant.USER_TABLE_MODEL, TSDataType.STRING)); - queriesTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(QUERIES, queriesTable); final TsTable databaseTable = new TsTable(DATABASES); @@ -94,7 +93,6 @@ public class InformationSchema { databaseTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.DATA_REGION_GROUP_NUM_TABLE_MODEL, TSDataType.INT32)); - databaseTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(DATABASES, databaseTable); final TsTable tableTable = new TsTable(TABLES); @@ -114,7 +112,6 @@ public class InformationSchema { ColumnHeaderConstant.COMMENT.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); tableTable.addColumnSchema( new AttributeColumnSchema(ColumnHeaderConstant.TABLE_TYPE_TABLE_MODEL, TSDataType.STRING)); - tableTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(TABLES, tableTable); final TsTable columnTable = new TsTable(COLUMNS); @@ -137,7 +134,6 @@ public class InformationSchema { columnTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.COMMENT.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); - columnTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(COLUMNS, columnTable); final TsTable regionTable = new TsTable(REGIONS); @@ -179,7 +175,6 @@ public class InformationSchema { regionTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.COMPRESSION_RATIO_TABLE_MODEL, TSDataType.DOUBLE)); - regionTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(REGIONS, regionTable); final TsTable pipeTable = new TsTable(PIPES); @@ -208,7 +203,6 @@ public class InformationSchema { pipeTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.ESTIMATED_REMAINING_SECONDS_TABLE_MODEL, TSDataType.DOUBLE)); - pipeTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(PIPES, pipeTable); final TsTable pipePluginTable = new TsTable(PIPE_PLUGINS); @@ -220,7 +214,6 @@ public class InformationSchema { new AttributeColumnSchema(ColumnHeaderConstant.CLASS_NAME_TABLE_MODEL, TSDataType.STRING)); pipePluginTable.addColumnSchema( new AttributeColumnSchema(ColumnHeaderConstant.PLUGIN_JAR_TABLE_MODEL, TSDataType.STRING)); - pipePluginTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(PIPE_PLUGINS, pipePluginTable); final TsTable topicTable = new TsTable(TOPICS); @@ -229,7 +222,6 @@ public class InformationSchema { topicTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.TOPIC_CONFIGS_TABLE_MODEL, TSDataType.STRING)); - topicTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(TOPICS, topicTable); final TsTable subscriptionTable = new TsTable(SUBSCRIPTIONS); @@ -241,7 +233,6 @@ public class InformationSchema { subscriptionTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.SUBSCRIBED_CONSUMERS_TABLE_MODEL, TSDataType.STRING)); - subscriptionTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(SUBSCRIPTIONS, subscriptionTable); final TsTable viewTable = new TsTable(VIEWS); @@ -253,7 +244,6 @@ public class InformationSchema { viewTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.VIEW_DEFINITION_TABLE_MODEL, TSDataType.STRING)); - viewTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(VIEWS, viewTable); final TsTable functionTable = new TsTable(FUNCTIONS); @@ -268,7 +258,6 @@ public class InformationSchema { functionTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.STATE.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); - functionTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(FUNCTIONS, functionTable); final TsTable configurationsTable = new TsTable(CONFIGURATIONS); @@ -278,7 +267,6 @@ public class InformationSchema { configurationsTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.VALUE.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); - configurationsTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(CONFIGURATIONS, configurationsTable); final TsTable keywordsTable = new TsTable(KEYWORDS); @@ -286,7 +274,6 @@ public class InformationSchema { new TagColumnSchema(ColumnHeaderConstant.WORD, TSDataType.STRING)); keywordsTable.addColumnSchema( new AttributeColumnSchema(ColumnHeaderConstant.RESERVED, TSDataType.INT32)); - keywordsTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(KEYWORDS, keywordsTable); final TsTable nodesTable = new TsTable(NODES); @@ -308,7 +295,6 @@ public class InformationSchema { ColumnHeaderConstant.VERSION.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); nodesTable.addColumnSchema( new AttributeColumnSchema(ColumnHeaderConstant.BUILD_INFO_TABLE_MODEL, TSDataType.STRING)); - nodesTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(NODES, nodesTable); final TsTable configNodesTable = new TsTable(CONFIG_NODES); @@ -320,7 +306,6 @@ public class InformationSchema { configNodesTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.ROLE.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); - configNodesTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(CONFIG_NODES, configNodesTable); final TsTable dataNodesTable = new TsTable(DATA_NODES); @@ -344,7 +329,6 @@ public class InformationSchema { dataNodesTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.SCHEMA_CONSENSUS_PORT_TABLE_MODEL, TSDataType.INT32)); - dataNodesTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(DATA_NODES, dataNodesTable); final TsTable connectionsTable = new TsTable(CONNECTIONS); @@ -360,7 +344,6 @@ public class InformationSchema { new AttributeColumnSchema(ColumnHeaderConstant.LAST_ACTIVE_TIME, TSDataType.TIMESTAMP)); connectionsTable.addColumnSchema( new AttributeColumnSchema(ColumnHeaderConstant.CLIENT_IP, TSDataType.STRING)); - connectionsTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(CONNECTIONS, connectionsTable); final TsTable currentQueriesTable = new TsTable(CURRENT_QUERIES); @@ -383,7 +366,6 @@ public class InformationSchema { new AttributeColumnSchema(ColumnHeaderConstant.USER_TABLE_MODEL, TSDataType.STRING)); currentQueriesTable.addColumnSchema( new AttributeColumnSchema(ColumnHeaderConstant.CLIENT_IP, TSDataType.STRING)); - currentQueriesTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(CURRENT_QUERIES, currentQueriesTable); final TsTable queriesCostsHistogramTable = new TsTable(QUERIES_COSTS_HISTOGRAM); @@ -393,7 +375,6 @@ public class InformationSchema { new AttributeColumnSchema(ColumnHeaderConstant.NUMS, TSDataType.INT32)); queriesCostsHistogramTable.addColumnSchema( new AttributeColumnSchema(ColumnHeaderConstant.DATANODE_ID, TSDataType.INT32)); - queriesCostsHistogramTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(QUERIES_COSTS_HISTOGRAM, queriesCostsHistogramTable); final TsTable servicesTable = new TsTable(SERVICES); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsFileTableSchemaUtil.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsFileTableSchemaUtil.java index 9a5abffbbcc65..1536dce04c2e1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsFileTableSchemaUtil.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsFileTableSchemaUtil.java @@ -34,6 +34,8 @@ import java.util.HashMap; import java.util.List; +import static org.apache.iotdb.commons.schema.table.TsTable.TIME_COLUMN_NAME; + /** Utility class for converting between TsTable and TSFile TableSchema */ public class TsFileTableSchemaUtil { @@ -43,9 +45,8 @@ private TsFileTableSchemaUtil() { /** Column category filter for efficient parsing */ public enum ColumnCategoryFilter { - /** Include TAG and FIELD only (exclude TIME and ATTRIBUTE) - for TsFile writing */ - NO_ATTRIBUTE( - cat -> cat != TsTableColumnCategory.TIME && cat != TsTableColumnCategory.ATTRIBUTE); + /** Include TAG, time, and FIELD only (exclude ATTRIBUTE) - for TsFile writing */ + NO_ATTRIBUTE(cat -> cat != TsTableColumnCategory.ATTRIBUTE); private final java.util.function.Predicate predicate; @@ -89,6 +90,14 @@ private static TableSchema tsTableBufferToTableSchemaInternal( final String columnName = ReadWriteIOUtils.readString(tsTableBuffer); final TSDataType dataType = ReadWriteIOUtils.readDataType(tsTableBuffer); + // if the time column position in first column and named as "time", skip it + if (i == 0 + && category == TsTableColumnCategory.TIME + && columnName.equalsIgnoreCase(TIME_COLUMN_NAME)) { + skipMap(tsTableBuffer); + continue; + } + if (category == TsTableColumnCategory.FIELD) { ReadWriteIOUtils.readEncoding(tsTableBuffer); ReadWriteIOUtils.readCompressionType(tsTableBuffer); @@ -157,11 +166,20 @@ public static TableSchema toTsFileTableSchemaNoAttribute(final TsTable table) { // Directly iterate through columns and filter out TIME and ATTRIBUTE columns int columnIndex = 0; - for (final TsTableColumnSchema columnSchema : tsTableColumnSchemas) { + + for (int i = 0; i < tsTableColumnSchemas.size(); i++) { + TsTableColumnSchema columnSchema = tsTableColumnSchemas.get(i); final TsTableColumnCategory category = columnSchema.getColumnCategory(); - // Skip TIME and ATTRIBUTE columns (only include TAG and FIELD) - if (category == TsTableColumnCategory.TIME || category == TsTableColumnCategory.ATTRIBUTE) { + // if the time columns is named as "time" and in first position, drop it + if (i == 0 + && category == TsTableColumnCategory.TIME + && columnSchema.getColumnName().equalsIgnoreCase(TIME_COLUMN_NAME)) { + continue; + } + + // Skip ATTRIBUTE columns (only include TIME, TAG and FIELD) + if (category == TsTableColumnCategory.ATTRIBUTE) { continue; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index 1b5a9d4ac34aa..8f484f4e231d5 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -25,14 +25,12 @@ import org.apache.iotdb.commons.schema.table.column.AttributeColumnSchema; import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; -import org.apache.iotdb.commons.schema.table.column.TimeColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import com.google.common.collect.ImmutableList; -import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -64,9 +62,6 @@ public class TsTable { public static final String TIME_COLUMN_NAME = "time"; public static final String COMMENT_KEY = "__comment"; - private static final TimeColumnSchema TIME_COLUMN_SCHEMA = - new TimeColumnSchema(TIME_COLUMN_NAME, TSDataType.TIMESTAMP); - public static final String TTL_PROPERTY = "ttl"; public static final Set TABLE_ALLOWED_PROPERTIES = Collections.singleton(TTL_PROPERTY); private static final String OBJECT_STRING_ERROR = @@ -101,7 +96,6 @@ public class TsTable { public TsTable(final String tableName) { this.tableName = tableName; - columnSchemaMap.put(TIME_COLUMN_NAME, TIME_COLUMN_SCHEMA); } // This interface is used by InformationSchema table, so time column is not necessary diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnCategory.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnCategory.java index ef620f3673a67..4b171a89c5c3d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnCategory.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnCategory.java @@ -82,6 +82,8 @@ public ColumnCategory toTsFileColumnType() { return ColumnCategory.ATTRIBUTE; case FIELD: return ColumnCategory.FIELD; + case TIME: + return ColumnCategory.TIME; default: throw new IllegalArgumentException("Unsupported column type in TsFile: " + this); } @@ -95,6 +97,8 @@ public static TsTableColumnCategory fromTsFileColumnCategory(ColumnCategory colu return TAG; case ATTRIBUTE: return ATTRIBUTE; + case TIME: + return TIME; default: throw new IllegalArgumentException("Unknown column type: " + columnType); } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/TsFileTableSchemaUtilTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/TsFileTableSchemaUtilTest.java index 4966b436a6e8e..f39eca4348d7f 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/TsFileTableSchemaUtilTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/TsFileTableSchemaUtilTest.java @@ -325,9 +325,6 @@ public void testColumnCategoryFilter() { final TsFileTableSchemaUtil.ColumnCategoryFilter filter = TsFileTableSchemaUtil.ColumnCategoryFilter.NO_ATTRIBUTE; - // TIME should be filtered out - Assert.assertFalse("TIME should be filtered out", filter.test(TsTableColumnCategory.TIME)); - // ATTRIBUTE should be filtered out Assert.assertFalse( "ATTRIBUTE should be filtered out", filter.test(TsTableColumnCategory.ATTRIBUTE)); @@ -340,8 +337,7 @@ public void testColumnCategoryFilter() { // Test all enum values for (TsTableColumnCategory category : TsTableColumnCategory.values()) { - final boolean shouldInclude = - category != TsTableColumnCategory.TIME && category != TsTableColumnCategory.ATTRIBUTE; + final boolean shouldInclude = category != TsTableColumnCategory.ATTRIBUTE; Assert.assertEquals( "Category " + category + " filter result mismatch", shouldInclude, filter.test(category)); } @@ -641,11 +637,10 @@ public void testToTsFileTableSchemaNoAttributeWithEmptyTable() { public void testToTsFileTableSchemaNoAttributeWithMixedOrder() { // Test with columns in mixed order (TIME, TAG, ATTRIBUTE, FIELD, etc.) final TsTable table = new TsTable("mixedOrderTable"); - table.addColumnSchema(new TimeColumnSchema("time", TSDataType.INT64)); table.addColumnSchema(new TagColumnSchema("tag1", TSDataType.STRING)); table.addColumnSchema(new AttributeColumnSchema("attr1", TSDataType.STRING)); table.addColumnSchema(new FieldColumnSchema("field1", TSDataType.DOUBLE)); - table.addColumnSchema(new TimeColumnSchema("time2", TSDataType.INT64)); + table.addColumnSchema(new TimeColumnSchema("time2", TSDataType.TIMESTAMP)); table.addColumnSchema(new TagColumnSchema("tag2", TSDataType.STRING)); table.addColumnSchema(new AttributeColumnSchema("attr2", TSDataType.STRING)); table.addColumnSchema(new FieldColumnSchema("field2", TSDataType.INT32)); @@ -653,7 +648,7 @@ public void testToTsFileTableSchemaNoAttributeWithMixedOrder() { final TableSchema result = TsFileTableSchemaUtil.toTsFileTableSchemaNoAttribute(table); // Should have 2 TAG + 2 FIELD = 4 columns - Assert.assertEquals("Should have 4 columns", 4, result.getColumnSchemas().size()); + Assert.assertEquals("Should have 5 columns", 5, result.getColumnSchemas().size()); // Verify order: tag1, tag2, field1, field2 (original order preserved) final List columnNames = @@ -662,8 +657,9 @@ public void testToTsFileTableSchemaNoAttributeWithMixedOrder() { .collect(Collectors.toList()); Assert.assertEquals("tag1", columnNames.get(0)); Assert.assertEquals("field1", columnNames.get(1)); - Assert.assertEquals("tag2", columnNames.get(2)); - Assert.assertEquals("field2", columnNames.get(3)); + Assert.assertEquals("time2", columnNames.get(2)); + Assert.assertEquals("tag2", columnNames.get(3)); + Assert.assertEquals("field2", columnNames.get(4)); } @Test diff --git a/pom.xml b/pom.xml index c3b80202a01ea..2e576729c40b0 100644 --- a/pom.xml +++ b/pom.xml @@ -173,7 +173,7 @@ 0.14.1 1.9 1.5.6-3 - 2.2.1-260115-SNAPSHOT + 2.2.1-260120-SNAPSHOT