From a4ae3d1fadef813f02c306460ece8786a488108e Mon Sep 17 00:00:00 2001 From: RemHero <1104304963@qq.com> Date: Tue, 4 Jun 2024 22:55:00 +0800 Subject: [PATCH 01/11] fix show columns --- .../engine/physical/storage/IStorage.java | 4 +- .../execute/StoragePhysicalTaskExecutor.java | 43 ++++++------------- .../iginx/filesystem/FileSystemStorage.java | 7 ++- .../iginx/filesystem/exec/Executor.java | 3 +- .../iginx/filesystem/exec/LocalExecutor.java | 39 +++++++++++++---- .../iginx/filesystem/exec/RemoteExecutor.java | 12 +++--- .../filesystem/server/FileSystemWorker.java | 4 +- .../tsinghua/iginx/iotdb/IoTDBStorage.java | 29 ++++++++++--- .../expansion/BaseCapacityExpansionIT.java | 17 ++++++++ .../filesystem}/datasource/DataSourceIT.java | 2 +- thrift/src/main/proto/filesystem.thrift | 2 +- 11 files changed, 104 insertions(+), 58 deletions(-) rename test/src/test/java/cn/edu/tsinghua/iginx/integration/{ => expansion/filesystem}/datasource/DataSourceIT.java (99%) diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/IStorage.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/IStorage.java index b2d83e47c7..dda2f745f7 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/IStorage.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/IStorage.java @@ -26,10 +26,12 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.Insert; import cn.edu.tsinghua.iginx.engine.shared.operator.Project; import cn.edu.tsinghua.iginx.engine.shared.operator.Select; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; import cn.edu.tsinghua.iginx.utils.Pair; import java.util.List; +import java.util.Set; public interface IStorage { /** 对非叠加分片查询数据 */ @@ -55,7 +57,7 @@ TaskExecuteResult executeProjectDummyWithSelect( TaskExecuteResult executeInsert(Insert insert, DataArea dataArea); /** 获取所有列信息 */ - List getColumns() throws PhysicalException; + List getColumns(Set pattern, TagFilter tagFilter) throws PhysicalException; /** 获取指定前缀的数据边界 */ Pair getBoundaryOfStorage(String prefix) throws PhysicalException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java index 074e5f1834..061cedc8a7 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java @@ -31,7 +31,6 @@ import cn.edu.tsinghua.iginx.engine.physical.storage.domain.Column; import cn.edu.tsinghua.iginx.engine.physical.storage.domain.DataArea; import cn.edu.tsinghua.iginx.engine.physical.storage.queue.StoragePhysicalTaskQueue; -import cn.edu.tsinghua.iginx.engine.physical.storage.utils.TagKVUtils; import cn.edu.tsinghua.iginx.engine.physical.task.GlobalPhysicalTask; import cn.edu.tsinghua.iginx.engine.physical.task.MemoryPhysicalTask; import cn.edu.tsinghua.iginx.engine.physical.task.StoragePhysicalTask; @@ -54,13 +53,12 @@ import cn.edu.tsinghua.iginx.monitor.HotSpotMonitor; import cn.edu.tsinghua.iginx.monitor.RequestsMonitor; import cn.edu.tsinghua.iginx.utils.Pair; -import cn.edu.tsinghua.iginx.utils.StringUtils; + import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; -import java.util.regex.Pattern; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -324,7 +322,12 @@ public TaskExecuteResult executeShowColumns(ShowColumns showColumns) { continue; } try { - List columnList = pair.k.getColumns(); + Set patternSet = showColumns.getPathRegexSet(); + TagFilter tagFilter = showColumns.getTagFilter(); + if (storage.getDataPrefix() != null) { + patternSet.add(storage.getDataPrefix()+".*"); + } + List columnList = pair.k.getColumns(patternSet, tagFilter); // fix the schemaPrefix String schemaPrefix = storage.getSchemaPrefix(); if (schemaPrefix != null) { @@ -340,40 +343,18 @@ public TaskExecuteResult executeShowColumns(ShowColumns showColumns) { } } - Set pathRegexSet = showColumns.getPathRegexSet(); - TagFilter tagFilter = showColumns.getTagFilter(); - - TreeSet tsSetAfterFilter = new TreeSet<>(Comparator.comparing(Column::getPhysicalPath)); - for (Column column : columnSet) { - boolean isTarget = true; - if (!pathRegexSet.isEmpty()) { - isTarget = false; - for (String pathRegex : pathRegexSet) { - if (Pattern.matches(StringUtils.reformatPath(pathRegex), column.getPath())) { - isTarget = true; - break; - } - } - } - if (tagFilter != null) { - if (!TagKVUtils.match(column.getTags(), tagFilter)) { - isTarget = false; - } - } - if (isTarget) { - tsSetAfterFilter.add(column); - } - } + TreeSet columnSetAfterFilter = new TreeSet<>(Comparator.comparing(Column::getPhysicalPath)); + columnSetAfterFilter.addAll(columnSet); int limit = showColumns.getLimit(); int offset = showColumns.getOffset(); if (limit == Integer.MAX_VALUE && offset == 0) { - return new TaskExecuteResult(Column.toRowStream(tsSetAfterFilter)); + return new TaskExecuteResult(Column.toRowStream(columnSetAfterFilter)); } else { // only need part of data. List tsList = new ArrayList<>(); - int cur = 0, size = tsSetAfterFilter.size(); - for (Iterator iter = tsSetAfterFilter.iterator(); iter.hasNext(); cur++) { + int cur = 0, size = columnSetAfterFilter.size(); + for (Iterator iter = columnSetAfterFilter.iterator(); iter.hasNext(); cur++) { if (cur >= size || cur - offset >= limit) { break; } diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/FileSystemStorage.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/FileSystemStorage.java index 9c32e4fd9d..b42e783c7b 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/FileSystemStorage.java +++ b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/FileSystemStorage.java @@ -32,6 +32,7 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.KeyFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.filesystem.exec.Executor; import cn.edu.tsinghua.iginx.filesystem.exec.LocalExecutor; import cn.edu.tsinghua.iginx.filesystem.exec.RemoteExecutor; @@ -43,6 +44,8 @@ import cn.edu.tsinghua.iginx.utils.Pair; import java.util.Arrays; import java.util.List; +import java.util.Set; + import org.apache.thrift.transport.TTransportException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -149,8 +152,8 @@ public TaskExecuteResult executeDelete(Delete delete, DataArea dataArea) { } @Override - public List getColumns() throws PhysicalException { - return executor.getColumnsOfStorageUnit(WILDCARD); + public List getColumns(Set pattern, TagFilter tagFilter) throws PhysicalException { + return executor.getColumnsOfStorageUnit(WILDCARD, pattern, tagFilter); } @Override diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/Executor.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/Executor.java index 5c59a40c50..66926d37d8 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/Executor.java +++ b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/Executor.java @@ -11,6 +11,7 @@ import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; import cn.edu.tsinghua.iginx.utils.Pair; import java.util.List; +import java.util.Set; public interface Executor { @@ -26,7 +27,7 @@ TaskExecuteResult executeProjectTask( TaskExecuteResult executeDeleteTask( List paths, List keyRanges, TagFilter tagFilter, String storageUnit); - List getColumnsOfStorageUnit(String storageUnit) throws PhysicalException; + List getColumnsOfStorageUnit(String storageUnit, Set pattern, TagFilter tagFilter) throws PhysicalException; Pair getBoundaryOfStorage(String dataPrefix) throws PhysicalException; diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java index e7510fb036..ff0787193a 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java +++ b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java @@ -7,6 +7,7 @@ import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream.EmptyRowStream; import cn.edu.tsinghua.iginx.engine.physical.storage.domain.Column; +import cn.edu.tsinghua.iginx.engine.physical.storage.utils.TagKVUtils; import cn.edu.tsinghua.iginx.engine.physical.task.TaskExecuteResult; import cn.edu.tsinghua.iginx.engine.shared.KeyRange; import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; @@ -34,6 +35,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +import cn.edu.tsinghua.iginx.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -309,27 +314,45 @@ public TaskExecuteResult executeDeleteTask( } @Override - public List getColumnsOfStorageUnit(String storageUnit) throws PhysicalException { + public List getColumnsOfStorageUnit(String storageUnit, Set pattern, TagFilter tagFilter) throws PhysicalException { List columns = new ArrayList<>(); if (root != null) { File directory = new File(FilePathUtils.toIginxPath(root, storageUnit, null)); for (File file : fileSystemManager.getAllFiles(directory, false)) { FileMeta meta = fileSystemManager.getFileMeta(file); + String columnPath = FilePathUtils.convertAbsolutePathToPath(root, file.getAbsolutePath(), storageUnit); + boolean isChosen = true; if (meta == null) { throw new PhysicalException( String.format( "encounter error when getting columns of storage unit because file meta %s is null", file.getAbsolutePath())); } - columns.add( - new Column( - FilePathUtils.convertAbsolutePathToPath(root, file.getAbsolutePath(), storageUnit), - meta.getDataType(), - meta.getTags(), - false)); + // get columns by pattern + if (!pattern.isEmpty()) { + for (String pathRegex : pattern) { + if (!Pattern.matches(StringUtils.reformatPath(pathRegex), columnPath)) { + isChosen = false; + break; + } + } + } + if (!isChosen) { + continue; + } + // get columns by tag filter + if (tagFilter != null && !TagKVUtils.match(meta.getTags(), tagFilter)) { + columns.add( + new Column( + columnPath, + meta.getDataType(), + meta.getTags(), + false)); + } } } - if (hasData && dummyRoot != null) { + // get columns from dummy storage unit + if (hasData && dummyRoot != null && tagFilter==null) { for (File file : fileSystemManager.getAllFiles(new File(realDummyRoot), true)) { columns.add( new Column( diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/RemoteExecutor.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/RemoteExecutor.java index 32c9c31087..2c55bf3a2d 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/RemoteExecutor.java +++ b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/RemoteExecutor.java @@ -28,10 +28,7 @@ import cn.edu.tsinghua.iginx.utils.Pair; import cn.edu.tsinghua.iginx.utils.ThriftConnPool; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import org.apache.thrift.TException; import org.apache.thrift.protocol.TBinaryProtocol; @@ -207,11 +204,11 @@ public TaskExecuteResult executeDeleteTask( } @Override - public List getColumnsOfStorageUnit(String storageUnit) throws PhysicalException { + public List getColumnsOfStorageUnit(String storageUnit, Set pattern, TagFilter tagFilter) throws PhysicalException { try { TTransport transport = thriftConnPool.borrowTransport(); Client client = new Client(new TBinaryProtocol(transport)); - GetColumnsOfStorageUnitResp resp = client.getColumnsOfStorageUnit(storageUnit); + GetColumnsOfStorageUnitResp resp = client.getColumnsOfStorageUnit(storageUnit, pattern, constructRawTagFilter(tagFilter)); thriftConnPool.returnTransport(transport); List columns = new ArrayList<>(); resp.getPathList() @@ -254,6 +251,9 @@ public void close() { private RawTagFilter constructRawTagFilter(TagFilter tagFilter) { RawTagFilter filter = null; + if(tagFilter == null) { + return null; + } switch (tagFilter.getType()) { case Base: { diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemWorker.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemWorker.java index 73383ae362..1503c8397e 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemWorker.java +++ b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemWorker.java @@ -198,10 +198,10 @@ public Status executeDelete(DeleteReq req) throws TException { } @Override - public GetColumnsOfStorageUnitResp getColumnsOfStorageUnit(String storageUnit) throws TException { + public GetColumnsOfStorageUnitResp getColumnsOfStorageUnit(String storageUnit, Set pattern, RawTagFilter tagFilter) throws TException { List ret = new ArrayList<>(); try { - List columns = executor.getColumnsOfStorageUnit(storageUnit); + List columns = executor.getColumnsOfStorageUnit(storageUnit, pattern, resolveRawTagFilter(tagFilter)); columns.forEach( column -> { FSColumn fsColumn = diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java index 74be91517d..b2e1cb4705 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java +++ b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java @@ -192,13 +192,13 @@ public void release() throws PhysicalException { } @Override - public List getColumns() throws PhysicalException { + public List getColumns(Set pattern, TagFilter tagFilter) throws PhysicalException { List columns = new ArrayList<>(); - getColumns2StorageUnit(columns, null); + getColumns2StorageUnit(columns, null, pattern, tagFilter); return columns; } - private void getColumns2StorageUnit(List columns, Map columns2StorageUnit) + private void getColumns2StorageUnit(List columns, Map columns2StorageUnit, Set pattern, TagFilter tagFilter) throws PhysicalException { try { SessionDataSetWrapper dataSet = sessionPool.executeQueryStatement(SHOW_TIMESERIES); @@ -221,6 +221,25 @@ private void getColumns2StorageUnit(List columns, Map co if (columns2StorageUnit != null) { columns2StorageUnit.put(pair.k, fragment); } + boolean isChosen = true; + // get columns by pattern + if (!pattern.isEmpty()) { + for (String pathRegex : pattern) { + if (!Pattern.matches(StringUtils.reformatPath(pathRegex), pair.k)) { + isChosen = false; + break; + } + } + } else { + if (isDummy) continue; + } + if (!isChosen) { + continue; + } + // get columns by tag filter + if (tagFilter != null && !TagKVUtils.match(pair.v, tagFilter)) { + continue; + } switch (dataTypeName) { case "BOOLEAN": @@ -790,7 +809,7 @@ private List determineDeletePathList(String storageUnit, Delete delete) } else { List patterns = delete.getPatterns(); TagFilter tagFilter = delete.getTagFilter(); - List timeSeries = getColumns(); + List timeSeries = getColumns(new HashSet<>(), null); List pathList = new ArrayList<>(); for (Column ts : timeSeries) { @@ -859,7 +878,7 @@ private String getFilterString(Filter filter, String storageUnit) throws Physica if (filterStr.contains("*")) { List columns = new ArrayList<>(); Map columns2Fragment = new HashMap<>(); - getColumns2StorageUnit(columns, columns2Fragment); + getColumns2StorageUnit(columns, columns2Fragment, new HashSet<>(), null); filterStr = FilterTransformer.toString( expandFilterWildcard(filter.copy(), columns, columns2Fragment, storageUnit)); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java index 1577c7587b..ece1882bf6 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java @@ -14,6 +14,7 @@ import cn.edu.tsinghua.iginx.integration.expansion.utils.SQLTestTools; import cn.edu.tsinghua.iginx.integration.tool.ConfLoader; import cn.edu.tsinghua.iginx.session.ClusterInfo; +import cn.edu.tsinghua.iginx.session.Column; import cn.edu.tsinghua.iginx.session.QueryDataSet; import cn.edu.tsinghua.iginx.session.Session; import cn.edu.tsinghua.iginx.thrift.RemovedStorageEngineInfo; @@ -465,6 +466,14 @@ private void testAddAndRemoveStorageEngineWithPrefix() { List> valuesList = EXP_VALUES_LIST1; + // 测试 show columns + try { + List columns = session.showColumns(); + LOGGER.info("columns: {}", columns); + } catch (SessionException e) { + LOGGER.error("show columns error: ", e); + } + // 添加不同 schemaPrefix，相同 dataPrefix addStorageEngine(expPort, true, true, dataPrefix1, schemaPrefix1, extraParams); @@ -473,6 +482,14 @@ private void testAddAndRemoveStorageEngineWithPrefix() { List pathList = Arrays.asList("nt.wf03.wt01.status2", "p1.nt.wf03.wt01.status2"); SQLTestTools.executeAndCompare(session, statement, pathList, REPEAT_EXP_VALUES_LIST1); + // 测试添加节点后的 show columns + try { + List columns = session.showColumns(); + LOGGER.info("columns: {}", columns); + } catch (SessionException e) { + LOGGER.error("show columns error: ", e); + } + addStorageEngine(expPort, true, true, dataPrefix1, schemaPrefix2, extraParams); addStorageEngine(expPort, true, true, dataPrefix1, null, extraParams); testShowClusterInfo(5); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/datasource/DataSourceIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/datasource/DataSourceIT.java similarity index 99% rename from test/src/test/java/cn/edu/tsinghua/iginx/integration/datasource/DataSourceIT.java rename to test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/datasource/DataSourceIT.java index aede81abd9..a004fbdadd 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/datasource/DataSourceIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/datasource/DataSourceIT.java @@ -1,4 +1,4 @@ -package cn.edu.tsinghua.iginx.integration.datasource; +package cn.edu.tsinghua.iginx.integration.expansion.filesystem.datasource; import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; diff --git a/thrift/src/main/proto/filesystem.thrift b/thrift/src/main/proto/filesystem.thrift index 51a6a5f180..384d407ff6 100644 --- a/thrift/src/main/proto/filesystem.thrift +++ b/thrift/src/main/proto/filesystem.thrift @@ -157,7 +157,7 @@ service FileSystemService { Status executeDelete(1: DeleteReq req); - GetColumnsOfStorageUnitResp getColumnsOfStorageUnit(1: string storageUnit); + GetColumnsOfStorageUnitResp getColumnsOfStorageUnit(1: string storageUnit, 2: set patterns, 3: RawTagFilter tagFilter); GetBoundaryOfStorageResp getBoundaryOfStorage(1: string dataPrefix); From a406f2ff409d995951ae53cd601bb902d13da708 Mon Sep 17 00:00:00 2001 From: RemHero <1104304963@qq.com> Date: Tue, 4 Jun 2024 22:56:36 +0800 Subject: [PATCH 02/11] test --- .github/workflows/DB-CE.yml | 2 +- .../workflows/standalone-test-pushdown.yml | 208 +++++----- .github/workflows/standalone-test.yml | 380 +++++++++--------- dataSources/pom.xml | 10 +- test/pom.xml | 8 +- 5 files changed, 304 insertions(+), 304 deletions(-) diff --git a/.github/workflows/DB-CE.yml b/.github/workflows/DB-CE.yml index a6a941df54..c331810ec8 100644 --- a/.github/workflows/DB-CE.yml +++ b/.github/workflows/DB-CE.yml @@ -27,7 +27,7 @@ on: description: "The database to run the test on" type: string required: false - default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' + default: '["FileSystem", "IoTDB12"]' env: VERSION: 0.6.0-SNAPSHOT diff --git a/.github/workflows/standalone-test-pushdown.yml b/.github/workflows/standalone-test-pushdown.yml index c4116604e3..cebca0d776 100644 --- a/.github/workflows/standalone-test-pushdown.yml +++ b/.github/workflows/standalone-test-pushdown.yml @@ -1,104 +1,104 @@ -name: "Union Database Test With Push Down" - -on: - workflow_call: - inputs: - java-matrix: - description: "The java version to run the test on" - type: string - required: false - default: '["8"]' - python-matrix: - description: "The python version to run the test on" - type: string - required: false - default: '["3.9"]' - os-matrix: - description: "The operating system to run the test on" - type: string - required: false - default: '["ubuntu-latest", "macos-13", "windows-latest"]' - metadata-matrix: - description: "The metadata to run the test on" - type: string - required: false - default: '["zookeeper", "etcd"]' - db-matrix: - description: "The database to run the test on" - type: string - required: false - default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' - -env: - VERSION: 0.6.0-SNAPSHOT - -jobs: - Union-DB-Test-Push_Down: - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - java: ${{ fromJSON(inputs.java-matrix) }} - python-version: ${{ fromJSON(inputs.python-matrix) }} - os: ${{ fromJSON(inputs.os-matrix) }} - metadata: ${{ fromJSON(inputs.metadata-matrix) }} - DB-name: ${{ fromJSON(inputs.db-matrix) }} - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} - - - if: runner.os == 'Windows' - name: Set JAVA_OPTS - run: echo "JAVA_OPTS=-Xmx4g -Xmx2g" >> $GITHUB_ENV - - - name: Run Metadata - uses: ./.github/actions/metadataRunner - with: - metadata: ${{ matrix.metadata }} - - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: ${{ matrix.DB-name }} - - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: ${{ matrix.DB-name }} - Push-Down: "true" - Set-Filter-Fragment-OFF: "true" - Metadata: ${{ matrix.metadata }} - - - name: Start IGinX - uses: ./.github/actions/iginxRunner - - - name: TestController IT - if: always() - shell: bash - env: - METADATA_STORAGE: ${{ matrix.metadata }} - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" - mvn test -q -Dtest=Controller -DfailIfNoTests=false -P-format - - - name: Show test result - if: always() - shell: bash - run: | - cat ${GITHUB_WORKSPACE}/test/src/test/resources/testResult.txt - - - name: Show IGinX log - if: always() - shell: bash - run: | - cat iginx-*.log +#name: "Union Database Test With Push Down" +# +#on: +# workflow_call: +# inputs: +# java-matrix: +# description: "The java version to run the test on" +# type: string +# required: false +# default: '["8"]' +# python-matrix: +# description: "The python version to run the test on" +# type: string +# required: false +# default: '["3.9"]' +# os-matrix: +# description: "The operating system to run the test on" +# type: string +# required: false +# default: '["ubuntu-latest", "macos-13", "windows-latest"]' +# metadata-matrix: +# description: "The metadata to run the test on" +# type: string +# required: false +# default: '["zookeeper", "etcd"]' +# db-matrix: +# description: "The database to run the test on" +# type: string +# required: false +# default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' +# +#env: +# VERSION: 0.6.0-SNAPSHOT +# +#jobs: +# Union-DB-Test-Push_Down: +# timeout-minutes: 35 +# strategy: +# fail-fast: false +# matrix: +# java: ${{ fromJSON(inputs.java-matrix) }} +# python-version: ${{ fromJSON(inputs.python-matrix) }} +# os: ${{ fromJSON(inputs.os-matrix) }} +# metadata: ${{ fromJSON(inputs.metadata-matrix) }} +# DB-name: ${{ fromJSON(inputs.db-matrix) }} +# runs-on: ${{ matrix.os }} +# steps: +# - uses: actions/checkout@v4 +# - name: Environment dependence +# uses: ./.github/actions/dependence +# with: +# python-version: ${{ matrix.python-version }} +# java: ${{ matrix.java }} +# +# - if: runner.os == 'Windows' +# name: Set JAVA_OPTS +# run: echo "JAVA_OPTS=-Xmx4g -Xmx2g" >> $GITHUB_ENV +# +# - name: Run Metadata +# uses: ./.github/actions/metadataRunner +# with: +# metadata: ${{ matrix.metadata }} +# +# - name: Run DB +# uses: ./.github/actions/dbRunner +# with: +# DB-name: ${{ matrix.DB-name }} +# +# - name: Install IGinX with Maven +# shell: bash +# run: | +# mvn clean package -DskipTests -P-format -q +# +# - name: Change IGinX config +# uses: ./.github/actions/confWriter +# with: +# DB-name: ${{ matrix.DB-name }} +# Push-Down: "true" +# Set-Filter-Fragment-OFF: "true" +# Metadata: ${{ matrix.metadata }} +# +# - name: Start IGinX +# uses: ./.github/actions/iginxRunner +# +# - name: TestController IT +# if: always() +# shell: bash +# env: +# METADATA_STORAGE: ${{ matrix.metadata }} +# run: | +# chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" +# mvn test -q -Dtest=Controller -DfailIfNoTests=false -P-format +# +# - name: Show test result +# if: always() +# shell: bash +# run: | +# cat ${GITHUB_WORKSPACE}/test/src/test/resources/testResult.txt +# +# - name: Show IGinX log +# if: always() +# shell: bash +# run: | +# cat iginx-*.log diff --git a/.github/workflows/standalone-test.yml b/.github/workflows/standalone-test.yml index e9f202c6e3..417ceeb6aa 100644 --- a/.github/workflows/standalone-test.yml +++ b/.github/workflows/standalone-test.yml @@ -1,190 +1,190 @@ -name: "Union Database Test" - -on: - workflow_call: - inputs: - java-matrix: - description: "The java version to run the test on" - type: string - required: false - default: '["8"]' - python-matrix: - description: "The python version to run the test on" - type: string - required: false - default: '["3.9"]' - os-matrix: - description: "The operating system to run the test on" - type: string - required: false - default: '["ubuntu-latest", "macos-13", "windows-latest"]' - metadata-matrix: - description: "The metadata to run the test on" - type: string - required: false - default: '["zookeeper", "etcd"]' - db-matrix: - description: "The database to run the test on" - type: string - required: false - default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' - -env: - VERSION: 0.6.0-SNAPSHOT - -jobs: - Union-DB-Test: - timeout-minutes: 40 - strategy: - fail-fast: false - matrix: - java: ${{ fromJSON(inputs.java-matrix) }} - python-version: ${{ fromJSON(inputs.python-matrix) }} - os: ${{ fromJSON(inputs.os-matrix) }} - metadata: ${{ fromJSON(inputs.metadata-matrix) }} - DB-name: ${{ fromJSON(inputs.db-matrix) }} - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} - - - name: Run Metadata - uses: ./.github/actions/metadataRunner - with: - metadata: ${{ matrix.metadata }} - - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: ${{ matrix.DB-name }} - - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: ${{ matrix.DB-name }} - Set-Filter-Fragment-OFF: "true" - Metadata: ${{ matrix.metadata }} - - # start udf path test first to avoid being effected - - name: Start IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-test-udf: "true" - - - name: Run UDF path test - if: always() - shell: bash - run: | - mvn test -q -Dtest=UDFPathIT -DfailIfNoTests=false -P-format - if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register.sh" - "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register.sh" ${VERSION} - elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_windows.sh" - "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_windows.sh" ${VERSION} - elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_macos.sh" - "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_macos.sh" ${VERSION} - fi - - - name: set client test context - uses: ./.github/actions/context - with: - work-name: restart-iginx-meta - metadata: ${{ matrix.metadata }} - - - name: set client test context - uses: ./.github/actions/context - with: - DB-name: ${{ matrix.DB-name }} - shell: client-before - - # large image export only tested in FileSystem and Parquet - - name: Test Client Export File - if: always() - shell: bash - run: | - if [[ "${{ matrix.DB-name }}" == "FileSystem" || "${{ matrix.DB-name }}" == "Parquet" ]]; then - mvn test -q -Dtest=ExportFileIT -DfailIfNoTests=false -P-format - else - mvn test -q -Dtest=ExportFileIT#checkExportByteStream -DfailIfNoTests=false -P-format - mvn test -q -Dtest=ExportFileIT#checkExportCsv -DfailIfNoTests=false -P-format - fi - - - name: Stop IGinX and Metadata, Clear Metadata Data, then Start Them - uses: ./.github/actions/context - with: - work-name: restart-iginx-meta - metadata: ${{ matrix.metadata }} - - - name: set client test context - uses: ./.github/actions/context - with: - shell: client-after - - - name: Test Client Import File - if: always() - shell: bash - run: | - mvn test -q -Dtest=ImportFileIT -DfailIfNoTests=false -P-format - - - name: clean metadata and restart IGinX - uses: ./.github/actions/context - with: - work-name: restart-iginx-meta - metadata: ${{ matrix.metadata }} - - - name: TestController IT - if: always() - shell: bash - env: - METADATA_STORAGE: ${{ matrix.metadata }} - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" - mvn test -q -Dtest=Controller -DfailIfNoTests=false -P-format - - - name: Show IGinX log - if: always() - shell: bash - run: | - cat iginx-*.log - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - Set-Key-Range-Test-Policy: "true" - - - name: clean metadata and restart IGinX - uses: ./.github/actions/context - with: - work-name: restart-iginx-meta - metadata: ${{ matrix.metadata }} - - - name: FilterFragmentRuleTest IT - if: always() - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" - mvn test -q -Dtest=SQLSessionIT#testFilterFragmentOptimizer -DfailIfNoTests=false -P-format - - - name: Show test result - if: always() - shell: bash - run: | - cat ${GITHUB_WORKSPACE}/test/src/test/resources/testResult.txt - - - name: Show IGinX log - if: always() - shell: bash - run: | - cat iginx-*.log +#name: "Union Database Test" +# +#on: +# workflow_call: +# inputs: +# java-matrix: +# description: "The java version to run the test on" +# type: string +# required: false +# default: '["8"]' +# python-matrix: +# description: "The python version to run the test on" +# type: string +# required: false +# default: '["3.9"]' +# os-matrix: +# description: "The operating system to run the test on" +# type: string +# required: false +# default: '["ubuntu-latest", "macos-13", "windows-latest"]' +# metadata-matrix: +# description: "The metadata to run the test on" +# type: string +# required: false +# default: '["zookeeper", "etcd"]' +# db-matrix: +# description: "The database to run the test on" +# type: string +# required: false +# default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' +# +#env: +# VERSION: 0.6.0-SNAPSHOT +# +#jobs: +# Union-DB-Test: +# timeout-minutes: 40 +# strategy: +# fail-fast: false +# matrix: +# java: ${{ fromJSON(inputs.java-matrix) }} +# python-version: ${{ fromJSON(inputs.python-matrix) }} +# os: ${{ fromJSON(inputs.os-matrix) }} +# metadata: ${{ fromJSON(inputs.metadata-matrix) }} +# DB-name: ${{ fromJSON(inputs.db-matrix) }} +# runs-on: ${{ matrix.os }} +# steps: +# - uses: actions/checkout@v4 +# - name: Environment dependence +# uses: ./.github/actions/dependence +# with: +# python-version: ${{ matrix.python-version }} +# java: ${{ matrix.java }} +# +# - name: Run Metadata +# uses: ./.github/actions/metadataRunner +# with: +# metadata: ${{ matrix.metadata }} +# +# - name: Run DB +# uses: ./.github/actions/dbRunner +# with: +# DB-name: ${{ matrix.DB-name }} +# +# - name: Install IGinX with Maven +# shell: bash +# run: | +# mvn clean package -DskipTests -P-format -q +# +# - name: Change IGinX config +# uses: ./.github/actions/confWriter +# with: +# DB-name: ${{ matrix.DB-name }} +# Set-Filter-Fragment-OFF: "true" +# Metadata: ${{ matrix.metadata }} +# +# # start udf path test first to avoid being effected +# - name: Start IGinX +# uses: ./.github/actions/iginxRunner +# with: +# version: ${VERSION} +# if-test-udf: "true" +# +# - name: Run UDF path test +# if: always() +# shell: bash +# run: | +# mvn test -q -Dtest=UDFPathIT -DfailIfNoTests=false -P-format +# if [ "$RUNNER_OS" == "Linux" ]; then +# chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register.sh" +# "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register.sh" ${VERSION} +# elif [ "$RUNNER_OS" == "Windows" ]; then +# chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_windows.sh" +# "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_windows.sh" ${VERSION} +# elif [ "$RUNNER_OS" == "macOS" ]; then +# chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_macos.sh" +# "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_macos.sh" ${VERSION} +# fi +# +# - name: set client test context +# uses: ./.github/actions/context +# with: +# work-name: restart-iginx-meta +# metadata: ${{ matrix.metadata }} +# +# - name: set client test context +# uses: ./.github/actions/context +# with: +# DB-name: ${{ matrix.DB-name }} +# shell: client-before +# +# # large image export only tested in FileSystem and Parquet +# - name: Test Client Export File +# if: always() +# shell: bash +# run: | +# if [[ "${{ matrix.DB-name }}" == "FileSystem" || "${{ matrix.DB-name }}" == "Parquet" ]]; then +# mvn test -q -Dtest=ExportFileIT -DfailIfNoTests=false -P-format +# else +# mvn test -q -Dtest=ExportFileIT#checkExportByteStream -DfailIfNoTests=false -P-format +# mvn test -q -Dtest=ExportFileIT#checkExportCsv -DfailIfNoTests=false -P-format +# fi +# +# - name: Stop IGinX and Metadata, Clear Metadata Data, then Start Them +# uses: ./.github/actions/context +# with: +# work-name: restart-iginx-meta +# metadata: ${{ matrix.metadata }} +# +# - name: set client test context +# uses: ./.github/actions/context +# with: +# shell: client-after +# +# - name: Test Client Import File +# if: always() +# shell: bash +# run: | +# mvn test -q -Dtest=ImportFileIT -DfailIfNoTests=false -P-format +# +# - name: clean metadata and restart IGinX +# uses: ./.github/actions/context +# with: +# work-name: restart-iginx-meta +# metadata: ${{ matrix.metadata }} +# +# - name: TestController IT +# if: always() +# shell: bash +# env: +# METADATA_STORAGE: ${{ matrix.metadata }} +# run: | +# chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" +# mvn test -q -Dtest=Controller -DfailIfNoTests=false -P-format +# +# - name: Show IGinX log +# if: always() +# shell: bash +# run: | +# cat iginx-*.log +# +# - name: Change IGinX config +# uses: ./.github/actions/confWriter +# with: +# Set-Key-Range-Test-Policy: "true" +# +# - name: clean metadata and restart IGinX +# uses: ./.github/actions/context +# with: +# work-name: restart-iginx-meta +# metadata: ${{ matrix.metadata }} +# +# - name: FilterFragmentRuleTest IT +# if: always() +# shell: bash +# run: | +# chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" +# mvn test -q -Dtest=SQLSessionIT#testFilterFragmentOptimizer -DfailIfNoTests=false -P-format +# +# - name: Show test result +# if: always() +# shell: bash +# run: | +# cat ${GITHUB_WORKSPACE}/test/src/test/resources/testResult.txt +# +# - name: Show IGinX log +# if: always() +# shell: bash +# run: | +# cat iginx-*.log diff --git a/dataSources/pom.xml b/dataSources/pom.xml index a1cdc62a91..4b7542f6b6 100644 --- a/dataSources/pom.xml +++ b/dataSources/pom.xml @@ -29,12 +29,12 @@ filesystem - influxdb + iotdb12 - mongodb - parquet - redis - relational + + + + diff --git a/test/pom.xml b/test/pom.xml index d2d0c8dc6c..3aa6b81cac 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -44,10 +44,10 @@ cn.edu.tsinghua parquet - - cn.edu.tsinghua - relational - + + + + cn.edu.tsinghua redis From 918a7604a0b4a9cc1260e19f559798e9b229ad44 Mon Sep 17 00:00:00 2001 From: RemHero <1104304963@qq.com> Date: Tue, 4 Jun 2024 23:18:36 +0800 Subject: [PATCH 03/11] test --- .../workflows/standalone-test-pushdown.yml | 208 +++++----- .github/workflows/standalone-test.yml | 380 +++++++++--------- 2 files changed, 294 insertions(+), 294 deletions(-) diff --git a/.github/workflows/standalone-test-pushdown.yml b/.github/workflows/standalone-test-pushdown.yml index cebca0d776..c4116604e3 100644 --- a/.github/workflows/standalone-test-pushdown.yml +++ b/.github/workflows/standalone-test-pushdown.yml @@ -1,104 +1,104 @@ -#name: "Union Database Test With Push Down" -# -#on: -# workflow_call: -# inputs: -# java-matrix: -# description: "The java version to run the test on" -# type: string -# required: false -# default: '["8"]' -# python-matrix: -# description: "The python version to run the test on" -# type: string -# required: false -# default: '["3.9"]' -# os-matrix: -# description: "The operating system to run the test on" -# type: string -# required: false -# default: '["ubuntu-latest", "macos-13", "windows-latest"]' -# metadata-matrix: -# description: "The metadata to run the test on" -# type: string -# required: false -# default: '["zookeeper", "etcd"]' -# db-matrix: -# description: "The database to run the test on" -# type: string -# required: false -# default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' -# -#env: -# VERSION: 0.6.0-SNAPSHOT -# -#jobs: -# Union-DB-Test-Push_Down: -# timeout-minutes: 35 -# strategy: -# fail-fast: false -# matrix: -# java: ${{ fromJSON(inputs.java-matrix) }} -# python-version: ${{ fromJSON(inputs.python-matrix) }} -# os: ${{ fromJSON(inputs.os-matrix) }} -# metadata: ${{ fromJSON(inputs.metadata-matrix) }} -# DB-name: ${{ fromJSON(inputs.db-matrix) }} -# runs-on: ${{ matrix.os }} -# steps: -# - uses: actions/checkout@v4 -# - name: Environment dependence -# uses: ./.github/actions/dependence -# with: -# python-version: ${{ matrix.python-version }} -# java: ${{ matrix.java }} -# -# - if: runner.os == 'Windows' -# name: Set JAVA_OPTS -# run: echo "JAVA_OPTS=-Xmx4g -Xmx2g" >> $GITHUB_ENV -# -# - name: Run Metadata -# uses: ./.github/actions/metadataRunner -# with: -# metadata: ${{ matrix.metadata }} -# -# - name: Run DB -# uses: ./.github/actions/dbRunner -# with: -# DB-name: ${{ matrix.DB-name }} -# -# - name: Install IGinX with Maven -# shell: bash -# run: | -# mvn clean package -DskipTests -P-format -q -# -# - name: Change IGinX config -# uses: ./.github/actions/confWriter -# with: -# DB-name: ${{ matrix.DB-name }} -# Push-Down: "true" -# Set-Filter-Fragment-OFF: "true" -# Metadata: ${{ matrix.metadata }} -# -# - name: Start IGinX -# uses: ./.github/actions/iginxRunner -# -# - name: TestController IT -# if: always() -# shell: bash -# env: -# METADATA_STORAGE: ${{ matrix.metadata }} -# run: | -# chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" -# mvn test -q -Dtest=Controller -DfailIfNoTests=false -P-format -# -# - name: Show test result -# if: always() -# shell: bash -# run: | -# cat ${GITHUB_WORKSPACE}/test/src/test/resources/testResult.txt -# -# - name: Show IGinX log -# if: always() -# shell: bash -# run: | -# cat iginx-*.log +name: "Union Database Test With Push Down" + +on: + workflow_call: + inputs: + java-matrix: + description: "The java version to run the test on" + type: string + required: false + default: '["8"]' + python-matrix: + description: "The python version to run the test on" + type: string + required: false + default: '["3.9"]' + os-matrix: + description: "The operating system to run the test on" + type: string + required: false + default: '["ubuntu-latest", "macos-13", "windows-latest"]' + metadata-matrix: + description: "The metadata to run the test on" + type: string + required: false + default: '["zookeeper", "etcd"]' + db-matrix: + description: "The database to run the test on" + type: string + required: false + default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' + +env: + VERSION: 0.6.0-SNAPSHOT + +jobs: + Union-DB-Test-Push_Down: + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + java: ${{ fromJSON(inputs.java-matrix) }} + python-version: ${{ fromJSON(inputs.python-matrix) }} + os: ${{ fromJSON(inputs.os-matrix) }} + metadata: ${{ fromJSON(inputs.metadata-matrix) }} + DB-name: ${{ fromJSON(inputs.db-matrix) }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - if: runner.os == 'Windows' + name: Set JAVA_OPTS + run: echo "JAVA_OPTS=-Xmx4g -Xmx2g" >> $GITHUB_ENV + + - name: Run Metadata + uses: ./.github/actions/metadataRunner + with: + metadata: ${{ matrix.metadata }} + + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: ${{ matrix.DB-name }} + + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: ${{ matrix.DB-name }} + Push-Down: "true" + Set-Filter-Fragment-OFF: "true" + Metadata: ${{ matrix.metadata }} + + - name: Start IGinX + uses: ./.github/actions/iginxRunner + + - name: TestController IT + if: always() + shell: bash + env: + METADATA_STORAGE: ${{ matrix.metadata }} + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" + mvn test -q -Dtest=Controller -DfailIfNoTests=false -P-format + + - name: Show test result + if: always() + shell: bash + run: | + cat ${GITHUB_WORKSPACE}/test/src/test/resources/testResult.txt + + - name: Show IGinX log + if: always() + shell: bash + run: | + cat iginx-*.log diff --git a/.github/workflows/standalone-test.yml b/.github/workflows/standalone-test.yml index 417ceeb6aa..e9f202c6e3 100644 --- a/.github/workflows/standalone-test.yml +++ b/.github/workflows/standalone-test.yml @@ -1,190 +1,190 @@ -#name: "Union Database Test" -# -#on: -# workflow_call: -# inputs: -# java-matrix: -# description: "The java version to run the test on" -# type: string -# required: false -# default: '["8"]' -# python-matrix: -# description: "The python version to run the test on" -# type: string -# required: false -# default: '["3.9"]' -# os-matrix: -# description: "The operating system to run the test on" -# type: string -# required: false -# default: '["ubuntu-latest", "macos-13", "windows-latest"]' -# metadata-matrix: -# description: "The metadata to run the test on" -# type: string -# required: false -# default: '["zookeeper", "etcd"]' -# db-matrix: -# description: "The database to run the test on" -# type: string -# required: false -# default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' -# -#env: -# VERSION: 0.6.0-SNAPSHOT -# -#jobs: -# Union-DB-Test: -# timeout-minutes: 40 -# strategy: -# fail-fast: false -# matrix: -# java: ${{ fromJSON(inputs.java-matrix) }} -# python-version: ${{ fromJSON(inputs.python-matrix) }} -# os: ${{ fromJSON(inputs.os-matrix) }} -# metadata: ${{ fromJSON(inputs.metadata-matrix) }} -# DB-name: ${{ fromJSON(inputs.db-matrix) }} -# runs-on: ${{ matrix.os }} -# steps: -# - uses: actions/checkout@v4 -# - name: Environment dependence -# uses: ./.github/actions/dependence -# with: -# python-version: ${{ matrix.python-version }} -# java: ${{ matrix.java }} -# -# - name: Run Metadata -# uses: ./.github/actions/metadataRunner -# with: -# metadata: ${{ matrix.metadata }} -# -# - name: Run DB -# uses: ./.github/actions/dbRunner -# with: -# DB-name: ${{ matrix.DB-name }} -# -# - name: Install IGinX with Maven -# shell: bash -# run: | -# mvn clean package -DskipTests -P-format -q -# -# - name: Change IGinX config -# uses: ./.github/actions/confWriter -# with: -# DB-name: ${{ matrix.DB-name }} -# Set-Filter-Fragment-OFF: "true" -# Metadata: ${{ matrix.metadata }} -# -# # start udf path test first to avoid being effected -# - name: Start IGinX -# uses: ./.github/actions/iginxRunner -# with: -# version: ${VERSION} -# if-test-udf: "true" -# -# - name: Run UDF path test -# if: always() -# shell: bash -# run: | -# mvn test -q -Dtest=UDFPathIT -DfailIfNoTests=false -P-format -# if [ "$RUNNER_OS" == "Linux" ]; then -# chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register.sh" -# "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register.sh" ${VERSION} -# elif [ "$RUNNER_OS" == "Windows" ]; then -# chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_windows.sh" -# "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_windows.sh" ${VERSION} -# elif [ "$RUNNER_OS" == "macOS" ]; then -# chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_macos.sh" -# "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_macos.sh" ${VERSION} -# fi -# -# - name: set client test context -# uses: ./.github/actions/context -# with: -# work-name: restart-iginx-meta -# metadata: ${{ matrix.metadata }} -# -# - name: set client test context -# uses: ./.github/actions/context -# with: -# DB-name: ${{ matrix.DB-name }} -# shell: client-before -# -# # large image export only tested in FileSystem and Parquet -# - name: Test Client Export File -# if: always() -# shell: bash -# run: | -# if [[ "${{ matrix.DB-name }}" == "FileSystem" || "${{ matrix.DB-name }}" == "Parquet" ]]; then -# mvn test -q -Dtest=ExportFileIT -DfailIfNoTests=false -P-format -# else -# mvn test -q -Dtest=ExportFileIT#checkExportByteStream -DfailIfNoTests=false -P-format -# mvn test -q -Dtest=ExportFileIT#checkExportCsv -DfailIfNoTests=false -P-format -# fi -# -# - name: Stop IGinX and Metadata, Clear Metadata Data, then Start Them -# uses: ./.github/actions/context -# with: -# work-name: restart-iginx-meta -# metadata: ${{ matrix.metadata }} -# -# - name: set client test context -# uses: ./.github/actions/context -# with: -# shell: client-after -# -# - name: Test Client Import File -# if: always() -# shell: bash -# run: | -# mvn test -q -Dtest=ImportFileIT -DfailIfNoTests=false -P-format -# -# - name: clean metadata and restart IGinX -# uses: ./.github/actions/context -# with: -# work-name: restart-iginx-meta -# metadata: ${{ matrix.metadata }} -# -# - name: TestController IT -# if: always() -# shell: bash -# env: -# METADATA_STORAGE: ${{ matrix.metadata }} -# run: | -# chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" -# mvn test -q -Dtest=Controller -DfailIfNoTests=false -P-format -# -# - name: Show IGinX log -# if: always() -# shell: bash -# run: | -# cat iginx-*.log -# -# - name: Change IGinX config -# uses: ./.github/actions/confWriter -# with: -# Set-Key-Range-Test-Policy: "true" -# -# - name: clean metadata and restart IGinX -# uses: ./.github/actions/context -# with: -# work-name: restart-iginx-meta -# metadata: ${{ matrix.metadata }} -# -# - name: FilterFragmentRuleTest IT -# if: always() -# shell: bash -# run: | -# chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" -# mvn test -q -Dtest=SQLSessionIT#testFilterFragmentOptimizer -DfailIfNoTests=false -P-format -# -# - name: Show test result -# if: always() -# shell: bash -# run: | -# cat ${GITHUB_WORKSPACE}/test/src/test/resources/testResult.txt -# -# - name: Show IGinX log -# if: always() -# shell: bash -# run: | -# cat iginx-*.log +name: "Union Database Test" + +on: + workflow_call: + inputs: + java-matrix: + description: "The java version to run the test on" + type: string + required: false + default: '["8"]' + python-matrix: + description: "The python version to run the test on" + type: string + required: false + default: '["3.9"]' + os-matrix: + description: "The operating system to run the test on" + type: string + required: false + default: '["ubuntu-latest", "macos-13", "windows-latest"]' + metadata-matrix: + description: "The metadata to run the test on" + type: string + required: false + default: '["zookeeper", "etcd"]' + db-matrix: + description: "The database to run the test on" + type: string + required: false + default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' + +env: + VERSION: 0.6.0-SNAPSHOT + +jobs: + Union-DB-Test: + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + java: ${{ fromJSON(inputs.java-matrix) }} + python-version: ${{ fromJSON(inputs.python-matrix) }} + os: ${{ fromJSON(inputs.os-matrix) }} + metadata: ${{ fromJSON(inputs.metadata-matrix) }} + DB-name: ${{ fromJSON(inputs.db-matrix) }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - name: Run Metadata + uses: ./.github/actions/metadataRunner + with: + metadata: ${{ matrix.metadata }} + + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: ${{ matrix.DB-name }} + + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: ${{ matrix.DB-name }} + Set-Filter-Fragment-OFF: "true" + Metadata: ${{ matrix.metadata }} + + # start udf path test first to avoid being effected + - name: Start IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" + + - name: Run UDF path test + if: always() + shell: bash + run: | + mvn test -q -Dtest=UDFPathIT -DfailIfNoTests=false -P-format + if [ "$RUNNER_OS" == "Linux" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register.sh" + "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register.sh" ${VERSION} + elif [ "$RUNNER_OS" == "Windows" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_windows.sh" + "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_windows.sh" ${VERSION} + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_macos.sh" + "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_macos.sh" ${VERSION} + fi + + - name: set client test context + uses: ./.github/actions/context + with: + work-name: restart-iginx-meta + metadata: ${{ matrix.metadata }} + + - name: set client test context + uses: ./.github/actions/context + with: + DB-name: ${{ matrix.DB-name }} + shell: client-before + + # large image export only tested in FileSystem and Parquet + - name: Test Client Export File + if: always() + shell: bash + run: | + if [[ "${{ matrix.DB-name }}" == "FileSystem" || "${{ matrix.DB-name }}" == "Parquet" ]]; then + mvn test -q -Dtest=ExportFileIT -DfailIfNoTests=false -P-format + else + mvn test -q -Dtest=ExportFileIT#checkExportByteStream -DfailIfNoTests=false -P-format + mvn test -q -Dtest=ExportFileIT#checkExportCsv -DfailIfNoTests=false -P-format + fi + + - name: Stop IGinX and Metadata, Clear Metadata Data, then Start Them + uses: ./.github/actions/context + with: + work-name: restart-iginx-meta + metadata: ${{ matrix.metadata }} + + - name: set client test context + uses: ./.github/actions/context + with: + shell: client-after + + - name: Test Client Import File + if: always() + shell: bash + run: | + mvn test -q -Dtest=ImportFileIT -DfailIfNoTests=false -P-format + + - name: clean metadata and restart IGinX + uses: ./.github/actions/context + with: + work-name: restart-iginx-meta + metadata: ${{ matrix.metadata }} + + - name: TestController IT + if: always() + shell: bash + env: + METADATA_STORAGE: ${{ matrix.metadata }} + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" + mvn test -q -Dtest=Controller -DfailIfNoTests=false -P-format + + - name: Show IGinX log + if: always() + shell: bash + run: | + cat iginx-*.log + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + Set-Key-Range-Test-Policy: "true" + + - name: clean metadata and restart IGinX + uses: ./.github/actions/context + with: + work-name: restart-iginx-meta + metadata: ${{ matrix.metadata }} + + - name: FilterFragmentRuleTest IT + if: always() + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" + mvn test -q -Dtest=SQLSessionIT#testFilterFragmentOptimizer -DfailIfNoTests=false -P-format + + - name: Show test result + if: always() + shell: bash + run: | + cat ${GITHUB_WORKSPACE}/test/src/test/resources/testResult.txt + + - name: Show IGinX log + if: always() + shell: bash + run: | + cat iginx-*.log From e083c74591948c663bb71d8d89c091cf707fd369 Mon Sep 17 00:00:00 2001 From: RemHero <1104304963@qq.com> Date: Wed, 5 Jun 2024 17:11:31 +0800 Subject: [PATCH 04/11] test --- .github/workflows/standard-test-suite.yml | 40 +++++----- .../execute/StoragePhysicalTaskExecutor.java | 6 +- .../iginx/filesystem/FileSystemStorage.java | 4 +- .../iginx/filesystem/exec/Executor.java | 3 +- .../iginx/filesystem/exec/LocalExecutor.java | 18 ++--- .../iginx/filesystem/exec/RemoteExecutor.java | 8 +- .../filesystem/server/FileSystemWorker.java | 6 +- .../tsinghua/iginx/iotdb/IoTDBStorage.java | 9 ++- dataSources/pom.xml | 10 +-- test/pom.xml | 8 +- .../expansion/BaseCapacityExpansionIT.java | 75 ++++++++++++++----- 11 files changed, 115 insertions(+), 72 deletions(-) diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index a4f8234894..f5b7a22f3b 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -11,27 +11,27 @@ concurrency: cancel-in-progress: true jobs: - unit-test: - uses: ./.github/workflows/unit-test.yml - unit-mds: - uses: ./.github/workflows/unit-mds.yml - case-regression: - uses: ./.github/workflows/case-regression.yml - with: - metadata-matrix: '["zookeeper"]' - standalone-test: - uses: ./.github/workflows/standalone-test.yml - with: - metadata-matrix: '["zookeeper"]' - standalone-test-pushdown: - uses: ./.github/workflows/standalone-test-pushdown.yml - with: - metadata-matrix: '["zookeeper"]' +# unit-test: +# uses: ./.github/workflows/unit-test.yml +# unit-mds: +# uses: ./.github/workflows/unit-mds.yml +# case-regression: +# uses: ./.github/workflows/case-regression.yml +# with: +# metadata-matrix: '["zookeeper"]' +# standalone-test: +# uses: ./.github/workflows/standalone-test.yml +# with: +# metadata-matrix: '["zookeeper"]' +# standalone-test-pushdown: +# uses: ./.github/workflows/standalone-test-pushdown.yml +# with: +# metadata-matrix: '["zookeeper"]' db-ce: uses: ./.github/workflows/DB-CE.yml with: metadata-matrix: '["zookeeper"]' - remote-test: - uses: ./.github/workflows/remote-test.yml - with: - metadata-matrix: '["zookeeper"]' +# remote-test: +# uses: ./.github/workflows/remote-test.yml +# with: +# metadata-matrix: '["zookeeper"]' diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java index 061cedc8a7..7735275b22 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java @@ -53,7 +53,6 @@ import cn.edu.tsinghua.iginx.monitor.HotSpotMonitor; import cn.edu.tsinghua.iginx.monitor.RequestsMonitor; import cn.edu.tsinghua.iginx.utils.Pair; - import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -325,7 +324,7 @@ public TaskExecuteResult executeShowColumns(ShowColumns showColumns) { Set patternSet = showColumns.getPathRegexSet(); TagFilter tagFilter = showColumns.getTagFilter(); if (storage.getDataPrefix() != null) { - patternSet.add(storage.getDataPrefix()+".*"); + patternSet.add(storage.getDataPrefix() + ".*"); } List columnList = pair.k.getColumns(patternSet, tagFilter); // fix the schemaPrefix @@ -343,7 +342,8 @@ public TaskExecuteResult executeShowColumns(ShowColumns showColumns) { } } - TreeSet columnSetAfterFilter = new TreeSet<>(Comparator.comparing(Column::getPhysicalPath)); + TreeSet columnSetAfterFilter = + new TreeSet<>(Comparator.comparing(Column::getPhysicalPath)); columnSetAfterFilter.addAll(columnSet); int limit = showColumns.getLimit(); diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/FileSystemStorage.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/FileSystemStorage.java index b42e783c7b..6f31d9e666 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/FileSystemStorage.java +++ b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/FileSystemStorage.java @@ -45,7 +45,6 @@ import java.util.Arrays; import java.util.List; import java.util.Set; - import org.apache.thrift.transport.TTransportException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -152,7 +151,8 @@ public TaskExecuteResult executeDelete(Delete delete, DataArea dataArea) { } @Override - public List getColumns(Set pattern, TagFilter tagFilter) throws PhysicalException { + public List getColumns(Set pattern, TagFilter tagFilter) + throws PhysicalException { return executor.getColumnsOfStorageUnit(WILDCARD, pattern, tagFilter); } diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/Executor.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/Executor.java index 66926d37d8..b5b71fa32a 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/Executor.java +++ b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/Executor.java @@ -27,7 +27,8 @@ TaskExecuteResult executeProjectTask( TaskExecuteResult executeDeleteTask( List paths, List keyRanges, TagFilter tagFilter, String storageUnit); - List getColumnsOfStorageUnit(String storageUnit, Set pattern, TagFilter tagFilter) throws PhysicalException; + List getColumnsOfStorageUnit(String storageUnit, Set pattern, TagFilter tagFilter) + throws PhysicalException; Pair getBoundaryOfStorage(String dataPrefix) throws PhysicalException; diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java index ff0787193a..90cbf66cd6 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java +++ b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java @@ -30,6 +30,7 @@ import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.utils.Pair; +import cn.edu.tsinghua.iginx.utils.StringUtils; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -37,8 +38,6 @@ import java.util.Map; import java.util.Set; import java.util.regex.Pattern; - -import cn.edu.tsinghua.iginx.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -314,13 +313,15 @@ public TaskExecuteResult executeDeleteTask( } @Override - public List getColumnsOfStorageUnit(String storageUnit, Set pattern, TagFilter tagFilter) throws PhysicalException { + public List getColumnsOfStorageUnit( + String storageUnit, Set pattern, TagFilter tagFilter) throws PhysicalException { List columns = new ArrayList<>(); if (root != null) { File directory = new File(FilePathUtils.toIginxPath(root, storageUnit, null)); for (File file : fileSystemManager.getAllFiles(directory, false)) { FileMeta meta = fileSystemManager.getFileMeta(file); - String columnPath = FilePathUtils.convertAbsolutePathToPath(root, file.getAbsolutePath(), storageUnit); + String columnPath = + FilePathUtils.convertAbsolutePathToPath(root, file.getAbsolutePath(), storageUnit); boolean isChosen = true; if (meta == null) { throw new PhysicalException( @@ -342,17 +343,12 @@ public List getColumnsOfStorageUnit(String storageUnit, Set patt } // get columns by tag filter if (tagFilter != null && !TagKVUtils.match(meta.getTags(), tagFilter)) { - columns.add( - new Column( - columnPath, - meta.getDataType(), - meta.getTags(), - false)); + columns.add(new Column(columnPath, meta.getDataType(), meta.getTags(), false)); } } } // get columns from dummy storage unit - if (hasData && dummyRoot != null && tagFilter==null) { + if (hasData && dummyRoot != null && tagFilter == null) { for (File file : fileSystemManager.getAllFiles(new File(realDummyRoot), true)) { columns.add( new Column( diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/RemoteExecutor.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/RemoteExecutor.java index 2c55bf3a2d..21e2dacd23 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/RemoteExecutor.java +++ b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/RemoteExecutor.java @@ -204,11 +204,13 @@ public TaskExecuteResult executeDeleteTask( } @Override - public List getColumnsOfStorageUnit(String storageUnit, Set pattern, TagFilter tagFilter) throws PhysicalException { + public List getColumnsOfStorageUnit( + String storageUnit, Set pattern, TagFilter tagFilter) throws PhysicalException { try { TTransport transport = thriftConnPool.borrowTransport(); Client client = new Client(new TBinaryProtocol(transport)); - GetColumnsOfStorageUnitResp resp = client.getColumnsOfStorageUnit(storageUnit, pattern, constructRawTagFilter(tagFilter)); + GetColumnsOfStorageUnitResp resp = + client.getColumnsOfStorageUnit(storageUnit, pattern, constructRawTagFilter(tagFilter)); thriftConnPool.returnTransport(transport); List columns = new ArrayList<>(); resp.getPathList() @@ -251,7 +253,7 @@ public void close() { private RawTagFilter constructRawTagFilter(TagFilter tagFilter) { RawTagFilter filter = null; - if(tagFilter == null) { + if (tagFilter == null) { return null; } switch (tagFilter.getType()) { diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemWorker.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemWorker.java index 1503c8397e..fb44573bf6 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemWorker.java +++ b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemWorker.java @@ -198,10 +198,12 @@ public Status executeDelete(DeleteReq req) throws TException { } @Override - public GetColumnsOfStorageUnitResp getColumnsOfStorageUnit(String storageUnit, Set pattern, RawTagFilter tagFilter) throws TException { + public GetColumnsOfStorageUnitResp getColumnsOfStorageUnit( + String storageUnit, Set pattern, RawTagFilter tagFilter) throws TException { List ret = new ArrayList<>(); try { - List columns = executor.getColumnsOfStorageUnit(storageUnit, pattern, resolveRawTagFilter(tagFilter)); + List columns = + executor.getColumnsOfStorageUnit(storageUnit, pattern, resolveRawTagFilter(tagFilter)); columns.forEach( column -> { FSColumn fsColumn = diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java index b2e1cb4705..a31aa6e56d 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java +++ b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java @@ -192,13 +192,18 @@ public void release() throws PhysicalException { } @Override - public List getColumns(Set pattern, TagFilter tagFilter) throws PhysicalException { + public List getColumns(Set pattern, TagFilter tagFilter) + throws PhysicalException { List columns = new ArrayList<>(); getColumns2StorageUnit(columns, null, pattern, tagFilter); return columns; } - private void getColumns2StorageUnit(List columns, Map columns2StorageUnit, Set pattern, TagFilter tagFilter) + private void getColumns2StorageUnit( + List columns, + Map columns2StorageUnit, + Set pattern, + TagFilter tagFilter) throws PhysicalException { try { SessionDataSetWrapper dataSet = sessionPool.executeQueryStatement(SHOW_TIMESERIES); diff --git a/dataSources/pom.xml b/dataSources/pom.xml index 4b7542f6b6..abf7da3715 100644 --- a/dataSources/pom.xml +++ b/dataSources/pom.xml @@ -29,12 +29,12 @@ filesystem - + iotdb12 - - - - + + + + diff --git a/test/pom.xml b/test/pom.xml index 3aa6b81cac..61b03d3eee 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -44,10 +44,10 @@ cn.edu.tsinghua parquet - - - - + + + + cn.edu.tsinghua redis diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java index ece1882bf6..a2959c9e1e 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java @@ -3,8 +3,7 @@ import static cn.edu.tsinghua.iginx.integration.controller.Controller.SUPPORT_KEY; import static cn.edu.tsinghua.iginx.integration.expansion.constant.Constant.*; import static cn.edu.tsinghua.iginx.integration.expansion.utils.SQLTestTools.executeShellScript; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; import cn.edu.tsinghua.iginx.exception.SessionException; import cn.edu.tsinghua.iginx.integration.controller.Controller; @@ -17,12 +16,15 @@ import cn.edu.tsinghua.iginx.session.Column; import cn.edu.tsinghua.iginx.session.QueryDataSet; import cn.edu.tsinghua.iginx.session.Session; +import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.thrift.RemovedStorageEngineInfo; import cn.edu.tsinghua.iginx.thrift.StorageEngineType; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; + import org.junit.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -324,6 +326,16 @@ protected void queryExtendedKeyDummy() { statement = "select wf05.wt01.status, wf05.wt01.temperature from tm;"; SQLTestTools.executeAndContainValue( session, statement, READ_ONLY_PATH_LIST, READ_ONLY_EXTEND_VALUES_LIST); + + // test show columns + testShowColumns(Arrays.asList( + new Column("mn.wf01.wt01.temperature", DataType.DOUBLE), + new Column("mn.wf01.wt01.status", DataType.LONG), + new Column("nt.wf03.wt01.status2", DataType.LONG), + new Column("nt.wf04.wt01.temperature", DataType.DOUBLE), + new Column("tm.wf05.wt01.status", DataType.LONG), + new Column("tm.wf05.wt01.temperature", DataType.DOUBLE) + )); } protected void queryExtendedColDummy() { @@ -456,6 +468,29 @@ private void queryAllNewData() { SQLTestTools.executeAndCompare(session, statement, expect); } + private void testShowColumns(List expectColumns) { + try { + List columns = session.showColumns(); + LOGGER.info("show columns: {}", columns); + + // 对期望列表和实际列表中的Column对象按路径排序 + List sortedExpectPaths = expectColumns.stream() + .map(Column::getPath) + .sorted() + .collect(Collectors.toList()); + + List sortedActualPaths = columns.stream() + .map(Column::getPath) + .sorted() + .collect(Collectors.toList()); + + // 检查排序后的路径列表是否相同 + assertArrayEquals(sortedExpectPaths.toArray(), sortedActualPaths.toArray()); + } catch (SessionException e) { + LOGGER.error("show columns error: ", e); + } + } + private void testAddAndRemoveStorageEngineWithPrefix() { String dataPrefix1 = "nt.wf03"; String dataPrefix2 = "nt.wf04"; @@ -466,30 +501,32 @@ private void testAddAndRemoveStorageEngineWithPrefix() { List> valuesList = EXP_VALUES_LIST1; - // 测试 show columns - try { - List columns = session.showColumns(); - LOGGER.info("columns: {}", columns); - } catch (SessionException e) { - LOGGER.error("show columns error: ", e); - } + testShowColumns(Arrays.asList( + new Column("b.b.b", DataType.LONG), + new Column("ln.wf02.status", DataType.BOOLEAN), + new Column("ln.wf02.version", DataType.BINARY), + new Column("nt.wf03.wt01.status2", DataType.LONG), + new Column("nt.wf04.wt01.temperature", DataType.DOUBLE), + new Column("zzzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzzzz", DataType.LONG) + )); // 添加不同 schemaPrefix，相同 dataPrefix addStorageEngine(expPort, true, true, dataPrefix1, schemaPrefix1, extraParams); + testShowColumns(Arrays.asList( + new Column("b.b.b", DataType.LONG), + new Column("ln.wf02.status", DataType.BOOLEAN), + new Column("ln.wf02.version", DataType.BINARY), + new Column("nt.wf03.wt01.status2", DataType.LONG),new Column("p1.nt.wf03.wt01.status2", DataType.LONG), + new Column("nt.wf04.wt01.temperature", DataType.DOUBLE), + new Column("zzzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzzzz", DataType.LONG) + )); + // 添加节点 dataPrefix = dataPrefix1 && schemaPrefix = p1 后查询 String statement = "select status2 from *;"; List pathList = Arrays.asList("nt.wf03.wt01.status2", "p1.nt.wf03.wt01.status2"); SQLTestTools.executeAndCompare(session, statement, pathList, REPEAT_EXP_VALUES_LIST1); - // 测试添加节点后的 show columns - try { - List columns = session.showColumns(); - LOGGER.info("columns: {}", columns); - } catch (SessionException e) { - LOGGER.error("show columns error: ", e); - } - addStorageEngine(expPort, true, true, dataPrefix1, schemaPrefix2, extraParams); addStorageEngine(expPort, true, true, dataPrefix1, null, extraParams); testShowClusterInfo(5); @@ -697,8 +734,8 @@ private void testSameKeyWarning() { QueryDataSet res = session.executeQuery(statement); if ((res.getWarningMsg() == null - || res.getWarningMsg().isEmpty() - || !res.getWarningMsg().contains("The query results contain overlapped keys.")) + || res.getWarningMsg().isEmpty() + || !res.getWarningMsg().contains("The query results contain overlapped keys.")) && SUPPORT_KEY.get(testConf.getStorageType())) { LOGGER.error("未抛出重叠key的警告"); fail(); From 4a4c7981fca09c2e8b9d13fe894d9589d697800d Mon Sep 17 00:00:00 2001 From: RemHero <1104304963@qq.com> Date: Wed, 5 Jun 2024 17:28:33 +0800 Subject: [PATCH 05/11] fix pom --- dataSources/pom.xml | 10 +++++----- test/pom.xml | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dataSources/pom.xml b/dataSources/pom.xml index abf7da3715..a1cdc62a91 100644 --- a/dataSources/pom.xml +++ b/dataSources/pom.xml @@ -29,12 +29,12 @@ filesystem - + influxdb iotdb12 - - - - + mongodb + parquet + redis + relational diff --git a/test/pom.xml b/test/pom.xml index 61b03d3eee..d2d0c8dc6c 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -44,10 +44,10 @@ cn.edu.tsinghua parquet - - - - + + cn.edu.tsinghua + relational + cn.edu.tsinghua redis From 621cf38f0e21338c3d40d125e56b896fd75080af Mon Sep 17 00:00:00 2001 From: RemHero <1104304963@qq.com> Date: Thu, 6 Jun 2024 10:11:06 +0800 Subject: [PATCH 06/11] add interface --- .../java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java | 2 +- .../java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java | 2 +- .../java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java | 5 ++++- .../main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java | 2 +- .../cn/edu/tsinghua/iginx/relational/RelationalStorage.java | 4 ++-- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java index ece91da52a..ca623f630b 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java +++ b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java @@ -230,7 +230,7 @@ private String findExtremeRecordPath( } @Override - public List getColumns() { + public List getColumns(Set pattern, TagFilter tagFilter) { List timeseries = new ArrayList<>(); for (Bucket bucket : diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java b/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java index 2dde3d2ca8..ca8b450d6b 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java +++ b/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java @@ -295,7 +295,7 @@ private static long getDuplicateKey(WriteError error) { } @Override - public List getColumns() { + public List getColumns(Set pattern, TagFilter tagFilter) { List columns = new ArrayList<>(); for (String dbName : getDatabaseNames(this.client)) { MongoDatabase db = this.client.getDatabase(dbName); diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java index d39b62d931..12c69e5149 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java +++ b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java @@ -32,6 +32,7 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.KeyFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.metadata.entity.*; import cn.edu.tsinghua.iginx.parquet.exec.Executor; import cn.edu.tsinghua.iginx.parquet.exec.LocalExecutor; @@ -44,6 +45,8 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Set; + import org.apache.thrift.transport.TTransportException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -174,7 +177,7 @@ public TaskExecuteResult executeInsert(Insert insert, DataArea dataArea) { } @Override - public List getColumns() throws PhysicalException { + public List getColumns(Set pattern, TagFilter tagFilter) throws PhysicalException { return executor.getColumnsOfStorageUnit("*"); } diff --git a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java b/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java index 831a7b9c1c..37ecff2dc2 100644 --- a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java +++ b/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java @@ -467,7 +467,7 @@ public TaskExecuteResult executeInsert(Insert insert, DataArea dataArea) { } @Override - public List getColumns() { + public List getColumns(Set pattern, TagFilter tagFilter) { List ret = new ArrayList<>(); getIginxColumns(ret::add); getDummyColumns(ret::add); diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java index 484814477d..072723874b 100644 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java @@ -335,7 +335,7 @@ private boolean filterContainsType(List types, Filter filter) { } @Override - public List getColumns() throws RelationalTaskExecuteFailureException { + public List getColumns(Set pattern, TagFilter tagFilter) throws RelationalTaskExecuteFailureException { List columns = new ArrayList<>(); Map extraParams = meta.getExtraParams(); try { @@ -1833,7 +1833,7 @@ private void executeBatchInsert( private List> determineDeletedPaths( List paths, TagFilter tagFilter) { try { - List columns = getColumns(); + List columns = getColumns(null, null); List> deletedPaths = new ArrayList<>(); for (Column column : columns) { From f0655629a03488e1716ce2ceee8fab3366eb796e Mon Sep 17 00:00:00 2001 From: RemHero <1104304963@qq.com> Date: Thu, 6 Jun 2024 11:30:12 +0800 Subject: [PATCH 07/11] fix --- .../execute/StoragePhysicalTaskExecutor.java | 39 ++++++++++++++----- .../iginx/filesystem/exec/LocalExecutor.java | 30 ++++++++------ .../tsinghua/iginx/iotdb/IoTDBStorage.java | 8 ++-- 3 files changed, 50 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java index 7735275b22..fc7b6314b1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java @@ -58,7 +58,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; +import java.util.regex.Pattern; import java.util.stream.Collectors; + +import cn.edu.tsinghua.iginx.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -314,6 +317,8 @@ public TaskExecuteResult executeGlobalTask(GlobalPhysicalTask task) { public TaskExecuteResult executeShowColumns(ShowColumns showColumns) { List storageList = metaManager.getStorageEngineList(); Set columnSet = new HashSet<>(); + TreeSet columnSetAfterFilter = + new TreeSet<>(Comparator.comparing(Column::getPhysicalPath)); for (StorageEngineMeta storage : storageList) { long id = storage.getId(); Pair pair = storageManager.getStorage(id); @@ -323,18 +328,36 @@ public TaskExecuteResult executeShowColumns(ShowColumns showColumns) { try { Set patternSet = showColumns.getPathRegexSet(); TagFilter tagFilter = showColumns.getTagFilter(); - if (storage.getDataPrefix() != null) { - patternSet.add(storage.getDataPrefix() + ".*"); - } + List columnList = pair.k.getColumns(patternSet, tagFilter); - // fix the schemaPrefix + + // fix the schemaPrefix and dataPrefix String schemaPrefix = storage.getSchemaPrefix(); - if (schemaPrefix != null) { + String dataPrefixRegex = StringUtils.reformatPath(storage.getDataPrefix() + ".*"); + if (tagFilter == null) { for (Column column : columnList) { if (column.isDummy()) { - column.setPath(schemaPrefix + "." + column.getPath()); + if (Pattern.matches(dataPrefixRegex, column.getPath())) { + if (schemaPrefix != null) { + column.setPath(schemaPrefix + "." + column.getPath()); + boolean isMatch = false; + for (String pathRegex : patternSet) { + if (Pattern.matches(StringUtils.reformatPath(pathRegex), column.getPath())) { + isMatch=true; + break; + } + } + if (isMatch) { + columnSetAfterFilter.add(column); + } + } + } + } else { + columnSetAfterFilter.addAll(columnList); } } + } else { + columnSetAfterFilter.addAll(columnList); } columnSet.addAll(columnList); } catch (PhysicalException e) { @@ -342,10 +365,6 @@ public TaskExecuteResult executeShowColumns(ShowColumns showColumns) { } } - TreeSet columnSetAfterFilter = - new TreeSet<>(Comparator.comparing(Column::getPhysicalPath)); - columnSetAfterFilter.addAll(columnSet); - int limit = showColumns.getLimit(); int offset = showColumns.getOffset(); if (limit == Integer.MAX_VALUE && offset == 0) { diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java index 90cbf66cd6..44c44696e3 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java +++ b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java @@ -312,6 +312,18 @@ public TaskExecuteResult executeDeleteTask( return new TaskExecuteResult(null, null); } + boolean isPathMatchPattern(String path, Set pattern) { + if (pattern.isEmpty()) { + return true; + } + for (String pathRegex : pattern) { + if (Pattern.matches(StringUtils.reformatPath(pathRegex), path)) { + return true; + } + } + return false; + } + @Override public List getColumnsOfStorageUnit( String storageUnit, Set pattern, TagFilter tagFilter) throws PhysicalException { @@ -322,7 +334,6 @@ public List getColumnsOfStorageUnit( FileMeta meta = fileSystemManager.getFileMeta(file); String columnPath = FilePathUtils.convertAbsolutePathToPath(root, file.getAbsolutePath(), storageUnit); - boolean isChosen = true; if (meta == null) { throw new PhysicalException( String.format( @@ -330,15 +341,7 @@ public List getColumnsOfStorageUnit( file.getAbsolutePath())); } // get columns by pattern - if (!pattern.isEmpty()) { - for (String pathRegex : pattern) { - if (!Pattern.matches(StringUtils.reformatPath(pathRegex), columnPath)) { - isChosen = false; - break; - } - } - } - if (!isChosen) { + if(!isPathMatchPattern(columnPath, pattern)) { continue; } // get columns by tag filter @@ -350,10 +353,13 @@ public List getColumnsOfStorageUnit( // get columns from dummy storage unit if (hasData && dummyRoot != null && tagFilter == null) { for (File file : fileSystemManager.getAllFiles(new File(realDummyRoot), true)) { + String dummyPath = FilePathUtils.convertAbsolutePathToPath(dummyRoot, file.getAbsolutePath(), storageUnit); + if(!isPathMatchPattern(dummyPath, pattern)) { + continue; + } columns.add( new Column( - FilePathUtils.convertAbsolutePathToPath( - dummyRoot, file.getAbsolutePath(), storageUnit), + dummyPath, DataType.BINARY, null, true)); diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java index a31aa6e56d..3685bbd823 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java +++ b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java @@ -226,17 +226,15 @@ private void getColumns2StorageUnit( if (columns2StorageUnit != null) { columns2StorageUnit.put(pair.k, fragment); } - boolean isChosen = true; + boolean isChosen = false; // get columns by pattern if (!pattern.isEmpty()) { for (String pathRegex : pattern) { - if (!Pattern.matches(StringUtils.reformatPath(pathRegex), pair.k)) { - isChosen = false; + if (Pattern.matches(StringUtils.reformatPath(pathRegex), pair.k)) { + isChosen = true; break; } } - } else { - if (isDummy) continue; } if (!isChosen) { continue; From e16e2aeba55ed15ae6046d16acad1869127786e8 Mon Sep 17 00:00:00 2001 From: RemHero <1104304963@qq.com> Date: Thu, 6 Jun 2024 13:45:43 +0800 Subject: [PATCH 08/11] fix --- .../iginx/filesystem/exec/LocalExecutor.java | 3 ++- .../tsinghua/iginx/iotdb/IoTDBStorage.java | 23 +++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java index 44c44696e3..85c7e3bcc2 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java +++ b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java @@ -346,8 +346,9 @@ public List getColumnsOfStorageUnit( } // get columns by tag filter if (tagFilter != null && !TagKVUtils.match(meta.getTags(), tagFilter)) { - columns.add(new Column(columnPath, meta.getDataType(), meta.getTags(), false)); + continue; } + columns.add(new Column(columnPath, meta.getDataType(), meta.getTags(), false)); } } // get columns from dummy storage unit diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java index 3685bbd823..092f4bcd6c 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java +++ b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java @@ -199,6 +199,18 @@ public List getColumns(Set pattern, TagFilter tagFilter) return columns; } + boolean isPathMatchPattern(String path, Set pattern) { + if (pattern.isEmpty()) { + return true; + } + for (String pathRegex : pattern) { + if (Pattern.matches(StringUtils.reformatPath(pathRegex), path)) { + return true; + } + } + return false; + } + private void getColumns2StorageUnit( List columns, Map columns2StorageUnit, @@ -226,17 +238,8 @@ private void getColumns2StorageUnit( if (columns2StorageUnit != null) { columns2StorageUnit.put(pair.k, fragment); } - boolean isChosen = false; // get columns by pattern - if (!pattern.isEmpty()) { - for (String pathRegex : pattern) { - if (Pattern.matches(StringUtils.reformatPath(pathRegex), pair.k)) { - isChosen = true; - break; - } - } - } - if (!isChosen) { + if (!isPathMatchPattern(pair.k, pattern)) { continue; } // get columns by tag filter From aa69181c1dc82d93ef9a942167e8a61ec8803fc6 Mon Sep 17 00:00:00 2001 From: RemHero <1104304963@qq.com> Date: Thu, 6 Jun 2024 15:57:35 +0800 Subject: [PATCH 09/11] format --- .github/workflows/DB-CE.yml | 2 +- .github/workflows/standard-test-suite.yml | 40 +++++------ .../execute/StoragePhysicalTaskExecutor.java | 5 +- .../iginx/filesystem/exec/LocalExecutor.java | 14 ++-- .../iginx/parquet/ParquetStorage.java | 4 +- .../iginx/relational/RelationalStorage.java | 3 +- .../expansion/BaseCapacityExpansionIT.java | 70 +++++++++---------- 7 files changed, 67 insertions(+), 71 deletions(-) diff --git a/.github/workflows/DB-CE.yml b/.github/workflows/DB-CE.yml index c331810ec8..a6a941df54 100644 --- a/.github/workflows/DB-CE.yml +++ b/.github/workflows/DB-CE.yml @@ -27,7 +27,7 @@ on: description: "The database to run the test on" type: string required: false - default: '["FileSystem", "IoTDB12"]' + default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' env: VERSION: 0.6.0-SNAPSHOT diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index f5b7a22f3b..a4f8234894 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -11,27 +11,27 @@ concurrency: cancel-in-progress: true jobs: -# unit-test: -# uses: ./.github/workflows/unit-test.yml -# unit-mds: -# uses: ./.github/workflows/unit-mds.yml -# case-regression: -# uses: ./.github/workflows/case-regression.yml -# with: -# metadata-matrix: '["zookeeper"]' -# standalone-test: -# uses: ./.github/workflows/standalone-test.yml -# with: -# metadata-matrix: '["zookeeper"]' -# standalone-test-pushdown: -# uses: ./.github/workflows/standalone-test-pushdown.yml -# with: -# metadata-matrix: '["zookeeper"]' + unit-test: + uses: ./.github/workflows/unit-test.yml + unit-mds: + uses: ./.github/workflows/unit-mds.yml + case-regression: + uses: ./.github/workflows/case-regression.yml + with: + metadata-matrix: '["zookeeper"]' + standalone-test: + uses: ./.github/workflows/standalone-test.yml + with: + metadata-matrix: '["zookeeper"]' + standalone-test-pushdown: + uses: ./.github/workflows/standalone-test-pushdown.yml + with: + metadata-matrix: '["zookeeper"]' db-ce: uses: ./.github/workflows/DB-CE.yml with: metadata-matrix: '["zookeeper"]' -# remote-test: -# uses: ./.github/workflows/remote-test.yml -# with: -# metadata-matrix: '["zookeeper"]' + remote-test: + uses: ./.github/workflows/remote-test.yml + with: + metadata-matrix: '["zookeeper"]' diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java index fc7b6314b1..917a55ceee 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java @@ -53,6 +53,7 @@ import cn.edu.tsinghua.iginx.monitor.HotSpotMonitor; import cn.edu.tsinghua.iginx.monitor.RequestsMonitor; import cn.edu.tsinghua.iginx.utils.Pair; +import cn.edu.tsinghua.iginx.utils.StringUtils; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -60,8 +61,6 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.regex.Pattern; import java.util.stream.Collectors; - -import cn.edu.tsinghua.iginx.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -343,7 +342,7 @@ public TaskExecuteResult executeShowColumns(ShowColumns showColumns) { boolean isMatch = false; for (String pathRegex : patternSet) { if (Pattern.matches(StringUtils.reformatPath(pathRegex), column.getPath())) { - isMatch=true; + isMatch = true; break; } } diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java index 85c7e3bcc2..d7f2160a2a 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java +++ b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java @@ -341,7 +341,7 @@ public List getColumnsOfStorageUnit( file.getAbsolutePath())); } // get columns by pattern - if(!isPathMatchPattern(columnPath, pattern)) { + if (!isPathMatchPattern(columnPath, pattern)) { continue; } // get columns by tag filter @@ -354,16 +354,12 @@ public List getColumnsOfStorageUnit( // get columns from dummy storage unit if (hasData && dummyRoot != null && tagFilter == null) { for (File file : fileSystemManager.getAllFiles(new File(realDummyRoot), true)) { - String dummyPath = FilePathUtils.convertAbsolutePathToPath(dummyRoot, file.getAbsolutePath(), storageUnit); - if(!isPathMatchPattern(dummyPath, pattern)) { + String dummyPath = + FilePathUtils.convertAbsolutePathToPath(dummyRoot, file.getAbsolutePath(), storageUnit); + if (!isPathMatchPattern(dummyPath, pattern)) { continue; } - columns.add( - new Column( - dummyPath, - DataType.BINARY, - null, - true)); + columns.add(new Column(dummyPath, DataType.BINARY, null, true)); } } return columns; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java index 12c69e5149..c2bc012f09 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java +++ b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java @@ -46,7 +46,6 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.thrift.transport.TTransportException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -177,7 +176,8 @@ public TaskExecuteResult executeInsert(Insert insert, DataArea dataArea) { } @Override - public List getColumns(Set pattern, TagFilter tagFilter) throws PhysicalException { + public List getColumns(Set pattern, TagFilter tagFilter) + throws PhysicalException { return executor.getColumnsOfStorageUnit("*"); } diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java index 072723874b..1084b6e69c 100644 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java @@ -335,7 +335,8 @@ private boolean filterContainsType(List types, Filter filter) { } @Override - public List getColumns(Set pattern, TagFilter tagFilter) throws RelationalTaskExecuteFailureException { + public List getColumns(Set pattern, TagFilter tagFilter) + throws RelationalTaskExecuteFailureException { List columns = new ArrayList<>(); Map extraParams = meta.getExtraParams(); try { diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java index a2959c9e1e..676ebc3e30 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java @@ -24,7 +24,6 @@ import java.util.Collections; import java.util.List; import java.util.stream.Collectors; - import org.junit.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -328,14 +327,14 @@ protected void queryExtendedKeyDummy() { session, statement, READ_ONLY_PATH_LIST, READ_ONLY_EXTEND_VALUES_LIST); // test show columns - testShowColumns(Arrays.asList( - new Column("mn.wf01.wt01.temperature", DataType.DOUBLE), - new Column("mn.wf01.wt01.status", DataType.LONG), - new Column("nt.wf03.wt01.status2", DataType.LONG), - new Column("nt.wf04.wt01.temperature", DataType.DOUBLE), - new Column("tm.wf05.wt01.status", DataType.LONG), - new Column("tm.wf05.wt01.temperature", DataType.DOUBLE) - )); + testShowColumns( + Arrays.asList( + new Column("mn.wf01.wt01.temperature", DataType.DOUBLE), + new Column("mn.wf01.wt01.status", DataType.LONG), + new Column("nt.wf03.wt01.status2", DataType.LONG), + new Column("nt.wf04.wt01.temperature", DataType.DOUBLE), + new Column("tm.wf05.wt01.status", DataType.LONG), + new Column("tm.wf05.wt01.temperature", DataType.DOUBLE))); } protected void queryExtendedColDummy() { @@ -474,15 +473,11 @@ private void testShowColumns(List expectColumns) { LOGGER.info("show columns: {}", columns); // 对期望列表和实际列表中的Column对象按路径排序 - List sortedExpectPaths = expectColumns.stream() - .map(Column::getPath) - .sorted() - .collect(Collectors.toList()); + List sortedExpectPaths = + expectColumns.stream().map(Column::getPath).sorted().collect(Collectors.toList()); - List sortedActualPaths = columns.stream() - .map(Column::getPath) - .sorted() - .collect(Collectors.toList()); + List sortedActualPaths = + columns.stream().map(Column::getPath).sorted().collect(Collectors.toList()); // 检查排序后的路径列表是否相同 assertArrayEquals(sortedExpectPaths.toArray(), sortedActualPaths.toArray()); @@ -501,26 +496,31 @@ private void testAddAndRemoveStorageEngineWithPrefix() { List> valuesList = EXP_VALUES_LIST1; - testShowColumns(Arrays.asList( - new Column("b.b.b", DataType.LONG), - new Column("ln.wf02.status", DataType.BOOLEAN), - new Column("ln.wf02.version", DataType.BINARY), - new Column("nt.wf03.wt01.status2", DataType.LONG), - new Column("nt.wf04.wt01.temperature", DataType.DOUBLE), - new Column("zzzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzzzz", DataType.LONG) - )); + testShowColumns( + Arrays.asList( + new Column("b.b.b", DataType.LONG), + new Column("ln.wf02.status", DataType.BOOLEAN), + new Column("ln.wf02.version", DataType.BINARY), + new Column("nt.wf03.wt01.status2", DataType.LONG), + new Column("nt.wf04.wt01.temperature", DataType.DOUBLE), + new Column( + "zzzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzzzz", + DataType.LONG))); // 添加不同 schemaPrefix，相同 dataPrefix addStorageEngine(expPort, true, true, dataPrefix1, schemaPrefix1, extraParams); - testShowColumns(Arrays.asList( - new Column("b.b.b", DataType.LONG), - new Column("ln.wf02.status", DataType.BOOLEAN), - new Column("ln.wf02.version", DataType.BINARY), - new Column("nt.wf03.wt01.status2", DataType.LONG),new Column("p1.nt.wf03.wt01.status2", DataType.LONG), - new Column("nt.wf04.wt01.temperature", DataType.DOUBLE), - new Column("zzzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzzzz", DataType.LONG) - )); + testShowColumns( + Arrays.asList( + new Column("b.b.b", DataType.LONG), + new Column("ln.wf02.status", DataType.BOOLEAN), + new Column("ln.wf02.version", DataType.BINARY), + new Column("nt.wf03.wt01.status2", DataType.LONG), + new Column("p1.nt.wf03.wt01.status2", DataType.LONG), + new Column("nt.wf04.wt01.temperature", DataType.DOUBLE), + new Column( + "zzzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzzzz", + DataType.LONG))); // 添加节点 dataPrefix = dataPrefix1 && schemaPrefix = p1 后查询 String statement = "select status2 from *;"; @@ -734,8 +734,8 @@ private void testSameKeyWarning() { QueryDataSet res = session.executeQuery(statement); if ((res.getWarningMsg() == null - || res.getWarningMsg().isEmpty() - || !res.getWarningMsg().contains("The query results contain overlapped keys.")) + || res.getWarningMsg().isEmpty() + || !res.getWarningMsg().contains("The query results contain overlapped keys.")) && SUPPORT_KEY.get(testConf.getStorageType())) { LOGGER.error("未抛出重叠key的警告"); fail(); From 4c73402cc3058622c44bca98f7451667b8ba2227 Mon Sep 17 00:00:00 2001 From: RemHero <1104304963@qq.com> Date: Fri, 7 Jun 2024 16:14:52 +0800 Subject: [PATCH 10/11] add test --- .../storage/execute/StoragePhysicalTaskExecutor.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java index 917a55ceee..658e615973 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java @@ -315,7 +315,6 @@ public TaskExecuteResult executeGlobalTask(GlobalPhysicalTask task) { public TaskExecuteResult executeShowColumns(ShowColumns showColumns) { List storageList = metaManager.getStorageEngineList(); - Set columnSet = new HashSet<>(); TreeSet columnSetAfterFilter = new TreeSet<>(Comparator.comparing(Column::getPhysicalPath)); for (StorageEngineMeta storage : storageList) { @@ -332,14 +331,14 @@ public TaskExecuteResult executeShowColumns(ShowColumns showColumns) { // fix the schemaPrefix and dataPrefix String schemaPrefix = storage.getSchemaPrefix(); - String dataPrefixRegex = StringUtils.reformatPath(storage.getDataPrefix() + ".*"); + String dataPrefixRegex = storage.getDataPrefix() == null ? null : StringUtils.reformatPath(storage.getDataPrefix() + ".*"); if (tagFilter == null) { for (Column column : columnList) { if (column.isDummy()) { - if (Pattern.matches(dataPrefixRegex, column.getPath())) { + if (dataPrefixRegex == null || Pattern.matches(dataPrefixRegex, column.getPath())) { if (schemaPrefix != null) { column.setPath(schemaPrefix + "." + column.getPath()); - boolean isMatch = false; + boolean isMatch = patternSet.isEmpty(); for (String pathRegex : patternSet) { if (Pattern.matches(StringUtils.reformatPath(pathRegex), column.getPath())) { isMatch = true; @@ -349,16 +348,17 @@ public TaskExecuteResult executeShowColumns(ShowColumns showColumns) { if (isMatch) { columnSetAfterFilter.add(column); } + } else { + columnSetAfterFilter.add(column); } } } else { - columnSetAfterFilter.addAll(columnList); + columnSetAfterFilter.add(column); } } } else { columnSetAfterFilter.addAll(columnList); } - columnSet.addAll(columnList); } catch (PhysicalException e) { return new TaskExecuteResult(e); } From 4a48e82ce9f1c187fa05cc265e6d417448e90ab7 Mon Sep 17 00:00:00 2001 From: RemHero <1104304963@qq.com> Date: Fri, 7 Jun 2024 16:21:44 +0800 Subject: [PATCH 11/11] remove test --- .github/workflows/DB-CE.yml | 2 +- .github/workflows/standard-test-suite.yml | 40 +++++++++++------------ 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/DB-CE.yml b/.github/workflows/DB-CE.yml index a6a941df54..c331810ec8 100644 --- a/.github/workflows/DB-CE.yml +++ b/.github/workflows/DB-CE.yml @@ -27,7 +27,7 @@ on: description: "The database to run the test on" type: string required: false - default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' + default: '["FileSystem", "IoTDB12"]' env: VERSION: 0.6.0-SNAPSHOT diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index a4f8234894..f5b7a22f3b 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -11,27 +11,27 @@ concurrency: cancel-in-progress: true jobs: - unit-test: - uses: ./.github/workflows/unit-test.yml - unit-mds: - uses: ./.github/workflows/unit-mds.yml - case-regression: - uses: ./.github/workflows/case-regression.yml - with: - metadata-matrix: '["zookeeper"]' - standalone-test: - uses: ./.github/workflows/standalone-test.yml - with: - metadata-matrix: '["zookeeper"]' - standalone-test-pushdown: - uses: ./.github/workflows/standalone-test-pushdown.yml - with: - metadata-matrix: '["zookeeper"]' +# unit-test: +# uses: ./.github/workflows/unit-test.yml +# unit-mds: +# uses: ./.github/workflows/unit-mds.yml +# case-regression: +# uses: ./.github/workflows/case-regression.yml +# with: +# metadata-matrix: '["zookeeper"]' +# standalone-test: +# uses: ./.github/workflows/standalone-test.yml +# with: +# metadata-matrix: '["zookeeper"]' +# standalone-test-pushdown: +# uses: ./.github/workflows/standalone-test-pushdown.yml +# with: +# metadata-matrix: '["zookeeper"]' db-ce: uses: ./.github/workflows/DB-CE.yml with: metadata-matrix: '["zookeeper"]' - remote-test: - uses: ./.github/workflows/remote-test.yml - with: - metadata-matrix: '["zookeeper"]' +# remote-test: +# uses: ./.github/workflows/remote-test.yml +# with: +# metadata-matrix: '["zookeeper"]'