Sj test1

.travis.yml

@@ -11,8 +11,14 @@ env:
   matrix:
     - MAVEN_CHECKS=true
     - WEBUI_CHECKS=true
-    - TEST_SPECIFIC_MODULES=presto-tests
-    - TEST_SPECIFIC_MODULES=presto-tests TEST_FLAGS="-P ci-only"
+    - TEST_SPECIFIC_MODULES=presto-tests TEST_FLAGS="-P presto-tests-execution-memory"
+    - TEST_SPECIFIC_MODULES=presto-tests TEST_FLAGS="-P presto-tests-general"
+    - TEST_SPECIFIC_MODULES=presto-tests TEST_FLAGS="-P ci-only-distributed-non-hash-gen"
+    - TEST_SPECIFIC_MODULES=presto-tests TEST_FLAGS="-P ci-only-tpch-distributed-queries"
+    - TEST_SPECIFIC_MODULES=presto-tests TEST_FLAGS="-P ci-only-local-queries"
+    - TEST_SPECIFIC_MODULES=presto-tests TEST_FLAGS="-P ci-only-distributed-queries"
+    - TEST_SPECIFIC_MODULES=presto-tests TEST_FLAGS="-P ci-only-aggregation-queries"
+    - TEST_SPECIFIC_MODULES=presto-tests TEST_FLAGS="-P ci-only-plan-determinism"
     - TEST_SPECIFIC_MODULES=presto-raptor
     - TEST_SPECIFIC_MODULES=presto-accumulo
     - TEST_SPECIFIC_MODULES=presto-cassandra TEST_FLAGS="-P test-cassandra-integration-smoke-test"

presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java

@@ -778,7 +778,7 @@ public void finishStatisticsCollection(Session session, AnalyzeTableHandle table
     {
         ConnectorId connectorId = tableHandle.getConnectorId();
         CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, connectorId);
-        catalogMetadata.getMetadata().finishStatisticsCollection(session.toConnectorSession(), tableHandle.getConnectorHandle(), computedStatistics);
+        catalogMetadata.getMetadata().finishStatisticsCollection(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), computedStatistics);
     }
 
     @Override

presto-main/src/main/java/com/facebook/presto/operator/project/InputPageProjection.java

@@ -64,7 +64,7 @@ public Work<Block> project(ConnectorSession session, DriverYieldSignal yieldSign
             result = block.copyPositions(selectedPositions.getPositions(), selectedPositions.getOffset(), selectedPositions.size());
         }
         else if (selectedPositions.getOffset() == 0 && selectedPositions.size() == page.getPositionCount()) {
-            result = block;
+            result = block.getLoadedBlock();
         }
         else {
             result = block.getRegion(selectedPositions.getOffset(), selectedPositions.size());

presto-main/src/main/java/com/facebook/presto/operator/repartition/OptimizedPartitionedOutputOperator.java

@@ -164,7 +164,6 @@ public void addInput(Page page)
             return;
         }
 
-        page = page.getLoadedPage();
         page = pagePreprocessor.apply(page);
         pagePartitioner.partitionPage(page);
 

presto-main/src/main/java/com/facebook/presto/server/TaskResource.java

@@ -114,12 +114,7 @@ public TaskResource(
         this.sessionPropertyManager = requireNonNull(sessionPropertyManager, "sessionPropertyManager is null");
         this.responseExecutor = requireNonNull(responseExecutor, "responseExecutor is null");
         this.timeoutExecutor = requireNonNull(timeoutExecutor, "timeoutExecutor is null");
-        if (communicationConfig.isBinaryTransportEnabled()) {
-            this.planFragmentCodec = planFragmentSmileCodec;
-        }
-        else {
-            this.planFragmentCodec = wrapJsonCodec(planFragmentJsonCodec);
-        }
+        this.planFragmentCodec = wrapJsonCodec(planFragmentJsonCodec);
     }
 
     @GET

presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java

@@ -771,10 +771,6 @@ private synchronized void abort(TaskStatus status)
         checkState(status.getState().isDone(), "cannot abort task with an incomplete status");
 
         try (SetThreadName ignored = new SetThreadName("HttpRemoteTask-%s", taskId)) {
-            // With recoverable grouped execution, failed task does not necessarily fail the whole query.
-            // Not updating task info makes query unable to finish in tests because failed task is stuck in RUNNING state.
-            // TODO: Investigate why this only happens in TestHiveRecoverableGroupedExecution when worker is closed, but not in production test via cli.
-            taskInfoFetcher.updateTaskInfo(getTaskInfo().withTaskStatus(status));
             taskStatusFetcher.updateTaskStatus(status);
 
             // send abort to task
@@ -880,7 +876,15 @@ private void failTask(Throwable cause)
             log.debug(cause, "Remote task %s failed with %s", taskStatus.getSelf(), cause);
         }
 
-        abort(failWith(getTaskStatus(), FAILED, ImmutableList.of(toFailure(cause))));
+        TaskStatus failedTaskStatus = failWith(getTaskStatus(), FAILED, ImmutableList.of(toFailure(cause)));
+        // Transition task to failed state without waiting for the final task info returned by the abort request.
+        // The abort request is very likely not to succeed, leaving the task and the stage in the limbo state for
+        // the entire duration of abort retries. If the task is failed, it is not that important to actually
+        // record the final statistics and the final information about a failed task.
+        taskInfoFetcher.updateTaskInfo(getTaskInfo().withTaskStatus(failedTaskStatus));
+
+        // Initiate abort request
+        abort(failedTaskStatus);
     }
 
     private HttpUriBuilder getHttpUriBuilder(TaskStatus taskStatus)

presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java

@@ -117,14 +117,13 @@ public HttpRemoteTaskFactory(
             this.taskStatusCodec = taskStatusSmileCodec;
             this.taskInfoCodec = taskInfoSmileCodec;
             this.taskUpdateRequestCodec = taskUpdateRequestSmileCodec;
-            this.planFragmentCodec = planFragmentSmileCodec;
         }
         else {
             this.taskStatusCodec = wrapJsonCodec(taskStatusJsonCodec);
             this.taskInfoCodec = wrapJsonCodec(taskInfoJsonCodec);
             this.taskUpdateRequestCodec = wrapJsonCodec(taskUpdateRequestJsonCodec);
-            this.planFragmentCodec = wrapJsonCodec(planFragmentJsonCodec);
         }
+        this.planFragmentCodec = wrapJsonCodec(planFragmentJsonCodec);
 
         this.updateScheduledExecutor = newSingleThreadScheduledExecutor(daemonThreadsNamed("task-info-update-scheduler-%s"));
         this.errorScheduledExecutor = newSingleThreadScheduledExecutor(daemonThreadsNamed("remote-task-error-delay-%s"));

presto-mongodb/src/test/java/com/facebook/presto/mongodb/TestMongoClientConfig.java

@@ -35,7 +35,7 @@ public void testDefaults()
                 .setConnectionsPerHost(100)
                 .setMaxWaitTime(120_000)
                 .setConnectionTimeout(10_000)
-                .setSocketTimeout(0)
+                .setSocketTimeout(10)
                 .setSocketKeepAlive(false)
                 .setSslEnabled(false)
                 .setCursorBatchSize(0)

presto-tests/pom.xml

@@ -220,8 +220,13 @@
                     <!-- these tests take a very long time so only run them in the CI server -->
                     <excludes>
                         <exclude>**/TestDistributedQueriesNoHashGeneration.java</exclude>
+                        <exclude>**/TestTpchDistributedQueries.java</exclude>
                         <exclude>**/TestLocalQueries.java</exclude>
                         <exclude>**/TestNonIterativeDistributedQueries.java</exclude>
+                        <exclude>**/TestAggregations.java</exclude>
+                        <exclude>**/TestOptimizeMixedDistinctAggregations.java</exclude>
+                        <exclude>**/TestSpilledAggregations.java</exclude>
+                        <exclude>**/TestQueryPlanDeterminism.java</exclude>
                     </excludes>
                 </configuration>
             </plugin>
@@ -245,7 +250,7 @@
         </profile>
 
         <profile>
-            <id>ci-only</id>
+            <id>ci-only-distributed-non-hash-gen</id>
             <build>
                 <plugins>
                     <plugin>
@@ -255,13 +260,144 @@
                             <excludes combine.self="override" />
                             <includes>
                                 <include>**/TestDistributedQueriesNoHashGeneration.java</include>
+                            </includes>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+        <profile>
+            <id>ci-only-tpch-distributed-queries</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <configuration>
+                            <excludes combine.self="override" />
+                            <includes>
+                                <include>**/TestTpchDistributedQueries.java</include>
+                            </includes>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+        <profile>
+            <id>ci-only-local-queries</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <configuration>
+                            <excludes combine.self="override" />
+                            <includes>
                                 <include>**/TestLocalQueries.java</include>
+                            </includes>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+        <profile>
+            <id>ci-only-distributed-queries</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <configuration>
+                            <excludes combine.self="override" />
+                            <includes>
                                 <include>**/TestNonIterativeDistributedQueries.java</include>
                             </includes>
                         </configuration>
                     </plugin>
                 </plugins>
             </build>
         </profile>
+        <profile>
+            <id>ci-only-plan-determinism</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <configuration>
+                            <excludes combine.self="override" />
+                            <includes>
+                                <include>**/TestQueryPlanDeterminism.java</include>
+                            </includes>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+
+        <profile>
+            <id>ci-only-aggregation-queries</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <configuration>
+                            <excludes combine.self="override" />
+                            <includes>
+                                <include>**/TestAggregations.java</include>
+                                <include>**/TestOptimizeMixedDistinctAggregations.java</include>
+                                <include>**/TestSpilledAggregations.java</include>
+                            </includes>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+
+        <profile>
+            <id>presto-tests-execution-memory</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <configuration>
+                            <includes>
+                                <include>**/execution/*.java</include>
+                                <include>**/connector/informationschema/*.java</include>
+                                <include>**/memory/*.java</include>
+                            </includes>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+        <profile>
+            <id>presto-tests-general</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <configuration>
+                            <includes>
+                                <include>**/tests/*.java</include>
+                            </includes>
+                            <excludes>
+                                <exclude>**/TestDistributedQueriesNoHashGeneration.java</exclude>
+                                <exclude>**/TestTpchDistributedQueries.java</exclude>
+                                <exclude>**/TestLocalQueries.java</exclude>
+                                <exclude>**/TestNonIterativeDistributedQueries.java</exclude>
+                                <exclude>**/TestAggregations.java</exclude>
+                                <exclude>**/TestOptimizeMixedDistinctAggregations.java</exclude>
+                                <exclude>**/TestSpilledAggregations.java</exclude>
+                                <exclude>**/TestQueryPlanDeterminism.java</exclude>
+                            </excludes>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
     </profiles>
 </project>

presto-verifier/src/main/java/com/facebook/presto/verifier/checksum/ChecksumResult.java

@@ -21,6 +21,7 @@
 import java.sql.Types;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
@@ -55,7 +56,7 @@ public Object getChecksum(String columnName)
         return checksums.get(columnName);
     }
 
-    public static ChecksumResult fromResultSet(ResultSet resultSet)
+    public static Optional<ChecksumResult> fromResultSet(ResultSet resultSet)
             throws SQLException
     {
         long rowCount = resultSet.getLong(1);
@@ -83,6 +84,6 @@ else if (metaData.getColumnType(i) == Types.BIGINT) {
                 checksums.put(columnName, new SqlVarbinary((byte[]) checksum));
             }
         }
-        return new ChecksumResult(rowCount, checksums);
+        return Optional.of(new ChecksumResult(rowCount, checksums));
     }
 }

presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerificationUtil.java

@@ -83,7 +83,7 @@ public static void teardownSafely(PrestoAction prestoAction, @Nullable QueryBund
     public static List<Column> getColumns(PrestoAction prestoAction, TypeManager typeManager, QualifiedName tableName)
     {
         return prestoAction
-                .execute(new ShowColumns(tableName), DESCRIBE, resultSet -> Column.fromResultSet(typeManager, resultSet))
+                .execute(new ShowColumns(tableName), DESCRIBE, resultSet -> Optional.of(Column.fromResultSet(typeManager, resultSet)))
                 .getResults();
     }