From 75891ca9b2bb80e3b7eebf0de9681802a4daa828 Mon Sep 17 00:00:00 2001 From: sonal Date: Tue, 13 Aug 2019 10:47:00 +0530 Subject: [PATCH 1/9] - Fix issue with registering SparkSubmit processes in case of multiple host tests. - Minor test fixes. --- .../snappydata/hydra/cluster/SnappyTest.java | 36 ++++++------------- .../hydra/streaming_sink/clusterRestart.inc | 2 +- .../streaming_sink/streamingWithLeadHA.inc | 3 +- .../streaming_sink/streamingWithLocatorHA.inc | 2 +- .../streaming_sink/streamingWithServerHA.inc | 2 +- .../streaming_sinkForConflation.conf | 2 +- .../streaming_sinkForUpdateAndDelete.conf | 3 +- ...streaming_sinkWithAddNewNodeRebalance.conf | 7 ++-- .../streaming_sinkWithClusterRestart.conf | 3 ++ .../streaming_sinkWithRebalance.conf | 5 +-- .../streaming_sinkWithRestartStreaming.conf | 2 +- .../hydra/testDMLOps/SnappyDMLOpsUtil.java | 2 ++ ...logConsistencyWithAddNewNodeRebalance.conf | 5 ++- .../hydra/testDMLOps/ValidateDMLOpApp.scala | 3 +- 14 files changed, 35 insertions(+), 42 deletions(-) diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java index 3b10634b43..871a7cd2e5 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java @@ -1850,20 +1850,7 @@ public void executeProcess(ProcessBuilder pb, File logFile) { } public synchronized void recordSnappyProcessIDinNukeRun(String pName) { - Process pr = null; - try { - File log = new File("."); - String dest = log.getCanonicalPath() + File.separator + "PIDs_" + pName + "_" + HostHelper - .getLocalHost() + - ".log"; - File logFile = new File(dest); - if (!logFile.exists()) { - recordProcessIds(pName, HostHelper.getLocalHost(), logFile); - } - } catch (IOException e) { - String s = "Problem while starting the process : " + pr; - throw new TestException(s, e); - } + recordSnappyProcessIDinNukeRun(pName, HostHelper.getLocalHost()); } protected synchronized void recordSnappyProcessIDinNukeRun(String pName, String hostName) { @@ -1874,19 +1861,11 @@ protected synchronized void recordSnappyProcessIDinNukeRun(String pName, String String dest = log.getCanonicalPath() + File.separator + "PIDs_" + pName + "_" + hostName + ".log"; File logFile = new File(dest); if (!logFile.exists()) { - String command; - command = "ssh -n -x -o PasswordAuthentication=no -o StrictHostKeyChecking=no " + hostName; - pb = new ProcessBuilder("/bin/bash", "-c", command); - pr = pb.start(); - pr.waitFor(); recordProcessIds(pName, hostName, logFile); } } catch (IOException e) { String s = "Problem while starting the process : " + pr; throw new TestException(s, e); - } catch (InterruptedException e) { - String s = "Exception occurred while waiting for the process execution : " + pr; - throw new TestException(s, e); } } @@ -1895,9 +1874,10 @@ protected static void recordProcessIds(String pName, String hostName, File logFi ProcessBuilder pb; Process pr = null; try { + command = "ssh -n -x -o PasswordAuthentication=no -o StrictHostKeyChecking=no " + hostName; if (pName.equals("Master")) command = "ps ax | grep -w " + pName + " | grep -v grep | awk '{print $1}'"; - else command = "jps | grep " + pName + " | awk '{print $1}'"; + else command += " jps | grep " + pName + " | awk '{print $1}'"; hd = TestConfig.getInstance().getMasterDescription() .getVmDescription().getHostDescription(); pb = new ProcessBuilder("/bin/bash", "-c", command); @@ -2848,6 +2828,7 @@ public static synchronized void HydraTask_stopSparkCluster() { */ public static synchronized void HydraTask_stopSnappyLeader() { File log = null; + if(snappyTest==null) snappyTest = new SnappyTest(); try { ProcessBuilder pb = new ProcessBuilder(snappyTest.getScriptLocation("snappy-leads.sh"), "stop"); @@ -2870,6 +2851,7 @@ public static synchronized void HydraTask_stopSnappyLeader() { */ public static synchronized void HydraTask_stopSnappyServers() { File log = null; + if(snappyTest==null) snappyTest = new SnappyTest(); try { ProcessBuilder pb = new ProcessBuilder(snappyTest.getScriptLocation("snappy-servers.sh"), "stop"); log = new File("."); @@ -2891,6 +2873,7 @@ public static synchronized void HydraTask_stopSnappyServers() { */ public static synchronized void HydraTask_stopSnappyLocator() { File log = null; + if(snappyTest==null) snappyTest = new SnappyTest(); try { ProcessBuilder pb = new ProcessBuilder(snappyTest.getScriptLocation("snappy-locators.sh") , "stop"); @@ -3610,13 +3593,16 @@ protected static void getLeadVM(ClientVmInfo info, ArrayList vmList, ArrayList s if (!isPrimaryLeadUp) { for (int i = 0; i < 3; i++) { if (isPrimaryLeadUp) break; + sleepForMs(5); isPrimaryLeadUp = isPrimaryLeadUpAndRunning(); } } if (cycleVms && !isPrimaryLeadUp) { - do { + for (int i = 0; i < 3; i++) { + if (isPrimaryLeadUp) break; + sleepForMs(5); isPrimaryLeadUp = isPrimaryLeadUpAndRunning(); - } while (isPrimaryLeadUp); + } } if (!isPrimaryLeadUp) { throw new TestException("Primary lead node is not up and running in the cluster."); diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/clusterRestart.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/clusterRestart.inc index 96c5983863..5a6c0a025a 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/clusterRestart.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/clusterRestart.inc @@ -3,7 +3,7 @@ TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMe io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "kafkaTopic=streamingSinkTopic,tableName=persoon,streaming.batch_interval=100,isConflation=false" io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar - threadGroups = leadThread + threadGroups = snappyThreads startInterval = 60 maxTimesToRun = 1; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc index 52a475c3ee..a6c1303992 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc @@ -13,9 +13,8 @@ TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest t io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar io.snappydata.hydra.cluster.SnappyPrms-sleepTimeSecsBeforeRestart = 60 - maxThreads = 1 startInterval = 120 - threadGroups = leadThread + threadGroups = snappyThreads maxTimesToRun = 1; CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc index 84da4240c7..74e94d75b0 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc @@ -9,7 +9,7 @@ INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_cycleLocatorVms startInterval = 60 - threadGroups = leadThread + threadGroups = snappyThreads io.snappydata.hydra.cluster.SnappyPrms-sleepTimeSecsBeforeRestart = 60 maxTimesToRun = 1; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc index d1b4bcee49..805c3debf2 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc @@ -9,7 +9,7 @@ INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = b TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_cycleStoreVms startInterval = 60 - threadGroups = leadThread + threadGroups = snappyThreads io.snappydata.hydra.cluster.SnappyPrms-sleepTimeSecsBeforeRestart = 60 maxTimesToRun = 1; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForConflation.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForConflation.conf index 401a907540..7ac7077bdb 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForConflation.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForConflation.conf @@ -20,7 +20,7 @@ INITTASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_generateAndPublishMethod //io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = "io.snappydata.hydra.streaming_sink.StringMessageProducer" - io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "300000,streamingSinkTopic,0,${opType}" + io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "150000,streamingSinkTopic,0,${opType}" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar io.snappydata.hydra.cluster.SnappyPrms-isConflationTest = true threadGroups = leadThread diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf index 942d50b5bd..9df1a81ede 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf @@ -22,7 +22,8 @@ TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest tas // io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = "io.snappydata.hydra.streaming_sink.StringMessageProducer" io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "500000,streamingSinkTopic,100000,4" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar - threadGroups = snappyThreads; + maxTimesToRun=1 + threadGroups = leadThread; TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSQLScripts io.snappydata.hydra.cluster.SnappyPrms-logFileName = sqlScriptResults_1.log diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithAddNewNodeRebalance.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithAddNewNodeRebalance.conf index 4fd235778d..dc5df919a1 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithAddNewNodeRebalance.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithAddNewNodeRebalance.conf @@ -10,7 +10,10 @@ TASK taskClass = io.snappydata.hydra.cdcConnector.SnappyCDCTest taskMethod // trigger rebalance TASK taskClass = io.snappydata.hydra.cdcConnector.SnappyCDCTest taskMethod = HydraTask_performRebalance - threadGroups = leadThread + threadGroups = snappyStoreThreads startInterval=180 - maxThreads = 1; + maxThreads = 1 + maxTimesToRun = 1; +io.snappydata.hydra.cdcConnector.SnappyCDCPrms-dataLocation = /nfs/users/$USER/snappyHydraLogs/newNodeTest; +io.snappydata.hydra.cdcConnector.SnappyCDCPrms-nodeName = $HOSTNAME; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithClusterRestart.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithClusterRestart.conf index cfb611f281..815a4bbaf2 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithClusterRestart.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithClusterRestart.conf @@ -1,3 +1,6 @@ + INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf; INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/clusterRestart.inc; + +hydra.Prms-maxResultWaitSec = 1200; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRebalance.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRebalance.conf index 19e1845a14..3bbd886365 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRebalance.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRebalance.conf @@ -2,6 +2,7 @@ INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDel // trigger rebalance TASK taskClass = io.snappydata.hydra.cdcConnector.SnappyCDCTest taskMethod = HydraTask_performRebalance - threadGroups = leadThread + threadGroups = snappyStoreThreads startInterval = 240 - maxThreads = 1; + maxThreads = 1 + maxTimesToRun = 1; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRestartStreaming.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRestartStreaming.conf index cc64ca2298..6f86b87ae4 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRestartStreaming.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRestartStreaming.conf @@ -5,7 +5,7 @@ TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest tas io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "kafkaTopic=streamingSinkTopic,tableName=persoon,isConflation=false" io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar - threadGroups = leadThread + threadGroups = snappyThreads startInterval = 120 maxTimesToRun = 1; diff --git a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/SnappyDMLOpsUtil.java b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/SnappyDMLOpsUtil.java index 43f8ae6a7e..e1b4508625 100644 --- a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/SnappyDMLOpsUtil.java +++ b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/SnappyDMLOpsUtil.java @@ -870,6 +870,7 @@ public void performInsertUsingBatch(Connection conn, String tableName, Log.getLogWriter().info("Executing batch statement for insert."); snappyPS.executeBatch(); if (hasDerbyServer) derbyPS.executeBatch(); + Log.getLogWriter().info("Executed batch statement for insert."); batchCnt = 0; } } @@ -877,6 +878,7 @@ public void performInsertUsingBatch(Connection conn, String tableName, if(!isPopulate) SnappyConsistencyTest.waitForBarrier(tid + "", 2); snappyPS.executeBatch(); snappyPS.close(); + Log.getLogWriter().info("Executed the batch."); if(!isPopulate) SnappyConsistencyTest.waitForBarrier(tid + "", 2); if (hasDerbyServer) { derbyPS.executeBatch(); diff --git a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithAddNewNodeRebalance.conf b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithAddNewNodeRebalance.conf index c01c78efd7..70c9a92630 100644 --- a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithAddNewNodeRebalance.conf +++ b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithAddNewNodeRebalance.conf @@ -37,6 +37,5 @@ hydra.Prms-maxResultWaitSec = 900; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-dmlOperations = ONEOF insert update insert insert delete insert FOENO; -// add in local.conf -// io.snappydata.hydra.cdcConnector.SnappyCDCPrms-dataLocation = <-dir path for new server> -// io.snappydata.hydra.cdcConnector.SnappyCDCPrms-nodeName = \ No newline at end of file +io.snappydata.hydra.cdcConnector.SnappyCDCPrms-dataLocation = /nfs/users/$USER/snappyHydraLogs/newNodeTest; +io.snappydata.hydra.cdcConnector.SnappyCDCPrms-nodeName = $HOSTNAME; diff --git a/dtests/src/test/scala/io/snappydata/hydra/testDMLOps/ValidateDMLOpApp.scala b/dtests/src/test/scala/io/snappydata/hydra/testDMLOps/ValidateDMLOpApp.scala index d9d31e7f9f..68d8789036 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/testDMLOps/ValidateDMLOpApp.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/testDMLOps/ValidateDMLOpApp.scala @@ -35,11 +35,10 @@ object ValidateDMLOpApp { val outputFile = "DMLOpsApp_thr_" + threadID + "_" + System.currentTimeMillis + ".out" val pw = new PrintWriter(new FileOutputStream(new File(outputFile), true)) val stmt = args(0) - // scalastyle:off println pw.println(s"stmt : ${stmt}") - val tableName = args(1) pw.println(s"Executing ${stmt}") + pw.flush() val startTime = System.currentTimeMillis val df = snc.sql(stmt) val endTime = System.currentTimeMillis From ca22e508f3c2bd0d468be6362d683ab768f2815e Mon Sep 17 00:00:00 2001 From: sonal Date: Wed, 14 Aug 2019 10:21:22 +0530 Subject: [PATCH 2/9] - Closing the snappy connection in case left open. - Minor test fixes. --- .../io/snappydata/hydra/cdcConnector/SnappyCDCTest.java | 6 +++++- dtests/src/test/java/io/snappydata/hydra/misc/misc.bt | 9 ++------- .../testDMLOpsCatalogConsistencyWithRebalance.conf | 2 ++ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/dtests/src/test/java/io/snappydata/hydra/cdcConnector/SnappyCDCTest.java b/dtests/src/test/java/io/snappydata/hydra/cdcConnector/SnappyCDCTest.java index 1fcc36df11..63fb23f818 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cdcConnector/SnappyCDCTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/cdcConnector/SnappyCDCTest.java @@ -105,8 +105,9 @@ public static void HydraTask_performRebalance() { } public void performRebalance() { + Connection conn = null; try { - Connection conn = SnappyTest.getLocatorConnection(); + conn = SnappyTest.getLocatorConnection(); Long startTime = System.currentTimeMillis(); conn.createStatement().execute("call sys.rebalance_all_buckets();"); Long endTime = System.currentTimeMillis(); @@ -114,6 +115,9 @@ public void performRebalance() { Log.getLogWriter().info("The rebalance procedure took " + totalTime + " ms"); } catch (SQLException ex) { throw new util.TestException("Caught exception in performRebalance() " + ex.getMessage()); + } finally { + if(conn!=null) + closeConnection(conn); } } diff --git a/dtests/src/test/java/io/snappydata/hydra/misc/misc.bt b/dtests/src/test/java/io/snappydata/hydra/misc/misc.bt index 14841da411..03dc9caba0 100644 --- a/dtests/src/test/java/io/snappydata/hydra/misc/misc.bt +++ b/dtests/src/test/java/io/snappydata/hydra/misc/misc.bt @@ -39,14 +39,13 @@ io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithDDLs.conf D=worker workerHosts=2 workerVMsPerHost=1 workerThreadsPerVM=1 persistenceMode="sync" redundantCopies=1 - io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkAppCatalogConsistencyWithDDLs.conf A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=2 workerVMsPerHost=1 workerThreadsPerVM=1 - persistenceMode="sync" redundantCopies=1 - + kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" + opType=4 isRowTable=false isPartitioned=true withKeyColumn=true io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithRebalance.conf A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 @@ -55,13 +54,9 @@ io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithRebalance.conf D=worker workerHosts=2 workerVMsPerHost=1 workerThreadsPerVM=1 persistenceMode="sync" redundantCopies=1 - io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithAddNewNodeRebalance.conf A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=2 workerVMsPerHost=1 workerThreadsPerVM=1 persistenceMode="sync" redundantCopies=1 - logPath= - newNode= - snappyPath= diff --git a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithRebalance.conf b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithRebalance.conf index 538ce0a392..d7e19b0f5b 100644 --- a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithRebalance.conf +++ b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithRebalance.conf @@ -21,6 +21,8 @@ TASK taskClass = io.snappydata.hydra.testDMLOps.SnappyDMLOpsUtil taskM // trigger rebalance TASK taskClass = io.snappydata.hydra.cdcConnector.SnappyCDCTest taskMethod = HydraTask_performRebalance threadGroups = snappyThreads + startInterval = 90 + endInterval = 90 maxThreads = 1; From 60a85d1724ac250061a4413bbc593da852826adc Mon Sep 17 00:00:00 2001 From: sonal Date: Wed, 23 Oct 2019 12:34:25 +0530 Subject: [PATCH 3/9] - Adding a flag to enable or disable the DEBUG logging for spark application. --- .../adAnalytics/SnappyAdAnalyticsTest.java | 4 +-- .../snappydata/hydra/cluster/SnappyPrms.java | 10 ++++++ .../snappydata/hydra/cluster/SnappyTest.java | 36 ++++++++++++++++--- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java b/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java index 3767b5a23d..f6af5908d0 100644 --- a/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java @@ -417,7 +417,7 @@ public void verifyResults(){ String aggType = SnappySchemaPrms.getAggregateType(); switch (aggType.toUpperCase()) { case "JOIN": - query = "select tp.*,pd.language from temp_person tp, person_details pd where tp.id=pd.id"; + query = "select tp.*,pd.language from temp_persoon tp, person_details pd where tp.id=pd.id"; break; case "AVG": query = "select id, avg(age) as avg_age, avg(numChild) as avg_numchild from temp_persoon group by id"; @@ -447,7 +447,7 @@ public void verifyResults(){ String errMsg = testInstance.compareFiles(queryResultDirPath, streamTableFile, tmpTableFile, false, "streaming"); if(errMsg.length()> 0 ){ - throw new TestException("Got exception while validating results"); + throw new TestException("Got exception while validating results : " + errMsg); } } catch (SQLException se) { Log.getLogWriter().info("Got exception while verifying results"); diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyPrms.java b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyPrms.java index 0b71ed5ace..033a6a19cf 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyPrms.java +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyPrms.java @@ -692,6 +692,11 @@ public class SnappyPrms extends BasePrms { */ public static Long numRowsList; + /** + *(boolean) Property to enable debugging at spark level. + */ + public static Long enableSparkDebug; + /** * (int)Number of stack dumps to be taken for each thread of locators, servers, leads */ @@ -999,6 +1004,11 @@ public static String getLogFileName() { return tasktab().stringAt(key, null); } + public static boolean isSparkDebugEnabled(){ + Long key = enableSparkDebug; + return tasktab().booleanAt(key, false); + } + public static Vector getSQLScriptNames() { Long key = sqlScriptNames; return BasePrms.tasktab().vecAt(key, BasePrms.tab().vecAt(key, null)); diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java index 871a7cd2e5..ef7f73b773 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java @@ -2294,6 +2294,29 @@ public String setCDCSparkAppCmds(String userAppArgs, String commonArgs, String s return command; } + public String getLog4jFile(String dest) { + String log4jFileName = ""; + try { + log4jFileName = getCurrentDirPath() + File.separator + "log4j_" + getMyTid() + ".properties"; + Log.getLogWriter().info("log4j path :" + log4jFileName); + File log4jFile = new File(log4jFileName); + FileWriter fw = new FileWriter(log4jFile); + fw.write("log4j.rootCategory=DEBUG, file\n"); + fw.write("log4j.appender.file=org.apache.log4j.RollingFileAppender\n"); + fw.write("log4j.appender.file.append=true\n"); + fw.write("log4j.appender.file.file=" + dest + "\n"); + fw.write("log4j.appender.file.MaxFileSize=1GB\n"); + fw.write("log4j.appender.file.MaxBackupIndex=10000\n"); + fw.write("log4j.appender.file.layout=io.snappydata.log4j.PatternLayout\n"); + fw.write("log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS zzz} %t %p %c{1}: %m%n\n"); + fw.close(); + } catch (IOException e) { + throw new TestException("IOException occurred while retriving log4j file at path " + + log4jFileName + "\nError Message:" + e.getMessage()); + } + return log4jFileName; + } + public void executeSparkJob(Vector jobClassNames, String logFileName) { String snappyJobScript = getScriptLocation("spark-submit"); boolean isCDCStream = SnappyCDCPrms.getIsCDCStream(); @@ -2301,6 +2324,7 @@ public void executeSparkJob(Vector jobClassNames, String logFileName) { File log = null, logFile = null; userAppJar = SnappyPrms.getUserAppJar(); snappyTest.verifyDataForJobExecution(jobClassNames, userAppJar); + boolean isSparkDebug = SnappyPrms.isSparkDebugEnabled(); try { for (int i = 0; i < jobClassNames.size(); i++) { String userJob = (String) jobClassNames.elementAt(i); @@ -2322,13 +2346,16 @@ public void executeSparkJob(Vector jobClassNames, String logFileName) { throw new TestException("dml props for thread " + getMyTid() + " is null)"); userAppArgs = userAppArgs + " " + dmlProps; } + String log4jFile = (isSparkDebug)?getLog4jFile(dest):""; if (SnappyCDCPrms.getIsCDC()) { command = setCDCSparkAppCmds(userAppArgs, commonArgs, snappyJobScript, userJob, masterHost, masterPort, logFile); } else { command = snappyJobScript + " --class " + userJob + " --master spark://" + masterHost + ":" + masterPort + " " + SnappyPrms.getExecutorMemory() + " " + - SnappyPrms.getSparkSubmitExtraPrms() + " " + commonArgs + " " + snappyTest.getUserAppJarLocation(userAppJar, jarPath) + " " + + SnappyPrms.getSparkSubmitExtraPrms() + " " + commonArgs + " " + + ((isSparkDebug)?(" --driver-java-options -Dlog4j.configuration=file://" + log4jFile):"") + " " + + snappyTest.getUserAppJarLocation(userAppJar, jarPath) + " " + userAppArgs + " " + primaryLocatorHost + ":" + primaryLocatorPort; } Log.getLogWriter().info("spark-submit command is : " + command); @@ -2511,11 +2538,12 @@ public boolean getSnappyJobsStatus(String snappyJobScript, File logFile, String FileInputStream fis = new FileInputStream(commandOutput); BufferedReader br = new BufferedReader(new InputStreamReader(fis)); line = null; - String searchString = "Connection reset by peer"; + String connResetErr = "Connection reset by peer"; + String askTimedOutErr = "Ask timed out on "; while ((line = br.readLine()) != null && !found) { - if (line.toLowerCase().contains(searchString.toLowerCase())) { + if (line.toLowerCase().contains(connResetErr.toLowerCase()) || line.toLowerCase().contains(askTimedOutErr.toLowerCase())) { found = true; - Log.getLogWriter().info("Connection reset by peer..."); + Log.getLogWriter().info("Job submission failed with : " + line); } } br.close(); From 09c490cedbbcc846e657f7cea8362db44ffdd089 Mon Sep 17 00:00:00 2001 From: Sonal Date: Sat, 16 Nov 2019 17:30:35 +0530 Subject: [PATCH 4/9] - Changes to have lead config restored before stopping the cluster. --- .../hydra/adAnalytics/SnappyAdAnalyticsTest.java | 13 ++++++------- .../io/snappydata/hydra/cluster/sampleLeadHA.conf | 3 --- .../java/io/snappydata/hydra/consistency/leadHA.inc | 2 +- .../io/snappydata/hydra/consistency/serverHA.inc | 2 +- .../java/io/snappydata/hydra/northwind/leadHA.inc | 2 +- .../io/snappydata/hydra/northwind/locatorHA.inc | 2 +- .../java/io/snappydata/hydra/northwind/serverHA.inc | 2 +- .../hydra/streaming_sink/streamingWithLeadHA.inc | 2 +- .../hydra/streaming_sink/streamingWithLocatorHA.inc | 2 +- .../hydra/streaming_sink/streamingWithServerHA.inc | 2 +- .../streaming_sink/streaming_sinkForConflation.conf | 3 +-- .../streaming_sink/streaming_sinkForDelete.conf | 3 +-- .../streaming_sinkForGenericColNames.conf | 3 +-- .../streaming_sink/streaming_sinkForUpdate.conf | 3 +-- .../streaming_sinkForUpdateAndDelete.conf | 3 +-- .../streaming_sinkWithMultipleWriters.conf | 3 +-- .../streaming_sink/streaming_sinkWithRandomOps.conf | 3 +-- 17 files changed, 21 insertions(+), 32 deletions(-) diff --git a/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java b/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java index 7a53c12e63..9fa1d6aa59 100644 --- a/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java @@ -146,10 +146,6 @@ protected void writeSnappyPocToSparkEnv() { /** * Start kafka zookeeper. */ - public static synchronized void HydraTask_StartKafkaZookeeper() { - snappyAdAnalyticsTest.startZookeeper(); - } - protected void startZookeeper() { ProcessBuilder pb = null; try { @@ -222,7 +218,7 @@ public static synchronized void HydraTask_StartKafkaBrokers() { } protected void startKafkaBroker() { - String command = ""; + int numServers = (int)SnappyBB.getBB().getSharedCounters().read(SnappyBB.numServers); Log.getLogWriter().info("Test will start " + numServers + " kafka brokers."); String script = snappyTest.getScriptLocation(kafkaDir + sep + "bin/kafka-server-start.sh"); @@ -230,6 +226,7 @@ protected void startKafkaBroker() { File orgPropFile = new File(orgPropFilePath); try { for (int i = 1; i <= numServers; i++) { + String command = ""; String broker = "broker" + i; String brokerLogDirPath = kafkaLogDir + sep + broker; new File(brokerLogDirPath).mkdir(); @@ -337,6 +334,7 @@ protected void executeSnappyStreamingJob(Vector jobClassNames, String logFileNam ProcessBuilder pb = null; File log = null; File logFile = null; + String userJarPath = getUserAppJarLocation(SnappyPrms.getUserAppJar(),jarPath); verifyDataForJobExecution(jobClassNames, userJarPath); leadHost = getLeadHost(); @@ -373,12 +371,12 @@ protected void executeSnappyStreamingJob(Vector jobClassNames, String logFileNam throw new TestException("Failed to start the streaming job. Please check the logs."); } else { Log.getLogWriter().info("JobID is : " + jobID); - SnappyBB.getBB().getSharedMap().put(appName, jobID); for (int j = 0; j < 3; j++) { if (!getJobStatus(jobID)) { throw new TestException("Got Exception while executing streaming job. Please check " + "the job status output."); } + SnappyBB.getBB().getSharedMap().put(appName, jobID); } } } @@ -417,7 +415,7 @@ public void verifyResults(){ String aggType = SnappySchemaPrms.getAggregateType(); switch (aggType.toUpperCase()) { case "JOIN": - query = "select tp.*,pd.language from temp_persoon tp, person_details pd where tp.id=pd.id"; + query = "select tp.*,pd.language from temp_persoon tp, persoon_details pd where tp.id=pd.id"; break; case "AVG": query = "select id, avg(age) as avg_age, avg(numChild) as avg_numchild from temp_persoon group by id"; @@ -784,6 +782,7 @@ protected void stopSnappyStreamingJob() { File logFile = null; leadHost = getLeadHost(); String appName = SnappyPrms.getUserAppName(); + Log.getLogWriter.info("User App Name is : " + appName); String leadPort = (String)SnappyBB.getBB().getSharedMap().get("primaryLeadPort"); String jobID = (String) SnappyBB.getBB().getSharedMap().get(appName); String snappyCmd = snappyJobScript + " stop --job-id " + jobID + " --lead " + leadHost + ":" + leadPort; diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/sampleLeadHA.conf b/dtests/src/test/java/io/snappydata/hydra/cluster/sampleLeadHA.conf index d59fd791dc..b45c51c450 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/sampleLeadHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/sampleLeadHA.conf @@ -30,9 +30,6 @@ CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = H io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar threadGroups = snappyStoreThreads, snappyThreads; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_stopSnappy - threadGroups = snappyThreads; - CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData threadGroups = snappyThreads; diff --git a/dtests/src/test/java/io/snappydata/hydra/consistency/leadHA.inc b/dtests/src/test/java/io/snappydata/hydra/consistency/leadHA.inc index 69615df040..9db364a383 100644 --- a/dtests/src/test/java/io/snappydata/hydra/consistency/leadHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/consistency/leadHA.inc @@ -11,7 +11,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = H maxTimesToRun = 3 ; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData threadGroups = snappyThreads; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-isHATest = true; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/consistency/serverHA.inc b/dtests/src/test/java/io/snappydata/hydra/consistency/serverHA.inc index 2487c57e7e..75efbba90e 100644 --- a/dtests/src/test/java/io/snappydata/hydra/consistency/serverHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/consistency/serverHA.inc @@ -11,7 +11,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy maxTimesToRun = 3 ; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData threadGroups = snappyThreads; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-isHATest = true; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/leadHA.inc b/dtests/src/test/java/io/snappydata/hydra/northwind/leadHA.inc index cb403e6226..43f2c9fcc8 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/leadHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/leadHA.inc @@ -16,5 +16,5 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy maxTimesToRun = 3 ; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData threadGroups = snappyThreads; diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/locatorHA.inc b/dtests/src/test/java/io/snappydata/hydra/northwind/locatorHA.inc index aefb466d7a..dd4b8a783b 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/locatorHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/locatorHA.inc @@ -15,7 +15,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy maxTimesToRun = 3 ; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLocatorConfigData +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLocatorConfigData threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/serverHA.inc b/dtests/src/test/java/io/snappydata/hydra/northwind/serverHA.inc index 3bc01c74ea..f3c287efb1 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/serverHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/serverHA.inc @@ -17,5 +17,5 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy maxTimesToRun = 3 ; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData threadGroups = snappyThreads; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc index a6c1303992..1e93a40f6b 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc @@ -17,6 +17,6 @@ TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest t threadGroups = snappyThreads maxTimesToRun = 1; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData threadGroups = snappyThreads; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc index 74e94d75b0..9623ca00db 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc @@ -13,5 +13,5 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy io.snappydata.hydra.cluster.SnappyPrms-sleepTimeSecsBeforeRestart = 60 maxTimesToRun = 1; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLocatorConfigData +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLocatorConfigData threadGroups = snappyThreads; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc index 805c3debf2..f530c1dc55 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc @@ -13,5 +13,5 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy io.snappydata.hydra.cluster.SnappyPrms-sleepTimeSecsBeforeRestart = 60 maxTimesToRun = 1; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData threadGroups = snappyThreads; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForConflation.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForConflation.conf index 7ac7077bdb..527eba242e 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForConflation.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForConflation.conf @@ -33,8 +33,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTa maxThreads = 1; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_stopStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.adAnalytics.StopSnappyStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_verifyResults diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForDelete.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForDelete.conf index 51f7b5699e..b52a14f967 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForDelete.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForDelete.conf @@ -32,8 +32,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTa maxThreads = 1; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_stopStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.adAnalytics.StopSnappyStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_verifyResults diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForGenericColNames.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForGenericColNames.conf index 8ba4195af4..918d7a4eaa 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForGenericColNames.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForGenericColNames.conf @@ -32,8 +32,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTa maxThreads = 1; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_stopStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.adAnalytics.StopSnappyStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_verifyResults diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdate.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdate.conf index e592de71dd..92d2dfa498 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdate.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdate.conf @@ -32,8 +32,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTa maxThreads = 1; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_stopStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.adAnalytics.StopSnappyStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_verifyResults diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf index 9df1a81ede..cdf9503f08 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf @@ -32,8 +32,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTa maxThreads = 1; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_stopStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.adAnalytics.StopSnappyStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_verifyResults diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithMultipleWriters.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithMultipleWriters.conf index 3955f5d68f..d3dcdac3f3 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithMultipleWriters.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithMultipleWriters.conf @@ -52,8 +52,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTa maxThreads = 1; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_stopStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.adAnalytics.StopSnappyStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_verifyResults diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRandomOps.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRandomOps.conf index 5b8871e89e..21c299d32f 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRandomOps.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRandomOps.conf @@ -25,8 +25,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTa threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_stopStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.adAnalytics.StopSnappyStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.testDMLOps.SnappyDMLOpsUtil taskMethod = HydraTask_verifyResults From 4242e059166c50d2ee6e1a0c91334f8abf27e8ed Mon Sep 17 00:00:00 2001 From: Sonal Date: Tue, 19 Nov 2019 15:07:54 +0530 Subject: [PATCH 5/9] - fixing few issues in test --- .../adAnalytics/SnappyAdAnalyticsTest.java | 9 ++++-- .../snappydata/hydra/consistency/leadHA.inc | 2 +- .../snappydata/hydra/consistency/serverHA.inc | 2 +- .../io/snappydata/hydra/northwind/leadHA.inc | 2 +- .../snappydata/hydra/northwind/locatorHA.inc | 2 +- .../snappydata/hydra/northwind/serverHA.inc | 2 +- .../snappydata/hydra/rowTableCPP/serverHA.inc | 4 +-- .../streaming_sink/streamingWithLeadHA.inc | 2 +- .../streaming_sink/streamingWithLocatorHA.inc | 2 +- .../streaming_sink/streamingWithServerHA.inc | 2 +- .../hydra/streaming_sink/streaming_sinkAgg.bt | 32 ++++++++++++++++--- 11 files changed, 44 insertions(+), 17 deletions(-) diff --git a/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java b/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java index 9fa1d6aa59..bd355d61a8 100644 --- a/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java @@ -146,6 +146,11 @@ protected void writeSnappyPocToSparkEnv() { /** * Start kafka zookeeper. */ + + public static synchronized void HydraTask_StartKafkaZookeeper() { + snappyAdAnalyticsTest.startZookeeper(); + } + protected void startZookeeper() { ProcessBuilder pb = null; try { @@ -415,7 +420,7 @@ public void verifyResults(){ String aggType = SnappySchemaPrms.getAggregateType(); switch (aggType.toUpperCase()) { case "JOIN": - query = "select tp.*,pd.language from temp_persoon tp, persoon_details pd where tp.id=pd.id"; + query = "select tp.*, pd.languages from temp_persoon tp, persoon_details pd where tp.id=pd.id"; break; case "AVG": query = "select id, avg(age) as avg_age, avg(numChild) as avg_numchild from temp_persoon group by id"; @@ -782,7 +787,7 @@ protected void stopSnappyStreamingJob() { File logFile = null; leadHost = getLeadHost(); String appName = SnappyPrms.getUserAppName(); - Log.getLogWriter.info("User App Name is : " + appName); + Log.getLogWriter().info("User App Name is : " + appName); String leadPort = (String)SnappyBB.getBB().getSharedMap().get("primaryLeadPort"); String jobID = (String) SnappyBB.getBB().getSharedMap().get(appName); String snappyCmd = snappyJobScript + " stop --job-id " + jobID + " --lead " + leadHost + ":" + leadPort; diff --git a/dtests/src/test/java/io/snappydata/hydra/consistency/leadHA.inc b/dtests/src/test/java/io/snappydata/hydra/consistency/leadHA.inc index 9db364a383..4e570e20d3 100644 --- a/dtests/src/test/java/io/snappydata/hydra/consistency/leadHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/consistency/leadHA.inc @@ -12,6 +12,6 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = H ; ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData - threadGroups = snappyThreads; + clientNames = locator1; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-isHATest = true; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/consistency/serverHA.inc b/dtests/src/test/java/io/snappydata/hydra/consistency/serverHA.inc index 75efbba90e..cbd6b4fe50 100644 --- a/dtests/src/test/java/io/snappydata/hydra/consistency/serverHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/consistency/serverHA.inc @@ -12,6 +12,6 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy ; ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData - threadGroups = snappyThreads; + clientNames = locator1; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-isHATest = true; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/leadHA.inc b/dtests/src/test/java/io/snappydata/hydra/northwind/leadHA.inc index 43f2c9fcc8..40afaf32d3 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/leadHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/leadHA.inc @@ -17,4 +17,4 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy ; ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData - threadGroups = snappyThreads; + clientNames = locator1; diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/locatorHA.inc b/dtests/src/test/java/io/snappydata/hydra/northwind/locatorHA.inc index dd4b8a783b..0c9c405678 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/locatorHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/locatorHA.inc @@ -16,7 +16,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy ; ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLocatorConfigData - threadGroups = snappyThreads; + clientNames = locator1; CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.northwind.ValidateNWQueriesJob diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/serverHA.inc b/dtests/src/test/java/io/snappydata/hydra/northwind/serverHA.inc index f3c287efb1..eaddc08a5f 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/serverHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/serverHA.inc @@ -18,4 +18,4 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy ; ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData - threadGroups = snappyThreads; + clientNames = locator1; diff --git a/dtests/src/test/java/io/snappydata/hydra/rowTableCPP/serverHA.inc b/dtests/src/test/java/io/snappydata/hydra/rowTableCPP/serverHA.inc index af7762bcad..152546b089 100644 --- a/dtests/src/test/java/io/snappydata/hydra/rowTableCPP/serverHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/rowTableCPP/serverHA.inc @@ -15,5 +15,5 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy maxTimesToRun = 5 ; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData - threadGroups = snappyThreads; +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData + clientNames = locator1; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc index 1e93a40f6b..78162e236a 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc @@ -18,5 +18,5 @@ TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest t maxTimesToRun = 1; ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData - threadGroups = snappyThreads; + clientNames = locator1; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc index 9623ca00db..77aba3e2b4 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc @@ -14,4 +14,4 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy maxTimesToRun = 1; ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLocatorConfigData - threadGroups = snappyThreads; \ No newline at end of file + clientNames = locator1; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc index f530c1dc55..7d777aabde 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc @@ -14,4 +14,4 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy maxTimesToRun = 1; ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData - threadGroups = snappyThreads; + clientNames = locator1; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkAgg.bt b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkAgg.bt index f1961b4b57..bbd37013d5 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkAgg.bt +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkAgg.bt @@ -3,21 +3,21 @@ io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkWithSum.conf B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=4 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update + opType=1 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkWithAvg.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=4 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update + opType=1 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkWithJoin.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=4 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update + opType=1 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithCount.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 @@ -25,7 +25,7 @@ io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorMode C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=2 workerVMsPerHost=1 workerThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=4 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update + opType=1 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithJoin.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 @@ -33,4 +33,26 @@ io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorMode C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=2 workerVMsPerHost=1 workerThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=4 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update \ No newline at end of file + opType=1 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update + +io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkWithSum.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" + opType=1 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=complete + +io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkWithAvg.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" + opType=1 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=complete + +io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithJoin.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=2 workerVMsPerHost=1 workerThreadsPerVM=1 + kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" + opType=1 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=complete \ No newline at end of file From 64826e38284575e43e95e0abff49858d31245d62 Mon Sep 17 00:00:00 2001 From: Sonal Date: Wed, 27 Nov 2019 13:55:50 +0530 Subject: [PATCH 6/9] - Test fixes --- .../streaming_sinkSmartConnectorModeWithServerHA.conf | 4 +++- .../streaming_sinkSmartConnectorModewithLeadHA.conf | 6 +++--- .../snappydata/hydra/streaming_sink/streamingWithLeadHA.inc | 3 --- .../hydra/streaming_sink/streamingWithLocatorHA.inc | 3 --- .../hydra/streaming_sink/streamingWithServerHA.inc | 3 --- .../hydra/streaming_sink/streaming_sinkWithLeadHA.conf | 5 +++-- .../hydra/streaming_sink/streaming_sinkWithLocatorHA.conf | 5 +++-- .../hydra/streaming_sink/streaming_sinkWithServerHA.conf | 4 +++- 8 files changed, 15 insertions(+), 18 deletions(-) diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModeWithServerHA.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModeWithServerHA.conf index a7e3807482..87a5cce309 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModeWithServerHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModeWithServerHA.conf @@ -1,3 +1,5 @@ +INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc; + INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModeForUpdateAndDelete.conf; -INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc; +INCLUDE $JTESTS/io/snappydata/hydra/northwind/backUpServerConfig.inc; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModewithLeadHA.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModewithLeadHA.conf index ec1ffa77c6..4f4d2432df 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModewithLeadHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModewithLeadHA.conf @@ -1,3 +1,6 @@ +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData + threadGroups = snappyThreads; + INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModeForUpdateAndDelete.conf; INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = backUpLeadConfigData @@ -14,9 +17,6 @@ TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest t threadGroups = snappyThreads maxTimesToRun = 1; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData - threadGroups = snappyThreads; - CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_executeSQLScriptsWithSleep io.snappydata.hydra.cluster.SnappyPrms-logFileName = sqlScriptResults_restart.log io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = streaming_sink_queries.sql diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc index 78162e236a..6b4e30a10f 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc @@ -4,9 +4,6 @@ io.snappydata.hydra.cluster.SnappyPrms-cycleVms = true; util.StopStartPrms-stopModes = NICE_KILL; util.StopStartPrms-numVMsToStop = 1; -INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = backUpLeadConfigData - threadGroups = snappyThreads; - TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_restartLeadVMWithStreaming io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.streaming_sink.SnappyStreamingSinkJob io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "kafkaTopic=streamingSinkTopic,tableName=persoon,streaming.batch_interval=100,isConflation=false" diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc index 77aba3e2b4..242a2600c4 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc @@ -4,9 +4,6 @@ io.snappydata.hydra.cluster.SnappyPrms-cycleVms = true; util.StopStartPrms-stopModes = NICE_KILL; util.StopStartPrms-numVMsToStop = 1; -INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = backUpLocatorConfigData - threadGroups = snappyThreads; - TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_cycleLocatorVms startInterval = 60 threadGroups = snappyThreads diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc index 7d777aabde..c2cfeec965 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc @@ -4,9 +4,6 @@ io.snappydata.hydra.cluster.SnappyPrms-cycleVms = true; util.StopStartPrms-stopModes = NICE_KILL; util.StopStartPrms-numVMsToStop = 1; -INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = backUpServerConfigData - threadGroups = snappyThreads; - TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_cycleStoreVms startInterval = 60 threadGroups = snappyThreads diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithLeadHA.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithLeadHA.conf index 3eb25ed6e2..433ccc68f0 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithLeadHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithLeadHA.conf @@ -1,4 +1,5 @@ -INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf; - INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc; +INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf; + +INCLUDE $JTESTS/io/snappydata/hydra/northwind/backUpLeadConfig.inc; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithLocatorHA.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithLocatorHA.conf index 8f42d6f612..e211ef8424 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithLocatorHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithLocatorHA.conf @@ -1,4 +1,5 @@ -INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf; - INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc; +INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf; + +INCLUDE $JTESTS/io/snappydata/hydra/northwind/backUpLocatorConfig.inc; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithServerHA.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithServerHA.conf index 50522eed99..b0af7c4bda 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithServerHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithServerHA.conf @@ -1,3 +1,5 @@ +INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc; + INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf; -INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc; +INCLUDE $JTESTS/io/snappydata/hydra/northwind/backUpServerConfig.inc; \ No newline at end of file From 6682e3dde38048dd341e3cbfd6a410083173f1c5 Mon Sep 17 00:00:00 2001 From: Sonal Date: Thu, 5 Dec 2019 10:29:09 +0530 Subject: [PATCH 7/9] -test fixes --- .../io/snappydata/hydra/cluster/SnappyTest.java | 3 +++ ...reaming_sinkSmartConnectorModewithLeadHA.conf | 2 +- .../streamingAgg/startClusterDualModeAgg.inc | 2 +- .../streaming_sink/streamingAgg/streamingAgg.inc | 2 +- ...treaming_sinkSmartConnectorModeWithCount.conf | 2 +- ...streaming_sinkSmartConnectorModeWithJoin.conf | 2 +- .../hydra/streaming_sink/streaming_sinkAgg.bt | 16 ++++++++-------- .../streaming_sinkWithMultipleWriters.conf | 2 +- .../streaming_sink/CreateTableForJoinJob.scala | 2 +- 9 files changed, 18 insertions(+), 15 deletions(-) diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java index 08d7da81b0..72132f6586 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java @@ -891,6 +891,7 @@ public static synchronized void backUpServerConfigData() { * the test for stopping all locators which have been started in the test. **/ public static synchronized void restoreLocatorConfigData() { + if(snappyTest == null) snappyTest = new SnappyTest(); snappyTest.restoreConfigData("locators"); } @@ -904,6 +905,7 @@ public static synchronized void restoreLocatorConfigData() { * which have been started in the test. **/ public static synchronized void restoreServerConfigData() { + if(snappyTest == null) snappyTest = new SnappyTest(); snappyTest.restoreConfigData("servers"); } @@ -917,6 +919,7 @@ public static synchronized void restoreServerConfigData() { * which have been started in the test. **/ public static synchronized void restoreLeadConfigData() { + if(snappyTest == null) snappyTest = new SnappyTest(); snappyTest.restoreConfigData("leads"); } diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModewithLeadHA.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModewithLeadHA.conf index 4f4d2432df..dee55e16f1 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModewithLeadHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModewithLeadHA.conf @@ -1,5 +1,5 @@ ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData - threadGroups = snappyThreads; + clientNames = locator1; INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModeForUpdateAndDelete.conf; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/startClusterDualModeAgg.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/startClusterDualModeAgg.inc index e3c07c3c3d..faaa33482e 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/startClusterDualModeAgg.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/startClusterDualModeAgg.inc @@ -112,7 +112,7 @@ runMode = always threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_generateAndPublishMethod -io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "500000,streamingSinkTopic,0,0" +io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "400000,streamingSinkTopic,0,0" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar threadGroups = snappyThreads; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streamingAgg.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streamingAgg.inc index 3daef32fbf..a9f6520395 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streamingAgg.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streamingAgg.inc @@ -1,6 +1,6 @@ TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_generateAndPublishMethod //io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = "io.snappydata.hydra.streaming_sink.StringMessageProducer" -io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "400000,streamingSinkTopic,200000,4" +io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "400000,streamingSinkTopic,200000,0" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar threadGroups = leadThread maxTimesToRun = 1; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithCount.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithCount.conf index 622034df81..2225489c3e 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithCount.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithCount.conf @@ -8,7 +8,7 @@ INITTASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest INITTASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_executeSnappyStreamingApp io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.streaming_sink.SnappyStreamingSinkAppWithAggregate - io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "streamingSinkTopic persoon false ${outputMode} count" + io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "streamingSinkTopic persoon false ${outputMode} count" io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingSinkApp io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar threadGroups = snappyThreads; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithJoin.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithJoin.conf index ae455a2f72..cd1bad68ee 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithJoin.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithJoin.conf @@ -13,7 +13,7 @@ INITTASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest INITTASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_executeSnappyStreamingApp io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.streaming_sink.SnappyStreamingSinkAppWithAggregate - io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "streamingSinkTopic persoon false ${outputMode} join" + io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "streamingSinkTopic persoon false ${outputMode} join" io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingSinkApp io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar threadGroups = snappyThreads; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkAgg.bt b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkAgg.bt index bbd37013d5..81ed6f11c7 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkAgg.bt +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkAgg.bt @@ -3,21 +3,21 @@ io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkWithSum.conf B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=1 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update + opType=0 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkWithAvg.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=1 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update + opType=0 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkWithJoin.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=1 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update + opType=0 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithCount.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 @@ -25,7 +25,7 @@ io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorMode C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=2 workerVMsPerHost=1 workerThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=1 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update + opType=0 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithJoin.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 @@ -33,21 +33,21 @@ io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorMode C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=2 workerVMsPerHost=1 workerThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=1 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update + opType=0 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkWithSum.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=1 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=complete + opType=0 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=complete io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkWithAvg.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=1 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=complete + opType=0 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=complete io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithJoin.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 @@ -55,4 +55,4 @@ io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorMode C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=2 workerVMsPerHost=1 workerThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=1 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=complete \ No newline at end of file + opType=0 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=complete \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithMultipleWriters.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithMultipleWriters.conf index d3dcdac3f3..a93167161b 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithMultipleWriters.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithMultipleWriters.conf @@ -68,7 +68,7 @@ INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/stopClusters.inc; /* end task must stop snappy members because they are not stopped by Hydra */ hydra.Prms-alwaysDoEndTasks = true; hydra.Prms-totalTaskTimeSec = 900; -hydra.Prms-maxResultWaitSec = 1200; +hydra.Prms-maxResultWaitSec = 1500; hydra.VmPrms-extraVMArgs += fcn "hydra.TestConfigFcns.duplicate (\"-Xms1512m -Xmx2g \", ${${A}Hosts}, true)" diff --git a/dtests/src/test/scala/io/snappydata/hydra/streaming_sink/CreateTableForJoinJob.scala b/dtests/src/test/scala/io/snappydata/hydra/streaming_sink/CreateTableForJoinJob.scala index 0fd003f64f..9f22b91d2e 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/streaming_sink/CreateTableForJoinJob.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/streaming_sink/CreateTableForJoinJob.scala @@ -74,7 +74,7 @@ class CreateTableForJoinJob extends SnappySQLJob{ s"hasChildren boolean," + s"numChild int," + s"hasSiblings boolean, " + - s"language varchar(10)" + + s"languages varchar(10)" + s" $primaryKey" + s") using $provider $options" pw.println(s"Creating table $s") From f6369bce263b603a17cc792993ec6da40c940510 Mon Sep 17 00:00:00 2001 From: Sonal Date: Fri, 6 Dec 2019 17:13:29 +0530 Subject: [PATCH 8/9] - Handle "Job ID not found" error in test and sleep for sometime before proceeding test in case of this error. --- .../io/snappydata/hydra/cluster/SnappyTest.java | 4 ++++ .../StructuredStreamingTestUtil.scala | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java index 72132f6586..c564bdbf17 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java @@ -2543,10 +2543,14 @@ public boolean getSnappyJobsStatus(String snappyJobScript, File logFile, String line = null; String connResetErr = "Connection reset by peer"; String askTimedOutErr = "Ask timed out on "; + String noSuchJobID ="No such job ID"; while ((line = br.readLine()) != null && !found) { if (line.toLowerCase().contains(connResetErr.toLowerCase()) || line.toLowerCase().contains(askTimedOutErr.toLowerCase())) { found = true; Log.getLogWriter().info("Job submission failed with : " + line); + } else if(line.toLowerCase().contains(noSuchJobID.toLowerCase())){ + Log.getLogWriter().info("Job ID " + str + " not found. Sleeping before proceeding test ... "); + sleepForMs(30); } } br.close(); diff --git a/dtests/src/test/scala/io/snappydata/hydra/streaming_sink/StructuredStreamingTestUtil.scala b/dtests/src/test/scala/io/snappydata/hydra/streaming_sink/StructuredStreamingTestUtil.scala index b6a74aa484..d333fbe981 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/streaming_sink/StructuredStreamingTestUtil.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/streaming_sink/StructuredStreamingTestUtil.scala @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package io.snappydata.hydra.streaming_sink From 76347cdfd1afa6b9163306ea8adc41b257227c28 Mon Sep 17 00:00:00 2001 From: Sonal Date: Mon, 20 Jan 2020 14:56:31 +0530 Subject: [PATCH 9/9] - Minor test fixes. --- .../streaming_sink/streamingAgg/startClusterDualModeAgg.inc | 2 ++ .../hydra/streaming_sink/streamingAgg/streamingAgg.inc | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/startClusterDualModeAgg.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/startClusterDualModeAgg.inc index faaa33482e..f9aefdcd32 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/startClusterDualModeAgg.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/startClusterDualModeAgg.inc @@ -121,3 +121,5 @@ TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMe io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = query_snappyTemp_table.sql threadGroups = snappyStoreThreads maxThreads = 1; + +io.snappydata.hydra.testDMLOps.SnappySchemaPrms-isAggregate=true; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streamingAgg.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streamingAgg.inc index a9f6520395..2937ac28c8 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streamingAgg.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streamingAgg.inc @@ -1,6 +1,6 @@ TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_generateAndPublishMethod //io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = "io.snappydata.hydra.streaming_sink.StringMessageProducer" -io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "400000,streamingSinkTopic,200000,0" +io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "400000,streamingSinkTopic,500000,0" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar threadGroups = leadThread maxTimesToRun = 1;