diff --git a/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java b/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java index 1f0bfff09d..bd355d61a8 100644 --- a/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java @@ -146,6 +146,7 @@ protected void writeSnappyPocToSparkEnv() { /** * Start kafka zookeeper. */ + public static synchronized void HydraTask_StartKafkaZookeeper() { snappyAdAnalyticsTest.startZookeeper(); } @@ -222,7 +223,7 @@ public static synchronized void HydraTask_StartKafkaBrokers() { } protected void startKafkaBroker() { - String command = ""; + int numServers = (int)SnappyBB.getBB().getSharedCounters().read(SnappyBB.numServers); Log.getLogWriter().info("Test will start " + numServers + " kafka brokers."); String script = snappyTest.getScriptLocation(kafkaDir + sep + "bin/kafka-server-start.sh"); @@ -230,6 +231,7 @@ protected void startKafkaBroker() { File orgPropFile = new File(orgPropFilePath); try { for (int i = 1; i <= numServers; i++) { + String command = ""; String broker = "broker" + i; String brokerLogDirPath = kafkaLogDir + sep + broker; new File(brokerLogDirPath).mkdir(); @@ -337,6 +339,7 @@ protected void executeSnappyStreamingJob(Vector jobClassNames, String logFileNam ProcessBuilder pb = null; File log = null; File logFile = null; + String userJarPath = getUserAppJarLocation(SnappyPrms.getUserAppJar(),jarPath); verifyDataForJobExecution(jobClassNames, userJarPath); leadHost = getLeadHost(); @@ -373,12 +376,12 @@ protected void executeSnappyStreamingJob(Vector jobClassNames, String logFileNam throw new TestException("Failed to start the streaming job. Please check the logs."); } else { Log.getLogWriter().info("JobID is : " + jobID); - SnappyBB.getBB().getSharedMap().put(appName, jobID); for (int j = 0; j < 3; j++) { if (!getJobStatus(jobID)) { throw new TestException("Got Exception while executing streaming job. Please check " + "the job status output."); } + SnappyBB.getBB().getSharedMap().put(appName, jobID); } } } @@ -417,7 +420,7 @@ public void verifyResults(){ String aggType = SnappySchemaPrms.getAggregateType(); switch (aggType.toUpperCase()) { case "JOIN": - query = "select tp.*,pd.language from temp_person tp, person_details pd where tp.id=pd.id"; + query = "select tp.*, pd.languages from temp_persoon tp, persoon_details pd where tp.id=pd.id"; break; case "AVG": query = "select id, avg(age) as avg_age, avg(numChild) as avg_numchild from temp_persoon group by id"; @@ -447,7 +450,7 @@ public void verifyResults(){ String errMsg = testInstance.compareFiles(queryResultDirPath, streamTableFile, tmpTableFile, false, "streaming"); if(errMsg.length()> 0 ){ - throw new TestException("Got exception while validating results"); + throw new TestException("Got exception while validating results : " + errMsg); } } catch (SQLException se) { Log.getLogWriter().info("Got exception while verifying results"); @@ -784,6 +787,7 @@ protected void stopSnappyStreamingJob() { File logFile = null; leadHost = getLeadHost(); String appName = SnappyPrms.getUserAppName(); + Log.getLogWriter().info("User App Name is : " + appName); String leadPort = (String)SnappyBB.getBB().getSharedMap().get("primaryLeadPort"); String jobID = (String) SnappyBB.getBB().getSharedMap().get(appName); String snappyCmd = snappyJobScript + " stop --job-id " + jobID + " --lead " + leadHost + ":" + leadPort; diff --git a/dtests/src/test/java/io/snappydata/hydra/cdcConnector/SnappyCDCTest.java b/dtests/src/test/java/io/snappydata/hydra/cdcConnector/SnappyCDCTest.java index cec434fc20..ad66a7f837 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cdcConnector/SnappyCDCTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/cdcConnector/SnappyCDCTest.java @@ -107,8 +107,9 @@ public static void HydraTask_performRebalance() { } public void performRebalance() { + Connection conn = null; try { - Connection conn = SnappyTest.getLocatorConnection(); + conn = SnappyTest.getLocatorConnection(); Long startTime = System.currentTimeMillis(); conn.createStatement().execute("call sys.rebalance_all_buckets();"); Long endTime = System.currentTimeMillis(); @@ -116,6 +117,9 @@ public void performRebalance() { Log.getLogWriter().info("The rebalance procedure took " + totalTime + " ms"); } catch (SQLException ex) { throw new util.TestException("Caught exception in performRebalance() " + ex.getMessage()); + } finally { + if(conn!=null) + closeConnection(conn); } } diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyPrms.java b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyPrms.java index da593b2fc1..73a9c0091d 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyPrms.java +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyPrms.java @@ -692,6 +692,11 @@ public class SnappyPrms extends BasePrms { */ public static Long numRowsList; + /** + *(boolean) Property to enable debugging at spark level. + */ + public static Long enableSparkDebug; + /** * (int)Number of stack dumps to be taken for each thread of locators, servers, leads */ @@ -999,6 +1004,11 @@ public static String getLogFileName() { return tasktab().stringAt(key, null); } + public static boolean isSparkDebugEnabled(){ + Long key = enableSparkDebug; + return tasktab().booleanAt(key, false); + } + public static Vector getSQLScriptNames() { Long key = sqlScriptNames; return BasePrms.tasktab().vecAt(key, BasePrms.tab().vecAt(key, null)); diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java index 4c85e47602..492cff4522 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java @@ -891,6 +891,7 @@ public static synchronized void backUpServerConfigData() { * the test for stopping all locators which have been started in the test. **/ public static synchronized void restoreLocatorConfigData() { + if(snappyTest == null) snappyTest = new SnappyTest(); snappyTest.restoreConfigData("locators"); } @@ -904,6 +905,7 @@ public static synchronized void restoreLocatorConfigData() { * which have been started in the test. **/ public static synchronized void restoreServerConfigData() { + if(snappyTest == null) snappyTest = new SnappyTest(); snappyTest.restoreConfigData("servers"); } @@ -917,6 +919,7 @@ public static synchronized void restoreServerConfigData() { * which have been started in the test. **/ public static synchronized void restoreLeadConfigData() { + if(snappyTest == null) snappyTest = new SnappyTest(); snappyTest.restoreConfigData("leads"); } @@ -1850,20 +1853,7 @@ public void executeProcess(ProcessBuilder pb, File logFile) { } public synchronized void recordSnappyProcessIDinNukeRun(String pName) { - Process pr = null; - try { - File log = new File("."); - String dest = log.getCanonicalPath() + File.separator + "PIDs_" + pName + "_" + HostHelper - .getLocalHost() + - ".log"; - File logFile = new File(dest); - if (!logFile.exists()) { - recordProcessIds(pName, HostHelper.getLocalHost(), logFile); - } - } catch (IOException e) { - String s = "Problem while starting the process : " + pr; - throw new TestException(s, e); - } + recordSnappyProcessIDinNukeRun(pName, HostHelper.getLocalHost()); } protected synchronized void recordSnappyProcessIDinNukeRun(String pName, String hostName) { @@ -1874,19 +1864,11 @@ protected synchronized void recordSnappyProcessIDinNukeRun(String pName, String String dest = log.getCanonicalPath() + File.separator + "PIDs_" + pName + "_" + hostName + ".log"; File logFile = new File(dest); if (!logFile.exists()) { - String command; - command = "ssh -n -x -o PasswordAuthentication=no -o StrictHostKeyChecking=no " + hostName; - pb = new ProcessBuilder("/bin/bash", "-c", command); - pr = pb.start(); - pr.waitFor(); recordProcessIds(pName, hostName, logFile); } } catch (IOException e) { String s = "Problem while starting the process : " + pr; throw new TestException(s, e); - } catch (InterruptedException e) { - String s = "Exception occurred while waiting for the process execution : " + pr; - throw new TestException(s, e); } } @@ -1895,9 +1877,10 @@ protected static void recordProcessIds(String pName, String hostName, File logFi ProcessBuilder pb; Process pr = null; try { + command = "ssh -n -x -o PasswordAuthentication=no -o StrictHostKeyChecking=no " + hostName; if (pName.equals("Master")) command = "ps ax | grep -w " + pName + " | grep -v grep | awk '{print $1}'"; - else command = "jps | grep " + pName + " | awk '{print $1}'"; + else command += " jps | grep " + pName + " | awk '{print $1}'"; hd = TestConfig.getInstance().getMasterDescription() .getVmDescription().getHostDescription(); pb = new ProcessBuilder("/bin/bash", "-c", command); @@ -2314,6 +2297,29 @@ public String setCDCSparkAppCmds(String userAppArgs, String commonArgs, String s return command; } + public String getLog4jFile(String dest) { + String log4jFileName = ""; + try { + log4jFileName = getCurrentDirPath() + File.separator + "log4j_" + getMyTid() + ".properties"; + Log.getLogWriter().info("log4j path :" + log4jFileName); + File log4jFile = new File(log4jFileName); + FileWriter fw = new FileWriter(log4jFile); + fw.write("log4j.rootCategory=DEBUG, file\n"); + fw.write("log4j.appender.file=org.apache.log4j.RollingFileAppender\n"); + fw.write("log4j.appender.file.append=true\n"); + fw.write("log4j.appender.file.file=" + dest + "\n"); + fw.write("log4j.appender.file.MaxFileSize=1GB\n"); + fw.write("log4j.appender.file.MaxBackupIndex=10000\n"); + fw.write("log4j.appender.file.layout=io.snappydata.log4j.PatternLayout\n"); + fw.write("log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS zzz} %t %p %c{1}: %m%n\n"); + fw.close(); + } catch (IOException e) { + throw new TestException("IOException occurred while retriving log4j file at path " + + log4jFileName + "\nError Message:" + e.getMessage()); + } + return log4jFileName; + } + public void executeSparkJob(Vector jobClassNames, String logFileName) { String snappyJobScript = getScriptLocation("spark-submit"); boolean isCDCStream = SnappyCDCPrms.getIsCDCStream(); @@ -2321,6 +2327,7 @@ public void executeSparkJob(Vector jobClassNames, String logFileName) { File log = null, logFile = null; userAppJar = SnappyPrms.getUserAppJar(); snappyTest.verifyDataForJobExecution(jobClassNames, userAppJar); + boolean isSparkDebug = SnappyPrms.isSparkDebugEnabled(); try { for (int i = 0; i < jobClassNames.size(); i++) { String userJob = (String) jobClassNames.elementAt(i); @@ -2342,13 +2349,16 @@ public void executeSparkJob(Vector jobClassNames, String logFileName) { throw new TestException("dml props for thread " + getMyTid() + " is null)"); userAppArgs = userAppArgs + " " + dmlProps; } + String log4jFile = (isSparkDebug)?getLog4jFile(dest):""; if (SnappyCDCPrms.getIsCDC()) { command = setCDCSparkAppCmds(userAppArgs, commonArgs, snappyJobScript, userJob, masterHost, masterPort, logFile); } else { command = snappyJobScript + " --class " + userJob + " --master spark://" + masterHost + ":" + masterPort + " " + SnappyPrms.getExecutorMemory() + " " + - SnappyPrms.getSparkSubmitExtraPrms() + " " + commonArgs + " " + snappyTest.getUserAppJarLocation(userAppJar, jarPath) + " " + + SnappyPrms.getSparkSubmitExtraPrms() + " " + commonArgs + " " + + ((isSparkDebug)?(" --driver-java-options -Dlog4j.configuration=file://" + log4jFile):"") + " " + + snappyTest.getUserAppJarLocation(userAppJar, jarPath) + " " + userAppArgs + " " + primaryLocatorHost + ":" + primaryLocatorPort; } Log.getLogWriter().info("spark-submit command is : " + command); @@ -2531,11 +2541,16 @@ public boolean getSnappyJobsStatus(String snappyJobScript, File logFile, String FileInputStream fis = new FileInputStream(commandOutput); BufferedReader br = new BufferedReader(new InputStreamReader(fis)); line = null; - String searchString = "Connection reset by peer"; + String connResetErr = "Connection reset by peer"; + String askTimedOutErr = "Ask timed out on "; + String noSuchJobID ="No such job ID"; while ((line = br.readLine()) != null && !found) { - if (line.toLowerCase().contains(searchString.toLowerCase())) { + if (line.toLowerCase().contains(connResetErr.toLowerCase()) || line.toLowerCase().contains(askTimedOutErr.toLowerCase())) { found = true; - Log.getLogWriter().info("Connection reset by peer..."); + Log.getLogWriter().info("Job submission failed with : " + line); + } else if(line.toLowerCase().contains(noSuchJobID.toLowerCase())){ + Log.getLogWriter().info("Job ID " + str + " not found. Sleeping before proceeding test ... "); + sleepForMs(30); } } br.close(); @@ -2848,6 +2863,7 @@ public static synchronized void HydraTask_stopSparkCluster() { */ public static synchronized void HydraTask_stopSnappyLeader() { File log = null; + if(snappyTest==null) snappyTest = new SnappyTest(); try { ProcessBuilder pb = new ProcessBuilder(snappyTest.getScriptLocation("snappy-leads.sh"), "stop"); @@ -2870,6 +2886,7 @@ public static synchronized void HydraTask_stopSnappyLeader() { */ public static synchronized void HydraTask_stopSnappyServers() { File log = null; + if(snappyTest==null) snappyTest = new SnappyTest(); try { ProcessBuilder pb = new ProcessBuilder(snappyTest.getScriptLocation("snappy-servers.sh"), "stop"); log = new File("."); @@ -2891,6 +2908,7 @@ public static synchronized void HydraTask_stopSnappyServers() { */ public static synchronized void HydraTask_stopSnappyLocator() { File log = null; + if(snappyTest==null) snappyTest = new SnappyTest(); try { ProcessBuilder pb = new ProcessBuilder(snappyTest.getScriptLocation("snappy-locators.sh") , "stop"); @@ -3610,13 +3628,16 @@ protected static void getLeadVM(ClientVmInfo info, ArrayList vmList, ArrayList s if (!isPrimaryLeadUp) { for (int i = 0; i < 3; i++) { if (isPrimaryLeadUp) break; + sleepForMs(5); isPrimaryLeadUp = isPrimaryLeadUpAndRunning(); } } if (cycleVms && !isPrimaryLeadUp) { - do { + for (int i = 0; i < 3; i++) { + if (isPrimaryLeadUp) break; + sleepForMs(5); isPrimaryLeadUp = isPrimaryLeadUpAndRunning(); - } while (isPrimaryLeadUp); + } } if (!isPrimaryLeadUp) { throw new TestException("Primary lead node is not up and running in the cluster."); diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/sampleLeadHA.conf b/dtests/src/test/java/io/snappydata/hydra/cluster/sampleLeadHA.conf index d59fd791dc..b45c51c450 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/sampleLeadHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/sampleLeadHA.conf @@ -30,9 +30,6 @@ CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = H io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar threadGroups = snappyStoreThreads, snappyThreads; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_stopSnappy - threadGroups = snappyThreads; - CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData threadGroups = snappyThreads; diff --git a/dtests/src/test/java/io/snappydata/hydra/consistency/leadHA.inc b/dtests/src/test/java/io/snappydata/hydra/consistency/leadHA.inc index 69615df040..4e570e20d3 100644 --- a/dtests/src/test/java/io/snappydata/hydra/consistency/leadHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/consistency/leadHA.inc @@ -11,7 +11,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = H maxTimesToRun = 3 ; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData - threadGroups = snappyThreads; +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData + clientNames = locator1; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-isHATest = true; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/consistency/serverHA.inc b/dtests/src/test/java/io/snappydata/hydra/consistency/serverHA.inc index 2487c57e7e..cbd6b4fe50 100644 --- a/dtests/src/test/java/io/snappydata/hydra/consistency/serverHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/consistency/serverHA.inc @@ -11,7 +11,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy maxTimesToRun = 3 ; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData - threadGroups = snappyThreads; +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData + clientNames = locator1; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-isHATest = true; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/misc/misc.bt b/dtests/src/test/java/io/snappydata/hydra/misc/misc.bt index 14841da411..03dc9caba0 100644 --- a/dtests/src/test/java/io/snappydata/hydra/misc/misc.bt +++ b/dtests/src/test/java/io/snappydata/hydra/misc/misc.bt @@ -39,14 +39,13 @@ io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithDDLs.conf D=worker workerHosts=2 workerVMsPerHost=1 workerThreadsPerVM=1 persistenceMode="sync" redundantCopies=1 - io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkAppCatalogConsistencyWithDDLs.conf A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=2 workerVMsPerHost=1 workerThreadsPerVM=1 - persistenceMode="sync" redundantCopies=1 - + kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" + opType=4 isRowTable=false isPartitioned=true withKeyColumn=true io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithRebalance.conf A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 @@ -55,13 +54,9 @@ io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithRebalance.conf D=worker workerHosts=2 workerVMsPerHost=1 workerThreadsPerVM=1 persistenceMode="sync" redundantCopies=1 - io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithAddNewNodeRebalance.conf A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=2 workerVMsPerHost=1 workerThreadsPerVM=1 persistenceMode="sync" redundantCopies=1 - logPath= - newNode= - snappyPath= diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/leadHA.inc b/dtests/src/test/java/io/snappydata/hydra/northwind/leadHA.inc index cb403e6226..40afaf32d3 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/leadHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/leadHA.inc @@ -16,5 +16,5 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy maxTimesToRun = 3 ; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData - threadGroups = snappyThreads; +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData + clientNames = locator1; diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/locatorHA.inc b/dtests/src/test/java/io/snappydata/hydra/northwind/locatorHA.inc index aefb466d7a..0c9c405678 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/locatorHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/locatorHA.inc @@ -15,8 +15,8 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy maxTimesToRun = 3 ; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLocatorConfigData - threadGroups = snappyThreads; +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLocatorConfigData + clientNames = locator1; CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.northwind.ValidateNWQueriesJob diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/serverHA.inc b/dtests/src/test/java/io/snappydata/hydra/northwind/serverHA.inc index 3bc01c74ea..eaddc08a5f 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/serverHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/serverHA.inc @@ -17,5 +17,5 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy maxTimesToRun = 3 ; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData - threadGroups = snappyThreads; +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData + clientNames = locator1; diff --git a/dtests/src/test/java/io/snappydata/hydra/rowTableCPP/serverHA.inc b/dtests/src/test/java/io/snappydata/hydra/rowTableCPP/serverHA.inc index af7762bcad..152546b089 100644 --- a/dtests/src/test/java/io/snappydata/hydra/rowTableCPP/serverHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/rowTableCPP/serverHA.inc @@ -15,5 +15,5 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy maxTimesToRun = 5 ; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData - threadGroups = snappyThreads; +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData + clientNames = locator1; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/clusterRestart.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/clusterRestart.inc index 96c5983863..5a6c0a025a 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/clusterRestart.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/clusterRestart.inc @@ -3,7 +3,7 @@ TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMe io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "kafkaTopic=streamingSinkTopic,tableName=persoon,streaming.batch_interval=100,isConflation=false" io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar - threadGroups = leadThread + threadGroups = snappyThreads startInterval = 60 maxTimesToRun = 1; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModeWithServerHA.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModeWithServerHA.conf index a7e3807482..87a5cce309 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModeWithServerHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModeWithServerHA.conf @@ -1,3 +1,5 @@ +INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc; + INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModeForUpdateAndDelete.conf; -INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc; +INCLUDE $JTESTS/io/snappydata/hydra/northwind/backUpServerConfig.inc; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModewithLeadHA.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModewithLeadHA.conf index ec1ffa77c6..dee55e16f1 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModewithLeadHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModewithLeadHA.conf @@ -1,3 +1,6 @@ +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData + clientNames = locator1; + INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/smartConnector/streaming_sinkSmartConnectorModeForUpdateAndDelete.conf; INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = backUpLeadConfigData @@ -14,9 +17,6 @@ TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest t threadGroups = snappyThreads maxTimesToRun = 1; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData - threadGroups = snappyThreads; - CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_executeSQLScriptsWithSleep io.snappydata.hydra.cluster.SnappyPrms-logFileName = sqlScriptResults_restart.log io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = streaming_sink_queries.sql diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/startClusterDualModeAgg.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/startClusterDualModeAgg.inc index e3c07c3c3d..f9aefdcd32 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/startClusterDualModeAgg.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/startClusterDualModeAgg.inc @@ -112,7 +112,7 @@ runMode = always threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_generateAndPublishMethod -io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "500000,streamingSinkTopic,0,0" +io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "400000,streamingSinkTopic,0,0" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar threadGroups = snappyThreads; @@ -121,3 +121,5 @@ TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMe io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = query_snappyTemp_table.sql threadGroups = snappyStoreThreads maxThreads = 1; + +io.snappydata.hydra.testDMLOps.SnappySchemaPrms-isAggregate=true; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streamingAgg.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streamingAgg.inc index 3daef32fbf..2937ac28c8 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streamingAgg.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streamingAgg.inc @@ -1,6 +1,6 @@ TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_generateAndPublishMethod //io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = "io.snappydata.hydra.streaming_sink.StringMessageProducer" -io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "400000,streamingSinkTopic,200000,4" +io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "400000,streamingSinkTopic,500000,0" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar threadGroups = leadThread maxTimesToRun = 1; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithCount.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithCount.conf index 622034df81..2225489c3e 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithCount.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithCount.conf @@ -8,7 +8,7 @@ INITTASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest INITTASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_executeSnappyStreamingApp io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.streaming_sink.SnappyStreamingSinkAppWithAggregate - io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "streamingSinkTopic persoon false ${outputMode} count" + io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "streamingSinkTopic persoon false ${outputMode} count" io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingSinkApp io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar threadGroups = snappyThreads; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithJoin.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithJoin.conf index ae455a2f72..cd1bad68ee 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithJoin.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithJoin.conf @@ -13,7 +13,7 @@ INITTASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest INITTASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_executeSnappyStreamingApp io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.streaming_sink.SnappyStreamingSinkAppWithAggregate - io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "streamingSinkTopic persoon false ${outputMode} join" + io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "streamingSinkTopic persoon false ${outputMode} join" io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingSinkApp io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar threadGroups = snappyThreads; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc index 52a475c3ee..6b4e30a10f 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc @@ -4,20 +4,16 @@ io.snappydata.hydra.cluster.SnappyPrms-cycleVms = true; util.StopStartPrms-stopModes = NICE_KILL; util.StopStartPrms-numVMsToStop = 1; -INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = backUpLeadConfigData - threadGroups = snappyThreads; - TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_restartLeadVMWithStreaming io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.streaming_sink.SnappyStreamingSinkJob io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "kafkaTopic=streamingSinkTopic,tableName=persoon,streaming.batch_interval=100,isConflation=false" io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar io.snappydata.hydra.cluster.SnappyPrms-sleepTimeSecsBeforeRestart = 60 - maxThreads = 1 startInterval = 120 - threadGroups = leadThread + threadGroups = snappyThreads maxTimesToRun = 1; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData - threadGroups = snappyThreads; +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLeadConfigData + clientNames = locator1; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc index 84da4240c7..242a2600c4 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc @@ -4,14 +4,11 @@ io.snappydata.hydra.cluster.SnappyPrms-cycleVms = true; util.StopStartPrms-stopModes = NICE_KILL; util.StopStartPrms-numVMsToStop = 1; -INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = backUpLocatorConfigData - threadGroups = snappyThreads; - TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_cycleLocatorVms startInterval = 60 - threadGroups = leadThread + threadGroups = snappyThreads io.snappydata.hydra.cluster.SnappyPrms-sleepTimeSecsBeforeRestart = 60 maxTimesToRun = 1; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLocatorConfigData - threadGroups = snappyThreads; \ No newline at end of file +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreLocatorConfigData + clientNames = locator1; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc index d1b4bcee49..c2cfeec965 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc @@ -4,14 +4,11 @@ io.snappydata.hydra.cluster.SnappyPrms-cycleVms = true; util.StopStartPrms-stopModes = NICE_KILL; util.StopStartPrms-numVMsToStop = 1; -INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = backUpServerConfigData - threadGroups = snappyThreads; - TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_cycleStoreVms startInterval = 60 - threadGroups = leadThread + threadGroups = snappyThreads io.snappydata.hydra.cluster.SnappyPrms-sleepTimeSecsBeforeRestart = 60 maxTimesToRun = 1; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData - threadGroups = snappyThreads; +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = restoreServerConfigData + clientNames = locator1; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkAgg.bt b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkAgg.bt index f1961b4b57..81ed6f11c7 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkAgg.bt +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkAgg.bt @@ -3,21 +3,21 @@ io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkWithSum.conf B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=4 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update + opType=0 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkWithAvg.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=4 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update + opType=0 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkWithJoin.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=4 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update + opType=0 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithCount.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 @@ -25,7 +25,7 @@ io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorMode C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=2 workerVMsPerHost=1 workerThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=4 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update + opType=0 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithJoin.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 @@ -33,4 +33,26 @@ io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorMode C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=2 workerVMsPerHost=1 workerThreadsPerVM=1 kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" - opType=4 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update \ No newline at end of file + opType=0 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=update + +io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkWithSum.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" + opType=0 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=complete + +io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkWithAvg.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" + opType=0 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=complete + +io/snappydata/hydra/streaming_sink/streamingAgg/streaming_sinkSmartConnectorModeWithJoin.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=2 workerVMsPerHost=1 workerThreadsPerVM=1 + kafkaDir="/export/shared/QA_DATA/kafka_2.11-0.10.2.2" + opType=0 isRowTable=false isPartitioned=true withKeyColumn=true outputMode=complete \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForConflation.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForConflation.conf index 401a907540..527eba242e 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForConflation.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForConflation.conf @@ -20,7 +20,7 @@ INITTASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_generateAndPublishMethod //io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = "io.snappydata.hydra.streaming_sink.StringMessageProducer" - io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "300000,streamingSinkTopic,0,${opType}" + io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "150000,streamingSinkTopic,0,${opType}" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar io.snappydata.hydra.cluster.SnappyPrms-isConflationTest = true threadGroups = leadThread @@ -33,8 +33,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTa maxThreads = 1; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_stopStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.adAnalytics.StopSnappyStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_verifyResults diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForDelete.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForDelete.conf index 51f7b5699e..b52a14f967 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForDelete.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForDelete.conf @@ -32,8 +32,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTa maxThreads = 1; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_stopStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.adAnalytics.StopSnappyStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_verifyResults diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForGenericColNames.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForGenericColNames.conf index 8ba4195af4..918d7a4eaa 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForGenericColNames.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForGenericColNames.conf @@ -32,8 +32,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTa maxThreads = 1; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_stopStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.adAnalytics.StopSnappyStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_verifyResults diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdate.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdate.conf index e592de71dd..92d2dfa498 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdate.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdate.conf @@ -32,8 +32,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTa maxThreads = 1; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_stopStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.adAnalytics.StopSnappyStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_verifyResults diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf index 942d50b5bd..cdf9503f08 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf @@ -22,7 +22,8 @@ TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest tas // io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = "io.snappydata.hydra.streaming_sink.StringMessageProducer" io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "500000,streamingSinkTopic,100000,4" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar - threadGroups = snappyThreads; + maxTimesToRun=1 + threadGroups = leadThread; TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSQLScripts io.snappydata.hydra.cluster.SnappyPrms-logFileName = sqlScriptResults_1.log @@ -31,8 +32,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTa maxThreads = 1; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_stopStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.adAnalytics.StopSnappyStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_verifyResults diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithAddNewNodeRebalance.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithAddNewNodeRebalance.conf index 4fd235778d..dc5df919a1 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithAddNewNodeRebalance.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithAddNewNodeRebalance.conf @@ -10,7 +10,10 @@ TASK taskClass = io.snappydata.hydra.cdcConnector.SnappyCDCTest taskMethod // trigger rebalance TASK taskClass = io.snappydata.hydra.cdcConnector.SnappyCDCTest taskMethod = HydraTask_performRebalance - threadGroups = leadThread + threadGroups = snappyStoreThreads startInterval=180 - maxThreads = 1; + maxThreads = 1 + maxTimesToRun = 1; +io.snappydata.hydra.cdcConnector.SnappyCDCPrms-dataLocation = /nfs/users/$USER/snappyHydraLogs/newNodeTest; +io.snappydata.hydra.cdcConnector.SnappyCDCPrms-nodeName = $HOSTNAME; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithClusterRestart.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithClusterRestart.conf index cfb611f281..815a4bbaf2 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithClusterRestart.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithClusterRestart.conf @@ -1,3 +1,6 @@ + INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf; INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/clusterRestart.inc; + +hydra.Prms-maxResultWaitSec = 1200; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithLeadHA.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithLeadHA.conf index 3eb25ed6e2..433ccc68f0 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithLeadHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithLeadHA.conf @@ -1,4 +1,5 @@ -INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf; - INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streamingWithLeadHA.inc; +INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf; + +INCLUDE $JTESTS/io/snappydata/hydra/northwind/backUpLeadConfig.inc; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithLocatorHA.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithLocatorHA.conf index 8f42d6f612..e211ef8424 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithLocatorHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithLocatorHA.conf @@ -1,4 +1,5 @@ -INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf; - INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streamingWithLocatorHA.inc; +INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf; + +INCLUDE $JTESTS/io/snappydata/hydra/northwind/backUpLocatorConfig.inc; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithMultipleWriters.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithMultipleWriters.conf index 3955f5d68f..a93167161b 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithMultipleWriters.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithMultipleWriters.conf @@ -52,8 +52,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTa maxThreads = 1; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_stopStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.adAnalytics.StopSnappyStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_verifyResults @@ -69,7 +68,7 @@ INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/stopClusters.inc; /* end task must stop snappy members because they are not stopped by Hydra */ hydra.Prms-alwaysDoEndTasks = true; hydra.Prms-totalTaskTimeSec = 900; -hydra.Prms-maxResultWaitSec = 1200; +hydra.Prms-maxResultWaitSec = 1500; hydra.VmPrms-extraVMArgs += fcn "hydra.TestConfigFcns.duplicate (\"-Xms1512m -Xmx2g \", ${${A}Hosts}, true)" diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRandomOps.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRandomOps.conf index 5b8871e89e..21c299d32f 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRandomOps.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRandomOps.conf @@ -25,8 +25,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTa threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest taskMethod = HydraTask_stopStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-streamingJobClassNames = io.snappydata.hydra.adAnalytics.StopSnappyStreamingJob - io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.testDMLOps.SnappyDMLOpsUtil taskMethod = HydraTask_verifyResults diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRebalance.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRebalance.conf index 19e1845a14..3bbd886365 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRebalance.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRebalance.conf @@ -2,6 +2,7 @@ INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDel // trigger rebalance TASK taskClass = io.snappydata.hydra.cdcConnector.SnappyCDCTest taskMethod = HydraTask_performRebalance - threadGroups = leadThread + threadGroups = snappyStoreThreads startInterval = 240 - maxThreads = 1; + maxThreads = 1 + maxTimesToRun = 1; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRestartStreaming.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRestartStreaming.conf index cc64ca2298..6f86b87ae4 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRestartStreaming.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithRestartStreaming.conf @@ -5,7 +5,7 @@ TASK taskClass = io.snappydata.hydra.adAnalytics.SnappyAdAnalyticsTest tas io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "kafkaTopic=streamingSinkTopic,tableName=persoon,isConflation=false" io.snappydata.hydra.cluster.SnappyPrms-userAppName = streamingApp io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar - threadGroups = leadThread + threadGroups = snappyThreads startInterval = 120 maxTimesToRun = 1; diff --git a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithServerHA.conf b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithServerHA.conf index 50522eed99..b0af7c4bda 100644 --- a/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithServerHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/streaming_sink/streaming_sinkWithServerHA.conf @@ -1,3 +1,5 @@ +INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc; + INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streaming_sinkForUpdateAndDelete.conf; -INCLUDE $JTESTS/io/snappydata/hydra/streaming_sink/streamingWithServerHA.inc; +INCLUDE $JTESTS/io/snappydata/hydra/northwind/backUpServerConfig.inc; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/SnappyDMLOpsUtil.java b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/SnappyDMLOpsUtil.java index 2e1cc24520..a0d194de46 100644 --- a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/SnappyDMLOpsUtil.java +++ b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/SnappyDMLOpsUtil.java @@ -870,6 +870,7 @@ public void performInsertUsingBatch(Connection conn, String tableName, Log.getLogWriter().info("Executing batch statement for insert."); snappyPS.executeBatch(); if (hasDerbyServer) derbyPS.executeBatch(); + Log.getLogWriter().info("Executed batch statement for insert."); batchCnt = 0; } } @@ -877,6 +878,7 @@ public void performInsertUsingBatch(Connection conn, String tableName, if(!isPopulate) SnappyConsistencyTest.waitForBarrier(tid + "", 2); snappyPS.executeBatch(); snappyPS.close(); + Log.getLogWriter().info("Executed the batch."); if(!isPopulate) SnappyConsistencyTest.waitForBarrier(tid + "", 2); if (hasDerbyServer) { derbyPS.executeBatch(); diff --git a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithAddNewNodeRebalance.conf b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithAddNewNodeRebalance.conf index c01c78efd7..70c9a92630 100644 --- a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithAddNewNodeRebalance.conf +++ b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithAddNewNodeRebalance.conf @@ -37,6 +37,5 @@ hydra.Prms-maxResultWaitSec = 900; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-dmlOperations = ONEOF insert update insert insert delete insert FOENO; -// add in local.conf -// io.snappydata.hydra.cdcConnector.SnappyCDCPrms-dataLocation = <-dir path for new server> -// io.snappydata.hydra.cdcConnector.SnappyCDCPrms-nodeName = \ No newline at end of file +io.snappydata.hydra.cdcConnector.SnappyCDCPrms-dataLocation = /nfs/users/$USER/snappyHydraLogs/newNodeTest; +io.snappydata.hydra.cdcConnector.SnappyCDCPrms-nodeName = $HOSTNAME; diff --git a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithRebalance.conf b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithRebalance.conf index 538ce0a392..d7e19b0f5b 100644 --- a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithRebalance.conf +++ b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/testDMLOpsCatalogConsistencyWithRebalance.conf @@ -21,6 +21,8 @@ TASK taskClass = io.snappydata.hydra.testDMLOps.SnappyDMLOpsUtil taskM // trigger rebalance TASK taskClass = io.snappydata.hydra.cdcConnector.SnappyCDCTest taskMethod = HydraTask_performRebalance threadGroups = snappyThreads + startInterval = 90 + endInterval = 90 maxThreads = 1; diff --git a/dtests/src/test/scala/io/snappydata/hydra/streaming_sink/CreateTableForJoinJob.scala b/dtests/src/test/scala/io/snappydata/hydra/streaming_sink/CreateTableForJoinJob.scala index 0fd003f64f..9f22b91d2e 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/streaming_sink/CreateTableForJoinJob.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/streaming_sink/CreateTableForJoinJob.scala @@ -74,7 +74,7 @@ class CreateTableForJoinJob extends SnappySQLJob{ s"hasChildren boolean," + s"numChild int," + s"hasSiblings boolean, " + - s"language varchar(10)" + + s"languages varchar(10)" + s" $primaryKey" + s") using $provider $options" pw.println(s"Creating table $s") diff --git a/dtests/src/test/scala/io/snappydata/hydra/streaming_sink/StructuredStreamingTestUtil.scala b/dtests/src/test/scala/io/snappydata/hydra/streaming_sink/StructuredStreamingTestUtil.scala index b6a74aa484..d333fbe981 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/streaming_sink/StructuredStreamingTestUtil.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/streaming_sink/StructuredStreamingTestUtil.scala @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package io.snappydata.hydra.streaming_sink diff --git a/dtests/src/test/scala/io/snappydata/hydra/testDMLOps/ValidateDMLOpApp.scala b/dtests/src/test/scala/io/snappydata/hydra/testDMLOps/ValidateDMLOpApp.scala index f2c2d14566..eb25a2d6fb 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/testDMLOps/ValidateDMLOpApp.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/testDMLOps/ValidateDMLOpApp.scala @@ -35,11 +35,10 @@ object ValidateDMLOpApp { val outputFile = "DMLOpsApp_thr_" + threadID + "_" + System.currentTimeMillis + ".out" val pw = new PrintWriter(new FileOutputStream(new File(outputFile), true)) val stmt = args(0) - // scalastyle:off println pw.println(s"stmt : ${stmt}") - val tableName = args(1) pw.println(s"Executing ${stmt}") + pw.flush() val startTime = System.currentTimeMillis val df = snc.sql(stmt) val endTime = System.currentTimeMillis