Add to the JDBC test framework the ability to set a performance SLA p…

…er file (babelfish-for-postgresql#700)

Description
Currently the tests developed on Babelfish do not have performance SLAs. This change adds SLAs to the JDBC test framework.
Added a new parameter in config.txt to specify the default SLA. This will be considered as default timeout unless it is explicitly mentioned in the test file.
One can specify the SLA value in milli seconds for a specific test file by adding a comment at the starting of the test file mentioning the SLA value. Ex: -- SLA 10000

test/JDBC/expected/sla_demo.out

@@ -0,0 +1,24 @@
+-- psql
+select 1;
+GO
+~~START~~
+int4
+1
+~~END~~
+
+
+select pg_sleep(3);
+GO
+~~START~~
+void
+
+~~END~~
+
+
+select 2;
+GO
+~~START~~
+int4
+2
+~~END~~
+

test/JDBC/input/BABEL-1756-vu-verify.sql

@@ -1,3 +1,4 @@
+-- sla 20000
 SELECT colid, name, collation_100 FROM sys.spt_tablecollations_view WHERE object_id = sys.object_id('babel_1756_vu_prepare_t1') ORDER BY colid
 GO
 

test/JDBC/input/BABEL-LOGIN-USER-EXT.mix

@@ -1,3 +1,4 @@
+-- sla 170000
 -- psql
 ALTER SYSTEM SET babelfishpg_tsql.allow_antlr_to_unsupported_grammar_for_testing = true;
 SELECT pg_reload_conf();

test/JDBC/input/BABEL-SP_COLUMN_PRIVILEGES.mix

@@ -1,3 +1,4 @@
+-- sla 10000
 -- tsql
 CREATE DATABASE db1
 GO

test/JDBC/input/BABEL-SP_FKEYS-dep-vu-verify.sql

@@ -1,2 +1,3 @@
+-- sla 20000
 exec babel_sp_fkeys_dep_vu_prepare_p1
 go

test/JDBC/input/BABEL-SP_FKEYS-vu-verify.sql

@@ -1,3 +1,4 @@
+-- sla 1000000
 use babel_sp_fkeys_vu_prepare_db1
 go
 

test/JDBC/input/BABEL-SP_FKEYS.sql

@@ -1,3 +1,4 @@
+-- sla 1000000
 create database db1
 go
 use db1

test/JDBC/input/BABEL-SP_SPECIAL_COLUMNS-vu-verify.sql

@@ -1,3 +1,4 @@
+-- sla 60000
 use babel_sp_special_columns_vu_prepare_db1
 go
 

test/JDBC/input/BABEL-SP_SPECIAL_COLUMNS.sql

@@ -1,3 +1,4 @@
+-- sla 60000
 create database db1
 go
 use db1

test/JDBC/input/BABEL-SP_SPROC_COLUMNS-vu-verify.sql

@@ -1,3 +1,4 @@
+-- sla 25000
 USE babel_sp_sproc_columns_vu_prepare_db1
 GO
 

test/JDBC/input/BABEL-SP_TABLE_PRIVILIGES-vu-verify.sql

@@ -1,3 +1,4 @@
+-- sla 20000
 use babel_sp_table_priviliges_vu_prepare_db1
 go
 

test/JDBC/input/babel_cursor.sql

@@ -1,3 +1,4 @@
+-- sla 10000
 CREATE TABLE babel_cursor_t1 (i INT, d numeric(8, 4), c varchar(10), u uniqueidentifier, v sql_variant);
 INSERT INTO babel_cursor_t1 VALUES (1, 1.1, 'a', '1E984725-C51C-4BF4-9960-E1C80E27ABA0', 1);
 INSERT INTO babel_cursor_t1 VALUES (2, 22.22, 'bb', '2E984725-C51C-4BF4-9960-E1C80E27ABA0', 22.22);

test/JDBC/input/sla_demo.mix

@@ -0,0 +1,10 @@
+-- sla 5000
+-- psql
+select 1;
+GO
+
+select pg_sleep(3);
+GO
+
+select 2;
+GO

test/JDBC/input/views/sys-all_sql_modules-dep-vu-verify.mix

@@ -1,3 +1,4 @@
+-- sla 60000
 EXEC sys_all_sql_modules_dep_vu_prepare_p1;
 GO
 

test/JDBC/input/views/sys-all_sql_modules-vu-verify.mix

@@ -1,3 +1,4 @@
+-- sla 60000
 -- NOTE: Some tests will be marked as "DISABLED DUE TO NON-UNIQUE OBJECT_ID"
 -- This is because during MVU, items in pg_class an pg_proc could have the same OBJECT_ID
 -- Some tests have no way of identification except for OBJECT_ID since definition will be missing

test/JDBC/input/views/sys-all_sql_modules.mix

@@ -1,3 +1,4 @@
+-- sla 80000
 -- Setup
 CREATE DATABASE sys_all_sql_modules_vu_prepare_db
 GO

test/JDBC/input/views/sys-columns-vu-verify.sql

@@ -1,3 +1,4 @@
+-- sla 10000
 -- Tests for sys.columns catalog view
 -- Test precision and scale for all numeric datatypes
 select name, column_id, precision, scale from sys.columns where object_id=OBJECT_ID('t1_sys_syscolumns') order by name;

test/JDBC/input/views/sys-objects-vu-verify.sql

@@ -1,3 +1,4 @@
+-- sla 50000
 USE sys_objects_vu_prepare_db1
 GO
 

test/JDBC/input/views/sys-procedures-vu-verify.sql

@@ -1,3 +1,4 @@
+-- sla 50000
 USE sys_procedures_vu_prepare_db1
 GO
 

test/JDBC/input/views/sys-sql_modules-dep-vu-verify.mix

@@ -1,3 +1,4 @@
+-- sla 50000
 EXEC sys_sql_modules_dep_vu_prepare_p1;
 GO
 

test/JDBC/input/views/sys-sql_modules-vu-verify.mix

@@ -1,3 +1,4 @@
+-- sla 35000
 -- NOTE: Some tests will be marked as "DISABLED DUE TO NON-UNIQUE OBJECT_ID"
 -- This is because during MVU, items in pg_class an pg_proc could have the same OBJECT_ID
 -- Some tests have no way of identification except for OBJECT_ID since definition will be missing

test/JDBC/input/views/sys-sql_modules.mix

@@ -1,3 +1,4 @@
+-- sla 35000
 -- Setup
 CREATE DATABASE sys_sql_modules_vu_prepare_db
 GO

test/JDBC/input/views/sys-syscolumns.sql

@@ -1,3 +1,4 @@
+-- sla 10000
 create database db1;
 go
 

test/JDBC/input/views/sys-system_sql_modules-dep-vu-verify.sql

@@ -1,3 +1,4 @@
+-- sla 50000
 USE sys_system_sql_modules_dep_vu_prepare_db1
 GO
 

test/JDBC/input/views/sys-system_sql_modules-vu-verify.sql

@@ -1,3 +1,4 @@
+-- sla 50000
 -- NOTE: Some tests will be marked as "DISABLED DUE TO NON-UNIQUE OBJECT_ID"
 -- This is because during MVU, items in pg_class an pg_proc could have the same OBJECT_ID
 -- Some tests have no way of identification except for OBJECT_ID since definition will be missing

test/JDBC/input/views/sys-system_sql_modules.sql

@@ -1,3 +1,4 @@
+-- sla 50000
 -- Test for system function
 SELECT
     definition,

test/JDBC/src/main/java/com/sqlsamples/Config.java

@@ -27,6 +27,7 @@ public class Config {
     static String scheduleFileName = properties.getProperty("scheduleFile");
     static String testFileRoot = properties.getProperty("testFileRoot");
     static boolean isUpgradeTestMode =  Boolean.parseBoolean(properties.getProperty("isUpgradeTestMode"));
+    static long defaultSLA = Long.parseLong(properties.getProperty("defaultSLA"));
 
     static String connectionString = constructConnectionString();
 

test/JDBC/src/main/java/com/sqlsamples/Statistics.java

@@ -7,7 +7,8 @@
 public class Statistics {
     DescriptiveStatistics descriptiveStatistics;
     static ArrayList<Long> exec_times = new ArrayList<>();
-
+    static long curr_exec_time = 0L;
+    static long sla = 0L;
     public Statistics(double[] values) { 
         descriptiveStatistics = new DescriptiveStatistics(values);
     }

test/JDBC/src/main/java/com/sqlsamples/batch_run.java

@@ -10,6 +10,8 @@
 
 import static com.sqlsamples.Config.*;
 import static com.sqlsamples.Statistics.exec_times;
+import static com.sqlsamples.Statistics.curr_exec_time;
+import static com.sqlsamples.Statistics.sla;
 
 public class batch_run {
 
@@ -20,6 +22,7 @@ static void batch_run_sql(Connection con_bbl, BufferedWriter bw, String testFile
         boolean isCrossDialectFile = false;
         boolean tsqlDialect = false;
         boolean psqlDialect = false;
+        boolean customSLA = false;
 
         if (testFilePath.contains(".mix")) {
             isCrossDialectFile = true;
@@ -65,12 +68,10 @@ static void batch_run_sql(Connection con_bbl, BufferedWriter bw, String testFile
                     }
                     continue;
                 }
-
                 long startTime = System.nanoTime();
 
                 // if line starts with keyword "prepst", it means it is either a prep exec statement or an exec statement
                 if (strLine.startsWith("prepst")) {
-
                     // Convert .NET input file format for prepared statement to JDBC
                     strLine = strLine.replaceAll("@[a-zA-Z0-9]+", "?");
 
@@ -91,7 +92,6 @@ static void batch_run_sql(Connection con_bbl, BufferedWriter bw, String testFile
                         jdbcPreparedStatement.testPreparedStatementWithFile(result, bw, strLine, logger);
 
                     } else if (!result[1].equals("exec")) {
-
                         jdbcPreparedStatement.closePreparedStatements(bw, logger);
 
                         SQL = result[1];
@@ -186,7 +186,6 @@ static void batch_run_sql(Connection con_bbl, BufferedWriter bw, String testFile
 
                 // if line starts with keyword "cursor", it means it is a cursor operation
                 } else if (strLine.startsWith("cursor")) {
-
                     // Convert .NET input file format for prepared statement to JDBC
                     // Used if cursor opened on a result set from a prepared statement
                     if (strLine.contains("prepst")) {
@@ -255,7 +254,14 @@ static void batch_run_sql(Connection con_bbl, BufferedWriter bw, String testFile
                 } else {
                     // execute statement as a normal SQL statement
                     if (isSQLFile) {
-                        if (!strLine.equalsIgnoreCase("GO")) {
+                        customSLA = strLine.toLowerCase().startsWith("-- sla");
+                        if (customSLA){
+                            String[] tokens=strLine.split(" ");  
+                            sla = Long.parseLong(tokens[2]);
+                            sla = sla*(1000000L);
+                            continue;
+                        }
+                        else if (!strLine.equalsIgnoreCase("GO")) {
                             sqlBatch.append(strLine).append(System.lineSeparator());
                             continue;
                         } else {
@@ -273,10 +279,10 @@ static void batch_run_sql(Connection con_bbl, BufferedWriter bw, String testFile
                     jdbcStatement.createStatements(con_bbl, bw, logger);
                     jdbcStatement.testStatementWithFile(SQL, bw, strLine, logger);
                 }
-
                 long endTime = System.nanoTime();
                 long duration = (endTime - startTime);
                 exec_times.add(duration);
+                curr_exec_time += duration;
             }
         } catch (IOException ioe) {
             logger.error("IO Exception: " + ioe.getMessage(), ioe);

test/JDBC/src/main/resources/config.txt

@@ -41,3 +41,6 @@ testFileRoot = ./
 
 # WHETHER TEST RUN MODE IS UPGRADE TEST RUN
 isUpgradeTestMode = false
+
+# Default SLA in milliseconds
+defaultSLA = 15000

test/JDBC/src/test/java/com/sqlsamples/TestQueryFile.java

@@ -16,6 +16,8 @@
 import static com.sqlsamples.Config.*;
 
 import static com.sqlsamples.Statistics.exec_times;
+import static com.sqlsamples.Statistics.curr_exec_time;
+import static com.sqlsamples.Statistics.sla;
 
 public class TestQueryFile {
 
@@ -26,6 +28,7 @@ public class TestQueryFile {
     static Logger summaryLogger = LogManager.getLogger("testSummaryLogger");    //logger to write summary of tests executed
     static Logger logger = LogManager.getLogger("eventLoggger");                //logger to log any test framework events
     static ArrayList<AbstractMap.SimpleEntry<String, Boolean>> summaryMap = new ArrayList<>(); //map to store test names and status
+    static ArrayList<AbstractMap.SimpleEntry<String, ArrayList<Long>>> slaMap = new ArrayList<>(); //map to store execution time and SLA
     static ArrayList<AbstractMap.SimpleEntry<String, ArrayList<Integer>>> testCountMap = new ArrayList<>(); //map to store test names and number of tests passed
     static ArrayList <String> fileList = new ArrayList<>();
     static HashMap<String, String> filePaths = new HashMap<>(); //map to store fileName and their paths
@@ -252,16 +255,23 @@ public static void logSummary() {
             String testMethodName = summaryMap.get(i).getKey();
 
             boolean status = summaryMap.get(i).getValue();
+            Long exec_time = slaMap.get(i).getValue().get(0);
+            Long sla_val = slaMap.get(i).getValue().get(1);
             int testsPassed = 0, totalTests = 0;
 
             if(status){
                 //extra spaces for right side padding
-                summaryLogger.info((testMethodName + ":" + "                                                     ").substring(0, maxlen+2) + "Passed!" + testStats);
+                summaryLogger.info((testMethodName + ":" + "                                                     ").substring(0, maxlen+2) + "Passed! (" + exec_time + "/" + sla_val + "ms OK)" + testStats);
                 passed++;
             }
+            else if(exec_time <= sla_val){
+                //extra spaces for right side padding
+                summaryLogger.info((testMethodName + ":" + "                                                     ").substring(0, maxlen+2) + "Failed! (" + exec_time + "/" + sla_val + "ms OK)" + testStats);
+                failed++;
+            }
             else{
                 //extra spaces for right side padding
-                summaryLogger.info((testMethodName + ":" + "                                                     ").substring(0, maxlen+2) + "Failed!" + testStats);
+                summaryLogger.info((testMethodName + ":" + "                                                     ").substring(0, maxlen+2) + "Failed! (" + exec_time + "/" + sla_val + "ms TIME OUT)" + testStats);
                 failed++;
             }
         }
@@ -378,8 +388,8 @@ public void TestQueryBatch(String inputFileName) throws SQLException, ClassNotFo
         String testFilePath = filePaths.get(inputFileName);
 
         boolean result; // whether test passed or failed
+        boolean timeout;
         int failed;
-
         String outputFileName;
 
         if(inputFilesDirectoryPath.substring(inputFilesDirectoryPath.length() - 1) == "/"){
@@ -394,12 +404,21 @@ public void TestQueryBatch(String inputFileName) throws SQLException, ClassNotFo
         // generate buffer reader associated with the file
         FileWriter fw = new FileWriter(outputFile);
         BufferedWriter bw = new BufferedWriter(fw);
+        curr_exec_time = 0L;
         batch_run.batch_run_sql(connection_bbl, bw, testFilePath, logger);
         bw.close();
-
+        if(sla == 0){
+            sla = defaultSLA*1000000L;
+        }
         File expectedFile = new File(generatedFilesDirectoryPath + outputFileName + ".out");
         File sqlExpectedFile = new File(sqlServerGeneratedFilesDirectoryPath + outputFileName + ".out");
 
+        if(curr_exec_time <= sla){
+            timeout = false;
+        }
+        else{
+            timeout = true;
+        }
         if (expectedFile.exists()) {
             // get the diff
             result = compareOutFiles(outputFile, expectedFile);
@@ -410,14 +429,31 @@ public void TestQueryBatch(String inputFileName) throws SQLException, ClassNotFo
             result = false;
         }
 
-        summaryMap.add(new AbstractMap.SimpleEntry<>(inputFileName, result)); //add test name and result to map
+        ArrayList<Long> tempSla = new ArrayList<>();
+        tempSla.add(curr_exec_time/1000000L);
+        tempSla.add(sla/1000000L);
+        summaryMap.add(new AbstractMap.SimpleEntry<>(inputFileName, result && !timeout)); //add test name and result to map
+        slaMap.add(new AbstractMap.SimpleEntry<>(inputFileName, tempSla)); //add execution time and SLA
+        sla = 0L;
 
         try {
-            Assertions.assertTrue(result);
+            Assertions.assertTrue(result && !timeout);
         } catch (AssertionError e) {
-            Throwable throwable = new Throwable(inputFileName + " FAILED! Output diff can be found in '" + diffFile.getAbsolutePath() + "'");
-            throwable.setStackTrace(new StackTraceElement[0]);
-            throw throwable;
+            if(timeout && result){
+                Throwable throwable = new Throwable(inputFileName + " FAILED! Execution timed out!!");
+                throwable.setStackTrace(new StackTraceElement[0]);
+                throw throwable;
+            }
+            else if(timeout && !result){
+                Throwable throwable = new Throwable(inputFileName + " FAILED! Execution timed out! Output diff can be found in '" + diffFile.getAbsolutePath() + "'");
+                throwable.setStackTrace(new StackTraceElement[0]);
+                throw throwable;
+            }
+            else{
+                Throwable throwable = new Throwable(inputFileName + " FAILED! Output diff can be found in '" + diffFile.getAbsolutePath() + "'");
+                throwable.setStackTrace(new StackTraceElement[0]);
+                throw throwable;
+            }
         }
     }
 }

test/JDBC/upgrade/latest/verification_cleanup/13_4/sys-columns-dep-vu-verify.sql

@@ -1,3 +1,4 @@
+-- sla 20000
 exec sys_columns_dep_vu_prepare_p1
 go
 

test/JDBC/upgrade/latest/verification_cleanup/13_5/sys-columns-dep-vu-verify.sql

@@ -1,3 +1,4 @@
+-- sla 20000
 exec sys_columns_dep_vu_prepare_p1
 go
 

test/JDBC/upgrade/latest/verification_cleanup/13_6/sys-all_sql_modules-vu-verify.mix

@@ -1,3 +1,4 @@
+-- sla 200000
 USE sys_all_sql_modules_vu_prepare_db
 GO
 

test/JDBC/upgrade/latest/verification_cleanup/13_6/sys-all_views-vu-verify.sql

@@ -1,3 +1,4 @@
+-- sla 10000
 SELECT * FROM sys_all_views_dep_view_vu_prepare
 GO
 

test/JDBC/upgrade/latest/verification_cleanup/13_6/sys-columns-dep-vu-verify.sql

@@ -1,3 +1,4 @@
+-- sla 20000
 exec sys_columns_dep_vu_prepare_p1
 go
 

test/JDBC/upgrade/latest/verification_cleanup/13_6/sys-default_constraints-dep-vu-verify.sql

@@ -1,3 +1,4 @@
+-- sla 10000
 EXEC sys_default_constraints_dep_vu_prepare_p1
 GO