Merge pull request #54 from IBMStreams/develop

2.1.4

com.ibm.streamsx.elasticsearch/info.xml

@@ -78,7 +78,7 @@ as they constitute serious security holes.
   you can use this parameter to temporarily disable hostnam verification.
 
 ]]></info:description>
-    <info:version>2.1.3</info:version>
+    <info:version>2.1.4</info:version>
     <info:requiredProductVersion>4.2.0.0</info:requiredProductVersion>
   </info:identity>
   <info:dependencies/>

com.ibm.streamsx.elasticsearch/pom.xml

@@ -4,7 +4,7 @@
   <groupId>com.ibm.streamsx.elasticsearch</groupId>
   <artifactId>streamsx.elasticsearch</artifactId>
   <packaging>jar</packaging>
-  <version>2.1.3</version>
+  <version>2.1.4</version>
   <dependencies>
     <dependency>
       <groupId>io.searchbox</groupId>

samples/AppConfigSample/README.md

@@ -34,7 +34,7 @@ You need to launch the application in distributed mode, otherwise the applicatio
 
 Verify that data is written into `index2` with the connection string of your instance:
 
-curl -u admin:PASSWORD 'https://portal-ssl11-22.bmix-eu-gb-yp-0a000dc-a22d-22bd-a333-4444c555ba66.77777777.composedb.com:15149/index2/_search?q=*&pretty'
+curl -k -u admin:PASSWORD 'https://portal-ssl11-22.bmix-eu-gb-yp-0a000dc-a22d-22bd-a333-4444c555ba66.77777777.composedb.com:15149/index2/_search?q=*&pretty'
 
 
 ### Clean:

samples/AppConfigSample/com.ibm.streamsx.elasticsearch.sample.AppConfig/Main.spl

@@ -20,7 +20,7 @@ use com.ibm.streamsx.elasticsearch::ElasticsearchIndex;
  * Optional parameter change the index name, default: index1
  *
   */
-composite Main
+public composite Main
 {
 
 	param

samples/AppConfigSample/info.xml

@@ -3,7 +3,7 @@
   <info:identity>
     <info:name>ElasticsearchAppConfigSample</info:name>
     <info:description>Demonstrates the use of the ElasticsearchIndex operator with connection settings in an application configuration.</info:description>
-    <info:version>1.0.0</info:version>
+    <info:version>1.0.1</info:version>
     <info:requiredProductVersion>4.2.0.0</info:requiredProductVersion>
   </info:identity>
   <info:dependencies>

samples/ECGSample/com.ibm.streamsx.elasticsearch.sample.ECG/Main.spl

@@ -17,7 +17,7 @@ namespace com.ibm.streamsx.elasticsearch.sample.ECG ;
 
 use com.ibm.streamsx.elasticsearch::ElasticsearchIndex ;
 
-composite Main
+public composite Main
 {
 	graph
 		/**

samples/SSLCloudSample/README.md

@@ -20,7 +20,7 @@ When launching the application in standalone mode, you need to update `nodeList`
 
 Verify that data is written into `index1` with the connection string of your instance:
 
-curl -u admin:PASSWORD 'https://portal-ssl11-22.bmix-eu-gb-yp-0a000dc-a22d-22bd-a333-4444c555ba66.77777777.composedb.com:15149/index1/_search?q=*&pretty'
+curl -k -u admin:PASSWORD 'https://portal-ssl11-22.bmix-eu-gb-yp-0a000dc-a22d-22bd-a333-4444c555ba66.77777777.composedb.com:15149/index1/_search?q=*&pretty'
 
 
 ### Clean:

samples/SSLCloudSample/com.ibm.streamsx.elasticsearch.sample.SSL/Main.spl

@@ -27,7 +27,7 @@ use com.ibm.streamsx.elasticsearch::ElasticsearchIndex ;
  * URL to connect to Elasticsearch server
  *
   */
-composite Main
+public composite Main
 {
 
 	param
@@ -54,9 +54,7 @@ composite Main
 				indexName : $indexName;
 				nodeList : $nodeList;
 				sslEnabled : true;
-				sslDebug : false;
-				sslTrustStore : "cacerts";
-				sslTrustStorePassword : "123456";
+				sslTrustAllCertificates: true;
 				userName : $userName;
 				password : $password;
 		}

samples/SSLCloudSample/info.xml

@@ -3,7 +3,7 @@
   <info:identity>
     <info:name>ElasticsearchSSLCloudSample</info:name>
     <info:description>Demonstrates the usage of using the ElasticsearchIndex operator with SSL connection to the IBM cloud service.</info:description>
-    <info:version>1.0.0</info:version>
+    <info:version>1.0.1</info:version>
     <info:requiredProductVersion>4.2.0.0</info:requiredProductVersion>
   </info:identity>
   <info:dependencies>

tests/cloud/es_test/Makefile

@@ -46,7 +46,7 @@ clean:
 
 configure:
 	-streamtool rmappconfig --noprompt es
-	streamtool mkappconfig --property userName=$(ES_USER) --property password=$(ES_PASSWORD) --property nodeList=$(ES_NODES) --property sslEnabled=true es
+	streamtool mkappconfig --property userName=$(ES_USER) --property password=$(ES_PASSWORD) --property nodeList=$(ES_NODES) --property sslEnabled=true --property sslTrustAllCertificates=true es
 
 tkidx:
 	$(STREAMS_INSTALL)/bin/spl-make-toolkit -i .

tests/cloud/test_es.py

@@ -6,6 +6,8 @@
 import streamsx.spl.toolkit as tk
 import streamsx.rest as sr
 import os, os.path
+import subprocess
+from subprocess import call, Popen, PIPE
 import elasticsearch
 from elasticsearch import Elasticsearch
 from elasticsearch.helpers import bulk
@@ -15,6 +17,14 @@
 import requests
 from urllib.parse import urlparse
 
+def streams_install_env_var():
+    result = True
+    try:
+        os.environ['STREAMS_INSTALL']
+    except KeyError: 
+        result = False
+    return result
+
 class Test(unittest.TestCase):
     """ Test invocations of composite operators in local Streams instance """
 
@@ -24,9 +34,16 @@ def setUpClass(self):
         print ("Setup Elasticsearch client ...")
         # ES client expects ES_URL environment variable with URL to Compose Elasticsearch service, e.g. https://user:password@portalxxx.composedb.com:port/
         es_url = os.environ['ES_URL']
+        print (str(es_url))
         self._es = Elasticsearch([es_url],verify_certs=False)
         self._indexName = 'test-index-cloud'
-        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) 
+        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+        creds = urlparse(es_url)
+        self._es_user_name = creds.username
+        self._es_password = creds.password
+        self._es_node_list = creds.hostname+':'+str(creds.port)
+
 
     def setUp(self):
         Tester.setup_distributed(self)
@@ -74,12 +91,29 @@ def _validate_count(self, indexName, expectedNum):
         print("Count: "+str(count['count']))
         assert (count['count'] == expectedNum), "Wrong tuple count (expected="+str(expectedNum)+"): "+str(count['count'])        
 
+    def _run_shell_command_line(self, command):
+        process = Popen(command, universal_newlines=True, shell=True, stdout=PIPE, stderr=PIPE)
+        stdout, stderr = process.communicate()
+        return stdout, stderr, process.returncode
+
+    def _create_app_config(self):
+        if ("TestICP" in str(self) or "TestCloud" in str(self) ):
+            print ("Ensure that application configuration 'es' is created.")
+        else:
+            if streams_install_env_var():
+                print ("Create elasticsearch application configuration with streamtool")
+                this_dir = os.path.dirname(os.path.realpath(__file__))
+                app_dir = this_dir+'/es_test'
+                stdout, stderr, err = self._run_shell_command_line('export ES_NODES='+self._es_node_list+';'+'export ES_USER='+self._es_user_name+';'+'export ES_PASSWORD='+self._es_password+';'+'cd '+app_dir+'; make configure')
+                print (str(err))
+
     # ------------------------------------
 
     # CONSISTENT REGION test with TopologyTester:
     # Resets triggered by ConsistentRegionResetter and Beacon re-submits the tuples
     def test_consistent_region_with_resets(self):
         self._indexName = 'test-index-cr'
+        self._create_app_config()
         # delete index before launching Streams job
         self._es.indices.delete(index=self._indexName, ignore=[400, 404]) 
         numResets = 3
@@ -93,6 +127,7 @@ def test_consistent_region_with_resets(self):
 
     def test_bulk(self):
         self._indexName = 'test-index-bulk'
+        self._create_app_config()
         # delete index before launching Streams job
         self._es.indices.delete(index=self._indexName, ignore=[400, 404]) 
         numTuples = 20000 # num generated tuples