diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/AppDeployerServiceComponent.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/AppDeployerServiceComponent.java index 7b7af54355..9304fd3655 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/AppDeployerServiceComponent.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/AppDeployerServiceComponent.java @@ -30,7 +30,6 @@ import org.osgi.service.component.annotations.ReferenceCardinality; import org.osgi.service.component.annotations.ReferencePolicy; import org.wso2.carbon.securevault.SecretCallbackHandlerService; -import org.wso2.config.mapper.ConfigParser; import org.wso2.micro.application.deployer.handler.DefaultAppDeployer; import org.wso2.micro.core.CarbonAxisConfigurator; import org.wso2.micro.integrator.dataservices.core.DBDeployer; @@ -39,18 +38,10 @@ import org.wso2.micro.integrator.initializer.deployment.application.deployer.CappDeployer; import org.wso2.micro.integrator.initializer.deployment.synapse.deployer.FileRegistryResourceDeployer; import org.wso2.micro.integrator.initializer.deployment.synapse.deployer.SynapseAppDeployer; -import org.wso2.micro.integrator.initializer.serviceCatalog.ServiceCatalogExecutor; import org.wso2.micro.integrator.initializer.services.SynapseEnvironmentService; import org.wso2.micro.integrator.initializer.utils.ConfigurationHolder; import org.wso2.micro.integrator.ndatasource.capp.deployer.DataSourceCappDeployer; -import java.util.ArrayList; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static org.wso2.micro.integrator.initializer.utils.Constants.ENABLE; -import static org.wso2.micro.integrator.initializer.utils.Constants.SERVICE_CATALOG_CONFIG; @Component(name = "org.wso2.micro.integrator.initializer.deployment.AppDeployerServiceComponent", immediate = true) public class AppDeployerServiceComponent { @@ -168,6 +159,7 @@ private void addCAppDeployer(DeploymentEngine deploymentEngine) { // Initialize CApp deployer here CappDeployer cappDeployer = new CappDeployer(); cappDeployer.setDirectory(artifactRepoPath + DeploymentConstants.CAPP_DIR_NAME); + cappDeployer.setSecretCallbackHandlerService(secretCallbackHandlerService); cappDeployer.init(configCtx); // Register application deployment handlers @@ -211,19 +203,6 @@ private void invokeRegisteredDeployers() { AxisConfigurator axisConfigurator = configCtx.getAxisConfiguration().getConfigurator(); if (axisConfigurator instanceof CarbonAxisConfigurator) { ((CarbonAxisConfigurator) axisConfigurator).deployServices(); - Map catalogProperties; - if (ConfigParser.getParsedConfigs().get(SERVICE_CATALOG_CONFIG) != null) { - catalogProperties = - (Map) ((ArrayList) ConfigParser.getParsedConfigs().get( - SERVICE_CATALOG_CONFIG)).get(0); - if ((boolean) catalogProperties.get(ENABLE)) { - String repoLocation = ((CarbonAxisConfigurator) axisConfigurator).getRepoLocation(); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.submit(new ServiceCatalogExecutor(repoLocation, secretCallbackHandlerService)); - // we are not interested in the result of this thread execution - executorService.shutdown(); - } - } } } diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/application/deployer/CappDeployer.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/application/deployer/CappDeployer.java index 909f3a6eba..55feb01e1e 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/application/deployer/CappDeployer.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/deployment/application/deployer/CappDeployer.java @@ -33,15 +33,19 @@ import org.apache.synapse.config.SynapseConfigUtils; import org.apache.synapse.config.SynapseConfiguration; import org.apache.synapse.api.API; +import org.wso2.carbon.securevault.SecretCallbackHandlerService; import org.wso2.micro.application.deployer.AppDeployerUtils; import org.wso2.micro.application.deployer.CarbonApplication; import org.wso2.micro.application.deployer.config.ApplicationConfiguration; import org.wso2.micro.application.deployer.config.Artifact; import org.wso2.micro.application.deployer.handler.AppDeploymentHandler; +import org.wso2.micro.core.CarbonAxisConfigurator; import org.wso2.micro.core.util.CarbonException; import org.wso2.micro.core.util.FileManipulator; import org.wso2.micro.integrator.initializer.dashboard.ArtifactDeploymentListener; +import org.wso2.micro.integrator.initializer.serviceCatalog.ServiceCatalogDeployer; import org.wso2.micro.integrator.initializer.utils.DeployerUtil; +import org.wso2.micro.integrator.initializer.utils.ServiceCatalogUtils; import java.io.File; import java.io.FileInputStream; @@ -55,6 +59,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import javax.xml.namespace.QName; import javax.xml.stream.XMLStreamException; @@ -86,6 +92,21 @@ public class CappDeployer extends AbstractDeployer { */ private String extension; + /** + * Service Catalog Executor threads for publishing Services to Service Catalog. + */ + private ExecutorService serviceCatalogExecutor; + + /** + * Map object to store Service Catalog configuration + */ + private Map serviceCatalogConfiguration; + + /** + * SecretCallbackHandlerService to read Service Catalog Configs + */ + private SecretCallbackHandlerService secretCallbackHandlerService; + public void init(ConfigurationContext configurationContext) { if (log.isDebugEnabled()) { log.debug("Initializing Capp Deployer.."); @@ -96,6 +117,12 @@ public void init(ConfigurationContext configurationContext) { String appUnzipDir = AppDeployerUtils.getAppUnzipDir() + File.separator + AppDeployerUtils.getTenantIdString(); FileManipulator.deleteDir(appUnzipDir); + + if (ServiceCatalogUtils.isServiceCatalogEnabled()) { + serviceCatalogConfiguration = ServiceCatalogUtils.readConfiguration(secretCallbackHandlerService); + serviceCatalogExecutor = Executors.newFixedThreadPool( + ServiceCatalogUtils.getExecutorThreadCount(serviceCatalogConfiguration, 10)); + } } public void setDirectory(String cAppDir) { @@ -193,6 +220,12 @@ private void deployCarbonApps(String artifactPath) throws CarbonException { faultyCAppObjects.add(currentApp); faultyCapps.add(cAppName); } + if (serviceCatalogConfiguration != null && !faultyCapps.contains(cAppName)) { + ServiceCatalogDeployer serviceDeployer = new ServiceCatalogDeployer(cAppName, + ((CarbonAxisConfigurator) axisConfig.getAxisConfiguration().getConfigurator()).getRepoLocation(), + serviceCatalogConfiguration); + serviceCatalogExecutor.execute(serviceDeployer); + } } /** @@ -658,6 +691,10 @@ public void cleanup() { faultyCAppObjects.clear(); } + public void setSecretCallbackHandlerService(SecretCallbackHandlerService secretCallbackHandlerService) { + this.secretCallbackHandlerService = secretCallbackHandlerService; + } + /** * Partially building the API to get the API name * diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogExecutor.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogDeployer.java similarity index 58% rename from components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogExecutor.java rename to components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogDeployer.java index 2cb72ea52b..d0e71831f5 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogExecutor.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/serviceCatalog/ServiceCatalogDeployer.java @@ -1,45 +1,47 @@ /* - * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. * - * WSO2 Inc. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at + * WSO2 Inc. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the + * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ + package org.wso2.micro.integrator.initializer.serviceCatalog; -import org.wso2.carbon.securevault.SecretCallbackHandlerService; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import java.io.File; import java.util.Map; import static org.wso2.micro.integrator.initializer.utils.Constants.*; +import static org.wso2.micro.integrator.initializer.utils.Constants.SERVICE_CATALOG; import static org.wso2.micro.integrator.initializer.utils.ServiceCatalogUtils.*; +import static org.wso2.micro.integrator.initializer.utils.ServiceCatalogUtils.publishToAPIM; -/** - * This class will publish the metadata to the service catalog in a separate thread. - */ -public class ServiceCatalogExecutor implements Runnable { - +public class ServiceCatalogDeployer implements Runnable { + private static final Log log = LogFactory.getLog(ServiceCatalogDeployer.class); private static final String CAPP_UNZIP_DIR; + private final String cAppName; + private final Map serviceCatalogConfiguration; private final String repoLocation; - private final SecretCallbackHandlerService secretCallbackHandlerService; - public ServiceCatalogExecutor(String repoLocation, - SecretCallbackHandlerService secretCallbackHandlerService) { + public ServiceCatalogDeployer(String name, String repoLocation, Map serviceCatalogConfiguration) { + this.cAppName = name; this.repoLocation = repoLocation; - this.secretCallbackHandlerService = secretCallbackHandlerService; + this.serviceCatalogConfiguration = serviceCatalogConfiguration; } static { @@ -50,13 +52,13 @@ public ServiceCatalogExecutor(String repoLocation, @Override public void run() { + log.info("Executing Service Catalog deployer for CApp : " + cAppName); + // check pre-conditions if (!checkPreConditions()) return; - Map apimConfigs = readConfiguration(secretCallbackHandlerService); - // call service catalog and get all services - Map md5MapOfAllService = getAllServices(apimConfigs); + Map md5MapOfAllService = getAllServices(serviceCatalogConfiguration); if (md5MapOfAllService == null) return; // error occurred while getting all services // create temporary directory to hold metadata. @@ -71,6 +73,6 @@ public void run() { if (!archiveDir(CAPP_UNZIP_DIR + File.separator + ZIP_FOLDER_NAME, tempDir.getPath())) return; // publish to service catalog endpoint. - publishToAPIM(apimConfigs, CAPP_UNZIP_DIR + File.separator + ZIP_FOLDER_NAME); + publishToAPIM(serviceCatalogConfiguration, CAPP_UNZIP_DIR + File.separator + ZIP_FOLDER_NAME); } } diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/Constants.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/Constants.java index 0fdc005e66..cc87317772 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/Constants.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/Constants.java @@ -41,6 +41,7 @@ public class Constants { public static final String ENABLE = "enable"; public static final String USER_NAME = "username"; public static final String PASSWORD = "password"; + public static final String SERVICE_CATALOG_EXECUTOR_THREADS = "executor_threads"; public static final String SERVICE_CATALOG_PUBLISH_ENDPOINT = "api/am/service-catalog/v0/services/import?overwrite" + "=true"; diff --git a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java index 5cc9018db4..295e8748ab 100644 --- a/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java +++ b/components/org.wso2.micro.integrator.initializer/src/main/java/org/wso2/micro/integrator/initializer/utils/ServiceCatalogUtils.java @@ -384,6 +384,7 @@ public static Map readConfiguration(SecretCallbackHandlerService String userName = catalogProperties.get(USER_NAME); String password = catalogProperties.get(PASSWORD); + String executorThreads = catalogProperties.get(SERVICE_CATALOG_EXECUTOR_THREADS); if (secretResolver == null) { secretResolver = SecretResolverFactory.create((OMElement) null, false); } @@ -402,6 +403,7 @@ public static Map readConfiguration(SecretCallbackHandlerService configMap.put(APIM_HOST, apimHost); configMap.put(USER_NAME, userName); configMap.put(PASSWORD, password); + configMap.put(SERVICE_CATALOG_EXECUTOR_THREADS, executorThreads); return configMap; } @@ -575,7 +577,7 @@ public static boolean archiveDir(String destArchiveName, String sourceDir) { if (alreadyUploaded) { log.info("Service catalog already contains the latest configs, aborting the service-catalog uploader"); } else { - log.info("Could not find metadata to upload, aborting the service-catalog uploader"); + log.info("Metadata not included, hence not publishing to Service Catalog"); } return false; } @@ -871,4 +873,51 @@ private static String getProxyServiceUrlFromMetadata(File yamlFile) { } return currentServiceUrl; } + + /** + * Checks whether Service Catalog configuration is Enabled. + * + * @return whether Service Catalog is enabled. + */ + public static boolean isServiceCatalogEnabled() { + Map catalogProperties; + if (ConfigParser.getParsedConfigs().get(SERVICE_CATALOG_CONFIG) != null) { + catalogProperties = + (Map) ((ArrayList) ConfigParser.getParsedConfigs().get( + SERVICE_CATALOG_CONFIG)).get(0); + if ((boolean) catalogProperties.get(ENABLE)) { + return true; + } + } + return false; + } + + /** + * Retrieve Executor Thread configuration for Service Catalog. + * + * @param serviceCatalogConfig Map containing deployment.toml configuration + * @param def Default value + * + * @return Executor Thread count. + */ + public static int getExecutorThreadCount(Map serviceCatalogConfig, int def) { + String executorThreads = serviceCatalogConfig.get(SERVICE_CATALOG_EXECUTOR_THREADS); + int threads; + if (executorThreads != null) { + try { + threads = Integer.parseInt(executorThreads); + if (log.isDebugEnabled()) { + log.debug("Service Catalog Executor Thread count is set to " + threads); + } + } catch (NumberFormatException e) { + log.warn("Invalid Service Catalog Executor Thread count. Setting to default " + def); + return def; + } + return threads ; + } + if (log.isDebugEnabled()) { + log.debug("Service Catalog Executor Thread count is not defined. Setting to default " + def); + } + return def; + } } diff --git a/integration/mediation-tests/tests-other/src/test/java/org/wso2/carbon/esb/serviceCatalog/test/ServiceCatalogTestCase.java b/integration/mediation-tests/tests-other/src/test/java/org/wso2/carbon/esb/serviceCatalog/test/ServiceCatalogTestCase.java index 847e4f4a61..4bdc63cea6 100644 --- a/integration/mediation-tests/tests-other/src/test/java/org/wso2/carbon/esb/serviceCatalog/test/ServiceCatalogTestCase.java +++ b/integration/mediation-tests/tests-other/src/test/java/org/wso2/carbon/esb/serviceCatalog/test/ServiceCatalogTestCase.java @@ -65,6 +65,7 @@ public class ServiceCatalogTestCase extends ESBIntegrationTest { private static final String FAULTY_CAPP = "invalidCompositeApplication_1.0.0.car"; private static final String CAPP_WITH_META_AND_ENV = "blaCompositeExporter_1.0.0-SNAPSHOT.car"; private static final String CAPP_WITH_PROXY_META = "proxyCompositeExporter_1.0.0-SNAPSHOT.car"; + private static final String CAPP_WITHOUT_META = "HelloWorldWithoutMetadataCompositeExporter_1.0.0-SNAPSHOT.car"; private static final String NEW_CAPP_NAME = "demoCompositeExporter_1.0.0-SNAPSHOT.car"; private static final String MODIFIED_NEW_CAPP_NAME = "changed_demoCompositeExporter_1.0.0-SNAPSHOT.car"; private static final String SH_FILE_NAME = "micro-integrator.sh"; @@ -144,12 +145,42 @@ public void testServiceCatalogWithoutFaultyCAPP() serverConfigurationManager.removeFromCarbonapps(FAULTY_CAPP); serverConfigurationManager.restartMicroIntegrator(); assertTrue(Utils.checkForLog(carbonLogReader, - "Could not find metadata to upload, aborting the service-catalog uploader", 10), + "Metadata not included, hence not publishing to Service Catalog", 10), "Did not receive the expected info log"); } @Test(groups = {"wso2.esb"}, - description = "Test service catalog without setting env variables", priority = 3) + description = "Test service catalog by hot deploying CAapp without Metadata)", priority = 3) + public void testServiceCatalogHotDeploymentWithoutMetaData() + throws IOException, URISyntaxException, AutomationUtilException, InterruptedException { + carbonLogReader.clearLogs(); + File metadataCAPP = new File( + getESBResourceLocation() + File.separator + SERVICE_CATALOG_FOLDER + File.separator + + CAPP_WITHOUT_META); + serverConfigurationManager.copyToCarbonapps(metadataCAPP); + assertTrue(Utils.checkForLog(carbonLogReader, + "Metadata not included, hence not publishing to Service Catalog", 20), + "Did not receive the expected info log"); + serverConfigurationManager.removeFromCarbonapps(CAPP_WITHOUT_META); + } + + @Test(groups = {"wso2.esb"}, + description = "Test service catalog by hot deploying CAapp with Metadata)", priority = 4) + public void testServiceCatalogHotDeploymentWithMetaData() + throws IOException, URISyntaxException, AutomationUtilException, InterruptedException { + carbonLogReader.clearLogs(); + File metadataCAPP = new File( + getESBResourceLocation() + File.separator + SERVICE_CATALOG_FOLDER + File.separator + + CAPP_WITH_META_AND_ENV); + serverConfigurationManager.copyToCarbonapps(metadataCAPP); + assertTrue(Utils.checkForLog(carbonLogReader, + "Successfully updated the service catalog", 20), + "Did not receive the expected info log"); + serverConfigurationManager.removeFromCarbonapps(CAPP_WITH_META_AND_ENV); + } + + @Test(groups = {"wso2.esb"}, + description = "Test service catalog without setting env variables", priority = 5) public void testServiceCatalogMetadataWithoutEnv() throws IOException, AutomationUtilException, InterruptedException { File metadataCAPP = new File( @@ -162,7 +193,7 @@ public void testServiceCatalogMetadataWithoutEnv() } @Test(groups = {"wso2.esb"}, - description = "Test service catalog after setting env variables", priority = 4) + description = "Test service catalog after setting env variables", priority = 6) public void testServiceCatalogMetadataWithEnv() throws IOException, AutomationUtilException, InterruptedException { @@ -186,7 +217,7 @@ public void testServiceCatalogMetadataWithEnv() } @Test(groups = {"wso2.esb"}, - description = "Test the ZIP file created by the service catalog", priority = 5) + description = "Test the ZIP file created by the service catalog", priority = 7) public void testServiceCatalogZipFile() throws CarbonException, FileNotFoundException { File extracted = chekAndExtractPayloadZip(); assertTrue(extracted.exists(), "Error occurred while extracting the ZIP"); @@ -202,7 +233,7 @@ public void testServiceCatalogZipFile() throws CarbonException, FileNotFoundExce } @Test(groups = {"wso2.esb"}, - description = "Test MI is uploading only newly added APIs", priority = 6) + description = "Test MI is uploading only newly added APIs", priority = 8) public void testUploadOnlyNewAPIs() throws CarbonException, IOException, AutomationUtilException, InterruptedException { File newCAPP = new File( @@ -224,7 +255,7 @@ public void testUploadOnlyNewAPIs() } @Test(groups = {"wso2.esb"}, - description = "Test MI is uploading only modified APIs", priority = 7) + description = "Test MI is uploading only modified APIs", priority = 9) public void testUploadOnlyModifiedAPIs() throws CarbonException, IOException, AutomationUtilException, URISyntaxException, InterruptedException { // remove CAPP and add the modified one @@ -248,7 +279,7 @@ public void testUploadOnlyModifiedAPIs() } @Test(groups = {"wso2.esb"}, - description = "Test restart MI without any CAPP changes", priority = 8) + description = "Test restart MI without any CAPP changes", priority = 10) public void testMIRestart() throws IOException, AutomationUtilException, InterruptedException { serverConfigurationManager.applyMIConfigurationWithRestart(new File( getESBResourceLocation() + File.separator + SERVICE_CATALOG_FOLDER + File.separator + "FourthAPI" + @@ -262,7 +293,7 @@ public void testMIRestart() throws IOException, AutomationUtilException, Interru assertFalse(zipFile.exists(), "Payload.zip file should not be created"); } - @Test(groups = {"wso2.esb"}, description = "Test service catalog with proxy services", priority = 9) + @Test(groups = {"wso2.esb"}, description = "Test service catalog with proxy services", priority = 11) public void testServiceCatalogProxyServiceMetadata() throws CarbonException, IOException, AutomationUtilException, InterruptedException { File metadataCAPP = new File(getESBResourceLocation() + File.separator diff --git a/integration/mediation-tests/tests-other/src/test/resources/artifacts/ESB/serviceCatalog/HelloWorldWithoutMetadataCompositeExporter_1.0.0-SNAPSHOT.car b/integration/mediation-tests/tests-other/src/test/resources/artifacts/ESB/serviceCatalog/HelloWorldWithoutMetadataCompositeExporter_1.0.0-SNAPSHOT.car new file mode 100644 index 0000000000..30b95e4ad4 Binary files /dev/null and b/integration/mediation-tests/tests-other/src/test/resources/artifacts/ESB/serviceCatalog/HelloWorldWithoutMetadataCompositeExporter_1.0.0-SNAPSHOT.car differ