diff --git a/.secrets.baseline b/.secrets.baseline index 2bcb69d..15cbb29 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "go.sum|^.secrets.baseline$", "lines": null }, - "generated_at": "2024-05-07T10:25:46Z", + "generated_at": "2024-05-07T12:00:08Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -82,7 +82,7 @@ "hashed_secret": "bbc4e9d52252171a3a306be55086c65b126189e8", "is_secret": false, "is_verified": false, - "line_number": 35, + "line_number": 38, "type": "Secret Keyword", "verified_result": null }, @@ -90,7 +90,7 @@ "hashed_secret": "d9e9019d9eb455a3d72a3bc252c26927bb148a10", "is_secret": false, "is_verified": false, - "line_number": 52, + "line_number": 55, "type": "Secret Keyword", "verified_result": null }, @@ -98,7 +98,7 @@ "hashed_secret": "b13d7622394e85c3b2694f426bc096b093764462", "is_secret": false, "is_verified": false, - "line_number": 56, + "line_number": 59, "type": "Secret Keyword", "verified_result": null } diff --git a/README.md b/README.md index 48bc4ea..9ac951e 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,45 @@ -# Retrieval Augmented Generation (RAG) stack +# Retrieval Augmented Generation Pattern for Watsonx on IBM Cloud -To run the full stack, follow these steps. These steps will be updated as development progresses on the stack and underlying DAs. +The following [deployable architecture](https://cloud.ibm.com/docs/secure-enterprise?topic=secure-enterprise-understand-module-da#what-is-da) automates the deployment of a sample GenAI Pattern on IBM Cloud, including all underlying infrastructure. This architecture implements the best practices for Watsonx GenAI Pattern deployment on IBM Cloud, as described in the [reference architecture](https://cloud.ibm.com/docs/pattern-genai-rag?topic=pattern-genai-rag-genai-pattern). -## 1. Deploy the stack in a new project from catalog +# Deployment Details -Catalog url: https://cloud.ibm.com/catalog/7df1e4ca-d54c-4fd0-82ce-3d13247308cd/architecture/Retrieval_Augmented_Generation_Pattern-5fdd0045-30fc-4013-a8bc-6db9d5447a52?bss_account=9f9af00a96104f49b6509aa715f9d6a5 +To run the full stack, follow these steps. These steps will be updated as development progresses on the stack and underlying deployable architectures. -Click the "Add to project" button, and select create in new project. +## 1. Deploy the Stack in a New Project from Catalog -## 2. Prereqs in target account +Catalog URL: https://cloud.ibm.com/catalog/7df1e4ca-d54c-4fd0-82ce-3d13247308cd/architecture/Retrieval_Augmented_Generation_Pattern-5fdd0045-30fc-4013-a8bc-6db9d5447a52?bss_account=9f9af00a96104f49b6509aa715f9d6a5 + +Click the "Add to Project" button and select "Create in new project." + +## 2. Prerequisites in Target Account Before deploying the stack, ensure you have: -- Created an API key in the target account with sufficient permissions. Note the API key, as it will be used later. -- For now, grant it admin privileges. The exact permissions required will be refined in future versions. -- Install the IBM Cloud CLI's Project addon using `ibmcloud plugin install project` command. More info here: https://cloud.ibm.com/docs/cli?topic=cli-projects-cli +* Created an API key in the target account with sufficient permissions. Note the API key, as it will be used later. For now, grant it admin privileges. The exact permissions required will be refined in future versions. +* Installed the IBM Cloud CLI's Project add-on using the `ibmcloud plugin install project` command. More information is available here: https://cloud.ibm.com/docs/cli?topic=cli-projects-cli -## 3. Set the input configuration for the stack +## 3. Set the Input Configuration for the Stack -- Clone this repository locally. -- Create a file with name ".def.json" with the following content. +* Clone this repository locally. +* Create a file named ".def.json" with the following content: **Important**: -- Ensure region is either us-south or eu-de as watsonx can only be deployed in those 2 locations for now. -- Ensure that the prefix is globally unique. It is used for the container registry namespace (which needs to be globally unique) in this alpha version. -- The signing key is the base64 key obtained from the `gpg --export-secret-key | base64` command. See https://cloud.ibm.com/docs/devsecops?topic=devsecops-devsecops-image-signing#cd-devsecops-gpg-export for details. -- If specifying `existing_secrets_manager_crn`, the ibmcloud_api_key that is passed as an input must have the documented read and write access to the instance -- If specifying `existing_secrets_manager_crn`, ensure that the default security group does not contain secrets named `signing-key` and `ibmcloud-api-key` . The RAG DA currently always attempt to create secret with those names (temporary issue - to be fixed). +* Ensure the region is either us-south or eu-de, as Watsonx can only be deployed in those two locations for now. +* Ensure that the prefix is globally unique. It is used for the container registry namespace (which needs to be globally unique) in this alpha version. +* If specifying `existing_secrets_manager_crn`, the `ibmcloud_api_key` that is passed as an input must have the documented read and write access to the instance. +* If specifying `existing_secrets_manager_crn`, ensure that the default security group does not contain secrets named `signing-key` and `ibmcloud-api-key`. The RAG DA currently always attempts to create a secret with those names (temporary issue - to be fixed). +* The signing key is the base64 key obtained from the `gpg --export-secret-key | base64` command. See https://cloud.ibm.com/docs/devsecops?topic=devsecops-devsecops-image-signing#cd-devsecops-gpg-export for details. ```json { "inputs": { "prefix": "", "ibmcloud_api_key": "", - "resource_group_name": "", + "resource_group_name": "", "region": "", "sample_app_git_url": "https://github.com/IBM/gen-ai-rag-watsonx-sample-application", - "watsonx_admin_api_key": "", + "watsonx_admin_api_key": "", "signing_key": "signing key used to sign build artifacts", "existing_secrets_manager_crn": " - reuse an existing secret manager instance", "enable_platform_logs_metrics": " - set to true to enable observability instance to capture regional logs" @@ -53,7 +56,7 @@ Example: "resource_group_name": "stack-service-rg", "region": "eu-de", "sample_app_git_url": "https://github.com/IBM/gen-ai-rag-watsonx-sample-application", - "watsonx_admin_api_key": "", + "watsonx_admin_api_key": "", "signing_key": "signing key used to sign build artifacts", "enable_platform_logs_metrics": "false", "existing_secrets_manager_crn": "crn:v1:bluemix:public:secrets-manager:us-south:a/190c293e9fda4c6684b5acf4b17871b8:14580411-4fa2-42d3-af3f-ab7fc6371b6d::" @@ -64,20 +67,20 @@ Example: ## 4. Run ./deploy-many.sh -- Ensure you are login into the account containing the Cloud project with the stack using ibmcloud login --sso -- Execute ./deploy-many.sh with project name, stack name and optional configuration name pattern. The selected non-stack configruations will be processed by their name in alphabetical order. Using configuration name pattern (regex can be used - make sure to enclose it in quotes) you can chose which configurations are deployed +* Ensure you are logged in to the account containing the Cloud project with the stack using `ibmcloud login --sso`. +* Execute `./deploy-many.sh` with the project name, stack name, and optional configuration name pattern. The selected non-stack configurations will be processed by their name in alphabetical order. Using the configuration name pattern (regex can be used - make sure to enclose it in quotes), you can choose which configurations are deployed. -Example 1 - update stack inputs for stack configuration `RAG` and process all non-stack configurations in the project: +Example 1 - Update stack inputs for stack configuration `RAG` and process all non-stack configurations in the project: ```bash ./deploy-many.sh my-test-project RAG ``` -Example 2 - update stack inputs and process some configurations in the project: +Example 2 - Update stack inputs and process some configurations in the project: ```bash ./deploy-many.sh my-test-project RAG 'RAG-1|RAG-4|RAG-5' ``` -Example 3 - simulate updating stack inputs and validating some configurations in the project in dry-run mode (no changes or actual validation or deployments is done): +Example 3 - Simulate updating stack inputs and validating some configurations in the project in dry-run mode (no changes or actual validation or deployments are done): ```bash DRY_RUN=true ./deploy-many.sh my-test-project RAG 'RAG-1|RAG-4|RAG-5' ``` diff --git a/common-dev-assets b/common-dev-assets index 0a4ea52..22ef5be 160000 --- a/common-dev-assets +++ b/common-dev-assets @@ -1 +1 @@ -Subproject commit 0a4ea52a38f1c23f6476d5f57c6813b44cf42f4f +Subproject commit 22ef5be99c68bcaf173da6ed9b7e3f9ae4cd3800 diff --git a/ibm_catalog.json b/ibm_catalog.json index 395de68..bcfddb6 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -14,7 +14,7 @@ "watson", "ai" ], - "short_description": "An automated solution that deploys a sample application illustrating how to implement the RAG Pattern with watsonx.ai and IBM Cloud services.", + "short_description": "An automated solution that deploys a sample application illustrating how to implement the RAG Pattern with watson.ai and IBM Cloud services.", "offering_icon_url": "https://globalcatalog.cloud.ibm.com/api/v1/1082e7d2-5e2f-0a11-a3bc-f88a8e1931fc/artifacts/solution.svg", "flavors": [ { @@ -104,7 +104,7 @@ "architecture": { "features": [ { - "title": "Deploy a customer care generative AI app to Code Engine using Continous Delivery", + "title": "Deploy a banking retrieval augmented generation (RAG) app to IBM Cloud Code Engine using Continous Delivery.", "description": "" } ], @@ -133,30 +133,28 @@ { "key": "prefix", "type": "string", - "default_value": "rag", - "description": "A prefix added to the name of all resources created by this solution. Used to avoid name clashes in the target account.", + "default_value": "sample", + "description": "A prefix added to the name of all resources created by this solution. Used to avoid name clashes in the target account when existing this solution multiple times.", "required": true }, - { - "key": "enable_platform_logs_metrics", - "type": "boolean", - "default_value": false, - "description": "Whether to provision logging and monitoring instances are configured to receive all platform logs and metrics in the target region. There can only be one instance per region provisioned for platform logs/metrics.", - "required": false - }, - { - "key": "existing_secrets_manager_crn", - "type": "string", - "default_value": "__NULL__", - "description": "The CRN of an existing secret manager instance to use in this solution. If not set, a new secret manager instance is provisioned. ", - "required": false - }, { "key": "ibmcloud_api_key", "type": "password", "description": "The API Key used to provision all resources created in this solution.", "required": true }, + { + "key": "signing_key", + "type": "password", + "description": "The key used to sign the application image built by the CI pipeline deployed in this solution. Please refer to the documentation for details on generating the key.", + "display_name": "Multiline secure value", + "required": true, + "custom_config": { + "type": "multiline_secure_value", + "grouping": "deployment", + "original_grouping": "deployment" + } + }, { "key": "region", "type": "string", @@ -177,15 +175,14 @@ { "key": "resource_group_name", "type": "string", - "default_value": "rag-services-rc", + "default_value": "rag-services", "description": "The name of the resource group that is created by this solution. The actual name is prefixed with the value of the input 'prefix'. All resources created by this solution are deployed in this resource group. ", "required": false }, { - "key": "sample_app_git_url", - "type": "string", - "default_value": "https://github.com/IBM/gen-ai-rag-watsonx-sample-application", - "description": "The URL to the public git repository containing the sample rag application code.", + "key": "watsonx_admin_api_key", + "type": "password", + "description": "The API Key used to provision the watson project resources. If not set, the ibmcloud_api_key is used.", "required": false }, { @@ -206,15 +203,24 @@ ] }, { - "key": "signing_key", - "type": "password", - "description": "The key used to sign the application image built by the CI pipeline deployed in this solution.", + "key": "enable_platform_logs_metrics", + "type": "boolean", + "default_value": false, + "description": "Whether to provision logging and monitoring instances are configured to receive all platform logs and metrics in the target region. There can only be one instance per region provisioned for platform logs/metrics.", "required": false }, { - "key": "watsonx_admin_api_key", - "type": "password", - "description": "The API Key used to provision the watson project resources. If not set, the ibmcloud_api_key is used.", + "key": "existing_secrets_manager_crn", + "type": "string", + "default_value": "__NULL__", + "description": "The CRN of an existing secret manager instance to use in this solution. If not set, a new secret manager instance is provisioned. ", + "required": false + }, + { + "key": "sample_app_git_url", + "type": "string", + "default_value": "https://github.com/IBM/gen-ai-rag-watsonx-sample-application", + "description": "The URL to the public git repository containing the sample rag application code.", "required": false } ], diff --git a/stack_definition.json b/stack_definition.json index bc1217d..2908eac 100644 --- a/stack_definition.json +++ b/stack_definition.json @@ -55,10 +55,9 @@ }, { "name": "signing_key", - "required": false, + "required": true, "type": "password", - "hidden": false, - "default": "replace" + "hidden": false }, { "name": "existing_secrets_manager_crn", @@ -138,6 +137,10 @@ "name": "ibmcloud_api_key", "value": "ref:../../inputs/ibmcloud_api_key" }, + { + "name": "region", + "value": "ref:../../inputs/region" + }, { "name": "region", "value": "ref:../../inputs/region" @@ -175,6 +178,10 @@ { "name": "existing_secrets_manager_crn", "value": "ref:../../inputs/existing_secrets_manager_crn" + }, + { + "name": "service_plan", + "value": "ref:../../inputs/secret_manager_service_plan" } ] }, @@ -214,7 +221,7 @@ }, { "name": "3 - Observability - Logging Monitoring Activity Tracker", - "version_locator": "7df1e4ca-d54c-4fd0-82ce-3d13247308cd.58843031-95a7-4e8e-9abc-13c478a8bd16", + "version_locator": "7df1e4ca-d54c-4fd0-82ce-3d13247308cd.3ae7c6ae-20c2-4214-b3b9-7110356b4b6c", "inputs": [ { "name": "ibmcloud_api_key", @@ -252,7 +259,7 @@ }, { "name": "4 - WatsonX SaaS services", - "version_locator": "8bfb1293-8b85-4d3f-a89f-015d0a0719df.02313717-33ba-43cd-8a6d-c661c0538cf3", + "version_locator": "8bfb1293-8b85-4d3f-a89f-015d0a0719df.3b21f2da-e498-4c35-bac2-47d25c7cfa55", "inputs": [ { "name": "ibmcloud_api_key", @@ -314,7 +321,7 @@ }, { "name": "toolchain_name", - "value": "Generative AI Sample App" + "value": "RAG Sample App" }, { "name": "toolchain_region", diff --git a/tests/pr_test.go b/tests/pr_test.go index 83258df..968c0c7 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -19,7 +19,7 @@ func TestProjectsFullTest(t *testing.T) { "2c - Security Service - Security Compliance Center", "3 - Observability - Logging Monitoring Activity Tracker", "4 - WatsonX SaaS services", - "5 - Generative AI Sample App - Code Engine Toolchain Config", + "5 - RAG Sample App - Code Engine Toolchain Config", "6 - Sample RAG app configuration", }, })