From a2cc861b583c7925b9f4fbe9ee15dc22dccf34c9 Mon Sep 17 00:00:00 2001
From: Jamie Anderson
 <127742609+jamie-anderson-sonarsource@users.noreply.github.com>
Date: Thu, 28 Nov 2024 13:02:26 +0000
Subject: [PATCH] APPSEC-1443 Update PostreSQL detection to avoid false
 positives (#467)

---
 .../secrets/configuration/postgresql.yaml     | 323 +++++++++++++-----
 1 file changed, 234 insertions(+), 89 deletions(-)

diff --git a/sonar-text-plugin/src/main/resources/org/sonar/plugins/secrets/configuration/postgresql.yaml b/sonar-text-plugin/src/main/resources/org/sonar/plugins/secrets/configuration/postgresql.yaml
index b841441e..550e16f5 100644
--- a/sonar-text-plugin/src/main/resources/org/sonar/plugins/secrets/configuration/postgresql.yaml
+++ b/sonar-text-plugin/src/main/resources/org/sonar/plugins/secrets/configuration/postgresql.yaml
@@ -5,35 +5,54 @@ provider:
       message: Make sure this PostgreSQL database password gets changed and removed from the code.
   detection:
     post:
-      # Potential FPs here are environment variables, templates and string substitutions, such as
-      # - $my_password, $$my_password
-      # - $db_password
-      # - [mypass123]
-      # - <example-password>
-      # - %s
-      # - {{MyPassword}}
-      # - $(mypassword)
-      # - ${mypassword}, $${mypassword}
-      # - #{mypassword}
-      # - os.getenv("PASS")
-      # - os.environ['PASS']
+      # Potential FPs here are environment variables, templates and string substitutions.
       patternNot:
-        - "(?i)^(\\${1,2}[a-z_]*)?(db|my)?_?pass(word|wd)?"
+        # ://test:test@
+        - "(?i)^test$"
+        # password
+        # db_pass
+        # my_passwd
+        - "(?i)^(db|my)?_?pass(word|wd)?$"
+        # $my_password
+        # $$my_password
+        # $db_password
+        - "(?i)^\\${1,2}\\w*pass(word)?$"
+        # [mypass123]
         - "^\\[[\\w\\t \\-]+\\]$"
+        # <example-password>
         - "^<[\\w\\t -]{1,10}>?"
+        # sprintf("postgres://%s:%s@%s:@s/", ...)
+        # %v
         - "^%[sv]$"
-        - "^\\${1,2}[{(]"
-        - "^\\${1,2}\\w+$"
-        - "\\$\\{\\w+(:-\\w+)?\\}"
-        - "^\\{+[^}]*\\}+$"
+        # $MY_PASSWORD
+        # $MY_PASSWORD$
+        - "^\\${1,2}\\w+\\$?$"
+        # $(MY_PASSWORD)
+        - "^\\$\\([^)]+\\)$"
+        # ${ENV_VAR}
+        # ${ENV_VAR:-default}
+        # ://${self:db.username}:${self:db.password}@
+        - "\\$\\{[^\\}]+\\}"
+        # {password}
+        # {{password}}
+        - "^\\{++[^}]++\\}++$"
+        # #{password}
+        - "^#\\{[^}]++}#?$"
+        # ***
         - "^\\*{3,}$"
-        - "^`[a-z]+"
+        # "postgres://"+user+":"+password+"@"+server+":"+port+"/"
+        - "^\"\\s*\\+[^\"]+\\+\\s*\""
+        # 'postgres://'+user+'+'password+'@'+server+':'+port+'/'
+        - "^'\\s*\\+[^']+\\+\\s*'"
+        # `postgres://`+user+`+`password+`@`+server+`:`+port+`/`
+        - "^`\\s*\\+[^`]+\\+\\s*`"
+        # "postgres://{}:{}@{}:{}/".format(...)
+        # string.Format("postgres://{0}:{1}@{2}:{3}/", ...)
+        - "^\\{\\d*\\}$"
+        # os.environ['DB_PASSWORD']
         - "\\b(get)?env(iron)?\\b"
-        - "^None$"
-        - "^\\${1,2}[a-z_]+pass(word)?$"
-        - "^process\\.env\\."
-        - "^\\{{2,}"
-        - "^#\\{[^}]++}#?$"
+        # `cat /dev/urandom | ...`
+        - "(?i)^`[\\w.\\-/]+\b"
 
   rules:
     # Note:
@@ -71,10 +90,14 @@ provider:
           # Certain special characters need to be percent-encoded so we can break matching when we find them.
           #
           # A negative lookbehind is added to avoid regex-based false positives.
-          pattern: "(?is)\
-                    (?<!regex.{1,50})\
-                    \\bpostg(?:res(?:ql)?|is)(?:\\+\\w+)?://[^:@/ ]++:([^@/ ]++)@"
+          pattern: "(?is)\\bpostg(?:res(?:ql)?|is)(?:\\+\\w+)?://[^:@/ ]++:([^@/ ]++)@"
+          context:
+            matchNot:
+              patternBefore:
+                pattern: (?i)regex
+                maxCharDistance: 50
       examples:
+        # From the RSPEC
         - text: |
             # Noncompliant code example
             uri = "postgres://foouser:foopass@example.com/testdb"
@@ -88,13 +111,38 @@ provider:
             password = os.environ["PG_PASSWORD"]
             uri      = f"postgres://{user}:{password}@example.com/testdb"
           containsSecret: false
+        # True positive matches
         - text: |
-            Hanami::Model.configure do
-              pass    = CGI.escape("#{$env.db_password}").gsub('+', '%20')
-              str_cnn = "postgres://#{$env.db_username}:#{pass}@#{$env.db_host}:#{$env.db_port}/#{$env.db_name}?max_connections=#{$env.db_max_connections}"
-              adapter :sql, str_cnn
-            end
+            SqlSettingsDefaultDataSource = "postgres://mmuser:mostest@localhost/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes"
+          containsSecret: true
+          match: mostest
+        - text: |
+            passwordFile := fs.String("password", "../../../tools/secrets/password.txt", "password file")
+            databasePrefix := fs.String("database-prefix", "postgres://postgres:postgres_password_padded_for_security@localhost:5432/ocr2vrf-test", "database prefix")
+            databaseSuffixes := fs.String("database-suffixes", "sslmode=disable", "database parameters to be added")
+          containsSecret: true
+          match: postgres_password_padded_for_security
+        - text: |
+            driver: postgres
+            dsn: postgres://foouser:foopass@localhost:5432/testdb?sslmode=disable
+            table: footable
+          containsSecret: true
+          match: foopass
+        - text: |
+            AIRFLOW_CONN_METADATA_DB=postgres+psycopg2://airflow:airflow@postgres:5432/airflow
+            AIRFLOW_VAR__METADATA_DB_SCHEMA=airflow
+          containsSecret: true
+          match: airflow
+        - text: |
+            let urlstr = "postgres://postgres:password!12345@nas.mynetwork.lan:55432/conn_test".to_owned();
+          containsSecret: true
+          match: password!12345
+        # Exclusions due to preconditions
+        - text: |
+            SqlSettingsDefaultDataSource = "postgres://mmuser:mostest@localhost/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes"
+          fileName: Doc.md
           containsSecret: false
+        # Exclusions due to context
         - text: |
               Regex regex("(postgis://.*:)(.*)(@.*)");
           containsSecret: false
@@ -103,11 +151,28 @@ provider:
                   "(postgres://.*:)(.*)(@.*)"
               );
           containsSecret: false
+        # Exclusions due to postconditions
+        - text: |
+            def test_it_returns_the_default_database_url
+              assert_equal "postgres://test:test@#{@host}:#{@port}/test", @container.database_url
+            end
+          containsSecret: false
         - text: |
             backend "pg" {
               conn_str = "postgres://user:pass@database.example/terraform_backend"
             }
           containsSecret: false
+        - text: |
+            backend "pg" {
+              conn_str = "postgres://$DB_USER$:$DB_PASS$@database.example/terraform_backend"
+            }
+          containsSecret: false
+        - text: |
+            DATABASE_URL="postgresql://postgres:[YOUR-PASSWORD]@db.uzapyutenknggogexmmw.supabase.co:5432/postgres"
+          containsSecret: false
+        - text: |
+            "postgres://<user>:<password>@<host>/<database>"
+          containsSecret: false
         - text: |
             connStr = fmt.Sprintf("postgres://%s:%s@:%s/%s%ssslmode=%s&host=%s",
               url.PathEscape(dbUser), url.PathEscape(dbPasswd), port, dbName, dbParam, dbsslMode, host)
@@ -115,6 +180,20 @@ provider:
         - text: |
             url := fmt.Sprintf("postgres://%v:%v@%v:%v/%v", user, pass, host, port, name)
           containsSecret: false
+        - text: |
+            Hanami::Model.configure do
+              pass    = CGI.escape("#{$env.db_password}").gsub('+', '%20')
+              str_cnn = "postgres://#{$env.db_username}:#{pass}@#{$env.db_host}:#{$env.db_port}/#{$env.db_name}?max_connections=#{$env.db_max_connections}"
+              adapter :sql, str_cnn
+            end
+          containsSecret: false
+        - text: |
+            spec:
+              containers:
+                - env:
+                  - name: POSTGRES_CONNECTION_URI
+                    value: "postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(POSTGRES_SERVER):$(POSTGRES_PORT)/$(POSTGRES_DATABASE)"
+          containsSecret: false
         - text: |
             if (!newConfig.DATABASE_URL) {
               const encodedUser = encodeURIComponent(newConfig.POSTHOG_DB_USER)
@@ -123,32 +202,84 @@ provider:
             }
           containsSecret: false
         - text: |
-            SqlSettingsDefaultDataSource = "postgres://mmuser:mostest@localhost/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes"
-          containsSecret: true
-          match: mostest
+            export const dbConnection = `postgres://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}/${process.env.DB_NAME}`
+          containsSecret: false
         - text: |
-            SqlSettingsDefaultDataSource = "postgres://mmuser:mostest@localhost/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes"
-          fileName: Doc.md
+            DATABASE_URL: "postgresql://${self:custom.db_credentials.username}:${self:custom.db_credentials.password}@${self:custom.db_host}:${self:custom.db_port}/blink?schema=public"
           containsSecret: false
         - text: |
-            passwordFile := fs.String("password", "../../../tools/secrets/password.txt", "password file")
-            databasePrefix := fs.String("database-prefix", "postgres://postgres:postgres_password_padded_for_security@localhost:5432/ocr2vrf-test", "database prefix")
-            databaseSuffixes := fs.String("database-suffixes", "sslmode=disable", "database parameters to be added")
-          containsSecret: true
-          match: postgres_password_padded_for_security
+            services:
+              w3bapp:
+                environment:
+                  SRV_APPLET_MGR__Postgres_Master: postgresql://${POSTGRES_USER:-w3badmin}:${POSTGRES_PASSWORD:-PaSsW0Rd}@postgres:5432/${POSTGRES_DB:-w3bstream}?sslmode=disable&application_name=mgr
+                  SRV_APPLET_MGR__Postgres_ConnMaxLifetime: 10m
+                  SRV_APPLET_MGR__Postgres_PoolSize: 5
+          # The password in this format is handled by the rule "postgres-url-password-from-env-with-default"
+          containsSecret: false
         - text: |
-            driver: postgres
-            dsn: postgres://foouser:foopass@localhost:5432/testdb?sslmode=disable
-            table: footable
-          containsSecret: true
-          match: foopass
+            string url = $"postgresql://{user}:{password}@{host}/{database}";
+          containsSecret: false
         - text: |
-            AIRFLOW_CONN_METADATA_DB=postgres+psycopg2://airflow:airflow@postgres:5432/airflow
-            AIRFLOW_VAR__METADATA_DB_SCHEMA=airflow
-          containsSecret: true
-          match: airflow
+            @property
+            def database_url(self) -> Optional[PostgresDsn]:
+                return (
+                    f"postgresql+asyncpg://{self.POSTGRES_USER}:{self.POSTGRES_PASSWORD}@"
+                    f"{self.POSTGRES_HOST}:{self.POSTGRES_PORT}/{self.POSTGRES_DB}"
+                )
+          containsSecret: false
         - text: |
-            export const dbConnection = `postgres://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}/${process.env.DB_NAME}`
+            const database =
+              process.env.NODE_ENV === "test"
+                ? process.env.POSTGRES_DB_TEST
+                : process.env.POSTGRES_DB;
+
+            const connectionString = `postgresql://${process.env.POSTGRES_USER}:${process.env.POSTGRES_PASSWORD}@${process.env.POSTGRES_HOST}:${process.env.POSTGRES_PORT}/${database}`;
+          containsSecret: false
+        - text: |
+            engine = create_engine(
+                f"postgresql://neylsoncrepalde:{os.environ['PGPASS']}@database-igti.cfowiwu0gidv.us-east-2.rds.amazonaws.com:5432/postgres"
+            )
+          containsSecret: false
+        - text: |
+            "postgresql://{{user}}:{{password}}@{{host}}/{{database}}"
+          containsSecret: false
+        - text: |
+            client.secrets.database.configure(
+                name='db-connection-name',
+                plugin_name='postgresql-database-plugin',
+                allowed_roles='role-name',
+                connection_url=f'postgresql://{{{{username}}}}:{{{{password}}}}@postgres:5432/postgres?sslmode=disable',
+                username='db-username',
+                password='db-password',
+            )
+          containsSecret: false
+        - text: |
+            "postgresql+psycopg2://scott123:*****@host.name.com/dbname"
+          containsSecret: false
+        - text: |
+            dbconn="postgresql+psycopg2://"+str(conn_login)+":"+str(conn_password)+"@"+str(conn_host)+":"+str(conn_port)+"/"+str(conn_schema)
+          containsSecret: false
+        - text: |
+            "p_engine = create_engine('postgresql://'+p_username+':'+p_pwd+'@'+p_host+':'+str(p_port)+'/'+p_dbname)\n",
+          containsSecret: false
+        - text: |
+            # postgresql://username:password@host:port/database
+            conn_string = "postgresql://{}:{}@{}:{}/{}" \
+                .format(DB_USER, DB_PASSWORD, DB_ENDPOINT, DB_PORT, DB)
+          containsSecret: false
+        - text: |
+            # Connect to Postgres DB
+            self.engine = create_engine('postgresql+psycopg2://{0}:{1}@localhost/cirtkit'.format(DB_USER, DB_PASSWD))
+          containsSecret: false
+        - text: |
+            export const PostgresUrl: DbEnvUrl = {
+              envVar: "POSTGRES_URL",
+              url:
+                `postgresql://${DefaultDbConnection.username}:` +
+                `${DefaultDbConnection.password}@${DefaultDbConnection.host}:` +
+                `${DefaultDbConnection.port}/${DefaultDbConnection.database}` +
+                `?schema=${process.env.DB_SCHEMA}`
+            };
           containsSecret: false
 
     - id: postgres-url-password-from-env-with-default
@@ -226,11 +357,11 @@ provider:
             paths:
               - "**/appsettings.Development.json"
               - "**/appsettings.Local.json"
-
         matching:
           # looks for occurrences of the string `PG_PASSWORD=` followed by a value that is not enclosed in single or double quotes.
-          pattern: "\\bPG_PASSWORD=(?!\\\\?[\"'])([^\\s]+)(?:$|\\s)"
+          pattern: "\\bPG_PASSWORD=(?!\\\\?[\"'`]|\\$\\()([^\\s]+)(?:$|\\s)"
       examples:
+        # From the RSPEC
         - text: |
             PG_USER=postgres
             PG_PASSWORD=password
@@ -242,37 +373,44 @@ provider:
             - PG_PASSWORD=<enter password>
             - PG_DB=postgres
           containsSecret: false
+        # True positive matches
         - text: |
-            PG_USER=$DB_USER
-            PG_PASSWORD=$DB_PASS
-            PG_PORT=5436
-          containsSecret: false
-        - text: |
-            PG_PASSWORD=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 10 | head -n 1)
-          containsSecret: false
-        - text: |
-            PG_USER=${PG_USER}
-            PG_PASSWORD=${PG_PASSWORD}
-            PG_DATABASE=${PG_DATABASE}
-          containsSecret: false
+            PG_HOST=localhost
+            PG_PASSWORD=P@ssw0rd
+            PG_USER=postgres
+          containsSecret: true
+          match: P@ssw0rd
         - text: |
             if [ "${PG_PASSWORD-}" == "" ]; then
               PG_PASSWORD=dev_DVwgY7H5p3QgiZQr3tCo5X
             fi
           containsSecret: true
           match: dev_DVwgY7H5p3QgiZQr3tCo5X
+        # True negative matches
+        - text: |
+            PG_PASSWORD=`cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 10 | head -n 1`
+          containsSecret: false
+        - text: |
+            PG_PASSWORD=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 10 | head -n 1)
+          containsSecret: false
+        # Exclusions due to preconditions
         - text: |
             if [ "${PG_PASSWORD-}" == "" ]; then
               PG_PASSWORD=dev_DVwgY7H5p3QgiZQr3tCo5X
             fi
           fileName: Doc.adoc
           containsSecret: false
+        # Exclusions due to postconditions
         - text: |
-            PG_HOST=localhost
-            PG_PASSWORD=P@ssw0rd
-            PG_USER=postgres
-          containsSecret: true
-          match: P@ssw0rd
+            PG_USER=$DB_USER
+            PG_PASSWORD=$DB_PASS
+            PG_PORT=5436
+          containsSecret: false
+        - text: |
+            PG_USER=${PG_USER}
+            PG_PASSWORD=${PG_PASSWORD}
+            PG_DATABASE=${PG_DATABASE}
+          containsSecret: false
         - text: |
             PG_USER=env("PG_USER")
             PG_PASSWORD=env("PG_PASSWORD")
@@ -311,10 +449,10 @@ provider:
             paths:
               - "**/appsettings.Development.json"
               - "**/appsettings.Local.json"
-
         matching:
-          pattern: "\\bPG_PASSWORD=\\\\?[\"']([^\\r\\n\"']+)\\\\?[\"']"
+          pattern: "\\bPG_PASSWORD=\\\\?[\"'](?!\\$\\()([^\\r\\n\"']+)\\\\?[\"']"
       examples:
+        # From the RSPEC
         - text: |
             PG_USER="postgres"
             PG_PASSWORD="password"
@@ -326,37 +464,44 @@ provider:
             - PG_PASSWORD="<enter password>"
             - PG_DB="postgres"
           containsSecret: false
+        # True positive matches
         - text: |
-            PG_USER="$DB_USER"
-            PG_PASSWORD="$DB_PASS"
-            PG_PORT="5436"
-          containsSecret: false
-        - text: |
-            PG_PASSWORD="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 10 | head -n 1)"
-          containsSecret: false
-        - text: |
-            PG_USER="${PG_USER}"
-            PG_PASSWORD="${PG_PASSWORD}"
-            PG_DATABASE="${PG_DATABASE}"
-          containsSecret: false
+            PG_HOST="localhost"
+            PG_PASSWORD="P@ssw0rd"
+            PG_USER="postgres"
+          containsSecret: true
+          match: P@ssw0rd
         - text: |
             if [ "${PG_PASSWORD-}" == "" ]; then
               PG_PASSWORD="dev_DVwgY7H5p3QgiZQr3tCo5X"
             fi
           containsSecret: true
           match: dev_DVwgY7H5p3QgiZQr3tCo5X
+        # True negative matches
+        - text: |
+            PG_PASSWORD=`cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 10 | head -n 1`
+          containsSecret: false
+        - text: |
+            PG_PASSWORD="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 10 | head -n 1)"
+          containsSecret: false
+        # Exclusions due to preconditions
         - text: |
             if [ "${PG_PASSWORD-}" == "" ]; then
               PG_PASSWORD="dev_DVwgY7H5p3QgiZQr3tCo5X"
             fi
           fileName: Doc.example
           containsSecret: false
+        # Exclusions due to postconditions
         - text: |
-            PG_HOST="localhost"
-            PG_PASSWORD="P@ssw0rd"
-            PG_USER="postgres"
-          containsSecret: true
-          match: P@ssw0rd
+            PG_USER="$DB_USER"
+            PG_PASSWORD="$DB_PASS"
+            PG_PORT="5436"
+          containsSecret: false
+        - text: |
+            PG_USER="${PG_USER}"
+            PG_PASSWORD="${PG_PASSWORD}"
+            PG_DATABASE="${PG_DATABASE}"
+          containsSecret: false
         - text: |
             PG_USER=env("PG_USER")
             PG_PASSWORD=env("PG_PASSWORD")