Merge branch 'v2.1.x' into develop

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 Shared Pipes Component
 
+## v2.1.1 [2025-02-11]
+* Get credentials from the platform-default native credentials store for turbot-hosted private plugins. ([#643](https://github.com/turbot/pipe-fittings/issues/643))
+
+## v2.1.0 [2025-02-11]
+* TailpipeConnection now supports indexes/partitions in addition to to/from. ([#632](https://github.com/turbot/pipe-fittings/issues/632)).
+* Move `secrets` package from steampipe-plugin-code.
+
 ## v2.0.1 [2025-02-02]
 
 * Add backend.NameFromConnectionString. ([#631](https://github.com/turbot/pipe-fittings/issues/631)).

ociinstaller/oci_downloader.go

@@ -77,8 +77,8 @@ func (o *OciDownloader[I, C]) Pull(ctx context.Context, ref string, mediaTypes [
 		return nil, nil, nil, nil, err
 	}
 
-	// Get credentials from the docker credentials store
-	storeOpts := credentials.StoreOptions{}
+	// Get credentials from the platform-default native credentials store
+	storeOpts := credentials.StoreOptions{DetectDefaultNativeStore: true}
 	var credStore *credentials.DynamicStore
 	if strings.HasPrefix(ref, o.baseImageRef) {
 		credStore, err = credentials.NewStore("", storeOpts)

secrets/aws_iam_access_key.go

@@ -0,0 +1,103 @@
+package secrets
+
+import (
+	"regexp"
+	"slices"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/sts"
+)
+
+func init() {
+	RegisterMatcher(&awsAccessKeyID{})
+}
+
+type awsAccessKeyID struct{}
+
+func (*awsAccessKeyID) Type() string {
+	return "aws_access_key_id"
+}
+
+// https://aws.amazon.com/blogs/security/a-safer-way-to-distribute-aws-credentials-to-ec2/
+
+// grep -RP '(?<![A-Z0-9])[A-Z0-9]{20}(?![A-Z0-9])' *
+// grep -RP '(?<![A-Za-z0-9/+=])[A-Za-z0-9/+=]{40}(?![A-Za-z0-9/+=])' *
+
+func (*awsAccessKeyID) DenyList() []*regexp.Regexp {
+	return []*regexp.Regexp{
+		// Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
+		regexp.MustCompile("(?m)((AKIA|ABIA|ACCA|AGPA|AIDA|AIPA|ANPA|ANVA|APKA|AROA|ASCA|ASIA)[0-9A-Z]{16})"),
+	}
+}
+
+func (*awsAccessKeyID) Authenticate(secret string, src string) (AuthenticatedResult, error) {
+	//  This examines the variable name to identify AWS secret tokens.
+	//  The order is important since we want to prefer finding `AKIA`-based
+	//  keys (since they can be tested), rather than the secret tokens.
+	// re := regexp.MustCompile("(?m)aws.{0,20}?['\"]([0-9a-zA-Z/+]{40})['\"]")
+	re := regexp.MustCompile(`(?m)([0-9a-zA-Z\/+]{40})`)
+
+	matchGroups := re.FindAllStringSubmatchIndex(src, -1)
+	var secrets = make([]string, 0)
+	for _, m := range matchGroups {
+		var startOffset, endOffset int
+		if len(m) > 2 {
+			// If the regexp in the secret matcher has a match group, then use it
+			// as the "secret" from the string. For example "user:(secret)".
+			startOffset = m[2]
+			endOffset = m[3]
+		} else {
+			// If the regexp has no match group, then use the full match as the secret.
+			// e.g. "tok-[a-z]+"
+			startOffset = m[0]
+			endOffset = m[1]
+		}
+
+		secret_key := src[startOffset:endOffset]
+		secrets = append(secrets, secret_key)
+	}
+
+	if len(secrets) == 0 {
+		return NOT_IMPLEMENTED, nil
+	}
+
+	for _, secret_key := range secrets {
+		sessionOptions := session.Options{
+			Config: aws.Config{
+				Region:      aws.String("us-east-1"),
+				MaxRetries:  aws.Int(0),
+				Credentials: credentials.NewStaticCredentials(secret, secret_key, ""),
+			},
+		}
+
+		sess, err := session.NewSessionWithOptions(sessionOptions)
+		if err != nil {
+			return nil, err
+		}
+		svc := sts.New(sess)
+
+		callerIdentity, err := svc.GetCallerIdentity(&sts.GetCallerIdentityInput{})
+		if err != nil {
+			if awsErr, ok := err.(awserr.Error); ok {
+				// Valid means keys match the regex
+				// SignatureDoesNotMatch - In case Access key is invalid but secret key is not is correct
+				// IncompleteSignature - When the access key is invalid
+				// InvalidClientTokenId - When access key and secret key are valid but expired
+				if slices.Contains([]string{"SignatureDoesNotMatch", "IncompleteSignature", "InvalidClientTokenId"}, awsErr.Code()) {
+					return UNAUTHENTICATED, nil
+				}
+			}
+		}
+
+		if callerIdentity != nil {
+			if callerIdentity.Account != nil {
+				return AUTHENTICATED, nil
+			}
+		}
+	}
+
+	return NOT_IMPLEMENTED, nil
+}

secrets/azure_storage_key.go

@@ -0,0 +1,26 @@
+package secrets
+
+import (
+	"regexp"
+)
+
+func init() {
+	RegisterMatcher(&azureStorageAccountKey{})
+}
+
+type azureStorageAccountKey struct{}
+
+func (*azureStorageAccountKey) Type() string {
+	return "azure_storage_account_key"
+}
+
+func (*azureStorageAccountKey) DenyList() []*regexp.Regexp {
+	return []*regexp.Regexp{
+		regexp.MustCompile(`AccountKey=[a-zA-Z0-9+/=]{88}`),
+		regexp.MustCompile(`[a-zA-Z0-9+/=]{88}`),
+	}
+}
+
+func (*azureStorageAccountKey) Authenticate(secret string, src string) (AuthenticatedResult, error) {
+	return NOT_IMPLEMENTED, nil
+}

secrets/basic_auth.go

@@ -0,0 +1,46 @@
+package secrets
+
+import (
+	"fmt"
+	"regexp"
+)
+
+func init() {
+	RegisterMatcher(&basicAuth{})
+}
+
+type basicAuth struct{}
+
+func (*basicAuth) Type() string {
+	return "basic_auth"
+}
+
+// These characters are derived from RFC 3986 Section 2.2.
+//
+// We don't expect any of these delimiter characters to appear in
+// the username/password component of the URL, seeing that this would probably
+// result in an unexpected URL parsing (and probably won't even work).
+
+func basicAuthReservedCharacters() string {
+	return `:/?#[]@`
+}
+
+func basicAuthSubDelimiterCharacters() string {
+	return `!$&\'()*+,;=`
+}
+
+func basicAuthRegexString() string {
+	allChars := basicAuthReservedCharacters() + basicAuthSubDelimiterCharacters()
+	return fmt.Sprintf(`(?m)([a-zA-Z0-9+-\.]+://[^%s\s]+:[^%s\s]+)@`, regexp.QuoteMeta(allChars), regexp.QuoteMeta(allChars))
+}
+
+func (*basicAuth) DenyList() []*regexp.Regexp {
+	return []*regexp.Regexp{
+		regexp.MustCompile(basicAuthRegexString()),
+	}
+}
+
+func (*basicAuth) Authenticate(secret string, src string) (AuthenticatedResult, error) {
+	// Not supported
+	return NOT_IMPLEMENTED, nil
+}

secrets/facebook_access_token.go

@@ -0,0 +1,25 @@
+package secrets
+
+import "regexp"
+
+func init() {
+	RegisterMatcher(&facebookAccessToken{})
+}
+
+// https://github.com/zricethezav/gitleaks/blob/6f5ad9dc0b385c872f652324188ce91da7157c7c/config/default.go
+// https://github.com/l4yton/RegHex#facebook-access-token
+type facebookAccessToken struct{}
+
+func (*facebookAccessToken) Type() string {
+	return "facebook_access_token"
+}
+
+func (*facebookAccessToken) DenyList() []*regexp.Regexp {
+	return []*regexp.Regexp{
+		regexp.MustCompile(`(?m)EAACEdEose0cBA[0-9A-Za-z]+`),
+	}
+}
+
+func (*facebookAccessToken) Authenticate(secret string, src string) (AuthenticatedResult, error) {
+	return NOT_IMPLEMENTED, nil
+}

secrets/facebook_oauth.go

@@ -0,0 +1,24 @@
+package secrets
+
+import "regexp"
+
+func init() {
+	RegisterMatcher(&facebookOauth{})
+}
+
+// https://github.com/l4yton/RegHex#facebook-oauth
+type facebookOauth struct{}
+
+func (*facebookOauth) Type() string {
+	return "facebook_oauth"
+}
+
+func (*facebookOauth) DenyList() []*regexp.Regexp {
+	return []*regexp.Regexp{
+		regexp.MustCompile(`(?m)[f|F][a|A][c|C][e|E][b|B][o|O][o|O][k|K].*['|\"][0-9a-f]{32}['|\"]`),
+	}
+}
+
+func (*facebookOauth) Authenticate(secret string, src string) (AuthenticatedResult, error) {
+	return NOT_IMPLEMENTED, nil
+}

secrets/facebook_secret_key.go

@@ -0,0 +1,24 @@
+package secrets
+
+import "regexp"
+
+func init() {
+	RegisterMatcher(&facebookSecretKey{})
+}
+
+// https://github.com/l4yton/RegHex#facebook-secret-key
+type facebookSecretKey struct{}
+
+func (*facebookSecretKey) Type() string {
+	return "facebook_secret_key"
+}
+
+func (*facebookSecretKey) DenyList() []*regexp.Regexp {
+	return []*regexp.Regexp{
+		regexp.MustCompile(`(?im)(facebook|fb)(.{0,20})?(?-i)['\"][0-9a-f]{32}`),
+	}
+}
+
+func (*facebookSecretKey) Authenticate(secret string, src string) (AuthenticatedResult, error) {
+	return NOT_IMPLEMENTED, nil
+}

secrets/github_app_token.go

@@ -0,0 +1,23 @@
+package secrets
+
+import "regexp"
+
+func init() {
+	RegisterMatcher(&githubAppToken{})
+}
+
+type githubAppToken struct{}
+
+func (*githubAppToken) Type() string {
+	return "github_app_token"
+}
+
+func (*githubAppToken) DenyList() []*regexp.Regexp {
+	return []*regexp.Regexp{
+		regexp.MustCompile(`(?m)(ghu|ghs)_[0-9a-zA-Z]{36}`),
+	}
+}
+
+func (*githubAppToken) Authenticate(secret string, src string) (AuthenticatedResult, error) {
+	return NOT_IMPLEMENTED, nil
+}

secrets/github_oauth_access_token.go

@@ -0,0 +1,23 @@
+package secrets
+
+import "regexp"
+
+func init() {
+	RegisterMatcher(&githubOAuthAccessToken{})
+}
+
+type githubOAuthAccessToken struct{}
+
+func (*githubOAuthAccessToken) Type() string {
+	return "github_oauth_access_token"
+}
+
+func (*githubOAuthAccessToken) DenyList() []*regexp.Regexp {
+	return []*regexp.Regexp{
+		regexp.MustCompile(`(?m)gho_[0-9a-zA-Z]{36}`),
+	}
+}
+
+func (*githubOAuthAccessToken) Authenticate(secret string, src string) (AuthenticatedResult, error) {
+	return NOT_IMPLEMENTED, nil
+}

secrets/github_personal_access_token.go

@@ -0,0 +1,25 @@
+package secrets
+
+import "regexp"
+
+func init() {
+	RegisterMatcher(&githubPersonalAccessToken{})
+}
+
+type githubPersonalAccessToken struct{}
+
+func (*githubPersonalAccessToken) Type() string {
+	return "github_personal_access_token"
+}
+
+func (*githubPersonalAccessToken) DenyList() []*regexp.Regexp {
+	return []*regexp.Regexp{
+		// https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/
+		regexp.MustCompile(`(?m)(ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{36}`),
+		regexp.MustCompile(`(?m)[0-9a-f]{40}`), // https://bl.ocks.org/magnetikonline/073afe7909ffdd6f10ef06a00bc3bc88
+	}
+}
+
+func (*githubPersonalAccessToken) Authenticate(secret string, src string) (AuthenticatedResult, error) {
+	return NOT_IMPLEMENTED, nil
+}

secrets/github_refresh_token.go

@@ -0,0 +1,25 @@
+package secrets
+
+import "regexp"
+
+func init() {
+	RegisterMatcher(&githubRefreshToken{})
+}
+
+//// Github Refresh Token
+
+type githubRefreshToken struct{}
+
+func (*githubRefreshToken) Type() string {
+	return "github_refresh_token"
+}
+
+func (*githubRefreshToken) DenyList() []*regexp.Regexp {
+	return []*regexp.Regexp{
+		regexp.MustCompile(`(?m)ghr_[0-9a-zA-Z]{76}`),
+	}
+}
+
+func (*githubRefreshToken) Authenticate(secret string, src string) (AuthenticatedResult, error) {
+	return NOT_IMPLEMENTED, nil
+}