Skip to content

Commit

Permalink
invert binary ext processing (#13)
Browse files Browse the repository at this point in the history 
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
  • Loading branch information
@wagoodman
wagoodman authored Oct 17, 2023
1 parent f6026d5 commit cca5e66
Show file tree
Hide file tree
Showing 2 changed files with 185 additions and 71 deletions.
52 changes: 37 additions & 15 deletions tool/githubrelease/installer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,11 +187,17 @@ func downloadAndExtractAsset(lgr logger.Logger, asset ghAsset, checksumAsset *gh
}

func isArchiveAsset(asset ghAsset) bool {
return archiveMimeTypes.Has(asset.ContentType) || (asset.ContentType == "" && hasArchiveExtension(asset.Name))
if archiveMimeTypes.Has(asset.ContentType) {
return true
}
return asset.ContentType == "" && hasArchiveExtension(asset.Name)
}

func isBinaryAsset(asset ghAsset) bool {
return binaryMimeTypes.Has(asset.ContentType) || (asset.ContentType == "" && (hasBinaryExtension(asset.Name) || !hasKnownNonBinaryExtension(asset.Name)))
if binaryMimeTypes.Has(asset.ContentType) {
return true
}
return asset.ContentType == "" && (hasBinaryExtension(asset.Name))
}

func hasArchiveExtension(name string) bool {
Expand Down Expand Up @@ -344,8 +350,8 @@ func selectChecksumAsset(lgr logger.Logger, assets []ghAsset) *ghAsset {

// this list is derived from https://github.com/golang/go/blob/master/src/go/build/syslist.go
var architectureAliases = map[string][]string{
"386": {"i386"},
"amd64": {"x86_64"},
"386": {"i386", "x32", "x86"},
"amd64": {"x86_64", "86_64", "x86-64", "86-64"},
"amd64p32": {},
"arm": {},
"arm64": {"aarch64"},
Expand Down Expand Up @@ -392,6 +398,20 @@ var osAliases = map[string][]string{
"zos": {},
}

var (
archKeys = strset.New(flattenAliases(architectureAliases)...)
osKeys = strset.New(flattenAliases(osAliases)...)
)

func flattenAliases(aliases map[string][]string) []string {
var as []string
for k, vs := range aliases {
as = append(as, k)
as = append(as, vs...)
}
return as
}

func selectBinaryAsset(lgr logger.Logger, assets []ghAsset, goOS, goArch string) *ghAsset {
// search for the asset by name with the OS and arch in the name
// e.g. chronicle_0.7.0_linux_amd64.tar.gz
Expand All @@ -415,7 +435,7 @@ func selectBinaryAsset(lgr logger.Logger, assets []ghAsset, goOS, goArch string)
continue
}

cleanName := strings.ReplaceAll(strings.ReplaceAll(strings.ToLower(asset.Name), ".", "_"), "-", "_")
cleanName := normalizedAssetName(asset.Name)

if !containsOneOf(cleanName, asSuffix(gooss)) {
lgr.WithFields("asset", asset.Name).Tracef("skipping asset (missing os %q)", gooss)
Expand All @@ -436,25 +456,27 @@ func selectBinaryAsset(lgr logger.Logger, assets []ghAsset, goOS, goArch string)
return nil
}

func normalizedAssetName(name string) string {
return strings.ReplaceAll(strings.ReplaceAll(strings.ToLower(name), ".", "_"), "-", "_")
}

func hasBinaryExtension(name string) bool {
ext := path.Ext(name)
switch ext {
case ".exe", "":
return true
}
return false
}

func hasKnownNonBinaryExtension(name string) bool {
if hasArchiveExtension(name) {
return true
}
ext := path.Ext(name)
switch ext {
// note: we only need to check for the last part of any extension (that is, only ".gz" not ".tar.gz")
case ".pem", ".sig", ".txt", ".md", ".json", ".xml":
cleanExt := normalizedAssetName(ext)
fields := strings.Split(cleanExt, "_")
// get the last field
cleanExt = fields[len(fields)-1]

if archKeys.Has(cleanExt) || osKeys.Has(cleanExt) {
// this is a loose confirmation that the suffix is not a file extension
return true
}

return false
}

Expand Down
204 changes: 148 additions & 56 deletions tool/githubrelease/installer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,76 @@ func Test_selectBinaryAsset(t *testing.T) {
},
want: nil,
},
{
name: "binary asset matching the target host last (with content type)",
args: args{
goOS: "linux",
goArch: "amd64",
assets: []ghAsset{
{
Name: "syft_0.89.0_linux_amd64.rpm",
ContentType: "application/x-rpm",
URL: "http://localhost:8080/syft_0.89.0_linux_amd64.rpm",
},
{
Name: "syft_0.89.0_linux_amd64.deb",
ContentType: "application/x-debian-package",
URL: "http://localhost:8080/syft_0.89.0_linux_amd64.deb",
},
{
Name: "syft_0.89.0_windows_amd64.msi",
ContentType: "application/x-msi",
URL: "http://localhost:8080/syft_0.89.0_windows_amd64.msi",
},
{
Name: "syft_0.89.0_linux_amd64",
ContentType: "application/x-executable",
URL: "http://localhost:8080/syft_0.89.0_linux_amd64",
},
},
},
want: &ghAsset{

Name: "syft_0.89.0_linux_amd64",
ContentType: "application/x-executable",
URL: "http://localhost:8080/syft_0.89.0_linux_amd64",
},
},
{
name: "binary asset matching the target host last (no content type)",
args: args{
goOS: "linux",
goArch: "amd64",
assets: []ghAsset{
{
Name: "syft_0.89.0_linux_amd64.rpm",
ContentType: "", // important!
URL: "http://localhost:8080/syft_0.89.0_linux_amd64.rpm",
},
{
Name: "syft_0.89.0_linux_amd64.deb",
ContentType: "", // important!
URL: "http://localhost:8080/syft_0.89.0_linux_amd64.deb",
},
{
Name: "syft_0.89.0_windows_amd64.msi",
ContentType: "", // important!
URL: "http://localhost:8080/syft_0.89.0_windows_amd64.msi",
},
{
Name: "syft_0.89.0_linux_amd64",
ContentType: "", // important!
URL: "http://localhost:8080/syft_0.89.0_linux_amd64",
},
},
},
want: &ghAsset{

Name: "syft_0.89.0_linux_amd64",
ContentType: "",
URL: "http://localhost:8080/syft_0.89.0_linux_amd64",
},
},
{
name: "no binary assets for target host",
args: args{
Expand Down Expand Up @@ -803,118 +873,140 @@ func Test_hasArchiveExtension(t *testing.T) {
}
}

func Test_hasKnownNonBinaryExtension(t *testing.T) {
func Test_isBinaryAsset(t *testing.T) {
tests := []struct {
name string
want bool
name string
asset ghAsset
want bool
}{
// positive cases
{
name: "syft_0.93.0_linux_amd64.tar.gz",
want: true,
},
{
name: "syft_0.93.0_linux_amd64.tar",
want: true,
},
{
name: "syft_0.93.0_linux_amd64.tgz",
want: true,
},
{
name: "checksums.txt",
name: "binary by content type",
asset: ghAsset{
Name: "thing.tar.gz", // important! mismatch with content type
ContentType: "application/x-executable", // this is the reason for the match
},
want: true,
},
{
name: "checksums.pem",
name: "binary by extension",
asset: ghAsset{
Name: "thing.exe",
ContentType: "", // important!
},
want: true,
},
{
name: "checksums.sig",
name: "binary by non extension",
asset: ghAsset{
Name: "thing",
ContentType: "", // important!
},
want: true,
},
// negative cases...
{
name: "thing.gz.does",
name: "not binary by extension",
asset: ghAsset{
Name: "thing.tar",
ContentType: "", // important!
},
want: false,
},
{
name: "yajsv.darwin.amd64",
name: "not binary by content type",
asset: ghAsset{
Name: "thing", // important! cannot have extension
ContentType: "application/x-lzx",
},
want: false,
},
{
name: "yajsv.darwin.arm64",
name: "not a binary (unknown extension)",
asset: ghAsset{
Name: "syft_0.89.0_linux_amd64.gdg",
ContentType: "", // important!
},
want: false,
},
{
name: "yajsv.linux.386",
name: "not a binary (rpm extension)",
asset: ghAsset{
Name: "syft_0.89.0_linux_amd64.rpm",
ContentType: "", // important!
},
want: false,
},
{
name: "yajsv.linux.amd64",
name: "not a binary (deb extension)",
asset: ghAsset{
Name: "syft_0.89.0_linux_amd64.deb",
ContentType: "", // important!
},
want: false,
},
{
name: "yajsv.windows.386.exe",
want: false,
name: "architecture, not extension",
asset: ghAsset{
Name: "syft.89.linux.amd64",
ContentType: "", // important!
},
want: true,
},
{
name: "yajsv.windows.amd64.exe",
want: false,
name: "architecture, not extension",
asset: ghAsset{
Name: "syft.89.linux-amd64",
ContentType: "", // important!
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, hasKnownNonBinaryExtension(tt.name))
})
}
}

func Test_isBinaryAsset(t *testing.T) {
tests := []struct {
name string
asset ghAsset
want bool
}{
{
name: "binary by content type",
name: "yajsv.darwin.amd64",
asset: ghAsset{
Name: "thing.tar.gz", // important! mismatch with content type
ContentType: "application/x-executable", // this is the reason for the match
Name: "yajsv.darwin.amd64",
ContentType: "", // important!
},
want: true,
},
{
name: "binary by extension",
name: "yajsv.darwin.arm64",
asset: ghAsset{
Name: "thing.exe",
Name: "yajsv.darwin.arm64",
ContentType: "", // important!
},
want: true,
},
{
name: "binary by non extension",
name: "yajsv.linux.386",
asset: ghAsset{
Name: "thing",
Name: "yajsv.linux.386",
ContentType: "", // important!
},
want: true,
},
{
name: "not binary by extension",
name: "yajsv.linux.amd64",
asset: ghAsset{
Name: "thing.tar",
Name: "yajsv.linux.amd64",
ContentType: "", // important!
},
want: false,
want: true,
},
{
name: "not binary by content type",
name: "yajsv.windows.386.exe",
asset: ghAsset{
Name: "thing", // important! cannot have extension
ContentType: "application/x-lzx",
Name: "yajsv.windows.386.exe",
ContentType: "", // important!
},
want: false,
want: true,
},
{
name: "yajsv.windows.amd64.exe",
asset: ghAsset{
Name: "yajsv.windows.amd64.exe",
ContentType: "", // important!
},
want: true,
},
}
for _, tt := range tests {
Expand Down

0 comments on commit cca5e66

Please sign in to comment.