diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 12dd6a8ef..22e2a9b6f 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -93,6 +93,7 @@ pipeline { parallel([ 'stack-command': generateTestCommandStage(command: 'test-stack-command', artifacts: ['build/elastic-stack-dump/stack/logs/*.log', 'build/elastic-stack-dump/stack/logs/fleet-server-internal/*']), 'check-packages': generateTestCommandStage(command: 'test-check-packages', artifacts: ['build/test-results/*.xml', 'build/kubectl-dump.txt', 'build/elastic-stack-dump/check/logs/*.log', 'build/elastic-stack-dump/check/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), + 'build-zip': generateTestCommandStage(command: 'test-build-zip', artifacts: ['build/elastic-stack-dump/build-zip/logs/*.log']), 'profiles-command': generateTestCommandStage(command: 'test-profiles-command'), ]) } diff --git a/Makefile b/Makefile index c20fff8ee..f86268ab0 100644 --- a/Makefile +++ b/Makefile @@ -44,10 +44,13 @@ test-stack-command: test-check-packages: ./scripts/test-check-packages.sh +test-build-zip: + ./scripts/test-build-zip.sh + test-profiles-command: ./scripts/test-profiles-command.sh -test: test-go test-stack-command test-check-packages test-profiles-command +test: test-go test-stack-command test-check-packages test-profiles-command test-build-zip check-git-clean: git update-index --really-refresh diff --git a/cmd/build.go b/cmd/build.go index cc08e3430..0007d90b3 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -34,6 +34,7 @@ func setupBuildCommand() *cobraext.Command { Long: buildLongDescription, RunE: buildCommandAction, } + cmd.Flags().Bool(cobraext.BuildZipFlagName, false, cobraext.BuildZipFlagDescription) return cobraext.NewCommand(cmd, cobraext.ContextPackage) } @@ -62,7 +63,11 @@ func buildCommandAction(cmd *cobra.Command, args []string) error { cmd.Printf("%s file rendered: %s\n", splitTarget[len(splitTarget)-1], target) } - target, err := builder.BuildPackage(packageRoot) + createZip, _ := cmd.Flags().GetBool(cobraext.BuildZipFlagName) + target, err := builder.BuildPackage(builder.BuildOptions{ + PackageRoot: packageRoot, + CreateZip: createZip, + }) if err != nil { return errors.Wrap(err, "building package failed") } diff --git a/go.mod b/go.mod index 9797acaf1..96016adef 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/kylelemons/godebug v1.1.0 github.com/magefile/mage v1.11.0 + github.com/mholt/archiver/v3 v3.5.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.2.1 @@ -47,9 +48,11 @@ require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect + github.com/andybalholm/brotli v1.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect github.com/creasty/defaults v1.5.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dsnet/compress v0.0.1 // indirect github.com/emirpasic/gods v1.12.0 // indirect github.com/evanphx/json-patch v4.11.0+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect @@ -64,6 +67,7 @@ require ( github.com/go-stack/stack v1.8.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.1 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/go-cmp v0.5.5 // indirect github.com/google/gofuzz v1.1.0 // indirect @@ -79,6 +83,8 @@ require ( github.com/json-iterator/go v1.1.11 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect + github.com/klauspost/compress v1.11.13 // indirect + github.com/klauspost/pgzip v1.2.4 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-colorable v0.1.9 // indirect @@ -93,15 +99,19 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect + github.com/nwaples/rardecode v1.1.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pierrec/lz4/v4 v4.0.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday v1.5.2 // indirect github.com/sergi/go-diff v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/ulikunitz/xz v0.5.7 // indirect github.com/xanzy/ssh-agent v0.3.0 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect + github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect go.mongodb.org/mongo-driver v1.1.2 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect diff --git a/go.sum b/go.sum index ddb312b0d..3068cc53c 100644 --- a/go.sum +++ b/go.sum @@ -118,6 +118,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= +github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= +github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -320,6 +322,9 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elastic/go-elasticsearch/v7 v7.15.1 h1:Wd8RLHb5D8xPBU8vGlnLXyflkso9G+rCmsXjqH8LLQQ= @@ -467,6 +472,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -612,8 +619,14 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A= +github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -679,6 +692,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mholt/archiver/v3 v3.5.0 h1:nE8gZIrw66cu4osS/U7UW7YDuGMHssxKutU8IfWxwWE= +github.com/mholt/archiver/v3 v3.5.0/go.mod h1:qqTTPUK/HZPFgFQ/TJ3BzvTpF/dPtFVJXdQbCmeMxwc= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= @@ -727,6 +742,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -781,6 +798,8 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= +github.com/pierrec/lz4/v4 v4.0.3 h1:vNQKSVZNYUEAvRY9FaUXAF1XPbSOHJtDTiP41kzDz2E= +github.com/pierrec/lz4/v4 v4.0.3/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -905,6 +924,9 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4= +github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -927,6 +949,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= diff --git a/internal/builder/packages.go b/internal/builder/packages.go index 2a7b23efc..fc2d881be 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -16,6 +16,14 @@ import ( "github.com/pkg/errors" ) +const buildIntegrationsFolder = "integrations" + +type BuildOptions struct { + PackageRoot string + + CreateZip bool +} + // BuildDirectory function locates the target build directory. If the directory doesn't exist, it will create it. func BuildDirectory() (string, error) { buildDir, found, err := findBuildDirectory() @@ -55,25 +63,49 @@ func findBuildDirectory() (string, bool, error) { return "", false, nil } -// BuildPackagesDirectory function locates the target build directory for packages. -// If the directories path doesn't exist, it will create it. +// BuildPackagesDirectory function locates the target build directory for the package. func BuildPackagesDirectory(packageRoot string) (string, error) { + buildDir, err := buildPackagesRootDirectory() + if err != nil { + return "", errors.Wrap(err, "can't locate build packages root directory") + } + m, err := packages.ReadPackageManifestFromPackageRoot(packageRoot) + if err != nil { + return "", errors.Wrapf(err, "reading package manifest failed (path: %s)", packageRoot) + } + return filepath.Join(buildDir, m.Name, m.Version), nil +} + +// buildPackagesZipPath function locates the target zipped package path. +func buildPackagesZipPath(packageRoot string) (string, error) { + buildDir, err := buildPackagesRootDirectory() + if err != nil { + return "", errors.Wrap(err, "can't locate build packages root directory") + } + m, err := packages.ReadPackageManifestFromPackageRoot(packageRoot) + if err != nil { + return "", errors.Wrapf(err, "reading package manifest failed (path: %s)", packageRoot) + } + return ZippedBuiltPackagePath(buildDir, *m), nil +} + +// ZippedBuiltPackagePath function returns the path to zipped built package. +func ZippedBuiltPackagePath(buildDir string, m packages.PackageManifest) string { + return filepath.Join(buildDir, fmt.Sprintf("%s-%s.zip", m.Name, m.Version)) +} + +func buildPackagesRootDirectory() (string, error) { buildDir, found, err := FindBuildPackagesDirectory() if err != nil { return "", errors.Wrap(err, "can't locate build directory") } if !found { - buildDir, err = createBuildDirectory("integrations") // TODO add support for other package types + buildDir, err = createBuildDirectory(buildIntegrationsFolder) // TODO add support for other package types if err != nil { return "", errors.Wrap(err, "can't create new build directory") } } - - m, err := packages.ReadPackageManifestFromPackageRoot(packageRoot) - if err != nil { - return "", errors.Wrapf(err, "reading package manifest failed (path: %s)", packageRoot) - } - return filepath.Join(buildDir, m.Name, m.Version), nil + return buildDir, nil } // FindBuildPackagesDirectory function locates the target build directory for packages. @@ -84,7 +116,7 @@ func FindBuildPackagesDirectory() (string, bool, error) { } if found { - path := filepath.Join(buildDir, "integrations") // TODO add support for other package types + path := filepath.Join(buildDir, buildIntegrationsFolder) // TODO add support for other package types fileInfo, err := os.Stat(path) if errors.Is(err, os.ErrNotExist) { return "", false, nil @@ -102,8 +134,8 @@ func FindBuildPackagesDirectory() (string, bool, error) { } // BuildPackage function builds the package. -func BuildPackage(packageRoot string) (string, error) { - destinationDir, err := BuildPackagesDirectory(packageRoot) +func BuildPackage(options BuildOptions) (string, error) { + destinationDir, err := BuildPackagesDirectory(options.PackageRoot) if err != nil { return "", errors.Wrap(err, "can't locate build directory") } @@ -115,8 +147,8 @@ func BuildPackage(packageRoot string) (string, error) { return "", errors.Wrap(err, "clearing package contents failed") } - logger.Debugf("Copy package content (source: %s)", packageRoot) - err = files.CopyWithoutDev(packageRoot, destinationDir) + logger.Debugf("Copy package content (source: %s)", options.PackageRoot) + err = files.CopyWithoutDev(options.PackageRoot, destinationDir) if err != nil { return "", errors.Wrap(err, "copying package contents failed") } @@ -128,11 +160,26 @@ func BuildPackage(packageRoot string) (string, error) { } logger.Debug("Resolve external fields") - err = resolveExternalFields(packageRoot, destinationDir) + err = resolveExternalFields(options.PackageRoot, destinationDir) if err != nil { return "", errors.Wrap(err, "resolving external fields failed") } - return destinationDir, nil + + if !options.CreateZip { + return destinationDir, nil + } + + logger.Debug("Build zipped package") + zippedPackagePath, err := buildPackagesZipPath(options.PackageRoot) + if err != nil { + return "", errors.Wrap(err, "can't evaluate path for the zipped package") + } + + err = files.Zip(destinationDir, zippedPackagePath) + if err != nil { + return "", errors.Wrapf(err, "can't compress the built package (compressed file path: %s)", zippedPackagePath) + } + return zippedPackagePath, nil } func createBuildDirectory(dirs ...string) (string, error) { diff --git a/internal/cleanup/build.go b/internal/cleanup/build.go index 3221237dd..bddfd7a97 100644 --- a/internal/cleanup/build.go +++ b/internal/cleanup/build.go @@ -56,5 +56,12 @@ func Build() (string, error) { if err != nil { return "", errors.Wrapf(err, "can't remove directory (path: %s)", destinationDir) } + + zippedBuildPackagePath := builder.ZippedBuiltPackagePath(buildDir, *m) + logger.Debugf("Remove zipped built package (path: %s)", zippedBuildPackagePath) + err = os.RemoveAll(zippedBuildPackagePath) + if err != nil { + return "", errors.Wrapf(err, "can't remove zipped built package (path: %s)", zippedBuildPackagePath) + } return destinationDir, nil } diff --git a/internal/cobraext/const.go b/internal/cobraext/const.go index e9ee5795a..5aba2768a 100644 --- a/internal/cobraext/const.go +++ b/internal/cobraext/const.go @@ -6,6 +6,9 @@ package cobraext // Flag names and descriptions used by CLI commands. const ( + BuildZipFlagName = "zip" + BuildZipFlagDescription = "archive the built package" + CheckConditionFlagName = "check-condition" CheckConditionFlagDescription = "check if the condition is met for the package, but don't install the package (e.g. kibana.version=7.10.0)" diff --git a/internal/files/compress.go b/internal/files/compress.go new file mode 100644 index 000000000..3325016cf --- /dev/null +++ b/internal/files/compress.go @@ -0,0 +1,63 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package files + +import ( + "compress/flate" + "os" + "path/filepath" + "strings" + + "github.com/elastic/elastic-package/internal/logger" + + "github.com/mholt/archiver/v3" + "github.com/pkg/errors" +) + +// Zip function creates the .zip archive from the source path (built package content). +func Zip(sourcePath, destinationFile string) error { + logger.Debugf("Compress using archiver.Zip (destination: %s)", destinationFile) + + z := archiver.Zip{ + CompressionLevel: flate.DefaultCompression, + MkdirAll: false, + SelectiveCompression: true, + ContinueOnError: false, + OverwriteExisting: true, + ImplicitTopLevelFolder: false, + } + + // Create a temporary work directory to properly name the root directory in the archive, e.g. aws-1.0.1 + tempDir, err := os.MkdirTemp("", "elastic-package-") + if err != nil { + return errors.Wrap(err, "can't prepare a temporary directory") + } + defer os.RemoveAll(tempDir) + workDir := filepath.Join(tempDir, folderNameFromFileName(destinationFile)) + os.MkdirAll(workDir, 0755) + + logger.Debugf("Create work directory for archiving: %s", workDir) + err = CopyAll(sourcePath, workDir) + if err != nil { + return errors.Wrapf(err, "can't create a work directory (path: %s)", workDir) + } + + err = z.Archive([]string{workDir}, destinationFile) + if err != nil { + return errors.Wrapf(err, "can't archive source directory (source path: %s)", sourcePath) + } + return nil +} + +// folderNameFromFileName returns the folder name from the destination file. +// Based on mholt/archiver: https://github.com/mholt/archiver/blob/d35d4ce7c5b2411973fb7bd96ca1741eb011011b/archiver.go#L397 +func folderNameFromFileName(filename string) string { + base := filepath.Base(filename) + firstDot := strings.LastIndex(base, ".") + if firstDot > -1 { + return base[:firstDot] + } + return base +} diff --git a/scripts/test-build-zip.sh b/scripts/test-build-zip.sh new file mode 100755 index 000000000..0bc714aa4 --- /dev/null +++ b/scripts/test-build-zip.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +set -euxo pipefail + +cleanup() { + r=$? + + # Dump stack logs + elastic-package stack dump -v --output build/elastic-stack-dump/build-zip + + # Take down the stack + elastic-package stack down -v + + # Clean used resources + for d in test/packages/*/; do + ( + cd $d + elastic-package clean -v + ) + done + + exit $r +} + +trap cleanup EXIT + +OLDPWD=$PWD +# Build packages +for d in test/packages/*/; do + ( + cd $d + elastic-package build --zip -v + ) +done +cd - + +# Remove unzipped built packages, leave .zip files +rm -r build/integrations/*/ + +# Boot up the stack +eval "$(elastic-package stack shellinit)" +elastic-package stack up -d -v + +# Install zipped packages +for d in test/packages/*/; do + ( + cd $d + elastic-package install -v + ) +cd - +done \ No newline at end of file