diff --git a/appveyor.yml b/appveyor.yml index 3adc4c375..bb5775dfc 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,8 +6,9 @@ environment: GOPATH: c:\gopath GO111MODULE: on GOPROXY: https://proxy.golang.org + SKIP_UNTIL_113: true -stack: go 1.12 +stack: go 1.13 test_script: - go test ./... \ No newline at end of file diff --git a/cmd/proxy/actions/app_proxy.go b/cmd/proxy/actions/app_proxy.go index b950ef2df..1a7058375 100644 --- a/cmd/proxy/actions/app_proxy.go +++ b/cmd/proxy/actions/app_proxy.go @@ -74,12 +74,23 @@ func addProxyRoutes( // 3. The stashpool manages limiting concurrent requests and passes them to stash. // 4. The plain stash.New just takes a request from upstream and saves it into storage. fs := afero.NewOsFs() - mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoProxy, fs) + + // TODO: remove before we release v0.7.0 + if c.GoProxy != "direct" && c.GoProxy != "" { + l.Error("GoProxy is deprecated, please use GoBinaryEnvVars") + } + if !c.GoBinaryEnvVars.HasKey("GONOSUMDB") { + c.GoBinaryEnvVars.Add("GONOSUMDB", strings.Join(c.NoSumPatterns, ",")) + } + if err := c.GoBinaryEnvVars.Validate(); err != nil { + return err + } + mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoBinaryEnvVars, fs) if err != nil { return err } - lister := module.NewVCSLister(c.GoBinary, c.GoProxy, fs) + lister := module.NewVCSLister(c.GoBinary, c.GoBinaryEnvVars, fs) withSingleFlight, err := getSingleFlight(c, s) if err != nil { diff --git a/config.dev.toml b/config.dev.toml index 370c0ce27..66f5027f6 100755 --- a/config.dev.toml +++ b/config.dev.toml @@ -15,6 +15,7 @@ GoBinary = "go" # Env override: GO_ENV GoEnv = "development" +# GoProxy IS DEPRECATED and will be removed in v0.7.0: PLEASE USE GoBinaryEnvVars # GoProxy specifies GOPROXY env for go list or mod download inside athens # which can be configured totally same with GOPROXY of Go Command. # Notes that the comma-separated GOPROXY (e.g. ,,direct) is only available in Go 1.13 or higher, @@ -22,6 +23,20 @@ GoEnv = "development" # Env override: GOPROXY GoProxy = "direct" +# GoBinaryEnvVars are environment variables that you'd like +# to pass directly to the Go command that Athens runs under the +# hood. Athens primarily runs two Go commands: +# 1. `go mod download -json @` +# 2. `go list -m -json @latest` +# The go command accepts multiple environment variables that +# can affect the two processes above such as GONOSUMDB and GOPROXY. +# Note that athens passes the NoSumPatterns to the two commands above, +# but the existence of GONOSUMDB in this configuration takes precedence. +# Although you can pass any key=value to the Go command here, you can see +# the list of possible env vars by running `go env`. +# Env override: ATHENS_GO_BINARY_ENV_VARS +GoBinaryEnvVars = ["GOPROXY=direct"] + # GoGetWorkers specifies how many times you can concurrently # go mod download, this is so that low performance instances # can manage go get more sanely and not run out of disk or memory. diff --git a/go.mod b/go.mod index cfce18967..87d51d9d2 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( cloud.google.com/go v0.26.0 contrib.go.opencensus.io/exporter/stackdriver v0.6.0 github.com/Azure/azure-storage-blob-go v0.7.0 + github.com/Azure/go-autorest/autorest/adal v0.6.0 // indirect github.com/BurntSushi/toml v0.3.1 github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 // indirect github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57 diff --git a/go.sum b/go.sum index dc547a565..11fc53ba6 100644 --- a/go.sum +++ b/go.sum @@ -4,14 +4,26 @@ contrib.go.opencensus.io/exporter/stackdriver v0.6.0 h1:U0FQWsZU3aO8W+BrZc88T8fd contrib.go.opencensus.io/exporter/stackdriver v0.6.0/go.mod h1:QeFzMJDAw8TXt5+aRaSuE8l5BwaMIOIlaVkBOPRuMuw= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999 h1:OR8VhtwhcAI3U48/rzBsVOuHi0zDPzYI1xASVcdSgR8= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/Azure/azure-pipeline-go v0.1.8 h1:KmVRa8oFMaargVesEuuEoiLCQ4zCCwQ8QX/xg++KS20= -github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= -github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c h1:Y5ueznoCekgCWBytF1Q9lTpZ3tJeX37dQtCcGjMCLYI= -github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= github.com/Azure/azure-storage-blob-go v0.7.0 h1:MuueVOYkufCxJw5YZzF842DY2MBsp+hLuh2apKY0mck= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= +github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.6.0 h1:UCTq22yE3RPgbU/8u4scfnnzuCW6pwQ9n+uBtV78ouo= +github.com/Azure/go-autorest/autorest/adal v0.6.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ= @@ -64,6 +76,7 @@ github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDA github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg= github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gobuffalo/envy v1.6.7 h1:XMZGuFqTupAXhZTriQ+qO38QvNOSU/0rl3hEPCFci/4= github.com/gobuffalo/envy v1.6.7/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ= @@ -165,9 +178,11 @@ github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZX github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= @@ -205,6 +220,7 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= @@ -249,8 +265,6 @@ go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029103014-dab2b1051b5d h1:5JyY8HlzxzYI+qHOOciM8s2lJbIEaefMUdtYt7dRDrg= -golang.org/x/crypto v0.0.0-20181029103014-dab2b1051b5d/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -262,8 +276,6 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181029044818-c44066c5c816 h1:mVFkLpejdFLXVUv9E42f3XJVfMdqd0IVLVIVLjZWn5o= -golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190502183928-7f726cade0ab h1:9RfW3ktsOZxgo9YNbBAjq1FWzc/igwEcUzZz8IXgSbk= golang.org/x/net v0.0.0-20190502183928-7f726cade0ab/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -276,8 +288,6 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc h1:SdCq5U4J+PpbSDIl9bM0V1e1Ug1jsnBkAFvTs1htn7U= -golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82 h1:vsphBvatvfbhlb4PO1BYSr9dzugGxJ/SQHoNufZJq1w= @@ -303,7 +313,6 @@ gopkg.in/DataDog/dd-trace-go.v1 v1.10.0 h1:aKIe93NsKAn5Gm/A4nNO4hlPuKTnhaf+khqu0 gopkg.in/DataDog/dd-trace-go.v1 v1.10.0/go.mod h1:DVp8HmDh8PuTu2Z0fVVlBsyWaC++fzwVCaGWylTe3tg= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= diff --git a/pkg/config/config.go b/pkg/config/config.go index 1ae599313..567491eab 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -7,6 +7,7 @@ import ( "path/filepath" "runtime" "strconv" + "strings" "github.com/BurntSushi/toml" "github.com/gomods/athens/pkg/download/mode" @@ -23,6 +24,7 @@ type Config struct { GoEnv string `validate:"required" envconfig:"GO_ENV"` GoBinary string `validate:"required" envconfig:"GO_BINARY_PATH"` GoProxy string `envconfig:"GOPROXY"` + GoBinaryEnvVars EnvList `envconfig:"ATHENS_GO_BINARY_ENV_VARS"` GoGetWorkers int `validate:"required" envconfig:"ATHENS_GOGET_WORKERS"` ProtocolWorkers int `validate:"required" envconfig:"ATHENS_PROTOCOL_WORKERS"` LogLevel string `validate:"required" envconfig:"ATHENS_LOG_LEVEL"` @@ -56,6 +58,40 @@ type Config struct { Storage *StorageConfig } +// EnvList is a list of key-value environment +// variables that are passed to the Go command +type EnvList []string + +// HasKey returns whether a key-value entry +// is present by only checking the left of +// key=value +func (el EnvList) HasKey(key string) bool { + for _, env := range el { + if strings.HasPrefix(env, key+"=") { + return true + } + } + return false +} + +// Add adds a key=value entry to the environment +// list +func (el *EnvList) Add(key, value string) { + *el = append(*el, key+"="+value) +} + +// Validate validates that all strings inside the +// list are of the key=value format +func (el EnvList) Validate() error { + const op errors.Op = "EnvList.Validate" + for _, env := range el { + if strings.Count(env, "=") != 1 { + return errors.E(op, fmt.Errorf("incorrect env format: %v", env)) + } + } + return nil +} + // Load loads the config from a file. // If file is not present returns default config func Load(configFile string) (*Config, error) { @@ -78,6 +114,7 @@ func Load(configFile string) (*Config, error) { func defaultConfig() *Config { return &Config{ GoBinary: "go", + GoBinaryEnvVars: EnvList{"GOPROXY=direct"}, GoEnv: "development", GoProxy: "direct", GoGetWorkers: 10, diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 0482d1d7d..9cd2236fd 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "runtime" "strconv" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -76,21 +77,22 @@ func TestEnvOverrides(t *testing.T) { TimeoutConf: TimeoutConf{ Timeout: 30, }, - StorageType: "minio", - GlobalEndpoint: "mytikas.gomods.io", - Port: ":7000", - EnablePprof: false, - PprofPort: ":3001", - BasicAuthUser: "testuser", - BasicAuthPass: "testpass", - ForceSSL: true, - ValidatorHook: "testhook.io", - PathPrefix: "prefix", - NETRCPath: "/test/path/.netrc", - HGRCPath: "/test/path/.hgrc", - Storage: &StorageConfig{}, - SingleFlight: &SingleFlight{}, - RobotsFile: "robots.txt", + StorageType: "minio", + GlobalEndpoint: "mytikas.gomods.io", + Port: ":7000", + EnablePprof: false, + PprofPort: ":3001", + BasicAuthUser: "testuser", + BasicAuthPass: "testpass", + ForceSSL: true, + ValidatorHook: "testhook.io", + PathPrefix: "prefix", + NETRCPath: "/test/path/.netrc", + HGRCPath: "/test/path/.hgrc", + Storage: &StorageConfig{}, + GoBinaryEnvVars: []string{"GOPROXY=direct"}, + SingleFlight: &SingleFlight{}, + RobotsFile: "robots.txt", } envVars := getEnvMap(expConf) @@ -279,6 +281,7 @@ func TestParseExampleConfig(t *testing.T) { TraceExporter: "", StatsExporter: "prometheus", SingleFlightType: "memory", + GoBinaryEnvVars: []string{"GOPROXY=direct"}, SingleFlight: &SingleFlight{}, SumDBs: []string{"https://sum.golang.org"}, NoSumPatterns: []string{}, @@ -325,6 +328,7 @@ func getEnvMap(config *Config) map[string]string { envVars["ATHENS_NETRC_PATH"] = config.NETRCPath envVars["ATHENS_HGRC_PATH"] = config.HGRCPath envVars["ATHENS_ROBOTS_FILE"] = config.RobotsFile + envVars["ATHENS_GO_BINARY_ENV_VARS"] = strings.Join(config.GoBinaryEnvVars, ",") storage := config.Storage if storage != nil { @@ -477,3 +481,24 @@ func TestDefaultConfigMatchesConfigFile(t *testing.T) { t.Errorf("Default values from the config file: %v should equal to the default values returned in case the config file isn't provided %v", parsedConf, defConf) } } + +func TestEnvList(t *testing.T) { + el := EnvList{"KEY=VALUE"} + if !el.HasKey("KEY") { + t.Fatal("expected KEY to be present") + } + if el.HasKey("KEY=") { + t.Fatal("expected KEY= to not be found") + } + el.Add("HELLO", "WORLD") + if !el.HasKey("HELLO") { + t.Fatal("expected HELLO key to be found") + } + if err := el.Validate(); err != nil { + t.Fatalf("expected err to be nil but got %v", err) + } + el = EnvList{"HELLO"} + if err := el.Validate(); err == nil { + t.Fatal("expected a validation error for incorrect formatting but got nil") + } +} diff --git a/pkg/download/protocol_test.go b/pkg/download/protocol_test.go index efb5ebee3..068e28870 100644 --- a/pkg/download/protocol_test.go +++ b/pkg/download/protocol_test.go @@ -35,9 +35,8 @@ func getDP(t *testing.T) Protocol { t.Fatalf("Unable to parse config file: %s", err.Error()) } goBin := conf.GoBinary - goProxy := conf.GoProxy fs := afero.NewOsFs() - mf, err := module.NewGoGetFetcher(goBin, goProxy, fs) + mf, err := module.NewGoGetFetcher(goBin, conf.GoBinaryEnvVars, fs) if err != nil { t.Fatal(err) } @@ -46,7 +45,7 @@ func getDP(t *testing.T) Protocol { t.Fatal(err) } st := stash.New(mf, s) - return New(&Opts{s, st, module.NewVCSLister(goBin, goProxy, fs), nil}) + return New(&Opts{s, st, module.NewVCSLister(goBin, conf.GoBinaryEnvVars, fs), nil}) } type listTest struct { diff --git a/pkg/module/all_test.go b/pkg/module/all_test.go index 56354f93a..975f81ba9 100644 --- a/pkg/module/all_test.go +++ b/pkg/module/all_test.go @@ -19,7 +19,7 @@ type ModuleSuite struct { suite.Suite fs afero.Fs goBinaryName string - goProxy string + env []string } func (m *ModuleSuite) SetupTest() { @@ -28,5 +28,5 @@ func (m *ModuleSuite) SetupTest() { func TestModules(t *testing.T) { goBinaryPath := envy.Get("GO_BINARY_PATH", "go") - suite.Run(t, &ModuleSuite{goBinaryName: goBinaryPath, goProxy: "direct"}) + suite.Run(t, &ModuleSuite{goBinaryName: goBinaryPath, env: []string{"GOPROXY=direct"}}) } diff --git a/pkg/module/go_get_fetcher.go b/pkg/module/go_get_fetcher.go index 81feac3d5..3f92a0201 100644 --- a/pkg/module/go_get_fetcher.go +++ b/pkg/module/go_get_fetcher.go @@ -19,7 +19,7 @@ import ( type goGetFetcher struct { fs afero.Fs goBinaryName string - goProxy string + envVars []string } type goModule struct { @@ -35,7 +35,7 @@ type goModule struct { } // NewGoGetFetcher creates fetcher which uses go get tool to fetch modules -func NewGoGetFetcher(goBinaryName string, goProxy string, fs afero.Fs) (Fetcher, error) { +func NewGoGetFetcher(goBinaryName string, envVars []string, fs afero.Fs) (Fetcher, error) { const op errors.Op = "module.NewGoGetFetcher" if err := validGoBinary(goBinaryName); err != nil { return nil, errors.E(op, err) @@ -43,7 +43,7 @@ func NewGoGetFetcher(goBinaryName string, goProxy string, fs afero.Fs) (Fetcher, return &goGetFetcher{ fs: fs, goBinaryName: goBinaryName, - goProxy: goProxy, + envVars: envVars, }, nil } @@ -66,7 +66,7 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver return nil, errors.E(op, err) } - m, err := downloadModule(g.goBinaryName, g.goProxy, g.fs, goPathRoot, modPath, mod, ver) + m, err := downloadModule(g.goBinaryName, g.envVars, g.fs, goPathRoot, modPath, mod, ver) if err != nil { clearFiles(g.fs, goPathRoot) return nil, errors.E(op, err) @@ -101,13 +101,13 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver // given a filesystem, gopath, repository root, module and version, runs 'go mod download -json' // on module@version from the repoRoot with GOPATH=gopath, and returns a non-nil error if anything went wrong. -func downloadModule(goBinaryName, goProxy string, fs afero.Fs, gopath, repoRoot, module, version string) (goModule, error) { +func downloadModule(goBinaryName string, envVars []string, fs afero.Fs, gopath, repoRoot, module, version string) (goModule, error) { const op errors.Op = "module.downloadModule" uri := strings.TrimSuffix(module, "/") fullURI := fmt.Sprintf("%s@%s", uri, version) cmd := exec.Command(goBinaryName, "mod", "download", "-json", fullURI) - cmd.Env = prepareEnv(gopath, goProxy) + cmd.Env = prepareEnv(gopath, envVars) cmd.Dir = repoRoot stdout := &bytes.Buffer{} stderr := &bytes.Buffer{} diff --git a/pkg/module/go_get_fetcher_test.go b/pkg/module/go_get_fetcher_test.go index 5cdb5f431..22009b7e5 100644 --- a/pkg/module/go_get_fetcher_test.go +++ b/pkg/module/go_get_fetcher_test.go @@ -3,6 +3,9 @@ package module import ( "context" "io/ioutil" + "net/http" + "net/http/httptest" + "os" "runtime" "github.com/gomods/athens/pkg/errors" @@ -14,14 +17,14 @@ var ctx = context.Background() func (s *ModuleSuite) TestNewGoGetFetcher() { r := s.Require() - fetcher, err := NewGoGetFetcher(s.goBinaryName, s.goProxy, s.fs) + fetcher, err := NewGoGetFetcher(s.goBinaryName, s.env, s.fs) r.NoError(err) _, ok := fetcher.(*goGetFetcher) r.True(ok) } func (s *ModuleSuite) TestGoGetFetcherError() { - fetcher, err := NewGoGetFetcher("invalidpath", "", afero.NewOsFs()) + fetcher, err := NewGoGetFetcher("invalidpath", s.env, afero.NewOsFs()) assert.Nil(s.T(), fetcher) if runtime.GOOS == "windows" { @@ -35,7 +38,7 @@ func (s *ModuleSuite) TestGoGetFetcherFetch() { r := s.Require() // we need to use an OS filesystem because fetch executes vgo on the command line, which // always writes to the filesystem - fetcher, err := NewGoGetFetcher(s.goBinaryName, s.goProxy, afero.NewOsFs()) + fetcher, err := NewGoGetFetcher(s.goBinaryName, s.env, afero.NewOsFs()) r.NoError(err) ver, err := fetcher.Fetch(ctx, repoURI, version) r.NoError(err) @@ -55,7 +58,7 @@ func (s *ModuleSuite) TestGoGetFetcherFetch() { func (s *ModuleSuite) TestNotFoundFetches() { r := s.Require() - fetcher, err := NewGoGetFetcher(s.goBinaryName, s.goProxy, afero.NewOsFs()) + fetcher, err := NewGoGetFetcher(s.goBinaryName, s.env, afero.NewOsFs()) r.NoError(err) // when someone buys laks47dfjoijskdvjxuyyd.com, and implements // a git server on top of it, this test will fail :) @@ -67,3 +70,48 @@ func (s *ModuleSuite) TestNotFoundFetches() { s.Failf("incorrect error kind", "expected a not found error but got %v", errors.Kind(err)) } } + +func (s *ModuleSuite) TestGoGetFetcherSumDB() { + if os.Getenv("SKIP_UNTIL_113") != "" { + return + } + r := s.Require() + zipBytes, err := ioutil.ReadFile("test_data/mockmod.xyz@v1.2.3.zip") + r.NoError(err) + mp := &mockProxy{paths: map[string][]byte{ + "/mockmod.xyz/@v/v1.2.3.info": []byte(`{"Version":"v1.2.3"}`), + "/mockmod.xyz/@v/v1.2.3.mod": []byte(`{"module mod}`), + "/mockmod.xyz/@v/v1.2.3.zip": zipBytes, + }} + proxyAddr, close := s.getProxy(mp) + defer close() + + fetcher, err := NewGoGetFetcher(s.goBinaryName, []string{"GOPROXY=" + proxyAddr}, afero.NewOsFs()) + r.NoError(err) + _, err = fetcher.Fetch(ctx, "mockmod.xyz", "v1.2.3") + if err == nil { + s.T().Fatal("expected a gosum error but got nil") + } + fetcher, err = NewGoGetFetcher(s.goBinaryName, []string{"GONOSUMDB=mockmod.xyz", "GOPROXY=" + proxyAddr}, afero.NewOsFs()) + r.NoError(err) + _, err = fetcher.Fetch(ctx, "mockmod.xyz", "v1.2.3") + r.NoError(err, "expected the go sum to not be consulted but got an error") +} + +func (s *ModuleSuite) getProxy(h http.Handler) (addr string, close func()) { + srv := httptest.NewServer(h) + return srv.URL, srv.Close +} + +type mockProxy struct { + paths map[string][]byte +} + +func (m *mockProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + resp, ok := m.paths[r.URL.Path] + if !ok { + w.WriteHeader(404) + return + } + w.Write(resp) +} diff --git a/pkg/module/go_vcs_lister.go b/pkg/module/go_vcs_lister.go index 57f54f85b..dde9e2571 100644 --- a/pkg/module/go_vcs_lister.go +++ b/pkg/module/go_vcs_lister.go @@ -24,7 +24,7 @@ type listResp struct { type vcsLister struct { goBinPath string - goProxy string + env []string fs afero.Fs } @@ -55,7 +55,7 @@ func (l *vcsLister) List(ctx context.Context, mod string) (*storage.RevInfo, []s return nil, nil, errors.E(op, err) } defer clearFiles(l.fs, gopath) - cmd.Env = prepareEnv(gopath, l.goProxy) + cmd.Env = prepareEnv(gopath, l.env) err = cmd.Run() if err != nil { @@ -83,6 +83,6 @@ func (l *vcsLister) List(ctx context.Context, mod string) (*storage.RevInfo, []s } // NewVCSLister creates an UpstreamLister which uses VCS to fetch a list of available versions -func NewVCSLister(goBinPath, goProxy string, fs afero.Fs) UpstreamLister { - return &vcsLister{goBinPath: goBinPath, goProxy: goProxy, fs: fs} +func NewVCSLister(goBinPath string, env []string, fs afero.Fs) UpstreamLister { + return &vcsLister{goBinPath: goBinPath, env: env, fs: fs} } diff --git a/pkg/module/prepare_env.go b/pkg/module/prepare_env.go index 2b3042da4..208f3fe73 100644 --- a/pkg/module/prepare_env.go +++ b/pkg/module/prepare_env.go @@ -10,7 +10,7 @@ import ( // prepareEnv will return all the appropriate // environment variables for a Go Command to run // successfully (such as GOPATH, GOCACHE, PATH etc) -func prepareEnv(gopath, goProxy string) []string { +func prepareEnv(gopath string, envVars []string) []string { pathEnv := fmt.Sprintf("PATH=%s", os.Getenv("PATH")) homeEnv := fmt.Sprintf("HOME=%s", os.Getenv("HOME")) httpProxy := fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY")) @@ -21,7 +21,6 @@ func prepareEnv(gopath, goProxy string) []string { httpsProxyLower := fmt.Sprintf("https_proxy=%s", os.Getenv("https_proxy")) noProxyLower := fmt.Sprintf("no_proxy=%s", os.Getenv("no_proxy")) gopathEnv := fmt.Sprintf("GOPATH=%s", gopath) - goProxyEnv := fmt.Sprintf("GOPROXY=%s", goProxy) cacheEnv := fmt.Sprintf("GOCACHE=%s", filepath.Join(gopath, "cache")) gitSSH := fmt.Sprintf("GIT_SSH=%s", os.Getenv("GIT_SSH")) gitSSHCmd := fmt.Sprintf("GIT_SSH_COMMAND=%s", os.Getenv("GIT_SSH_COMMAND")) @@ -31,7 +30,6 @@ func prepareEnv(gopath, goProxy string) []string { pathEnv, homeEnv, gopathEnv, - goProxyEnv, cacheEnv, disableCgo, enableGoModules, @@ -44,6 +42,7 @@ func prepareEnv(gopath, goProxy string) []string { gitSSH, gitSSHCmd, } + cmdEnv = append(cmdEnv, envVars...) if sshAuthSockVal, hasSSHAuthSock := os.LookupEnv("SSH_AUTH_SOCK"); hasSSHAuthSock { // Verify that the ssh agent unix socket exists and is a unix socket. diff --git a/pkg/module/test_data/mockmod.xyz@v1.2.3.zip b/pkg/module/test_data/mockmod.xyz@v1.2.3.zip new file mode 100644 index 000000000..627806ee2 Binary files /dev/null and b/pkg/module/test_data/mockmod.xyz@v1.2.3.zip differ