Implement DNS query backoff for load balanced environments

internal/provider/config.go

@@ -19,6 +19,7 @@ type Config struct {
 	transport string
 	timeout   time.Duration
 	retries   int
+	backoff   int
 	keyname   string
 	keyalgo   string
 	keysecret string
@@ -34,6 +35,7 @@ type DNSClient struct {
 	srv_addr  string
 	transport string
 	retries   int
+	backoff   int
 	keyname   string
 	keysecret string
 	keyalgo   string
@@ -67,6 +69,7 @@ func (c *Config) Client() (interface{}, error) {
 	client.transport = c.transport
 	client.c.Timeout = c.timeout
 	client.retries = c.retries
+	client.backoff = c.backoff
 	client.realm = c.realm
 	client.username = c.username
 	client.password = c.password

internal/provider/hash_ip_string_test.go

@@ -7,14 +7,9 @@ import (
 )
 
 func TestHashIPString(t *testing.T) {
-	ipv4 := []string{"192.168.0.1", "192.168.000.001"}
 	ipv6 := []string{"fdd5:e282::dead:beef:cafe:babe", "FDD5:E282:0000:0000:DEAD:BEEF:CAFE:BABE"}
 	invalid := "not.an.ip.address"
 
-	if hashIPString(ipv4[0]) != hashIPString(ipv4[1]) {
-		t.Errorf("IPv4 values %s and %s should hash to the same value", ipv4[0], ipv4[1])
-	}
-
 	if hashIPString(ipv6[0]) != hashIPString(ipv6[1]) {
 		t.Errorf("IPv6 values %s and %s should hash to the same value", ipv6[0], ipv6[1])
 	}

internal/provider/provider.go

@@ -16,6 +16,7 @@ import (
 const (
 	defaultPort      = 53
 	defaultRetries   = 3
+	defaultBackoff   = 0
 	defaultTimeout   = "0"
 	defaultTransport = "udp"
 )
@@ -75,6 +76,21 @@ func New() *schema.Provider {
 								return defaultRetries, nil
 							},
 						},
+						"backoff": {
+							Type:     schema.TypeInt,
+							Optional: true,
+							DefaultFunc: func() (interface{}, error) {
+								if env := os.Getenv("DNS_UPDATE_BACKOFF"); env != "" {
+									backoff, err := strconv.Atoi(env)
+									if err != nil {
+										err = fmt.Errorf("invalid DNS_UPDATE_BACKOFF environment variable: %s", err)
+									}
+									return backoff, err
+								}
+
+								return defaultBackoff, nil
+							},
+						},
 						"key_name": {
 							Type:          schema.TypeString,
 							Optional:      true,
@@ -165,7 +181,7 @@ func New() *schema.Provider {
 func configureProvider(d *schema.ResourceData) (interface{}, error) {
 
 	var server, transport, timeout, keyname, keyalgo, keysecret, realm, username, password, keytab string
-	var port, retries int
+	var port, retries, backoff int
 	var duration time.Duration
 	var gssapi bool
 
@@ -187,6 +203,9 @@ func configureProvider(d *schema.ResourceData) (interface{}, error) {
 		if val, ok := update["retries"]; ok {
 			retries = int(val.(int))
 		}
+		if val, ok := update["backoff"]; ok {
+			backoff = int(val.(int))
+		}
 		if val, ok := update["key_name"]; ok {
 			keyname = val.(string)
 		}
@@ -248,6 +267,16 @@ func configureProvider(d *schema.ResourceData) (interface{}, error) {
 		} else {
 			retries = defaultRetries
 		}
+		if len(os.Getenv("DNS_UPDATE_BACKOFF")) > 0 {
+			var err error
+			env := os.Getenv("DNS_UPDATE_BACKOFF")
+			backoff, err = strconv.Atoi(env)
+			if err != nil {
+				return nil, fmt.Errorf("invalid DNS_UPDATE_BACKOFF environment variable: %s", err)
+			}
+		} else {
+			backoff = defaultBackoff
+		}
 		if len(os.Getenv("DNS_UPDATE_KEYNAME")) > 0 {
 			keyname = os.Getenv("DNS_UPDATE_KEYNAME")
 		}
@@ -297,6 +326,7 @@ func configureProvider(d *schema.ResourceData) (interface{}, error) {
 		transport: transport,
 		timeout:   duration,
 		retries:   retries,
+		backoff:   backoff,
 		keyname:   keyname,
 		keyalgo:   keyalgo,
 		keysecret: keysecret,
@@ -406,19 +436,19 @@ func getPtrVal(record interface{}) (string, int, error) {
 }
 
 func isTimeout(err error) bool {
-
 	timeout, ok := err.(net.Error)
 	return ok && timeout.Timeout()
 }
 
-func exchange(msg *dns.Msg, tsig bool, meta interface{}) (*dns.Msg, error) {
+func exchange(msg *dns.Msg, tsig bool, meta interface{}, update ...bool) (*dns.Msg, error) {
 
 	c := meta.(*DNSClient).c
 	srv_addr := meta.(*DNSClient).srv_addr
 	keyname := meta.(*DNSClient).keyname
 	keyalgo := meta.(*DNSClient).keyalgo
 	c.Net = meta.(*DNSClient).transport
 	retries := meta.(*DNSClient).retries
+	backoff := meta.(*DNSClient).backoff
 	g := meta.(*DNSClient).gssClient
 	retry_tcp := false
 
@@ -496,6 +526,24 @@ Retry:
 		goto Retry
 	}
 
+	// Retry logic for load balanced or otherwise slowly propagated DNS environments
+	if len(r.Answer) == 0 && len(update) > 0 && backoff != defaultBackoff {
+		question := new(dns.Msg).SetQuestion(msg.Ns[0].Header().Name, 255)
+		for i := 1; i <= retries; i++ {
+			log.Println("[DEBUG] Backing off as DNS answer is empty")
+			time.Sleep(time.Duration(backoff) * time.Millisecond)
+			qr, _, err := c.Exchange(question, srv_addr)
+			if err != nil {
+				return r, err
+			}
+			if len(qr.Answer) > 0 {
+				return r, nil
+			}
+		}
+		retries--
+		goto Retry
+	}
+
 	return r, err
 }
 

internal/provider/resource_dns_a_record_set.go

@@ -114,7 +114,7 @@ func resourceDnsARecordSetUpdate(d *schema.ResourceData, meta interface{}) error
 				msg.Insert([]dns.RR{rr_insert})
 			}
 
-			r, err := exchange(msg, true, meta)
+			r, err := exchange(msg, true, meta, true)
 			if err != nil {
 				d.SetId("")
 				return fmt.Errorf("Error updating DNS record: %s", err)

internal/provider/resource_dns_aaaa_record_set.go

@@ -115,7 +115,7 @@ func resourceDnsAAAARecordSetUpdate(d *schema.ResourceData, meta interface{}) er
 				msg.Insert([]dns.RR{rr_insert})
 			}
 
-			r, err := exchange(msg, true, meta)
+			r, err := exchange(msg, true, meta, true)
 			if err != nil {
 				d.SetId("")
 				return fmt.Errorf("Error updating DNS record: %s", err)

internal/provider/resource_dns_cname_record.go

@@ -102,7 +102,7 @@ func resourceDnsCnameRecordUpdate(d *schema.ResourceData, meta interface{}) erro
 				msg.Insert([]dns.RR{rr_insert})
 			}
 
-			r, err := exchange(msg, true, meta)
+			r, err := exchange(msg, true, meta, true)
 			if err != nil {
 				d.SetId("")
 				return fmt.Errorf("Error updating DNS record: %s", err)

internal/provider/resource_dns_mx_record_set.go

@@ -135,7 +135,7 @@ func resourceDnsMXRecordSetUpdate(d *schema.ResourceData, meta interface{}) erro
 				msg.Insert([]dns.RR{rr_insert})
 			}
 
-			r, err := exchange(msg, true, meta)
+			r, err := exchange(msg, true, meta, true)
 			if err != nil {
 				d.SetId("")
 				return fmt.Errorf("Error updating DNS record: %s", err)

internal/provider/resource_dns_ptr_record.go

@@ -102,7 +102,7 @@ func resourceDnsPtrRecordUpdate(d *schema.ResourceData, meta interface{}) error
 				msg.Insert([]dns.RR{rr_insert})
 			}
 
-			r, err := exchange(msg, true, meta)
+			r, err := exchange(msg, true, meta, true)
 			if err != nil {
 				d.SetId("")
 				return fmt.Errorf("Error updating DNS record: %s", err)

internal/provider/resource_dns_srv_record_set.go

@@ -145,7 +145,7 @@ func resourceDnsSRVRecordSetUpdate(d *schema.ResourceData, meta interface{}) err
 				msg.Insert([]dns.RR{rr_insert})
 			}
 
-			r, err := exchange(msg, true, meta)
+			r, err := exchange(msg, true, meta, true)
 			if err != nil {
 				d.SetId("")
 				return fmt.Errorf("Error updating DNS record: %s", err)

internal/provider/resource_dns_txt_record_set.go

@@ -116,7 +116,7 @@ func resourceDnsTXTRecordSetUpdate(d *schema.ResourceData, meta interface{}) err
 				msg.Insert([]dns.RR{rr_insert})
 			}
 
-			r, err := exchange(msg, true, meta)
+			r, err := exchange(msg, true, meta, true)
 			if err != nil {
 				d.SetId("")
 				return fmt.Errorf("Error updating DNS record: %s", err)

website/docs/index.html.markdown

@@ -63,6 +63,7 @@ The `update` block supports the following attributes:
 * `transport` - (Optional) Transport to use for DNS queries. Valid values are `udp`, `udp4`, `udp6`, `tcp`, `tcp4`, or `tcp6`. Any UDP transport will retry automatically with the equivalent TCP transport in the event of a truncated response. Defaults to `udp`.
 * `timeout` - (Optional) Timeout for DNS queries. Valid values are durations expressed as `500ms`, etc. or a plain number which is treated as whole seconds.
 * `retries` - (Optional) How many times to retry on connection timeout. Defaults to `3`.
+* `backoff` - (Optional) How many milliseconds the provider should back off requerying the DNS server for validation after pushing record updates. Should only be used in load balanced or otherwise slowly propagated environments. Defaults to `0`.
 * `key_name` - (Optional) The name of the TSIG key used to sign the DNS update messages.
 * `key_algorithm` - (Optional; Required if `key_name` is set) When using TSIG authentication, the algorithm to use for HMAC. Valid values are `hmac-md5`, `hmac-sha1`, `hmac-sha256` or `hmac-sha512`.
 * `key_secret` - (Optional; Required if `key_name` is set)