On Demand Hashing

Dockerfile

@@ -1,11 +1,11 @@
-FROM golang:1 as builder
+FROM golang:1 AS builder
 LABEL maintainer="Joel Messerli <hi.github@peg.nu>"
 WORKDIR /go/src/github.com/jmesserli/nx
 COPY . .
 RUN go get -d -v ./...
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o /go/bin/nx .
 
-FROM alpine:latest
+FROM alpine:3
 RUN apk --no-cache add ca-certificates tzdata
 WORKDIR /root/
 COPY --from=builder /go/bin/nx .

cache/cached_writer.go

@@ -3,152 +3,128 @@ package cache
 import (
 	"bytes"
 	"crypto/sha1"
-	"encoding/json"
 	"fmt"
 	"io"
 	"log"
 	"os"
 	"regexp"
+	"strings"
 	"text/tabwriter"
 	"text/template"
 )
 
 var logger = log.New(os.Stdout, "[cached_writer] ", log.LstdFlags)
 
 type CachedTemplateWriter struct {
-	hashFile       string
-	fileHashes     map[string]string
-	newHashes      map[string]string
-	buf            bytes.Buffer
-	ProcessedFiles []string
-	UpdatedFiles   []string
+	template        *template.Template
+	ignorePatterns  []*regexp.Regexp
+	useTabbedWriter bool
+	newHashes       map[string]string
+	ProcessedFiles  []string
+	UpdatedFiles    []string
 }
 
-func empty(hashFile string) *CachedTemplateWriter {
+func New(template *template.Template, ignorePatterns []*regexp.Regexp, useTabbedWriter bool) *CachedTemplateWriter {
 	return &CachedTemplateWriter{
-		hashFile:   hashFile,
-		fileHashes: map[string]string{},
-		newHashes:  map[string]string{},
+		template:        template,
+		ignorePatterns:  ignorePatterns,
+		useTabbedWriter: useTabbedWriter,
+		newHashes:       map[string]string{},
 	}
 }
 
-func New(hashFile string) *CachedTemplateWriter {
-	if _, err := os.Stat(hashFile); os.IsNotExist(err) {
-		return empty(hashFile)
-	}
-
-	jsonBytes, err := os.ReadFile(hashFile)
-	if err != nil {
-		logger.Println("could not open json hash file, discarding")
-		return empty(hashFile)
-	}
-
-	data := map[string]string{}
-	err = json.Unmarshal(jsonBytes, &data)
-	if err != nil {
-		logger.Println("could not parse json hash file, discarding")
-		return empty(hashFile)
-	}
-
-	return &CachedTemplateWriter{
-		hashFile:   hashFile,
-		fileHashes: data,
-		newHashes:  map[string]string{},
-	}
-}
-
-func (w *CachedTemplateWriter) WriteTemplate(
+func (cw *CachedTemplateWriter) WriteTemplate(
 	file string,
-	tpl *template.Template,
 	data interface{},
-	ignorePatterns []*regexp.Regexp,
-	useTabbedWriter bool,
 ) (bool, error) {
-	// Reset buffer
-	w.buf = bytes.Buffer{}
+	buf := bytes.Buffer{}
+	err := func() error {
+		var bufWriter io.Writer
+		if cw.useTabbedWriter {
+			bufWriter = tabwriter.NewWriter(&buf, 2, 2, 2, ' ', 0)
+			defer bufWriter.(*tabwriter.Writer).Flush()
+		} else {
+			bufWriter = &buf
+		}
 
-	err := tpl.Execute(&w.buf, data)
+		err := cw.template.Execute(bufWriter, data)
+		if err != nil {
+			return err
+		}
+		return nil
+	}()
 	if err != nil {
 		return false, err
 	}
 
-	str := string(w.buf.Bytes())
-	for _, regex := range ignorePatterns {
-		str = regex.ReplaceAllString(str, "-hash:omit-")
-	}
-
-	hash := sha1.New()
-	hash.Write([]byte(str))
-	hashBytes := hash.Sum(nil)
-	hashStr := fmt.Sprintf("%x", hashBytes)
-
-	existingHash, ok := w.fileHashes[file]
-	if ok && existingHash == hashStr {
-		//logger.Printf("File fresh: %s\n", file)
-		w.ProcessedFiles = append(w.ProcessedFiles, file)
-		w.newHashes[file] = w.fileHashes[file]
-		w.updateJson()
-		return false, nil
+	str := string(buf.Bytes())
+	hashStr := cw.hash(str)
+
+	existingFileStr, err := cw.getFileContent(file)
+	if err == nil {
+		existingHash := cw.hash(existingFileStr)
+		if existingHash == hashStr {
+			//logger.Printf("File fresh: %s\n", file)
+			cw.ProcessedFiles = append(cw.ProcessedFiles, file)
+			cw.newHashes[file] = existingHash
+			return false, nil
+		}
+	} else {
+		logger.Printf("ignored error while reading existing file %s: %s\n", file, err.Error())
 	}
 
 	f, err := os.Create(file)
 	if err != nil {
 		return false, err
 	}
-
 	defer func(closeable io.Closer) {
 		err := closeable.Close()
 		if err != nil {
 			panic(err)
 		}
 	}(f)
 
-	var writer io.Writer
-	if useTabbedWriter {
-		writer = tabwriter.NewWriter(f, 2, 2, 2, ' ', 0)
-	} else {
-		writer = f
-	}
-
-	if useTabbedWriter {
-		wr := tabwriter.NewWriter(f, 2, 2, 2, ' ', 0)
-		_, err = wr.Write(w.buf.Bytes())
-		if err != nil {
-			return false, err
-		}
-		_ = wr.Flush()
-	} else {
-		_, err = writer.Write(w.buf.Bytes())
-		if err != nil {
-			return false, err
-		}
+	_, err = f.Write(buf.Bytes())
+	if err != nil {
+		return false, err
 	}
 
 	logger.Printf("New hash %s for file %s\n", hashStr, file)
-	w.ProcessedFiles = append(w.ProcessedFiles, file)
-	w.UpdatedFiles = append(w.UpdatedFiles, file)
-	w.fileHashes[file] = hashStr
-	w.newHashes[file] = hashStr
-	w.updateJson()
+	cw.ProcessedFiles = append(cw.ProcessedFiles, file)
+	cw.UpdatedFiles = append(cw.UpdatedFiles, file)
+	cw.newHashes[file] = hashStr
 
 	return true, nil
 }
 
-func (w *CachedTemplateWriter) updateJson() {
-	jsonBytes, err := json.Marshal(w.newHashes)
+func (cw *CachedTemplateWriter) getFileContent(file string) (string, error) {
+	stat, err := os.Stat(file)
 	if err != nil {
-		panic(err)
+		return "", err
+	}
+	if stat.IsDir() {
+		return "", fmt.Errorf("%s is a directory", file)
 	}
 
-	f, err := os.Create(w.hashFile)
+	fileBytes, err := os.ReadFile(file)
 	if err != nil {
-		panic(err)
+		return "", err
 	}
+	return string(fileBytes), nil
+}
 
-	_, err = f.Write(jsonBytes)
-	if err != nil {
-		panic(err)
+func (cw *CachedTemplateWriter) hash(content string) string {
+	content = strings.ReplaceAll(content, "\r\n", "\n")
+	content = strings.TrimSpace(content)
+
+	if cw.ignorePatterns != nil {
+		for _, regex := range cw.ignorePatterns {
+			content = regex.ReplaceAllString(content, "-hash:omit-")
+		}
 	}
 
-	_ = f.Close()
+	hash := sha1.New()
+	hash.Write([]byte(content))
+	hashBytes := hash.Sum(nil)
+	return fmt.Sprintf("%x", hashBytes)
 }

main.go

@@ -33,7 +33,7 @@ func main() {
 
 	prefixIPsList := loadPrefixes(prefixes, nc)
 	sortPrefixList(prefixIPsList)
-	generateAll(prefixIPsList, dnsIps, wgIps, iplIps, conf)
+	generateAll(prefixIPsList, dnsIps, wgIps, iplIps, &conf)
 
 	logger.Println("Writing updated files report")
 	err := os.WriteFile("generated/updated_files.txt", []byte(strings.Join(conf.UpdatedFiles, "\n")), os.ModePerm)
@@ -78,7 +78,7 @@ func sortPrefixList(prefixIPsList []prefixIPs) {
 	}
 }
 
-func generateAll(prefixIPsList []prefixIPs, dnsIps []model.IPAddress, wgIps []model.IPAddress, iplIps []model.IPAddress, conf config.NXConfig) {
+func generateAll(prefixIPsList []prefixIPs, dnsIps []model.IPAddress, wgIps []model.IPAddress, iplIps []model.IPAddress, conf *config.NXConfig) {
 	defer util.DurationSince(util.StartTracking("generateAll"))
 
 	for _, prefixIP := range prefixIPsList {
@@ -103,14 +103,14 @@ func generateAll(prefixIPsList []prefixIPs, dnsIps []model.IPAddress, wgIps []mo
 
 		DottedMailResponsible: "unknown\\.admin.local",
 		NameserverFQDN:        "unknown-nameserver.local.",
-	}, &conf)
+	}, conf)
 
 	logger.Println("Generating BIND config files")
-	dns.GenerateConfigs(generatedZones, &conf)
+	dns.GenerateConfigs(generatedZones, conf)
 	logger.Println("Generating Wireguard config files")
-	wg.GenerateWgConfigs(wgIps, &conf)
+	wg.GenerateWgConfigs(wgIps, conf)
 	logger.Println("Generating IP lists")
-	ipl.GenerateIPLists(iplIps, &conf)
+	ipl.GenerateIPLists(iplIps, conf)
 }
 
 type prefixIPs struct {

ns/dns/configgenerator.go

@@ -95,8 +95,10 @@ func GenerateConfigs(zones []string, conf *config.NXConfig) {
 		panic(err)
 	}
 	configTemplate := template.Must(template.New("config").Parse(string(templateString)))
-	cw := cache.New("generated/hashes/bind-config.json")
-	ignoreRegex := regexp.MustCompile("(?m)^ \\* Generated at.*$")
+	ignoreRegexes := []*regexp.Regexp{
+		regexp.MustCompile("(?m)^ \\* Generated at.*$"),
+	}
+	cw := cache.New(configTemplate, ignoreRegexes, false)
 
 	templateVars := configTemplateVars{
 		GeneratedAt: time.Now().Format(time.RFC3339),
@@ -156,10 +158,7 @@ func GenerateConfigs(zones []string, conf *config.NXConfig) {
 
 		_, err = cw.WriteTemplate(
 			fmt.Sprintf("generated/bind-config/%s.conf", currentMaster.Name),
-			configTemplate,
 			templateVars,
-			[]*regexp.Regexp{ignoreRegex},
-			false,
 		)
 		if err != nil {
 			panic(err)

ns/dns/zonegenerator.go

@@ -238,9 +238,11 @@ func GenerateZones(addresses []model.IPAddress, defaultSoaInfo SOAInfo, conf *co
 		panic(err)
 	}
 	zoneTemplate := template.Must(template.New("zone").Parse(string(templateString)))
-
-	cw := cache.New("generated/hashes/zones.json")
-	ignoreRegex := regexp.MustCompile("(?m)^(\\s+\\d+\\s+; serial.*|; Generated at .*)$")
+	ignoreRegexes := []*regexp.Regexp{
+		regexp.MustCompile("(?m)^; Generated at .*$"),
+		regexp.MustCompile("(?m)^\\s+\\d+\\s+; serial.*$"),
+	}
+	cw := cache.New(zoneTemplate, ignoreRegexes, true)
 
 	for zone, records := range zoneRecordsMap {
 		templateArgs.Records = records
@@ -264,10 +266,7 @@ func GenerateZones(addresses []model.IPAddress, defaultSoaInfo SOAInfo, conf *co
 
 		_, err := cw.WriteTemplate(
 			fmt.Sprintf("generated/zones/%s.db", zone),
-			zoneTemplate,
 			templateArgs,
-			[]*regexp.Regexp{ignoreRegex},
-			true,
 		)
 		if err != nil {
 			panic(err)

ns/ipl/ipl.go

@@ -58,20 +58,19 @@ func GenerateIPLists(addresses []model.IPAddress, conf *config.NXConfig) {
 		panic(err)
 	}
 	iplTemplate := template.Must(template.New("ipl").Parse(string(templateString)))
+	ignoreRegexes := []*regexp.Regexp{
+		regexp.MustCompile("(?m)^# Generated at .*$"),
+	}
 
-	cw := cache.New("generated/hashes/ipl.json")
-	ignoreRegex := regexp.MustCompile("(?m)^# Generated at .*$")
+	cw := cache.New(iplTemplate, ignoreRegexes, false)
 
 	for group, ips := range groupMap {
 		vars.Name = group
 		vars.IPs = ips
 
 		_, err := cw.WriteTemplate(
 			fmt.Sprintf("generated/ipl/%s.ipl.txt", group),
-			iplTemplate,
 			vars,
-			[]*regexp.Regexp{ignoreRegex},
-			false,
 		)
 		if err != nil {
 			panic(err)

ns/wg/wireguard.go

@@ -62,8 +62,8 @@ func GenerateWgConfigs(ips []model.IPAddress, conf *config.NXConfig) {
 	if err != nil {
 		panic(err)
 	}
-	zoneTemplate := template.Must(template.New("wg-config").Parse(string(templateString)))
-	cw := cache.New("generated/hashes/wg.json")
+	wgTemplate := template.Must(template.New("wg-config").Parse(string(templateString)))
+	cw := cache.New(wgTemplate, []*regexp.Regexp{}, false)
 
 	for vpnName, peers := range vpnPeers {
 		for _, peer := range peers {
@@ -82,11 +82,8 @@ func GenerateWgConfigs(ips []model.IPAddress, conf *config.NXConfig) {
 			}
 
 			_, err := cw.WriteTemplate(
-				fmt.Sprintf("generated/wg/%s_%s.conf", vpnName, data.ServerName),
-				zoneTemplate,
+				fmt.Sprintf("generated/wg/%s-%s.conf", vpnName, data.ServerName),
 				data,
-				[]*regexp.Regexp{},
-				false,
 			)
 			if err != nil {
 				panic(err)