diff --git a/Dockerfile b/Dockerfile index 640b4d1..c104392 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ -FROM golang:1 as builder +FROM golang:1 AS builder LABEL maintainer="Joel Messerli " WORKDIR /go/src/github.com/jmesserli/nx COPY . . RUN go get -d -v ./... RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o /go/bin/nx . -FROM alpine:latest +FROM alpine:3 RUN apk --no-cache add ca-certificates tzdata WORKDIR /root/ COPY --from=builder /go/bin/nx . diff --git a/cache/cached_writer.go b/cache/cached_writer.go index b2bc463..98b1539 100644 --- a/cache/cached_writer.go +++ b/cache/cached_writer.go @@ -3,12 +3,12 @@ package cache import ( "bytes" "crypto/sha1" - "encoding/json" "fmt" "io" "log" "os" "regexp" + "strings" "text/tabwriter" "text/template" ) @@ -16,86 +16,67 @@ import ( var logger = log.New(os.Stdout, "[cached_writer] ", log.LstdFlags) type CachedTemplateWriter struct { - hashFile string - fileHashes map[string]string - newHashes map[string]string - buf bytes.Buffer - ProcessedFiles []string - UpdatedFiles []string + template *template.Template + ignorePatterns []*regexp.Regexp + useTabbedWriter bool + newHashes map[string]string + ProcessedFiles []string + UpdatedFiles []string } -func empty(hashFile string) *CachedTemplateWriter { +func New(template *template.Template, ignorePatterns []*regexp.Regexp, useTabbedWriter bool) *CachedTemplateWriter { return &CachedTemplateWriter{ - hashFile: hashFile, - fileHashes: map[string]string{}, - newHashes: map[string]string{}, + template: template, + ignorePatterns: ignorePatterns, + useTabbedWriter: useTabbedWriter, + newHashes: map[string]string{}, } } -func New(hashFile string) *CachedTemplateWriter { - if _, err := os.Stat(hashFile); os.IsNotExist(err) { - return empty(hashFile) - } - - jsonBytes, err := os.ReadFile(hashFile) - if err != nil { - logger.Println("could not open json hash file, discarding") - return empty(hashFile) - } - - data := map[string]string{} - err = json.Unmarshal(jsonBytes, &data) - if err != nil { - logger.Println("could not parse json hash file, discarding") - return empty(hashFile) - } - - return &CachedTemplateWriter{ - hashFile: hashFile, - fileHashes: data, - newHashes: map[string]string{}, - } -} - -func (w *CachedTemplateWriter) WriteTemplate( +func (cw *CachedTemplateWriter) WriteTemplate( file string, - tpl *template.Template, data interface{}, - ignorePatterns []*regexp.Regexp, - useTabbedWriter bool, ) (bool, error) { - // Reset buffer - w.buf = bytes.Buffer{} + buf := bytes.Buffer{} + err := func() error { + var bufWriter io.Writer + if cw.useTabbedWriter { + bufWriter = tabwriter.NewWriter(&buf, 2, 2, 2, ' ', 0) + defer bufWriter.(*tabwriter.Writer).Flush() + } else { + bufWriter = &buf + } - err := tpl.Execute(&w.buf, data) + err := cw.template.Execute(bufWriter, data) + if err != nil { + return err + } + return nil + }() if err != nil { return false, err } - str := string(w.buf.Bytes()) - for _, regex := range ignorePatterns { - str = regex.ReplaceAllString(str, "-hash:omit-") - } - - hash := sha1.New() - hash.Write([]byte(str)) - hashBytes := hash.Sum(nil) - hashStr := fmt.Sprintf("%x", hashBytes) - - existingHash, ok := w.fileHashes[file] - if ok && existingHash == hashStr { - //logger.Printf("File fresh: %s\n", file) - w.ProcessedFiles = append(w.ProcessedFiles, file) - w.newHashes[file] = w.fileHashes[file] - w.updateJson() - return false, nil + str := string(buf.Bytes()) + hashStr := cw.hash(str) + + existingFileStr, err := cw.getFileContent(file) + if err == nil { + existingHash := cw.hash(existingFileStr) + if existingHash == hashStr { + //logger.Printf("File fresh: %s\n", file) + cw.ProcessedFiles = append(cw.ProcessedFiles, file) + cw.newHashes[file] = existingHash + return false, nil + } + } else { + logger.Printf("ignored error while reading existing file %s: %s\n", file, err.Error()) } f, err := os.Create(file) if err != nil { return false, err } - defer func(closeable io.Closer) { err := closeable.Close() if err != nil { @@ -103,52 +84,47 @@ func (w *CachedTemplateWriter) WriteTemplate( } }(f) - var writer io.Writer - if useTabbedWriter { - writer = tabwriter.NewWriter(f, 2, 2, 2, ' ', 0) - } else { - writer = f - } - - if useTabbedWriter { - wr := tabwriter.NewWriter(f, 2, 2, 2, ' ', 0) - _, err = wr.Write(w.buf.Bytes()) - if err != nil { - return false, err - } - _ = wr.Flush() - } else { - _, err = writer.Write(w.buf.Bytes()) - if err != nil { - return false, err - } + _, err = f.Write(buf.Bytes()) + if err != nil { + return false, err } logger.Printf("New hash %s for file %s\n", hashStr, file) - w.ProcessedFiles = append(w.ProcessedFiles, file) - w.UpdatedFiles = append(w.UpdatedFiles, file) - w.fileHashes[file] = hashStr - w.newHashes[file] = hashStr - w.updateJson() + cw.ProcessedFiles = append(cw.ProcessedFiles, file) + cw.UpdatedFiles = append(cw.UpdatedFiles, file) + cw.newHashes[file] = hashStr return true, nil } -func (w *CachedTemplateWriter) updateJson() { - jsonBytes, err := json.Marshal(w.newHashes) +func (cw *CachedTemplateWriter) getFileContent(file string) (string, error) { + stat, err := os.Stat(file) if err != nil { - panic(err) + return "", err + } + if stat.IsDir() { + return "", fmt.Errorf("%s is a directory", file) } - f, err := os.Create(w.hashFile) + fileBytes, err := os.ReadFile(file) if err != nil { - panic(err) + return "", err } + return string(fileBytes), nil +} - _, err = f.Write(jsonBytes) - if err != nil { - panic(err) +func (cw *CachedTemplateWriter) hash(content string) string { + content = strings.ReplaceAll(content, "\r\n", "\n") + content = strings.TrimSpace(content) + + if cw.ignorePatterns != nil { + for _, regex := range cw.ignorePatterns { + content = regex.ReplaceAllString(content, "-hash:omit-") + } } - _ = f.Close() + hash := sha1.New() + hash.Write([]byte(content)) + hashBytes := hash.Sum(nil) + return fmt.Sprintf("%x", hashBytes) } diff --git a/main.go b/main.go index 80491c5..7114ec4 100644 --- a/main.go +++ b/main.go @@ -33,7 +33,7 @@ func main() { prefixIPsList := loadPrefixes(prefixes, nc) sortPrefixList(prefixIPsList) - generateAll(prefixIPsList, dnsIps, wgIps, iplIps, conf) + generateAll(prefixIPsList, dnsIps, wgIps, iplIps, &conf) logger.Println("Writing updated files report") err := os.WriteFile("generated/updated_files.txt", []byte(strings.Join(conf.UpdatedFiles, "\n")), os.ModePerm) @@ -78,7 +78,7 @@ func sortPrefixList(prefixIPsList []prefixIPs) { } } -func generateAll(prefixIPsList []prefixIPs, dnsIps []model.IPAddress, wgIps []model.IPAddress, iplIps []model.IPAddress, conf config.NXConfig) { +func generateAll(prefixIPsList []prefixIPs, dnsIps []model.IPAddress, wgIps []model.IPAddress, iplIps []model.IPAddress, conf *config.NXConfig) { defer util.DurationSince(util.StartTracking("generateAll")) for _, prefixIP := range prefixIPsList { @@ -103,14 +103,14 @@ func generateAll(prefixIPsList []prefixIPs, dnsIps []model.IPAddress, wgIps []mo DottedMailResponsible: "unknown\\.admin.local", NameserverFQDN: "unknown-nameserver.local.", - }, &conf) + }, conf) logger.Println("Generating BIND config files") - dns.GenerateConfigs(generatedZones, &conf) + dns.GenerateConfigs(generatedZones, conf) logger.Println("Generating Wireguard config files") - wg.GenerateWgConfigs(wgIps, &conf) + wg.GenerateWgConfigs(wgIps, conf) logger.Println("Generating IP lists") - ipl.GenerateIPLists(iplIps, &conf) + ipl.GenerateIPLists(iplIps, conf) } type prefixIPs struct { diff --git a/ns/dns/configgenerator.go b/ns/dns/configgenerator.go index 534cfc5..fc5571f 100644 --- a/ns/dns/configgenerator.go +++ b/ns/dns/configgenerator.go @@ -95,8 +95,10 @@ func GenerateConfigs(zones []string, conf *config.NXConfig) { panic(err) } configTemplate := template.Must(template.New("config").Parse(string(templateString))) - cw := cache.New("generated/hashes/bind-config.json") - ignoreRegex := regexp.MustCompile("(?m)^ \\* Generated at.*$") + ignoreRegexes := []*regexp.Regexp{ + regexp.MustCompile("(?m)^ \\* Generated at.*$"), + } + cw := cache.New(configTemplate, ignoreRegexes, false) templateVars := configTemplateVars{ GeneratedAt: time.Now().Format(time.RFC3339), @@ -156,10 +158,7 @@ func GenerateConfigs(zones []string, conf *config.NXConfig) { _, err = cw.WriteTemplate( fmt.Sprintf("generated/bind-config/%s.conf", currentMaster.Name), - configTemplate, templateVars, - []*regexp.Regexp{ignoreRegex}, - false, ) if err != nil { panic(err) diff --git a/ns/dns/zonegenerator.go b/ns/dns/zonegenerator.go index b0a4c32..110e47e 100644 --- a/ns/dns/zonegenerator.go +++ b/ns/dns/zonegenerator.go @@ -238,9 +238,11 @@ func GenerateZones(addresses []model.IPAddress, defaultSoaInfo SOAInfo, conf *co panic(err) } zoneTemplate := template.Must(template.New("zone").Parse(string(templateString))) - - cw := cache.New("generated/hashes/zones.json") - ignoreRegex := regexp.MustCompile("(?m)^(\\s+\\d+\\s+; serial.*|; Generated at .*)$") + ignoreRegexes := []*regexp.Regexp{ + regexp.MustCompile("(?m)^; Generated at .*$"), + regexp.MustCompile("(?m)^\\s+\\d+\\s+; serial.*$"), + } + cw := cache.New(zoneTemplate, ignoreRegexes, true) for zone, records := range zoneRecordsMap { templateArgs.Records = records @@ -264,10 +266,7 @@ func GenerateZones(addresses []model.IPAddress, defaultSoaInfo SOAInfo, conf *co _, err := cw.WriteTemplate( fmt.Sprintf("generated/zones/%s.db", zone), - zoneTemplate, templateArgs, - []*regexp.Regexp{ignoreRegex}, - true, ) if err != nil { panic(err) diff --git a/ns/ipl/ipl.go b/ns/ipl/ipl.go index fe4579f..48bf6ac 100644 --- a/ns/ipl/ipl.go +++ b/ns/ipl/ipl.go @@ -58,9 +58,11 @@ func GenerateIPLists(addresses []model.IPAddress, conf *config.NXConfig) { panic(err) } iplTemplate := template.Must(template.New("ipl").Parse(string(templateString))) + ignoreRegexes := []*regexp.Regexp{ + regexp.MustCompile("(?m)^# Generated at .*$"), + } - cw := cache.New("generated/hashes/ipl.json") - ignoreRegex := regexp.MustCompile("(?m)^# Generated at .*$") + cw := cache.New(iplTemplate, ignoreRegexes, false) for group, ips := range groupMap { vars.Name = group @@ -68,10 +70,7 @@ func GenerateIPLists(addresses []model.IPAddress, conf *config.NXConfig) { _, err := cw.WriteTemplate( fmt.Sprintf("generated/ipl/%s.ipl.txt", group), - iplTemplate, vars, - []*regexp.Regexp{ignoreRegex}, - false, ) if err != nil { panic(err) diff --git a/ns/wg/wireguard.go b/ns/wg/wireguard.go index 7a2f7b7..803eb5b 100644 --- a/ns/wg/wireguard.go +++ b/ns/wg/wireguard.go @@ -62,8 +62,8 @@ func GenerateWgConfigs(ips []model.IPAddress, conf *config.NXConfig) { if err != nil { panic(err) } - zoneTemplate := template.Must(template.New("wg-config").Parse(string(templateString))) - cw := cache.New("generated/hashes/wg.json") + wgTemplate := template.Must(template.New("wg-config").Parse(string(templateString))) + cw := cache.New(wgTemplate, []*regexp.Regexp{}, false) for vpnName, peers := range vpnPeers { for _, peer := range peers { @@ -82,11 +82,8 @@ func GenerateWgConfigs(ips []model.IPAddress, conf *config.NXConfig) { } _, err := cw.WriteTemplate( - fmt.Sprintf("generated/wg/%s_%s.conf", vpnName, data.ServerName), - zoneTemplate, + fmt.Sprintf("generated/wg/%s-%s.conf", vpnName, data.ServerName), data, - []*regexp.Regexp{}, - false, ) if err != nil { panic(err) diff --git a/tagparser/tagparser.go b/tagparser/tagparser.go index 468023e..294667d 100644 --- a/tagparser/tagparser.go +++ b/tagparser/tagparser.go @@ -8,6 +8,8 @@ import ( "strconv" ) +const NoValueErrStr = "no value available" + var tagRegex = regexp.MustCompile("^(\\w+),ns:(\\w+)$") type annotatedField struct { @@ -103,7 +105,7 @@ func findValueForField(field annotatedField, tags []model.Tag) (interface{}, err if sKind == reflect.String { if len(strValues) == 0 { - return nil, fmt.Errorf("no value available") + return nil, fmt.Errorf(NoValueErrStr) } return strValues, nil } else if sKind == reflect.Int { @@ -123,14 +125,14 @@ func findValueForField(field annotatedField, tags []model.Tag) (interface{}, err } else if fKind == reflect.String { if len(strValues) == 0 { //fmt.Printf("warn: No values available for string field <%s>. Returning empty string.\n", field.sField.Name) - return "", fmt.Errorf("no value available") + return "", fmt.Errorf(NoValueErrStr) } return strValues[0], nil } else if fKind == reflect.Int { if len(strValues) == 0 { //fmt.Printf("warn: No values available for int field <%s>. Returning 0.\n", field.sField.Name) - return 0, fmt.Errorf("no value available") + return 0, fmt.Errorf(NoValueErrStr) } for _, val := range strValues { @@ -144,11 +146,11 @@ func findValueForField(field annotatedField, tags []model.Tag) (interface{}, err } //fmt.Printf("warn: No values available for int field <%s>. Returning 0.\n", field.sField.Name) - return 0, fmt.Errorf("no value available") + return 0, fmt.Errorf(NoValueErrStr) } else if fKind == reflect.Bool { if len(strValues) == 0 { //fmt.Printf("warn: No values available for bool field <%s>. Returning false.\n", field.sField.Name) - return false, fmt.Errorf("no value available") + return false, fmt.Errorf(NoValueErrStr) } for _, val := range strValues { @@ -162,7 +164,7 @@ func findValueForField(field annotatedField, tags []model.Tag) (interface{}, err } //fmt.Printf("warn: No values available for bool field <%s>. Returning false.\n", field.sField.Name) - return false, fmt.Errorf("no value available") + return false, fmt.Errorf(NoValueErrStr) } panic(fmt.Sprintf("Unsupported field type <%v>", fKind))