From 285712549863549e305c85d883ef26333038b865 Mon Sep 17 00:00:00 2001
From: Joel Messerli <git@pegnu.dev>
Date: Mon, 13 May 2024 13:33:27 +0200
Subject: [PATCH 1/3] fix: hash files are no longer necessary and updated files
are written again
---
cache/cached_writer.go | 165 ++++++++++++++++----------------------
main.go | 12 +--
ns/dns/configgenerator.go | 9 +--
ns/dns/zonegenerator.go | 11 ++-
ns/ipl/ipl.go | 9 +--
ns/wg/wireguard.go | 9 +--
6 files changed, 93 insertions(+), 122 deletions(-)
diff --git a/cache/cached_writer.go b/cache/cached_writer.go
index b2bc463..0cede3d 100644
--- a/cache/cached_writer.go
+++ b/cache/cached_writer.go
@@ -3,12 +3,12 @@ package cache
import (
"bytes"
"crypto/sha1"
- "encoding/json"
"fmt"
"io"
"log"
"os"
"regexp"
+ "strings"
"text/tabwriter"
"text/template"
)
@@ -16,86 +16,68 @@ import (
var logger = log.New(os.Stdout, "[cached_writer] ", log.LstdFlags)
type CachedTemplateWriter struct {
- hashFile string
- fileHashes map[string]string
- newHashes map[string]string
- buf bytes.Buffer
- ProcessedFiles []string
- UpdatedFiles []string
+ template *template.Template
+ ignorePatterns []*regexp.Regexp
+ useTabbedWriter bool
+ newHashes map[string]string
+ ProcessedFiles []string
+ UpdatedFiles []string
}
-func empty(hashFile string) *CachedTemplateWriter {
+func New(template *template.Template, ignorePatterns []*regexp.Regexp, useTabbedWriter bool) *CachedTemplateWriter {
return &CachedTemplateWriter{
- hashFile: hashFile,
- fileHashes: map[string]string{},
- newHashes: map[string]string{},
+ template: template,
+ ignorePatterns: ignorePatterns,
+ useTabbedWriter: useTabbedWriter,
+ newHashes: map[string]string{},
}
}
-func New(hashFile string) *CachedTemplateWriter {
- if _, err := os.Stat(hashFile); os.IsNotExist(err) {
- return empty(hashFile)
- }
-
- jsonBytes, err := os.ReadFile(hashFile)
- if err != nil {
- logger.Println("could not open json hash file, discarding")
- return empty(hashFile)
- }
-
- data := map[string]string{}
- err = json.Unmarshal(jsonBytes, &data)
- if err != nil {
- logger.Println("could not parse json hash file, discarding")
- return empty(hashFile)
- }
-
- return &CachedTemplateWriter{
- hashFile: hashFile,
- fileHashes: data,
- newHashes: map[string]string{},
- }
-}
-
-func (w *CachedTemplateWriter) WriteTemplate(
+func (cw *CachedTemplateWriter) WriteTemplate(
file string,
- tpl *template.Template,
data interface{},
- ignorePatterns []*regexp.Regexp,
- useTabbedWriter bool,
) (bool, error) {
- // Reset buffer
- w.buf = bytes.Buffer{}
+ buf := bytes.Buffer{}
+ err := func() error {
+ var bufWriter io.Writer
+ if cw.useTabbedWriter {
+ tw := tabwriter.NewWriter(&buf, 2, 2, 2, ' ', 0)
+ bufWriter = tw
+ defer tw.Flush()
+ } else {
+ bufWriter = &buf
+ }
- err := tpl.Execute(&w.buf, data)
+ err := cw.template.Execute(bufWriter, data)
+ if err != nil {
+ return err
+ }
+ return nil
+ }()
if err != nil {
return false, err
}
- str := string(w.buf.Bytes())
- for _, regex := range ignorePatterns {
- str = regex.ReplaceAllString(str, "-hash:omit-")
- }
-
- hash := sha1.New()
- hash.Write([]byte(str))
- hashBytes := hash.Sum(nil)
- hashStr := fmt.Sprintf("%x", hashBytes)
-
- existingHash, ok := w.fileHashes[file]
- if ok && existingHash == hashStr {
- //logger.Printf("File fresh: %s\n", file)
- w.ProcessedFiles = append(w.ProcessedFiles, file)
- w.newHashes[file] = w.fileHashes[file]
- w.updateJson()
- return false, nil
+ str := string(buf.Bytes())
+ hashStr := cw.hash(str)
+
+ existingFileStr, err := cw.getFileContent(file)
+ if err == nil {
+ existingHash := cw.hash(existingFileStr)
+ if existingHash == hashStr {
+ //logger.Printf("File fresh: %s\n", file)
+ cw.ProcessedFiles = append(cw.ProcessedFiles, file)
+ cw.newHashes[file] = existingHash
+ return false, nil
+ }
+ } else {
+ logger.Printf("ignored error while reading existing file %s: %s\n", file, err.Error())
}
f, err := os.Create(file)
if err != nil {
return false, err
}
-
defer func(closeable io.Closer) {
err := closeable.Close()
if err != nil {
@@ -103,52 +85,47 @@ func (w *CachedTemplateWriter) WriteTemplate(
}
}(f)
- var writer io.Writer
- if useTabbedWriter {
- writer = tabwriter.NewWriter(f, 2, 2, 2, ' ', 0)
- } else {
- writer = f
- }
-
- if useTabbedWriter {
- wr := tabwriter.NewWriter(f, 2, 2, 2, ' ', 0)
- _, err = wr.Write(w.buf.Bytes())
- if err != nil {
- return false, err
- }
- _ = wr.Flush()
- } else {
- _, err = writer.Write(w.buf.Bytes())
- if err != nil {
- return false, err
- }
+ _, err = f.Write(buf.Bytes())
+ if err != nil {
+ return false, err
}
logger.Printf("New hash %s for file %s\n", hashStr, file)
- w.ProcessedFiles = append(w.ProcessedFiles, file)
- w.UpdatedFiles = append(w.UpdatedFiles, file)
- w.fileHashes[file] = hashStr
- w.newHashes[file] = hashStr
- w.updateJson()
+ cw.ProcessedFiles = append(cw.ProcessedFiles, file)
+ cw.UpdatedFiles = append(cw.UpdatedFiles, file)
+ cw.newHashes[file] = hashStr
return true, nil
}
-func (w *CachedTemplateWriter) updateJson() {
- jsonBytes, err := json.Marshal(w.newHashes)
+func (cw *CachedTemplateWriter) getFileContent(file string) (string, error) {
+ stat, err := os.Stat(file)
if err != nil {
- panic(err)
+ return "", err
+ }
+ if stat.IsDir() {
+ return "", fmt.Errorf("%s is a directory", file)
}
- f, err := os.Create(w.hashFile)
+ fileBytes, err := os.ReadFile(file)
if err != nil {
- panic(err)
+ return "", err
}
+ return string(fileBytes), nil
+}
- _, err = f.Write(jsonBytes)
- if err != nil {
- panic(err)
+func (cw *CachedTemplateWriter) hash(content string) string {
+ content = strings.ReplaceAll(content, "\r\n", "\n")
+ content = strings.TrimSpace(content)
+
+ if cw.ignorePatterns != nil {
+ for _, regex := range cw.ignorePatterns {
+ content = regex.ReplaceAllString(content, "-hash:omit-")
+ }
}
- _ = f.Close()
+ hash := sha1.New()
+ hash.Write([]byte(content))
+ hashBytes := hash.Sum(nil)
+ return fmt.Sprintf("%x", hashBytes)
}
diff --git a/main.go b/main.go
index 80491c5..7114ec4 100644
--- a/main.go
+++ b/main.go
@@ -33,7 +33,7 @@ func main() {
prefixIPsList := loadPrefixes(prefixes, nc)
sortPrefixList(prefixIPsList)
- generateAll(prefixIPsList, dnsIps, wgIps, iplIps, conf)
+ generateAll(prefixIPsList, dnsIps, wgIps, iplIps, &conf)
logger.Println("Writing updated files report")
err := os.WriteFile("generated/updated_files.txt", []byte(strings.Join(conf.UpdatedFiles, "\n")), os.ModePerm)
@@ -78,7 +78,7 @@ func sortPrefixList(prefixIPsList []prefixIPs) {
}
}
-func generateAll(prefixIPsList []prefixIPs, dnsIps []model.IPAddress, wgIps []model.IPAddress, iplIps []model.IPAddress, conf config.NXConfig) {
+func generateAll(prefixIPsList []prefixIPs, dnsIps []model.IPAddress, wgIps []model.IPAddress, iplIps []model.IPAddress, conf *config.NXConfig) {
defer util.DurationSince(util.StartTracking("generateAll"))
for _, prefixIP := range prefixIPsList {
@@ -103,14 +103,14 @@ func generateAll(prefixIPsList []prefixIPs, dnsIps []model.IPAddress, wgIps []mo
DottedMailResponsible: "unknown\\.admin.local",
NameserverFQDN: "unknown-nameserver.local.",
- }, &conf)
+ }, conf)
logger.Println("Generating BIND config files")
- dns.GenerateConfigs(generatedZones, &conf)
+ dns.GenerateConfigs(generatedZones, conf)
logger.Println("Generating Wireguard config files")
- wg.GenerateWgConfigs(wgIps, &conf)
+ wg.GenerateWgConfigs(wgIps, conf)
logger.Println("Generating IP lists")
- ipl.GenerateIPLists(iplIps, &conf)
+ ipl.GenerateIPLists(iplIps, conf)
}
type prefixIPs struct {
diff --git a/ns/dns/configgenerator.go b/ns/dns/configgenerator.go
index 534cfc5..fc5571f 100644
--- a/ns/dns/configgenerator.go
+++ b/ns/dns/configgenerator.go
@@ -95,8 +95,10 @@ func GenerateConfigs(zones []string, conf *config.NXConfig) {
panic(err)
}
configTemplate := template.Must(template.New("config").Parse(string(templateString)))
- cw := cache.New("generated/hashes/bind-config.json")
- ignoreRegex := regexp.MustCompile("(?m)^ \\* Generated at.*$")
+ ignoreRegexes := []*regexp.Regexp{
+ regexp.MustCompile("(?m)^ \\* Generated at.*$"),
+ }
+ cw := cache.New(configTemplate, ignoreRegexes, false)
templateVars := configTemplateVars{
GeneratedAt: time.Now().Format(time.RFC3339),
@@ -156,10 +158,7 @@ func GenerateConfigs(zones []string, conf *config.NXConfig) {
_, err = cw.WriteTemplate(
fmt.Sprintf("generated/bind-config/%s.conf", currentMaster.Name),
- configTemplate,
templateVars,
- []*regexp.Regexp{ignoreRegex},
- false,
)
if err != nil {
panic(err)
diff --git a/ns/dns/zonegenerator.go b/ns/dns/zonegenerator.go
index b0a4c32..110e47e 100644
--- a/ns/dns/zonegenerator.go
+++ b/ns/dns/zonegenerator.go
@@ -238,9 +238,11 @@ func GenerateZones(addresses []model.IPAddress, defaultSoaInfo SOAInfo, conf *co
panic(err)
}
zoneTemplate := template.Must(template.New("zone").Parse(string(templateString)))
-
- cw := cache.New("generated/hashes/zones.json")
- ignoreRegex := regexp.MustCompile("(?m)^(\\s+\\d+\\s+; serial.*|; Generated at .*)$")
+ ignoreRegexes := []*regexp.Regexp{
+ regexp.MustCompile("(?m)^; Generated at .*$"),
+ regexp.MustCompile("(?m)^\\s+\\d+\\s+; serial.*$"),
+ }
+ cw := cache.New(zoneTemplate, ignoreRegexes, true)
for zone, records := range zoneRecordsMap {
templateArgs.Records = records
@@ -264,10 +266,7 @@ func GenerateZones(addresses []model.IPAddress, defaultSoaInfo SOAInfo, conf *co
_, err := cw.WriteTemplate(
fmt.Sprintf("generated/zones/%s.db", zone),
- zoneTemplate,
templateArgs,
- []*regexp.Regexp{ignoreRegex},
- true,
)
if err != nil {
panic(err)
diff --git a/ns/ipl/ipl.go b/ns/ipl/ipl.go
index fe4579f..48bf6ac 100644
--- a/ns/ipl/ipl.go
+++ b/ns/ipl/ipl.go
@@ -58,9 +58,11 @@ func GenerateIPLists(addresses []model.IPAddress, conf *config.NXConfig) {
panic(err)
}
iplTemplate := template.Must(template.New("ipl").Parse(string(templateString)))
+ ignoreRegexes := []*regexp.Regexp{
+ regexp.MustCompile("(?m)^# Generated at .*$"),
+ }
- cw := cache.New("generated/hashes/ipl.json")
- ignoreRegex := regexp.MustCompile("(?m)^# Generated at .*$")
+ cw := cache.New(iplTemplate, ignoreRegexes, false)
for group, ips := range groupMap {
vars.Name = group
@@ -68,10 +70,7 @@ func GenerateIPLists(addresses []model.IPAddress, conf *config.NXConfig) {
_, err := cw.WriteTemplate(
fmt.Sprintf("generated/ipl/%s.ipl.txt", group),
- iplTemplate,
vars,
- []*regexp.Regexp{ignoreRegex},
- false,
)
if err != nil {
panic(err)
diff --git a/ns/wg/wireguard.go b/ns/wg/wireguard.go
index 7a2f7b7..803eb5b 100644
--- a/ns/wg/wireguard.go
+++ b/ns/wg/wireguard.go
@@ -62,8 +62,8 @@ func GenerateWgConfigs(ips []model.IPAddress, conf *config.NXConfig) {
if err != nil {
panic(err)
}
- zoneTemplate := template.Must(template.New("wg-config").Parse(string(templateString)))
- cw := cache.New("generated/hashes/wg.json")
+ wgTemplate := template.Must(template.New("wg-config").Parse(string(templateString)))
+ cw := cache.New(wgTemplate, []*regexp.Regexp{}, false)
for vpnName, peers := range vpnPeers {
for _, peer := range peers {
@@ -82,11 +82,8 @@ func GenerateWgConfigs(ips []model.IPAddress, conf *config.NXConfig) {
}
_, err := cw.WriteTemplate(
- fmt.Sprintf("generated/wg/%s_%s.conf", vpnName, data.ServerName),
- zoneTemplate,
+ fmt.Sprintf("generated/wg/%s-%s.conf", vpnName, data.ServerName),
data,
- []*regexp.Regexp{},
- false,
)
if err != nil {
panic(err)
From 21bff6641b87af08314a40573b230fcfe3029896 Mon Sep 17 00:00:00 2001
From: Joel Messerli <git@pegnu.dev>
Date: Mon, 13 May 2024 13:39:11 +0200
Subject: [PATCH 2/3] chore: sonarlint
---
Dockerfile | 4 ++--
tagparser/tagparser.go | 14 ++++++++------
2 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 640b4d1..c104392 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,11 +1,11 @@
-FROM golang:1 as builder
+FROM golang:1 AS builder
LABEL maintainer="Joel Messerli <hi.github@peg.nu>"
WORKDIR /go/src/github.com/jmesserli/nx
COPY . .
RUN go get -d -v ./...
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o /go/bin/nx .
-FROM alpine:latest
+FROM alpine:3
RUN apk --no-cache add ca-certificates tzdata
WORKDIR /root/
COPY --from=builder /go/bin/nx .
diff --git a/tagparser/tagparser.go b/tagparser/tagparser.go
index 468023e..294667d 100644
--- a/tagparser/tagparser.go
+++ b/tagparser/tagparser.go
@@ -8,6 +8,8 @@ import (
"strconv"
)
+const NoValueErrStr = "no value available"
+
var tagRegex = regexp.MustCompile("^(\\w+),ns:(\\w+)$")
type annotatedField struct {
@@ -103,7 +105,7 @@ func findValueForField(field annotatedField, tags []model.Tag) (interface{}, err
if sKind == reflect.String {
if len(strValues) == 0 {
- return nil, fmt.Errorf("no value available")
+ return nil, fmt.Errorf(NoValueErrStr)
}
return strValues, nil
} else if sKind == reflect.Int {
@@ -123,14 +125,14 @@ func findValueForField(field annotatedField, tags []model.Tag) (interface{}, err
} else if fKind == reflect.String {
if len(strValues) == 0 {
//fmt.Printf("warn: No values available for string field <%s>. Returning empty string.\n", field.sField.Name)
- return "", fmt.Errorf("no value available")
+ return "", fmt.Errorf(NoValueErrStr)
}
return strValues[0], nil
} else if fKind == reflect.Int {
if len(strValues) == 0 {
//fmt.Printf("warn: No values available for int field <%s>. Returning 0.\n", field.sField.Name)
- return 0, fmt.Errorf("no value available")
+ return 0, fmt.Errorf(NoValueErrStr)
}
for _, val := range strValues {
@@ -144,11 +146,11 @@ func findValueForField(field annotatedField, tags []model.Tag) (interface{}, err
}
//fmt.Printf("warn: No values available for int field <%s>. Returning 0.\n", field.sField.Name)
- return 0, fmt.Errorf("no value available")
+ return 0, fmt.Errorf(NoValueErrStr)
} else if fKind == reflect.Bool {
if len(strValues) == 0 {
//fmt.Printf("warn: No values available for bool field <%s>. Returning false.\n", field.sField.Name)
- return false, fmt.Errorf("no value available")
+ return false, fmt.Errorf(NoValueErrStr)
}
for _, val := range strValues {
@@ -162,7 +164,7 @@ func findValueForField(field annotatedField, tags []model.Tag) (interface{}, err
}
//fmt.Printf("warn: No values available for bool field <%s>. Returning false.\n", field.sField.Name)
- return false, fmt.Errorf("no value available")
+ return false, fmt.Errorf(NoValueErrStr)
}
panic(fmt.Sprintf("Unsupported field type <%v>", fKind))
From bc84750e0e25635c1ea9468a604d395047430ea5 Mon Sep 17 00:00:00 2001
From: Joel Messerli <git@pegnu.dev>
Date: Mon, 13 May 2024 13:44:28 +0200
Subject: [PATCH 3/3] chore: eliminate unnecessary variable
---
cache/cached_writer.go | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/cache/cached_writer.go b/cache/cached_writer.go
index 0cede3d..98b1539 100644
--- a/cache/cached_writer.go
+++ b/cache/cached_writer.go
@@ -41,9 +41,8 @@ func (cw *CachedTemplateWriter) WriteTemplate(
err := func() error {
var bufWriter io.Writer
if cw.useTabbedWriter {
- tw := tabwriter.NewWriter(&buf, 2, 2, 2, ' ', 0)
- bufWriter = tw
- defer tw.Flush()
+ bufWriter = tabwriter.NewWriter(&buf, 2, 2, 2, ' ', 0)
+ defer bufWriter.(*tabwriter.Writer).Flush()
} else {
bufWriter = &buf
}