use a semaphore for up to 5 concurrent link checks and show when resu…

…lts will be removed in results page

go.mod

@@ -2,4 +2,7 @@ module codeberg.org/meadowingc/fido
 
 go 1.22.0
 
-require github.com/google/uuid v1.6.0
+require (
+	github.com/google/uuid v1.6.0
+	golang.org/x/sync v0.6.0
+)

go.sum

@@ -1,2 +1,4 @@
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=

linkchecker/scheduler.go

@@ -1,11 +1,13 @@
 package linkchecker
 
 import (
+	"context"
 	"log"
 	"sync"
 	"time"
 
 	"github.com/google/uuid"
+	"golang.org/x/sync/semaphore"
 )
 
 type CheckLinkRequest struct {
@@ -14,10 +16,11 @@ type CheckLinkRequest struct {
 }
 
 type SchedulerResult struct {
-	UUID   string
-	URL    string
-	Status string
-	Result *LinkCheckResult
+	UUID       string
+	URL        string
+	Status     string
+	ValidUntil time.Time
+	Result     *LinkCheckResult
 }
 
 var (
@@ -31,19 +34,29 @@ func GetResultForUUID(uid string) *SchedulerResult {
 	return uidToResultMap[uid]
 }
 
+var semaphoreLinkCheck = semaphore.NewWeighted(5)
+
 func SubmitLinkForCheck(link string) string {
 	// TODO better for this to be the sha256 of the link
 	uid := uuid.New().String()
 
 	uidToResultMap[uid] = &SchedulerResult{
-		UUID:   uid,
-		URL:    link,
-		Status: "PENDING",
+		UUID:       uid,
+		URL:        link,
+		ValidUntil: time.Now().Add(2 * 24 * time.Hour),
+		Status:     "PENDING",
 	}
 
 	log.Printf("Submitted link %s for check with UUID %s", link, uid)
 
 	go func() {
+		// Acquire a spot in the semaphore
+		if err := semaphoreLinkCheck.Acquire(context.Background(), 1); err != nil {
+			log.Printf("Failed to acquire semaphore: %v", err)
+			return
+		}
+		defer semaphoreLinkCheck.Release(1)
+
 		result, err := CheckLink(link)
 		log.Printf("Link check for %s completed with %d errors", link, len(result.FoundErrors))
 
@@ -59,7 +72,8 @@ func SubmitLinkForCheck(link string) string {
 
 		// keep the results around for a while and then remove from the map
 		go func() {
-			<-time.After(2 * 24 * time.Hour)
+			// wait until uidToResultMap[uid].ValidUntil
+			<-time.After(time.Until(uidToResultMap[uid].ValidUntil))
 			mapLock.Lock()
 			delete(uidToResultMap, uid)
 			mapLock.Unlock()

main.go

@@ -3,13 +3,21 @@ package main
 import (
 	"fmt"
 	"html/template"
+	"log"
 	"net/http"
+	"os/exec"
 	"strings"
 
 	"codeberg.org/meadowingc/fido/linkchecker"
 )
 
 func main() {
+	// check that `linkchecker` is actually found on PATH
+	_, err := exec.LookPath("linkchecker")
+	if err != nil {
+		log.Fatalf("linkchecker is not found in system's PATH: %v", err)
+	}
+
 	templates := template.Must(template.New("").Funcs(template.FuncMap{
 		"add": func(a, b int) int {
 			return a + b
@@ -47,6 +55,12 @@ func main() {
 	http.HandleFunc("/result/{operation_id}", func(w http.ResponseWriter, r *http.Request) {
 		opId := r.PathValue("operation_id")
 
+		templates = template.Must(template.New("").Funcs(template.FuncMap{
+			"add": func(a, b int) int {
+				return a + b
+			},
+		}).ParseGlob("templates/*.tmpl.html"))
+
 		// get result for operation ID
 		result := linkchecker.GetResultForUUID(opId)
 		if result == nil {

templates/result.tmpl.html

@@ -27,6 +27,9 @@ <h1>Results</h1>
         </ul>
         {{- end -}}
 
+        <p>
+            <small>These results will be removed from Fido on <code>{{.ValidUntil.Local}}</code></small>
+        </p>
 
         <p>
             <a href="/">Submit another link</a>