Added logging on allowed domains middleware and duplicate requests

middleware/duplicate_requests.go

@@ -2,6 +2,7 @@ package middleware
 
 import (
 	"github.com/geziyor/geziyor/client"
+	"log"
 	"sync"
 )
 
@@ -9,12 +10,16 @@ import (
 type DuplicateRequests struct {
 	RevisitEnabled bool
 	visitedURLs    sync.Map
+	logOnlyOnce    sync.Map
 }
 
 func (a *DuplicateRequests) ProcessRequest(r *client.Request) {
 	if !a.RevisitEnabled && r.Request.Method == "GET" {
-		if _, visited := a.visitedURLs.LoadOrStore(r.Request.URL.String(), struct{}{}); visited {
-			//log.Printf("URL already visited %s\n")
+		requestURL := r.Request.URL.String()
+		if _, visited := a.visitedURLs.LoadOrStore(requestURL, struct{}{}); visited {
+			if _, logged := a.logOnlyOnce.LoadOrStore(requestURL, struct{}{}); !logged {
+				log.Printf("URL already visited %s\n", requestURL)
+			}
 			r.Cancel()
 		}
 	}