Robots.txt support implemented

2019-07-06 16:18:03 +03:00
parent 2cab68d2ce
commit 42faa92ece
9 changed files with 154 additions and 64 deletions
--- a/middleware/middleware.go
+++ b/middleware/middleware.go
@ -4,6 +4,7 @@ import (
 	"github.com/geziyor/geziyor/client"
 )

+// RequestResponseProcessor interface is for middlewares that needs to process both requests and responses
 type RequestResponseProcessor interface {
 	RequestProcessor
 	ResponseProcessor