初始化

middleware/allowed_domains.go

@@ -1,8 +1,8 @@
 package middleware
 
 import (
-	"github.com/geziyor/geziyor/client"
-	"github.com/geziyor/geziyor/internal"
+	"softdown.com/shusou/geziyor/client"
+	"softdown.com/shusou/geziyor/internal"
 	"sync"
 )
 

middleware/delay.go

@@ -1,8 +1,8 @@
 package middleware
 
 import (
-	"github.com/geziyor/geziyor/client"
 	"math/rand"
+	"softdown.com/shusou/geziyor/client"
 	"time"
 )
 

middleware/duplicate_requests.go

@@ -1,8 +1,8 @@
 package middleware
 
 import (
-	"github.com/geziyor/geziyor/client"
-	"github.com/geziyor/geziyor/internal"
+	"softdown.com/shusou/geziyor/client"
+	"softdown.com/shusou/geziyor/internal"
 	"sync"
 )
 

middleware/duplicate_requests_test.go

@@ -1,8 +1,8 @@
 package middleware
 
 import (
-	"github.com/geziyor/geziyor/client"
 	"github.com/stretchr/testify/assert"
+	"softdown.com/shusou/geziyor/client"
 	"strings"
 	"testing"
 )

middleware/headers.go

@@ -1,7 +1,7 @@
 package middleware
 
 import (
-	"github.com/geziyor/geziyor/client"
+	"softdown.com/shusou/geziyor/client"
 )
 
 // Headers sets default request headers

middleware/log_stats.go

@@ -1,8 +1,8 @@
 package middleware
 
 import (
-	"github.com/geziyor/geziyor/client"
-	"github.com/geziyor/geziyor/internal"
+	"softdown.com/shusou/geziyor/client"
+	"softdown.com/shusou/geziyor/internal"
 )
 
 // LogStats logs responses

middleware/metrics.go

@@ -1,8 +1,8 @@
 package middleware
 
 import (
-	"github.com/geziyor/geziyor/client"
-	"github.com/geziyor/geziyor/metrics"
+	"softdown.com/shusou/geziyor/client"
+	"softdown.com/shusou/geziyor/metrics"
 	"strconv"
 )
 

middleware/middleware.go

@@ -1,7 +1,7 @@
 package middleware
 
 import (
-	"github.com/geziyor/geziyor/client"
+	"softdown.com/shusou/geziyor/client"
 )
 
 // RequestResponseProcessor interface is for middlewares that needs to process both requests and responses

middleware/parse_html.go

@@ -3,8 +3,8 @@ package middleware
 import (
 	"bytes"
 	"github.com/PuerkitoBio/goquery"
-	"github.com/geziyor/geziyor/client"
-	"github.com/geziyor/geziyor/internal"
+	"softdown.com/shusou/geziyor/client"
+	"softdown.com/shusou/geziyor/internal"
 )
 
 // ParseHTML parses response if response is HTML

middleware/robotstxt.go

@@ -1,10 +1,11 @@
 package middleware
 
 import (
-	"github.com/geziyor/geziyor/client"
-	"github.com/geziyor/geziyor/internal"
-	"github.com/geziyor/geziyor/metrics"
 	"github.com/temoto/robotstxt"
+	"net/http"
+	"softdown.com/shusou/geziyor/client"
+	"softdown.com/shusou/geziyor/internal"
+	"softdown.com/shusou/geziyor/metrics"
 	"strconv"
 	"sync"
 )
@@ -38,7 +39,7 @@ func (m *RobotsTxt) ProcessRequest(r *client.Request) {
 	m.mut.RUnlock()
 
 	if !exists {
-		robotsReq, err := client.NewRequest("GET", r.URL.Scheme+"://"+r.Host+"/robots.txt", nil)
+		robotsReq, err := client.NewRequest(http.MethodGet, r.URL.Scheme+"://"+r.Host+"/robots.txt", "", nil)
 		if err != nil {
 			return // Don't Do anything
 		}