Retrying support added for chrome. Fixed robots.txt retry issue. Fixed Meta issue

2019-07-07 19:50:15 +03:00
parent 90d2be2210
commit d3c4389c46
4 changed files with 41 additions and 28 deletions
--- a/middleware/robotstxt.go
+++ b/middleware/robotstxt.go
@ -34,7 +34,6 @@ func (m *RobotsTxt) ProcessRequest(r *client.Request) {
 	m.mut.RUnlock()

 	if !exists {
-		// TODO: Disable retry
 		robotsReq, err := client.NewRequest("GET", r.URL.Scheme+"://"+r.Host+"/robots.txt", nil)
 		if err != nil {
 			return // Don't Do anything