Call ErrorFunc on errors. Unexport DoRequestClient and DoRequestChrome

client/client.go

@@ -89,9 +89,9 @@ func newClientDefault() *Client {
 // DoRequest selects appropriate request handler, client or Chrome
 func (c *Client) DoRequest(req *Request) (resp *Response, err error) {
 	if req.Rendered {
-		resp, err = c.DoRequestChrome(req)
+		resp, err = c.doRequestChrome(req)
 	} else {
-		resp, err = c.DoRequestClient(req)
+		resp, err = c.doRequestClient(req)
 	}
 
 	// Retry on Error
@@ -120,8 +120,8 @@ func (c *Client) DoRequest(req *Request) (resp *Response, err error) {
 	return
 }
 
-// DoRequestClient is a simple wrapper to read response according to options.
-func (c *Client) DoRequestClient(req *Request) (*Response, error) {
+// doRequestClient is a simple wrapper to read response according to options.
+func (c *Client) doRequestClient(req *Request) (*Response, error) {
 	// Do request
 	resp, err := c.Do(req.Request)
 	if resp != nil {
@@ -164,8 +164,8 @@ func (c *Client) DoRequestClient(req *Request) (*Response, error) {
 	return &response, nil
 }
 
-// DoRequestChrome opens up a new chrome instance and makes request
-func (c *Client) DoRequestChrome(req *Request) (*Response, error) {
+// doRequestChrome opens up a new chrome instance and makes request
+func (c *Client) doRequestChrome(req *Request) (*Response, error) {
 	var body string
 	var res *network.Response
 

geziyor.go

@@ -251,7 +251,11 @@ func (g *Geziyor) do(req *client.Request, callback func(g *Geziyor, r *client.Re
 
 	res, err := g.Client.DoRequest(req)
 	if err != nil {
-		log.Println(err)
+		if g.Opt.ErrorFunc != nil {
+			g.Opt.ErrorFunc(g, req, err)
+		} else {
+			log.Println(err)
+		}
 		return
 	}
 

middleware/robotstxt.go

@@ -44,7 +44,7 @@ func (m *RobotsTxt) ProcessRequest(r *client.Request) {
 		}
 
 		m.metrics.RobotsTxtRequestCounter.Add(1)
-		robotsResp, err := m.client.DoRequestClient(robotsReq)
+		robotsResp, err := m.client.DoRequest(robotsReq)
 		if err != nil {
 			return // Don't Do anything
 		}

options.go

@@ -44,6 +44,10 @@ type Options struct {
 	// If set true, cookies won't send.
 	CookiesDisabled bool
 
+	// ErrorFunc is callback of errors.
+	// If not defined, all errors will be logged.
+	ErrorFunc func(g *Geziyor, r *client.Request, err error)
+
 	// For extracting data
 	Exporters []export.Exporter