diff --git a/client/client.go b/client/client.go index 24ae4c2..16ae6ca 100644 --- a/client/client.go +++ b/client/client.go @@ -89,9 +89,9 @@ func newClientDefault() *Client { // DoRequest selects appropriate request handler, client or Chrome func (c *Client) DoRequest(req *Request) (resp *Response, err error) { if req.Rendered { - resp, err = c.DoRequestChrome(req) + resp, err = c.doRequestChrome(req) } else { - resp, err = c.DoRequestClient(req) + resp, err = c.doRequestClient(req) } // Retry on Error @@ -120,8 +120,8 @@ func (c *Client) DoRequest(req *Request) (resp *Response, err error) { return } -// DoRequestClient is a simple wrapper to read response according to options. -func (c *Client) DoRequestClient(req *Request) (*Response, error) { +// doRequestClient is a simple wrapper to read response according to options. +func (c *Client) doRequestClient(req *Request) (*Response, error) { // Do request resp, err := c.Do(req.Request) if resp != nil { @@ -164,8 +164,8 @@ func (c *Client) DoRequestClient(req *Request) (*Response, error) { return &response, nil } -// DoRequestChrome opens up a new chrome instance and makes request -func (c *Client) DoRequestChrome(req *Request) (*Response, error) { +// doRequestChrome opens up a new chrome instance and makes request +func (c *Client) doRequestChrome(req *Request) (*Response, error) { var body string var res *network.Response diff --git a/geziyor.go b/geziyor.go index d19768c..eca6236 100644 --- a/geziyor.go +++ b/geziyor.go @@ -251,7 +251,11 @@ func (g *Geziyor) do(req *client.Request, callback func(g *Geziyor, r *client.Re res, err := g.Client.DoRequest(req) if err != nil { - log.Println(err) + if g.Opt.ErrorFunc != nil { + g.Opt.ErrorFunc(g, req, err) + } else { + log.Println(err) + } return } diff --git a/middleware/robotstxt.go b/middleware/robotstxt.go index d6bb4c6..8dc549f 100644 --- a/middleware/robotstxt.go +++ b/middleware/robotstxt.go @@ -44,7 +44,7 @@ func (m *RobotsTxt) ProcessRequest(r *client.Request) { } m.metrics.RobotsTxtRequestCounter.Add(1) - robotsResp, err := m.client.DoRequestClient(robotsReq) + robotsResp, err := m.client.DoRequest(robotsReq) if err != nil { return // Don't Do anything } diff --git a/options.go b/options.go index 55979b6..701ce5e 100644 --- a/options.go +++ b/options.go @@ -44,6 +44,10 @@ type Options struct { // If set true, cookies won't send. CookiesDisabled bool + // ErrorFunc is callback of errors. + // If not defined, all errors will be logged. + ErrorFunc func(g *Geziyor, r *client.Request, err error) + // For extracting data Exporters []export.Exporter