gitea/geziyor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Call ErrorFunc on errors. Unexport DoRequestClient and DoRequestChrome

Browse Source
This commit is contained in:
Musab Gültekin 2019-12-13 00:03:44 +03:00
parent 7d2fe57bab
commit cfb16fe1ee
4 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
client/client.go
View File

@ -89,9 +89,9 @@ func newClientDefault() *Client {
// DoRequest selects appropriate request handler, client or Chrome // DoRequest selects appropriate request handler, client or Chrome
func (c *Client) DoRequest(req *Request) (resp *Response, err error) { func (c *Client) DoRequest(req *Request) (resp *Response, err error) {
if req.Rendered { if req.Rendered {
resp, err = c.DoRequestChrome(req) resp, err = c.doRequestChrome(req)
} else { } else {
resp, err = c.DoRequestClient(req) resp, err = c.doRequestClient(req)
} }
// Retry on Error // Retry on Error
@ -120,8 +120,8 @@ func (c *Client) DoRequest(req *Request) (resp *Response, err error) {
return return
} }
// DoRequestClient is a simple wrapper to read response according to options. // doRequestClient is a simple wrapper to read response according to options.
func (c *Client) DoRequestClient(req *Request) (*Response, error) { func (c *Client) doRequestClient(req *Request) (*Response, error) {
// Do request // Do request
resp, err := c.Do(req.Request) resp, err := c.Do(req.Request)
if resp != nil { if resp != nil {
@ -164,8 +164,8 @@ func (c *Client) DoRequestClient(req *Request) (*Response, error) {
return &response, nil return &response, nil
} }
// DoRequestChrome opens up a new chrome instance and makes request // doRequestChrome opens up a new chrome instance and makes request
func (c *Client) DoRequestChrome(req *Request) (*Response, error) { func (c *Client) doRequestChrome(req *Request) (*Response, error) {
var body string var body string
var res *network.Response var res *network.Response

6
geziyor.go
View File

@ -251,7 +251,11 @@ func (g *Geziyor) do(req *client.Request, callback func(g *Geziyor, r *client.Re
res, err := g.Client.DoRequest(req) res, err := g.Client.DoRequest(req)
if err != nil { if err != nil {
log.Println(err) if g.Opt.ErrorFunc != nil {
g.Opt.ErrorFunc(g, req, err)
} else {
log.Println(err)
}
return return
} }

2
middleware/robotstxt.go
View File

@ -44,7 +44,7 @@ func (m *RobotsTxt) ProcessRequest(r *client.Request) {
} }
m.metrics.RobotsTxtRequestCounter.Add(1) m.metrics.RobotsTxtRequestCounter.Add(1)
robotsResp, err := m.client.DoRequestClient(robotsReq) robotsResp, err := m.client.DoRequest(robotsReq)
if err != nil { if err != nil {
return // Don't Do anything return // Don't Do anything
} }

4
options.go
View File

@ -44,6 +44,10 @@ type Options struct {
// If set true, cookies won't send. // If set true, cookies won't send.
CookiesDisabled bool CookiesDisabled bool
// ErrorFunc is callback of errors.
// If not defined, all errors will be logged.
ErrorFunc func(g *Geziyor, r *client.Request, err error)
// For extracting data // For extracting data
Exporters []export.Exporter Exporters []export.Exporter