gitea/geziyor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Error handling improved

Browse Source
This commit is contained in:
Musab Gültekin 2019-06-20 10:14:36 +03:00
parent f88b88986c
commit 141bab0d05
5 changed files with 11 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
exporter/csv.go
View File

@ -20,7 +20,7 @@ func (e *CSVExporter) Export(exports chan interface{}) {
// Create file
newFile, err := os.OpenFile(internal.PreferFirst(e.FileName, "out.csv"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
log.Printf("output file creation error: %v", err)
log.Printf("Output file creation error: %v\n", err)
return
}

2
exporter/json.go
View File

@ -19,7 +19,7 @@ func (e *JSONExporter) Export(exports chan interface{}) {
// Create file
newFile, err := os.OpenFile(internal.PreferFirst(e.FileName, "out.json"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
log.Printf("output file creation error: %v", err)
log.Printf("Output file creation error: %v\n", err)
return
}

14
geziyor.go
View File

@ -7,6 +7,7 @@ import (
"github.com/chromedp/chromedp"
"github.com/fpfeng/httpcache"
"github.com/geziyor/geziyor/internal"
"github.com/pkg/errors"
"golang.org/x/net/html/charset"
"io"
"io/ioutil"
@ -183,6 +184,7 @@ func (g *Geziyor) do(req *Request, callback func(g *Geziyor, r *Response)) {
response, err = g.doRequestChrome(req)
}
if err != nil {
log.Println(err)
return
}
@ -208,8 +210,7 @@ func (g *Geziyor) doRequestClient(req *Request) (*Response, error) {
defer resp.Body.Close()
}
if err != nil {
log.Printf("Response error: %v\n", err)
return nil, err
return nil, errors.Wrap(err, "Response error")
}
// Limit response body reading
@ -219,15 +220,13 @@ func (g *Geziyor) doRequestClient(req *Request) (*Response, error) {
if !g.Opt.CharsetDetectDisabled && resp.Request.Method != "HEAD" {
bodyReader, err = charset.NewReader(bodyReader, resp.Header.Get("Content-Type"))
if err != nil {
log.Printf("Determine encoding error: %v\n", err)
return nil, err
return nil, errors.Wrap(err, "Determine encoding error")
}
}
body, err := ioutil.ReadAll(bodyReader)
if err != nil {
log.Printf("Reading Body error: %v\n", err)
return nil, err
return nil, errors.Wrap(err, "Reading body error")
}
response := Response{
@ -277,8 +276,7 @@ func (g *Geziyor) doRequestChrome(req *Request) (*Response, error) {
return err
}),
); err != nil {
log.Printf("Request getting rendered error: %v\n", err)
return nil, err
return nil, errors.Wrap(err, "Request getting rendered error")
}
// Set new URL in case of redirection

1
go.mod
View File

@ -8,6 +8,7 @@ require (
github.com/chromedp/chromedp v0.3.1-0.20190617065505-d55cf9043e05
github.com/fortytw2/leaktest v1.3.0
github.com/fpfeng/httpcache v0.0.0-20181220155740-6b8f16a92be3
github.com/pkg/errors v0.8.1
golang.org/x/net v0.0.0-20190522155817-f3200d17e092
golang.org/x/text v0.3.2 // indirect
)

2
go.sum
View File

@ -20,6 +20,8 @@ github.com/knq/sysutil v0.0.0-20181215143952-f05b59f0f307 h1:vl4eIlySbjertFaNwiM
github.com/knq/sysutil v0.0.0-20181215143952-f05b59f0f307/go.mod h1:BjPj+aVjl9FW/cCGiF3nGh5v+9Gd3VCgBQbod/GlMaQ=
github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983 h1:wL11wNW7dhKIcRCHSm4sHKPWz0tt4mwBsVodG7+Xyqg=
github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=