gitea/geziyor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Error handling improved

Browse Source
This commit is contained in:
Musab Gültekin 2019-06-20 10:14:36 +03:00
parent f88b88986c
commit 141bab0d05
5 changed files with 11 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
exporter/csv.go
View File

@ -20,7 +20,7 @@ func (e *CSVExporter) Export(exports chan interface{}) {
// Create file // Create file
newFile, err := os.OpenFile(internal.PreferFirst(e.FileName, "out.csv"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) newFile, err := os.OpenFile(internal.PreferFirst(e.FileName, "out.csv"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil { if err != nil {
log.Printf("output file creation error: %v", err) log.Printf("Output file creation error: %v\n", err)
return return
} }

2
exporter/json.go
View File

@ -19,7 +19,7 @@ func (e *JSONExporter) Export(exports chan interface{}) {
// Create file // Create file
newFile, err := os.OpenFile(internal.PreferFirst(e.FileName, "out.json"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) newFile, err := os.OpenFile(internal.PreferFirst(e.FileName, "out.json"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil { if err != nil {
log.Printf("output file creation error: %v", err) log.Printf("Output file creation error: %v\n", err)
return return
} }

14
geziyor.go
View File

@ -7,6 +7,7 @@ import (
"github.com/chromedp/chromedp" "github.com/chromedp/chromedp"
"github.com/fpfeng/httpcache" "github.com/fpfeng/httpcache"
"github.com/geziyor/geziyor/internal" "github.com/geziyor/geziyor/internal"
"github.com/pkg/errors"
"golang.org/x/net/html/charset" "golang.org/x/net/html/charset"
"io" "io"
"io/ioutil" "io/ioutil"
@ -183,6 +184,7 @@ func (g *Geziyor) do(req *Request, callback func(g *Geziyor, r *Response)) {
response, err = g.doRequestChrome(req) response, err = g.doRequestChrome(req)
} }
if err != nil { if err != nil {
log.Println(err)
return return
} }
@ -208,8 +210,7 @@ func (g *Geziyor) doRequestClient(req *Request) (*Response, error) {
defer resp.Body.Close() defer resp.Body.Close()
} }
if err != nil { if err != nil {
log.Printf("Response error: %v\n", err) return nil, errors.Wrap(err, "Response error")
return nil, err
} }
// Limit response body reading // Limit response body reading
@ -219,15 +220,13 @@ func (g *Geziyor) doRequestClient(req *Request) (*Response, error) {
if !g.Opt.CharsetDetectDisabled && resp.Request.Method != "HEAD" { if !g.Opt.CharsetDetectDisabled && resp.Request.Method != "HEAD" {
bodyReader, err = charset.NewReader(bodyReader, resp.Header.Get("Content-Type")) bodyReader, err = charset.NewReader(bodyReader, resp.Header.Get("Content-Type"))
if err != nil { if err != nil {
log.Printf("Determine encoding error: %v\n", err) return nil, errors.Wrap(err, "Determine encoding error")
return nil, err
} }
} }
body, err := ioutil.ReadAll(bodyReader) body, err := ioutil.ReadAll(bodyReader)
if err != nil { if err != nil {
log.Printf("Reading Body error: %v\n", err) return nil, errors.Wrap(err, "Reading body error")
return nil, err
} }
response := Response{ response := Response{
@ -277,8 +276,7 @@ func (g *Geziyor) doRequestChrome(req *Request) (*Response, error) {
return err return err
}), }),
); err != nil { ); err != nil {
log.Printf("Request getting rendered error: %v\n", err) return nil, errors.Wrap(err, "Request getting rendered error")
return nil, err
} }
// Set new URL in case of redirection // Set new URL in case of redirection

1
go.mod
View File

@ -8,6 +8,7 @@ require (
github.com/chromedp/chromedp v0.3.1-0.20190617065505-d55cf9043e05 github.com/chromedp/chromedp v0.3.1-0.20190617065505-d55cf9043e05
github.com/fortytw2/leaktest v1.3.0 github.com/fortytw2/leaktest v1.3.0
github.com/fpfeng/httpcache v0.0.0-20181220155740-6b8f16a92be3 github.com/fpfeng/httpcache v0.0.0-20181220155740-6b8f16a92be3
github.com/pkg/errors v0.8.1
golang.org/x/net v0.0.0-20190522155817-f3200d17e092 golang.org/x/net v0.0.0-20190522155817-f3200d17e092
golang.org/x/text v0.3.2 // indirect golang.org/x/text v0.3.2 // indirect
) )

2
go.sum
View File

@ -20,6 +20,8 @@ github.com/knq/sysutil v0.0.0-20181215143952-f05b59f0f307 h1:vl4eIlySbjertFaNwiM
github.com/knq/sysutil v0.0.0-20181215143952-f05b59f0f307/go.mod h1:BjPj+aVjl9FW/cCGiF3nGh5v+9Gd3VCgBQbod/GlMaQ= github.com/knq/sysutil v0.0.0-20181215143952-f05b59f0f307/go.mod h1:BjPj+aVjl9FW/cCGiF3nGh5v+9Gd3VCgBQbod/GlMaQ=
github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983 h1:wL11wNW7dhKIcRCHSm4sHKPWz0tt4mwBsVodG7+Xyqg= github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983 h1:wL11wNW7dhKIcRCHSm4sHKPWz0tt4mwBsVodG7+Xyqg=
github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=