gitea/geziyor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Callback are now mandatory as almost all the scrapers use it.

Browse Source
This commit is contained in:
Musab Gültekin
2019-06-11 14:24:48 +03:00
parent ca2414c5c8
commit e4e8723426
5 changed files with 52 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
geziyor.go
View File

@ -73,7 +73,7 @@ func NewGeziyor(opt Options) *Geziyor {
// Start starts scraping
func (g *Geziyor) Start() {
for _, startURL := range g.opt.StartURLs {
go g.Get(startURL)
go g.Get(startURL, g.opt.ParseFunc)
}
time.Sleep(time.Millisecond)
@ -81,27 +81,27 @@ func (g *Geziyor) Start() {
}
// Get issues a GET to the specified URL.
func (g *Geziyor) Get(url string, callbacks ...func(resp *Response)) {
func (g *Geziyor) Get(url string, callback func(resp *Response)) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Printf("Request creating error %v\n", err)
return
}
g.Do(req, callbacks...)
g.Do(req, callback)
}
// Head issues a HEAD to the specified URL
func (g *Geziyor) Head(url string, callbacks ...func(resp *Response)) {
func (g *Geziyor) Head(url string, callback func(resp *Response)) {
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
log.Printf("Request creating error %v\n", err)
return
}
g.Do(req, callbacks...)
g.Do(req, callback)
}
// Do sends an HTTP request
func (g *Geziyor) Do(req *http.Request, callbacks ...func(resp *Response)) {
func (g *Geziyor) Do(req *http.Request, callback func(resp *Response)) {
g.wg.Add(1)
defer g.wg.Done()
defer func() {
@ -179,12 +179,12 @@ func (g *Geziyor) Do(req *http.Request, callbacks ...func(resp *Response)) {
// Export Function
go Export(&response)
// ParseFunc response
if len(callbacks) == 0 && g.opt.ParseFunc != nil {
g.opt.ParseFunc(&response)
// Callbacks
if callback != nil {
callback(&response)
} else {
for _, callback := range callbacks {
callback(&response)
if g.opt.ParseFunc != nil {
g.opt.ParseFunc(&response)
}
}
time.Sleep(time.Millisecond)