Options used instead of direct parameter passing.
Tests updated.
This commit is contained in:
parent
6358b87472
commit
944bd3bada
22
gezer.go
22
gezer.go
@ -12,11 +12,12 @@ import (
|
||||
type Gezer struct {
|
||||
client *http.Client
|
||||
wg sync.WaitGroup
|
||||
Parse func(response *Response)
|
||||
opt Opt
|
||||
}
|
||||
|
||||
startURLs []string
|
||||
startedProcessing int
|
||||
finishedProcessing int
|
||||
type Opt struct {
|
||||
StartURLs []string
|
||||
ParseFunc func(response *Response)
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
@ -25,20 +26,19 @@ type Response struct {
|
||||
Doc *goquery.Document
|
||||
}
|
||||
|
||||
func NewGezer(parse func(response *Response), startURLs ...string) *Gezer {
|
||||
func NewGezer(opt Opt) *Gezer {
|
||||
return &Gezer{
|
||||
client: &http.Client{
|
||||
Timeout: time.Second * 10,
|
||||
},
|
||||
Parse: parse,
|
||||
startURLs: startURLs,
|
||||
opt: opt,
|
||||
}
|
||||
}
|
||||
|
||||
func (g *Gezer) Start() {
|
||||
g.wg.Add(len(g.startURLs))
|
||||
g.wg.Add(len(g.opt.StartURLs))
|
||||
|
||||
for _, url := range g.startURLs {
|
||||
for _, url := range g.opt.StartURLs {
|
||||
go g.getRequest(url)
|
||||
}
|
||||
|
||||
@ -73,6 +73,6 @@ func (g *Gezer) getRequest(url string) {
|
||||
Doc: doc,
|
||||
}
|
||||
|
||||
// Parse response
|
||||
g.Parse(&response)
|
||||
// ParseFunc response
|
||||
g.opt.ParseFunc(&response)
|
||||
}
|
||||
|
@ -2,25 +2,28 @@ package gezer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGezer_StartURLs_Simple(t *testing.T) {
|
||||
gezer := NewGezer(parse, "https://api.ipify.org", "https://api.ipify.org")
|
||||
gezer := NewGezer(Opt{
|
||||
StartURLs: []string{"https://api.ipify.org", "https://api.ipify.org"},
|
||||
ParseFunc: func(r *Response) {
|
||||
fmt.Println(string(r.Body))
|
||||
},
|
||||
})
|
||||
gezer.Start()
|
||||
}
|
||||
|
||||
func parse(response *Response) {
|
||||
fmt.Println(string(response.Body))
|
||||
func TestGezer_StartURLs_HTML(t *testing.T) {
|
||||
gezer := NewGezer(Opt{
|
||||
StartURLs: []string{"http://quotes.toscrape.com/"},
|
||||
ParseFunc: func(r *Response) {
|
||||
r.Doc.Find("div.quote").Each(func(i int, s *goquery.Selection) {
|
||||
fmt.Println(i, s.Find("span.text").Text(), s.Find("small.author").Text())
|
||||
})
|
||||
},
|
||||
})
|
||||
gezer.Start()
|
||||
}
|
||||
|
||||
//func TestGezer_StartURLs_HTML(t *testing.T) {
|
||||
// gezer := NewGezer(parse, "http://quotes.toscrape.com/")
|
||||
// gezer.Start()
|
||||
// for result := range gezer.Results {
|
||||
// result.Doc.Find("div.quote").Each(func(_ int, s *goquery.Selection) {
|
||||
// fmt.Println(s.Find("span.text").Text())
|
||||
// fmt.Println(s.Find("small.author").Text())
|
||||
// })
|
||||
// }
|
||||
//}
|
||||
|
Loading…
x
Reference in New Issue
Block a user