Options used instead of direct parameter passing.
Tests updated.
This commit is contained in:
parent
6358b87472
commit
944bd3bada
22
gezer.go
22
gezer.go
@ -12,11 +12,12 @@ import (
|
|||||||
type Gezer struct {
|
type Gezer struct {
|
||||||
client *http.Client
|
client *http.Client
|
||||||
wg sync.WaitGroup
|
wg sync.WaitGroup
|
||||||
Parse func(response *Response)
|
opt Opt
|
||||||
|
}
|
||||||
|
|
||||||
startURLs []string
|
type Opt struct {
|
||||||
startedProcessing int
|
StartURLs []string
|
||||||
finishedProcessing int
|
ParseFunc func(response *Response)
|
||||||
}
|
}
|
||||||
|
|
||||||
type Response struct {
|
type Response struct {
|
||||||
@ -25,20 +26,19 @@ type Response struct {
|
|||||||
Doc *goquery.Document
|
Doc *goquery.Document
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewGezer(parse func(response *Response), startURLs ...string) *Gezer {
|
func NewGezer(opt Opt) *Gezer {
|
||||||
return &Gezer{
|
return &Gezer{
|
||||||
client: &http.Client{
|
client: &http.Client{
|
||||||
Timeout: time.Second * 10,
|
Timeout: time.Second * 10,
|
||||||
},
|
},
|
||||||
Parse: parse,
|
opt: opt,
|
||||||
startURLs: startURLs,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (g *Gezer) Start() {
|
func (g *Gezer) Start() {
|
||||||
g.wg.Add(len(g.startURLs))
|
g.wg.Add(len(g.opt.StartURLs))
|
||||||
|
|
||||||
for _, url := range g.startURLs {
|
for _, url := range g.opt.StartURLs {
|
||||||
go g.getRequest(url)
|
go g.getRequest(url)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -73,6 +73,6 @@ func (g *Gezer) getRequest(url string) {
|
|||||||
Doc: doc,
|
Doc: doc,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse response
|
// ParseFunc response
|
||||||
g.Parse(&response)
|
g.opt.ParseFunc(&response)
|
||||||
}
|
}
|
||||||
|
@ -2,25 +2,28 @@ package gezer
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/PuerkitoBio/goquery"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestGezer_StartURLs_Simple(t *testing.T) {
|
func TestGezer_StartURLs_Simple(t *testing.T) {
|
||||||
gezer := NewGezer(parse, "https://api.ipify.org", "https://api.ipify.org")
|
gezer := NewGezer(Opt{
|
||||||
|
StartURLs: []string{"https://api.ipify.org", "https://api.ipify.org"},
|
||||||
|
ParseFunc: func(r *Response) {
|
||||||
|
fmt.Println(string(r.Body))
|
||||||
|
},
|
||||||
|
})
|
||||||
gezer.Start()
|
gezer.Start()
|
||||||
}
|
}
|
||||||
|
|
||||||
func parse(response *Response) {
|
func TestGezer_StartURLs_HTML(t *testing.T) {
|
||||||
fmt.Println(string(response.Body))
|
gezer := NewGezer(Opt{
|
||||||
|
StartURLs: []string{"http://quotes.toscrape.com/"},
|
||||||
|
ParseFunc: func(r *Response) {
|
||||||
|
r.Doc.Find("div.quote").Each(func(i int, s *goquery.Selection) {
|
||||||
|
fmt.Println(i, s.Find("span.text").Text(), s.Find("small.author").Text())
|
||||||
|
})
|
||||||
|
},
|
||||||
|
})
|
||||||
|
gezer.Start()
|
||||||
}
|
}
|
||||||
|
|
||||||
//func TestGezer_StartURLs_HTML(t *testing.T) {
|
|
||||||
// gezer := NewGezer(parse, "http://quotes.toscrape.com/")
|
|
||||||
// gezer.Start()
|
|
||||||
// for result := range gezer.Results {
|
|
||||||
// result.Doc.Find("div.quote").Each(func(_ int, s *goquery.Selection) {
|
|
||||||
// fmt.Println(s.Find("span.text").Text())
|
|
||||||
// fmt.Println(s.Find("small.author").Text())
|
|
||||||
// })
|
|
||||||
// }
|
|
||||||
//}
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user