Options used instead of direct parameter passing.

Tests updated.
This commit is contained in:
Musab Gültekin 2019-06-06 23:49:42 +03:00
parent 6358b87472
commit 944bd3bada
2 changed files with 28 additions and 25 deletions

View File

@ -12,11 +12,12 @@ import (
type Gezer struct { type Gezer struct {
client *http.Client client *http.Client
wg sync.WaitGroup wg sync.WaitGroup
Parse func(response *Response) opt Opt
}
startURLs []string type Opt struct {
startedProcessing int StartURLs []string
finishedProcessing int ParseFunc func(response *Response)
} }
type Response struct { type Response struct {
@ -25,20 +26,19 @@ type Response struct {
Doc *goquery.Document Doc *goquery.Document
} }
func NewGezer(parse func(response *Response), startURLs ...string) *Gezer { func NewGezer(opt Opt) *Gezer {
return &Gezer{ return &Gezer{
client: &http.Client{ client: &http.Client{
Timeout: time.Second * 10, Timeout: time.Second * 10,
}, },
Parse: parse, opt: opt,
startURLs: startURLs,
} }
} }
func (g *Gezer) Start() { func (g *Gezer) Start() {
g.wg.Add(len(g.startURLs)) g.wg.Add(len(g.opt.StartURLs))
for _, url := range g.startURLs { for _, url := range g.opt.StartURLs {
go g.getRequest(url) go g.getRequest(url)
} }
@ -73,6 +73,6 @@ func (g *Gezer) getRequest(url string) {
Doc: doc, Doc: doc,
} }
// Parse response // ParseFunc response
g.Parse(&response) g.opt.ParseFunc(&response)
} }

View File

@ -2,25 +2,28 @@ package gezer
import ( import (
"fmt" "fmt"
"github.com/PuerkitoBio/goquery"
"testing" "testing"
) )
func TestGezer_StartURLs_Simple(t *testing.T) { func TestGezer_StartURLs_Simple(t *testing.T) {
gezer := NewGezer(parse, "https://api.ipify.org", "https://api.ipify.org") gezer := NewGezer(Opt{
StartURLs: []string{"https://api.ipify.org", "https://api.ipify.org"},
ParseFunc: func(r *Response) {
fmt.Println(string(r.Body))
},
})
gezer.Start() gezer.Start()
} }
func parse(response *Response) { func TestGezer_StartURLs_HTML(t *testing.T) {
fmt.Println(string(response.Body)) gezer := NewGezer(Opt{
StartURLs: []string{"http://quotes.toscrape.com/"},
ParseFunc: func(r *Response) {
r.Doc.Find("div.quote").Each(func(i int, s *goquery.Selection) {
fmt.Println(i, s.Find("span.text").Text(), s.Find("small.author").Text())
})
},
})
gezer.Start()
} }
//func TestGezer_StartURLs_HTML(t *testing.T) {
// gezer := NewGezer(parse, "http://quotes.toscrape.com/")
// gezer.Start()
// for result := range gezer.Results {
// result.Doc.Find("div.quote").Each(func(_ int, s *goquery.Selection) {
// fmt.Println(s.Find("span.text").Text())
// fmt.Println(s.Find("small.author").Text())
// })
// }
//}