diff --git a/gezer.go b/gezer.go index ad97d6b..4496ce1 100644 --- a/gezer.go +++ b/gezer.go @@ -12,11 +12,12 @@ import ( type Gezer struct { client *http.Client wg sync.WaitGroup - Parse func(response *Response) + opt Opt +} - startURLs []string - startedProcessing int - finishedProcessing int +type Opt struct { + StartURLs []string + ParseFunc func(response *Response) } type Response struct { @@ -25,20 +26,19 @@ type Response struct { Doc *goquery.Document } -func NewGezer(parse func(response *Response), startURLs ...string) *Gezer { +func NewGezer(opt Opt) *Gezer { return &Gezer{ client: &http.Client{ Timeout: time.Second * 10, }, - Parse: parse, - startURLs: startURLs, + opt: opt, } } func (g *Gezer) Start() { - g.wg.Add(len(g.startURLs)) + g.wg.Add(len(g.opt.StartURLs)) - for _, url := range g.startURLs { + for _, url := range g.opt.StartURLs { go g.getRequest(url) } @@ -73,6 +73,6 @@ func (g *Gezer) getRequest(url string) { Doc: doc, } - // Parse response - g.Parse(&response) + // ParseFunc response + g.opt.ParseFunc(&response) } diff --git a/gezer_test.go b/gezer_test.go index bcc6868..3ab5b61 100644 --- a/gezer_test.go +++ b/gezer_test.go @@ -2,25 +2,28 @@ package gezer import ( "fmt" + "github.com/PuerkitoBio/goquery" "testing" ) func TestGezer_StartURLs_Simple(t *testing.T) { - gezer := NewGezer(parse, "https://api.ipify.org", "https://api.ipify.org") + gezer := NewGezer(Opt{ + StartURLs: []string{"https://api.ipify.org", "https://api.ipify.org"}, + ParseFunc: func(r *Response) { + fmt.Println(string(r.Body)) + }, + }) gezer.Start() } -func parse(response *Response) { - fmt.Println(string(response.Body)) +func TestGezer_StartURLs_HTML(t *testing.T) { + gezer := NewGezer(Opt{ + StartURLs: []string{"http://quotes.toscrape.com/"}, + ParseFunc: func(r *Response) { + r.Doc.Find("div.quote").Each(func(i int, s *goquery.Selection) { + fmt.Println(i, s.Find("span.text").Text(), s.Find("small.author").Text()) + }) + }, + }) + gezer.Start() } - -//func TestGezer_StartURLs_HTML(t *testing.T) { -// gezer := NewGezer(parse, "http://quotes.toscrape.com/") -// gezer.Start() -// for result := range gezer.Results { -// result.Doc.Find("div.quote").Each(func(_ int, s *goquery.Selection) { -// fmt.Println(s.Find("span.text").Text()) -// fmt.Println(s.Find("small.author").Text()) -// }) -// } -//}