Options used instead of direct parameter passing.
Tests updated.
This commit is contained in:
		
							
								
								
									
										22
									
								
								gezer.go
									
									
									
									
									
								
							
							
						
						
									
										22
									
								
								gezer.go
									
									
									
									
									
								
							| @@ -12,11 +12,12 @@ import ( | ||||
| type Gezer struct { | ||||
| 	client *http.Client | ||||
| 	wg     sync.WaitGroup | ||||
| 	Parse  func(response *Response) | ||||
| 	opt    Opt | ||||
| } | ||||
|  | ||||
| 	startURLs          []string | ||||
| 	startedProcessing  int | ||||
| 	finishedProcessing int | ||||
| type Opt struct { | ||||
| 	StartURLs []string | ||||
| 	ParseFunc func(response *Response) | ||||
| } | ||||
|  | ||||
| type Response struct { | ||||
| @@ -25,20 +26,19 @@ type Response struct { | ||||
| 	Doc  *goquery.Document | ||||
| } | ||||
|  | ||||
| func NewGezer(parse func(response *Response), startURLs ...string) *Gezer { | ||||
| func NewGezer(opt Opt) *Gezer { | ||||
| 	return &Gezer{ | ||||
| 		client: &http.Client{ | ||||
| 			Timeout: time.Second * 10, | ||||
| 		}, | ||||
| 		Parse:     parse, | ||||
| 		startURLs: startURLs, | ||||
| 		opt: opt, | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func (g *Gezer) Start() { | ||||
| 	g.wg.Add(len(g.startURLs)) | ||||
| 	g.wg.Add(len(g.opt.StartURLs)) | ||||
|  | ||||
| 	for _, url := range g.startURLs { | ||||
| 	for _, url := range g.opt.StartURLs { | ||||
| 		go g.getRequest(url) | ||||
| 	} | ||||
|  | ||||
| @@ -73,6 +73,6 @@ func (g *Gezer) getRequest(url string) { | ||||
| 		Doc:      doc, | ||||
| 	} | ||||
|  | ||||
| 	// Parse response | ||||
| 	g.Parse(&response) | ||||
| 	// ParseFunc response | ||||
| 	g.opt.ParseFunc(&response) | ||||
| } | ||||
|   | ||||
| @@ -2,25 +2,28 @@ package gezer | ||||
|  | ||||
| import ( | ||||
| 	"fmt" | ||||
| 	"github.com/PuerkitoBio/goquery" | ||||
| 	"testing" | ||||
| ) | ||||
|  | ||||
| func TestGezer_StartURLs_Simple(t *testing.T) { | ||||
| 	gezer := NewGezer(parse, "https://api.ipify.org", "https://api.ipify.org") | ||||
| 	gezer := NewGezer(Opt{ | ||||
| 		StartURLs: []string{"https://api.ipify.org", "https://api.ipify.org"}, | ||||
| 		ParseFunc: func(r *Response) { | ||||
| 			fmt.Println(string(r.Body)) | ||||
| 		}, | ||||
| 	}) | ||||
| 	gezer.Start() | ||||
| } | ||||
|  | ||||
| func parse(response *Response) { | ||||
| 	fmt.Println(string(response.Body)) | ||||
| func TestGezer_StartURLs_HTML(t *testing.T) { | ||||
| 	gezer := NewGezer(Opt{ | ||||
| 		StartURLs: []string{"http://quotes.toscrape.com/"}, | ||||
| 		ParseFunc: func(r *Response) { | ||||
| 			r.Doc.Find("div.quote").Each(func(i int, s *goquery.Selection) { | ||||
| 				fmt.Println(i, s.Find("span.text").Text(), s.Find("small.author").Text()) | ||||
| 			}) | ||||
| 		}, | ||||
| 	}) | ||||
| 	gezer.Start() | ||||
| } | ||||
|  | ||||
| //func TestGezer_StartURLs_HTML(t *testing.T) { | ||||
| //	gezer := NewGezer(parse, "http://quotes.toscrape.com/") | ||||
| //	gezer.Start() | ||||
| //	for result := range gezer.Results { | ||||
| //		result.Doc.Find("div.quote").Each(func(_ int, s *goquery.Selection) { | ||||
| //			fmt.Println(s.Find("span.text").Text()) | ||||
| //			fmt.Println(s.Find("small.author").Text()) | ||||
| //		}) | ||||
| //	} | ||||
| //} | ||||
|   | ||||
		Reference in New Issue
	
	Block a user