Options used instead of direct parameter passing.
Tests updated.
This commit is contained in:
		
							
								
								
									
										22
									
								
								gezer.go
									
									
									
									
									
								
							
							
						
						
									
										22
									
								
								gezer.go
									
									
									
									
									
								
							| @@ -12,11 +12,12 @@ import ( | |||||||
| type Gezer struct { | type Gezer struct { | ||||||
| 	client *http.Client | 	client *http.Client | ||||||
| 	wg     sync.WaitGroup | 	wg     sync.WaitGroup | ||||||
| 	Parse  func(response *Response) | 	opt    Opt | ||||||
|  | } | ||||||
|  |  | ||||||
| 	startURLs          []string | type Opt struct { | ||||||
| 	startedProcessing  int | 	StartURLs []string | ||||||
| 	finishedProcessing int | 	ParseFunc func(response *Response) | ||||||
| } | } | ||||||
|  |  | ||||||
| type Response struct { | type Response struct { | ||||||
| @@ -25,20 +26,19 @@ type Response struct { | |||||||
| 	Doc  *goquery.Document | 	Doc  *goquery.Document | ||||||
| } | } | ||||||
|  |  | ||||||
| func NewGezer(parse func(response *Response), startURLs ...string) *Gezer { | func NewGezer(opt Opt) *Gezer { | ||||||
| 	return &Gezer{ | 	return &Gezer{ | ||||||
| 		client: &http.Client{ | 		client: &http.Client{ | ||||||
| 			Timeout: time.Second * 10, | 			Timeout: time.Second * 10, | ||||||
| 		}, | 		}, | ||||||
| 		Parse:     parse, | 		opt: opt, | ||||||
| 		startURLs: startURLs, |  | ||||||
| 	} | 	} | ||||||
| } | } | ||||||
|  |  | ||||||
| func (g *Gezer) Start() { | func (g *Gezer) Start() { | ||||||
| 	g.wg.Add(len(g.startURLs)) | 	g.wg.Add(len(g.opt.StartURLs)) | ||||||
|  |  | ||||||
| 	for _, url := range g.startURLs { | 	for _, url := range g.opt.StartURLs { | ||||||
| 		go g.getRequest(url) | 		go g.getRequest(url) | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| @@ -73,6 +73,6 @@ func (g *Gezer) getRequest(url string) { | |||||||
| 		Doc:      doc, | 		Doc:      doc, | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	// Parse response | 	// ParseFunc response | ||||||
| 	g.Parse(&response) | 	g.opt.ParseFunc(&response) | ||||||
| } | } | ||||||
|   | |||||||
| @@ -2,25 +2,28 @@ package gezer | |||||||
|  |  | ||||||
| import ( | import ( | ||||||
| 	"fmt" | 	"fmt" | ||||||
|  | 	"github.com/PuerkitoBio/goquery" | ||||||
| 	"testing" | 	"testing" | ||||||
| ) | ) | ||||||
|  |  | ||||||
| func TestGezer_StartURLs_Simple(t *testing.T) { | func TestGezer_StartURLs_Simple(t *testing.T) { | ||||||
| 	gezer := NewGezer(parse, "https://api.ipify.org", "https://api.ipify.org") | 	gezer := NewGezer(Opt{ | ||||||
|  | 		StartURLs: []string{"https://api.ipify.org", "https://api.ipify.org"}, | ||||||
|  | 		ParseFunc: func(r *Response) { | ||||||
|  | 			fmt.Println(string(r.Body)) | ||||||
|  | 		}, | ||||||
|  | 	}) | ||||||
| 	gezer.Start() | 	gezer.Start() | ||||||
| } | } | ||||||
|  |  | ||||||
| func parse(response *Response) { | func TestGezer_StartURLs_HTML(t *testing.T) { | ||||||
| 	fmt.Println(string(response.Body)) | 	gezer := NewGezer(Opt{ | ||||||
|  | 		StartURLs: []string{"http://quotes.toscrape.com/"}, | ||||||
|  | 		ParseFunc: func(r *Response) { | ||||||
|  | 			r.Doc.Find("div.quote").Each(func(i int, s *goquery.Selection) { | ||||||
|  | 				fmt.Println(i, s.Find("span.text").Text(), s.Find("small.author").Text()) | ||||||
|  | 			}) | ||||||
|  | 		}, | ||||||
|  | 	}) | ||||||
|  | 	gezer.Start() | ||||||
| } | } | ||||||
|  |  | ||||||
| //func TestGezer_StartURLs_HTML(t *testing.T) { |  | ||||||
| //	gezer := NewGezer(parse, "http://quotes.toscrape.com/") |  | ||||||
| //	gezer.Start() |  | ||||||
| //	for result := range gezer.Results { |  | ||||||
| //		result.Doc.Find("div.quote").Each(func(_ int, s *goquery.Selection) { |  | ||||||
| //			fmt.Println(s.Find("span.text").Text()) |  | ||||||
| //			fmt.Println(s.Find("small.author").Text()) |  | ||||||
| //		}) |  | ||||||
| //	} |  | ||||||
| //} |  | ||||||
|   | |||||||
		Reference in New Issue
	
	Block a user