gitea/geziyor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HTTP Client can be changed now. Docs updated.

Browse Source
This commit is contained in:
Musab Gültekin
2019-06-22 13:12:05 +03:00
parent 7bc782400c
commit a64a262554
6 changed files with 45 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
geziyor_test.go
View File

@@ -46,7 +46,7 @@ func TestQuotes(t *testing.T) {
}
func quotesParse(g *geziyor.Geziyor, r *geziyor.Response) {
r.DocHTML.Find("div.quote").Each(func(i int, s *goquery.Selection) {
r.HTMLDoc.Find("div.quote").Each(func(i int, s *goquery.Selection) {
// Export Data
g.Exports <- map[string]interface{}{
"number": i,
@@ -59,7 +59,7 @@ func quotesParse(g *geziyor.Geziyor, r *geziyor.Response) {
})
// Next Page
if href, ok := r.DocHTML.Find("li.next > a").Attr("href"); ok {
if href, ok := r.HTMLDoc.Find("li.next > a").Attr("href"); ok {
g.Get(r.JoinURL(href), quotesParse)
}
}
@@ -72,13 +72,14 @@ func TestAllLinks(t *testing.T) {
StartURLs: []string{"http://books.toscrape.com/"},
ParseFunc: func(g *geziyor.Geziyor, r *geziyor.Response) {
g.Exports <- []string{r.Request.URL.String()}
r.DocHTML.Find("a").Each(func(i int, s *goquery.Selection) {
r.HTMLDoc.Find("a").Each(func(i int, s *goquery.Selection) {
if href, ok := s.Attr("href"); ok {
g.Get(r.JoinURL(href), g.Opt.ParseFunc)
}
})
},
Exporters: []geziyor.Exporter{&exporter.CSVExporter{}},
Exporters: []geziyor.Exporter{&exporter.CSVExporter{}},
MetricsType: metrics.Prometheus,
}).Start()
}
@@ -97,7 +98,7 @@ func TestStartRequestsFunc(t *testing.T) {
g.Get("http://quotes.toscrape.com/", g.Opt.ParseFunc)
},
ParseFunc: func(g *geziyor.Geziyor, r *geziyor.Response) {
r.DocHTML.Find("a").Each(func(_ int, s *goquery.Selection) {
r.HTMLDoc.Find("a").Each(func(_ int, s *goquery.Selection) {
g.Exports <- s.AttrOr("href", "")
})
},