Middleware support added. HTML Parsing disable option added.

Goroutine leaks will be tested using leaktest lib.
This commit is contained in:
Musab Gültekin
2019-06-15 17:55:40 +03:00
parent 4799b0f7b4
commit 7b23596a2d
9 changed files with 53 additions and 17 deletions

View File

@@ -3,6 +3,7 @@ package geziyor_test
import (
"fmt"
"github.com/PuerkitoBio/goquery"
"github.com/fortytw2/leaktest"
"github.com/fpfeng/httpcache"
"github.com/geziyor/geziyor"
"github.com/geziyor/geziyor/exporter"
@@ -21,6 +22,7 @@ func TestSimple(t *testing.T) {
}
func TestSimpleCache(t *testing.T) {
defer leaktest.Check(t)()
gez := geziyor.NewGeziyor(geziyor.Options{
StartURLs: []string{"http://api.ipify.org"},
Cache: httpcache.NewMemoryCache(),
@@ -34,6 +36,7 @@ func TestSimpleCache(t *testing.T) {
}
func TestQuotes(t *testing.T) {
defer leaktest.Check(t)()
geziyor.NewGeziyor(geziyor.Options{
StartURLs: []string{"http://quotes.toscrape.com/"},
ParseFunc: quotesParse,
@@ -56,7 +59,7 @@ func quotesParse(r *geziyor.Response) {
// Next Page
if href, ok := r.DocHTML.Find("li.next > a").Attr("href"); ok {
go r.Geziyor.Get(r.JoinURL(href), quotesParse)
r.Geziyor.Get(r.JoinURL(href), quotesParse)
}
}