From edbddf74d802ccc348cf7438b5d4b8fb714faf27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Musab=20G=C3=BCltekin?= Date: Sat, 8 Jun 2019 16:01:31 +0300 Subject: [PATCH] Cache added to options --- gezer.go | 19 ++++++++++++++----- gezer_test.go | 2 ++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/gezer.go b/gezer.go index f5c72c8..afe5d8b 100644 --- a/gezer.go +++ b/gezer.go @@ -4,7 +4,7 @@ import ( "bytes" "fmt" "github.com/PuerkitoBio/goquery" - httpcacheDumb "github.com/fpfeng/httpcache" + "github.com/fpfeng/httpcache" "io/ioutil" "log" "net/http" @@ -24,17 +24,26 @@ type Opt struct { AllowedDomains []string StartURLs []string ParseFunc func(response *Response) + Cache httpcache.Cache +} + +func init() { + log.SetOutput(os.Stdout) } func NewGezer(opt Opt) *Gezer { - log.SetOutput(os.Stdout) - return &Gezer{ + gezer := &Gezer{ client: &http.Client{ - Timeout: time.Second * 10, - Transport: httpcacheDumb.NewMemoryCacheTransport(), + Timeout: time.Second * 10, }, opt: opt, } + + if opt.Cache != nil { + gezer.client.Transport = httpcache.NewTransport(opt.Cache) + } + + return gezer } func (g *Gezer) Start() { diff --git a/gezer_test.go b/gezer_test.go index 2c22753..b3960dc 100644 --- a/gezer_test.go +++ b/gezer_test.go @@ -3,12 +3,14 @@ package gezer import ( "fmt" "github.com/PuerkitoBio/goquery" + "github.com/fpfeng/httpcache" "testing" ) func TestGezer_StartURLs_Simple(t *testing.T) { gezer := NewGezer(Opt{ StartURLs: []string{"http://api.ipify.org"}, + Cache: httpcache.NewMemoryCache(), ParseFunc: func(r *Response) { fmt.Println(string(r.Body)) r.Gezer.Get("http://api.ipify.org")