Cache added to options
This commit is contained in:
parent
ca197ff06a
commit
edbddf74d8
17
gezer.go
17
gezer.go
@ -4,7 +4,7 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
httpcacheDumb "github.com/fpfeng/httpcache"
|
||||
"github.com/fpfeng/httpcache"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
@ -24,17 +24,26 @@ type Opt struct {
|
||||
AllowedDomains []string
|
||||
StartURLs []string
|
||||
ParseFunc func(response *Response)
|
||||
Cache httpcache.Cache
|
||||
}
|
||||
|
||||
func init() {
|
||||
log.SetOutput(os.Stdout)
|
||||
}
|
||||
|
||||
func NewGezer(opt Opt) *Gezer {
|
||||
log.SetOutput(os.Stdout)
|
||||
return &Gezer{
|
||||
gezer := &Gezer{
|
||||
client: &http.Client{
|
||||
Timeout: time.Second * 10,
|
||||
Transport: httpcacheDumb.NewMemoryCacheTransport(),
|
||||
},
|
||||
opt: opt,
|
||||
}
|
||||
|
||||
if opt.Cache != nil {
|
||||
gezer.client.Transport = httpcache.NewTransport(opt.Cache)
|
||||
}
|
||||
|
||||
return gezer
|
||||
}
|
||||
|
||||
func (g *Gezer) Start() {
|
||||
|
@ -3,12 +3,14 @@ package gezer
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/fpfeng/httpcache"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGezer_StartURLs_Simple(t *testing.T) {
|
||||
gezer := NewGezer(Opt{
|
||||
StartURLs: []string{"http://api.ipify.org"},
|
||||
Cache: httpcache.NewMemoryCache(),
|
||||
ParseFunc: func(r *Response) {
|
||||
fmt.Println(string(r.Body))
|
||||
r.Gezer.Get("http://api.ipify.org")
|
||||
|
Loading…
x
Reference in New Issue
Block a user