Cache added to options
This commit is contained in:
parent
ca197ff06a
commit
edbddf74d8
19
gezer.go
19
gezer.go
@ -4,7 +4,7 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
httpcacheDumb "github.com/fpfeng/httpcache"
|
"github.com/fpfeng/httpcache"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
@ -24,17 +24,26 @@ type Opt struct {
|
|||||||
AllowedDomains []string
|
AllowedDomains []string
|
||||||
StartURLs []string
|
StartURLs []string
|
||||||
ParseFunc func(response *Response)
|
ParseFunc func(response *Response)
|
||||||
|
Cache httpcache.Cache
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
log.SetOutput(os.Stdout)
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewGezer(opt Opt) *Gezer {
|
func NewGezer(opt Opt) *Gezer {
|
||||||
log.SetOutput(os.Stdout)
|
gezer := &Gezer{
|
||||||
return &Gezer{
|
|
||||||
client: &http.Client{
|
client: &http.Client{
|
||||||
Timeout: time.Second * 10,
|
Timeout: time.Second * 10,
|
||||||
Transport: httpcacheDumb.NewMemoryCacheTransport(),
|
|
||||||
},
|
},
|
||||||
opt: opt,
|
opt: opt,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if opt.Cache != nil {
|
||||||
|
gezer.client.Transport = httpcache.NewTransport(opt.Cache)
|
||||||
|
}
|
||||||
|
|
||||||
|
return gezer
|
||||||
}
|
}
|
||||||
|
|
||||||
func (g *Gezer) Start() {
|
func (g *Gezer) Start() {
|
||||||
|
@ -3,12 +3,14 @@ package gezer
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
|
"github.com/fpfeng/httpcache"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestGezer_StartURLs_Simple(t *testing.T) {
|
func TestGezer_StartURLs_Simple(t *testing.T) {
|
||||||
gezer := NewGezer(Opt{
|
gezer := NewGezer(Opt{
|
||||||
StartURLs: []string{"http://api.ipify.org"},
|
StartURLs: []string{"http://api.ipify.org"},
|
||||||
|
Cache: httpcache.NewMemoryCache(),
|
||||||
ParseFunc: func(r *Response) {
|
ParseFunc: func(r *Response) {
|
||||||
fmt.Println(string(r.Body))
|
fmt.Println(string(r.Body))
|
||||||
r.Gezer.Get("http://api.ipify.org")
|
r.Gezer.Get("http://api.ipify.org")
|
||||||
|
Loading…
x
Reference in New Issue
Block a user