diff --git a/geziyor.go b/geziyor.go index 05dea22..3d4dd07 100644 --- a/geziyor.go +++ b/geziyor.go @@ -12,6 +12,7 @@ import ( "log" "math/rand" "net/http" + "net/http/cookiejar" "os" "runtime/debug" "sync" @@ -49,7 +50,7 @@ func init() { // If options provided, options func NewGeziyor(opt *Options) *Geziyor { geziyor := &Geziyor{ - Client: internal.NewClient(opt.CookiesDisabled), + Client: internal.NewClient(), Opt: opt, Exports: make(chan interface{}), requestMiddlewares: []RequestMiddleware{ @@ -75,6 +76,9 @@ func NewGeziyor(opt *Options) *Geziyor { if opt.Timeout != 0 { geziyor.Client.Timeout = opt.Timeout } + if !opt.CookiesDisabled { + geziyor.Client.Jar, _ = cookiejar.New(nil) + } if opt.ConcurrentRequests != 0 { geziyor.semGlobal = make(chan struct{}, opt.ConcurrentRequests) } diff --git a/geziyor_test.go b/geziyor_test.go index 0d16456..44a1dfc 100644 --- a/geziyor_test.go +++ b/geziyor_test.go @@ -146,3 +146,13 @@ func TestCookies(t *testing.T) { CookiesDisabled: true, }).Start() } + +func TestBasicAuth(t *testing.T) { + geziyor.NewGeziyor(&geziyor.Options{ + StartRequestsFunc: func(g *geziyor.Geziyor) { + req, _ := geziyor.NewRequest("GET", "https://httpbin.org/anything", nil) + req.SetBasicAuth("username", "password") + g.Do(req, nil) + }, + }).Start() +} diff --git a/internal/http.go b/internal/http.go index ee79bdd..9fa0787 100644 --- a/internal/http.go +++ b/internal/http.go @@ -4,7 +4,6 @@ import ( "errors" "net" "net/http" - "net/http/cookiejar" "net/url" "time" ) @@ -20,7 +19,7 @@ type Client struct { } // NewClient creates http.Client with modified values for typical web scraper -func NewClient(cookiesDisabled bool) *Client { +func NewClient() *Client { client := &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, @@ -37,9 +36,6 @@ func NewClient(cookiesDisabled bool) *Client { }, Timeout: time.Second * 180, // Google's timeout } - if !cookiesDisabled { - client.Jar, _ = cookiejar.New(nil) - } return &Client{Client: client} } diff --git a/request.go b/request.go index ec74626..b6d7331 100644 --- a/request.go +++ b/request.go @@ -1,6 +1,7 @@ package geziyor import ( + "io" "net/http" ) @@ -11,3 +12,13 @@ type Request struct { Rendered bool Cancelled bool } + +// NewRequest returns a new Request given a method, URL, and optional body. +func NewRequest(method, url string, body io.Reader) (*Request, error) { + req, err := http.NewRequest(method, url, body) + if err != nil { + return nil, err + } + + return &Request{Request: req}, nil +}