gitea/geziyor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remote endpoint support added for js rendered requests. Geziyor is beta now.

Browse Source
This commit is contained in:
Musab Gültekin
2019-08-05 15:14:47 +03:00
parent c117d71fef
commit 0e5230eac8
6 changed files with 54 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
client/client.go
View File

@ -31,6 +31,7 @@ type Client struct {
charsetDetectDisabled bool
retryTimes int
retryHTTPCodes []int
remoteAllocatorURL string
}
const (
@ -44,7 +45,7 @@ var (
)
// NewClient creates http.Client with modified values for typical web scraper
func NewClient(maxBodySize int64, charsetDetectDisabled bool, retryTimes int, retryHTTPCodes []int) *Client {
func NewClient(maxBodySize int64, charsetDetectDisabled bool, retryTimes int, retryHTTPCodes []int, remoteAllocatorURL string) *Client {
httpClient := &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
@ -68,6 +69,7 @@ func NewClient(maxBodySize int64, charsetDetectDisabled bool, retryTimes int, re
charsetDetectDisabled: charsetDetectDisabled,
retryTimes: retryTimes,
retryHTTPCodes: retryHTTPCodes,
remoteAllocatorURL: remoteAllocatorURL,
}
return &client
@ -156,12 +158,16 @@ func (c *Client) DoRequestChrome(req *Request) (*Response, error) {
var body string
var res *network.Response
ctx, cancel := chromedp.NewContext(context.Background())
ctx := context.Background()
if c.remoteAllocatorURL != "" {
ctx, _ = chromedp.NewRemoteAllocator(ctx, c.remoteAllocatorURL)
}
ctx, cancel := chromedp.NewContext(ctx)
defer cancel()
if err := chromedp.Run(ctx,
network.Enable(),
network.SetExtraHTTPHeaders(network.Headers(ConvertHeaderToMap(req.Header))),
network.SetExtraHTTPHeaders(ConvertHeaderToMap(req.Header)),
chromedp.ActionFunc(func(ctx context.Context) error {
var reqID network.RequestID
chromedp.ListenTarget(ctx, func(ev interface{}) {

8
client/client_test.go
View File

@ -101,7 +101,7 @@ func TestCharsetFromHeaders(t *testing.T) {
defer ts.Close()
req, _ := NewRequest("GET", ts.URL, nil)
res, _ := NewClient(DefaultMaxBody, false, DefaultRetryTimes, DefaultRetryHTTPCodes).DoRequest(req)
res, _ := NewClient(DefaultMaxBody, false, DefaultRetryTimes, DefaultRetryHTTPCodes, "").DoRequest(req)
if string(res.Body) != "Gültekin" {
t.Fatal(string(res.Body))
@ -116,7 +116,7 @@ func TestCharsetFromBody(t *testing.T) {
defer ts.Close()
req, _ := NewRequest("GET", ts.URL, nil)
res, _ := NewClient(DefaultMaxBody, false, DefaultRetryTimes, DefaultRetryHTTPCodes).DoRequest(req)
res, _ := NewClient(DefaultMaxBody, false, DefaultRetryTimes, DefaultRetryHTTPCodes, "").DoRequest(req)
if string(res.Body) != "Gültekin" {
t.Fatal(string(res.Body))
@ -132,7 +132,7 @@ func TestCharsetProvidedWithRequest(t *testing.T) {
req, _ := NewRequest("GET", ts.URL, nil)
req.Encoding = "windows-1254"
res, _ := NewClient(DefaultMaxBody, false, DefaultRetryTimes, DefaultRetryHTTPCodes).DoRequest(req)
res, _ := NewClient(DefaultMaxBody, false, DefaultRetryTimes, DefaultRetryHTTPCodes, "").DoRequest(req)
if string(res.Body) != "Gültekin" {
t.Fatal(string(res.Body))
@ -141,7 +141,7 @@ func TestCharsetProvidedWithRequest(t *testing.T) {
func TestRetry(t *testing.T) {
req, _ := NewRequest("GET", "https://httpbin.org/status/500", nil)
res, err := NewClient(DefaultMaxBody, false, DefaultRetryTimes, DefaultRetryHTTPCodes).DoRequest(req)
res, err := NewClient(DefaultMaxBody, false, DefaultRetryTimes, DefaultRetryHTTPCodes, "").DoRequest(req)
assert.Nil(t, res)
assert.Error(t, err)
}