gitea/geziyor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed Chrome response not right on some sites.

Browse Source
This commit is contained in:
Musab Gültekin 2019-07-01 12:32:15 +03:00
parent fb5b4e3406
commit 80f3500a69
3 changed files with 8 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
client/client.go
View File

@ -12,6 +12,7 @@ import (
"net" "net"
"net/http" "net/http"
"net/url" "net/url"
"strings"
"time" "time"
) )
@ -95,7 +96,6 @@ func (c *Client) DoRequestClient(req *Request, maxBodySize int64, charsetDetectD
// DoRequestChrome opens up a new chrome instance and makes request // DoRequestChrome opens up a new chrome instance and makes request
func (c *Client) DoRequestChrome(req *Request) (*Response, error) { func (c *Client) DoRequestChrome(req *Request) (*Response, error) {
var body string var body string
var reqID network.RequestID
var res *network.Response var res *network.Response
ctx, cancel := chromedp.NewContext(context.Background()) ctx, cancel := chromedp.NewContext(context.Background())
@ -105,16 +105,16 @@ func (c *Client) DoRequestChrome(req *Request) (*Response, error) {
network.Enable(), network.Enable(),
network.SetExtraHTTPHeaders(network.Headers(ConvertHeaderToMap(req.Header))), network.SetExtraHTTPHeaders(network.Headers(ConvertHeaderToMap(req.Header))),
chromedp.ActionFunc(func(ctx context.Context) error { chromedp.ActionFunc(func(ctx context.Context) error {
var reqID network.RequestID
chromedp.ListenTarget(ctx, func(ev interface{}) { chromedp.ListenTarget(ctx, func(ev interface{}) {
switch ev.(type) { switch ev.(type) {
case *network.EventRequestWillBeSent: case *network.EventRequestWillBeSent:
reqEvent := ev.(*network.EventRequestWillBeSent) reqEvent := ev.(*network.EventRequestWillBeSent)
if _, exists := reqEvent.Request.Headers["Referer"]; !exists { if _, exists := reqEvent.Request.Headers["Referer"]; !exists {
if strings.HasPrefix(reqEvent.Request.URL, "http") {
reqID = reqEvent.RequestID reqID = reqEvent.RequestID
} }
//if reqEvent := ev.(*network.EventRequestWillBeSent); reqEvent.Request.URL == req.URL.String() { }
// reqID = reqEvent.RequestID
//}
case *network.EventResponseReceived: case *network.EventResponseReceived:
if resEvent := ev.(*network.EventResponseReceived); resEvent.RequestID == reqID { if resEvent := ev.(*network.EventResponseReceived); resEvent.RequestID == reqID {
res = resEvent.Response res = resEvent.Response
@ -144,6 +144,7 @@ func (c *Client) DoRequestChrome(req *Request) (*Response, error) {
Response: &http.Response{ Response: &http.Response{
Request: req.Request, Request: req.Request,
StatusCode: int(res.Status), StatusCode: int(res.Status),
Proto: res.Protocol,
Header: ConvertMapToHeader(res.Headers), Header: ConvertMapToHeader(res.Headers),
}, },
Body: []byte(body), Body: []byte(body),

2
geziyor.go
View File

@ -153,7 +153,7 @@ func (g *Geziyor) Get(url string, callback func(g *Geziyor, r *client.Response))
} }
// GetRendered issues GET request using headless browser // GetRendered issues GET request using headless browser
// Opens up a new Chrome instance, makes request, waits for 1 second to render HTML DOM and closed. // Opens up a new Chrome instance, makes request, waits for rendering HTML DOM and closed.
// Rendered requests only supported for GET requests. // Rendered requests only supported for GET requests.
func (g *Geziyor) GetRendered(url string, callback func(g *Geziyor, r *client.Response)) { func (g *Geziyor) GetRendered(url string, callback func(g *Geziyor, r *client.Response)) {
req, err := client.NewRequest("GET", url, nil) req, err := client.NewRequest("GET", url, nil)

2
geziyor_test.go
View File

@ -110,7 +110,7 @@ func TestGetRendered(t *testing.T) {
}, },
ParseFunc: func(g *geziyor.Geziyor, r *client.Response) { ParseFunc: func(g *geziyor.Geziyor, r *client.Response) {
fmt.Println(string(r.Body)) fmt.Println(string(r.Body))
fmt.Println(r.Header) fmt.Println(r.Request.URL.String(), r.Header)
}, },
//URLRevisitEnabled: true, //URLRevisitEnabled: true,
}).Start() }).Start()