diff --git a/client/client.go b/client/client.go index b56b8b9..24ae4c2 100644 --- a/client/client.go +++ b/client/client.go @@ -169,6 +169,7 @@ func (c *Client) DoRequestChrome(req *Request) (*Response, error) { var body string var res *network.Response + // Set remote allocator or use local chrome instance ctx := context.Background() if c.opt.RemoteAllocatorURL != "" { ctx, _ = chromedp.NewRemoteAllocator(ctx, c.opt.RemoteAllocatorURL) @@ -183,6 +184,7 @@ func (c *Client) DoRequestChrome(req *Request) (*Response, error) { var reqID network.RequestID chromedp.ListenTarget(ctx, func(ev interface{}) { switch ev.(type) { + // Save main request ID to get response of it case *network.EventRequestWillBeSent: reqEvent := ev.(*network.EventRequestWillBeSent) if _, exists := reqEvent.Request.Headers["Referer"]; !exists { @@ -190,6 +192,7 @@ func (c *Client) DoRequestChrome(req *Request) (*Response, error) { reqID = reqEvent.RequestID } } + // Save response using main request ID case *network.EventResponseReceived: if resEvent := ev.(*network.EventResponseReceived); resEvent.RequestID == reqID { res = resEvent.Response diff --git a/client/response_test.go b/client/response_test.go index fddd384..19b9cb5 100644 --- a/client/response_test.go +++ b/client/response_test.go @@ -7,9 +7,7 @@ import ( func TestResponse_JoinURL(t *testing.T) { req, _ := NewRequest("GET", "https://localhost.com/test/a.html", nil) - resp := Response{ - Response: nil, - Request: req, - } + resp := Response{Request: req} + assert.Equal(t, "https://localhost.com/source", resp.JoinURL("/source")) } diff --git a/go.mod b/go.mod index 2bba0e6..cf40838 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.12 require ( github.com/PuerkitoBio/goquery v1.5.0 github.com/VividCortex/gohistogram v1.0.0 // indirect - github.com/chromedp/cdproto v0.0.0-20190812224334-39ef923dcb8d - github.com/chromedp/chromedp v0.4.1-0.20190910162850-42693d2528b1 + github.com/chromedp/cdproto v0.0.0-20191114225735-6626966fbae4 + github.com/chromedp/chromedp v0.5.2-0.20191114231622-97580065bae3 github.com/fortytw2/leaktest v1.3.0 github.com/go-kit/kit v0.8.0 github.com/google/btree v1.0.0 // indirect diff --git a/go.sum b/go.sum index 3d116cc..c547696 100644 --- a/go.sum +++ b/go.sum @@ -9,14 +9,10 @@ github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9Pq github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/chromedp/cdproto v0.0.0-20190712010927-387ddc1113d4 h1:xe2O+jIdrblRgMr8wMHGcR9ZUQwfp3Vx105jmTpcpKU= -github.com/chromedp/cdproto v0.0.0-20190712010927-387ddc1113d4/go.mod h1:0YChpVzuLJC5CPr+x3xkHN6Z8KOSXjNbL7qV8Wc4GW0= -github.com/chromedp/cdproto v0.0.0-20190812224334-39ef923dcb8d h1:00kLGv5nKzpFchNhGDXDRbKtYx/WoT983Ka2t8/pzRE= -github.com/chromedp/cdproto v0.0.0-20190812224334-39ef923dcb8d/go.mod h1:0YChpVzuLJC5CPr+x3xkHN6Z8KOSXjNbL7qV8Wc4GW0= -github.com/chromedp/chromedp v0.3.1-0.20190714134543-f6a51f2349aa h1:rUwJ1xjoJMkE2Tucw8MsdDV+XGSLg3o/fhCEGxFNsXo= -github.com/chromedp/chromedp v0.3.1-0.20190714134543-f6a51f2349aa/go.mod h1:zmXqbhGztPqJGTOgYSg9SnkUwjASK0lFHWPfHAiHZrk= -github.com/chromedp/chromedp v0.4.1-0.20190910162850-42693d2528b1 h1:X4XQDbZ9CbAXr8T0aWyhvgrSbX0gJcxRzo+71lboh2g= -github.com/chromedp/chromedp v0.4.1-0.20190910162850-42693d2528b1/go.mod h1:DC3QUn4mJ24dwjcaGQLoZrhm4X/uPHZ6spDbS2uFhm4= +github.com/chromedp/cdproto v0.0.0-20191114225735-6626966fbae4 h1:QD3KxSJ59L2lxG6MXBjNHxiQO2RmxTQ3XcK+wO44WOg= +github.com/chromedp/cdproto v0.0.0-20191114225735-6626966fbae4/go.mod h1:PfAWWKJqjlGFYJEidUM6aVIWPr0EpobeyVWEEmplX7g= +github.com/chromedp/chromedp v0.5.2-0.20191114231622-97580065bae3 h1:q4nIFhMOT6vP8ByPijzv8QBMWENfpdtxbArNgk0/hLg= +github.com/chromedp/chromedp v0.5.2-0.20191114231622-97580065bae3/go.mod h1:rsTo/xRo23KZZwFmWk2Ui79rBaVRRATCjLzNQlOFSiA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -46,12 +42,12 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/knq/sysutil v0.0.0-20181215143952-f05b59f0f307 h1:vl4eIlySbjertFaNwiMjXsGrFVK25aOWLq7n+3gh2ls= -github.com/knq/sysutil v0.0.0-20181215143952-f05b59f0f307/go.mod h1:BjPj+aVjl9FW/cCGiF3nGh5v+9Gd3VCgBQbod/GlMaQ= +github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08 h1:V0an7KRw92wmJysvFvtqtKMAPmvS5O0jtB0nYo6t+gs= +github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08/go.mod h1:dFWs1zEqDjFtnBXsd1vPOZaLsESovai349994nHx3e0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -105,10 +101,8 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056 h1:dHtDnRWQtSx0Hjq9kvKFpBh9uPPKfQN70NZZmvssGwk= +golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=