From 9b8a3837bd4a1e08cc258fd535adbc65c8b59a06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Musab=20G=C3=BCltekin?= Date: Fri, 13 Sep 2019 14:34:29 +0300 Subject: [PATCH] Added response joinURL test and updated chromedp. --- .travis.yml | 1 + README.md | 11 ----------- client/response_test.go | 15 +++++++++++++++ go.mod | 4 ++-- go.sum | 6 ++++++ 5 files changed, 24 insertions(+), 13 deletions(-) create mode 100644 client/response_test.go diff --git a/.travis.yml b/.travis.yml index d91e0e3..322c7e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ go: - 1.10.x - 1.11.x - 1.12.x + - 1.13.x - tip env: diff --git a/README.md b/README.md index 47a8fea..2e57662 100644 --- a/README.md +++ b/README.md @@ -162,14 +162,3 @@ BenchmarkRequests-8 200000 108710 ns/op PASS ok github.com/geziyor/geziyor 22.861s ``` - -## Roadmap - -If you're interested in helping this project, please consider these features: - -- Command line tool for: pausing and resuming scraper etc. (like [this](https://docs.scrapy.org/en/latest/topics/commands.html)) -- ~~Automatically exporting extracted data to multiple places (AWS, FTP, DB, JSON, CSV etc)~~ -- Downloading media (Images, Videos etc) (like [this](https://docs.scrapy.org/en/latest/topics/media-pipeline.html)) -- ~~Realtime metrics (Prometheus etc.)~~ - - \ No newline at end of file diff --git a/client/response_test.go b/client/response_test.go new file mode 100644 index 0000000..fddd384 --- /dev/null +++ b/client/response_test.go @@ -0,0 +1,15 @@ +package client + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestResponse_JoinURL(t *testing.T) { + req, _ := NewRequest("GET", "https://localhost.com/test/a.html", nil) + resp := Response{ + Response: nil, + Request: req, + } + assert.Equal(t, "https://localhost.com/source", resp.JoinURL("/source")) +} diff --git a/go.mod b/go.mod index 533bb53..2bba0e6 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.12 require ( github.com/PuerkitoBio/goquery v1.5.0 github.com/VividCortex/gohistogram v1.0.0 // indirect - github.com/chromedp/cdproto v0.0.0-20190712010927-387ddc1113d4 - github.com/chromedp/chromedp v0.3.1-0.20190714134543-f6a51f2349aa + github.com/chromedp/cdproto v0.0.0-20190812224334-39ef923dcb8d + github.com/chromedp/chromedp v0.4.1-0.20190910162850-42693d2528b1 github.com/fortytw2/leaktest v1.3.0 github.com/go-kit/kit v0.8.0 github.com/google/btree v1.0.0 // indirect diff --git a/go.sum b/go.sum index 51788b5..3d116cc 100644 --- a/go.sum +++ b/go.sum @@ -11,8 +11,12 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/chromedp/cdproto v0.0.0-20190712010927-387ddc1113d4 h1:xe2O+jIdrblRgMr8wMHGcR9ZUQwfp3Vx105jmTpcpKU= github.com/chromedp/cdproto v0.0.0-20190712010927-387ddc1113d4/go.mod h1:0YChpVzuLJC5CPr+x3xkHN6Z8KOSXjNbL7qV8Wc4GW0= +github.com/chromedp/cdproto v0.0.0-20190812224334-39ef923dcb8d h1:00kLGv5nKzpFchNhGDXDRbKtYx/WoT983Ka2t8/pzRE= +github.com/chromedp/cdproto v0.0.0-20190812224334-39ef923dcb8d/go.mod h1:0YChpVzuLJC5CPr+x3xkHN6Z8KOSXjNbL7qV8Wc4GW0= github.com/chromedp/chromedp v0.3.1-0.20190714134543-f6a51f2349aa h1:rUwJ1xjoJMkE2Tucw8MsdDV+XGSLg3o/fhCEGxFNsXo= github.com/chromedp/chromedp v0.3.1-0.20190714134543-f6a51f2349aa/go.mod h1:zmXqbhGztPqJGTOgYSg9SnkUwjASK0lFHWPfHAiHZrk= +github.com/chromedp/chromedp v0.4.1-0.20190910162850-42693d2528b1 h1:X4XQDbZ9CbAXr8T0aWyhvgrSbX0gJcxRzo+71lboh2g= +github.com/chromedp/chromedp v0.4.1-0.20190910162850-42693d2528b1/go.mod h1:DC3QUn4mJ24dwjcaGQLoZrhm4X/uPHZ6spDbS2uFhm4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -103,6 +107,8 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=