gitea/geziyor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

JoinURL now returns *url.URL and error

Browse Source
This commit is contained in:
Musab Gültekin 2021-04-17 11:12:22 +03:00
parent 9ea67b3554
commit 6a23efd175
3 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
client/response.go
View File

@ -22,14 +22,14 @@ type Response struct {
} }
// JoinURL joins base response URL and provided relative URL. // JoinURL joins base response URL and provided relative URL.
func (r *Response) JoinURL(relativeURL string) string { func (r *Response) JoinURL(relativeURL string) (*url.URL, error) {
parsedRelativeURL, err := url.Parse(relativeURL) parsedRelativeURL, err := url.Parse(relativeURL)
if err != nil { if err != nil {
return "" return nil, err
} }
joinedURL := r.Request.URL.ResolveReference(parsedRelativeURL) joinedURL := r.Request.URL.ResolveReference(parsedRelativeURL)
return joinedURL.String() return joinedURL, nil
} }
// IsHTML checks if response content is HTML by looking content-type header // IsHTML checks if response content is HTML by looking content-type header

4
client/response_test.go
View File

@ -9,5 +9,7 @@ func TestResponse_JoinURL(t *testing.T) {
req, _ := NewRequest("GET", "https://localhost.com/test/a.html", nil) req, _ := NewRequest("GET", "https://localhost.com/test/a.html", nil)
resp := Response{Request: req} resp := Response{Request: req}
assert.Equal(t, "https://localhost.com/source", resp.JoinURL("/source")) joinedURL, err := resp.JoinURL("/source")
assert.NoError(t, err)
assert.Equal(t, "https://localhost.com/source", joinedURL.String())
} }

9
geziyor_test.go
View File

@ -78,7 +78,8 @@ func quotesParse(g *geziyor.Geziyor, r *client.Response) {
// Next Page // Next Page
if href, ok := r.HTMLDoc.Find("li.next > a").Attr("href"); ok { if href, ok := r.HTMLDoc.Find("li.next > a").Attr("href"); ok {
g.Get(r.JoinURL(href), quotesParse) absoluteURL, _ := r.JoinURL(href)
g.Get(absoluteURL.String(), quotesParse)
} }
} }
@ -94,7 +95,8 @@ func TestAllLinks(t *testing.T) {
g.Exports <- []string{r.Request.URL.String()} g.Exports <- []string{r.Request.URL.String()}
r.HTMLDoc.Find("a").Each(func(i int, s *goquery.Selection) { r.HTMLDoc.Find("a").Each(func(i int, s *goquery.Selection) {
if href, ok := s.Attr("href"); ok { if href, ok := s.Attr("href"); ok {
g.Get(r.JoinURL(href), g.Opt.ParseFunc) absoluteURL, _ := r.JoinURL(href)
g.Get(absoluteURL.String(), g.Opt.ParseFunc)
} }
}) })
}, },
@ -269,7 +271,8 @@ func BenchmarkWhole(b *testing.B) {
g.Exports <- []string{r.Request.URL.String()} g.Exports <- []string{r.Request.URL.String()}
r.HTMLDoc.Find("a").Each(func(i int, s *goquery.Selection) { r.HTMLDoc.Find("a").Each(func(i int, s *goquery.Selection) {
if href, ok := s.Attr("href"); ok { if href, ok := s.Attr("href"); ok {
g.Get(r.JoinURL(href), g.Opt.ParseFunc) absoluteURL, _ := r.JoinURL(href)
g.Get(absoluteURL.String(), g.Opt.ParseFunc)
} }
}) })
}, },