gitea/geziyor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Do request support added. Updated docs.

Browse Source
This commit is contained in:
Musab Gültekin
2019-06-08 19:45:48 +03:00
parent 54c7d3550f
commit 815ae7eec5
3 changed files with 46 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
geziyor.go
View File

@ -14,6 +14,7 @@ import (
"time"
)
// Geziyor is our main scraper type
type Geziyor struct {
client *http.Client
wg sync.WaitGroup
@ -22,6 +23,7 @@ type Geziyor struct {
visitedURLS []string
}
// Opt is custom options type for Geziyor
type Opt struct {
AllowedDomains []string
StartURLs []string
@ -33,6 +35,8 @@ func init() {
log.SetOutput(os.Stdout)
}
// NewGeziyor creates new Geziyor with default values.
// If options provided, options
func NewGeziyor(opt Opt) *Geziyor {
geziyor := &Geziyor{
client: &http.Client{
@ -48,6 +52,7 @@ func NewGeziyor(opt Opt) *Geziyor {
return geziyor
}
// Start starts scraping
func (g *Geziyor) Start() {
for _, startURL := range g.opt.StartURLs {
go g.Get(startURL)
@ -57,19 +62,30 @@ func (g *Geziyor) Start() {
g.wg.Wait()
}
func (g *Geziyor) Get(rawURL string) {
// Get issues a GET to the specified URL.
func (g *Geziyor) Get(url string) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Printf("Request creating error %v\n", err)
return
}
g.Do(req)
}
// Do sends an HTTP request
func (g *Geziyor) Do(req *http.Request) {
g.wg.Add(1)
defer g.wg.Done()
if !checkURL(rawURL, g) {
if !checkURL(req.URL, g) {
return
}
// Log
log.Println("Fetching: ", rawURL)
log.Println("Fetching: ", req.URL.String())
// Get request
resp, err := g.client.Get(rawURL)
// Do request
resp, err := g.client.Do(req)
if resp != nil {
defer resp.Body.Close()
}
@ -104,23 +120,17 @@ func (g *Geziyor) Get(rawURL string) {
time.Sleep(time.Millisecond)
}
func checkURL(rawURL string, g *Geziyor) bool {
// Parse URL
parsedURL, err := url.Parse(rawURL)
if err != nil {
fmt.Fprintf(os.Stderr, "url parsing error: %v\n", err)
return false
}
func checkURL(parsedURL *url.URL, g *Geziyor) bool {
rawURL := parsedURL.String()
// Check for allowed domains
if len(g.opt.AllowedDomains) != 0 && !Contains(g.opt.AllowedDomains, parsedURL.Host) {
if len(g.opt.AllowedDomains) != 0 && !contains(g.opt.AllowedDomains, parsedURL.Host) {
log.Printf("Domain not allowed: %s\n", parsedURL.Host)
return false
}
// Check for duplicate requests
if Contains(g.visitedURLS, rawURL) {
if contains(g.visitedURLS, rawURL) {
log.Printf("URL already visited %s\n", rawURL)
return false
}
@ -129,8 +139,8 @@ func checkURL(rawURL string, g *Geziyor) bool {
return true
}
// Contains checks whether []string contains string
func Contains(s []string, e string) bool {
// contains checks whether []string contains string
func contains(s []string, e string) bool {
for _, a := range s {
if a == e {
return true