Global and Domain Concurrency limit implemented. Updated README

geziyor.go

@@ -21,6 +21,11 @@ type Geziyor struct {
 	opt    Options
 
 	visitedURLS []string
+	semGlobal   chan struct{}
+	semHosts    struct {
+		sync.RWMutex
+		hostSems map[string]chan struct{}
+	}
 }
 
 func init() {
@@ -43,6 +48,15 @@ func NewGeziyor(opt Options) *Geziyor {
 	if opt.Timeout != 0 {
 		geziyor.client.Timeout = opt.Timeout
 	}
+	if opt.ConcurrentRequests != 0 {
+		geziyor.semGlobal = make(chan struct{}, opt.ConcurrentRequests)
+	}
+	if opt.ConcurrentRequestsPerDomain != 0 {
+		geziyor.semHosts = struct {
+			sync.RWMutex
+			hostSems map[string]chan struct{}
+		}{hostSems: make(map[string]chan struct{})}
+	}
 
 	return geziyor
 }
@@ -86,12 +100,15 @@ func (g *Geziyor) Do(req *http.Request) {
 		return
 	}
 
-	// Log
-	log.Println("Fetching: ", req.URL.String())
-
 	// Modify Request
 	req.Header.Set("Accept-Charset", "utf-8")
 
+	// Acquire Semaphore
+	g.acquire(req)
+
+	// Log
+	log.Println("Fetching: ", req.URL.String())
+
 	// Do request
 	resp, err := g.client.Do(req)
 	if resp != nil {
@@ -99,6 +116,7 @@ func (g *Geziyor) Do(req *http.Request) {
 	}
 	if err != nil {
 		log.Printf("Response error: %v\n", err)
+		g.release(req)
 		return
 	}
 
@@ -106,6 +124,7 @@ func (g *Geziyor) Do(req *http.Request) {
 	reader, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
 	if err != nil {
 		log.Printf("Determine encoding error: %v\n", err)
+		g.release(req)
 		return
 	}
 
@@ -113,9 +132,13 @@ func (g *Geziyor) Do(req *http.Request) {
 	body, err := ioutil.ReadAll(reader)
 	if err != nil {
 		log.Printf("Reading Body error: %v\n", err)
+		g.release(req)
 		return
 	}
 
+	// Release Semaphore
+	g.release(req)
+
 	// Create Document
 	doc, _ := goquery.NewDocumentFromReader(bytes.NewReader(body))
 
@@ -136,18 +159,45 @@ func (g *Geziyor) Do(req *http.Request) {
 	time.Sleep(time.Millisecond)
 }
 
+func (g *Geziyor) acquire(req *http.Request) {
+	if g.opt.ConcurrentRequests != 0 {
+		g.semGlobal <- struct{}{}
+	}
+
+	if g.opt.ConcurrentRequestsPerDomain != 0 {
+		g.semHosts.RLock()
+		hostSem, exists := g.semHosts.hostSems[req.Host]
+		g.semHosts.RUnlock()
+		if !exists {
+			hostSem = make(chan struct{}, g.opt.ConcurrentRequestsPerDomain)
+			g.semHosts.Lock()
+			g.semHosts.hostSems[req.Host] = hostSem
+			g.semHosts.Unlock()
+		}
+		hostSem <- struct{}{}
+	}
+}
+
+func (g *Geziyor) release(req *http.Request) {
+	if g.opt.ConcurrentRequests != 0 {
+		<-g.semGlobal
+	}
+	if g.opt.ConcurrentRequestsPerDomain != 0 {
+		<-g.semHosts.hostSems[req.Host]
+	}
+}
+
 func checkURL(parsedURL *url.URL, g *Geziyor) bool {
 	rawURL := parsedURL.String()
-
 	// Check for allowed domains
 	if len(g.opt.AllowedDomains) != 0 && !contains(g.opt.AllowedDomains, parsedURL.Host) {
-		log.Printf("Domain not allowed: %s\n", parsedURL.Host)
+		//log.Printf("Domain not allowed: %s\n", parsedURL.Host)
 		return false
 	}
 
 	// Check for duplicate requests
 	if contains(g.visitedURLS, rawURL) {
-		log.Printf("URL already visited %s\n", rawURL)
+		//log.Printf("URL already visited %s\n", rawURL)
 		return false
 	}
 	g.visitedURLS = append(g.visitedURLS, rawURL)