Attribute extractor added. HTML extractor added. Outer HTML Extractor added.

exporter package renamed to export, extractor package renamed to extract for simplicity.

export/csv.go

@@ -0,0 +1,58 @@
+package export
+
+import (
+	"encoding/csv"
+	"fmt"
+	"github.com/geziyor/geziyor/internal"
+	"log"
+	"os"
+	"reflect"
+	"sort"
+)
+
+// CSV exports response data as CSV streaming file
+type CSV struct {
+	FileName string
+	Comma    rune
+	UseCRLF  bool
+}
+
+// Export exports response data as CSV streaming file
+func (e *CSV) Export(exports chan interface{}) {
+
+	// Create or append file
+	file, err := os.OpenFile(internal.PreferFirst(e.FileName, "out.csv"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+	if err != nil {
+		log.Printf("Output file creation error: %v\n", err)
+		return
+	}
+	defer file.Close()
+
+	writer := csv.NewWriter(file)
+	writer.Comma = internal.PreferFirstRune(e.Comma, ',')
+	writer.UseCRLF = e.UseCRLF
+
+	// Export data as responses came
+	for res := range exports {
+		var values []string
+
+		// Detect type and extract CSV values
+		val := reflect.ValueOf(res)
+		switch val.Kind() {
+		case reflect.Slice:
+			for i := 0; i < val.Len(); i++ {
+				values = append(values, fmt.Sprint(val.Index(i)))
+			}
+		case reflect.Map:
+			iter := val.MapRange()
+			for iter.Next() {
+				values = append(values, fmt.Sprint(iter.Value()))
+			}
+			sort.Strings(values)
+		}
+		if err := writer.Write(values); err != nil {
+			log.Printf("CSV writing error on exporter: %v\n", err)
+		}
+	}
+	writer.Flush()
+}

export/csv_test.go

@@ -0,0 +1,17 @@
+package export
+
+import "testing"
+
+func TestCSVExporter_Export(t *testing.T) {
+	ch := make(chan interface{})
+	defer close(ch)
+
+	exporter := &CSV{
+		FileName: "out.csv",
+		Comma:    ';',
+	}
+	go exporter.Export(ch)
+
+	ch <- []string{"1", "2"}
+	ch <- map[string]string{"key1": "value1", "key2": "value2"}
+}

export/json.go

@@ -0,0 +1,39 @@
+package export
+
+import (
+	"encoding/json"
+	"github.com/geziyor/geziyor/internal"
+	"log"
+	"os"
+)
+
+// JSON exports response data as JSON streaming file
+type JSON struct {
+	FileName   string
+	EscapeHTML bool
+	Prefix     string
+	Indent     string
+}
+
+// Export exports response data as JSON streaming file
+func (e *JSON) Export(exports chan interface{}) {
+
+	// Create or append file
+	file, err := os.OpenFile(internal.PreferFirst(e.FileName, "out.json"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+	if err != nil {
+		log.Printf("Output file creation error: %v\n", err)
+		return
+	}
+	defer file.Close()
+
+	encoder := json.NewEncoder(file)
+	encoder.SetEscapeHTML(e.EscapeHTML)
+	encoder.SetIndent(e.Prefix, e.Indent)
+
+	// Export data as responses came
+	for res := range exports {
+		if err := encoder.Encode(res); err != nil {
+			log.Printf("JSON encoding error on exporter: %v\n", err)
+		}
+	}
+}

export/json_test.go

@@ -0,0 +1,16 @@
+package export
+
+import "testing"
+
+func TestJSONExporter_Export(t *testing.T) {
+	ch := make(chan interface{})
+	defer close(ch)
+
+	exporter := &JSON{
+		FileName: "out.json",
+		Indent:   " ",
+	}
+	go exporter.Export(ch)
+
+	ch <- map[string]string{"key": "value"}
+}

export/pprint.go

@@ -0,0 +1,20 @@
+package export
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// PrettyPrint logs exported data to console as pretty printed
+type PrettyPrint struct{}
+
+// Export logs exported data to console as pretty printed
+func (*PrettyPrint) Export(exports chan interface{}) {
+	for res := range exports {
+		dat, err := json.MarshalIndent(res, "", "  ")
+		if err != nil {
+			continue
+		}
+		fmt.Println(string(dat))
+	}
+}