diff --git a/geziyor.go b/geziyor.go index 5e3b639..85f7169 100644 --- a/geziyor.go +++ b/geziyor.go @@ -146,24 +146,7 @@ func (g *Geziyor) Start() { } // Start Exporters - if len(g.Opt.Exporters) != 0 { - g.wgExporters.Add(len(g.Opt.Exporters)) - for _, exporter := range g.Opt.Exporters { - go func(exporter export.Exporter) { - defer g.wgExporters.Done() - if err := exporter.Export(g.Exports); err != nil { - internal.Logger.Printf("exporter error: %s\n", err) - } - }(exporter) - } - } else { - g.wgExporters.Add(1) - go func() { - for range g.Exports { - } - g.wgExporters.Done() - }() - } + g.startExporters() // Wait for SIGINT (interrupt) signal. shutdownChan := make(chan os.Signal, 1) @@ -326,3 +309,43 @@ func (g *Geziyor) interruptSignalWaiter(shutdownChan chan os.Signal, shutdownDon } } } + +func (g *Geziyor) startExporters() { + if len(g.Opt.Exporters) != 0 { + var exporterChans []chan interface{} + + g.wgExporters.Add(len(g.Opt.Exporters)) + for _, exporter := range g.Opt.Exporters { + exporterChan := make(chan interface{}) + exporterChans = append(exporterChans, exporterChan) + go func(exporter export.Exporter) { + defer g.wgExporters.Done() + if err := exporter.Export(exporterChan); err != nil { + internal.Logger.Printf("exporter error: %s\n", err) + } + }(exporter) + } + go func() { + // When exports closed, close the exporter chans. + // Exports chan will be closed after all requests are handled. + defer func() { + for _, exporterChan := range exporterChans { + close(exporterChan) + } + }() + // Send incoming data from exports to all of the exporter's chans + for data := range g.Exports { + for _, exporterChan := range exporterChans { + exporterChan <- data + } + } + }() + } else { + g.wgExporters.Add(1) + go func() { + for range g.Exports { + } + g.wgExporters.Done() + }() + } +} diff --git a/geziyor_test.go b/geziyor_test.go index bfea4fd..34f8d51 100644 --- a/geziyor_test.go +++ b/geziyor_test.go @@ -61,7 +61,7 @@ func TestQuotes(t *testing.T) { geziyor.NewGeziyor(&geziyor.Options{ StartURLs: []string{"http://quotes.toscrape.com/"}, ParseFunc: quotesParse, - Exporters: []export.Exporter{&export.JSON{}}, + Exporters: []export.Exporter{&export.JSONLine{FileName: "1.jsonl"}, &export.JSON{FileName: "2.json"}}, }).Start() }