up

Author: Oleksandr Pavlov

Examples/WebReaper.ConsoleApplication/Program.cs

@@ -1,26 +1,17 @@
 using WebReaper.Builders;
 
 var engine = await new ScraperEngineBuilder()
-    .GetWithBrowser(
-        new[] { "https://www.reddit.com/r/dotnet/" },
-        actions => actions
-            .ScrollToEnd()
-            .RepeatWithDelay(10, 2000)
-            .Build())
-    .Follow("a.SQnoC3ObvgnGjWt90zD9Z._2INHSNB8V5eaWp4P0rY_mE")
+    .GetWithBrowser("https://www.alexpavlov.dev/blog")
+    .FollowWithBrowser(".text-gray-900.transition")
     .Parse(new()
     {
-        new("title", "._eYtD2XCVieq6emjKBH3m"),
-        new("text", "._3xX726aBn29LDbsDtzr_6E._1Ap4F5maDtT1E1YuCiaO0r.D3IL3FD0RFy_mkKLPwL4")
+        new("title", ".text-3xl.font-bold"),
+        new("text", ".max-w-max.prose.prose-dark")
     })
-    .WriteToJsonFile("data/output.json", dataCleanupOnStart: true)
-    .TrackVisitedLinksInFile("data/visited.txt", dataCleanupOnStart: true)
-    .WithTextFileScheduler("data/jobs.txt", "data/currentJob.txt", dataCleanupOnStart: true)
-    .WithFileConfigStorage("data/config.txt")
-    .LogToConsole()
-    .PageCrawlLimit(500)
-    .HeadlessMode(true)
+    .WriteToJsonFile("output.json")
+    .PageCrawlLimit(10)
     .WithParallelismDegree(30)
+    .LogToConsole()
     .BuildAsync();
 
 await engine.RunAsync();

README.md

@@ -76,17 +76,21 @@ Parsing single page applications is super simple, just use the *GetWithBrowser*
 case Puppeteer will be used to load the pages.
 
 ```C#
+using WebReaper.Builders;
+
 var engine = await new ScraperEngineBuilder()
-    .GetWithBrowser("https://www.reddit.com/r/dotnet/")
-    .Follow("a.SQnoC3ObvgnGjWt90zD9Z._2INHSNB8V5eaWp4P0rY_mE")
+    .GetWithBrowser("https://www.alexpavlov.dev/blog")
+    .FollowWithBrowser(".text-gray-900.transition")
     .Parse(new()
     {
-        new("title", "._eYtD2XCVieq6emjKBH3m"),
-        new("text", "._3xX726aBn29LDbsDtzr_6E._1Ap4F5maDtT1E1YuCiaO0r.D3IL3FD0RFy_mkKLPwL4")
+        new("title", ".text-3xl.font-bold"),
+        new("text", ".max-w-max.prose.prose-dark")
     })
     .WriteToJsonFile("output.json")
+    .PageCrawlLimit(10)
+    .WithParallelismDegree(30)
     .LogToConsole()
-    .BuildAsync()
+    .BuildAsync();
 
 await engine.RunAsync();
 ```
@@ -120,7 +124,7 @@ It can be helpful if the required content is loaded only after some user interac
 
 ### Persist the progress locally
 
-If you want to persist the vistited links and job queue locally, so that you can start crawling where you left off you
+If you want to persist the visited links and job queue locally, so that you can start crawling where you left off you
 can use *ScheduleWithTextFile* and *TrackVisitedLinksInFile* methods:
 
 ```C#

WebReaper/Builders/ScraperEngineBuilder.cs

@@ -155,7 +155,7 @@ public ScraperEngineBuilder WriteToCsvFile(string filePath, bool dataCleanupOnSt
         return this;
     }
 
-    public ScraperEngineBuilder WriteToJsonFile(string filePath, bool dataCleanupOnStart)
+    public ScraperEngineBuilder WriteToJsonFile(string filePath, bool dataCleanupOnStart = true)
     {
         SpiderBuilder.WriteToJsonFile(filePath, dataCleanupOnStart);
         return this;
@@ -186,6 +186,12 @@ public ScraperEngineBuilder GetWithBrowser(
         ConfigBuilder.GetWithBrowser(startUrls, actionBuilder?.Invoke(new PageActionBuilder()));
         return this;
     }
+    
+    public ScraperEngineBuilder GetWithBrowser(params string[] startUrls)
+    {
+        ConfigBuilder.GetWithBrowser(startUrls);
+        return this;
+    }
 
     public ScraperEngineBuilder Follow(string linkSelector)
     {

WebReaper/Core/ScraperEngine.cs

@@ -76,7 +76,6 @@ await Parallel.ForEachAsync(Scheduler.GetAllAsync(cancellationToken), options, a
         catch (PageCrawlLimitException ex)
         {
             Logger.LogWarning(ex, "Shutting down due to page crawl limit {Limit}", ex.PageCrawlLimit);
-            throw;
         }
         catch (TaskCanceledException ex)
         {

WebReaper/Core/Spider/Concrete/Spider.cs

@@ -58,15 +58,7 @@ public async Task<List<Job>> CrawlAsync(Job job, CancellationToken cancellationT
 
         if (config.UrlBlackList.Contains(job.Url)) return Enumerable.Empty<Job>().ToList();
 
-        if (await LinkTracker.GetVisitedLinksCount() >= config.PageCrawlLimit)
-        {
-            Logger.LogInformation("Page crawl limit has been reached");
-
-            throw new PageCrawlLimitException("Page crawl limit has been reached.")
-            {
-                PageCrawlLimit = config.PageCrawlLimit
-            };
-        }
+        await CheckCrawlLimit(config);
 
         await LinkTracker.AddVisitedLinkAsync(job.Url);
 
@@ -80,6 +72,8 @@ public async Task<List<Job>> CrawlAsync(Job job, CancellationToken cancellationT
         if (job.PageCategory == PageCategory.TargetPage)
         {
             await ProcessTargetPage(job, doc, cancellationToken);
+            
+            await CheckCrawlLimit(config);
 
             return Enumerable.Empty<Job>().ToList();
         }
@@ -108,6 +102,19 @@ public async Task<List<Job>> CrawlAsync(Job job, CancellationToken cancellationT
         return newJobs;
     }
 
+    private async Task CheckCrawlLimit(ScraperConfig config)
+    {
+        if (await LinkTracker.GetVisitedLinksCount() >= config.PageCrawlLimit)
+        {
+            Logger.LogInformation("Page crawl limit has been reached");
+
+            throw new PageCrawlLimitException("Page crawl limit has been reached.")
+            {
+                PageCrawlLimit = config.PageCrawlLimit
+            };
+        }
+    }
+
     public event Action<ParsedData>? ScrapedData;
 
     public event Func<Metadata, JObject, Task>? PostProcessor;