Return HTTP_429_DETECTED in list of urls instead of as string

Author: opsdisk

README.md

@@ -97,7 +97,40 @@ using it.
 
 If you do not want `yagooglesearch` to handle HTTP 429 cool off period and would rather handle it yourself, pass
 `yagooglesearch_manages_http_429s=False` when instantiating the yagooglesearch object.  If an HTTP 429 is detected, the
-string "HTTP_429_DETECTED" will be returned and it's up to you on what the next step should be.
+string "HTTP_429_DETECTED" is added to a list object that will be returned, and it's up to you on what the next step
+should be.  The list object will contain all the URLs found before the HTTP 429 was detected.
+
+```python
+import yagooglesearch
+
+query = "site:twitter.com"
+
+client = yagooglesearch.SearchClient(
+    query,
+    tbs="li:1",
+    verbosity=4,
+    num=10,
+    max_search_result_urls_to_return=1000,
+    minimum_delay_between_paged_results_in_seconds=1,
+    yagooglesearch_manages_http_429s=False,  # Add to manage HTTP 429s.
+)
+client.assign_random_user_agent()
+
+urls = client.search()
+
+if "HTTP_429_DETECTED" in urls:
+    print("HTTP 429 detected...it's up to you to modify your search.")
+
+    # Remove HTTP_429_DETECTED from list.
+    urls.remove("HTTP_429_DETECTED")
+
+    print("URLs found before HTTP 429 detected...")
+
+    for url in urls:
+        print(url)
+```
+
+![http429_detection_string_in_returned_list.png](img/http429_detection_string_in_returned_list.png)
 
 ## HTTP and SOCKS5 proxy support
 

yagooglesearch/__init__.py

@@ -376,7 +376,7 @@ def get_page(self, url):
             # notification string.
             if not self.yagooglesearch_manages_http_429s:
                 ROOT_LOGGER.info("Since yagooglesearch_manages_http_429s=False, yagooglesearch is done.")
-                return "HTTP_429_detected"
+                return "HTTP_429_DETECTED"
 
             ROOT_LOGGER.info(f"Sleeping for {self.http_429_cool_off_time_in_minutes} minutes...")
             time.sleep(self.http_429_cool_off_time_in_minutes * 60)
@@ -447,9 +447,12 @@ def search(self):
             # Request Google search results.
             html = self.get_page(url)
 
-            # HTTP 429 message returned from get_page() function, return to calling script.
-            if html == "HTTP_429_detected":
-                return "HTTP_429_detected"
+            # HTTP 429 message returned from get_page() function, add "HTTP_429_DETECTED" to the set and return to the
+            # calling script.
+            if html == "HTTP_429_DETECTED":
+                unique_urls_set.add("HTTP_429_DETECTED")
+                self.unique_urls_list = list(unique_urls_set)
+                return self.unique_urls_list
 
             # Create the BeautifulSoup object.
             soup = BeautifulSoup(html, "html.parser")