Merge pull request #4 from opsdisk/issue-3-make-ssl-verify-optional

Added verify_ssl option

Author: opsdisk

README.md

@@ -13,6 +13,7 @@ heavily based off the [googlesearch](https://github.com/MarioVilas/googlesearch)
 * Randomizing delay times between retrieving paged search results (i.e., clicking on page 2 for more results)
 * HTTP(S) and SOCKS5 proxy support
 * Leveraging `requests` library for HTTP requests and cookie management
+* Adds "&filter=0" by default to search URLs to prevent any omission or filtering of search results by Google
 * Console and file logging
 * Python 3.6+
 
@@ -61,7 +62,7 @@ client = yagooglesearch.SearchClient(
     http_429_cool_off_time_in_minutes=45,
     http_429_cool_off_factor=1.5,
     proxy="socks5h://127.0.0.1:9050",
-    verbosity=5
+    verbosity=5,
 )
 client.assign_random_user_agent()
 
@@ -123,6 +124,40 @@ Supported proxy schemes are based off those supported in the Python `requests` l
 * `socks5h` - "If you want to resolve the domains on the proxy server, use socks5h as the scheme."  This is the
   **best**  option if you are using SOCKS because the DNS lookup and Google search is sourced from the proxy IP address.
 
+## HTTPS proxies and SSL/TLS certificates
+
+If you are using a self-signed certificate for an HTTPS proxy, you will likely need to disable SSL/TLS verification when
+either:
+
+1) Instantiating the `yagooglesearch.SearchClient` object:
+
+```python
+import yagooglesearch
+
+query = "site:github.com"
+
+client = yagooglesearch.SearchClient(
+    query,
+    proxy="http://127.0.0.1:8080",
+    verify_ssl=False,
+    verbosity=5,
+)
+```
+
+2) or after instantiation:
+
+```python
+query = "site:github.com"
+
+client = yagooglesearch.SearchClient(
+    query,
+    proxy="http://127.0.0.1:8080",
+    verbosity=5,
+)
+
+client.verify_ssl = False
+```
+
 ## Multiple proxies
 
 If you want to use multiple proxies, that burden is on the script utilizing the `yagooglesearch` library to instantiate
@@ -135,7 +170,7 @@ import yagooglesearch
 proxies = [
     "socks5h://127.0.0.1:9050",
     "socks5h://127.0.0.1:9051",
-    "socks5h://127.0.0.1:9052",
+    "http://127.0.0.1:9052",  # HTTPS proxy with a self-signed SSL/TLS certificate.
 ]
 
 search_queries = [
@@ -158,6 +193,10 @@ for search_query in search_queries:
         proxy=proxies[proxy_index],
     )
 
+    # Only disable SSL/TLS verification for the HTTPS proxy using a self-signed certificate.
+    if proxies[proxy_index].startswith("http://"):
+        client.verify_ssl = False
+
     urls_list = client.search()
 
     print(urls_list)

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="yagooglesearch",
-    version="1.1.0",
+    version="1.2.0",
     author="Brennon Thomas",
     author_email="info@opsdisk.com",
     description="A Python library for executing intelligent, realistic-looking, and tunable Google searches.",

yagooglesearch/__init__.py

@@ -12,7 +12,7 @@
 
 # Custom Python libraries.
 
-__version__ = "1.1.0"
+__version__ = "1.2.0"
 
 # Logging
 ROOT_LOGGER = logging.getLogger("yagooglesearch")
@@ -82,6 +82,7 @@ def __init__(
         http_429_cool_off_time_in_minutes=60,
         http_429_cool_off_factor=1.1,
         proxy="",
+        verify_ssl=True,
         verbosity=5,
     ):
 
@@ -108,6 +109,8 @@ def __init__(
         :param float http_429_cool_off_factor: Factor to multiply by http_429_cool_off_time_in_minutes for each HTTP 429
             detected.
         :param str proxy: HTTP(S) or SOCKS5 proxy to use.
+        :param bool verify_ssl: Verify the SSL certificate to prevent traffic interception attacks. Defaults to True.
+            This may need to be disabled in some HTTPS proxy instances.
         :param int verbosity: Logging and console output verbosity.
 
         :rtype: List of str
@@ -129,6 +132,7 @@ def __init__(
         self.http_429_cool_off_time_in_minutes = http_429_cool_off_time_in_minutes
         self.http_429_cool_off_factor = http_429_cool_off_factor
         self.proxy = proxy
+        self.verify_ssl = verify_ssl
         self.verbosity = verbosity
 
         # Assign log level.
@@ -176,6 +180,10 @@ def __init__(
                 "https": self.proxy,
             }
 
+        # Suppress warning messages if verify_ssl is disabled.
+        if not self.verify_ssl:
+            requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
+
     def update_urls(self):
         """Update search URLs being used."""
 
@@ -234,8 +242,9 @@ def filter_search_result_urls(self, link):
         ROOT_LOGGER.debug(f"pre filter_search_result_urls() link: {link}")
 
         try:
-            # Extract URL from parameter.
-            if link.startswith("/url?"):
+            # Extract URL from parameter.  Once in a while the full "http://www.google.com/url?" exists instead of just
+            # "/url?".  After a re-run, it disappears and "/url?" is present...might be a caching thing?
+            if link.startswith("/url?") or link.startswith("http://www.google.com/url?"):
                 urlparse_object = urllib.parse.urlparse(link, scheme="http")
 
                 # The "q" key exists most of the time.
@@ -294,7 +303,9 @@ def get_page(self, url):
         }
 
         ROOT_LOGGER.info(f"Requesting URL: {url}")
-        response = requests.get(url, proxies=self.proxy_dict, headers=headers, cookies=self.cookies, timeout=15)
+        response = requests.get(
+            url, proxies=self.proxy_dict, headers=headers, cookies=self.cookies, timeout=15, verify=self.verify_ssl
+        )
 
         # Update the cookies.
         self.cookies = response.cookies
@@ -303,10 +314,11 @@ def get_page(self, url):
         http_response_code = response.status_code
 
         # debug_requests_response(response)
-        ROOT_LOGGER.debug(f"     status_code: {http_response_code}")
-        ROOT_LOGGER.debug(f"     proxy: {self.proxy}")
-        ROOT_LOGGER.debug(f"     headers: {headers}")
-        ROOT_LOGGER.debug(f"     cookies: {self.cookies}")
+        ROOT_LOGGER.debug(f"    status_code: {http_response_code}")
+        ROOT_LOGGER.debug(f"    headers: {headers}")
+        ROOT_LOGGER.debug(f"    cookies: {self.cookies}")
+        ROOT_LOGGER.debug(f"    proxy: {self.proxy}")
+        ROOT_LOGGER.debug(f"    verify_ssl: {self.verify_ssl}")
 
         html = ""