Return to fetch_url

Author: YoussefKhalidAli

changelogs/fragments/10346-jenkins-plugins-fixes.yml

@@ -1,6 +1,6 @@
 bugfixes:
-  - "jenkins plugins plugin - install latest compatible version instead of latest (https://github.com/ansible-collections/community.general/pull/10346)."
-  - "jenkins plugins plugin - seperate Jenkins and external url credentials (https://github.com/ansible-collections/community.general/pull/10346)."
+  - "jenkins_plugin - install latest compatible version instead of latest (https://github.com/ansible-collections/community.general/issues/854)."
+  - "jenkins_plugin - seperate Jenkins and external URL credentials (https://github.com/ansible-collections/community.general/issues/4419)."
 
 minor_changes:
-  - "jenkins plugins plugin - install dependencies for specific version (https://github.com/ansible-collections/community.general/pull/10346)."
+  - "jenkins_plugin - install dependencies for specific version (https://github.com/ansible-collections/community.general/issue/4995)."

plugins/modules/jenkins_plugin.py

@@ -76,14 +76,16 @@
     default: ['https://updates.jenkins.io', 'http://mirrors.jenkins.io']
   updates_url_username:
     description:
-      - If using a custom O(updates_url), set this as the username of the user with access to the url.
+      - If using a custom O(updates_url), set this as the username of the user with access to the URL.
       - If the custom O(updates_url) does not require authentication, this can be left empty.
     type: str
+    version_added: 11.1.0
   updates_url_password:
     description:
-      - If using a custom O(updates_url), set this as the password of the user with access to the url.
+      - If using a custom O(updates_url), set this as the password of the user with access to the URL.
       - If the custom O(updates_url) does not require authentication, this can be left empty.
     type: str
+    version_added: 11.1.0
   update_json_url_segment:
     type: list
     elements: str
@@ -122,6 +124,8 @@
   with_dependencies:
     description:
       - Defines whether to install plugin dependencies.
+      - In earlier versions, this option had no effect when a specific C(version) was set.
+      - Since community.general 11.1.0, dependencies are also installed for versioned plugins.
     type: bool
     default: true
 
@@ -325,13 +329,12 @@
 import os
 import tempfile
 import time
-import base64
 from collections import OrderedDict
 
 from ansible.module_utils.basic import AnsibleModule, to_bytes
 from ansible.module_utils.six.moves import http_cookiejar as cookiejar
 from ansible.module_utils.six.moves.urllib.parse import urlencode
-from ansible.module_utils.urls import fetch_url, url_argument_spec, open_url
+from ansible.module_utils.urls import fetch_url, url_argument_spec, basic_auth_header
 from ansible.module_utils.six import text_type, binary_type
 from ansible.module_utils.common.text.converters import to_native
 
@@ -355,18 +358,14 @@ def __init__(self, module):
         # Authentication for non-Jenkins calls
         self.updates_url_credentials = {}
         if self.params.get('updates_url_username') and self.params.get('updates_url_password'):
-            auth = "{}:{}".format(self.params['updates_url_username'], self.params['updates_url_password']).encode("utf-8")
-            b64_auth = base64.b64encode(auth).decode("ascii")
-            self.updates_url_credentials["Authorization"] = "Basic {}".format(b64_auth)
+            self.updates_url_credentials["Authorization"] = basic_auth_header(self.params['updates_url_username'], self.params['updates_url_password'])
 
         # Crumb
         self.crumb = {}
 
         # Authentication for Jenkins calls
         if self.params.get('url_username') and self.params.get('url_password'):
-            auth = "{}:{}".format(self.params['url_username'], self.params['url_password']).encode("utf-8")
-            b64_auth = base64.b64encode(auth).decode("ascii")
-            self.crumb["Authorization"] = "Basic {}".format(b64_auth)
+            self.crumb["Authorization"] = basic_auth_header(self.params['url_username'], self.params['url_password'])
 
         # Cookie jar for crumb session
         self.cookies = None
@@ -418,16 +417,18 @@ def _get_urls_data(self, urls, what=None, msg_status=None, msg_exception=None, *
                 self.module.debug("fetching url: %s" % url)
 
                 is_jenkins_call = url.startswith(self.url)
+                self.module.params['force_basic_auth'] = is_jenkins_call
 
-                response = open_url(
-                    url, timeout=self.timeout,
-                    cookies=self.cookies if is_jenkins_call else None,
-                    headers=self.crumb if is_jenkins_call else self.updates_url_credentials, **kwargs)
-                if response.getcode() == 200:
+                response, info = fetch_url(
+                    self.module, url, timeout=self.timeout, cookies=self.cookies,
+                    headers=self.crumb if is_jenkins_call else self.updates_url_credentials or self.crumb,
+                    **kwargs)
+                if info['status'] == 200:
                     return response
                 else:
                     err_msg = ("%s. fetching url %s failed. response code: %s" % (msg_status, url, response.getcode()))
-
+                    if info['status'] > 400:  # extend error message
+                        err_msg = "%s. response body: %s" % (err_msg, info['body'])
             except Exception as e:
                 err_msg = "%s. fetching url %s failed. error msg: %s" % (msg_status, url, to_native(e))
             finally:
@@ -451,19 +452,18 @@ def _get_url_data(
         # Get the URL data
         try:
             is_jenkins_call = url.startswith(self.url)
-            response = open_url(
-                url, timeout=self.timeout,
-                cookies=self.cookies if is_jenkins_call else None,
-                headers=self.crumb if is_jenkins_call else self.updates_url_credentials, **kwargs)
+            self.module.params['force_basic_auth'] = is_jenkins_call
+
+            response, info = fetch_url(
+                self.module, url, timeout=self.timeout, cookies=self.cookies,
+                headers=self.crumb if is_jenkins_call else self.updates_url_credentials or self.crumb,
+                **kwargs)
 
-            if response.getcode() != 200:
+            if info['status'] != 200:
                 if dont_fail:
                     raise FailedInstallingWithPluginManager("HTTP {}".format(response.getcode()))
                 else:
-                    self.module.fail_json(
-                        msg=msg_status,
-                        details="Received status code {} from {}".format(response.getcode(), url)
-                    )
+                    self.module.fail_json(msg=msg_status, details=info['msg'], info=info)
         except Exception as e:
             if dont_fail:
                 raise FailedInstallingWithPluginManager(e)
@@ -679,6 +679,7 @@ def _get_latest_plugin_urls(self):
 
     def _get_latest_compatible_plugin_version(self, plugin_name=None):
         if not hasattr(self, 'jenkins_version'):
+            self.module.params['force_basic_auth'] = True
             resp, info = fetch_url(self.module, self.url)
             raw_version = info.get("x-jenkins")
             self.jenkins_version = self.parse_version(raw_version)
@@ -694,9 +695,9 @@ def _get_latest_compatible_plugin_version(self, plugin_name=None):
                 else:
                     raise FileNotFoundError("Cache file is outdated.")
         except Exception:
-            response = open_url("https://updates.jenkins.io/current/plugin-versions.json")   # Get list of plugins and their dependencies
+            response, info = fetch_url(self.module, "https://updates.jenkins.io/current/plugin-versions.json")   # Get list of plugins and their dependencies
 
-            if response.getcode() != 200:
+            if info['status'] != 200:
                 self.module.fail_json(msg="Failed to fetch plugin-versions.json", details=info)
 
             try:
@@ -949,9 +950,6 @@ def main():
         supports_check_mode=True,
     )
 
-    # Force basic authentication
-    module.params['force_basic_auth'] = True
-
     # Convert timeout to float
     try:
         module.params['timeout'] = float(module.params['timeout'])

tests/unit/plugins/modules/test_jenkins_plugin.py

@@ -7,7 +7,6 @@
 
 from io import BytesIO
 import json
-import socket
 from collections import OrderedDict
 
 from ansible_collections.community.general.plugins.modules.jenkins_plugin import JenkinsPlugin
@@ -16,6 +15,7 @@
     MagicMock,
     patch,
 )
+from ansible.module_utils.urls import basic_auth_header
 
 
 def pass_function(*args, **kwargs):
@@ -199,9 +199,8 @@ def isInList(l, i):
     return False
 
 
-@patch("ansible_collections.community.general.plugins.modules.jenkins_plugin.open_url")
 @patch("ansible_collections.community.general.plugins.modules.jenkins_plugin.fetch_url")
-def test__get_latest_compatible_plugin_version(fetch_mock, open_mock, mocker):
+def test__get_latest_compatible_plugin_version(fetch_mock, mocker):
     "test the latest compatible plugin version retrieval"
 
     params = {
@@ -216,37 +215,79 @@ def test__get_latest_compatible_plugin_version(fetch_mock, open_mock, mocker):
     module = mocker.Mock()
     module.params = params
 
-    mock_response = MagicMock()
-    mock_response.read.return_value = b""
-    fetch_mock.return_value = (mock_response, {"x-jenkins": "2.263.1"})
-
-    try:
-        socket.gethostbyname("updates.jenkins.io")
-        online = True
-    except socket.gaierror:
-        online = False
-
-    # Mock the open_url to simulate the response from Jenkins update center if tests are run offline
-    if not online:
-        plugin_data = {
-            "plugins": {
-                "git": OrderedDict([
-                    ("4.8.2", {"requiredCore": "2.263.1"}),
-                    ("4.8.3", {"requiredCore": "2.263.1"}),
-                    ("4.9.0", {"requiredCore": "2.289.1"}),
-                    ("4.9.1", {"requiredCore": "2.289.1"}),
-                ])
-            }
+    jenkins_info = {"x-jenkins": "2.263.1"}
+    jenkins_response = MagicMock()
+    jenkins_response.read.return_value = b"{}"
+
+    plugin_data = {
+        "plugins": {
+            "git": OrderedDict([
+                ("4.8.2", {"requiredCore": "2.263.1"}),
+                ("4.8.3", {"requiredCore": "2.263.1"}),
+                ("4.9.0", {"requiredCore": "2.289.1"}),
+                ("4.9.1", {"requiredCore": "2.289.1"}),
+            ])
         }
-        mock_open_resp = MagicMock()
-        mock_open_resp.getcode.return_value = 200
-        mock_open_resp.read.return_value = json.dumps(plugin_data).encode("utf-8")
-        open_mock.return_value = mock_open_resp
+    }
+    plugin_versions_response = MagicMock()
+    plugin_versions_response.read.return_value = json.dumps(plugin_data).encode("utf-8")
+    plugin_versions_info = {"status": 200}
 
-    JenkinsPlugin._csrf_enabled = pass_function
-    JenkinsPlugin._get_installed_plugins = pass_function
+    def fetch_url_side_effect(module, url, **kwargs):
+        if "plugin-versions.json" in url:
+            return (plugin_versions_response, plugin_versions_info)
+        else:
+            return (jenkins_response, jenkins_info)
 
-    jenkins_plugin = JenkinsPlugin(module)
+    fetch_mock.side_effect = fetch_url_side_effect
+
+    JenkinsPlugin._csrf_enabled = lambda self: False
+    JenkinsPlugin._get_installed_plugins = lambda self: None
 
+    jenkins_plugin = JenkinsPlugin(module)
     latest_version = jenkins_plugin._get_latest_compatible_plugin_version()
     assert latest_version == '4.8.3'
+
+
+@patch("ansible_collections.community.general.plugins.modules.jenkins_plugin.fetch_url")
+def test__get_urls_data_sets_correct_headers(fetch_mock, mocker):
+    params = {
+        "url": "http://jenkins.example.com",
+        "timeout": 30,
+        "name": "git",
+        "jenkins_home": "/var/lib/jenkins",
+        "updates_url": ["http://updates.example.com"],
+        "latest_plugins_url_segments": ["latest"],
+        "update_json_url_segment": ["update-center.json"],
+        "versioned_plugins_url_segments": ["plugins"],
+        "url_username": "jenkins_user",
+        "url_password": "jenkins_pass",
+        "updates_url_username": "update_user",
+        "updates_url_password": "update_pass",
+    }
+    module = mocker.Mock()
+    module.params = params
+
+    dummy_response = MagicMock()
+    fetch_mock.return_value = (dummy_response, {"status": 200})
+
+    JenkinsPlugin._csrf_enabled = lambda self: False
+    JenkinsPlugin._get_installed_plugins = lambda self: None
+
+    jp = JenkinsPlugin(module)
+
+    update_url = "http://updates.example.com/plugin-versions.json"
+    jp._get_urls_data([update_url])
+
+    jenkins_url = "http://jenkins.example.com/some-endpoint"
+    jp._get_urls_data([jenkins_url])
+
+    calls = fetch_mock.call_args_list
+
+    dummy, kwargs_2 = calls[1]
+    jenkins_auth = basic_auth_header("jenkins_user", "jenkins_pass")
+    assert kwargs_2["headers"]["Authorization"] == jenkins_auth
+
+    dummy, kwargs_1 = calls[0]
+    updates_auth = basic_auth_header("update_user", "update_pass")
+    assert kwargs_1["headers"]["Authorization"] == updates_auth