Addressing multiple jenkins_plugins module issue

changelogs/fragments/10346-jenkins-plugins-fixes.yml

@@ -0,0 +1,6 @@
+bugfixes:
+  - "jenkins plugins plugin - install latest compatible version instead of latest (https://github.com/ansible-collections/community.general/pull/10346)."
+  - "jenkins plugins plugin - seperate Jenkins and external url credentials (https://github.com/ansible-collections/community.general/pull/10346)."
+
+minor_changes:
+  - "jenkins plugins plugin - install dependencies for specific version (https://github.com/ansible-collections/community.general/pull/10346)."

plugins/modules/jenkins_plugin.py

@@ -74,6 +74,16 @@
       - A list of base URL(s) to retrieve C(update-center.json), and direct plugin files from.
       - This can be a list since community.general 3.3.0.
     default: ['https://updates.jenkins.io', 'http://mirrors.jenkins.io']
+  updates_url_username:
+    description:
+      - If using a custom O(updates_url), set this as the username of the user with access to the url.
+      - If the custom O(updates_url) does not require authentication, this can be left empty.
+    type: str
+  updates_url_password:
+    description:
+      - If using a custom O(updates_url), set this as the password of the user with access to the url.
+      - If the custom O(updates_url) does not require authentication, this can be left empty.
+    type: str
   update_json_url_segment:
     type: list
     elements: str
@@ -112,7 +122,6 @@
   with_dependencies:
     description:
       - Defines whether to install plugin dependencies.
-      - This option takes effect only if the O(version) is not defined.
     type: bool
     default: true
 
@@ -315,11 +324,14 @@
 import json
 import os
 import tempfile
+import time
+import base64
+from collections import OrderedDict
 
 from ansible.module_utils.basic import AnsibleModule, to_bytes
 from ansible.module_utils.six.moves import http_cookiejar as cookiejar
 from ansible.module_utils.six.moves.urllib.parse import urlencode
-from ansible.module_utils.urls import fetch_url, url_argument_spec
+from ansible.module_utils.urls import fetch_url, url_argument_spec, open_url
 from ansible.module_utils.six import text_type, binary_type
 from ansible.module_utils.common.text.converters import to_native
 
@@ -340,14 +352,28 @@ def __init__(self, module):
         self.url = self.params['url']
         self.timeout = self.params['timeout']
 
+        # Authentication for non-Jenkins calls
+        self.updates_url_credentials = {}
+        if self.params.get('updates_url_username') and self.params.get('updates_url_password'):
+            auth = "{}:{}".format(self.params['updates_url_username'], self.params['updates_url_password']).encode("utf-8")
+            b64_auth = base64.b64encode(auth).decode("ascii")
+            self.updates_url_credentials["Authorization"] = "Basic {}".format(b64_auth)
+
         # Crumb
         self.crumb = {}
+
+        # Authentication for Jenkins calls
+        if self.params.get('url_username') and self.params.get('url_password'):
+            auth = "{}:{}".format(self.params['url_username'], self.params['url_password']).encode("utf-8")
+            b64_auth = base64.b64encode(auth).decode("ascii")
+            self.crumb["Authorization"] = "Basic {}".format(b64_auth)
+
         # Cookie jar for crumb session
         self.cookies = None
 
         if self._csrf_enabled():
             self.cookies = cookiejar.LWPCookieJar()
-            self.crumb = self._get_crumb()
+            self._get_crumb()
 
         # Get list of installed plugins
         self._get_installed_plugins()
@@ -390,16 +416,18 @@ def _get_urls_data(self, urls, what=None, msg_status=None, msg_exception=None, *
             err_msg = None
             try:
                 self.module.debug("fetching url: %s" % url)
-                response, info = fetch_url(
-                    self.module, url, timeout=self.timeout, cookies=self.cookies,
-                    headers=self.crumb, **kwargs)
 
-                if info['status'] == 200:
+                is_jenkins_call = url.startswith(self.url)
+
+                response = open_url(
+                    url, timeout=self.timeout,
+                    cookies=self.cookies if is_jenkins_call else None,
+                    headers=self.crumb if is_jenkins_call else self.updates_url_credentials, **kwargs)
+                if response.getcode() == 200:
                     return response
                 else:
-                    err_msg = ("%s. fetching url %s failed. response code: %s" % (msg_status, url, info['status']))
-                    if info['status'] > 400:  # extend error message
-                        err_msg = "%s. response body: %s" % (err_msg, info['body'])
+                    err_msg = ("%s. fetching url %s failed. response code: %s" % (msg_status, url, response.getcode()))
+
             except Exception as e:
                 err_msg = "%s. fetching url %s failed. error msg: %s" % (msg_status, url, to_native(e))
             finally:
@@ -422,15 +450,20 @@ def _get_url_data(
 
         # Get the URL data
         try:
-            response, info = fetch_url(
-                self.module, url, timeout=self.timeout, cookies=self.cookies,
-                headers=self.crumb, **kwargs)
+            is_jenkins_call = url.startswith(self.url)
+            response = open_url(
+                url, timeout=self.timeout,
+                cookies=self.cookies if is_jenkins_call else None,
+                headers=self.crumb if is_jenkins_call else self.updates_url_credentials, **kwargs)
 
-            if info['status'] != 200:
+            if response.getcode() != 200:
                 if dont_fail:
-                    raise FailedInstallingWithPluginManager(info['msg'])
+                    raise FailedInstallingWithPluginManager("HTTP {}".format(response.getcode()))
                 else:
-                    self.module.fail_json(msg=msg_status, details=info['msg'])
+                    self.module.fail_json(
+                        msg=msg_status,
+                        details="Received status code {} from {}".format(response.getcode(), url)
+                    )
         except Exception as e:
             if dont_fail:
                 raise FailedInstallingWithPluginManager(e)
@@ -444,16 +477,12 @@ def _get_crumb(self):
             "%s/%s" % (self.url, "crumbIssuer/api/json"), 'Crumb')
 
         if 'crumbRequestField' in crumb_data and 'crumb' in crumb_data:
-            ret = {
-                crumb_data['crumbRequestField']: crumb_data['crumb']
-            }
+            self.crumb[crumb_data['crumbRequestField']] = crumb_data['crumb']
         else:
             self.module.fail_json(
                 msg="Required fields not found in the Crum response.",
                 details=crumb_data)
 
-        return ret
-
     def _get_installed_plugins(self):
         plugins_data = self._get_json_data(
             "%s/%s" % (self.url, "pluginManager/api/json?depth=1"),
@@ -467,6 +496,7 @@ def _get_installed_plugins(self):
         self.is_installed = False
         self.is_pinned = False
         self.is_enabled = False
+        self.installed_plugins = plugins_data['plugins']
 
         for p in plugins_data['plugins']:
             if p['shortName'] == self.params['name']:
@@ -480,6 +510,37 @@ def _get_installed_plugins(self):
 
                 break
 
+    def _install_dependencies(self):
+        dependencies = self._get_versioned_dependencies()
+        self.dependencies_states = []
+
+        for dep_name, dep_version in dependencies.items():
+            if not any(p['shortName'] == dep_name and p['version'] == dep_version for p in self.installed_plugins):
+                dep_params = self.params.copy()
+                dep_params['name'] = dep_name
+                dep_params['version'] = dep_version
+                dep_module = self.module
+                dep_module.params = dep_params
+                dep_plugin = JenkinsPlugin(dep_module)
+                if not dep_plugin.install():
+                    self.dependencies_states.append(
+                        {
+                            'name': dep_name,
+                            'version': dep_version,
+                            'state': 'absent'})
+                else:
+                    self.dependencies_states.append(
+                        {
+                            'name': dep_name,
+                            'version': dep_version,
+                            'state': 'present'})
+            else:
+                self.dependencies_states.append(
+                    {
+                        'name': dep_name,
+                        'version': dep_version,
+                        'state': 'present'})
+
     def _install_with_plugin_manager(self):
         if not self.module.check_mode:
             # Install the plugin (with dependencies)
@@ -540,6 +601,10 @@ def install(self):
                     plugin_content = plugin_fh.read()
                 checksum_old = hashlib.sha1(plugin_content).hexdigest()
 
+            # Install dependencies
+            if self.params['with_dependencies']:
+                self._install_dependencies()
+
             if self.params['version'] in [None, 'latest']:
                 # Take latest version
                 plugin_urls = self._get_latest_plugin_urls()
@@ -612,6 +677,53 @@ def _get_latest_plugin_urls(self):
                 urls.append("{0}/{1}/{2}.hpi".format(base_url, update_segment, self.params['name']))
         return urls
 
+    def _get_latest_compatible_plugin_version(self, plugin_name=None):
+        if not hasattr(self, 'jenkins_version'):
+            resp, info = fetch_url(self.module, self.url)
+            raw_version = info.get("x-jenkins")
+            self.jenkins_version = self.parse_version(raw_version)
+        name = plugin_name or self.params['name']
+        cache_path = "{}/ansible_jenkins_plugin_cache.json".format(self.params['jenkins_home'])
+
+        try:  # Check if file is saved localy
+            with open(cache_path, "r") as f:
+                file_mtime = os.path.getmtime(cache_path)
+                now = time.time()
+                if now - file_mtime < 86400:
+                    plugin_data = json.load(f)
+                else:
+                    raise FileNotFoundError("Cache file is outdated.")
+        except Exception:
+            response = open_url("https://updates.jenkins.io/current/plugin-versions.json")   # Get list of plugins and their dependencies
+
+            if response.getcode() != 200:
+                self.module.fail_json(msg="Failed to fetch plugin-versions.json", details=info)
+
+            try:
+                plugin_data = json.loads(to_native(response.read()), object_pairs_hook=OrderedDict)
+
+                # Save it to file for next time
+                with open(cache_path, "w") as f:
+                    json.dump(plugin_data, f)
+            except Exception as e:
+                self.module.fail_json(msg="Failed to parse plugin-versions.json", details=to_native(e))
+
+        plugin_versions = plugin_data.get("plugins", {}).get(name)
+        if not plugin_versions:
+            self.module.fail_json(msg="Plugin '{}' not found.".format(name))
+
+        sorted_versions = list(reversed(plugin_versions.items()))
+
+        for idx, (version_title, version_info) in enumerate(sorted_versions):
+            required_core = version_info.get("requiredCore", "0.0")
+            if self.parse_version(required_core) <= self.jenkins_version:
+                return 'latest' if idx == 0 else version_title
+
+        self.module.warn(
+            "No compatible version found for plugin '{}'. "
+            "Installing latest version.".format(name))
+        return 'latest'
+
     def _get_versioned_plugin_urls(self):
         urls = []
         for base_url in self.params['updates_url']:
@@ -626,6 +738,18 @@ def _get_update_center_urls(self):
                 urls.append("{0}/{1}".format(base_url, update_json))
         return urls
 
+    def _get_versioned_dependencies(self):
+        # Get dependencies for the specified plugin version
+        plugin_data = self._download_updates()['dependencies']
+
+        dependencies_info = {
+            dep["name"]: self._get_latest_compatible_plugin_version(dep["name"])
+            for dep in plugin_data
+            if not dep.get("optional", False)
+        }
+
+        return dependencies_info
+
     def _download_updates(self):
         try:
             updates_file, download_updates = download_updates_file(self.params['updates_expiration'])
@@ -779,6 +903,10 @@ def _pm_query(self, action, msg):
             msg_exception="%s has failed." % msg,
             method="POST")
 
+    @staticmethod
+    def parse_version(version_str):
+        return tuple(int(x) for x in version_str.split('.'))
+
 
 def main():
     # Module arguments
@@ -803,6 +931,8 @@ def main():
         updates_expiration=dict(default=86400, type="int"),
         updates_url=dict(type="list", elements="str", default=['https://updates.jenkins.io',
                                                                'http://mirrors.jenkins.io']),
+        updates_url_username=dict(type="str"),
+        updates_url_password=dict(type="str", no_log=True),
         update_json_url_segment=dict(type="list", elements="str", default=['update-center.json',
                                                                            'updates/update-center.json']),
         latest_plugins_url_segments=dict(type="list", elements="str", default=['latest']),
@@ -829,11 +959,17 @@ def main():
         module.fail_json(
             msg='Cannot convert %s to float.' % module.params['timeout'],
             details=to_native(e))
+    # Instantiate the JenkinsPlugin object
+    jp = JenkinsPlugin(module)
 
     # Set version to latest if state is latest
     if module.params['state'] == 'latest':
         module.params['state'] = 'present'
-        module.params['version'] = 'latest'
+        module.params['version'] = jp._get_latest_compatible_plugin_version()
+
+    # Ser version to latest compatible version if version is latest
+    if module.params['version'] == 'latest':
+        module.params['version'] = jp._get_latest_compatible_plugin_version()
 
     # Create some shortcuts
     name = module.params['name']
@@ -842,9 +978,6 @@ def main():
     # Initial change state of the task
     changed = False
 
-    # Instantiate the JenkinsPlugin object
-    jp = JenkinsPlugin(module)
-
     # Perform action depending on the requested state
     if state == 'present':
         changed = jp.install()
@@ -860,7 +993,7 @@ def main():
         changed = jp.disable()
 
     # Print status of the change
-    module.exit_json(changed=changed, plugin=name, state=state)
+    module.exit_json(changed=changed, plugin=name, state=state, dependencies=jp.dependencies_states if hasattr(jp, 'dependencies_states') else None)
 
 
 if __name__ == '__main__':