Pack dependency management

CHANGELOG.rst

@@ -11,6 +11,7 @@ Added
   ``url_hosts_blacklist`` and ``url_hosts_whitelist`` runner attribute. (new feature)
   #4757
 * Add ``user`` parameter to ``re_run`` method of st2client. #4785
+* Install pack dependencies automatically. #4769
 
 Changed
 ~~~~~~~

contrib/core/actions/error.yaml

@@ -0,0 +1,24 @@
+---
+description: Action that executes the Linux echo command (to stderr) on the localhost.
+runner_type: "local-shell-cmd"
+enabled: true
+entry_point: ''
+name: error
+parameters:
+  message:
+    description: The message that the command will echo to stderr.
+    type: string
+    required: true
+  cmd:
+    description: Arbitrary Linux command to be executed on the local host.
+    required: true
+    type: string
+    default: '>&2 echo "{{message}}"'
+    immutable: true
+  kwarg_op:
+    immutable: true
+  sudo:
+    default: false
+    immutable: true
+  sudo_password:
+    immutable: true

contrib/hello_st2/pack.yaml

@@ -16,6 +16,12 @@ version: 0.1.0
 python_versions:
   - "2"
   - "3"
+# New in StackStorm 3.2
+# Specify a list of dependency packs to install. If the pack is in StackStorm Exchange you can use
+# the pack name, or you can specify a full Git repository URL. Optionally, you can specify the
+# exact version, tag, or branch.
+dependencies:
+  - core
 # Name of the pack author.
 author: StackStorm, Inc.
 # Email of the pack author.

contrib/packs/actions/delete.yaml

@@ -14,3 +14,8 @@
       type: "string"
       default: "/opt/stackstorm/packs/"
       immutable: true
+    delete_env:
+      type: "boolean"
+      description: Delete virtual environment for pack when set to True.
+      default: true
+      required: false

contrib/packs/actions/download.yaml

@@ -27,3 +27,9 @@
       description: "True to use Python 3 binary for this pack."
       required: false
       default: false
+    dependency_list:
+      type: "array"
+      description: "Dependency list that needs to be downloaded."
+      items:
+        type: "string"
+      required: false

contrib/packs/actions/get_pack_dependencies.yaml

@@ -0,0 +1,20 @@
+
+
+---
+  name: "get_pack_dependencies"
+  runner_type: "python-script"
+  description: "Get pack dependencies specified in pack.yaml"
+  enabled: true
+  pack: packs
+  entry_point: "pack_mgmt/get_pack_dependencies.py"
+  parameters:
+    packs_status:
+      type: object
+      description: Name of the pack in Exchange or a git repo URL and download status.
+      required: true
+      default: null
+    nested:
+      type: integer
+      description: Nested level of dependencies to prevent infinite or really long download loops.
+      required: true
+      default: 3

contrib/packs/actions/install.meta.yaml

@@ -1,6 +1,6 @@
 ---
   name: "install"
-  runner_type: "action-chain"
+  runner_type: "orquesta"
   description: "Installs or upgrades a pack into local content repository, either by
                 git URL or a short name matching an index entry.
                 Will download pack, load the actions, sensors and rules from the pack.
@@ -32,3 +32,8 @@
       description: "Use Python 3 binary when creating a virtualenv for this pack."
       required: false
       default: false
+    skip_dependencies:
+      type: "boolean"
+      description: "Set to True to skip pack dependency installations."
+      required: false
+      default: false

contrib/packs/actions/pack_mgmt/delete.py

@@ -30,7 +30,7 @@ def __init__(self, config=None, action_service=None):
         self._base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path,
                                                    'virtualenvs/')
 
-    def run(self, packs, abs_repo_base):
+    def run(self, packs, abs_repo_base, delete_env=True):
         intersection = BLOCKED_PACKS & frozenset(packs)
         if len(intersection) > 0:
             names = ', '.join(list(intersection))
@@ -43,12 +43,13 @@ def run(self, packs, abs_repo_base):
                 self.logger.debug('Deleting pack directory "%s"' % (abs_fp))
                 shutil.rmtree(abs_fp)
 
-        # 2. Delete pack virtual environment
-        for pack_name in packs:
-            pack_name = quote_unix(pack_name)
-            virtualenv_path = os.path.join(self._base_virtualenvs_path, pack_name)
+        if delete_env:
+            # 2. Delete pack virtual environment
+            for pack_name in packs:
+                pack_name = quote_unix(pack_name)
+                virtualenv_path = os.path.join(self._base_virtualenvs_path, pack_name)
 
-            if os.path.isdir(virtualenv_path):
-                self.logger.debug('Deleting virtualenv "%s" for pack "%s"' %
-                                  (virtualenv_path, pack_name))
-                shutil.rmtree(virtualenv_path)
+                if os.path.isdir(virtualenv_path):
+                    self.logger.debug('Deleting virtualenv "%s" for pack "%s"' %
+                                      (virtualenv_path, pack_name))
+                    shutil.rmtree(virtualenv_path)

contrib/packs/actions/pack_mgmt/download.py

@@ -62,18 +62,29 @@ def __init__(self, config=None, action_service=None):
         if self.proxy_ca_bundle_path and not os.environ.get('proxy_ca_bundle_path', None):
             os.environ['no_proxy'] = self.no_proxy
 
-    def run(self, packs, abs_repo_base, verifyssl=True, force=False, python3=False):
+    def run(self, packs, abs_repo_base, verifyssl=True, force=False, python3=False,
+            dependency_list=None):
         result = {}
-
-        for pack in packs:
-            pack_result = download_pack(pack=pack, abs_repo_base=abs_repo_base,
-                                        verify_ssl=verifyssl, force=force,
-                                        proxy_config=self.proxy_config,
-                                        force_permissions=True,
-                                        use_python3=python3,
-                                        logger=self.logger)
-            pack_url, pack_ref, pack_result = pack_result
-            result[pack_ref] = pack_result
+        pack_url = None
+
+        if dependency_list:
+            for pack_dependency in dependency_list:
+                pack_result = download_pack(pack=pack_dependency, abs_repo_base=abs_repo_base,
+                                            verify_ssl=verifyssl, force=force,
+                                            proxy_config=self.proxy_config, force_permissions=True,
+                                            use_python3=python3, logger=self.logger)
+                pack_url, pack_ref, pack_result = pack_result
+                result[pack_ref] = pack_result
+        else:
+            for pack in packs:
+                pack_result = download_pack(pack=pack, abs_repo_base=abs_repo_base,
+                                            verify_ssl=verifyssl, force=force,
+                                            proxy_config=self.proxy_config,
+                                            force_permissions=True,
+                                            use_python3=python3,
+                                            logger=self.logger)
+                pack_url, pack_ref, pack_result = pack_result
+                result[pack_ref] = pack_result
 
         return self._validate_result(result=result, repo_url=pack_url)
 

contrib/packs/actions/pack_mgmt/get_pack_dependencies.py

@@ -0,0 +1,131 @@
+# Copyright 2019 Extreme Networks, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import print_function
+
+import six
+
+from st2common.constants.pack import PACK_VERSION_SEPARATOR
+from st2common.content.utils import get_pack_base_path
+from st2common.runners.base_action import Action
+from st2common.util.pack import get_pack_metadata
+
+
+class GetPackDependencies(Action):
+    def run(self, packs_status, nested):
+        """
+        :param packs_status: Name of the pack in Exchange or a git repo URL and download status.
+        :type: packs_status: ``dict``
+
+        :param nested: Nested level of dependencies to prevent infinite or really
+        long download loops.
+        :type nested: ``integer``
+        """
+        result = {}
+        dependency_list = []
+        conflict_list = []
+
+        if not packs_status or nested == 0:
+            return result
+
+        for pack, status in six.iteritems(packs_status):
+            if 'success' not in status.lower():
+                continue
+
+            dependency_packs = get_dependency_list(pack)
+            if not dependency_packs:
+                continue
+
+            for dep_pack in dependency_packs:
+                name_or_url, pack_version = self.get_name_and_version(dep_pack)
+
+                if len(name_or_url.split('/')) == 1:
+                    pack_name = name_or_url
+                else:
+                    name_or_git = name_or_url.split("/")[-1]
+                    pack_name = name_or_git if '.git' not in name_or_git else \
+                        name_or_git.split('.')[0]
+
+                # Check existing pack by pack name
+                existing_pack_version = get_pack_version(pack_name)
+
+                # Try one more time to get existing pack version by name if 'stackstorm-' is in
+                # pack name
+                if not existing_pack_version and 'stackstorm-' in pack_name.lower():
+                    existing_pack_version = get_pack_version(pack_name.split('stackstorm-')[-1])
+
+                if existing_pack_version:
+                    if existing_pack_version and not existing_pack_version.startswith('v'):
+                        existing_pack_version = 'v' + existing_pack_version
+                    if pack_version and not pack_version.startswith('v'):
+                        pack_version = 'v' + pack_version
+                    if pack_version and existing_pack_version != pack_version \
+                            and dep_pack not in conflict_list:
+                        conflict_list.append(dep_pack)
+                else:
+                    conflict = self.check_dependency_list_for_conflict(name_or_url, pack_version,
+                                                                       dependency_list)
+                    if conflict:
+                        conflict_list.append(dep_pack)
+                    elif dep_pack not in dependency_list:
+                        dependency_list.append(dep_pack)
+
+        result['dependency_list'] = dependency_list
+        result['conflict_list'] = conflict_list
+        result['nested'] = nested - 1
+
+        return result
+
+    def check_dependency_list_for_conflict(self, name, version, dependency_list):
+        conflict = False
+
+        for pack in dependency_list:
+            name_or_url, pack_version = self.get_name_and_version(pack)
+            if name == name_or_url:
+                if version != pack_version:
+                    conflict = True
+                break
+
+        return conflict
+
+    @staticmethod
+    def get_name_and_version(pack):
+        pack_and_version = pack.split(PACK_VERSION_SEPARATOR)
+        name_or_url = pack_and_version[0]
+        pack_version = pack_and_version[1] if len(pack_and_version) > 1 else None
+
+        return name_or_url, pack_version
+
+
+def get_pack_version(pack=None):
+    pack_path = get_pack_base_path(pack)
+    try:
+        pack_metadata = get_pack_metadata(pack_dir=pack_path)
+        result = pack_metadata.get('version', None)
+    except Exception:
+        result = None
+    finally:
+        return result
+
+
+def get_dependency_list(pack=None):
+    pack_path = get_pack_base_path(pack)
+
+    try:
+        pack_metadata = get_pack_metadata(pack_dir=pack_path)
+        result = pack_metadata.get('dependencies', None)
+    except Exception:
+        print('Could not open pack.yaml at location %s' % pack_path)
+        result = None
+    finally:
+        return result

contrib/packs/actions/pack_mgmt/register.py

@@ -72,6 +72,7 @@ def run(self, register, packs=None):
             'types': types
         }
 
+        packs.reverse()
         if packs:
             method_kwargs['packs'] = packs
 

contrib/packs/actions/pack_mgmt/virtualenv_setup_prerun.py

@@ -18,13 +18,22 @@
 
 
 class PacksTransformationAction(Action):
-    def run(self, packs_status):
+    def run(self, packs_status, packs_list=None):
         """
         :param packs_status: Result from packs.download action.
         :type: packs_status: ``dict``
+
+        :param packs_list: Names of the pack in Exchange, a git repo URL or local file system.
+        :type: packs_list: ``list``
         """
+        if not packs_list:
+            packs_list = []
+
         packs = []
         for pack_name, status in six.iteritems(packs_status):
             if 'success' in status.lower():
                 packs.append(pack_name)
-        return packs
+
+        packs_list.extend(packs)
+
+        return packs_list

contrib/packs/actions/virtualenv_prerun.yaml

@@ -7,3 +7,10 @@
   parameters:
     packs_status:
       type: "object"
+    packs_list:
+      type: array
+      items:
+        type: string
+      required: false
+      default: null
+      description: Names of the pack in Exchange, a git repo URL or local file system.