Pack dependency management

Fixes https://github.com/StackStorm/discussions/issues/354
1. `packs.install` workflow changed from `achtion-chain` to `orquesta` for workflow complexity.
2. Download all packs and dependency packs, check for conflict packs before `install pack requirements` and `register pack`.
3. Workflow fails on any time if conflict pack is recognized,  downloaded packs will be removed.
4.  Register packs in reverse order, it means the last dependency pack will be installed first.
5. New option `--skip` (default value is False) for skipping dependencies check:  `st2 install pack --skip`
6. Only case that will check conflict is that if pack was installed and dependency pack has specified pack version.
7. Nested level of dependencies to prevent infinite or long download loops is 3.
8.  Support StackStorm Exchange packs, the private repo ULR and local file system for conflict pack checking.

Author: jinpingh

CHANGELOG.rst

@@ -10,6 +10,8 @@ Added
 * Add support for blacklisting / whitelisting hosts to the HTTP runner by adding new
   ``url_hosts_blacklist`` and ``url_hosts_whitelist`` runner attribute. (new feature)
   #4757
+* Add support for Pack dependency management to install dependency packs and find out confilct packs.
+  #4769
 
 Changed
 ~~~~~~~

contrib/packs/actions/delete.yaml

@@ -14,3 +14,8 @@
       type: "string"
       default: "/opt/stackstorm/packs/"
       immutable: true
+    delete_env:
+      type: "boolean"
+      description: Flag for if need to delete virtual environment.
+      default: true
+      required: false

contrib/packs/actions/download.yaml

@@ -27,3 +27,9 @@
       description: "True to use Python 3 binary for this pack."
       required: false
       default: false
+    dependency_list:
+      type: "array"
+      description: "Dependency list that needs to be downloaded."
+      items:
+        type: "string"
+      required: false

contrib/packs/actions/get_pack_dependencies.yaml

@@ -0,0 +1,20 @@
+
+
+---
+  name: "get_pack_dependencies"
+  runner_type: "python-script"
+  description: "Get pack dependencies specified in pack.yaml"
+  enabled: true
+  pack: packs
+  entry_point: "pack_mgmt/get_pack_dependencies.py"
+  parameters:
+    packs_status:
+      type: object
+      description: Name of the pack in Exchange or a git repo URL and download status.
+      required: true
+      default: null
+    nested:
+      type: integer
+      description: Nested level of dependencies to prevent infinite or really long download loops.
+      required: true
+      default: 3

contrib/packs/actions/install.meta.yaml

@@ -1,6 +1,6 @@
 ---
   name: "install"
-  runner_type: "action-chain"
+  runner_type: "orquesta"
   description: "Installs or upgrades a pack into local content repository, either by
                 git URL or a short name matching an index entry.
                 Will download pack, load the actions, sensors and rules from the pack.
@@ -32,3 +32,8 @@
       description: "Use Python 3 binary when creating a virtualenv for this pack."
       required: false
       default: false
+    skip:
+      type: "boolean"
+      description: "Set to True to skip pack dependency installations."
+      required: false
+      default: false

contrib/packs/actions/pack_mgmt/delete.py

@@ -30,7 +30,7 @@ def __init__(self, config=None, action_service=None):
         self._base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path,
                                                    'virtualenvs/')
 
-    def run(self, packs, abs_repo_base):
+    def run(self, packs, abs_repo_base, delete_env=True):
         intersection = BLOCKED_PACKS & frozenset(packs)
         if len(intersection) > 0:
             names = ', '.join(list(intersection))
@@ -43,12 +43,13 @@ def run(self, packs, abs_repo_base):
                 self.logger.debug('Deleting pack directory "%s"' % (abs_fp))
                 shutil.rmtree(abs_fp)
 
-        # 2. Delete pack virtual environment
-        for pack_name in packs:
-            pack_name = quote_unix(pack_name)
-            virtualenv_path = os.path.join(self._base_virtualenvs_path, pack_name)
+        if delete_env:
+            # 2. Delete pack virtual environment
+            for pack_name in packs:
+                pack_name = quote_unix(pack_name)
+                virtualenv_path = os.path.join(self._base_virtualenvs_path, pack_name)
 
-            if os.path.isdir(virtualenv_path):
-                self.logger.debug('Deleting virtualenv "%s" for pack "%s"' %
-                                  (virtualenv_path, pack_name))
-                shutil.rmtree(virtualenv_path)
+                if os.path.isdir(virtualenv_path):
+                    self.logger.debug('Deleting virtualenv "%s" for pack "%s"' %
+                                      (virtualenv_path, pack_name))
+                    shutil.rmtree(virtualenv_path)

contrib/packs/actions/pack_mgmt/download.py

@@ -62,18 +62,29 @@ def __init__(self, config=None, action_service=None):
         if self.proxy_ca_bundle_path and not os.environ.get('proxy_ca_bundle_path', None):
             os.environ['no_proxy'] = self.no_proxy
 
-    def run(self, packs, abs_repo_base, verifyssl=True, force=False, python3=False):
+    def run(self, packs, abs_repo_base, verifyssl=True, force=False, python3=False,
+            dependency_list=None):
         result = {}
-
-        for pack in packs:
-            pack_result = download_pack(pack=pack, abs_repo_base=abs_repo_base,
-                                        verify_ssl=verifyssl, force=force,
-                                        proxy_config=self.proxy_config,
-                                        force_permissions=True,
-                                        use_python3=python3,
-                                        logger=self.logger)
-            pack_url, pack_ref, pack_result = pack_result
-            result[pack_ref] = pack_result
+        pack_url = None
+
+        if dependency_list:
+            for pack_dependency in dependency_list:
+                pack_result = download_pack(pack=pack_dependency, abs_repo_base=abs_repo_base,
+                                            verify_ssl=verifyssl, force=force,
+                                            proxy_config=self.proxy_config, force_permissions=True,
+                                            use_python3=python3, logger=self.logger)
+                pack_url, pack_ref, pack_result = pack_result
+                result[pack_ref] = pack_result
+        else:
+            for pack in packs:
+                pack_result = download_pack(pack=pack, abs_repo_base=abs_repo_base,
+                                            verify_ssl=verifyssl, force=force,
+                                            proxy_config=self.proxy_config,
+                                            force_permissions=True,
+                                            use_python3=python3,
+                                            logger=self.logger)
+                pack_url, pack_ref, pack_result = pack_result
+                result[pack_ref] = pack_result
 
         return self._validate_result(result=result, repo_url=pack_url)
 

contrib/packs/actions/pack_mgmt/get_pack_dependencies.py

@@ -0,0 +1,133 @@
+# Copyright 2019 Extreme Networks, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import six
+
+from st2common.constants.pack import PACK_VERSION_SEPARATOR
+from st2common.content.utils import get_pack_base_path
+from st2common.runners.base_action import Action
+from st2common.util.pack import get_pack_metadata
+
+STACKSTORM_EXCHANGE_PACK = 'StackStorm-Exchange/stackstorm-'
+LOCAL_FILE_PREFIX = 'file://'
+
+
+class GetPackDependencies(Action):
+    def __init__(self, config=None, action_service=None):
+        self.dependency_list = []
+        self.conflict_list = []
+
+    def run(self, packs_status, nested):
+        """
+        :param packs_status: Name of the pack in Exchange or a git repo URL and download status.
+        :type: packs_status: ``dict``
+
+        :param nested: Nested level of dependencies to prevent infinite or really
+        long download loops.
+        :type nested: ``integer``
+        """
+        result = {}
+
+        if not packs_status or nested == 0:
+            return result
+
+        for pack, status in six.iteritems(packs_status):
+            if 'success' not in status.lower():
+                continue
+
+            dependency_packs = get_dependency_list(pack)
+            if not dependency_packs:
+                continue
+
+            for dependency_pack in dependency_packs:
+                pack_and_version = dependency_pack.split(PACK_VERSION_SEPARATOR)
+                name_or_url = pack_and_version[0]
+                pack_version = pack_and_version[1] if len(pack_and_version) > 1 else None
+
+                if name_or_url.startswith(LOCAL_FILE_PREFIX):
+                    self.get_local_pack_dependencies(name_or_url, pack_version, dependency_pack)
+                elif (len(name_or_url.split('/')) == 1 and STACKSTORM_EXCHANGE_PACK not in
+                      name_or_url) or STACKSTORM_EXCHANGE_PACK in name_or_url:
+                    self.get_exchange_pack_dependencies(name_or_url, pack_version, dependency_pack)
+                else:
+                    self.get_none_exchange_pack_dependencies(name_or_url, pack_version,
+                                                             dependency_pack)
+
+        result['dependency_list'] = self.dependency_list
+        result['conflict_list'] = self.conflict_list
+        result['nested'] = nested - 1
+
+        return result
+
+    # For StackStorm Exchange packs. E.g.
+    # email
+    # https://github.com/StackStorm-Exchange/stackstorm-email
+    # https://github.com/StackStorm-Exchange/stackstorm-email.git
+    def get_exchange_pack_dependencies(self, name_or_url, pack_version, dependency_pack):
+        if len(name_or_url.split('/')) == 1:
+            pack_name = name_or_url
+        else:
+            name_or_git = name_or_url.split(STACKSTORM_EXCHANGE_PACK)[-1]
+            pack_name = name_or_git if '.git' not in name_or_git else name_or_git.split('.')[0]
+
+        existing_pack_version = get_pack_version(pack_name)
+        self.check_conflict(dependency_pack, pack_version, existing_pack_version)
+
+    # For None StackStorm Exchange packs. E.g.
+    # https://github.com/EncoreTechnologies/stackstorm-freeipa.git
+    # https://github.com/EncoreTechnologies/stackstorm-freeipa
+    # https://github.com/EncoreTechnologies/pack.git
+    def get_none_exchange_pack_dependencies(self, name_or_url, pack_version, dependency_pack):
+        name_or_git = name_or_url.split('/')[-1]
+        name = name_or_git if '.git' not in name_or_git else name_or_git.split('.')[0]
+        pack_name = name.split('-')[-1] if "stackstorm-" in name else name
+
+        existing_pack_version = get_pack_version(pack_name)
+        self.check_conflict(dependency_pack, pack_version, existing_pack_version)
+
+    # For local file. E.g
+    # file:///opt/stackstorm/st2/lib/python3.6/site-packages/st2tests/fixtures/packs/dummy_pack_3
+    def get_local_pack_dependencies(self, name_or_url, pack_version, dependency_pack):
+        pack_name = name_or_url.split("/")[-1]
+
+        existing_pack_version = get_pack_version(pack_name)
+        self.check_conflict(dependency_pack, pack_version, existing_pack_version)
+
+    def check_conflict(self, pack, version, existing_version):
+        if existing_version:
+            existing_version = 'v' + existing_version
+            if version and existing_version != version and pack not in self.conflict_list:
+                self.conflict_list.append(pack)
+        elif pack not in self.dependency_list:
+            self.dependency_list.append(pack)
+
+
+def get_pack_version(pack=None):
+    pack_path = get_pack_base_path(pack)
+    try:
+        pack_metadata = get_pack_metadata(pack_dir=pack_path)
+        return pack_metadata.get('version', None)
+    except Exception:
+        return None
+
+
+def get_dependency_list(pack=None):
+    pack_path = get_pack_base_path(pack)
+
+    try:
+        pack_metadata = get_pack_metadata(pack_dir=pack_path)
+        return pack_metadata.get('dependencies', None)
+    except Exception:
+        print ('Could not open pack.yaml at location %s' % pack_path)
+        return None

contrib/packs/actions/pack_mgmt/register.py

@@ -72,6 +72,7 @@ def run(self, register, packs=None):
             'types': types
         }
 
+        packs.reverse()
         if packs:
             method_kwargs['packs'] = packs
 

contrib/packs/actions/pack_mgmt/virtualenv_setup_prerun.py

@@ -18,13 +18,22 @@
 
 
 class PacksTransformationAction(Action):
-    def run(self, packs_status):
+    def run(self, packs_status, packs_list=None):
         """
         :param packs_status: Result from packs.download action.
         :type: packs_status: ``dict``
+
+        :param packs_list: Names of the pack in Exchange, a git repo URL or local file system.
+        :type: packs_list: ``list``
         """
+        if not packs_list:
+            packs_list = []
+
         packs = []
         for pack_name, status in six.iteritems(packs_status):
             if 'success' in status.lower():
                 packs.append(pack_name)
-        return packs
+
+        packs_list.extend(packs)
+
+        return packs_list