Pack dependency management

CHANGELOG.rst

@@ -10,6 +10,8 @@ Added
 * Add support for blacklisting / whitelisting hosts to the HTTP runner by adding new
   ``url_hosts_blacklist`` and ``url_hosts_whitelist`` runner attribute. (new feature)
   #4757
+* Add support for Pack dependency management to install dependency packs and find out confilct packs.
+  #4769
 
 Changed
 ~~~~~~~

contrib/packs/actions/delete.yaml

@@ -14,3 +14,8 @@
       type: "string"
       default: "/opt/stackstorm/packs/"
       immutable: true
+    delete_env:
+      type: "boolean"
+      description: Flag for if need to delete virtual environment.
+      default: true
+      required: false

contrib/packs/actions/download.yaml

@@ -27,3 +27,9 @@
       description: "True to use Python 3 binary for this pack."
       required: false
       default: false
+    dependency_list:
+      type: "array"
+      description: "Dependency list that needs to be downloaded."
+      items:
+        type: "string"
+      required: false

contrib/packs/actions/get_pack_dependencies.yaml

@@ -0,0 +1,20 @@
+
+
+---
+  name: "get_pack_dependencies"
+  runner_type: "python-script"
+  description: "Get pack dependencies specified in pack.yaml"
+  enabled: true
+  pack: packs
+  entry_point: "pack_mgmt/get_pack_dependencies.py"
+  parameters:
+    packs_status:
+      type: object
+      description: Name of the pack in Exchange or a git repo URL and download status.
+      required: true
+      default: null
+    nested:
+      type: integer
+      description: Nested level of dependencies to prevent infinite or really long download loops.
+      required: true
+      default: 3

contrib/packs/actions/install.meta.yaml

@@ -1,6 +1,6 @@
 ---
   name: "install"
-  runner_type: "action-chain"
+  runner_type: "orquesta"
   description: "Installs or upgrades a pack into local content repository, either by
                 git URL or a short name matching an index entry.
                 Will download pack, load the actions, sensors and rules from the pack.
@@ -32,3 +32,8 @@
       description: "Use Python 3 binary when creating a virtualenv for this pack."
       required: false
       default: false
+    skip:
+      type: "boolean"
+      description: "Set to True to skip pack dependency installations."
+      required: false
+      default: false

contrib/packs/actions/pack_mgmt/delete.py

@@ -30,7 +30,7 @@ def __init__(self, config=None, action_service=None):
         self._base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path,
                                                    'virtualenvs/')
 
-    def run(self, packs, abs_repo_base):
+    def run(self, packs, abs_repo_base, delete_env=True):
         intersection = BLOCKED_PACKS & frozenset(packs)
         if len(intersection) > 0:
             names = ', '.join(list(intersection))
@@ -43,12 +43,13 @@ def run(self, packs, abs_repo_base):
                 self.logger.debug('Deleting pack directory "%s"' % (abs_fp))
                 shutil.rmtree(abs_fp)
 
-        # 2. Delete pack virtual environment
-        for pack_name in packs:
-            pack_name = quote_unix(pack_name)
-            virtualenv_path = os.path.join(self._base_virtualenvs_path, pack_name)
+        if delete_env:
+            # 2. Delete pack virtual environment
+            for pack_name in packs:
+                pack_name = quote_unix(pack_name)
+                virtualenv_path = os.path.join(self._base_virtualenvs_path, pack_name)
 
-            if os.path.isdir(virtualenv_path):
-                self.logger.debug('Deleting virtualenv "%s" for pack "%s"' %
-                                  (virtualenv_path, pack_name))
-                shutil.rmtree(virtualenv_path)
+                if os.path.isdir(virtualenv_path):
+                    self.logger.debug('Deleting virtualenv "%s" for pack "%s"' %
+                                      (virtualenv_path, pack_name))
+                    shutil.rmtree(virtualenv_path)

contrib/packs/actions/pack_mgmt/download.py

@@ -62,18 +62,29 @@ def __init__(self, config=None, action_service=None):
         if self.proxy_ca_bundle_path and not os.environ.get('proxy_ca_bundle_path', None):
             os.environ['no_proxy'] = self.no_proxy
 
-    def run(self, packs, abs_repo_base, verifyssl=True, force=False, python3=False):
+    def run(self, packs, abs_repo_base, verifyssl=True, force=False, python3=False,
+            dependency_list=None):
         result = {}
-
-        for pack in packs:
-            pack_result = download_pack(pack=pack, abs_repo_base=abs_repo_base,
-                                        verify_ssl=verifyssl, force=force,
-                                        proxy_config=self.proxy_config,
-                                        force_permissions=True,
-                                        use_python3=python3,
-                                        logger=self.logger)
-            pack_url, pack_ref, pack_result = pack_result
-            result[pack_ref] = pack_result
+        pack_url = None
+
+        if dependency_list:
+            for pack_dependency in dependency_list:
+                pack_result = download_pack(pack=pack_dependency, abs_repo_base=abs_repo_base,
+                                            verify_ssl=verifyssl, force=force,
+                                            proxy_config=self.proxy_config, force_permissions=True,
+                                            use_python3=python3, logger=self.logger)
+                pack_url, pack_ref, pack_result = pack_result
+                result[pack_ref] = pack_result
+        else:
+            for pack in packs:
+                pack_result = download_pack(pack=pack, abs_repo_base=abs_repo_base,
+                                            verify_ssl=verifyssl, force=force,
+                                            proxy_config=self.proxy_config,
+                                            force_permissions=True,
+                                            use_python3=python3,
+                                            logger=self.logger)
+                pack_url, pack_ref, pack_result = pack_result
+                result[pack_ref] = pack_result
 
         return self._validate_result(result=result, repo_url=pack_url)
 

contrib/packs/actions/pack_mgmt/get_pack_dependencies.py

@@ -0,0 +1,133 @@
+# Copyright 2019 Extreme Networks, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import six
+
+from st2common.constants.pack import PACK_VERSION_SEPARATOR
+from st2common.content.utils import get_pack_base_path
+from st2common.runners.base_action import Action
+from st2common.util.pack import get_pack_metadata
+
+STACKSTORM_EXCHANGE_PACK = 'StackStorm-Exchange/stackstorm-'
+LOCAL_FILE_PREFIX = 'file://'
+
+
+class GetPackDependencies(Action):
+    def __init__(self, config=None, action_service=None):
+        self.dependency_list = []
+        self.conflict_list = []
+
+    def run(self, packs_status, nested):
+        """
+        :param packs_status: Name of the pack in Exchange or a git repo URL and download status.
+        :type: packs_status: ``dict``
+
+        :param nested: Nested level of dependencies to prevent infinite or really
+        long download loops.
+        :type nested: ``integer``
+        """
+        result = {}
+
+        if not packs_status or nested == 0:
+            return result
+
+        for pack, status in six.iteritems(packs_status):
+            if 'success' not in status.lower():
+                continue
+
+            dependency_packs = get_dependency_list(pack)
+            if not dependency_packs:
+                continue
+
+            for dependency_pack in dependency_packs:
+                pack_and_version = dependency_pack.split(PACK_VERSION_SEPARATOR)
+                name_or_url = pack_and_version[0]
+                pack_version = pack_and_version[1] if len(pack_and_version) > 1 else None
+
+                if name_or_url.startswith(LOCAL_FILE_PREFIX):
+                    self.get_local_pack_dependencies(name_or_url, pack_version, dependency_pack)
+                elif (len(name_or_url.split('/')) == 1 and STACKSTORM_EXCHANGE_PACK not in
+                      name_or_url) or STACKSTORM_EXCHANGE_PACK in name_or_url:
+                    self.get_exchange_pack_dependencies(name_or_url, pack_version, dependency_pack)
+                else:
+                    self.get_none_exchange_pack_dependencies(name_or_url, pack_version,
+                                                             dependency_pack)
+
+        result['dependency_list'] = self.dependency_list
+        result['conflict_list'] = self.conflict_list
+        result['nested'] = nested - 1
+
+        return result
+
+    # For StackStorm Exchange packs. E.g.
+    # email
+    # https://github.com/StackStorm-Exchange/stackstorm-email
+    # https://github.com/StackStorm-Exchange/stackstorm-email.git
+    def get_exchange_pack_dependencies(self, name_or_url, pack_version, dependency_pack):
+        if len(name_or_url.split('/')) == 1:
+            pack_name = name_or_url
+        else:
+            name_or_git = name_or_url.split(STACKSTORM_EXCHANGE_PACK)[-1]
+            pack_name = name_or_git if '.git' not in name_or_git else name_or_git.split('.')[0]
+
+        existing_pack_version = get_pack_version(pack_name)
+        self.check_conflict(dependency_pack, pack_version, existing_pack_version)
+
+    # For None StackStorm Exchange packs. E.g.
+    # https://github.com/EncoreTechnologies/stackstorm-freeipa.git
+    # https://github.com/EncoreTechnologies/stackstorm-freeipa
+    # https://github.com/EncoreTechnologies/pack.git
+    def get_none_exchange_pack_dependencies(self, name_or_url, pack_version, dependency_pack):
+        name_or_git = name_or_url.split('/')[-1]
+        name = name_or_git if '.git' not in name_or_git else name_or_git.split('.')[0]
+        pack_name = name.split('-')[-1] if "stackstorm-" in name else name
+
+        existing_pack_version = get_pack_version(pack_name)
+        self.check_conflict(dependency_pack, pack_version, existing_pack_version)
+
+    # For local file. E.g
+    # file:///opt/stackstorm/st2/lib/python3.6/site-packages/st2tests/fixtures/packs/dummy_pack_3
+    def get_local_pack_dependencies(self, name_or_url, pack_version, dependency_pack):
+        pack_name = name_or_url.split("/")[-1]
+
+        existing_pack_version = get_pack_version(pack_name)
+        self.check_conflict(dependency_pack, pack_version, existing_pack_version)
+
+    def check_conflict(self, pack, version, existing_version):
+        if existing_version:
+            existing_version = 'v' + existing_version
+            if version and existing_version != version and pack not in self.conflict_list:
+                self.conflict_list.append(pack)
+        elif pack not in self.dependency_list:
+            self.dependency_list.append(pack)
+
+
+def get_pack_version(pack=None):
+    pack_path = get_pack_base_path(pack)
+    try:
+        pack_metadata = get_pack_metadata(pack_dir=pack_path)
+        return pack_metadata.get('version', None)
+    except Exception:
+        return None
+
+
+def get_dependency_list(pack=None):
+    pack_path = get_pack_base_path(pack)
+
+    try:
+        pack_metadata = get_pack_metadata(pack_dir=pack_path)
+        return pack_metadata.get('dependencies', None)
+    except Exception:
+        print ('Could not open pack.yaml at location %s' % pack_path)
+        return None

contrib/packs/actions/pack_mgmt/register.py

@@ -72,6 +72,7 @@ def run(self, register, packs=None):
             'types': types
         }
 
+        packs.reverse()
         if packs:
             method_kwargs['packs'] = packs
 

contrib/packs/actions/pack_mgmt/virtualenv_setup_prerun.py

@@ -18,13 +18,22 @@
 
 
 class PacksTransformationAction(Action):
-    def run(self, packs_status):
+    def run(self, packs_status, packs_list=None):
         """
         :param packs_status: Result from packs.download action.
         :type: packs_status: ``dict``
+
+        :param packs_list: Names of the pack in Exchange, a git repo URL or local file system.
+        :type: packs_list: ``list``
         """
+        if not packs_list:
+            packs_list = []
+
         packs = []
         for pack_name, status in six.iteritems(packs_status):
             if 'success' in status.lower():
                 packs.append(pack_name)
-        return packs
+
+        packs_list.extend(packs)
+
+        return packs_list

contrib/packs/actions/virtualenv_prerun.yaml

@@ -7,3 +7,10 @@
   parameters:
     packs_status:
       type: "object"
+    packs_list:
+      type: array
+      items:
+        type: string
+      required: false
+      default: null
+      description: Names of the pack in Exchange, a git repo URL or local file system.