master: travis and trillian smoketests fixes and stabilisation (#3476)

Fix failing test, add more component tests, optimise and refactor test jobs.

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

Author: yadvr

.travis.yml

@@ -43,6 +43,7 @@ env:
              smoke/test_affinity_groups
              smoke/test_affinity_groups_projects
              smoke/test_async_job
+             smoke/test_create_list_domain_account_project
              smoke/test_deploy_vgpu_enabled_vm
              smoke/test_deploy_vm_iso
              smoke/test_deploy_vm_root_resize
@@ -52,13 +53,12 @@ env:
              smoke/test_disk_offerings
              smoke/test_dynamicroles
              smoke/test_global_settings
-             smoke/test_guest_vlan_range"
-
-    - TESTS="smoke/test_hostha_kvm
-             smoke/test_hostha_simulator
+             smoke/test_guest_vlan_range
              smoke/test_host_annotations
              smoke/test_host_maintenance
-             smoke/test_internal_lb
+             smoke/test_hostha_simulator"
+
+    - TESTS="smoke/test_internal_lb
              smoke/test_iso
              smoke/test_list_ids_parameter
              smoke/test_loadbalance
@@ -76,14 +76,12 @@ env:
              smoke/test_over_provisioning
              smoke/test_password_server
              smoke/test_portable_publicip
-             smoke/test_portforwardingrules"
-    
-    - TESTS="smoke/test_primary_storage"
-    
-    - TESTS="smoke/test_privategw_acl
+             smoke/test_portforwardingrules
+             smoke/test_privategw_acl
              smoke/test_projects
-             smoke/test_public_ip_range
-             smoke/test_pvlan
+             smoke/test_public_ip_range"
+
+    - TESTS="smoke/test_pvlan
              smoke/test_regions
              smoke/test_reset_vm_on_reboot
              smoke/test_resource_accounting
@@ -101,79 +99,80 @@ env:
              smoke/test_staticroles
              smoke/test_templates
              smoke/test_usage
-             smoke/test_usage_events
-             smoke/test_vm_life_cycle
+             smoke/test_usage_events"
+
+    - TESTS="smoke/test_vm_life_cycle
              smoke/test_vm_snapshots
              smoke/test_volumes
              smoke/test_vpc_redundant
              smoke/test_vpc_router_nics
-             smoke/test_vpc_vpn"
+             smoke/test_vpc_vpn
+             smoke/test_primary_storage"
 
-    - TESTS="smoke/misc/test_deploy_vm
-             smoke/misc/test_escalations_templates
-             smoke/misc/test_vm_ha
-             smoke/misc/test_vm_sync
-             component/find_hosts_for_migration
+    - TESTS="component/find_hosts_for_migration
+             component/test_acl_isolatednetwork
              component/test_acl_isolatednetwork_delete
-             component/test_acl_listsnapshot
-             component/test_acl_listvm
+             component/test_acl_listsnapshot"
+
+    - TESTS="component/test_acl_listvm
              component/test_acl_listvolume"
 
     - TESTS="component/test_acl_sharednetwork
              component/test_acl_sharednetwork_deployVM-impersonation"
 
     - TESTS="component/test_affinity_groups_projects
-             component/test_allocation_states"
+             component/test_allocation_states
+             component/test_assign_vm"
 
-    - TESTS="component/test_cpu_domain_limits
+    - TESTS="component/test_concurrent_snapshots_limit
+             component/test_cpu_domain_limits
              component/test_cpu_limits
              component/test_cpu_max_limits
-             component/test_cpu_project_limits"
+             component/test_cpu_project_limits
+             component/test_deploy_vm_userdata_multi_nic"
 
-    - TESTS="component/test_deploy_vm_userdata_multi_nic
-             component/test_egress_fw_rules
+    - TESTS="component/test_egress_fw_rules
              component/test_invalid_gw_nm
-             component/test_ip_reservation
-             component/test_lb_secondary_ip"
+             component/test_ip_reservation"
 
-    - TESTS="component/test_memory_limits
-             component/test_mm_domain_limits
-             component/test_mm_max_limits
-             component/test_mm_project_limits"
+    - TESTS="component/test_lb_secondary_ip
+             component/test_list_nics
+             component/test_list_pod
+             component/test_memory_limits"
 
-    - TESTS="component/test_network_offering
-             component/test_non_contiguous_vlan
-             component/test_persistent_networks"
+    - TESTS="component/test_mm_domain_limits
+             component/test_mm_max_limits
+             component/test_mm_project_limits
+             component/test_network_offering
+             component/test_non_contiguous_vlan"
 
-    - TESTS="component/test_project_limits
+    - TESTS="component/test_persistent_networks
              component/test_project_configs
-             component/test_project_usage
+             component/test_project_limits
              component/test_project_resources"
 
+    - TESTS="component/test_project_usage
+             component/test_resource_limits"
+
     - TESTS="component/test_regions_accounts
              component/test_routers
              component/test_snapshots
-             component/test_stopped_vm"
-
-    - TESTS="component/test_project_resources"
-
-    - TESTS="component/test_project_limits
-             component/test_resource_limits"
-
-    - TESTS="component/test_tags
+             component/test_stopped_vm
+             component/test_tags
              component/test_templates
-             component/test_update_vm
-             component/test_volumes"
+             component/test_updateResourceCount
+             component/test_update_vm"
 
-    - TESTS="component/test_vpc
-             component/test_vpc_network
+    - TESTS="component/test_volumes
+             component/test_vpc
+             component/test_vpc_distributed_routing_offering
+             component/test_vpc_network"
+
+    - TESTS="component/test_vpc_network_lbrules
              component/test_vpc_offerings
+             component/test_vpc_routers
              component/test_vpn_users"
 
-# FIXME: fix following tests and include them in Travis
-#    - TESTS="component/test_vpc" Please add when PR: https://github.com/apache/cloudstack/pull/955 CLOUDSTACK-8969 is fixed
-#    - TESTS="component/test_organization_states" Please add when CLOUDSTACK-7735 is fixed
-
 before_install: travis_wait 30 ./tools/travis/before_install.sh
 install: ./tools/travis/install.sh
 before_script: travis_wait 30 ./tools/travis/before_script.sh
@@ -198,25 +197,15 @@ after_script: ./tools/travis/after_script.sh
   after_success: true
   after_failure: true
 
-jobs:
-  include:
-    - stage: package
-      services: docker
-      env: IMAGE=khos2ow/cloudstack-rpm-builder:centos7 PARAMS="--distribution centos7 --pack oss"
-      <<: *package_job
-
-    - stage: package
-      services: docker
-      env: IMAGE=khos2ow/cloudstack-rpm-builder:centos6 PARAMS="--distribution centos63 --pack oss"
-      <<: *package_job
-
-    - stage: package
-      services: docker
-      env: IMAGE=khos2ow/cloudstack-deb-builder:ubuntu1804 PARAMS=""
-      <<: *package_job
-
-    - stage: package
-      services: docker
-      env: IMAGE=khos2ow/cloudstack-deb-builder:ubuntu1604 PARAMS=""
-      <<: *package_job
-
+#Disable due to intermittent failures
+#jobs:
+#  include:
+#    - stage: package
+#      services: docker
+#      env: IMAGE=khos2ow/cloudstack-rpm-builder:centos7 PARAMS="--distribution centos7 --pack oss"
+#      <<: *package_job
+#
+#    - stage: package
+#      services: docker
+#      env: IMAGE=khos2ow/cloudstack-deb-builder:ubuntu1804 PARAMS=""
+#      <<: *package_job

INSTALL.md

@@ -5,31 +5,32 @@ CloudStack. For more information please refer to the official [documentation](ht
 or the developer [wiki](https://cwiki.apache.org/confluence/display/CLOUDSTACK/Home).
 
 Apache CloudStack developers use various platforms for development, this guide
-was tested against a CentOS 6.5 x86_64 setup.
+was tested against a CentOS 7 x86_64 setup.
 
 * [Setting up development environment](https://cwiki.apache.org/confluence/display/CLOUDSTACK/Setting+up+CloudStack+Development+Environment) for Apache CloudStack.
 * [Building](https://cwiki.apache.org/confluence/display/CLOUDSTACK/How+to+build+CloudStack) Apache CloudStack.
+* [Appliance based development](https://github.com/rhtyd/monkeybox)
 
 ## Setting up Development Environment
 
 Install tools and dependencies used for development:
 
-    $ yum install git ant ant-devel java-1.6.0-openjdk java-1.6.0-openjdk-devel
+    $ yum install git java-1.8.0-openjdk java-1.8.0-openjdk-devel \
     mysql mysql-server mkisofs gcc python MySQL-python openssh-clients wget
 
     # yum -y update
-    # yum -y install java-1.7.0-openjdk
-    # yum -y install java-1.7.0-openjdk-devel
+    # yum -y install java-1.8.0-openjdk
+    # yum -y install java-1.8.0-openjdk-devel
     # yum -y install mysql-server
     # yum -y install git
     # yum -y install genisoimage
 
-Set up Maven (3.0.5):
+Set up Maven (3.6.0):
 
-    # wget http://www.us.apache.org/dist/maven/maven-3/3.0.5/binaries/apache-maven-3.0.5-bin.tar.gz
-    # tar -zxvf apache-maven-3.0.5-bin.tar.gz -C /usr/local
+    # wget http://www.us.apache.org/dist/maven/maven-3/3.6.0/binaries/apache-maven-3.6.0-bin.tar.gz
+    # tar -zxvf apache-maven-3.6.0-bin.tar.gz -C /usr/local
     # cd /usr/local
-    # ln -s apache-maven-3.0.5 maven
+    # ln -s apache-maven-3.6.0 maven
     # echo export M2_HOME=/usr/local/maven >> ~/.bashrc # or .zshrc or .profile
     # echo export PATH=/usr/local/maven/bin:${PATH} >> ~/.bashrc # or .zshrc or .profile
     # source ~/.bashrc
@@ -44,10 +45,10 @@ CloudStack is built using Java and Python.  To make selection of these tools ver
 
 Following installation, execute the following commands to configure jenv and pyenv for use with CloudStack development:
 
-    # pyenv install 2.7.11                                          ## Install Python 2.7.11
-    # pyenv virtualenv 2.7.11 cloudstack                            ## Create a cloudstack virtualenv using Python 2.7.11
+    # pyenv install 2.7.16                                          ## Install Python 2.7.16
+    # pyenv virtualenv 2.7.16 cloudstack                            ## Create a cloudstack virtualenv using Python 2.7.16
     # pip install -r <root CloudStack source tree>/requirements.txt ## Install cloudstack Python dependencies
-    # jenv add <path to JDK 1.7 installation>                       ## Add Java7 to jenv
+    # jenv add <path to JDK 1.8 installation>                       ## Add Java7 to jenv
 
 *N.B.* If you are running Linux, you may need to install additional packages to allow pyenv to build Python.
 
@@ -61,10 +62,10 @@ You may get the source code from the repository hosted on Apache:
 
 Or, you may fork the repository from the official Apache CloudStack mirror on [Github](https://github.com/apache/cloudstack)
 
-To checkout a specific branch, for example 4.4, do:
+To checkout a specific branch, for example 4.11, do:
 
     $ git fetch origin
-    $ git checkout -b 4.4 origin/4.4
+    $ git checkout -b 4.11 origin/4.11
 
 ## Building
 
@@ -142,7 +143,7 @@ To create rpms, install the following extra packages:
 
 Then:
 
-    $ cd packaging/centos63
+    $ cd packaging
     $ package.sh
 
 To create packages for noredist add the `-p noredist` option to the package script.

README.md

@@ -1,4 +1,4 @@
-# Apache CloudStack [![Build Status](https://travis-ci.org/apache/cloudstack.svg?branch=master)](https://travis-ci.org/apache/cloudstack) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apachecloudstack&metric=alert_status)](https://sonarcloud.io/dashboard?id=apachecloudstack) [![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=apachecloudstack&metric=ncloc)](https://sonarcloud.io/dashboard?id=apachecloudstack)
+# Apache CloudStack [![Build Status](https://travis-ci.org/apache/cloudstack.svg?branch=master)](https://travis-ci.org/apache/cloudstack) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apachecloudstack&metric=alert_status)](https://sonarcloud.io/dashboard?id=apachecloudstack) [![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=apachecloudstack&metric=ncloc)](https://sonarcloud.io/dashboard?id=apachecloudstack) ![GitHub language count](https://img.shields.io/github/languages/count/apache/cloudstack.svg) ![GitHub top language](https://img.shields.io/github/languages/top/apache/cloudstack.svg)
 
 ![Apache CloudStack](tools/logo/apache_cloudstack.png)
 

api/src/main/java/org/apache/cloudstack/api/command/user/project/CreateProjectCmd.java

@@ -55,7 +55,7 @@ public class CreateProjectCmd extends BaseAsyncCreateCmd {
     @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, validations = ApiArgValidator.NotNullOrEmpty, description = "name of the project")
     private String name;
 
-    @Parameter(name = ApiConstants.DISPLAY_TEXT, type = CommandType.STRING, required = true, validations = ApiArgValidator.NotNullOrEmpty, description = "display text of the project")
+    @Parameter(name = ApiConstants.DISPLAY_TEXT, type = CommandType.STRING, required = true, description = "display text of the project")
     private String displayText;
 
     // ///////////////////////////////////////////////////

plugins/hypervisors/simulator/src/main/java/com/cloud/resource/AgentRoutingResource.java

@@ -198,6 +198,7 @@ public StartupCommand[] initialize() {
         cmd.setName(agentHost.getName());
         cmd.setGuid(agentHost.getGuid());
         cmd.setVersion(agentHost.getVersion());
+        cmd.setHypervisorVersion(agentHost.getVersion());
         cmd.setAgentTag("agent-simulator");
         cmd.setDataCenter(String.valueOf(agentHost.getDataCenterId()));
         cmd.setPod(String.valueOf(agentHost.getPodId()));

server/src/main/java/com/cloud/api/query/QueryManagerImpl.java

@@ -181,6 +181,7 @@
 import com.cloud.projects.Project.ListProjectResourcesCriteria;
 import com.cloud.projects.ProjectInvitation;
 import com.cloud.projects.ProjectManager;
+import com.cloud.projects.ProjectVO;
 import com.cloud.projects.dao.ProjectAccountDao;
 import com.cloud.projects.dao.ProjectDao;
 import com.cloud.resource.ResourceManager;
@@ -640,10 +641,22 @@ private Pair<List<ResourceTagJoinVO>, Integer> listTagsInternal(ListTagsCmd cmd)
         String resourceType = cmd.getResourceType();
         String customerName = cmd.getCustomer();
         boolean listAll = cmd.listAll();
+        Long projectId = cmd.getProjectId();
+
+        if (projectId == null && ResourceObjectType.Project.name().equalsIgnoreCase(resourceType) && !Strings.isNullOrEmpty(resourceId)) {
+            try {
+                projectId = Long.parseLong(resourceId);
+            } catch (final NumberFormatException e) {
+                final ProjectVO project = _projectDao.findByUuidIncludingRemoved(resourceId);
+                if (project != null) {
+                    projectId = project.getId();
+                }
+            }
+        }
 
         Ternary<Long, Boolean, ListProjectResourcesCriteria> domainIdRecursiveListProject = new Ternary<Long, Boolean, ListProjectResourcesCriteria>(cmd.getDomainId(), cmd.isRecursive(), null);
 
-        _accountMgr.buildACLSearchParameters(caller, null, cmd.getAccountName(), cmd.getProjectId(), permittedAccounts, domainIdRecursiveListProject, listAll, false);
+        _accountMgr.buildACLSearchParameters(caller, null, cmd.getAccountName(), projectId, permittedAccounts, domainIdRecursiveListProject, listAll, false);
         Long domainId = domainIdRecursiveListProject.first();
         Boolean isRecursive = domainIdRecursiveListProject.second();
         ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third();

server/src/main/java/com/cloud/projects/ProjectManagerImpl.java

@@ -38,17 +38,12 @@
 import javax.mail.internet.InternetAddress;
 import javax.naming.ConfigurationException;
 
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
-
-import com.sun.mail.smtp.SMTPMessage;
-import com.sun.mail.smtp.SMTPSSLTransport;
-import com.sun.mail.smtp.SMTPTransport;
-
 import org.apache.cloudstack.acl.SecurityChecker.AccessType;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
 
 import com.cloud.api.ApiDBUtils;
 import com.cloud.api.query.dao.ProjectAccountJoinDao;
@@ -90,6 +85,9 @@
 import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn;
 import com.cloud.utils.db.TransactionStatus;
 import com.cloud.utils.exception.CloudRuntimeException;
+import com.sun.mail.smtp.SMTPMessage;
+import com.sun.mail.smtp.SMTPSSLTransport;
+import com.sun.mail.smtp.SMTPTransport;
 
 @Component
 public class ProjectManagerImpl extends ManagerBase implements ProjectManager {
@@ -207,15 +205,15 @@ public Project createProject(final String name, final String displayText, String
             @Override
             public Project doInTransaction(TransactionStatus status) {
 
-        //Create an account associated with the project
-        StringBuilder acctNm = new StringBuilder("PrjAcct-");
+                //Create an account associated with the project
+                StringBuilder acctNm = new StringBuilder("PrjAcct-");
                 acctNm.append(name).append("-").append(ownerFinal.getDomainId());
 
-        Account projectAccount = _accountMgr.createAccount(acctNm.toString(), Account.ACCOUNT_TYPE_PROJECT, null, domainId, null, null, UUID.randomUUID().toString());
+                Account projectAccount = _accountMgr.createAccount(acctNm.toString(), Account.ACCOUNT_TYPE_PROJECT, null, domainId, null, null, UUID.randomUUID().toString());
 
                 Project project = _projectDao.persist(new ProjectVO(name, displayText, ownerFinal.getDomainId(), projectAccount.getId()));
 
-        //assign owner to the project
+                //assign owner to the project
                 assignAccountToProject(project, ownerFinal.getId(), ProjectAccount.Role.Admin);
 
         if (project != null) {