Merge branch '4.19'

api/src/main/java/com/cloud/user/AccountService.java

@@ -116,6 +116,8 @@ User createUser(String userName, String password, String firstName, String lastN
 
     void checkAccess(Account account, AccessType accessType, boolean sameOwner, String apiName, ControlledEntity... entities) throws PermissionDeniedException;
 
+    void validateAccountHasAccessToResource(Account account, AccessType accessType, Object resource);
+
     Long finalyzeAccountId(String accountName, Long domainId, Long projectId, boolean enabledOnly);
 
     /**

engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java

@@ -1958,25 +1958,26 @@ protected MigrationOptions createFullCloneMigrationOptions(VolumeInfo srcVolumeI
      * - Full clones (no backing file): Take snapshot of the VM prior disk creation
      * Return this information
      */
-    protected void setVolumeMigrationOptions(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo,
-                                             VirtualMachineTO vmTO, Host srcHost, StoragePoolVO destStoragePool) {
-        if (!destStoragePool.isManaged()) {
-            String srcVolumeBackingFile = getVolumeBackingFile(srcVolumeInfo);
-
-            String srcPoolUuid = srcVolumeInfo.getDataStore().getUuid();
-            StoragePoolVO srcPool = _storagePoolDao.findById(srcVolumeInfo.getPoolId());
-            Storage.StoragePoolType srcPoolType = srcPool.getPoolType();
-
-            MigrationOptions migrationOptions;
-            if (StringUtils.isNotBlank(srcVolumeBackingFile)) {
-                migrationOptions = createLinkedCloneMigrationOptions(srcVolumeInfo, destVolumeInfo,
-                        srcVolumeBackingFile, srcPoolUuid, srcPoolType);
-            } else {
-                migrationOptions = createFullCloneMigrationOptions(srcVolumeInfo, vmTO, srcHost, srcPoolUuid, srcPoolType);
-            }
-            migrationOptions.setTimeout(StorageManager.KvmStorageOnlineMigrationWait.value());
-            destVolumeInfo.setMigrationOptions(migrationOptions);
+    protected void setVolumeMigrationOptions(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo, VirtualMachineTO vmTO, Host srcHost, StoragePoolVO destStoragePool,
+                                             MigrationOptions.Type migrationType) {
+        if (destStoragePool.isManaged()) {
+            return;
         }
+
+        String srcVolumeBackingFile = getVolumeBackingFile(srcVolumeInfo);
+
+        String srcPoolUuid = srcVolumeInfo.getDataStore().getUuid();
+        StoragePoolVO srcPool = _storagePoolDao.findById(srcVolumeInfo.getPoolId());
+        Storage.StoragePoolType srcPoolType = srcPool.getPoolType();
+
+        MigrationOptions migrationOptions;
+        if (MigrationOptions.Type.LinkedClone.equals(migrationType)) {
+            migrationOptions = createLinkedCloneMigrationOptions(srcVolumeInfo, destVolumeInfo, srcVolumeBackingFile, srcPoolUuid, srcPoolType);
+        } else {
+            migrationOptions = createFullCloneMigrationOptions(srcVolumeInfo, vmTO, srcHost, srcPoolUuid, srcPoolType);
+        }
+        migrationOptions.setTimeout(StorageManager.KvmStorageOnlineMigrationWait.value());
+        destVolumeInfo.setMigrationOptions(migrationOptions);
     }
 
     /**
@@ -2007,6 +2008,7 @@ public void copyAsync(Map<VolumeInfo, DataStore> volumeDataStoreMap, VirtualMach
             Map<VolumeInfo, VolumeInfo> srcVolumeInfoToDestVolumeInfo = new HashMap<>();
 
             boolean managedStorageDestination = false;
+            boolean migrateNonSharedInc = false;
             for (Map.Entry<VolumeInfo, DataStore> entry : volumeDataStoreMap.entrySet()) {
                 VolumeInfo srcVolumeInfo = entry.getKey();
                 DataStore destDataStore = entry.getValue();
@@ -2034,6 +2036,9 @@ public void copyAsync(Map<VolumeInfo, DataStore> volumeDataStoreMap, VirtualMach
                     logger.debug(String.format("Skipping copy template from source storage pool [%s] to target storage pool [%s] before migration due to volume [%s] does not have a template.", sourceStoragePool.getId(), destStoragePool.getId(), srcVolumeInfo.getId()));
                 }
 
+                MigrationOptions.Type migrationType = decideMigrationTypeAndCopyTemplateIfNeeded(destHost, vmInstance, srcVolumeInfo, sourceStoragePool, destStoragePool, destDataStore);
+                migrateNonSharedInc = migrateNonSharedInc || MigrationOptions.Type.LinkedClone.equals(migrationType);
+
                 VolumeVO destVolume = duplicateVolumeOnAnotherStorage(srcVolume, destStoragePool);
                 VolumeInfo destVolumeInfo = _volumeDataFactory.getVolume(destVolume.getId(), destDataStore);
 
@@ -2044,7 +2049,7 @@ public void copyAsync(Map<VolumeInfo, DataStore> volumeDataStoreMap, VirtualMach
                 // move the volume from Ready to Migrating
                 destVolumeInfo.processEvent(Event.MigrationRequested);
 
-                setVolumeMigrationOptions(srcVolumeInfo, destVolumeInfo, vmTO, srcHost, destStoragePool);
+                setVolumeMigrationOptions(srcVolumeInfo, destVolumeInfo, vmTO, srcHost, destStoragePool, migrationType);
 
                 // create a volume on the destination storage
                 destDataStore.getDriver().createAsync(destDataStore, destVolumeInfo, null);
@@ -2059,7 +2064,7 @@ public void copyAsync(Map<VolumeInfo, DataStore> volumeDataStoreMap, VirtualMach
 
                 _volumeDao.update(destVolume.getId(), destVolume);
 
-                postVolumeCreationActions(srcVolumeInfo, destVolumeInfo, vmTO, srcHost);
+                postVolumeCreationActions(srcVolumeInfo, destVolumeInfo);
 
                 destVolumeInfo = _volumeDataFactory.getVolume(destVolume.getId(), destDataStore);
 
@@ -2110,8 +2115,6 @@ public void copyAsync(Map<VolumeInfo, DataStore> volumeDataStoreMap, VirtualMach
             VMInstanceVO vm = _vmDao.findById(vmTO.getId());
             boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
 
-            boolean migrateNonSharedInc = isSourceAndDestinationPoolTypeOfNfs(volumeDataStoreMap);
-
             MigrateCommand migrateCommand = new MigrateCommand(vmTO.getName(), destHost.getPrivateIpAddress(), isWindows, vmTO, true);
             migrateCommand.setWait(StorageManager.KvmStorageOnlineMigrationWait.value());
             migrateCommand.setMigrateStorage(migrateStorage);
@@ -2161,6 +2164,22 @@ public void copyAsync(Map<VolumeInfo, DataStore> volumeDataStoreMap, VirtualMach
         }
     }
 
+    private MigrationOptions.Type decideMigrationTypeAndCopyTemplateIfNeeded(Host destHost, VMInstanceVO vmInstance, VolumeInfo srcVolumeInfo, StoragePoolVO sourceStoragePool, StoragePoolVO destStoragePool, DataStore destDataStore) {
+        VMTemplateVO vmTemplate = _vmTemplateDao.findById(vmInstance.getTemplateId());
+        String srcVolumeBackingFile = getVolumeBackingFile(srcVolumeInfo);
+        if (StringUtils.isNotBlank(srcVolumeBackingFile) && supportStoragePoolType(destStoragePool.getPoolType(), StoragePoolType.Filesystem) &&
+                srcVolumeInfo.getTemplateId() != null &&
+                Objects.nonNull(vmTemplate) &&
+                !Arrays.asList(KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME, VM_IMPORT_DEFAULT_TEMPLATE_NAME).contains(vmTemplate.getName())) {
+            LOGGER.debug(String.format("Copying template [%s] of volume [%s] from source storage pool [%s] to target storage pool [%s].", srcVolumeInfo.getTemplateId(), srcVolumeInfo.getId(), sourceStoragePool.getId(), destStoragePool.getId()));
+            copyTemplateToTargetFilesystemStorageIfNeeded(srcVolumeInfo, sourceStoragePool, destDataStore, destStoragePool, destHost);
+            return MigrationOptions.Type.LinkedClone;
+        }
+        LOGGER.debug(String.format("Skipping copy template from source storage pool [%s] to target storage pool [%s] before migration due to volume [%s] does not have a " +
+                "template or we are doing full clone migration.", sourceStoragePool.getId(), destStoragePool.getId(), srcVolumeInfo.getId()));
+        return MigrationOptions.Type.FullClone;
+    }
+
     protected String formatMigrationElementsAsJsonToDisplayOnLog(String objectName, Object object, Object from, Object to){
         return String.format("{%s: \"%s\", from: \"%s\", to:\"%s\"}", objectName, object, from, to);
     }
@@ -2422,7 +2441,7 @@ protected void updateCopiedTemplateReference(VolumeInfo srcVolumeInfo, VolumeInf
     /**
      * Handle post destination volume creation actions depending on the migrating volume type: full clone or linked clone
      */
-    protected void postVolumeCreationActions(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo, VirtualMachineTO vmTO, Host srcHost) {
+    protected void postVolumeCreationActions(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo) {
         MigrationOptions migrationOptions = destVolumeInfo.getMigrationOptions();
         if (migrationOptions != null) {
             if (migrationOptions.getType() == MigrationOptions.Type.LinkedClone && migrationOptions.isCopySrcTemplate()) {

plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java

@@ -21,6 +21,9 @@
 
 import javax.inject.Inject;
 
+import com.cloud.user.Account;
+
+import org.apache.cloudstack.api.ACL;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseCmd;
@@ -40,6 +43,7 @@ public class QuotaBalanceCmd extends BaseCmd {
     @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = true, description = "Account Id for which statement needs to be generated")
     private String accountName;
 
+    @ACL
     @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "If domain Id is given and the caller is domain admin then the statement is generated for domain.")
     private Long domainId;
 
@@ -51,6 +55,7 @@ public class QuotaBalanceCmd extends BaseCmd {
             ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS)
     private Date startDate;
 
+    @ACL
     @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "List usage records for the specified account")
     private Long accountId;
 
@@ -104,7 +109,14 @@ public void setStartDate(Date startDate) {
 
     @Override
     public long getEntityOwnerId() {
-       return _accountService.getActiveAccountByName(accountName, domainId).getAccountId();
+        if (accountId != null) {
+            return accountId;
+        }
+        Account account = _accountService.getActiveAccountByName(accountName, domainId);
+        if (account != null) {
+            return account.getAccountId();
+        }
+        return Account.ACCOUNT_ID_SYSTEM;
     }
 
     @Override

plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaCreditsCmd.java

@@ -18,6 +18,7 @@
 
 import com.cloud.user.Account;
 
+import org.apache.cloudstack.api.ACL;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.ApiErrorCode;
@@ -46,6 +47,7 @@ public class QuotaCreditsCmd extends BaseCmd {
     @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = true, description = "Account Id for which quota credits need to be added")
     private String accountName;
 
+    @ACL
     @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "Domain for which quota credits need to be added")
     private Long domainId;
 
@@ -130,6 +132,10 @@ public void execute() {
 
     @Override
     public long getEntityOwnerId() {
+        Account account = _accountService.getActiveAccountByName(accountName, domainId);
+        if (account != null) {
+            return account.getAccountId();
+        }
         return Account.ACCOUNT_ID_SYSTEM;
     }
 

plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java

@@ -21,6 +21,7 @@
 
 import javax.inject.Inject;
 
+import org.apache.cloudstack.api.ACL;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseCmd;
@@ -42,6 +43,7 @@ public class QuotaStatementCmd extends BaseCmd {
     @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = true, description = "Optional, Account Id for which statement needs to be generated")
     private String accountName;
 
+    @ACL
     @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "Optional, If domain Id is given and the caller is domain admin then the statement is generated for domain.")
     private Long domainId;
 
@@ -56,6 +58,7 @@ public class QuotaStatementCmd extends BaseCmd {
     @Parameter(name = ApiConstants.TYPE, type = CommandType.INTEGER, description = "List quota usage records for the specified usage type")
     private Integer usageType;
 
+    @ACL
     @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "List usage records for the specified account")
     private Long accountId;
 
@@ -112,6 +115,9 @@ public void setStartDate(Date startDate) {
 
     @Override
     public long getEntityOwnerId() {
+        if (accountId != null) {
+            return accountId;
+        }
         Account activeAccountByName = _accountService.getActiveAccountByName(accountName, domainId);
         if (activeAccountByName != null) {
             return activeAccountByName.getAccountId();

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/MigrateKVMAsync.java

@@ -123,8 +123,10 @@ public Domain call() throws LibvirtException {
             if (migrateNonSharedInc) {
                 flags |= VIR_MIGRATE_PERSIST_DEST;
                 flags |= VIR_MIGRATE_NON_SHARED_INC;
+                logger.debug("Setting VIR_MIGRATE_NON_SHARED_INC for linked clone migration.");
             } else {
                 flags |= VIR_MIGRATE_NON_SHARED_DISK;
+                logger.debug("Setting VIR_MIGRATE_NON_SHARED_DISK for full clone migration.");
             }
         }
 

plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java

@@ -454,6 +454,11 @@ public void checkAccess(Account account, AccessType accessType, boolean sameOwne
         // TODO Auto-generated method stub
     }
 
+    @Override
+    public void validateAccountHasAccessToResource(Account account, AccessType accessType, Object resource) {
+        // TODO Auto-generated method stub
+    }
+
     @Override
     public Long finalyzeAccountId(String accountName, Long domainId, Long projectId, boolean enabledOnly) {
         // TODO Auto-generated method stub

plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/api/command/ListAndSwitchSAMLAccountCmd.java

@@ -47,6 +47,7 @@
 import org.apache.cloudstack.saml.SAML2AuthManager;
 import org.apache.cloudstack.saml.SAMLUtils;
 
+import com.cloud.api.ApiServer;
 import com.cloud.api.response.ApiResponseSerializer;
 import com.cloud.domain.Domain;
 import com.cloud.domain.dao.DomainDao;
@@ -59,6 +60,8 @@
 import com.cloud.user.dao.UserDao;
 import com.cloud.utils.HttpUtils;
 
+import org.apache.commons.lang3.EnumUtils;
+
 @APICommand(name = "listAndSwitchSamlAccount", description = "Lists and switches to other SAML accounts owned by the SAML user", responseObject = SuccessResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
 public class ListAndSwitchSAMLAccountCmd extends BaseCmd implements APIAuthenticator {
 
@@ -102,7 +105,9 @@ public String authenticate(final String command, final Map<String, Object[]> par
                     params, responseType));
         }
 
-        if (!HttpUtils.validateSessionKey(session, params, req.getCookies(), ApiConstants.SESSIONKEY)) {
+        HttpUtils.ApiSessionKeyCheckOption sessionKeyCheckOption = EnumUtils.getEnumIgnoreCase(HttpUtils.ApiSessionKeyCheckOption.class,
+                ApiServer.ApiSessionKeyCheckLocations.value(), HttpUtils.ApiSessionKeyCheckOption.CookieAndParameter);
+        if (!HttpUtils.validateSessionKey(session, params, req.getCookies(), ApiConstants.SESSIONKEY, sessionKeyCheckOption)) {
             throw new ServerApiException(ApiErrorCode.UNAUTHORIZED, _apiServer.getSerializedApiError(ApiErrorCode.UNAUTHORIZED.getHttpCode(),
                     "Unauthorized session, please re-login",
                     params, responseType));

plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java

@@ -73,6 +73,9 @@ public interface SAML2AuthManager extends PluggableAPIAuthenticator, PluggableSe
     ConfigKey<Boolean> SAMLCheckSignature = new ConfigKey<Boolean>("Advanced", Boolean.class, "saml2.check.signature", "true",
             "When enabled (default and recommended), SAML2 signature checks are enforced and lack of signature in the SAML SSO response will cause login exception. Disabling this is not advisable but provided for backward compatibility for users who are able to accept the risks.", false);
 
+    ConfigKey<String> SAMLUserSessionKeyPathAttribute = new ConfigKey<String>("Advanced", String.class, "saml2.user.sessionkey.path", "",
+            "The Path attribute of sessionkey cookie when SAML users have logged in. If not set, it will be set to the path of SAML redirection URL (saml2.redirect.url).", true);
+
     SAMLProviderMetadata getSPMetadata();
     SAMLProviderMetadata getIdPMetadata(String entityId);
     Collection<SAMLProviderMetadata> getAllIdPMetadata();

plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java

@@ -540,6 +540,7 @@ public ConfigKey<?>[] getConfigKeys() {
                 SAMLServiceProviderSingleSignOnURL, SAMLServiceProviderSingleLogOutURL,
                 SAMLCloudStackRedirectionUrl, SAMLUserAttributeName,
                 SAMLIdentityProviderMetadataURL, SAMLDefaultIdentityProviderId,
-                SAMLSignatureAlgorithm, SAMLAppendDomainSuffix, SAMLTimeout, SAMLCheckSignature};
+                SAMLSignatureAlgorithm, SAMLAppendDomainSuffix, SAMLTimeout, SAMLCheckSignature,
+                SAMLUserSessionKeyPathAttribute};
     }
 }

plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAMLUtils.java

@@ -25,6 +25,8 @@
 import java.io.StringWriter;
 import java.io.UnsupportedEncodingException;
 import java.math.BigInteger;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URLEncoder;
 import java.nio.charset.Charset;
 import java.security.InvalidKeyException;
@@ -102,7 +104,9 @@
 import org.w3c.dom.Element;
 import org.xml.sax.SAXException;
 
+import com.cloud.api.ApiServlet;
 import com.cloud.utils.HttpUtils;
+import com.cloud.utils.exception.CloudRuntimeException;
 
 public class SAMLUtils {
     protected static Logger LOGGER = LogManager.getLogger(SAMLUtils.class);
@@ -297,7 +301,26 @@ public static void setupSamlUserCookies(final LoginCmdResponse loginResponse, fi
             resp.addCookie(new Cookie("timezone", URLEncoder.encode(timezone, HttpUtils.UTF_8)));
         }
         resp.addCookie(new Cookie("userfullname", URLEncoder.encode(loginResponse.getFirstName() + " " + loginResponse.getLastName(), HttpUtils.UTF_8).replace("+", "%20")));
-        resp.addHeader("SET-COOKIE", String.format("%s=%s;HttpOnly;Path=/client/api", ApiConstants.SESSIONKEY, loginResponse.getSessionKey()));
+
+        String redirectUrl = SAML2AuthManager.SAMLCloudStackRedirectionUrl.value();
+        String path = SAML2AuthManager.SAMLUserSessionKeyPathAttribute.value();
+        String domain = null;
+        try {
+            URI redirectUri = new URI(redirectUrl);
+            domain = redirectUri.getHost();
+            if (StringUtils.isBlank(path)) {
+                path = redirectUri.getPath();
+            }
+            if (StringUtils.isBlank(path)) {
+                path = "/";
+            }
+        } catch (URISyntaxException ex) {
+            throw new CloudRuntimeException("Invalid URI: " + redirectUrl);
+        }
+        String sameSite = ApiServlet.getApiSessionKeySameSite();
+        String sessionKeyCookie = String.format("%s=%s;Domain=%s;Path=%s;%s", ApiConstants.SESSIONKEY, loginResponse.getSessionKey(), domain, path, sameSite);
+        s_logger.debug("Adding sessionkey cookie to response: " + sessionKeyCookie);
+        resp.addHeader("SET-COOKIE", sessionKeyCookie);
     }
 
     /**