Add support for providing userdata to system VMs

api/src/main/java/org/apache/cloudstack/userdata/UserDataManager.java

@@ -22,11 +22,23 @@
 
 import com.cloud.utils.component.Manager;
 
+import java.io.IOException;
+
 public interface UserDataManager extends Manager, Configurable {
     String VM_USERDATA_MAX_LENGTH_STRING = "vm.userdata.max.length";
     ConfigKey<Integer> VM_USERDATA_MAX_LENGTH = new ConfigKey<>("Advanced", Integer.class, VM_USERDATA_MAX_LENGTH_STRING, "32768",
             "Max length of vm userdata after base64 encoding. Default is 32768 and maximum is 1048576", true);
 
     String concatenateUserData(String userdata1, String userdata2, String userdataProvider);
     String validateUserData(String userData, BaseCmd.HTTPMethod httpmethod);
+
+    /**
+     * This method validates the user data uuid for system VMs and returns the user data
+     * after compression and base64 encoding for the system VM to consume.
+     *
+     * @param userDataUuid
+     * @return a String containing the user data after compression and base64 encoding
+     * @throws IOException
+     */
+    String validateAndGetUserDataForSystemVM(String userDataUuid) throws IOException;
 }

engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java

@@ -106,6 +106,9 @@ public interface VirtualMachineManager extends Manager {
     ConfigKey<Boolean> VmSyncPowerStateTransitioning = new ConfigKey<>("Advanced", Boolean.class, "vm.sync.power.state.transitioning", "true",
             "Whether to sync power states of the transitioning and stalled VMs while processing VM power reports.", false);
 
+    ConfigKey<Boolean> SystemVmEnableUserData = new ConfigKey<>(Boolean.class, "systemvm.userdata.enabled", "Advanced", "false",
+            "Enable user data for system VMs. When enabled, the CPVM, SSVM, and Router system VMs will use the values from the global settings consoleproxy.userdata, secstorage.userdata, and router.userdata, respectively, to provide cloud-init user data to the VM.",
+            true, ConfigKey.Scope.Zone, null);
 
     interface Topics {
         String VM_POWER_STATE = "vm.powerstate";

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -5121,7 +5121,7 @@ public ConfigKey<?>[] getConfigKeys() {
                 VmConfigDriveLabel, VmConfigDriveOnPrimaryPool, VmConfigDriveForceHostCacheUse, VmConfigDriveUseHostCacheOnUnsupportedPool,
                 HaVmRestartHostUp, ResourceCountRunningVMsonly, AllowExposeHypervisorHostname, AllowExposeHypervisorHostnameAccountLevel, SystemVmRootDiskSize,
                 AllowExposeDomainInMetadata, MetadataCustomCloudName, VmMetadataManufacturer, VmMetadataProductName,
-                VmSyncPowerStateTransitioning
+                VmSyncPowerStateTransitioning, SystemVmEnableUserData
         };
     }
 

engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java

@@ -16,12 +16,18 @@
 // under the License.
 package org.apache.cloudstack.userdata;
 
+import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.URLDecoder;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import com.cloud.domain.Domain;
+import com.cloud.user.User;
+import com.cloud.user.UserDataVO;
+import com.cloud.user.dao.UserDataDao;
+import com.cloud.utils.compression.CompressionUtil;
 import org.apache.cloudstack.api.BaseCmd;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.commons.codec.binary.Base64;
@@ -31,7 +37,12 @@
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.exception.CloudRuntimeException;
 
+import javax.inject.Inject;
+
 public class UserDataManagerImpl extends ManagerBase implements UserDataManager {
+    @Inject
+    UserDataDao userDataDao;
+
     private static final int MAX_USER_DATA_LENGTH_BYTES = 2048;
     private static final int MAX_HTTP_GET_LENGTH = 2 * MAX_USER_DATA_LENGTH_BYTES; // 4KB
     private static final int NUM_OF_2K_BLOCKS = 512;
@@ -118,6 +129,25 @@ public String validateUserData(String userData, BaseCmd.HTTPMethod httpmethod) {
         return Base64.encodeBase64String(decodedUserData);
     }
 
+    @Override
+    public String validateAndGetUserDataForSystemVM(String userDataUuid) throws IOException {
+        if (StringUtils.isBlank(userDataUuid)) {
+            return null;
+        }
+        UserDataVO userDataVo = userDataDao.findByUuid(userDataUuid);
+        if (userDataVo == null) {
+            return null;
+        }
+        if (userDataVo.getDomainId() == Domain.ROOT_DOMAIN && userDataVo.getAccountId() == User.UID_ADMIN) {
+            // Decode base64 user data, compress it, then re-encode to reduce command line length
+            String plainTextUserData = new String(java.util.Base64.getDecoder().decode(userDataVo.getUserData()));
+            CompressionUtil compressionUtil = new CompressionUtil();
+            byte[] compressedUserData = compressionUtil.compressString(plainTextUserData);
+            return java.util.Base64.getEncoder().encodeToString(compressedUserData);
+        }
+        throw new CloudRuntimeException("User data can only be used by system VMs if it belongs to the ROOT domain and ADMIN account.");
+    }
+
     private byte[] validateAndDecodeByHTTPMethod(String userData, int maxHTTPLength, BaseCmd.HTTPMethod httpMethod) {
         byte[] decodedUserData = Base64.decodeBase64(userData.getBytes());
         if (decodedUserData == null || decodedUserData.length < 1) {

framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java

@@ -43,6 +43,64 @@ public class ConfigKey<T> {
     public static final String CATEGORY_NETWORK = "Network";
     public static final String CATEGORY_SYSTEM = "System";
 
+    // Configuration Groups to be used to define group for a config key
+    // Group name, description, precedence
+    public static final Ternary<String, String, Long> GROUP_MISCELLANEOUS = new Ternary<>("Miscellaneous", "Miscellaneous configuration", 999L);
+    public static final Ternary<String, String, Long> GROUP_ACCESS = new Ternary<>("Access", "Identity and Access management configuration", 1L);
+    public static final Ternary<String, String, Long> GROUP_COMPUTE = new Ternary<>("Compute", "Compute configuration", 2L);
+    public static final Ternary<String, String, Long> GROUP_STORAGE = new Ternary<>("Storage", "Storage configuration", 3L);
+    public static final Ternary<String, String, Long> GROUP_NETWORK = new Ternary<>("Network", "Network configuration", 4L);
+    public static final Ternary<String, String, Long> GROUP_HYPERVISOR = new Ternary<>("Hypervisor", "Hypervisor specific configuration", 5L);
+    public static final Ternary<String, String, Long> GROUP_MANAGEMENT_SERVER = new Ternary<>("Management Server", "Management Server configuration", 6L);
+    public static final Ternary<String, String, Long> GROUP_SYSTEM_VMS = new Ternary<>("System VMs", "System VMs related configuration", 7L);
+    public static final Ternary<String, String, Long> GROUP_INFRASTRUCTURE = new Ternary<>("Infrastructure", "Infrastructure configuration", 8L);
+    public static final Ternary<String, String, Long> GROUP_USAGE_SERVER = new Ternary<>("Usage Server", "Usage Server related configuration", 9L);
+
+    // Configuration Subgroups to be used to define subgroup for a config key
+    // Subgroup name, description, precedence
+    public static final Pair<String, Long> SUBGROUP_OTHERS = new Pair<>("Others", 999L);
+    public static final Pair<String, Long> SUBGROUP_ACCOUNT = new Pair<>("Account", 1L);
+    public static final Pair<String, Long> SUBGROUP_DOMAIN = new Pair<>("Domain", 2L);
+    public static final Pair<String, Long> SUBGROUP_PROJECT = new Pair<>("Project", 3L);
+    public static final Pair<String, Long> SUBGROUP_LDAP = new Pair<>("LDAP", 4L);
+    public static final Pair<String, Long> SUBGROUP_SAML = new Pair<>("SAML", 5L);
+    public static final Pair<String, Long> SUBGROUP_VIRTUAL_MACHINE = new Pair<>("Virtual Machine", 1L);
+    public static final Pair<String, Long> SUBGROUP_KUBERNETES = new Pair<>("Kubernetes", 2L);
+    public static final Pair<String, Long> SUBGROUP_HIGH_AVAILABILITY = new Pair<>("High Availability", 3L);
+    public static final Pair<String, Long> SUBGROUP_IMAGES = new Pair<>("Images", 1L);
+    public static final Pair<String, Long> SUBGROUP_VOLUME = new Pair<>("Volume", 2L);
+    public static final Pair<String, Long> SUBGROUP_SNAPSHOT = new Pair<>("Snapshot", 3L);
+    public static final Pair<String, Long> SUBGROUP_VM_SNAPSHOT = new Pair<>("VM Snapshot", 4L);
+    public static final Pair<String, Long> SUBGROUP_NETWORK = new Pair<>("Network", 1L);
+    public static final Pair<String, Long> SUBGROUP_DHCP = new Pair<>("DHCP", 2L);
+    public static final Pair<String, Long> SUBGROUP_VPC = new Pair<>("VPC", 3L);
+    public static final Pair<String, Long> SUBGROUP_LOADBALANCER = new Pair<>("LoadBalancer", 4L);
+    public static final Pair<String, Long> SUBGROUP_API = new Pair<>("API", 1L);
+    public static final Pair<String, Long> SUBGROUP_ALERTS = new Pair<>("Alerts", 2L);
+    public static final Pair<String, Long> SUBGROUP_EVENTS = new Pair<>("Events", 3L);
+    public static final Pair<String, Long> SUBGROUP_SECURITY = new Pair<>("Security", 4L);
+    public static final Pair<String, Long> SUBGROUP_USAGE = new Pair<>("Usage", 1L);
+    public static final Pair<String, Long> SUBGROUP_LIMITS = new Pair<>("Limits", 6L);
+    public static final Pair<String, Long> SUBGROUP_JOBS = new Pair<>("Jobs", 7L);
+    public static final Pair<String, Long> SUBGROUP_AGENT = new Pair<>("Agent", 8L);
+    public static final Pair<String, Long> SUBGROUP_HYPERVISOR = new Pair<>("Hypervisor", 1L);
+    public static final Pair<String, Long> SUBGROUP_KVM = new Pair<>("KVM", 2L);
+    public static final Pair<String, Long> SUBGROUP_VMWARE = new Pair<>("VMware", 3L);
+    public static final Pair<String, Long> SUBGROUP_XENSERVER = new Pair<>("XenServer", 4L);
+    public static final Pair<String, Long> SUBGROUP_OVM = new Pair<>("OVM", 5L);
+    public static final Pair<String, Long> SUBGROUP_BAREMETAL = new Pair<>("Baremetal", 6L);
+    public static final Pair<String, Long> SUBGROUP_CONSOLE_PROXY_VM = new Pair<>("ConsoleProxyVM", 1L);
+    public static final Pair<String, Long> SUBGROUP_SEC_STORAGE_VM = new Pair<>("SecStorageVM", 2L);
+    public static final Pair<String, Long> SUBGROUP_VIRTUAL_ROUTER = new Pair<>("VirtualRouter", 3L);
+    public static final Pair<String, Long> SUBGROUP_DIAGNOSTICS = new Pair<>("Diagnostics", 4L);
+    public static final Pair<String, Long> SUBGROUP_PRIMARY_STORAGE = new Pair<>("Primary Storage", 1L);
+    public static final Pair<String, Long> SUBGROUP_SECONDARY_STORAGE = new Pair<>("Secondary Storage", 2L);
+    public static final Pair<String, Long> SUBGROUP_BACKUP_AND_RECOVERY = new Pair<>("Backup & Recovery", 1L);
+    public static final Pair<String, Long> SUBGROUP_CERTIFICATE_AUTHORITY = new Pair<>("Certificate Authority", 2L);
+    public static final Pair<String, Long> SUBGROUP_QUOTA = new Pair<>("Quota", 3L);
+    public static final Pair<String, Long> SUBGROUP_CLOUDIAN = new Pair<>("Cloudian", 4L);
+    public static final Pair<String, Long> SUBGROUP_DRS = new Pair<>("DRS", 4L);
+
     public enum Scope {
         Global(null, 1),
         Zone(Global, 1 << 1),

plugins/network-elements/elastic-loadbalancer/src/main/java/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java

@@ -35,6 +35,8 @@
 import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
+import org.apache.cloudstack.userdata.UserDataManager;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Component;
 
 import com.cloud.agent.AgentManager;
@@ -101,6 +103,9 @@
 import com.cloud.vm.dao.DomainRouterDao;
 import com.cloud.vm.dao.NicDao;
 
+import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterUserData;
+import static com.cloud.vm.VirtualMachineManager.SystemVmEnableUserData;
+
 @Component
 public class ElasticLoadBalancerManagerImpl extends ManagerBase implements ElasticLoadBalancerManager, VirtualMachineGuru {
 
@@ -136,6 +141,8 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast
     private ElasticLbVmMapDao _elbVmMapDao;
     @Inject
     private NicDao _nicDao;
+    @Inject
+    private UserDataManager userDataManager;
 
     String _instance;
 
@@ -477,6 +484,19 @@ public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, Depl
         }
         String msPublicKey = _configDao.getValue("ssh.publickey");
         buf.append(" authorized_key=").append(VirtualMachineGuru.getEncodedMsPublicKey(msPublicKey));
+
+        if (SystemVmEnableUserData.valueIn(dc.getId())) {
+            String userDataUuid = RouterUserData.valueIn(dc.getId());
+            try {
+                String userData = userDataManager.validateAndGetUserDataForSystemVM(userDataUuid);
+                if (StringUtils.isNotBlank(userData)) {
+                    buf.append(" userdata=").append(userData);
+                }
+            } catch (Exception e) {
+                logger.warn("Failed to load user data for the elastic lb vm, ignored", e);
+            }
+        }
+
         if (logger.isDebugEnabled()) {
             logger.debug("Boot Args for " + profile + ": " + buf.toString());
         }

plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java

@@ -21,6 +21,8 @@
 import static com.cloud.hypervisor.Hypervisor.HypervisorType.LXC;
 import static com.cloud.hypervisor.Hypervisor.HypervisorType.VMware;
 import static com.cloud.hypervisor.Hypervisor.HypervisorType.XenServer;
+import static com.cloud.network.router.VirtualNetworkApplianceManager.RouterUserData;
+import static com.cloud.vm.VirtualMachineManager.SystemVmEnableUserData;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -39,6 +41,7 @@
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.lb.ApplicationLoadBalancerRuleVO;
 import org.apache.cloudstack.lb.dao.ApplicationLoadBalancerRuleDao;
+import org.apache.cloudstack.userdata.UserDataManager;
 import org.apache.commons.collections.CollectionUtils;
 
 import com.cloud.agent.AgentManager;
@@ -126,6 +129,7 @@
 import com.cloud.vm.VirtualMachineProfile.Param;
 import com.cloud.vm.dao.DomainRouterDao;
 import com.cloud.vm.dao.NicDao;
+import org.apache.commons.lang3.StringUtils;
 
 public class InternalLoadBalancerVMManagerImpl extends ManagerBase implements InternalLoadBalancerVMManager, InternalLoadBalancerVMService, VirtualMachineGuru {
     static final private String InternalLbVmNamePrefix = "b";
@@ -175,6 +179,8 @@ public class InternalLoadBalancerVMManagerImpl extends ManagerBase implements In
     ResourceManager _resourceMgr;
     @Inject
     UserDao _userDao;
+    @Inject
+    private UserDataManager userDataManager;
 
     @Override
     public boolean finalizeVirtualMachineProfile(final VirtualMachineProfile profile, final DeployDestination dest, final ReservationContext context) {
@@ -243,6 +249,19 @@ public boolean finalizeVirtualMachineProfile(final VirtualMachineProfile profile
         final String type = "ilbvm";
         buf.append(" type=" + type);
 
+        long dcId = profile.getVirtualMachine().getDataCenterId();
+        if (SystemVmEnableUserData.valueIn(dcId)) {
+            String userDataUuid = RouterUserData.valueIn(dcId);
+            try {
+                String userData = userDataManager.validateAndGetUserDataForSystemVM(userDataUuid);
+                if (StringUtils.isNotBlank(userData)) {
+                    buf.append(" userdata=").append(userData);
+                }
+            } catch (Exception e) {
+                logger.warn("Failed to load user data for the internal lb vm, ignored", e);
+            }
+        }
+
         if (logger.isDebugEnabled()) {
             logger.debug("Boot Args for " + profile + ": " + buf.toString());
         }

plugins/network-elements/internal-loadbalancer/src/test/java/org/apache/cloudstack/internallbvmmgr/LbChildTestConfiguration.java

@@ -18,6 +18,7 @@
 
 import java.io.IOException;
 
+import org.apache.cloudstack.userdata.UserDataManager;
 import org.mockito.Mockito;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
@@ -166,6 +167,11 @@ public AccountDao accountDao() {
             return Mockito.mock(AccountDao.class);
         }
 
+        @Bean
+        public UserDataManager userDataManager() {
+            return Mockito.mock(UserDataManager.class);
+        }
+
         @Override
         public boolean match(MetadataReader mdr, MetadataReaderFactory arg1) throws IOException {
             mdr.getClassMetadata().getClassName();

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -50,6 +50,9 @@
 import javax.naming.ConfigurationException;
 
 import com.cloud.consoleproxy.ConsoleProxyManager;
+import com.cloud.network.router.VirtualNetworkApplianceManager;
+import com.cloud.storage.secondary.SecondaryStorageVmManager;
+import com.cloud.vm.VirtualMachineManager;
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.acl.SecurityChecker;
 import org.apache.cloudstack.affinity.AffinityGroup;
@@ -638,6 +641,11 @@ protected void overProvisioningFactorsForValidation() {
     protected void populateConfigKeysAllowedOnlyForDefaultAdmin() {
         configKeysAllowedOnlyForDefaultAdmin.add(AccountManagerImpl.listOfRoleTypesAllowedForOperationsOfSameRoleType.key());
         configKeysAllowedOnlyForDefaultAdmin.add(AccountManagerImpl.allowOperationsOnUsersInSameAccount.key());
+
+        configKeysAllowedOnlyForDefaultAdmin.add(VirtualMachineManager.SystemVmEnableUserData.key());
+        configKeysAllowedOnlyForDefaultAdmin.add(ConsoleProxyManager.ConsoleProxyUserData.key());
+        configKeysAllowedOnlyForDefaultAdmin.add(SecondaryStorageVmManager.SecondaryStorageUserData.key());
+        configKeysAllowedOnlyForDefaultAdmin.add(VirtualNetworkApplianceManager.RouterUserData.key());
     }
 
     private void initMessageBusListener() {

server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManager.java

@@ -93,6 +93,12 @@ public interface ConsoleProxyManager extends Manager, ConsoleProxyService {
     ConfigKey<String> ConsoleProxyManagementLastState = new ConfigKey<String>(ConfigKey.CATEGORY_ADVANCED, String.class, "consoleproxy.management.state.last", com.cloud.consoleproxy.ConsoleProxyManagementState.Auto.toString(),
             "last console proxy service management state", false, ConfigKey.Kind.Select, consoleProxyManagementStates);
 
+    ConfigKey<String> ConsoleProxyUserData = new ConfigKey<>(String.class, "consoleproxy.userdata",
+            ConfigKey.CATEGORY_ADVANCED, "",
+            "UUID for user data for console proxy VMs. This works only when systemvm.userdata.enabled is set to true",
+            true, ConfigKey.Scope.Zone, null, "User Data for CPVMs",
+            null, ConfigKey.GROUP_SYSTEM_VMS, ConfigKey.SUBGROUP_CONSOLE_PROXY_VM);
+
     void setManagementState(ConsoleProxyManagementState state);
 
     ConsoleProxyManagementState getManagementState();