enable vsphere cloud provider on worker node

this will enable vsphere cloud provider on worker node,
although we could need further change from discussion:
vmware-archive/kubernetes-archived#490

this also fix several configuration issues on worker template.

Author: figo

master_templates-v1.10.0/master.tf

@@ -68,7 +68,7 @@ data "template_file" "master" {
     kube_controller_manager_artifact = "${ var.kube_controller_manager_artifact }"
     kube_scheduler = "${ base64encode(data.template_file.kube_scheduler.rendered) }"
     kube_scheduler_artifact = "${ var.kube_scheduler_artifact }"
-    cloud_config_file = "${ var.cloud_config_file }"
+    cloud_config_file = "${ base64encode(var.cloud_config_file) }"
     ca = "${ base64encode(var.ca) }"
     ca_key = "${ base64encode(var.ca_key) }"
     apiserver = "${ base64encode(var.apiserver) }"

master_templates-v1.10.0/master.yml

@@ -6,15 +6,12 @@ systemd:
   units:
     - name: systemd-resolved.service
       enable: true
-      #command: restart
 
     - name: systemd-hostnamed.service
       enable: true
-      #command: restart
 
     - name: docker.service
       enable: true
-      #command: start
       drop-ins:
         - name: docker.conf
           contents: |
@@ -23,7 +20,6 @@ systemd:
 
     - name: bins.service
       enable: true
-      #command: start
       contents: |
         [Unit]
         Description=Download Binaries
@@ -77,7 +73,6 @@ systemd:
         
     - name: kube-apiserver.service
       enable: true
-      #command: start
       contents: |
         [Unit]
         Description=Kubernetes API Server
@@ -97,7 +92,6 @@ systemd:
 
     - name: kube-controller-manager.service
       enable: true
-      #command: start
       contents: |
         [Unit]
         Description=Kubernetes Controller Manager
@@ -117,7 +111,6 @@ systemd:
 
     - name: kube-scheduler.service
       enable: true
-      #command: start
       contents: |
         [Unit]
         Description=Kubernetes Scheduler

vsphere/cloud.conf

@@ -0,0 +1,8 @@
+[Global]
+user="${ vsphere_user }"
+password="${ vsphere_password }"
+server="${ vsphere_server }"
+insecure-flag="1"
+datacenter="${ vsphere_datacenter }"
+datastore="${ vsphere_datastore }"
+working-dir="${ vsphere_vm_folder }"

vsphere/input.tf

@@ -1,6 +1,6 @@
 # General cluster settings
 variable "name" { default = "vsphere" }
-variable "data_dir" { default = "/tmp/cncf/data/vsphere" }
+variable "data_dir" { default = "/cncf/data/vsphere" }
 
 # vSphere Auth Settings
 variable "vsphere_server" {}
@@ -17,6 +17,7 @@ variable "resource_pool" { default = "Compute-ResourcePool" }
 variable "datastore_name" { default = "WorkloadDatastore" }
 variable "virtual_machine_domain" { default = "vsphere.local" }
 variable "virtual_machine_dns_servers" { default = [ "8.8.8.8",] }
+variable "vm_folder" { default = "Workloads" }
 
 
 # Master Configuration

vsphere/modules.tf

@@ -51,7 +51,7 @@ module "master_templates" {
   controller_key = "${ module.tls.controller_key }"
   scheduler = "${ module.tls.scheduler }"
   scheduler_key = "${ module.tls.scheduler_key }"
-  cloud_config_file = ""
+  cloud_config_file = "${ data.template_file.cloud_conf.rendered }"
 
   dns_conf = "${ module.dns.dns_conf }"
   dns_dhcp = ""
@@ -107,7 +107,7 @@ module "worker_templates" {
   proxy = "${ module.tls.proxy }"
   proxy_key = "${ module.tls.proxy_key }"
   bootstrap = "${ module.master_templates.bootstrap }"
-  cloud_config_file = ""
+  cloud_config_file = "${ data.template_file.cloud_conf.rendered }"
 
   dns_conf = "${ module.dns.dns_conf }"
   dns_dhcp = ""

vsphere/vsphere.tf

@@ -23,3 +23,17 @@ provider "gzip" {
 # coreos container-linux-config-transpiler
 provider "ct" {
 }
+
+
+data "template_file" "cloud_conf" {
+  template = "${ file( "${ path.module }/cloud.conf" )}"
+  vars {
+    vsphere_user = "${ var.vsphere_user }"
+    vsphere_server = "${ var.vsphere_server }"
+    vsphere_password = "${ var.vsphere_password }"
+    vsphere_datacenter = "${ var.datacenter }"
+    vsphere_datastore = "${ var.datastore_name }"
+    vsphere_vm_folder = "${ var.vm_folder }"
+  }
+}
+

worker_templates-v1.10.0/kubelet

@@ -18,4 +18,4 @@ KUBELET_OPTS="--allow-privileged=true \
 --runtime-request-timeout=15m \
 --tls-cert-file=/etc/srv/kubernetes/pki/kubelet.crt \
 --tls-private-key-file=/etc/srv/kubernetes/pki/kubelet.key \
---v=2"
+--v=2"

worker_templates-v1.10.0/worker.tf

@@ -55,7 +55,7 @@ data "template_file" "worker" {
   vars {
     hostname = "${ var.hostname }-${ count.index + 1 }.${ var.hostname_suffix }"
     hostname_path = "${ var.hostname_path }"
-    cloud_config_file = "${ base64gzip(var.cloud_config_file) }"
+    cloud_config_file = "${ base64encode(var.cloud_config_file) }"
     ca = "${ base64encode(var.ca) }"
     kubelet_crt = "${ base64encode(var.kubelet) }"
     kubelet_key = "${ base64encode(var.kubelet_key) }"

worker_templates-v1.10.0/worker.yml

@@ -14,13 +14,13 @@ systemd:
       enable: true
       drop-ins:
         - name: docker.conf
-          content: |
+          contents: |
             [Service]
             EnvironmentFile=/etc/default/docker
 
     - name: bins.service
       enable: true
-      content: |
+      contents: |
         [Unit]
         Description=Download Binaries
         [Service]
@@ -39,7 +39,7 @@ systemd:
 
     - name: socat.service
       enable: true
-      content: |
+      contents: |
         [Unit]
         Description=Install Socat
         After=docker.service
@@ -53,11 +53,11 @@ systemd:
 
     - name: kubelet.service
       enable: true
-      content: |
+      contents: |
         [Unit]
         Description=Kubernetes kubelet
-        After=socat.service
-        Requires=socat.service
+        After=socat.service bins.service
+        Requires=socat.service bins.service
 
         [Service]
         Restart=always
@@ -76,43 +76,43 @@ storage:
   files:
     - path: ${ hostname_path }
       filesystem: "root"
-      content: 
+      contents: 
         inline: |
           ${ hostname }
 
     - path: /etc/default/docker
       filesystem: "root"
-      content: 
+      contents: 
         inline: |
           DOCKER_OPTS="--ip-masq=false --iptables=false --log-driver=json-file --log-level=warn --log-opt=max-file=5 --log-opt=max-size=10m --storage-driver=overlay"
 
     - path: /etc/default/kubelet
       filesystem: "root"
       mode:  0644
-      content: 
+      contents: 
         base64: true
         inline: |
           ${ kubelet }
 
     - path: /var/lib/kubelet/bootstrap.kubeconfig
       filesystem: "root"
-      content: 
+      contents: 
         base64: true
         inline: |
           ${ kubelet_bootstrap_kubeconfig }
 
     - path: /etc/kubernetes/manifests/kube-proxy.yml
       filesystem: "root"
       mode:  0644
-      content: 
+      contents: 
         base64: true
         inline: |
           ${ kube_proxy }
 
     - path: /var/lib/kube-proxy/kubeconfig
       filesystem: "root"
       mode:  0644
-      content: 
+      contents: 
         base64: true
         inline: |
           ${ proxy_kubeconfig }
@@ -121,47 +121,47 @@ storage:
     - path: /etc/srv/kubernetes/pki/ca-certificates.crt
       filesystem: "root"
       mode:  0644
-      content: 
+      contents: 
         base64: true
         inline: |
           ${ ca }
 
     - path: /etc/srv/kubernetes/pki/kubelet.crt
       filesystem: "root"
       mode:  0644
-      content: 
+      contents: 
         base64: true
         inline: |
           ${ kubelet_crt }
       
     - path: /etc/srv/kubernetes/pki/kubelet.key
       filesystem: "root"
       mode:  0644
-      content: 
+      contents: 
         base64: true
         inline: |
           ${ kubelet_key }
         
     - path: /etc/srv/kubernetes/pki/proxy.crt
       filesystem: "root"
       mode:  0644
-      content: 
+      contents: 
         base64: true
         inline: |
           ${ proxy }
         
     - path: /etc/srv/kubernetes/pki/proxy.key
       filesystem: "root"
       mode:  0644
-      content: 
+      contents: 
         base64: true
         inline: |
           ${ proxy_key }
 
     - path: /etc/srv/kubernetes/cloud-config
       filesystem: "root"
       mode:  0644
-      content: 
+      contents: 
         base64: true
         inline: |
           ${ cloud_config_file }
@@ -171,7 +171,7 @@ storage:
       mode:  0644
       user:
         name: "root"
-      content: 
+      contents: 
         base64: true
         inline: |
           ${ dns_conf }
@@ -181,7 +181,7 @@ storage:
       mode:  0644
       user:
         name: "root"
-      content: 
+      contents: 
         base64: true
         inline: |
           ${ dns_dhcp }