Skip to content

Commit

Permalink
Merge pull request #212 from ccamacho/test-ovn
Browse files Browse the repository at this point in the history
feat: allow deploying the service node in any chassis
  • Loading branch information
ccamacho authored Mar 9, 2021
2 parents 1bd8ee6 + 351b653 commit 0c2f6d0
Show file tree
Hide file tree
Showing 11 changed files with 185 additions and 110 deletions.
2 changes: 1 addition & 1 deletion ci/install_gitlab_node.sh
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ if [ -f /etc/redhat-release ] || [ -f /etc/fedora-release ]; then
curl -LJO https://gitlab-runner-downloads.s3.amazonaws.com/latest/rpm/gitlab-runner_amd64.rpm

if ! rpm -qa | grep gitlab-runner; then
sudo rpm -ivh -replacepkgs gitlab-runner_amd64.rpm
sudo rpm -ivh --replacepkgs gitlab-runner_amd64.rpm
fi
fi

Expand Down
15 changes: 14 additions & 1 deletion kubeinit/roles/kubeinit_cdk/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,25 @@
public: yes
tags: provision_libvirt

- name: run infra prepare steps
ansible.builtin.include_role:
name: "../../roles/kubeinit_libvirt"
tasks_from: 00_prepare.yml
public: yes
# We prepare all the available hypervisors
with_items:
- "{{ groups['all'] | map('regex_search','^.*(hypervisor).*$') | select('string') | list }}"
loop_control:
loop_var: cluster_role_item
vars:
kubeinit_deployment_node_name: "{{ cluster_role_item }}"
tags: provision_libvirt

- name: provision the initial virtual infra
ansible.builtin.include_role:
name: "../../roles/kubeinit_libvirt"
tasks_from: main.yml
public: yes
# We prepare all the available hypervisors
with_items:
- "{{ groups['all'] | map('regex_search','^.*(hypervisor).*$') | select('string') | list }}"
loop_control:
Expand Down
15 changes: 14 additions & 1 deletion kubeinit/roles/kubeinit_eks/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,26 @@
public: yes
tags: provision_libvirt

- name: run infra prepare steps
ansible.builtin.include_role:
name: "../../roles/kubeinit_libvirt"
tasks_from: 00_prepare.yml
public: yes
# We prepare all the available hypervisors
with_items:
- "{{ groups['all'] | map('regex_search','^.*(hypervisor).*$') | select('string') | list }}"
loop_control:
loop_var: cluster_role_item
vars:
kubeinit_deployment_node_name: "{{ cluster_role_item }}"
tags: provision_libvirt

- name: provision the initial virtual infra
ansible.builtin.include_role:
name: "../../roles/kubeinit_libvirt"
tasks_from: main.yml
public: yes
tags: provision_libvirt
# We prepare all the available hypervisors
with_items:
- "{{ groups['all'] | map('regex_search','^.*(hypervisor).*$') | select('string') | list }}"
loop_control:
Expand Down
15 changes: 14 additions & 1 deletion kubeinit/roles/kubeinit_k8s/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,25 @@
public: yes
tags: provision_libvirt

- name: run infra prepare steps
ansible.builtin.include_role:
name: "../../roles/kubeinit_libvirt"
tasks_from: 00_prepare.yml
public: yes
# We prepare all the available hypervisors
with_items:
- "{{ groups['all'] | map('regex_search','^.*(hypervisor).*$') | select('string') | list }}"
loop_control:
loop_var: cluster_role_item
vars:
kubeinit_deployment_node_name: "{{ cluster_role_item }}"
tags: provision_libvirt

- name: provision the initial virtual infra
ansible.builtin.include_role:
name: "../../roles/kubeinit_libvirt"
tasks_from: main.yml
public: yes
# We prepare all the available hypervisors
with_items:
- "{{ groups['all'] | map('regex_search','^.*(hypervisor).*$') | select('string') | list }}"
loop_control:
Expand Down
6 changes: 5 additions & 1 deletion kubeinit/roles/kubeinit_libvirt/tasks/00_cleanup.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,9 +80,13 @@
##
- name: Clean OVN/OVS resources
ansible.builtin.shell: |
###############################################
# WARNING: We can not remove the external #
# bridge interface kiextbr0, the connection #
# will be lost if this is executed in the HVs #
###############################################
ovs-vsctl del-br br-int || true
ovs-vsctl del-br br-ex || true
ovs-vsctl del-br kiextbr0 || true
ovn-nbctl ls-del sw0 || true
ovn-nbctl ls-del sw1 || true
ovn-nbctl lr-del lr0 || true
Expand Down
44 changes: 22 additions & 22 deletions kubeinit/roles/kubeinit_libvirt/tasks/00_ovn_post_setup.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@
#
# Create a logical switch
#
ovn-nbctl ls-add sw0
ovn-nbctl --wait=hv ls-add sw0
{% if kubeinit_libvirt_external_service_interface_enabled %}
ovn-nbctl ls-add sw1
ovn-nbctl --wait=hv ls-add sw1
{% endif %}
when: >
groups['hypervisor_nodes'][0] in kubeinit_deployment_node_name
Expand Down Expand Up @@ -80,7 +80,7 @@
#
# We create an OVN port using the interface ID and the mac address of the VM
#
ovn-nbctl --db=tcp:{{ hostvars[groups['hypervisor_nodes'][0]]['ansible_default_ipv4']['address'] }}:{{ kubeinit_libvirt_ovn_northbound_port }} \
ovn-nbctl --wait=hv --db=tcp:{{ hostvars[groups['hypervisor_nodes'][0]]['ansible_default_ipv4']['address'] }}:{{ kubeinit_libvirt_ovn_northbound_port }} \
lsp-add sw0 {{ hostvars[item].interfaceid }}
#
Expand All @@ -96,14 +96,20 @@
lsp-set-dhcpv4-options {{ hostvars[item].interfaceid }} {{ kubeinit_provision_dhcp_options }}
{% if kubeinit_libvirt_external_service_interface_enabled and 'service' in item %}
ovn-nbctl --db=tcp:{{ hostvars[groups['hypervisor_nodes'][0]]['ansible_default_ipv4']['address'] }}:{{ kubeinit_libvirt_ovn_northbound_port }} \
ovn-nbctl --wait=hv --db=tcp:{{ hostvars[groups['hypervisor_nodes'][0]]['ansible_default_ipv4']['address'] }}:{{ kubeinit_libvirt_ovn_northbound_port }} \
lsp-add sw1 {{ kubeinit_inventory_network_bridge_external_interfaceid }}
ovn-nbctl --db=tcp:{{ hostvars[groups['hypervisor_nodes'][0]]['ansible_default_ipv4']['address'] }}:{{ kubeinit_libvirt_ovn_northbound_port }} \
lsp-set-addresses {{ kubeinit_inventory_network_bridge_external_interfaceid }} "{{ kubeinit_libvirt_external_service_interface.mac }} {{ kubeinit_libvirt_external_service_interface.ip }}"
ovn-nbctl --db=tcp:{{ hostvars[groups['hypervisor_nodes'][0]]['ansible_default_ipv4']['address'] }}:{{ kubeinit_libvirt_ovn_northbound_port }} \
lsp-set-port-security {{ kubeinit_inventory_network_bridge_external_interfaceid }} "{{ kubeinit_libvirt_external_service_interface.mac }} {{ kubeinit_libvirt_external_service_interface.ip }}"
# The following command will be executed in the hypervisor where the service machine is.
# We need to make sure the bridge binding exists between the OVN interface (Logical SWitch Port) and
# the local bridge (OVS local bridge)
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=externalserv:{{ kubeinit_libvirt_external_service_interface.attached }},provider:br-ex
{% endif %}
with_items: "{{ groups['all_nodes'] }}"
# We run this bucle in each hypervisor, so, we execute these tasks
Expand All @@ -116,25 +122,25 @@
#
# Create a logical router to connect the VMs switch
#
ovn-nbctl lr-add lr0
ovn-nbctl lrp-add lr0 lr0-sw0 00:00:00:65:77:09 {{ kubeinit_inventory_network_gateway }}/{{ kubeinit_inventory_network_cidr }}
ovn-nbctl lsp-add sw0 sw0-lr0
ovn-nbctl --wait=hv lr-add lr0
ovn-nbctl --wait=hv lrp-add lr0 lr0-sw0 00:00:00:65:77:09 {{ kubeinit_inventory_network_gateway }}/{{ kubeinit_inventory_network_cidr }}
ovn-nbctl --wait=hv lsp-add sw0 sw0-lr0
ovn-nbctl lsp-set-type sw0-lr0 router
ovn-nbctl lsp-set-addresses sw0-lr0 router
ovn-nbctl lsp-set-options sw0-lr0 router-port=lr0-sw0
#
# We create the external access switch
#
ovn-nbctl ls-add public
ovn-nbctl lrp-add lr0 lr0-public 00:00:20:20:12:13 172.16.0.1/24
ovn-nbctl lsp-add public public-lr0
ovn-nbctl --wait=hv ls-add public
ovn-nbctl --wait=hv lrp-add lr0 lr0-public 00:00:20:20:12:13 172.16.0.1/24
ovn-nbctl --wait=hv lsp-add public public-lr0
ovn-nbctl lsp-set-type public-lr0 router
ovn-nbctl lsp-set-addresses public-lr0 router
ovn-nbctl lsp-set-options public-lr0 router-port=lr0-public
#
# Create a localnet port
#
ovn-nbctl lsp-add public public-ln
ovn-nbctl --wait=hv lsp-add public public-ln
ovn-nbctl lsp-set-type public-ln localnet
ovn-nbctl lsp-set-addresses public-ln unknown
ovn-nbctl lsp-set-options public-ln network_name=provider
Expand All @@ -156,27 +162,21 @@
ovs-vsctl --may-exist add-br br-ex
ip addr add 172.16.0.254/24 dev br-ex
ip link set br-ex up
#
# Create a localnet port for the external interface
#
{% if kubeinit_libvirt_external_service_interface_enabled %}
ovs-vsctl --may-exist add-br br-kiext
ovn-nbctl lsp-add sw1 sw1-ln
ovn-nbctl --wait=hv lsp-add sw1 sw1-ln
ovn-nbctl lsp-set-type sw1-ln localnet
ovn-nbctl lsp-set-addresses sw1-ln unknown
ovn-nbctl lsp-set-options sw1-ln network_name=externalserv
#
# We add a bridge mapping from br-kiext called provider
# We add a bridge mapping from the external interface bridge called externalserv
#
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=externalserv:br-kiext,provider:br-ex
ovs-vsctl add-port ovs-bridge ovs-bridge-patch
ovs-vsctl add-port br-kiext br-kiext-patch
ovs-vsctl set interface ovs-bridge-patch type=patch
ovs-vsctl set interface ovs-bridge-patch options:peer=br-kiext-patch
ovs-vsctl set interface br-kiext-patch type=patch
ovs-vsctl set interface br-kiext-patch options:peer=ovs-bridge-patch
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=externalserv:{{ kubeinit_libvirt_external_service_interface.attached }},provider:br-ex
{% endif %}
#
# Routes
#
Expand Down
74 changes: 74 additions & 0 deletions kubeinit/roles/kubeinit_libvirt/tasks/00_prepare.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
---
# Copyright kubeinit.com
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

##
## Hypervisor installing dependencies and rebooting.
##
- name: Set up an Hypervisors
block:

- name: "check if the external bridge is created when using the external interface"
ansible.builtin.shell: |
set -o pipefail
nmcli con show | grep {{ kubeinit_libvirt_external_service_interface.attached }}
register: bridge_status
ignore_errors: yes
when: |
(hostvars[ groups['all'] | map('regex_search','^.*service.*$') | select('string') | list | first ].target in kubeinit_deployment_node_name) and
kubeinit_libvirt_external_service_interface_enabled
- name: Fail if the external bridge is not created when using the external interface
ansible.builtin.fail:
msg:
- "The bridge {{ kubeinit_libvirt_external_service_interface.attached }} to provide external"
- "connectivity is not created. This is a requirement that needs to be"
- "created before running the playbook."
- "You must create {{ kubeinit_libvirt_external_service_interface.attached }} in"
- "{{ hostvars[ groups['all'] | map('regex_search','^.*service.*$') | select('string') | list | first ].target }}"
- "before continue."
- "Run `nmcli con show` and check it is created correctly."
- "********************************************************************"
- "* WARNING: Make sure that if you use OVN, you *"
- "* create an OVS bridge, and if you use libvirt then, *"
- "* a Linux bridge. *"
- "* Refer to: http://docs.kubeinit.com/usage.html#external-interface *"
- "* for further details. *"
- "********************************************************************"
when: |
(hostvars[ groups['all'] | map('regex_search','^.*service.*$') | select('string') | list | first ].target in kubeinit_deployment_node_name) and
kubeinit_libvirt_external_service_interface_enabled and
bridge_status.rc != 0
- name: Fails if OS is not supported
ansible.builtin.fail:
msg: The hypervisor machine needs to be CentOS/RHEL
when: not (ansible_distribution == 'CentOS' or ansible_distribution == 'RedHat' or ansible_distribution == 'Fedora' or ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu')

- name: Fails if CentOS 7 hypervisor
ansible.builtin.fail:
msg:
- "CentOS 7 is not supported as CentOS 8 guests are used."
- "Refer to: https://access.redhat.com/solutions/4073061"
- "This issue is caused by the forward compatibility of xfs"
- "file system between CentOS 7 and CentOS 8."
- "Xfs file system in CentOS 8 uses reflink and sparse files,"
- "but CentOS 7 kernel does not understand them and refuses to mount it."
when: (ansible_distribution == 'CentOS' and ansible_distribution_major_version == "7")

# In this case the node will be an hypervisor
delegate_to: "{{ kubeinit_deployment_node_name }}"
tags:
- provision_libvirt
68 changes: 0 additions & 68 deletions kubeinit/roles/kubeinit_libvirt/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,74 +33,6 @@
- '|-------------------------------------------------------|'
when: kubeinit_libvirt_ovn_enabled

- name: "check if bridge is created in the single node deployment"
ansible.builtin.shell: |
set -o pipefail
nmcli con show | grep {{ kubeinit_libvirt_external_service_interface.attached }}
register: bridge_status
ignore_errors: yes
when: |
(hostvars[ groups['all'] | map('regex_search','^.*service.*$') | select('string') | list | first ].target in kubeinit_deployment_node_name) and
kubeinit_libvirt_external_service_interface_enabled and
not kubeinit_libvirt_ovn_enabled
- name: "check if bridge is created in the multi node deployment"
ansible.builtin.shell: |
set -o pipefail
nmcli con show | grep {{ kubeinit_libvirt_external_service_interface.attached }}
register: bridge_ovn_status
ignore_errors: yes
when: |
groups['hypervisor_nodes'][0] in kubeinit_deployment_node_name and
kubeinit_libvirt_ovn_enabled
- name: Fail if the external bridge is not created in the single node deployment
ansible.builtin.fail:
msg:
- "The bridge {{ kubeinit_libvirt_external_service_interface.attached }} to provide external"
- "connectivity is not created. This is a requirement that needs to be"
- "created before running the playbook."
- "You must create {{ kubeinit_libvirt_external_service_interface.attached }} in"
- "{{ hostvars[ groups['all'] | map('regex_search','^.*service.*$') | select('string') | list | first ].target }}"
- "before continue."
- "Run `nmcli con show` and check it is created correctly."
when: |
(hostvars[ groups['all'] | map('regex_search','^.*service.*$') | select('string') | list | first ].target in kubeinit_deployment_node_name) and
kubeinit_libvirt_external_service_interface_enabled and
not kubeinit_libvirt_ovn_enabled and
bridge_status.rc != 0
- name: Fail if the external bridge is created in the multi node deployment
ansible.builtin.fail:
msg:
- "The bridge {{ kubeinit_libvirt_external_service_interface.attached }} to provide external"
- "connectivity is created automatically by OVN. This is a requirement that needs to be"
- "checked before running the playbook."
- "You must delete {{ kubeinit_libvirt_external_service_interface.attached }} in"
- "{{ groups['hypervisor_nodes'][0] }} and have it configured as it was before,"
- "before continuing the deployment."
- "Run `nmcli con show` and check it is created correctly."
when: |
(groups['hypervisor_nodes'][0] in kubeinit_deployment_node_name) and
kubeinit_libvirt_ovn_enabled and
bridge_ovn_status.rc == 0
- name: Fails if OS is not supported
ansible.builtin.fail:
msg: The hypervisor machine needs to be CentOS/RHEL
when: not (ansible_distribution == 'CentOS' or ansible_distribution == 'RedHat' or ansible_distribution == 'Fedora' or ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu')

- name: Fails if CentOS 7 hypervisor
ansible.builtin.fail:
msg:
- "CentOS 7 is not supported as CentOS 8 guests are used."
- "Refer to: https://access.redhat.com/solutions/4073061"
- "This issue is caused by the forward compatibility of xfs"
- "file system between CentOS 7 and CentOS 8."
- "Xfs file system in CentOS 8 uses reflink and sparse files,"
- "but CentOS 7 kernel does not understand them and refuses to mount it."
when: (ansible_distribution == 'CentOS' and ansible_distribution_major_version == "7")

- name: Check if Intel virtualization is supported
ansible.builtin.shell: |
set -e
Expand Down
Loading

0 comments on commit 0c2f6d0

Please sign in to comment.