Cluster setup and client config

The cluster setup should work reliable now and servers will join after etcd is up.
Furthermore, the client configuration will be added to the output directory now.
Other files have seen improvements, too. But it is as it is, in heavy development :)
main
dschier 2 years ago
parent 03b973a868
commit 31d7ff5983

55
.gitignore vendored

@ -0,0 +1,55 @@
# reference: https://git-scm.com/docs/gitignore
output/*.*
!output/.gitkeep
# ansible
*.retry
# compiled source
bin/
*.com
*.class
*.dll
*.exe
*.o
*.so
# database
*.sql
*.sqlite
# lock files
*.lock
package-lock.json
# logs
logs/
*.log
# OS generated files
.DS_Store*
ehthumbs.db
Icon?
Thumbs.db
# packages
*.7z
*.dmg
*.gz
*.iso
*.rar
*.tar
*.zip
# python
*.py[cod]
__pycache__/
# temporary files
*.swp
*.swo
*~
# Vagrant
.vagrant/

@ -0,0 +1,64 @@
# Ansible for k3s deployment
An Ansible repository to deploy a k3s kubernetes cluster.
## Motivation
Having different shapres of kubernetes on hand is somewhat mandatory for typical
DevOps and development tasks.
## Description
This repository allows to build different types of k3s kubernetes setups locally
and on specified machines.
### Features
This section describes the features of the repository.
#### Cluster location
You can set up the cluster, wherever you want. You just need to configure the
inventory accordingly and you are good to go.
##### Vagrant
TBD
##### Your own machines
TBD
#### Cluster Size
You can build different clusters by changing the amount of servers and agents.
Some typical scenarios, you may be interested in, are described below.
##### Single instance
A single kubernetes instance is useful for local development and testing. It is
similar to microk8s or minikube setups.
##### HA Setup
You can setup a high available kubernetes cluster with 3 servers. This is useful
to develop kubernetes services, ingress, HA testing and loadbalancer
functionality.
##### Single server, multiple agents
If you want to test pod affinity, horizontal auto scaling or just want to use
multiple agents, you can define one server and multiple agents.
##### HA Setup with multiple agents
Define at minimum 3 servers and 1 agent to get a high available setup. This is
very close to a production cluster and can be useful to test production setups.
#### Container deployments
The repository will offer options to configure some additional scenarios.
##### TBD
TBD

28
Vagrantfile vendored

@ -1,6 +1,6 @@
K3S_SERVER_COUNT = 1
K3S_SERVER_COUNT = 3
K3S_AGENT_COUNT = 0
K3S_VERSION = "v1.21.1-rc2+k3s1"
K3S_VERSION = "v1.21.1+k3s1"
K3S_TOKEN = "VeryCrypticToken"
Vagrant.configure("2") do |config|
@ -27,10 +27,11 @@ Vagrant.configure("2") do |config|
hv.linked_clone = true
end
config.vm.box = "fedora/34-cloud-base"
# k3s server machines
(1..K3S_SERVER_COUNT).each do |i|
config.vm.define "k3s_server#{i}" do |server|
server.vm.box = "fedora/34-cloud-base"
server.vm.hostname = "k3s-server#{i}"
end
end
@ -38,26 +39,25 @@ Vagrant.configure("2") do |config|
# k3s agent machines
(1..K3S_AGENT_COUNT).each do |i|
config.vm.define "k3s_agent#{i}" do |agent|
agent.vm.box = "fedora/34-cloud-base"
agent.vm.hostname = "k3s-agent#{i}"
end
end
# provision
config.vm.provision "ansible" do |ansible|
ansible.playbook = "k3s_cluster.yml"
ansible.groups = {
"k3s_cluster:children" => ["k3s_server", "k3s_agent"],
config.vm.provision "ansible" do |cluster|
cluster.playbook = "k3s_cluster.yml"
cluster.compatibility_mode = "2.0"
cluster.groups = {
"k3s_cluster:children" => ["k3s_servers", "k3s_agents"],
"k3s_cluster:vars" => {"k3s_token" => K3S_TOKEN,
"k3s_version" => K3S_VERSION },
"k3s_server" => ["k3s_server1", "k3s_server2", "k3s_server3"],
"k3s_server:vars" => {"k3s_role" => "server"},
"k3s_agent" => ["k3s_agent1", "k3s_agent2", "k3s_agent3"],
"k3s_agent:vars" => {"k3s_role" => "agent"}
"k3s_servers" => ["k3s_server[1:#{K3S_SERVER_COUNT}]"],
"k3s_servers:vars" => {"k3s_role" => "server"},
"k3s_agents" => ["k3s_agent1", "k3s_agent2", "k3s_agent3"],
"k3s_agents:vars" => {"k3s_role" => "agent"}
}
ansible.host_vars = {
cluster.host_vars = {
"k3s_server1" => {"first_node" => true}
}
end
end

@ -1,5 +1,6 @@
[defaults]
use_persistent_connections=true
deprecation_warnings=False
[connection]
pipelining=true

@ -3,12 +3,175 @@
tasks:
- name: "Import k3s_cluster Tasks"
import_tasks: "tasks/k3s_cluster.yml"
# Prerequisites
- name: "Install Dependencies"
ansible.builtin.dnf:
name:
- "container-selinux"
- "selinux-policy-base"
state: "present"
become: true
post_tasks:
- name: "Install k3s SELinux"
ansible.builtin.dnf:
name: "https://rpm.rancher.io/k3s/latest/common/centos/7/noarch/k3s-selinux-0.2-1.el7_8.noarch.rpm"
disable_gpg_check: true
state: "present"
become: true
- name: "Message"
debug:
msg: "foo"
- name: "Enable IPv4 forwarding"
ansible.builtin.sysctl:
name: "net.ipv4.ip_forward"
value: 1
state: "present"
reload: true
become: true
- name: "Enable IPv6 forwarding"
ansible.builtin.sysctl:
name: "net.ipv6.conf.all.forwarding"
value: 1
state: "present"
reload: true
become: true
- name: "Manage br_netfilter Module File"
ansible.builtin.copy:
dest: "/etc/modules-load.d/br_netfilter.conf"
content: "br_netfilter"
mode: "u=rw,g=,o="
become: true
- name: "Load br_netfilter Module"
community.general.modprobe:
name: "br_netfilter"
state: "present"
become: true
- name: "Manage bridge-nf-call-iptables Sysctl"
ansible.builtin.sysctl:
name: "{{ item }}"
value: "1"
state: "present"
reload: true
become: true
loop:
- "net.bridge.bridge-nf-call-iptables"
- "net.bridge.bridge-nf-call-ip6tables"
# K3S Installation
- name: "Download k3s Binary"
ansible.builtin.get_url:
url: "https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s"
checksum: "sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-amd64.txt"
dest: "/usr/local/bin/k3s"
owner: "root"
group: "root"
mode: 0755
become: true
- hosts: "k3s_servers"
tasks:
- name: "Create kubectl symlink"
ansible.builtin.file:
src: "/usr/local/bin/k3s"
dest: "/usr/local/bin/kubectl"
state: "link"
become: true
- name: "Create crictl symlink"
ansible.builtin.file:
src: "/usr/local/bin/k3s"
dest: "/usr/local/bin/crictl"
state: "link"
become: true
- name: "Create ctr symlink"
ansible.builtin.file:
src: "/usr/local/bin/k3s"
dest: "/usr/local/bin/ctr"
state: "link"
become: true
- hosts: "k3s_server1"
tasks:
- name: "Manage k3s Service Unit File"
ansible.builtin.template:
src: "k3s.service.j2"
dest: "/etc/systemd/system/k3s.service"
owner: "root"
group: "root"
mode: 0644
become: true
- name: "Manage k3s Service"
ansible.builtin.systemd:
name: "k3s.service"
state: "started"
enabled: true
daemon_reload: true
become: true
- name: "Wait for etcd"
ansible.builtin.wait_for:
port: 2379
timeout: 120
- name: "Create kubeconfig"
ansible.builtin.template:
src: "admin.kubeconfig.j2"
dest: "output/admin.kubeconfig"
delegate_to: "localhost"
- name: "Fetch CA"
ansible.builtin.fetch:
src: "/var/lib/rancher/k3s/server/tls/server-ca.crt"
dest: "output/"
flat: true
become: true
- name: "Fetch CRT"
ansible.builtin.fetch:
src: "/var/lib/rancher/k3s/server/tls/client-admin.crt"
dest: "output/"
flat: true
become: true
- name: "Fetch KEY"
ansible.builtin.fetch:
src: "/var/lib/rancher/k3s/server/tls/client-admin.key"
dest: "output/"
flat: true
become: true
- hosts: "all !k3s_server1"
tasks:
- name: "Wait for etcd on server 1"
ansible.builtin.wait_for:
host: "{{ hostvars['k3s_server1']['ansible_host'] }}"
port: 2379
timeout: 120
- name: "Manage k3s Service Unit File"
ansible.builtin.template:
src: "k3s.service.j2"
dest: "/etc/systemd/system/k3s.service"
owner: "root"
group: "root"
mode: 0644
become: true
- name: "Manage k3s Service"
ansible.builtin.systemd:
name: "k3s.service"
state: "started"
enabled: true
daemon_reload: true
become: true
...

@ -0,0 +1,9 @@
---
- hosts: "k3s_server1"
tasks:
- name: "Just a message"
debug:
msg: "{{ inventory_hostname }}"
...

@ -0,0 +1 @@
foo

@ -1,93 +0,0 @@
---
- name: "Install Dependencies"
ansible.builtin.dnf:
name:
- "container-selinux"
- "selinux-policy-base"
state: "present"
become: true
- name: "Install k3s SELinux"
ansible.builtin.dnf:
name: "https://rpm.rancher.io/k3s/latest/common/centos/7/noarch/k3s-selinux-0.2-1.el7_8.noarch.rpm"
disable_gpg_check: true
state: "present"
become: true
#- name: "Manage firewalld Service"
# ansible.builtin.service:
# name: "firewalld.service"
# state: "stopped"
# enabled: false
# become: true
- name: "Enable IPv4 forwarding"
ansible.builtin.sysctl:
name: "net.ipv4.ip_forward"
value: 1
state: "present"
reload: true
become: true
- name: "Enable IPv6 forwarding"
ansible.builtin.sysctl:
name: "net.ipv6.conf.all.forwarding"
value: 1
state: "present"
reload: true
become: true
- name: "Download k3s Binary"
ansible.builtin.get_url:
url: "https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s"
checksum: "sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-amd64.txt"
dest: "/usr/local/bin/k3s"
owner: "root"
group: "root"
mode: 0755
become: true
- name: "Create kubectl symlink"
ansible.builtin.file:
src: "/usr/local/bin/k3s"
dest: "/usr/local/bin/kubectl"
state: "link"
become: true
- name: "Create crictl symlink"
ansible.builtin.file:
src: "/usr/local/bin/k3s"
dest: "/usr/local/bin/crictl"
state: "link"
become: true
- name: "Create ctr symlink"
ansible.builtin.file:
src: "/usr/local/bin/k3s"
dest: "/usr/local/bin/ctr"
state: "link"
become: true
- name: "Manage k3s Service Unit File"
ansible.builtin.template:
src: "k3s.service.j2"
dest: "/etc/systemd/system/k3s.service"
owner: "root"
group: "root"
mode: 0644
become: true
- name: "Manage k3s Service"
ansible.builtin.systemd:
name: "k3s.service"
state: "started"
enabled: true
daemon_reload: true
become: true
- name: "Create kubeconfig"
ansible.builtin.copy:
dest: "kubeconfig"
content: "foo"
delegate_to: "localhost"
...

@ -0,0 +1,20 @@
apiVersion: v1
clusters:
- cluster:
server: https://{{ ansible_host }}:6443
certificate-authority: server-ca.crt
name: local
contexts:
- context:
cluster: local
namespace: default
user: user
name: Default
current-context: Default
kind: Config
preferences: {}
users:
- name: user
user:
client-certificate: client-admin.crt
client-key: client-admin.key

@ -3,10 +3,14 @@
[Unit]
Description=Lightweight Kubernetes
Documentation=https://k3s.io
Wants=network-online.target
After=network-online.target
[Service]
Type=notify
EnvironmentFile=-/etc/default/%N
EnvironmentFile=-/etc/sysconfig/%N
ExecStartPre=-/sbin/modprobe br_netfilter
ExecStartPre=-/sbin/modprobe overlay

Loading…
Cancel
Save