Howto kubernetes archlinux

configure os

download latest archlinux cloud image:

wget https://linuximages.de/openstack/arch/arch-openstack-LATEST-image-bootstrap.qcow2

make sure we have libguestfs installed:

pacman -S libguestfs guestfs-tools

resize image:

cp arch-openstack-LATEST-image-bootstrap.qcow2 arch-openstack-LATEST-image-bootstrap_100G.qcow2
qemu-img resize arch-openstack-LATEST-image-bootstrap_100G.qcow2 +99G

expand image:

virt-resize --expand /dev/sda1 arch-openstack-LATEST-image-bootstrap.qcow2 arch-openstack-LATEST-image-bootstrap_100G.qcow2

change password:

virt-sysprep -a arch-openstack-LATEST-image-bootstrap_100G.qcow2 -q --root-password password:vidalinux

uninstall cloud-init:

virt-sysprep -a arch-openstack-LATEST-image-bootstrap_100G.qcow2 --run-command "pacman -R cloud-init --noconfirm"

on centos copy the image to this directory:

cd /var/lib/libvirt/images/

mount image:

qemu-nbd -c /dev/nbd0 arch-openstack-LATEST-image-bootstrap_100G.qcow2 
kpartx -a /dev/nbd0
mkdir /mnt/cloudimg
mount /dev/mapper/nbd0p1 /mnt/cloudimg
mount -o bind /dev /mnt/cloudimg/dev
mount -o bind /proc /mnt/cloudimg/proc

enter enviroment with arch-chroot:

arch-chroot /mnt/cloudimg

initialize the pacman keyring and populate signing keys:

pacman-key --init
pacman-key --populate archlinux

configure /etc/hosts:

cat > /etc/hosts << EOF
127.0.0.1 localhost
192.168.24.10 archlinux.ovoxcloud.com
EOF

edit /etc/ssh/sshd_config:

sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/g' /etc/ssh/sshd_config
sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config

enable sshd service:

systemctl enable sshd

ensure net.bridge.bridge-nf-call-iptables is set to 1 in your sysctl config:

cat <<EOF > /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF

configure networkmanager:

cat > /root/net << EOF
systemctl start NetworkManager && systemctl enable NetworkManager
nmcli con del eth0
nmcli con del Wired\ connection\ 1
nmcli con add con-name eth0 ipv4.method manual type ethernet ifname eth0 ipv4.addresses 192.168.24.10/24 ipv4.gateway 192.168.24.254 ipv4.dns 4.2.2.1,4.2.2.2 autoconnect yes
EOF

umount qcow2 image:

umount /mnt/cloudimg/proc
umount /mnt/cloudimg/dev
umount /mnt/cloudimg 
nbd-client -d /dev/nbd0
dmsetup remove /dev/mapper/nbd0p1

configure hostname:

hostnamectl set-hostname archlinux.ovoxcloud.com

configure timezone:

timedatectl set-timezone America/Puerto_Rico

network configuration:

ip addr add 192.168.24.10/24 dev eth0
ip route add default via 192.168.24.254
echo "nameserver 4.2.2.1" > /etc/resolv.conf

stop and disable systemd-resolved:

systemctl disable systemd-resolved && systemctl stop systemd-resolved

update entire os:

pacman -Syuu --noconfirm

install packages:

pacman -Sy curl vim screen nano net-tools bind-tools containerd networkmanager ebtables ethtool wget unzip socat cni-plugins conntrack-tools cri-o parted gptfdisk lvm2 git

install iptables:

pacman -S iptables

add the following registries to /etc/containers/registries.conf

cat >> /etc/containers/registries.conf << "EOF"
[registries.search]
registries = ['docker.io']
EOF

add the following config to crio:

cat > /etc/crio/crio.conf.d/00-plugin-dir.conf << EOF 
[crio.network]
plugin_dirs = [
  "/opt/cni/bin/",
]
EOF

enable cri-o:

systemctl enable crio 

install chrony:

pacman -S chrony --noconfirm

enable and start chrony:

systemctl enable chronyd

reboot machine:

reboot

install kubernetes

installing CNI:

CNI_VERSION="v1.3.0"
mkdir -p /opt/cni/bin
curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz

installing CRI:

CRICTL_VERSION="v1.27.0"
mkdir -p /opt/bin
curl -L "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz

installing kubeadm, kubelet, kubectl

RELEASE="$(curl -sSL https://dl.k8s.io/release/stable.txt)"
mkdir -p /opt/bin
cd /opt/bin
curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl}
chmod +x {kubeadm,kubelet,kubectl}
curl -sSL "https://raw.githubusercontent.com/kubernetes/release/master/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service
mkdir -p /etc/systemd/system/kubelet.service.d
curl -sSL "https://raw.githubusercontent.com/kubernetes/release/master/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf

create symbolic links for executables:

for u in $(ls|grep -v bins > bins && sed 'H;1h;$!d;x;s/\n/ /g' bins); do ln -s /opt/bin/$u /usr/local/bin/$u &&  chmod +x /usr/local/bin/$u; done

start kube cluster:

kubeadm init --pod-network-cidr 10.234.0.0/16 --apiserver-advertise-address=0.0.0.0 --cri-socket /var/run/crio/crio.sock --node-name archlinux.ovoxcloud.com

to start using your cluster, you need to run the following as a regular user:

mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

enable kubelet service:

systemctl enable kubelet

configure kubernetes

untaint the master so you can run pods

kubectl taint nodes --all node.kubernetes.io/not-ready:NoSchedule-
kubectl taint nodes --all node-role.kubernetes.io/control-plane:NoSchedule-

watch kubelet lot for errors:

journalctl -u kubelet -f

install calico operator:

kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/tigera-operator.yaml

download calico custom-resources:

wget https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/custom-resources.yaml

edit subnet on custom-resources yaml:

sed -i 's|192.168.0.0/16|10.234.0.0/16|g' custom-resources.yaml

create calico config:

kubectl create -f custom-resources.yaml

verify everthing is ok:

kubectl get nodes

if everything is ok you got this message:

NAME                      STATUS   ROLES    AGE     VERSION
archlinux.ovoxcloud.com   Ready    master   3m45s   v1.27.2

set up nginx-ingress:

kubectl apply -f https://raw.githubusercontent.com/vidalinux/kubernetes/main/ingress-nginx/deploy-nginx.yaml

deploy rook-ceph storage

create qcow2 images for rook-ceph:

qemu-img create -f qcow2 /static/rook/rook_disk1_300G.qcow2 300G
qemu-img create -f qcow2 /static/rook/rook_disk2_300G.qcow2 300G
qemu-img create -f qcow2 /static/rook/rook_disk3_300G.qcow2 300G

edit virtual machine:

virsh edit archlinux

add the following:

    <disk type='file' device='disk'>
     <driver name='qemu' type='qcow2'/>
     <source file='/static/rook/rook_disk1_300G.qcow2'/>
     <target dev='vdb' bus='virtio'/>
   </disk>
    <disk type='file' device='disk'>
     <driver name='qemu' type='qcow2'/>
     <source file='/static/rook/rook_disk2_300G.qcow2'/>
     <target dev='vdc' bus='virtio'/>
   </disk>
    <disk type='file' device='disk'>
     <driver name='qemu' type='qcow2'/>
     <source file='/static/rook/rook_disk3_300G.qcow2'/>
     <target dev='vdd' bus='virtio'/>
   </disk>

install devel package:

pacman -Sy base-devel --noconfirm

create normal user:

useradd -m linux

add user linux to sudoers:

cat > /etc/sudoers.d/10-linux << EOF
archy ALL=(ALL:ALL) ALL
EOF

installing yay as normal user:

su - linux
git clone https://aur.archlinux.org/yay.git
cd yay
makepkg -si

install ceph-bin and ceph-libs-bin:

yay -S ceph-libs-bin ceph-bin

load rbd kernel module:

depmod -a
modprobe rbd

clone rook git repo:

git clone --single-branch --branch v1.11.7 https://github.com/rook/rook.git

deploy rook-ceph operator:

cd rook/deploy/examples
kubectl create -f crds.yaml -f common.yaml -f operator.yaml

override rook config:

cat > override_poolsize.yaml << EOF
kind: ConfigMap
apiVersion: v1
metadata:
  name: rook-config-override
  namespace: rook-ceph
data:
  config: |
    [global]
    osd_pool_default_size = 1
---
EOF

deploy config:

kubectl create -f override_poolsize.yaml

edit the following settings on cluster.yaml:

  mon:
    count: 1
    allowMultiplePerNode: true
  mgr:
    count: 1
    allowMultiplePerNode: true 
  storage:
    config:
      osdsPerDevice: "1" 
    nodes:
      - name: "archlinux.ovoxcloud.com"
        devices: # specific devices to use for storage can be specified for each node
          - name: "vdb"
          - name: "vdc"
          - name: "vdd"

deploy cluster:

kubectl create -f cluster.yaml

verify installation status:

kubectl --namespace rook-ceph get cephclusters.ceph.rook.io rook-ceph

deploy storageclass:

kubectl create -f deploy/examples/csi/rbd/storageclass-test.yaml

edit cephfs storageclass yaml:

nano /root/rook/deploy/examples/csi/cephfs/storageclass.yaml

change the following settings:

 fsName: test-fs
 pool: test-fs-data0

deploy cephfs storageclass:

kubectl create -f /root/rook/deploy/examples/csi/cephfs/storageclass.yaml

test ceph-block storage:

kubectl create -f https://raw.githubusercontent.com/vidalinux/kubernetes/main/rook/ceph/ceph-block-pvc.yaml

verify ceph-block storage status:

kubectl -n default get pvc

test cephfs storage:

kubectl create -f https://raw.githubusercontent.com/vidalinux/kubernetes/main/rook/ceph/ceph-fs-pvc.yaml

verify cephfs storage status:

kubectl -n default get pvc

mount cephfs as filesystem:

cat > /usr/local/bin/mount-cephfs << EOF
#!/bin/bash

MON_HOST=$(kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- grep mon_host /etc/ceph/ceph.conf | cut -d " " -f 3 | tr -d '\r')
CEPH_SECRET=$(kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- grep key /etc/ceph/keyring | cut -d " " -f 3 | tr -d '\r')
MOUNT_DIR=/mnt/ceph-test

if [ ! -d ${MOUNT_DIR} ];
then
mkdir -p ${MOUNT_DIR}
fi

mount -t ceph -o mds_namespace=test-fs,name=admin,secret=${CEPH_SECRET} ${MON_HOST}:/ ${MOUNT_DIR}
EOF

fix mount-cephfs script permissions:

chmod +x /usr/local/bin/mount-cephfs

run script

mount-cephfs

confirm cephfs is mounted:

mount|grep /mnt/cephfs

if you want to remove rook-ceph deployment:

cd /root/rook/deploy/examples
kubectl delete -f cluster.yaml
kubectl delete -f crds.yaml -f common.yaml -f operator.yaml
rm -rf /var/lib/rook/*

deploy wordpress app

clone vidalinux kubernetes repo:

git clone https://github.com/vidalinux/kubernetes.git

edit settings on wordpress yaml:

     # wordpress container
       - name: WORDPRESS_DB_NAME
         value: "wordpressdb"
       - name: WORDPRESS_DB_USER
         value: "wordpress"
       - name: WORDPRESS_DB_PASSWORD
         value: "wordpress"
       - name: WORDPRESS_DB_HOST
         value: "wpvidalinux-db"
       - name: TZ
         value: "America/Puerto_Rico"
     # mariadb container
       - name: MYSQL_DATABASE
         value: wordpressdb
       - name: MYSQL_ROOT_PASSWORD
         value: root
       - name: MYSQL_USER
         value: wordpress
       - name: MYSQL_PASSWORD
         value: wordpress
       - name: TZ
         value: "America/Puerto_Rico"

get geniune ssl cert and key:

cat domain.crt |base64 -w 0
cat domain.key |base64 -w 0

use crt and key to create secret yaml:

echo > $PWD/secret.yaml << EOF
apiVersion: v1
kind: Secret
metadata:
  name: ovoxcloud.com-tls
  namespace: wpvidalinux
data:
  tls.crt: $(cat ovoxcloud.crt |base64 -w 0)
  tls.key: $(cat ovoxcloud.key |base64 -w 0)
type: kubernetes.io/tls
EOF

deploy secret yaml:

kubectl create -f secret.yaml

deploy wordpress app:

kubectl create -f wordpress/wordpress.yml

change name space:

kubectl config set-context --current --namespace=wpvidalinux-app

display pods running in current namespace:

kubectl get pods

show all pods running in all namespaces:

kubectl get pods -A

deploy seafile app

copy geniune ssl certifcates to seafile ssl directory:

cp domain.crt /root/kubernetes/seafile/ssl/seafile.domain.com.key
cp domain.key /root/kubernetes/seafile/ssl/seafile.domain.com.crt

edit configs on seafile yaml:

      # mariadb container
       - name: MYSQL_LOG_CONSOLE
         value: "true"
       - name: MYSQL_ROOT_PASSWORD
         value: "livinglavidalinux"
       - name: TZ
         value: "America/Puerto_Rico"
      # seafile container
       - name: DB_HOST
         value: "seafile-db"
       - name: DB_ROOT_PASSWD
         value: "livinglavidalinux"
       - name: SEAFILE_ADMIN_EMAIL
         value: "junior@ovoxcloud.com"
       - name: SEAFILE_ADMIN_PASSWORD
         value: "livinglavidalinux"
       - name: SEAFILE_SERVER_HOSTNAME
         value: "seafile.ovoxcloud.com"
       - name: SEAFILE_SERVER_LETSENCRYPT
         value: "true"
       - name: TZ
         value: "America/Puerto_Rico"

execute copy-certs.sh:

bash /root/kubernetes/seafile/copy-certs.sh

deploy seafile application using yaml:

kubectl create -f seafile.yaml

show container log:

kubectl logs seafile-srv-697c787f5c-px7cw

deploy awx ansible app

install the latest version of kustomize:

curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"  | bash

deploy awx tower operator:

cd /root/kubernetes/awx
kubectl apply -k .

look at the operator pod logs:

kubectl logs -f deployments/awx-operator-controller-manager -c awx-manager

use crt and key to create secret yaml:

echo > $PWD/secret.yaml << EOF
apiVersion: v1
kind: Secret
metadata:
  name: ovoxcloud.com-tls
  namespace: awx
data:
  tls.crt: $(cat ovoxcloud.crt |base64 -w 0)
  tls.key: $(cat ovoxcloud.key |base64 -w 0)
type: kubernetes.io/tls
EOF

deploy secret yaml:

kubectl create -f secret.yaml

deploy ingress yaml:

kubectl create -f ingress.yaml

get user admin password for web access:

kubectl -n awx get secret awx-tower-admin-password -o jsonpath="{.data.password}" -n awx | base64 --decode ; echo

access web interface:

https://awx.ovoxcloud.com

if you want to uninstall awx:

kustomize build '/root/kubernetes/awx' | kubectl delete -f -

helm

install helm:

curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash

renew certificates

verify certificates:

kubeadm certs check-expiration

renew all certificates:

kubeadm certs renew all

replace the config:

cp -i /etc/kubernetes/admin.conf $HOME/.kube/config

references

[latest version]

• https://github.com/kubernetes/release

[cni latest version]

• https://github.com/containernetworking/plugins/releases

[cri-tools latest version]

• https://github.com/kubernetes-sigs/cri-tools

[nginx supported versions]

• https://github.com/kubernetes/ingress-nginx#supported-versions-table

[single-node ceph cluster kubernetes]

• https://www.rusinov.ie/en/posts/2020/setting-up-single-node-ceph-cluster-for-kubernetes/