Ubuntu Desktop QEMU/KVM中使用Ubuntu Server 22.04配置k8s集群
1.本机部署个代理
这里我是自己用docker+sing-box 部署的 http://192.168.2.105:10808
如果 用v2rayN等软件部署的,记得开启局域网共享
2.安装虚拟机
我本机使用的是Ubuntu Desktop ,这里在QEMU/KVM 中安装 Ubuntu Server 22.04
可以去官网下载:https://cn.ubuntu.com/download/server/step1
具体安装步骤参照:https://blog.csdn.net/u010080562/article/details/127708329
安装后拷贝三个供安装集群使用
3. 通用配置
- 参照上边的教程 把ssh配置好
- ssh远程链接
- 配置hostname
hostnamectl set-hostname $host_name
hostname
sudo reboot
- 配置网络
# 网关
gateway4=192.168.122.1
# 静态IP地址
ip=192.168.122.4
# 上边的根据自己实际情况改
apt update && apt install iputils-ping vim net-tools netplan.io wget -y
sudo tee /etc/netplan/01-network-manager-all.yaml > /dev/null <<EOF
network:
version: 2
renderer: networkd
ethernets:
enp1s0:
addresses: [$ip/24]
gateway4: $gateway4
nameservers:
addresses: [8.8.8.8]
EOF
chmod 600 /etc/netplan/01-network-manager-all.yaml
netplan apply
- 配置防火墙
echo "配置防火墙"
sudo ufw allow 6443/tcp
sudo ufw allow 2379-2380/tcp
sudo ufw allow 10250-10259/tcp
- 安装docker
echo "==> 卸载旧版本 Docker(如果存在)..."
sudo apt remove -y docker docker-engine docker.io containerd runc || true
echo "==> 安装依赖工具..."
sudo apt update
sudo apt install -y ca-certificates gnupg lsb-release
echo "==> 添加 Docker GPG 密钥..."
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | \
sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
echo "==> 添加阿里云 Docker 软件源..."
UBUNTU_CODENAME=$(lsb_release -cs)
echo \
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \
https://mirrors.aliyun.com/docker-ce/linux/ubuntu ${UBUNTU_CODENAME} stable" | \
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
echo "==> 更新软件源并安装 Docker..."
sudo apt update
sudo apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
echo "==> 启动并设置 Docker 开机自启..."
sudo systemctl enable docker
sudo systemctl start docker
echo "==> 配置国内镜像加速器(阿里云)..."
sudo mkdir -p /etc/docker
cat <<EOF | sudo tee /etc/docker/daemon.json
{
"registry-mirrors": [
"https://docker.m.daocloud.io",
"https://docker.imgdb.de",
"https://docker-0.unsee.tech",
"https://docker.hlmirror.com",
"https://docker.1ms.run",
"https://func.ink",
"https://lispy.org",
"https://docker.xiaogenban1993.com"
]
}
EOF
# 配置拉取镜像的网络
sudo mkdir -p /etc/systemd/system/docker.service.d && \
echo -e \
"[Service]\nEnvironment=\"HTTP_PROXY=$proxy\"\nEnvironment=\"HTTPS_PROXY=$proxy\"\nEnvironment=\"NO_PROXY=localhost,127.0.0.1,::1,10.0.0.0/8,192.168.0.0/16\"" \
| sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf && \
sudo systemctl daemon-reexec && \
sudo systemctl daemon-reload && \
sudo systemctl restart docker && \
docker info | grep -i proxy
echo "==> 重新加载配置并重启 Docker..."
sudo systemctl daemon-reexec
sudo systemctl restart docker
echo " Docker 安装完成!版本信息如下:"
docker --version
- 安装k8s
echo "==> 设置 kubeadm 软件源"
sudo apt-get update
sudo apt install -y apt-transport-https ca-certificates
curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | sudo apt-key add -
cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF
echo "==> 安装 kubelet、kubeadm 和 kubectl,并锁定其版本"
sudo apt-get update
sudo apt-get install -y kubelet=1.23.15-00 kubeadm=1.23.15-00 kubectl=1.23.15-00
sudo apt-mark hold kubelet kubeadm kubectl
# 加载 br_netfilter 内核模块
sudo modprobe br_netfilter
sudo sysctl -w net.bridge.bridge-nf-call-iptables=1
sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1
echo "br_netfilter" | sudo tee /etc/modules-load.d/br_netfilter.conf
echo "net.bridge.bridge-nf-call-iptables=1" | sudo tee /etc/sysctl.d/k8s.conf
echo "net.bridge.bridge-nf-call-ip6tables=1" | sudo tee -a /etc/sysctl.d/k8s.conf
sudo sysctl --system
sudo kubeadm reset -f
sudo rm -rf /etc/kubernetes/pki/ # 强制删除残留证书
sudo rm -rf /var/lib/kubelet/
sudo rm -f $HOME/.kube/config
sudo sed -i '/\/swap.img/ { /^[^#]/ s/^/#/ }' /etc/fstab
sudo swapoff -a
free -h
至此 通用配置完成
4. master配置
ip=192.168.122.4
# 加载 br_netfilter 内核模块
sudo modprobe br_netfilter
# 设置 sysctl 参数
sudo sysctl -w net.bridge.bridge-nf-call-iptables=1
sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1
# 持久化配置(重启后生效)
echo "br_netfilter" | sudo tee /etc/modules-load.d/br_netfilter.conf
echo "net.bridge.bridge-nf-call-iptables=1" | sudo tee /etc/sysctl.d/k8s.conf
echo "net.bridge.bridge-nf-call-ip6tables=1" | sudo tee -a /etc/sysctl.d/k8s.conf
# 应用 sysctl 配置
sudo sysctl --system
# 使用阿里云镜像仓库拉取镜像
kubeadm config images pull \
--image-repository registry.aliyuncs.com/google_containers \
--kubernetes-version v1.23.15
# 初始化集群
sudo kubeadm init \
--image-repository registry.aliyuncs.com/google_containers \
--kubernetes-version v1.23.15 \
--pod-network-cidr=10.244.0.0/16 \
--apiserver-advertise-address=$ip \
--ignore-preflight-errors=FileContent--proc-sys-net-bridge-bridge-nf-call-iptables,SystemVerification
# 配置 kubectl
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
# 安装 Pod 网络
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
kubectl apply -f kube-flannel.yml
5. worker 添加到集群里
在master节点执行 命令 将结果粘贴到worker节点执行
kubeadm token create --print-join-command
如成功 则输出如图
如出错 重新执行 命令 清理缓存 处理问题 重新执行上边master输出的命令
# 清理之前的残留配置
sudo kubeadm reset -f
sudo rm -rf /etc/kubernetes/pki/ # 强制删除残留证书
sudo rm -rf /var/lib/kubelet/
sudo rm -f $HOME/.kube/config
sudo sed -i '/\/swap.img/ { /^[^#]/ s/^/#/ }' /etc/fstab
sudo swapoff -a
free -h
至此 所有安装配置完成,重复worker安装步骤在其他节点上即可
成果展示
常用命令记录:
# 查看集群节点
kubectl get nodes
# 删除集群节点
kubectl delete nodes 节点名
# 修改节点标签名
kubectl label node worker02 node-role.kubernetes.io/worker02=
shell脚本中加入 set -e 启动严格模式 碰到命令输出错误直接终止
问题记录:
- KVM 虚拟机 网卡问题
sudo virsh net-list --all
sudo virsh net-start default
sudo virsh net-list --all
- master not ready 问题处理 ----CNI 未正确启动(一般都是 kube-flannel 里的镜像拉取有问题)
# 配置代理
sudo mkdir -p /etc/systemd/system/docker.service.d && \
echo -e \
"[Service]\nEnvironment=\"HTTP_PROXY=http://192.168.2.105:10808\"\nEnvironment=\"HTTPS_PROXY=http://192.168.2.105:10808\"\nEnvironment=\"NO_PROXY=localhost,127.0.0.1,::1,10.0.0.0/8,192.168.0.0/16\"" \
| sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf && \
sudo systemctl daemon-reexec && \
sudo systemctl daemon-reload && \
sudo systemctl restart docker && \
docker info | grep -i proxy
# 安装 CNI
wget https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz && \
mv cni-plugins-linux-amd64-v0.8.6.tgz /opt/cni/bin/ && \
tar -zxvf cni-plugins-linux-amd64-v0.8.6.tgz
# 手动拉取镜像
docker pull ghcr.io/flannel-io/flannel:v0.26.7 && \
docker tag ghcr.io/flannel-io/flannel:v0.26.7 quay.io/coreos/flannel:v0.26.7 && \
docker images | grep flannel && \
wget https://raw.githubusercontent.com/flannel-io/flannel/v0.26.7/Documentation/kube-flannel.yml -O kube-flannel.yml
# vim进去修改
'''
containers:
- name: kube-flannel
image: quay.io/coreos/flannel:v0.26.7
'''
kubectl apply -f kube-flannel.yml && \
kubectl get pods -n kube-flannel -o wide && \
kubectl get nodes
检查代理网络是否可用 执行命令看输出即可
curl https://www.youtube.com/
代理不好使会一直卡住
代理好使会输出一堆html信息
第一次ssh链接 不知道ip可以在KVM中查看