文章内容：

• 1、containerd的安装和使用
• 2、基于kubeadm和containerd部署单master k8s

containerd的安装和使用

1、二进制安装containerd

下载containerd

https://github.com/containerd/containerd

# cd /usr/local/src
# wget https://github.com/containerd/containerd/releases/download/v1.6.6/containerd-1.6.6-linux-amd64.tar.gz
# tar  xvf containerd-1.6.6-linux-amd64.tar.gz
# cp bin/* /usr/local/bin/

service文件

https://github.com/containerd/containerd/blob/main/containerd.service

# vim /lib/systemd/system/containerd.service

# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target


[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd


Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=infinity
# Comment TasksMax if your systemd version does not supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
OOMScoreAdjust=-999


[Install]
WantedBy=multi-user.target

核对二进制文件路径

ExecStart=/usr/local/bin/containerd

配置文件

mkdir /etc/containerd/
# containerd config default  > /etc/containerd/config.toml

镜像加速 （153行）

     [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
          endpoint = ["https://9916w1ow.mirror.aliyuncs.com"]

启动服务containerd

# systemctl restart containerd
# systemctl enable containerd
# systemctl status  containerd

部署runc

https://github.com/opencontainers/runc

# wget https://github.com/opencontainers/runc/releases/download/v1.1.3/runc.amd64
# chmod +x runc.amd64
# cp runc.amd64 /usr/bin/runc

下载镜像并运行容器

# ctr images pull docker.io/library/alpine:latest

查看镜像
# ctr images ls

# ctr run -t --net-host  docker.io/library/alpine:latest container sh

containerd客户端工具扩展

crictl

https://github.com/kubernetes-sigs/cri-tools

下载crictl

# wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.24.2/crictl-v1.24.2-linux-amd64.tar.gz
# tar zxf crictl-v1.24.2-linux-amd64.tar.gz
# cp crictl /usr/local/bin/

配置crictl运行环境

默认链接 unix:///var/run/docker.sock
但是containerd在以下路径/run/containerd/containerd.sock

修改配置文件

runtime-endpoint: "unix:///run/containerd/containerd.sock"
image-endpoint: "unix:///run/containerd/containerd.sock"
timeout: 10
debug: false

下载并验证镜像

# crictl  pull nginx:1.20.2
# crictl images list

nerdctl 推荐使用

https://github.com/containerd/nerdctl

# wget https://github.com/containerd/nerdctl/releases/download/v0.22.0/nerdctl-0.22.0-linux-amd64.tar.gz

# tar xvf nerdctl-0.22.0-linux-amd64.tar.gz
nerdctl
containerd-rootless-setuptool.sh
containerd-rootless.sh
# cp  nerdctl /usr/local/bin/

安装cni

https://github.com/containernetworking/plugins

# mkdir -p /opt/cni/bin
# tar xvf cni-plugins-linux-amd64-v1.1.1.tgz -C /opt/cni/bin/

创建容器并指定端口

# nerdctl run -d -p 80:80 --name=nginx-web1 --restart=always nginx

# nerdctl ps

基于kubeadm和containerd部署单master k8s v1.24.x

准备3台虚拟机

一、下载安装containerd

# cd /usr/local/src/
# wget https://github.com/containerd/containerd/releases/download/v1.6.6/containerd-1.6.6-linux-amd64.tar.gz
# tar xvf containerd-1.6.6-linux-amd64.tar.gz
# cp bin/* /usr/local/bin/

# vim /lib/systemd/system/containerd.service

# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target


[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd


Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=infinity
# Comment TasksMax if your systemd version does not supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
OOMScoreAdjust=-999


[Install]
WantedBy=multi-user.target

# mkdir /etc/containerd/
# containerd config default  > /etc/containerd/config.toml

镜像加速 （153行）

      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
          endpoint = ["https://9916w1ow.mirror.aliyuncs.com"]

sandbox_image(61行)

sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.7"

systemctl restart containerd && systemctl enable containerd && systemctl status containerd

部署runc

 # wget https://github.com/opencontainers/runc/releases/download/v1.1.3/runc.amd64
# chmod +x runc.amd64
# cp runc.amd64 /usr/bin/runc

二、安装kubeadm基础环境

2.1：安装kubeadm、kubectl、kubelet

Kubernetes镜像配置方法
https://developer.aliyun.com/mirror/kubernetes?spm=a2c6h.13651102.0.0.3e221b11n5yXhY

apt-get update && apt-get install -y apt-transport-https

curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add -
cat <<EOF >/etc/apt/sources.list.d/kubernetes.listdeb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF

apt-get update
apt-cache madison kubeadm

apt-get install -y kubelet=1.24.3-00 kubeadm=1.24.3-00 kubectl=1.24.3-00

安装nerdctl

# wget https://github.com/containerd/nerdctl/releases/download/v0.22.0/nerdctl-0.22.0-linux-amd64.tar.gz

# tar xvf nerdctl-0.22.0-linux-amd64.tar.gz
nerdctl
containerd-rootless-setuptool.sh
containerd-rootless.sh
# cp  nerdctl /usr/local/bin/

安装cni

# wget https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz
# mkdir -p /opt/cni/bin
# tar xvf cni-plugins-linux-amd64-v1.1.1.tgz -C /opt/cni/bin/

三、初始化kubernetes （在管理节点）

3.1、下载镜像

# kubeadm config images list --kubernetes-version v1.24.3

下载镜像 （ 注意镜像下载空间 ）
images-down.sh

#/bin/bash
nerdctl -n k8s.io pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver:v1.24.3
nerdctl -n k8s.io pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager:v1.24.3
nerdctl -n k8s.io pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:v1.24.3
nerdctl -n k8s.io pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.24.3
nerdctl -n k8s.io pull registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.7
nerdctl -n k8s.io pull registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.5.3-0
nerdctl -n k8s.io pull registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:v1.8.6

3.2:初始化k8s集群 （在管理节点）

kubeadm init --apiserver-advertise-address=192.168.3.150 --apiserver-bind-port=6443 --kubernetes-version=v1.24.3  --pod-network-cidr=10.100.0.0/16  --service-cidr=10.200.0.0/16 --service-dns-domain=cluster.local --image-repository=registry.aliyuncs.com/google_containers  --ignore-preflight-errors=swap

初始化报错 需要优化内核参数

root@k8s-master1:/usr/local/src# kubeadm init --apiserver-advertise-address=192.168.3.150 --apiserver-bind-port=6443 --kubernetes-version=v1.24.3  --pod-network-cidr=10.100.0.0/16  --service-cidr=10.200.0.0/16 --service-dns-domain=cluster.local --image-repository=registry.aliyuncs.com/google_containers  --ignore-preflight-errors=swap
[init] Using Kubernetes version: v1.24.3
[preflight] Running pre-flight checks
error execution phase preflight: [preflight] Some fatal errors occurred:
    [ERROR FileContent--proc-sys-net-bridge-bridge-nf-call-iptables]: /proc/sys/net/bridge/bridge-nf-call-iptables does not exist
    [ERROR FileContent--proc-sys-net-ipv4-ip_forward]: /proc/sys/net/ipv4/ip_forward contents are not set to 1
[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`
To see the stack trace of this error execute with --v=5 or higher

优化内核参数

modprobe br_netfilter
vim /etc/sysctl.conf
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1

Your Kubernetes control-plane has initialized successfully!




To start using your cluster, you need to run the following as a regular user:




  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config






Alternatively, if you are the root user, you can run:




  export KUBECONFIG=/etc/kubernetes/admin.conf




You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/




Then you can join any number of worker nodes by running the following on each as root:




kubeadm join 192.168.3.150:6443 --token 837nq7.v32nzy3t88tz6qbz \
    --discovery-token-ca-cert-hash sha256:5b76c3398e18b1e517deaaf4a27eab86537f800a0f72b03c32ff8fa55a05a00a

安装网络组件

kubectl apply -f calico-ipip.yaml

添加node节点

node节点内核优化

modprobe br_netfilter

vim /etc/sysctl.conf
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1

kubeadm join 192.168.3.150:6443 --token 837nq7.v32nzy3t88tz6qbz \
    --discovery-token-ca-cert-hash sha256:5b76c3398e18b1e517deaaf4a27eab86537f800a0f72b03c32ff8fa55a05a00a

kubectl get node
检查pod
kubectl get pod -A

root@k8s-master1:/usr/local/src# kubectl  get node
NAME                      STATUS   ROLES           AGE     VERSION
k8s-master1.example.com   Ready    control-plane   16m     v1.24.3
k8s-master2.example.com   Ready    <none>          2m49s   v1.24.3
k8s-master3.example.com   Ready    <none>          2m44s   v1.24.3

# nerdctl  -n k8s.io images

# kubectl  describe pod calico-kube-controllers-56cdb7c587-q7psp  -n kube-system

报错

Warning  FailedCreatePodSandBox  11m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "94674bddd9039d24174ef42e0afa37719b7e58a3d4460a9e326273a47847bc7d": plugin type="calico" failed (add): error creating calico client: stat /root/.kube/config: no such file or directory
  Warning  FailedCreatePodSandBox  2m56s (x38 over 10m)  kubelet            (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "55da7de2de11eb60b2fd864ee86f0e5ff5f7beb6589f540471fa848007553ed7": plugin type="calico" failed (add): error creating calico client: stat /root/.kube/config: no such file or directory

解决
node节点 kube config文件

mkdir -p /root/.kube
scp /root/.kube/config  192.168.3.152:/root/.kube/

root@k8s-master1:/usr/local/src# kubectl  get pod -A
NAMESPACE     NAME                                              READY   STATUS    RESTARTS   AGE
kube-system   calico-kube-controllers-56cdb7c587-q7psp          1/1     Running   0          24m
kube-system   calico-node-bzbfm                                 1/1     Running   0          17m
kube-system   calico-node-csmf7                                 1/1     Running   0          17m
kube-system   calico-node-s2ch7                                 1/1     Running   0          24m
kube-system   coredns-74586cf9b6-mjms6                          1/1     Running   0          30m
kube-system   coredns-74586cf9b6-zsrjf                          1/1     Running   0          30m
kube-system   etcd-k8s-master1.example.com                      1/1     Running   0          30m
kube-system   kube-apiserver-k8s-master1.example.com            1/1     Running   0          30m
kube-system   kube-controller-manager-k8s-master1.example.com   1/1     Running   0          30m
kube-system   kube-proxy-4xxcc                                  1/1     Running   0          17m
kube-system   kube-proxy-5dgw6                                  1/1     Running   0          17m
kube-system   kube-proxy-dppr7                                  1/1     Running   0          30m
kube-system   kube-scheduler-k8s-master1.example.com            1/1     Running   0          30m

测试及验证

k8s创建nginx服务

root@k8s-master1:/usr/local/src# kubectl create  ns myserver
namespace/myserver created
root@k8s-master1:/usr/local/src# kubectl  apply -f nginx.yaml
deployment.apps/myserver-nginx-deployment created
service/myserver-nginx-service created

root@k8s-master1:/usr/local/src# kubectl get pod -n myserver
NAME                                         READY   STATUS    RESTARTS   AGE
myserver-nginx-deployment-56f4ccb9bd-wqjxr   1/1     Running   0          61s