二进制安装k8s 1.25.2 高可用集群

一、环境准备

1、环境说明(安装时配置IP及主机名）

序号主机IP主机名系统备注
1192.168.3.101master1rockylinux8.6最小化安装控制节点
2192.168.3.102master2rockylinux8.6最小化安装控制节点
3192.168.3.103master3rockylinux8.6最小化安装控制节点
4192.168.3.104node1rockylinux8.6最小化安装工作节点
5192.168.3.105node2rockylinux8.6最小化安装工作节点

2、关闭selinux,firewalld及swap分区(在五台设备上执行）

sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
systemctl disable firewalld
swapoff -a
reboot

注：swapoff -a 为临时关闭swap分区。永久关闭swap分区，vi /etc/fstab  注释swap分区一行

3、修改/etc/hosts文件，增加如下三行（五台设备）

cat << EOF >> /etc/hosts
192.168.3.101 master1
192.168.3.102 master2
192.168.3.103 master3
192.168.3.104 node1
192.168.3.105 node2
EOF

4、更改yum源为阿里云（五台设备执行）

sed -e 's|^mirrorlist=|#mirrorlist=|g' \
    -e 's|^#baseurl=http://dl.rockylinux.org/$contentdir|baseurl=https://mirrors.aliyun.com/rockylinux|g' \
    -i.bak \
    /etc/yum.repos.d/Rocky-*.repo

dnf makecache

5、配置命令补全及vim工具（五台设备执行）

dnf install -y wget bash-completion vim

6、配置免密登录（三台master上执行，可省）

ssh-keygen

for host in { master1 master2 master3 node1 node2 };do ssh-copy-id $host;done

7、配置时间同步（五台设备执行）

dnf install -y chrony

更改  /etc/chrony.conf  配置文件

将pool 2.pool.ntp.org iburst

改为

server ntp1.aliyun.com iburst
server ntp2.aliyun.com iburst
server ntp1.tencent.com iburst
server ntp2.tencent.com iburst

systemctl enable --now chronyd
chronyc sources

for host in { master1 master2 master3 node1 node2 };do ssh $host  date;done

8、修改内核参数（五台设备上执行）

modprobe br_netfilter
lsmod | grep br_netfilter

cat > /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF

sysctl -p /etc/sysctl.d/k8s.conf

 9、安装依整包及配置docker源、k8s源（五台设备执行）

dnf install -y yum-utils device-mapper-persistent-data lvm2 ipvsadm net-tools
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
sed -i 's+download.docker.com+mirrors.aliyun.com/docker-ce+' /etc/yum.repos.d/docker-ce.repo

cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

dnf makecache

10、开启Ipvs 五台设备

lsmod|grep ip_vs
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh

lsmod|grep ip_vs

modprobe br_netfilter
echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables
echo 1 > /proc/sys/net/ipv4/ip_forward

11、安装containerd（五台设备执行）

dnf install -y containerd

containerd config default > /etc/containerd/config.toml

更改配置文件

sed -i 's#SystemdCgroup = false#SystemdCgroup = true#g' /etc/containerd/config.toml
sed -i "s#k8s.gcr.io/pause#registry.cn-hangzhou.aliyuncs.com/google_containers/pause#g"  /etc/containerd/config.toml

配置镜像加速

sed -i '/registry.mirrors]/a\ \ \ \ \ \ \ \ [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]' /etc/containerd/config.toml
sed -i '/registry.mirrors."docker.io"]/a\ \ \ \ \ \ \ \ \ \ endpoint = ["https://0x3urqgf.mirror.aliyuncs.com"]' /etc/containerd/config.toml

启动containerd

systemctl enable --now containerd.service
systemctl status containerd.service

二、安装kubernetes 组件

1、安装ETCD

wget https://github.com/cloudflare/cfssl/releases/download/v1.6.1/cfssl_1.6.1_linux_amd64
wget https://github.com/cloudflare/cfssl/releases/download/v1.6.1/cfssl-certinfo_1.6.1_linux_amd64
wget https://github.com/cloudflare/cfssl/releases/download/v1.6.1/cfssljson_1.6.1_linux_amd64
mv cfssl_1.6.1_linux_amd64 /usr/local/bin/cfssl
mv cfssljson_1.6.1_linux_amd64 /usr/local/bin/cfssljson
mv cfssl-certinfo_1.6.1_linux_amd64 /usr/local/bin/cfssl-certinfo
chmod +x /usr/local/bin/cfssl*
mkdir /cfspki
mkdir -p /etc/etcd/pki
mkdir -p /var/lib/etcd/default.etcd
cd /cfspki/
cat > ca-csr.json  << EOF
{
        "CN": "kubernetes",
        "key": {
                "algo": "rsa",
                "size": 2048
},
        "names": [
        {
        "C": "CN",
        "ST": "Xinjiang",
        "L": "Urumqi",
        "O": "k8s",
        "OU": "system"
        }
],
        "ca": {
                "expiry": "87600h"
        }
}

EOF

cfssl gencert -initca ca-csr.json | cfssljson -bare ca

cat >  ca-config.json  << EOF
{
    "signing": {
        "default": {
            "expiry": "87600h"
    },
    "profiles": {
        "kubernetes": {
            "usages": [
                "signing",
                "key encipherment",
                "server auth",
                "client auth"
            ],
            "expiry": "87600h"
            }
        }
    }
}

EOF

cat > etcd-csr.json  << EOF
{
"CN": "etcd",
"hosts": [
    "127.0.0.1",
    "192.168.3.101",
    "192.168.3.102",
    "192.168.3.103",
    "192.168.3.110"
],
    "key": {
    "algo": "rsa",
    "size": 2048
},
    "names": [{
    "C": "CN",
    "ST": "Xinjiang",
    "L": "Urumqi",
    "O": "k8s",
    "OU": "system"
    }]
}

EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
ls etcd*.pem
cp ca*.pem /etc/etcd/pki/
cp etcd*.pem /etc/etcd/pki/
cd

wget https://github.com/etcd-io/etcd/releases/download/v3.4.21/etcd-v3.4.21-linux-amd64.tar.gz
tar -xvf etcd-v3.4.21-linux-amd64.tar.gz
cp -ar etcd-v3.4.21-linux-amd64/etcd* /usr/local/bin/
chmod +x /usr/local/bin/etcd*

cat > /etc/etcd/etcd.conf << EOF
#[Member]
ETCD_NAME="etcd1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.3.101:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.3.101:2379,http://127.0.0.1:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.3.101:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.3.101:2379"
ETCD_INITIAL_CLUSTER="etcd1=https://192.168.3.101:2380,etcd2=https://192.168.3.102:2380,etcd3=https://192.168.3.103:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF

cat > /usr/lib/systemd/system/etcd.service << EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=-/etc/etcd/etcd.conf
WorkingDirectory=/var/lib/etcd/
ExecStart=/usr/local/bin/etcd \\
--cert-file=/etc/etcd/pki/etcd.pem \\
--key-file=/etc/etcd/pki/etcd-key.pem \\
--trusted-ca-file=/etc/etcd/pki/ca.pem \\
--peer-cert-file=/etc/etcd/pki/etcd.pem \\
--peer-key-file=/etc/etcd/pki/etcd-key.pem \\
--peer-trusted-ca-file=/etc/etcd/pki/ca.pem \\
--peer-client-cert-auth \\
--client-cert-auth
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

EOF

scp -r etcd-v3.4.21-linux-amd64/etcd* master2:/usr/local/bin/
scp -r etcd-v3.4.21-linux-amd64/etcd* master3:/usr/local/bin/
scp /usr/lib/systemd/system/etcd.service master2:/usr/lib/systemd/system/
scp /usr/lib/systemd/system/etcd.service master3:/usr/lib/systemd/system/
systemctl daemon-reload
systemctl start etcd
systemctl enable etcd

 master2 

mkdir -p /etc/etcd/pki
mkdir -p /var/lib/etcd/default.etcd
chmod +x /usr/local/bin/etcd*

cat > /etc/etcd/etcd.conf << EOF
#[Member]
ETCD_NAME="etcd2"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.3.102:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.3.102:2379,http://127.0.0.1:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.3.102:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.3.102:2379"
ETCD_INITIAL_CLUSTER="etcd1=https://192.168.3.101:2380,etcd2=https://192.168.3.102:2380,etcd3=https://192.168.3.103:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF

scp master1:/etc/etcd/pki/*.pem /etc/etcd/pki/
systemctl daemon-reload
systemctl start etcd
systemctl enable etcd

master3

mkdir -p /etc/etcd/pki
mkdir -p /var/lib/etcd/default.etcd
chmod +x /usr/local/bin/etcd*

cat > /etc/etcd/etcd.conf << EOF
#[Member]
ETCD_NAME="etcd3"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.3.103:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.3.103:2379,http://127.0.0.1:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.3.103:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.3.103:2379"
ETCD_INITIAL_CLUSTER="etcd1=https://192.168.3.101:2380,etcd2=https://192.168.3.102:2380,etcd3=https://192.168.3.103:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF

scp master1:/etc/etcd/pki/*.pem /etc/etcd/pki/
systemctl daemon-reload
systemctl start etcd
systemctl enable etcd

etcd状态检查  master1

/usr/local/bin/etcdctl --write-out=table --cacert=/etc/etcd/pki/ca.pem \
--cert=/etc/etcd/pki/etcd.pem --key=/etc/etcd/pki/etcd-key.pem \
--endpoints=https://192.168.3.101:2379,https://192.168.3.102:2379,\
https://192.168.3.103:2379 endpoint  health

2、安装 apiserver

wget https://dl.k8s.io/v1.25.3/kubernetes-server-linux-amd64.tar.gz
tar -xvf kubernetes-server-linux-amd64.tar.gz
cd kubernetes/server/bin/
cp kube-apiserver kube-controller-manager kube-scheduler kubectl /usr/local/bin/
scp kube-apiserver kube-controller-manager kube-scheduler kubectl master2:/usr/local/bin/
scp kube-apiserver kube-controller-manager kube-scheduler kubectl master3:/usr/local/bin/
scp kubelet kube-proxy node1:/usr/local/bin/
scp kubelet kube-proxy node2:/usr/local/bin/
mkdir -p /etc/kubernetes/pki
mkdir /var/log/kubernetes
cd /cfspki/
cat > token.csv << EOF
$(head -c 16 /dev/urandom | od -An -t x | tr -d ' '),kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF

cat > kube-apiserver-csr.json << EOF
{
	"CN": "kubernetes",
	"hosts": [
	    "127.0.0.1",
	    "192.168.3.101",
	    "192.168.3.102",
	    "192.168.3.103",
	    "192.168.3.110",
	    "10.10.0.1",
	    "kubernetes",
	    "kubernetes.default",
	    "kubernetes.default.svc",
	    "kubernetes.default.svc.cluster",
	    "kubernetes.default.svc.cluster.local"
	],
	"key": {
	    "algo": "rsa",
	    "size": 2048
	},
	"names": [
	{
	    "C": "CN",
	    "ST": "Xinjiang",
	    "L": "Urumqi",
	    "O": "k8s",
	    "OU": "system"
	}
        ]
}

EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-apiserver-csr.json | cfssljson -bare kube-apiserver
cp ca*.pem kube-apiserver*.pem /etc/kubernetes/pki
cp token.csv /etc/kubernetes/
cd
ls /etc/kubernetes/pki/


cat >  /etc/kubernetes/kube-apiserver.conf << EOF
KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \\
--anonymous-auth=false \\
--bind-address=192.168.3.101 \\
--secure-port=6443 \\
--advertise-address=192.168.3.101 \\
--authorization-mode=Node,RBAC \\
--runtime-config=api/all=true \\
--enable-bootstrap-token-auth \\
--service-cluster-ip-range=10.10.0.0/16 \\
--token-auth-file=/etc/kubernetes/token.csv \\
--service-node-port-range=30000-50000 \\
--tls-cert-file=/etc/kubernetes/pki/kube-apiserver.pem \\
--tls-private-key-file=/etc/kubernetes/pki/kube-apiserver-key.pem \\
--client-ca-file=/etc/kubernetes/pki/ca.pem \\
--kubelet-client-certificate=/etc/kubernetes/pki/kube-apiserver.pem \\
--kubelet-client-key=/etc/kubernetes/pki/kube-apiserver-key.pem \\
--service-account-key-file=/etc/kubernetes/pki/ca-key.pem \\
--service-account-signing-key-file=/etc/kubernetes/pki/ca-key.pem \\
--service-account-issuer=https://kubernetes.default.svc.cluster.local \\
--etcd-cafile=/etc/etcd/pki/ca.pem \\
--etcd-certfile=/etc/etcd/pki/etcd.pem \\
--etcd-keyfile=/etc/etcd/pki/etcd-key.pem \\
--etcd-servers=https://192.168.3.101:2379,https://192.168.3.102:2379,https://192.168.3.103:2379 \\
--allow-privileged=true \\
--apiserver-count=3 \\
--audit-log-maxage=30 \\
--audit-log-maxbackup=3 \\
--audit-log-maxsize=100 \\
--audit-log-path=/var/log/kube-apiserver-audit.log \\
--event-ttl=1h \\
--alsologtostderr=true \\
--logtostderr=false \\
--log-dir=/var/log/kubernetes \\
--v=4"

EOF

cat > /usr/lib/systemd/system/kube-apiserver.service  << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=etcd.service
Wants=etcd.service

[Service]
EnvironmentFile=-/etc/kubernetes/kube-apiserver.conf
ExecStart=/usr/local/bin/kube-apiserver \$KUBE_APISERVER_OPTS
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

EOF

scp /usr/lib/systemd/system/kube-apiserver.service master2:/usr/lib/systemd/system/
scp /usr/lib/systemd/system/kube-apiserver.service master3:/usr/lib/systemd/system/
systemctl daemon-reload
systemctl start kube-apiserver
systemctl enable kube-apiserver
systemctl status kube-apiserver

master2

mkdir -p /etc/kubernetes/pki
mkdir /var/log/kubernetes
scp master1:/etc/kubernetes/token.csv  /etc/kubernetes/
scp master1:/etc/kubernetes/pki/*  /etc/kubernetes/pki/

cat >  /etc/kubernetes/kube-apiserver.conf << EOF
KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \\
--anonymous-auth=false \\
--bind-address=192.168.3.102 \\
--secure-port=6443 \\
--advertise-address=192.168.3.102 \\
--authorization-mode=Node,RBAC \\
--runtime-config=api/all=true \\
--enable-bootstrap-token-auth \\
--service-cluster-ip-range=10.10.0.0/16 \\
--token-auth-file=/etc/kubernetes/token.csv \\
--service-node-port-range=30000-50000 \\
--tls-cert-file=/etc/kubernetes/pki/kube-apiserver.pem \\
--tls-private-key-file=/etc/kubernetes/pki/kube-apiserver-key.pem \\
--client-ca-file=/etc/kubernetes/pki/ca.pem \\
--kubelet-client-certificate=/etc/kubernetes/pki/kube-apiserver.pem \\
--kubelet-client-key=/etc/kubernetes/pki/kube-apiserver-key.pem \\
--service-account-key-file=/etc/kubernetes/pki/ca-key.pem \\
--service-account-signing-key-file=/etc/kubernetes/pki/ca-key.pem \\
--service-account-issuer=https://kubernetes.default.svc.cluster.local \\
--etcd-cafile=/etc/etcd/pki/ca.pem \\
--etcd-certfile=/etc/etcd/pki/etcd.pem \\
--etcd-keyfile=/etc/etcd/pki/etcd-key.pem \\
--etcd-servers=https://192.168.3.101:2379,https://192.168.3.102:2379,https://192.168.3.103:2379 \\
--allow-privileged=true \\
--apiserver-count=3 \\
--audit-log-maxage=30 \\
--audit-log-maxbackup=3 \\
--audit-log-maxsize=100 \\
--audit-log-path=/var/log/kube-apiserver-audit.log \\
--event-ttl=1h \\
--alsologtostderr=true \\
--logtostderr=false \\
--log-dir=/var/log/kubernetes \\
--v=4"

EOF

systemctl daemon-reload
systemctl start kube-apiserver
systemctl enable kube-apiserver
systemctl status kube-apiserver

master3

mkdir -p /etc/kubernetes/pki
mkdir /var/log/kubernetes
scp master1:/etc/kubernetes/token.csv  /etc/kubernetes/
scp master1:/etc/kubernetes/pki/*  /etc/kubernetes/pki/

cat >  /etc/kubernetes/kube-apiserver.conf << EOF
KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \\
--anonymous-auth=false \\
--bind-address=192.168.3.103 \\
--secure-port=6443 \\
--advertise-address=192.168.3.103 \\
--authorization-mode=Node,RBAC \\
--runtime-config=api/all=true \\
--enable-bootstrap-token-auth \\
--service-cluster-ip-range=10.10.0.0/16 \\
--token-auth-file=/etc/kubernetes/token.csv \\
--service-node-port-range=30000-50000 \\
--tls-cert-file=/etc/kubernetes/pki/kube-apiserver.pem \\
--tls-private-key-file=/etc/kubernetes/pki/kube-apiserver-key.pem \\
--client-ca-file=/etc/kubernetes/pki/ca.pem \\
--kubelet-client-certificate=/etc/kubernetes/pki/kube-apiserver.pem \\
--kubelet-client-key=/etc/kubernetes/pki/kube-apiserver-key.pem \\
--service-account-key-file=/etc/kubernetes/pki/ca-key.pem \\
--service-account-signing-key-file=/etc/kubernetes/pki/ca-key.pem \\
--service-account-issuer=https://kubernetes.default.svc.cluster.local \\
--etcd-cafile=/etc/etcd/pki/ca.pem \\
--etcd-certfile=/etc/etcd/pki/etcd.pem \\
--etcd-keyfile=/etc/etcd/pki/etcd-key.pem \\
--etcd-servers=https://192.168.3.101:2379,https://192.168.3.102:2379,https://192.168.3.103:2379 \\
--allow-privileged=true \\
--apiserver-count=3 \\
--audit-log-maxage=30 \\
--audit-log-maxbackup=3 \\
--audit-log-maxsize=100 \\
--audit-log-path=/var/log/kube-apiserver-audit.log \\
--event-ttl=1h \\
--alsologtostderr=true \\
--logtostderr=false \\
--log-dir=/var/log/kubernetes \\
--v=4"

EOF

systemctl daemon-reload
systemctl start kube-apiserver
systemctl enable kube-apiserver
systemctl status kube-apiserver

验证  master1

curl --insecure https://192.168.3.101:6443/

3、安装kubelet

cd /cfspki/
cat  > admin-csr.json << EOF
{
    "CN": "admin",
    "hosts":[],
    "key":{
    "algo": "rsa",
    "size": 2048
},
    "names": [
{
    "C": "CN",
    "ST": "Xinjiang",
    "L": "Urumqi",
    "O": "system:masters",
    "OU": "system"
    }
  ]
}

EOF


cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
cp admin*.pem /etc/kubernetes/pki/
kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true \
--server=https://192.168.3.101:6443 --kubeconfig=kube.config
kubectl config set-credentials admin --client-certificate=admin.pem \
--client-key=admin-key.pem --embed-certs=true --kubeconfig=kube.config
kubectl config set-context kubernetes --cluster=kubernetes --user=admin \
--kubeconfig=kube.config
kubectl config use-context kubernetes --kubeconfig=kube.config
mkdir ~/.kube -p
cp kube.config ~/.kube/config
cp kube.config /etc/kubernetes/admin.conf
kubectl create clusterrolebinding kube-apiserver:kubelet-apis \
--clusterrole=system:kubelet-api-admin --user kubernetes
kubectl cluster-info
kubectl get componentstatuses
kubectl get all --all-namespaces


source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)
kubectl completion bash > ~/.kube/completion.bash.inc
source '/root/.kube/completion.bash.inc'
source $HOME/.bash_profile

 master2  master3:

mkdir /root/.kube/
scp master1:/root/.kube/config ./kube

4、安装kube-controller-manager

cd /cfspki
cat >  kube-controller-manager-csr.json  << EOF
{
    "CN": "system:kube-controller-manager",
    "key": {
    "algo": "rsa",
    "size": 2048
    },
    "hosts": [
    "127.0.0.1",
    "192.168.3.101",
    "192.168.3.102",
    "192.168.3.103",
    "192.168.3.110"
    ],
    "names": [
    {
    "C": "CN",
    "ST": "Xinjiang",
    "L": "Urumqi",
    "O": "system:kube-controller-manager",
    "OU": "system"
    }
  ]
}

EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json \
-profile=kubernetes kube-controller-manager-csr.json | cfssljson \
-bare kube-controller-manager
kubectl config set-cluster kubernetes --certificate-authority=ca.pem \
--embed-certs=true --server=https://192.168.3.101:6443 \
--kubeconfig=kube-controller-manager.kubeconfig
kubectl config set-credentials system:kube-controller-manager \
--client-certificate=kube-controller-manager.pem \
--client-key=kube-controller-manager-key.pem --embed-certs=true \
--kubeconfig=kube-controller-manager.kubeconfig
kubectl config set-context system:kube-controller-manager --cluster=kubernetes \
--user=system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig
kubectl config use-context system:kube-controller-manager \
--kubeconfig=kube-controller-manager.kubeconfig
cp kube-controller-manager*.pem /etc/kubernetes/pki/
cp kube-controller-manager.kubeconfig /etc/kubernetes/
cd

cat > /usr/lib/systemd/system/kube-controller-manager.service  << EOF
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/etc/kubernetes/kube-controller-manager.conf
ExecStart=/usr/local/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target

EOF

cat > /etc/kubernetes/kube-controller-manager.conf  << EOF
KUBE_CONTROLLER_MANAGER_OPTS="--secure-port=10257  \\
--bind-address=127.0.0.1  \\
--kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig  \\
--service-cluster-ip-range=10.10.0.0/16  \\
--cluster-name=kubernetes \\
--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem \\
--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem \\
--allocate-node-cidrs=true \\
--cluster-cidr=10.0.0.0/16 \\
--root-ca-file=/etc/kubernetes/pki/ca.pem \\
--service-account-private-key-file=/etc/kubernetes/pki/ca-key.pem \\
--leader-elect=true \\
--feature-gates=RotateKubeletServerCertificate=true \\
--controllers=*,bootstrapsigner,tokencleaner \\
--horizontal-pod-autoscaler-sync-period=10s \\
--tls-cert-file=/etc/kubernetes/pki/kube-controller-manager.pem \\
--tls-private-key-file=/etc/kubernetes/pki/kube-controller-manager-key.pem \\
--use-service-account-credentials=true \\
--alsologtostderr=true \\
--logtostderr=false \\
--log-dir=/var/log/kubernetes \\
--v=2"

EOF

scp /etc/kubernetes/pki/kube-controller-manager*.pem master2:/etc/kubernetes/pki/
scp /etc/kubernetes/kube-controller-manager.kubeconfig master2:/etc/kubernetes/
scp /etc/kubernetes/kube-controller-manager.conf master2:/etc/kubernetes/
scp /usr/lib/systemd/system/kube-controller-manager.service master2:/usr/lib/systemd/system/
scp /etc/kubernetes/pki/kube-controller-manager*.pem master3:/etc/kubernetes/pki/
scp /etc/kubernetes/kube-controller-manager.kubeconfig master3:/etc/kubernetes/
scp /etc/kubernetes/kube-controller-manager.conf master3:/etc/kubernetes/
scp /usr/lib/systemd/system/kube-controller-manager.service master3:/usr/lib/systemd/system/

systemctl daemon-reload
systemctl start kube-controller-manager
systemctl enable kube-controller-manager
systemctl status kube-controller-manager

master2 master3

systemctl daemon-reload
systemctl start kube-controller-manager
systemctl enable kube-controller-manager
systemctl status kube-controller-manager

5、安装kube-scheduler

cd /cfspki
cat >  kube-scheduler-csr.json  << EOF
{
    "CN": "system:kube-scheduler",
    "hosts": [
    "127.0.0.1",
    "192.168.3.101",
    "192.168.3.102",
    "192.168.3.103",
    "192.168.3.110"
  ],
    "key": {
    "algo": "rsa",
    "size": 2048
  },
    "names": [
  {
    "C": "CN",
    "ST": "Xinjiang",
    "L": "Urumqi",
    "O": "system:kube-scheduler",
    "OU": "system"
    }
  ]
}

EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json \
-profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler
kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true \
--server=https://192.168.3.101:6443 --kubeconfig=kube-scheduler.kubeconfig
kubectl config set-credentials system:kube-scheduler \
--client-certificate=kube-scheduler.pem --client-key=kube-scheduler-key.pem \
--embed-certs=true  --kubeconfig=kube-scheduler.kubeconfig
kubectl config set-context system:kube-scheduler --cluster=kubernetes \
--user=system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig
kubectl config use-context system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig
cp kube-scheduler*.pem /etc/kubernetes/pki/
cp kube-scheduler.kubeconfig /etc/kubernetes/

cat >  /etc/kubernetes/kube-scheduler.conf  << EOF
KUBE_SCHEDULER_OPTS="--kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \\
--leader-elect=true \\
--alsologtostderr=true \\
--logtostderr=false \\
--log-dir=/var/log/kubernetes \\
--v=2"

EOF

cat > /usr/lib/systemd/system/kube-scheduler.service  << EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/etc/kubernetes/kube-scheduler.conf
ExecStart=/usr/local/bin/kube-scheduler \$KUBE_SCHEDULER_OPTS
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target

EOF

scp /etc/kubernetes/pki/kube-scheduler*.pem master2:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/kube-scheduler*.pem master3:/etc/kubernetes/pki/
scp /etc/kubernetes/kube-scheduler.conf master2:/etc/kubernetes/
scp /etc/kubernetes/kube-scheduler.conf master3:/etc/kubernetes/
scp /etc/kubernetes/kube-scheduler.kubeconfig master2:/etc/kubernetes/
scp /etc/kubernetes/kube-scheduler.kubeconfig master3:/etc/kubernetes/
scp /usr/lib/systemd/system/kube-scheduler.service master2:/usr/lib/systemd/system/
scp /usr/lib/systemd/system/kube-scheduler.service master3:/usr/lib/systemd/system/

systemctl daemon-reload
systemctl start kube-scheduler
systemctl enable kube-scheduler
systemctl status kube-scheduler

master2  master3

systemctl daemon-reload
systemctl start kube-scheduler
systemctl enable kube-scheduler
systemctl status kube-scheduler

6、安装kubelet

cd /cfspki/
BOOTSTRAP_TOKEN=$(awk -F "," '{print $1}' /etc/kubernetes/token.csv)
kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true \
--server=https://192.168.3.101:6443 --kubeconfig=kubelet-bootstrap.kubeconfig
kubectl config set-credentials kubelet-bootstrap --token=${BOOTSTRAP_TOKEN} \
--kubeconfig=kubelet-bootstrap.kubeconfig
kubectl config set-context default --cluster=kubernetes --user=kubelet-bootstrap \
--kubeconfig=kubelet-bootstrap.kubeconfig
kubectl config use-context default --kubeconfig=kubelet-bootstrap.kubeconfig
kubectl create clusterrolebinding kubelet-bootstrap \
--clusterrole=system:node-bootstrapper --user=kubelet-bootstrap

node1

mkdir /etc/kubernetes/pki -p
mkdir /var/lib/kubelet
mkdir /var/log/kubernetes
scp master1:/cfspki/kubelet-bootstrap.kubeconfig /etc/kubernetes/
scp master1:/cfspki/ca.pem  /etc/kubernetes/pki/

cat > /etc/kubernetes/kubelet.json << EOF
{
    "kind": "KubeletConfiguration",
    "apiVersion": "kubelet.config.k8s.io/v1beta1",
    "authentication": {
    "x509": {
    "clientCAFile": "/etc/kubernetes/pki/ca.pem"
  },
    "webhook": {
    "enabled": true,
    "cacheTTL": "2m0s"
  },
    "anonymous": {
    "enabled": false
  }
  },
    "authorization": {
    "mode": "Webhook",
    "webhook": {
    "cacheAuthorizedTTL": "5m0s",
    "cacheUnauthorizedTTL": "30s"
  }
  },
    "address": "192.168.3.104",
    "port": 10250,
    "readOnlyPort": 10255,
    "cgroupDriver": "systemd",
    "hairpinMode": "promiscuous-bridge",
    "serializeImagePulls": false,
    "featureGates": {
    "RotateKubeletServerCertificate": true
  },
    "clusterDomain": "cluster.local.",
    "clusterDNS": ["10.10.0.2"]
}

EOF

cat  >  /usr/lib/systemd/system/kubelet.service  << EOF
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=containerd.service
Requires=containerd.service

[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/usr/local/bin/kubelet \\
--bootstrap-kubeconfig=/etc/kubernetes/kubelet-bootstrap.kubeconfig \\
--cert-dir=/etc/kubernetes/pki \\
--kubeconfig=/etc/kubernetes/kubelet.kubeconfig \\
--config=/etc/kubernetes/kubelet.json \\
--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.6 \\
--container-runtime=remote \\
--container-runtime-endpoint=unix:///run//containerd/containerd.sock   
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target

EOF

systemctl daemon-reload
systemctl restart kubelet.service
systemctl enable kubelet.service
systemctl status kubelet.service

master1

kubectl get csr

kubectl certificate approve node-csr-6U7vaxCjWJbRTX6oSsrgJuXt3I81JzgV87mIkSf1Fn4
kubectl get csr
kubectl get nodes

 7、安装kube-proxy

cd /cfspki/
cat > kube-proxy-csr.json  << EOF
{
    "CN": "system:kube-proxy",
    "key": {
    "algo": "rsa",
    "size": 2048
  },
    "names": [
  {
    "C": "CN",
    "ST": "Xinjiang",
    "L": "Urumqi",
    "O": "k8s",
    "OU": "system"
    }
  ]
}

EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json \
-profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true \
--server=https://192.168.3.101:6443 --kubeconfig=kube-proxy.kubeconfig
kubectl config set-credentials kube-proxy --client-certificate=kube-proxy.pem \
--client-key=kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig
kubectl config set-context default --cluster=kubernetes --user=kube-proxy \
--kubeconfig=kube-proxy.kubeconfig
kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
scp kube-proxy.kubeconfig node1:/etc/kubernetes/

node1:

mkdir -p /var/lib/kube-proxy

cat >  /etc/kubernetes/kube-proxy.yaml << EOF
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 192.168.3.104
clientConnection:
  kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
clusterCIDR: 10.0.0.0/16
healthzBindAddress: 192.168.3.104:10256
kind: KubeProxyConfiguration
metricsBindAddress: 192.168.3.104:10249
mode: "ipvs"

EOF

cat > /usr/lib/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/usr/local/bin/kube-proxy \
--config=/etc/kubernetes/kube-proxy.yaml \
--alsologtostderr=true \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=2
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

EOF

systemctl daemon-reload
systemctl start kube-proxy.service
systemctl enable kube-proxy.service
systemctl status kube-proxy.service

8、部署 calico

kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml
kubectl get pods -n kube-system

版权声明：本文为w975121565原创文章，遵循CC 4.0 BY-SA版权协议，转载请附上原文出处链接和本声明。