# 设定IP
pod=172.20.0.0/16
service_cidr=172.21.0.0/16
TMPDIR=/mytmp

# 添加所需IP
SSH_IPS=(
10.13.14.20
10.13.14.21
10.13.14.22
10.13.14.23
)
SSH_USR=(root) # 对方用户名
SSH_PASSWD="JingMiao0616" # 对方密码




RPM_FILE="cri-dockerd-0.3.1-3.el7.x86_64.rpm"
if [ -f "$RPM_FILE" ]; then
	continue;
else
	wget https://share.zrp233.top/cri-dockerd-0.3.1-3.el7.x86_64.rpm
    #echo error !!!!!!!!!
    #echo "❌ 错误：文件 $RPM_FILE 不存在！"
    #exit 1
fi


set_master(){
hostnamectl set-hostname k8s-master
}



k8s(){
# 获取本机IP
MYIP=$(ip route get 1.1.1.1|awk '{print $7}')
#MYIP=$(ip addr show eth0 | grep "inet " | awk '{print $2}')
#MYIP=$(ip addr show ens33 | grep "inet " | awk '{print $2}')
echo $MYIP

systemctl stop firewalld
systemctl disable firewalld


# 禁用swap分区
sed -i '/swap/s/^/#/' /etc/fstab
swapoff /dev/mapper/centos-swap


# 修改linux的内核参数，添加网桥过滤和地址转发功能
cat >/etc/sysctl.d/kubernetes.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF

# 加载网桥过滤模块
modprobe br_netfilter

# 重新加载配置
sysctl -p /etc/sysctl.d/kubernetes.conf

#永久生效
tee /etc/sysconfig/modules/br_netfilter.modules << EOF
modprobe br_netfilter
EOF

chmod +x /etc/sysconfig/modules/br_netfilter.modules

tee /etc/rc.sysinit << EOF
#!/bin/bash
for file in /etc/sysconfig/modules/*.modules ; do
[ -x $file ] && $file
done
EOF


# 1 安装ipset和ipvsadm
yum install ipset ipvsadm -y

# 2 添加需要加载的模块写入脚本文件

cat <<EOF >  /etc/sysconfig/modules/ipvs.modules
#!/bin/bash
modprobe ip_vs
modprobe ip_vs_rr
modprobe ip_vs_wrr
modprobe ip_vs_sh
modprobe nf_conntrack_ipv4
EOF

# 3 为脚本文件添加执行权限
chmod +x /etc/sysconfig/modules/ipvs.modules

# 4 执行脚本文件
/bin/bash /etc/sysconfig/modules/ipvs.modules

# 安装docker
wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo

yum install --setopt=obsoletes=0 docker-ce-24.0.9-1.el7 -y

cat <<EOF >  /etc/docker/daemon.json
{
    "registry-mirrors": [
	"https://docker.xuanyuan.me",
	"https://cr.console.aliyun.com",
	"https://docker.1panel.live",
        "https://hub.uuuadc.top",
        "https://docker.anyhub.us.kg",
        "https://dockerhub.jobcher.com",
        "https://docker.ckyl.me",
        "https://docker.awsl9527.cn"]
}
EOF

# 5 启动docker
systemctl restart docker
systemctl enable docker



yum install -y ./cri-dockerd-0.3.1-3.el7.x86_64.rpm
systemctl daemon-reload
systemctl enable --now cri-docker.service
systemctl status cri-docker.service



# 编辑

cat > /etc/yum.repos.d/mykubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
       http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

# 安装kubeadm、kubelet和kubectl
yum install --setopt=obsoletes=0 kubeadm-1.28.2-0 kubelet-1.28.2-0 kubectl-1.28.2-0 -y



# 编辑/etc/sysconfig/kubelet，添加下面的配置

cat > /etc/sysconfig/kubelet << EOF
KUBELET_CGROUP_ARGS="--cgroup-driver=systemd"
KUBE_PROXY_MODE="ipvs"
EOF

systemctl enable --now kubelet


images=(
kube-apiserver:v1.28.2
kube-controller-manager:v1.28.2
kube-scheduler:v1.28.2
kube-proxy:v1.28.2
pause:3.9
pause:3.6
etcd:3.5.9-0
coredns:v1.10.1
)

for imageName in ${images[@]} ; do
        docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/$imageName
        docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/$imageName       registry.k8s.io/$imageName
        docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/$imageName
done
				docker tag registry.k8s.io/coredns:v1.10.1 registry.k8s.io/coredns/coredns:v1.10.1

}


master(){
cat > kubeadm_init.sh <<EOF
#!/bin/bash
kubeadm init \
    --kubernetes-version=v1.28.2 \
    --pod-network-cidr=${pod} \
    --service-cidr=${service_cidr} \
    --v=5 \
    --cri-socket  unix:///var/run/cri-dockerd.sock \
    --apiserver-advertise-address=${MYIP}
EOF

cat > kube-flannel.yml << EOF

---
kind: Namespace
apiVersion: v1
metadata:
  name: kube-flannel
  labels:
    k8s-app: flannel
    pod-security.kubernetes.io/enforce: privileged
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: flannel
  name: flannel
rules:
- apiGroups:
  - ""
  resources:
  - pods
  verbs:
  - get
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes/status
  verbs:
  - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: flannel
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-flannel
---
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: flannel
  name: flannel
  namespace: kube-flannel
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-flannel
  labels:
    tier: node
    k8s-app: flannel
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "${pod}",
      "EnableNFTables": false,
      "Backend": {
        "Type": "vxlan"
      }
    }
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds
  namespace: kube-flannel
  labels:
    tier: node
    app: flannel
    k8s-app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/os
                operator: In
                values:
                - linux
      hostNetwork: true
      priorityClassName: system-node-critical
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni-plugin
        image: ghcr.io/flannel-io/flannel-cni-plugin:v1.7.1-flannel1
        command:
        - cp
        args:
        - -f
        - /flannel
        - /opt/cni/bin/flannel
        volumeMounts:
        - name: cni-plugin
          mountPath: /opt/cni/bin
      - name: install-cni
        image: ghcr.io/flannel-io/flannel:v0.27.3
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: ghcr.io/flannel-io/flannel:v0.27.3
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
            add: ["NET_ADMIN", "NET_RAW"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: EVENT_QUEUE_DEPTH
          value: "5000"
        - name: CONT_WHEN_CACHE_NOT_READY
          value: "false"
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
        - name: xtables-lock
          mountPath: /run/xtables.lock
      volumes:
      - name: run
        hostPath:
          path: /run/flannel
      - name: cni-plugin
        hostPath:
          path: /opt/cni/bin
      - name: cni
        hostPath:
          path: /etc/cni/net.d
      - name: flannel-cfg
        configMap:
          name: kube-flannel-cfg
      - name: xtables-lock
        hostPath:
          path: /run/xtables.lock
          type: FileOrCreate
EOF
}







node_sethostname(){
i=1
for ip in ${SSH_IPS[@]}; do

cat > node.sh << EOF
hostnamectl set-hostname k8s-node$i
EOF


cat > tmp.yaml << EOF
- hosts: server${i}
  tasks:
    - name: hostnamectl set-hostname k8s-node
      script: node.sh
EOF

((i++))
ansible-playbook tmp.yaml

done

rm -rf tmp.yaml node.sh

}










node_yml(){
i=1
#for ip in ${SSH_IPS[@]}; do

sed  "s/hostnamectl set-hostname k8s-master/hostnamectl set-hostname k8s-node$i/" $0 > lib.sh
cat > node.sh << EOF
source ./lib.sh
k8s
$JOIN
EOF


cat > tmp.yaml << EOF
- hosts: all
  tasks:
    - name: 确保 /mytmp 是目录
      shell: mkdir -p ${TMPDIR}
    - name: 先复制 lib.sh
      copy: 
        src: lib.sh 
        dest: ${TMPDIR}
        mode: u+x
    - name: 先复制 lib.sh
      copy: 
        src: cri-dockerd-0.3.1-3.el7.x86_64.rpm
        dest: ${TMPDIR}
    - name: 
      script: node.sh
      args:
        chdir: ${TMPDIR}
    - name:
      file:
        path: ${TMPDIR}
        state: absent
EOF

((i++))
ansible-playbook tmp.yaml

#done

rm -rf tmp.yaml lib.sh node.sh

}





ip_test(){
for ip in "${SSH_IPS[@]}"; do
    #echo -n "PING $ip: "
    if ping -c1 -W1 "$ip" &>/dev/null;then
	continue
    else
        echo "$ip wrong"
	exit 1
    fi
done

}



inatall_ansible(){
# 安装扩展软件包
yum install -y epel-release

# 列出可用ansible包
yum list | grep ansible

# 安装ansible
yum install -y ansible
}
# inatall_ansible


ansible_add_hosts(){
cp /etc/hosts{,-bak-$(date +%Y%m%d-%H%M%S)}
cp /etc/ansible/hosts{,-bak-$(date +%Y%m%d-%H%M%S)}
# 10.0.1.28 ansible
MY_IP=$(ip route get 1.1.1.1|awk '{print $7}')
    if [[ -z "$MY_IP" ]]; then
        echo "无法获取本机 IP"
        return 1
    fi

# 写入 ansible IP
if ! grep -q "^${MY_IP}[[:space:]]\\+sansible\\>" /etc/hosts; then
    echo "${MY_IP} ansible" >> /etc/hosts
    echo "add ${MY_IP} ansible to /etc/hosts"
else
    echo "exist ${MY_IP} ansible in /etc/hosts"
fi

# 写入 server IP
i=1
for ip in "${SSH_IPS[@]}";
do
    if ! grep -q "^$ip[[:space:]]\\+server$i\\>" /etc/hosts; then
        echo "$ip server$i" >> /etc/hosts
        echo "add $ip server$i to /etc/hosts"
    else
        echo "exist $ip server$i in /etc/hosts"
    fi

    if ! grep -q "^server$i" /etc/ansible/hosts; then
        echo "server$i" >> /etc/ansible/hosts
        echo " add server$i to /etc/ansible/hosts"
    else
        echo "exist server$i in /etc/ansible/hosts"
    fi
    ssh-keyscan server$i >> ~/.ssh/known_hosts

	((i++))
done

}

#ansible_add_hosts


ssh_auto_pass()
{
# 安装 sshpass
yum install -y sshpass

# 生成 SSH RSA 密钥对
ssh-keygen -t rsa -b 2048 -f ~/.ssh/id_rsa -N "" -q -o -C "auto-generated" <<< y

# 获取目标主机的公钥并写入 known_hosts
for SSH_IP in ${SSH_IPS[@]};do
ssh-keyscan ${SSH_IP} >> ~/.ssh/known_hosts
done

# 批量免密
for SSH_IP in ${SSH_IPS[@]};do
    sshpass -p "${SSH_PASSWD}" ssh-copy-id ${SSH_USR}@${SSH_IP}
done
}

#ssh_auto_pass



get_join(){

JOIN=$(kubeadm token create --print-join-command)
JOIN="$JOIN --v=5 --cri-socket unix:///var/run/cri-dockerd.sock"

}

kubeadm_init(){

kubeadm init \
--kubernetes-version=v1.28.2 \
--pod-network-cidr=${pod} \
--service-cidr=${service_cidr} \
--v=5 \
--cri-socket  unix:///var/run/cri-dockerd.sock \
--apiserver-advertise-address=${MYIP}

mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

}





if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then

ip_test

inatall_ansible
ssh_auto_pass
ansible_add_hosts

set_master

k8s

master

kubeadm_init

get_join

sleep 3

echo $JOIN

node_sethostname


node_yml


kubectl apply -f kube-flannel.yml

rm -rf kube-flannel.yml kubeadm_init.sh

echo "kubectl completion bash >> ~/.bashrc"
echo "source ~/.bashrc"

echo "kubectl get pods -A -w"


fi

