#!/usr/bin/env bash
# author: joe.zheng
# version: 24.04.02

set -e

# @self {
# AUTO-GENERATED, DO NOT EDIT!

SELF="$(basename $0)"
SELF_VERSION="$(sed -n '1,4s/# version: \(.*\)/\1/p' $0)"

# @self }

# IP of the primary interface on the host
HOST_IP=$(ip route get 1 | head -1 | awk '{print $7}')
HOST_DNS=$(perl -e 'use Net::Domain hostfqdn; print hostfqdn' | sed 's/[.]*$//')
HOST_NAME=$(perl -e 'use Net::Domain hostname; print hostname')

K8S_NOPROXY=".svc,.svc.cluster.local"
K8S_PKGS="kubelet kubeadm kubectl"
INIT_CONFIG="kubeadm-init.yaml"       # config file for "kubeadm init"
JOIN_MANUAL="kubeadm-join.txt"        # "kubeadm join" instructions

INIT_WORKDIR="${INIT_WORKDIR:-n}"     # initialize workdir and quit if "y"

ADDRESS=$HOST_IP      # API server IP address
PORT=6443             # API server IP port
BACKEND="vxlan"       # flannel backend
BACKENDS="none vxlan wireguard" # valid flannel backend options
ENDPOINT=""           # endpoint for all control-plane nodes
SANS=$HOST_DNS        # API server certificate SANs
NODE=""               # specify node name
VERSION=1.23.0        # k8s stable version
MIRROR=auto           # mirror option
MIRRORS="yes no auto" # valid mirror options
RESET=n               # reset cluster first
UNINSTALL=n           # reset cluster and uninstall components
NO_CLUSTER=n          # do not init cluster
NO_WORKLOAD=n         # do not schedule workload on control-plane node

CRI_RUNTIMES="docker containerd"            # valid CRI runtimes
for r in $CRI_RUNTIMES; do
  if which $r >/dev/null; then
    DEF_RUNTIME="$r"
    break
  fi
done
CRI_RUNTIME="${CRI_RUNTIME:-$DEF_RUNTIME}"  # CRI runtime to use

CRI_CLIENT="docker"   # CRI client, such as docker, ctr

FLANNEL_VERSION="${FLANNEL_VERSION:-0.23.0}"                  # flannel version
FLANNEL_BASEURL="https://github.com/flannel-io/flannel"

LOCALPP_VERSION="${LOCALPP_VERSION:-0.0.24}"                  # local path provisioner version
LOCALPP_BASEURL="https://github.com/rancher/local-path-provisioner"

CRI_DOCKERD_VERSION="${CRI_DOCKERD_VERSION:-0.3.8}"           # cri-dockerd version
CRI_DOCKERD_BASEURL="https://github.com/Mirantis/cri-dockerd" # cri-dockerd repo url
CRI_DOCKERD_SOCKET="unix:///var/run/cri-dockerd.sock"         # cri-dockerd socket path

CRI_CONTAINERD_SOCKET="unix:///run/containerd/containerd.sock" # containerd socket path

K8S_PKG_REPO="https://pkgs.k8s.io/core:/stable:"              # k8s package repo
K8S_PKG_MIRROR="https://mirrors.aliyun.com/kubernetes"        # k8s package mirror
K8S_REG_REPO="registry.k8s.io"                                # k8s registry repo
K8S_REG_MIRROR="registry.aliyuncs.com/google_containers"      # k8s registry mirror

# @dry-run {
# AUTO-GENERATED, DO NOT EDIT!

DRY_RUN="${DRY_RUN:-n}"        # "y" to enable
RUN=''                         # command prefix for dry run
if [[ $DRY_RUN == 'y' ]]; then
  RUN='echo'
fi

# @dry-run }

# @sudo {
# AUTO-GENERATED, DO NOT EDIT!

SUDO="sudo"
if [[ $(id -u) == "0" ]]; then
  SUDO=""
fi

function validate_sudo() {
  if [[ -n $SUDO ]]; then
    msg "validate sudo"
    sudo -v
  fi
}

# @sudo }

function usage() {
  cat <<EOF
Usage: $SELF [-a <addr>] [-b <type>] [-c <cri>] [-e <addr>] [-m <type>]
  [-N <name>] [-s <sans>] [-v <ver>] [-r] [-R] [-U] [-C] [-W] [-n] [-h]

  Deploy Kubernetes and initialize a cluster the easy way

  -a <addr>: API server IP address, default: $ADDRESS
  -b <type>: Flannel backend [$BACKENDS], default: $BACKEND
  -c <cri>:  CRI runtime [$CRI_RUNTIMES], default: $CRI_RUNTIME
  -e <addr>: endpoint for all control-plane nodes, default: "$ENDPOINT"
  -s <sans>: API server certificate SANs separated by ",", default: $SANS
  -m <type>: mirror option [$MIRRORS], default: $MIRROR
  -N <name>: set node name, instead of the actual hostname, default: "$NODE"
  -v <ver>:  k8s version, in format MAJOR.MINOR.PATCH, default: $VERSION
  -r:        reset k8s cluster first, default: $RESET
  -R:        reset k8s cluster and stop, same as -rC
  -U:        reset cluster and uninstall components, default: $UNINSTALL
  -C:        do not initialize cluster, default: $NO_CLUSTER
  -W:        do not schedule workload on control-plane node, default: $NO_WORKLOAD
  -n:        dry run, print out information only, default: $DRY_RUN
  -h:        print the usage message

Offline mode:

  To support deployment in offline mode, the required packages and containers
  will be cached in the workdir "$CACHE/" when deploying it on a machine with
  internet connection, then copy the cache to the target machine and deploy it

  You can save all the downloaded container images by this command:

    $CACHE_IMAGE save

  These cached files will be loaded in offline mode when necessary.

  The workdir structure:

  - $CACHE
    |- images    (exported container images)
    |- manifests (manifest files in YAML format)
    |- packages  (software packages, such as *.deb)
       |- *      (by os, e.g.: ubuntu, centos, common)
    |- run       (runtime generated files)
       |- *      (by component, e.g.: $SELF)

  The offline mode will be enabled automatically through network detection.
  You can also force it by setting environment variable "OFFLINE=y"

Environment variables:
  MIRROR_URL:          mirror to use for GitHub, default: $MIRROR_URL
  OFFLINE:             force into offline mode [y n], default: $OFFLINE
  INIT_WORKDIR:        initialize workdir and quit if "y", default: $INIT_WORKDIR
  FLANNEL_VERSION:     flannel version, default: $FLANNEL_VERSION
  LOCALPP_VERSION:     local path provisioner version, default: $LOCALPP_VERSION
  CRI_DOCKERD_VERSION: cri-dockerd version, default: $CRI_DOCKERD_VERSION

Examples:

  1. Deploy a single node cluster
     $SELF

  2. Reset the cluster and deploy again
     $SELF -r

  3. Reset the cluster and deploy the specific version
     $SELF -r -v $VERSION

  4. Deploy a cluster with one control-plane and one worker node
     # on control-plane node
     $SELF -W
     # check the output command for "kubeadm join"

     # on worker node
     $SELF -C
     # use "kubeadm join" command to join the cluster

  5. Deploy a control-plane node with DNS name vip.k8s.io
     $SELF -W -e vip.k8s.io

  6. Reset the cluster and uninstall components
     $SELF -U

  7. Initialize workdir ($CACHE) and quit
     INIT_WORKDIR=y $SELF

Version: $SELF_VERSION
EOF

}

function current_version() {
  local version=
  if [[ -n $(which kubeadm) ]]; then
    version="$(kubeadm version -o short)"
    version=${version#v}
  fi
  echo $version
}

function main() {
  # adjust default version if already deployed
  CURRENT="$(current_version)"
  VERSION="${CURRENT:-$VERSION}"

  while getopts ":a:b:c:e:s:m:N:v:hCWnRrU" opt
  do
    case $opt in
      a ) ADDRESS=$OPTARG;;
      b ) BACKEND=$OPTARG
          if echo $BACKENDS | grep -v -w $BACKEND >/dev/null 2>&1; then
            echo "invalid backend option $BACKEND"
            usage && exit 1
          fi
          ;;
      c ) CRI_RUNTIME=$OPTARG
          if echo $CRI_RUNTIMES | grep -v -w $CRI_RUNTIME >/dev/null 2>&1; then
            echo "invalid backend option $CRI_RUNTIME"
            usage && exit 1
          fi
          ;;
      e ) ENDPOINT=$OPTARG;;
      s ) SANS=$OPTARG;;
      m ) MIRROR=$OPTARG
          if echo $MIRRORS | grep -v -w $MIRROR >/dev/null 2>&1; then
            echo "invalid mirror option $MIRROR"
            usage && exit 1
          fi
          ;;
      N ) NODE=$OPTARG;;
      v ) VERSION=${OPTARG#v}
         dots="${VERSION//[^.]}"
         if [[ ${#dots} != 2 ]]; then
           echo "invalid version: $VERSION"
           usage && exit 1
         fi
         ;;
      r ) RESET=y;;
      R ) RESET=y && NO_CLUSTER=y;;
      U ) RESET=y && NO_CLUSTER=y && UNINSTALL=y;;
      n ) DRY_RUN=y;;
      C ) NO_CLUSTER=y;;
      W ) NO_WORKLOAD=y;;
      h ) usage && exit;;
      * ) usage && echo "invalid option: -$OPTARG" && exit 1;;
    esac
  done
  shift $((OPTIND-1))

  CACHE_IMAGE="$CACHE/image" # script to save or load images

  if [[ $CRI_RUNTIME == 'docker' ]]; then
    CRI_CLIENT="docker"
  else
    CRI_CLIENT="ctr"
  fi
  if [[ -n $SUDO ]]; then
    CRI_CLIENT="$SUDO $CRI_CLIENT"
  fi

  for v in ADDRESS BACKEND CRI_RUNTIME CRI_CLIENT CURRENT DRY_RUN ENDPOINT HOST_IP MIRROR MIRROR_URL \
    NODE NO_CLUSTER NO_WORKLOAD OFFLINE RESET SANS UNINSTALL VERSION FLANNEL_VERSION LOCALPP_VERSION \
    CRI_DOCKERD_VERSION
  do
    eval echo "$v: \${$v}"
  done

  [[ $DRY_RUN == "y" ]] && exit

  warn_as_root
  if [[ $INIT_WORKDIR == "y" ]]; then
    init_workdir && exit
  fi
  validate_sudo
  check_prerequisites
  check_network
  check_container
  check_mirror && check_k8s_mirror
  init_workdir
  ensure_swapoff
  ensure_iptables
  ensure_selinuxoff
  ensure_firewalloff
  ensure_installed
  if [[ $RESET == "y" ]]; then
    reset_cluster
  fi
  if [[ $NO_CLUSTER != "y" ]]; then
    init_cluster
  fi
  if [[ $UNINSTALL == "y" ]]; then
    ensure_uninstalled
  fi

  msg "done"
}

function realfile() {
  [[ -L "$1" ]] && readlink -f $1 || echo $1
}

function dist_name() {
  dist="$(cat /etc/*-release | sed -ne 's/^ID=//p' | xargs echo)"
  if [[ -z $dist ]]; then
    dist="$(lsb_release -s -i)"
  fi
  dist="$(echo $dist | tr '[:upper:]' '[:lower:]')"
  echo $dist
}

function minimal_version() {
  echo $(printf "%s\n" $@ | sort -V | head -n1)
}

function check_prerequisites() {
  msg "check prerequisites"
  if [[ -z $(which perl) ]]; then
    err "perl is not available"
    exit 1
  fi
  if [[ -z $(which curl) ]]; then
    err "curl is not available"
    exit 1
  fi
  if [[ -z $(which openssl) ]]; then
    err "openssl is not available"
    exit 1
  fi
  if [[ -z $(which $CRI_RUNTIME) ]]; then
    err "$CRI_RUNTIME is not available"
    exit 1
  fi
}

function check_container() {
  msg "check container image downloading"
  local image="docker.io/library/hello-world:latest"
  if ! $CRI_CLIENT image pull $image >/dev/null 2>&1; then
    err "$CRI_CLIENT image pull $image failed, check $CRI_RUNTIME configuration"
    exit 1
  fi
}

function warn_as_root() {
  if [[ $(id -u) == "0" ]]; then
    msg "you are running as root user, are you sure?"
    msg "continue in 5 seconds, CTRL + C to interrupt"
    read -t 5 || true
  fi
}

function init_workdir() {
  ensure_workdir && ensure_script
}

USE_K8S_MIRROR="n"
function check_k8s_mirror() {
  msg "is k8s mirror needed"

  local mirror;
  if [[ $MIRROR == "auto" ]]; then
    mirror="yes" # default as needed
    if [[ $OFFLINE == "y" ]]; then
      msg "offline mode, assume mirror is needed"
    else
      msg "check if we can access k8s pkgs and images"
      local target="$K8S_PKG_REPO"
      if can_access $target; then
        msg "curl $target is OK"
        target="$K8S_REG_REPO/pause:latest"
        if $CRI_CLIENT image pull $target >/dev/null 2>&1; then
          msg "$CRI_CLIENT image pull $target is OK"
          mirror="no"
        else
          msg "$CRI_CLIENT image pull $target is FAILED"
        fi
      else
        msg "curl $target is FAILED"
      fi
    fi
  fi

  local since="1.24.0" # official repo only has k8s >= 1.24
  if [[ $mirror == "no" && $(minimal_version $since $VERSION) != $since ]]; then
    msg "request legacy version $VERSION, only available in the mirror"
    msg "use the mirror forcibly"
    mirror="yes"
  fi

  if [[ $MIRROR == "yes" || $mirror == "yes" ]]; then
    USE_K8S_MIRROR="y"
  fi
  msg "need k8s mirror: $USE_K8S_MIRROR"
}

# @base {
# AUTO-GENERATED, DO NOT EDIT!

function msg {
  echo "> $@"
}

function err {
  echo "> $@" >&2
}

function has() {
  [[ -z "${1##*$2*}" ]] && [[ -z "$2" || -n "$1" ]]
}

# @base }

# @network {
# AUTO-GENERATED, DO NOT EDIT!

OFFLINE="${OFFLINE:-n}"                           # no external network
MIRROR="${MIRROR:-auto}"                          # mirror option to use
MIRRORS="yes no auto"                             # valid mirror options
USE_MIRROR="${USE_MIRROR:-n}"                     # need to use mirror
MIRROR_URL="${MIRROR_URL:-https://iffi.me/proxy}" # mirror or http proxy to use
URL_TO_CHECK="${URL_TO_CHECK:-https://raw.githubusercontent.com}"

# the target URL is reachable but the response may be 4xx or 5xx
function can_access() {
   curl -IL -s -m 5 $1 >/dev/null 2>&1
}

function check_network() {
  msg "check network"
  if [[ $OFFLINE != 'y' ]]; then
    if ! can_access example.com; then
      msg "can't access external network"
      OFFLINE="y"
    fi
  fi
  msg "offline mode: $OFFLINE"
}

function check_mirror() {
  msg "use mirror or not"

  local mirror;
  if [[ $MIRROR == "auto" ]]; then
    mirror="yes" # default as needed
    if [[ $OFFLINE == "y" ]]; then
      msg "offline mode, assume mirror is needed"
    else
      local target="$URL_TO_CHECK"
      msg "check whether we can access $target"
      if can_access $target; then
        msg "curl $target is OK"
        mirror="no"
      else
        msg "curl $target is FAILED"
      fi
    fi
  fi
  if [[ $MIRROR == "yes" || $mirror == "yes" ]]; then
    msg "mirror is needed"
    if can_access $MIRROR_URL; then
      USE_MIRROR="y"
    else
      err "failed to access mirror ($MIRROR_URL)"
    fi
  fi
  msg "use mirror: $USE_MIRROR"
}

function mirror_url() {
  local origin="${1:?argument missing}"

  if [[ $USE_MIRROR == "y" ]]; then
    echo -n "$MIRROR_URL/$origin"
  else
    echo -n "$origin"
  fi
}

# @network }

# @cache {
# AUTO-GENERATED, DO NOT EDIT!

CACHE="${CACHE:-cache}"                                    # the cache directory
CACHE_RUN="${CACHE_RUN:-$CACHE/run/$SELF}"                 # runtime generated files
CACHE_IMAGES="${CACHE_IMAGES:-$CACHE/images}"              # exported container images
CACHE_PACKAGES="${CACHE_PACKAGES:-$CACHE/packages}"        # software packages, e.g. .deb, .tar.gz
CACHE_MANIFESTS="${CACHE_MANIFESTS:-$CACHE/manifests}"     # manifest files in YAML format
CACHE_DOWNLOADING="${CACHE_DOWNLOADING:-$CACHE/.download}" # temporary directory for downloading

function ensure_workdir() {
  msg "ensure workdir"

  for d in $CACHE_RUN $CACHE_IMAGES $CACHE_PACKAGES $CACHE_MANIFESTS $CACHE_DOWNLOADING; do
    if [[ ! -d $d ]]; then
      msg "create dir: $d"
      mkdir -p $d
    fi
  done
}

function download_file() {
  local src="${1:?missing source url}"
  local dst="${2:?missing destination}"

  if [[ -f "$dst" ]]; then
    msg "already exist: $dst"
  else
    msg "download: $src"
    local tmp="$CACHE_DOWNLOADING/$(basename $dst)"
    $RUN curl -fsSL $src > $tmp
    $RUN mkdir -p $(dirname $dst) && mv $tmp $dst
    msg "saved at: $dst"
  fi
}

# @cache }

function save_package_list() {
  local path="${1:?argument missing}"
  local dist=$(dist_name)
  msg "save current package list to $path"

  mkdir -p $(dirname $path)

  if [[ $dist == "ubuntu" ]]; then
    ls -1 /var/cache/apt/archives/*.deb | sort > $path
  elif [[ $dist == "centos" || $dist == "openeuler" ]]; then
    if [[ -d /var/cache/dnf ]]; then
      ls -1 /var/cache/dnf/*/packages/*.rpm | sort > $path
    else
      ls -1 /var/cache/yum/*/*/*/packages/*.rpm | sort > $path
    fi
  else
    err "dist \"$dist\" is not supported!"
    exit 1
  fi
}

NEED_CRI_DOCKERD=n
function need_cri_dockerd() {
  msg "is cri-dockerd needed"

  local since="1.24.0" # dockershim was removed since v1.24.0
  if [[ $(minimal_version $since $VERSION) == $since ]]; then
    msg "the requested k8s version $VERSION is >= $since"
    NEED_CRI_DOCKERD=y

    if [[ $(minimal_version $since $CURRENT) != $since ]]; then
      err "there is a known issue with k8s version ($CURRENT), upgrade to version >= 1.24.0 first"
      err "details in https://github.com/Mirantis/cri-dockerd/issues/167"
      msg "uninstall current k8s with command \"$SELF -U\" first, and try again"
      exit 1
    fi
  fi
  msg "need cri-dockerd: $NEED_CRI_DOCKERD"
}

function ensure_cri_dockerd_uninstalled() {
  msg "ensure cri-dockerd uninstalled"

  if systemctl is-active cri-docker.socket >/dev/null; then
    $SUDO systemctl stop cri-docker
    $SUDO rm -f /usr/bin/cri-dockerd
    $SUDO rm -f /etc/systemd/system/cri-docker.{service,socket}
    $SUDO systemctl daemon-reload
  fi
}

function install_cri_dockerd_if_needed() {
  msg "install cri-dockerd if needed"

  need_cri_dockerd

  if [[ $NEED_CRI_DOCKERD == "y" ]]; then
    local ver="$CRI_DOCKERD_VERSION"
    local url="$(mirror_url $CRI_DOCKERD_BASEURL)"

    if systemctl is-active cri-docker.socket >/dev/null; then
      v="$(cri-dockerd --version 2>&1 | awk '{print $2}')"
      msg "cri-dockerd already installed, version: $v"
      if [[ $RESET == "y" || $v != $ver ]]; then
        msg "uninstall current version $v first"
        ensure_cri_dockerd_uninstalled
      else
        msg "same as target version $ver, skip"
        return
      fi
    fi

    local arch="$(uname -m | sed 's/x86_64/amd64/')"
    local name="cri-dockerd-$ver.$arch.tgz"
    local dir="$CACHE_PACKAGES/common/cri-dockerd-$ver"
    local pkg="$dir/$name"

    mkdir -p $dir

    if [[ $OFFLINE == "y" ]]; then
      msg "WARNING: offline mode, cache must be ready"
      msg "try to install from the cache"
    else
      download_file "$url/releases/download/v$ver/$name" $pkg
      for f in cri-docker.{service,socket}; do
        download_file "$url/raw/v${ver}/packaging/systemd/$f" $dir/$f
      done
    fi

    msg "install cri-dockerd $ver"
    tar xzf $pkg -C $CACHE_RUN --strip-components=1
    $SUDO install $CACHE_RUN/cri-dockerd /usr/bin/
    $SUDO install $dir/cri-docker.{service,socket} /etc/systemd/system/
    $SUDO systemctl daemon-reload
    $SUDO systemctl enable cri-docker.service
    $SUDO systemctl enable --now cri-docker.socket
  fi
}

function install_k8s() {
  local dist=$(dist_name)
  msg "install k8s on \"$dist\""

  local cachedir="$CACHE_PACKAGES/$dist/k8s-$VERSION"

  if [[ $OFFLINE == "y" ]]; then
    msg "WARNING: offline mode, cache must be ready"
    msg "try to install package from the cache"
    if [[ $dist == "ubuntu" ]]; then
      msg "install $cachedir/*.deb"
      $SUDO dpkg -i $cachedir/*.deb
      msg "no auto update for $K8S_PKGS"
      $SUDO apt-mark hold $K8S_PKGS
    elif [[ $dist == "centos" || $dist == "openeuler" ]]; then
      msg "install $cachedir/*.rpm"
      $SUDO yum install -y -C --disablerepo=* $cachedir/*.rpm
      msg "enabled and start kubelet"
      $SUDO systemctl enable --now kubelet
    else
      err "dist \"$dist\" is not supported!"
      exit 1
    fi

    if [[ -z $(which kubeadm) ]]; then
      msg "kubeadm is not available"
      exit 1
    fi

    # offline mode is done
    return
  fi

  local version="v${VERSION%.*}" # major.minor
  local packages_old="$CACHE_RUN/packages-old.txt"
  local packages_new="$CACHE_RUN/packages-new.txt"

  save_package_list "$packages_old"
  if [[ $dist == "ubuntu" ]]; then
    local keydir="/etc/apt/keyrings"
    local gpgkey="$keydir/kubernetes-apt-keyring.gpg"
    local list="/etc/apt/sources.list.d/kubernetes.list"
    local key="$K8S_PKG_REPO/$version/deb/Release.key"
    local src="$K8S_PKG_REPO/$version/deb/ /"

    if [[ $USE_K8S_MIRROR == "y" ]]; then
      key="$K8S_PKG_MIRROR/apt/doc/apt-key.gpg"
      src="$K8S_PKG_MIRROR/apt/ kubernetes-xenial main"
    fi

    msg "add gpg key from $key"
    $SUDO install -m 0755 -d $keydir
    curl -fsSL $key | $SUDO gpg --dearmor --yes -o $gpgkey
    $SUDO chmod a+r $gpgkey

    msg "add repo source to $list"
    cat <<EOF | $SUDO tee $list
deb [signed-by=$gpgkey] $src
EOF

    msg "search packages for version $VERSION"
    $SUDO apt update
    local allver="$(apt-cache madison kubeadm | awk '{ print $3 }')"
    local pkgver="$(echo "$allver" | grep "$VERSION-" | head -n1)"
    if [[ -z $pkgver ]]; then
      msg "available versions:"
      echo "$(echo $allver | fold -s)"
      err "$VERSION not found"
      exit 1
    else
      msg "found version $pkgver"
      local pkgsss="$K8S_PKGS " # keep the last space
      local wanted="${pkgsss// /=$pkgver }"
      msg "install packages: $wanted"
      # use apt-get instead of apt to keep the downloaded packages
      $SUDO apt-get install -y $wanted
      msg "no auto update for $K8S_PKGS"
      $SUDO apt-mark hold $K8S_PKGS
    fi

  elif [[ $dist == "centos" || $dist == "openeuler" ]]; then
    local list="/etc/yum.repos.d/kubernetes.repo"
    local key="$K8S_PKG_REPO/$version/rpm/repodata/repomd.xml.key"
    local src="$K8S_PKG_REPO/$version/rpm"

    if [[ $USE_K8S_MIRROR == "y" ]]; then
      local name="el7-$(uname -m)"
      local base="$K8S_PKG_MIRROR/yum/doc"
      key="$base/yum-key.gpg $base/rpm-package-key.gpg"
      src="$K8S_PKG_MIRROR/yum/repos/kubernetes-$name"
    fi

    msg "add $src to $list"
    # repo_gpgcheck is disabled for centos7
    # details in https://github.com/kubernetes/kubernetes/issues/100757
cat <<EOF | $SUDO tee $list
[kubernetes]
name=Kubernetes
baseurl=$src
enabled=1
gpgcheck=1
repo_gpgcheck=0
gpgkey=$key
exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
EOF

    msg "enable yum cache"
    local conf="$(realfile /etc/yum.conf)"
    if grep "keepcache" $conf >/dev/null; then
      $SUDO perl -pi -e 's/(keepcache\s*=\s*).*$/${1}1/' $conf
    else
      $SUDO perl -pi -e 's/^(\s*\[main\].*)$/${1}\nkeepcache=1/' $conf
    fi
    $SUDO yum makecache

    local pkgsss="$K8S_PKGS "
    local wanted="${pkgsss// /-$VERSION }"
    msg "install $wanted"
    $SUDO yum install -y $wanted --disableexcludes=kubernetes
    msg "enabled and start kubelet"
    $SUDO systemctl enable --now kubelet

    # openEuler use containernetworking-plugins instead of kubernetes-cni
    if [[ $dist == "openeuler" ]] && [[ -d "/usr/libexec/cni" ]]; then
      msg "setup cni for $dist"
      $SUDO mkdir -p /opt/cni/bin && $SUDO ln -sf -t $_ /usr/libexec/cni/*
    fi

  else
    err "dist \"$dist\" is not supported!"
    exit 1
  fi
  save_package_list "$packages_new"

  msg "save the installed packages to $cachedir"
  local packages_delta="$CACHE_RUN/packages-delta.txt"
  comm -13 $packages_old $packages_new > $packages_delta

  mkdir -p $cachedir
  while read f; do
    msg "saving $f"
    cp -t $cachedir $f
  done <$packages_delta
}

function ensure_installed() {
  msg "ensure k8s installed"

  if [[ -n $CURRENT ]]; then
    msg "already installed, version: $CURRENT"
    if [[ $VERSION != $CURRENT ]]; then
      msg "the requested version ($VERSION) is different from the installed one"
      msg "we only support to switch control-plane version without reinstalling"
      msg "if you want to switch k8s version, reset cluster and delete k8s first"
      msg "use the command:"
      msg "  $SELF -U"
    fi
  else
    install_k8s
  fi

  if [[ $CRI_RUNTIME == 'docker' ]]; then
    install_cri_dockerd_if_needed
  fi
}

function ensure_uninstalled() {
  msg "ensure k8s uninstalled"

  if [[ -n $(which kubelet) ]]; then
    local dist=$(dist_name)
    if [[ $dist == "ubuntu" ]]; then
      $SUDO apt autoremove $K8S_PKGS
    elif [[ $dist == "centos" || $dist == "openeuler" ]]; then
      $SUDO yum remove $K8S_PKGS
    else
      err "dist \"$dist\" is not supported!"
      exit 1
    fi
  fi

  ensure_cri_dockerd_uninstalled
}

function ensure_selinuxoff() {
  msg "ensure SELinux is off"
  if [[ -n "$(which getenforce)" && "$(getenforce)" == "Enforcing" ]]; then
    msg "set SELinux in permissive mode (effectively disabling it)"
    $SUDO setenforce 0
    $SUDO sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config
  fi
}

function ensure_swapoff() {
  # k8s requires swapoff
  msg "ensure swap is off"
  local n=`cat /proc/swaps | wc -l`
  if (( n > 1 )); then
    local swap=$(systemctl list-units | perl -ne 'print $1 if /^\s+(dev-\w+\.swap)\s+/')
    if [[ -n $swap ]]
    then
      msg "disable swap device $swap"
      $SUDO systemctl mask $swap
    fi
    msg "disable any swap device in fstab"
    $SUDO perl -pi -e 's/^(.+(none|swap)\s+swap.+)/#$1/ unless /^#/' $(realfile /etc/fstab)
    msg "swapoff -a"
    $SUDO swapoff -a
  else
    msg "no swap device"
  fi
}

function ensure_firewalloff() {
  msg "ensure firewall is off for simplicity"
  if [[ "$(systemctl is-active firewalld)" == "active" ]]; then
    echo "firewall is active, stop and disable it"
    $SUDO systemctl stop firewalld
    $SUDO systemctl disable firewalld
  fi
}

function ensure_iptables() {
  msg "ensure iptables see bridged traffic"
  if ! lsmod | grep -wq br_netfilter; then
    echo "br_netfilter is not loaded, load it and make it auto-loaded"
    $SUDO modprobe br_netfilter
    cat <<EOF | $SUDO tee /etc/modules-load.d/k8s.conf
br_netfilter
EOF
  fi
  local prefix="/proc/sys/net/bridge/bridge-nf-call-ip"
  local forward="/proc/sys/net/ipv4/ip_forward"
  if [[ "$(cat $forward)" != "1" || "$(cat ${prefix}tables)" != "1" || "$(cat ${prefix}6tables)" != "1" ]]; then
    msg "bridge-nf-call-iptables is disabled, enable it"
    cat <<EOF | $SUDO tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF
    $SUDO sysctl --system
  fi
}

function ensure_script() {
  msg "ensure script"

  if [[ ! -f $CACHE_IMAGE ]]; then
    msg "create script: $CACHE_IMAGE"
cat <<'EOFOF' > $CACHE_IMAGE
#!/usr/bin/env bash
# author: joe.zheng
# version: 24.03.10

set -e

SELF="$(basename $0)"
SELF_VERSION="$(sed -n '1,4s/# version: \(.*\)/\1/p' $0)"

SUDO=""
if [[ $(id -u) != "0" ]]; then
  SUDO="sudo"
fi

DEF_CLIENT="docker"
if which docker >/dev/null; then
  DEF_CLIENT="docker"
elif which ctr >/dev/null; then
  DEF_CLIENT="ctr"
fi

OCI_CLIENT="${OCI_CLIENT:-$DEF_CLIENT}"
DIR_IMAGES="${DIR_IMAGES:-$(dirname $0)/images}"
NO_CLOBBER="${NO_CLOBBER:-n}"
AUTO_PULL="${AUTO_PULL:-y}"

NAMESPACE="k8s.io"

function usage() {
  cat <<EOF
Usage: $SELF {save | load} [<name>...]
  Save or load container images

  <name>: image name, in the format "repo:tag" for "save", e.g. nginx:latest,
          or "name[.tar.gz]" for "load", e.g. nginx:latest or nginx-latest.tar.gz

  If no image name is provided, it will try to read from STDIN first, otherwise
  all the cached container images will be used for "save", and all the saved
  image files under DIR_IMAGES will be used for "load".

  If both image names and IMAGE_LIST are provided, they will be merged.

  Duplicated image names will be removed.

Environment variables:
  DRY_RUN:     print out information only if it is "y"
  DIR_IMAGES:  dir to save images or load from, default: $DIR_IMAGES
  NO_CLOBBER:  do not overwrite existing files, default: $NO_CLOBBER
  OCI_CLIENT:  OCI client to use [docker, ctr], default: $OCI_CLIENT
  IMAGE_LIST:  file containing the list of image names (repo:tag)
  AUTO_PULL:   pull the image automatically if it doesn't exist, default: $AUTO_PULL
  RE_IGNORE:   regex to ignore unwanted images
  RE_WANTED:   regex to get wanted images

Version: $SELF_VERSION
EOF
}

function msg {
  echo "> $@"
}

function read_image_list {
  if [[ -e "$1" ]]; then
    awk '!/^#/ { print $1 }' $1
  fi
}

function ensure_image {
  local name="$1"
  msg "ensure image exists: $name"
  local todo
  if [[ "$OCI_CLIENT" == "docker" ]]; then
    if [[ -z "$($SUDO docker image ls -q $name)" ]]; then
      todo="$SUDO docker image pull $name"
    fi
  else
    if [[ -z "$($SUDO ctr -n $NAMESPACE image ls -q name==$name)" ]]; then
      todo="$SUDO ctr -n $NAMESPACE image pull $name"
    fi
  fi
  if [[ -n "$todo" ]]; then
    msg "pulling $name"
    if [[ "$DRY_RUN" != 'y' ]]; then
      eval $todo
    fi
  fi
}

if [[ -z "$1" || "$1" == "-h" ]]; then
  usage && exit
else
  cmd="$1"
  shift
  if ! [[ "$cmd" == "save" || "$cmd" == "load" ]]; then
    msg "not supported cmd: $cmd"
    usage && exit 1
  fi
fi

for v in DRY_RUN DIR_IMAGES IMAGE_LIST NO_CLOBBER OCI_CLIENT RE_IGNORE RE_WANTED
do
  eval echo "$v: \${$v}"
done

if [[ "$OCI_CLIENT" == "docker" ]]; then
  CMD_LIST="$SUDO docker image list -f 'dangling=false' --format '{{.Repository}}:{{.Tag}}'"
  CMD_LOAD="$SUDO docker image load"
  CMD_SAVE="$SUDO docker image save"
else
  CMD_LIST="$SUDO ctr -n $NAMESPACE image list -q | grep -v sha256:"
  CMD_LOAD="$SUDO ctr -n $NAMESPACE image import -"
  CMD_SAVE="$SUDO ctr -n $NAMESPACE image export -"
fi

if [[ -n "$SUDO" ]]; then
  $SUDO -v # validate first
fi

msg "$cmd images"
mkdir -p $DIR_IMAGES # all the files are stored in this dir

if [[ -t 0 ]]; then
  # stdin from terminal and no args are provided
  if (( $# < 1 )); then
    if [[ "$cmd" == "save" ]]; then
      images="$(eval $CMD_LIST)"
    else
      images="$(cd $DIR_IMAGES && find . -type f -name '*tar.gz')"
    fi
  fi
else
  # stdin from pipe or redirection
  images="$(cat)"
fi

# merge from image list file if provided
if [[ -n "$IMAGE_LIST" ]]; then
  images="$images $(read_image_list $IMAGE_LIST)"
fi

msg "enter directory: $DIR_IMAGES"
cd $DIR_IMAGES

# merge, sort and remove duplicated ones
for i in $(printf "%s\n" $images $@ | sort | uniq); do
  msg "processing $i"
  if [[ -n "$RE_WANTED" && ! $i =~ $RE_WANTED ]]; then
    msg "not wanted $i" && continue
  fi
  if [[ -n "$RE_IGNORE" && $i =~ $RE_IGNORE ]]; then
    msg "ignore $i" && continue
  fi
  if [[ "$cmd" == "save" ]]; then
    if [[ "$AUTO_PULL" == 'y' ]]; then
      ensure_image $i
    fi
    file="${i//:/-}.tar.gz"
    msg "saving $file"
    if [[ "$NO_CLOBBER" == 'y' ]]; then
      if [[ -e "$file" ]]; then
        msg "file exists, skip"
        continue
      fi
    fi
    if [[ "$DRY_RUN" != 'y' ]]; then
      mkdir -p $(dirname $i) && $CMD_SAVE $i | gzip > $file;
    fi
  else
    file="${i#./}" # assume it is a file name first
    if [[ "$file" != *.tar.gz && ! -e "$file" ]]; then
      file="${file//:/-}.tar.gz" # image name to file name
    fi
    msg "loading $file"
    if [[ "$DRY_RUN" != 'y' ]]; then
      zcat $file | $CMD_LOAD
    fi
  fi
done

msg "done"
EOFOF

    chmod a+x $CACHE_IMAGE
  fi
}

function reset_cluster() {
  msg "reset cluster"
  local params=
  if [[ $NEED_CRI_DOCKERD == "y" ]]; then
    params="$params --cri-socket $CRI_DOCKERD_SOCKET"
    msg "set cri-socket as $CRI_DOCKERD_SOCKET"
  elif [[ $CRI_RUNTIME == "containerd" ]]; then
    params="$params --cri-socket $CRI_CONTAINERD_SOCKET"
    msg "set cri-socket as $CRI_CONTAINERD_SOCKET"
  fi
  $SUDO kubeadm reset $params

  msg "release cni0 if any"
  if ip link show cni0 >/dev/null 2>&1; then
    $SUDO ip link set cni0 down
    $SUDO ip link delete cni0
  fi
  msg "clean cni config"
  $SUDO rm -rf /etc/cni/net.d/
}

function init_cluster() {
  msg "initialize cluster"

  if kubectl cluster-info >/dev/null 2>&1; then
    msg "already initialized"
    return
  fi

  [[ $OFFLINE == "y" ]] && msg "WARNING: offline mode, cache must be ready"

  local flannel_dir="$CACHE_MANIFESTS/flannel-$FLANNEL_VERSION"
  mkdir -p $flannel_dir

  local flannel="$flannel_dir/kube-flannel.yml"
  local flannel_mod="$CACHE_RUN/kube-flannel.mod.yml"
  msg "download flannel manifest"
  if [[ ! -e $flannel ]]; then
    local u="$(mirror_url $FLANNEL_BASEURL)/raw/v$FLANNEL_VERSION/Documentation/kube-flannel.yml"
    download_file $u $flannel
  fi

  local localpp_dir="$CACHE_MANIFESTS/local-path-provisioner-$LOCALPP_VERSION"
  mkdir -p $localpp_dir

  local localpp="$localpp_dir/local-path-storage.yaml"
  msg "download local-path-provisioner manifest"
  if [[ ! -e $localpp ]]; then
    local u="$(mirror_url $LOCALPP_BASEURL)/raw/v$LOCALPP_VERSION/deploy/local-path-storage.yaml"
    download_file $u $localpp
  fi

  # different CNI addon requires different pod-network-cider, here is for flannel
  local cidr=$(perl -ne 'print $1 if /"Network":\s*"([^"]+)"/' $flannel)
  local cidr_svc="10.96.0.0/16"

  local name="$HOST_NAME"
  local ckey="$(kubeadm certs certificate-key)"
  local token="$(kubeadm token generate)"
  local extras=
  local repo_config=
  local sans_config='  - "localhost"'
  local cpep_config=
  local ckey_config=
  local sock_config=
  local sock_option=

  if [[ -n $SANS ]]; then
    for s in $(echo $SANS | tr , '\n')
    do
      sans_config="$(printf '%s\n  - "%s"' "$sans_config" "$s")"
    done
  fi
  if [[ -n $ENDPOINT ]]; then
    # need HA
    cpep_config="controlPlaneEndpoint: \"$ENDPOINT\""
    ckey_config="certificateKey: \"$ckey\""
    extras="$extras --upload-certs"
    msg "control-plane endpoint specified, upload the certificates for HA setup"
  fi
  if [[ -n $NODE ]]; then
    name="$NODE"
  fi
  if [[ $USE_K8S_MIRROR == "y" ]]; then
    local repo="$K8S_REG_MIRROR"
    repo_config="imageRepository: \"$repo\""
    msg "mirror is required, use image repository: $repo"
  fi
  if [[ $NEED_CRI_DOCKERD == "y" ]]; then
    sock_option="--cri-socket $CRI_DOCKERD_SOCKET"
    sock_config="  criSocket: \"$CRI_DOCKERD_SOCKET\""
    msg "set cri-socket as $CRI_DOCKERD_SOCKET"
  elif [[ $CRI_RUNTIME == "containerd" ]]; then
    sock_option="--cri-socket $CRI_CONTAINERD_SOCKET"
    sock_config="  criSocket: \"$CRI_CONTAINERD_SOCKET\""
    msg "set cri-socket as $CRI_CONTAINERD_SOCKET"
  fi

  # use the same cgroup driver as the container runtime
  local cgroupdriver="systemd"
  if [[ $CRI_RUNTIME == 'docker' ]]; then
    cgroupdriver="$($CRI_CLIENT info -f {{.CgroupDriver}})"
  else
    if ! $CRI_RUNTIME config dump | grep -E 'SystemdCgroup\s*=\s*true'; then
      cgroupdriver="cgroupfs"
    fi
  fi

  cat <<EOF > $INIT_CONFIG
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
bootstrapTokens:
- token: "$token"
  description: "kubeadm bootstrap token"
  ttl: "24h"
localAPIEndpoint:
  advertiseAddress: "$ADDRESS"
nodeRegistration:
  name: "$name"
$sock_config
$ckey_config
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
kubernetesVersion: $VERSION
$cpep_config
networking:
  serviceSubnet: $cidr_svc
  podSubnet: "$cidr"
apiServer:
  certSANs:
$sans_config
$repo_config
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: $cgroupdriver
EOF

  # patch proxy env vars
  local no_proxy_more="$K8S_NOPROXY,$cidr,$cidr_svc"
  for e in no_proxy NO_PROXY
  do
    if [[ -n ${!e} ]]; then
      msg "patch $e: $no_proxy_more"
      eval $e="$no_proxy_more,${!e}"
    fi
  done

  if [[ $OFFLINE == "y" && -f $CACHE_IMAGE ]]; then
    msg "try to load cached container images"
    OCI_CLIENT="${CRI_CLIENT##* }" $CACHE_IMAGE load
  fi

  msg "init k8s control-plane node, ver:$VERSION, api:$ADDRESS"
  msg "details in $INIT_CONFIG"
  $SUDO env $(env | grep -i proxy) kubeadm init --config $INIT_CONFIG $extras

  # config
  msg "setup config to access cluster"
  mkdir -p $HOME/.kube
  $SUDO cp /etc/kubernetes/admin.conf $HOME/.kube/config
  $SUDO chown $(id -u):$(id -g) $HOME/.kube/config

  # untaint control-plane
  if [[ $NO_WORKLOAD != "y" ]]; then
    msg "allow to schedule pods on the control-plane"
    # new name was added at v1.24.0 and the old one was removed at v1.25.0
    # try them all for better compatibility
    for n in master control-plane; do
      kubectl taint nodes --all node-role.kubernetes.io/$n- 2>/dev/null || true
    done
  fi

  # install flannel to manage the network
  if [[ $BACKEND == 'none' ]]; then
    msg "no flannel is required, you should deploy CNI plugin by yourself"
  else
    msg "deploy flannel: $flannel_mod"
    local cmds=$(cat <<EOF
      if (/^\s+"Backend":\s*\{/ .. /^\s+\}\s*$/) {
        s/^(\s+)("Type":\s*).+$/\$1\$2"$BACKEND",\n\$1"PersistentKeepaliveInterval": 25/
      }
EOF
    )
    perl -pe "$cmds" $flannel > $flannel_mod
    kubectl apply -f $flannel_mod
  fi

  msg "deploy local-path-provisioner: $localpp"
  kubectl apply -f $localpp
  msg "set local-path as the default StorageClass"
  kubectl patch storageclass local-path -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'

  msg "generate instructions to join the cluster"
  msg "details in $JOIN_MANUAL"
  local apiserver="$ADDRESS:$PORT"
  if [[ -n $ENDPOINT ]]; then
    local host port
    IFS=":" read host port <<< "$ENDPOINT"
    if [[ -z $port ]]; then
      port="$PORT"
    fi
    apiserver="$host:$port"
  fi
  local algo="sha256"
  local capath="/etc/kubernetes/pki/ca.crt"
  local cahash="$(openssl x509 -pubkey -in $capath \
        | openssl rsa -pubin -outform der 2>/dev/null \
        | openssl dgst -$algo -hex | sed 's/^.* //')"

  cat <<EOF | tee $JOIN_MANUAL
# to join a worker node
sudo kubeadm join $apiserver $sock_option \\
  --token $token \\
  --discovery-token-ca-cert-hash $algo:$cahash

EOF

  if [[ -n $ENDPOINT ]]; then
    cat <<EOF | tee -a $JOIN_MANUAL
# to join a control-plane node
sudo kubeadm join $apiserver $sock_option \\
  --token $token \\
  --discovery-token-ca-cert-hash $algo:$cahash \\
  --control-plane \\
  --certificate-key $ckey

EOF
  fi

  cat <<EOF
# here are some useful commands:
# check status
kubectl get pods -A

# enable bash completion
source <(kubectl completion bash)
EOF
}

main "$@"