1 of 30

kube-apiserver

K8S Deep Dive

Kick D2

2 of 30

# -*- mode: ruby -*-

# vi: set ft=ruby :

BOX_IMAGE = "bento/ubuntu-18.04"

DOCKER_VERSION = "5:19.03.15~3-0~ubuntu-bionic"

K8S_GIT_TAG = "v1.19.16"

GO_VERSION = "1.17.6"

HOSTNAME = "kube-node"

Vagrant.configure("2") do |config|

config.vm.provision :shell, privileged: true, env: {"DOCKER_VERSION"=>DOCKER_VERSION, "GO_VERSION"=>GO_VERSION}, inline: $install_common_tools

config.vm.provision :shell, privileged: false, env: {"K8S_GIT_TAG"=>K8S_GIT_TAG}, inline: $git_clone

config.vm.define HOSTNAME do |subconfig|

subconfig.vm.box = BOX_IMAGE

subconfig.vm.hostname = HOSTNAME

subconfig.vm.network :private_network, ip: "192.168.100.2"

config.vm.provider "virtualbox" do |v|

v.memory = 8192

v.cpus = 8

end

# privileged: true (root)

$install_common_tools = <<-SCRIPT

## disable swap

swapoff -a

sed -i '/swap/d' /etc/fstab

## apt-get noninteractive

export DEBIAN_FRONTEND=noninteractive

## -qq : really quiet (except errors)

apt-get -qq update

## pre-requisite - gcc make

apt-get -qq install gcc make tree jq

## install Docker

apt-get -qq install apt-transport-https ca-certificates curl gnupg-agent software-properties-common &&

curl --stderr /dev/null -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - &&

add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" &&

apt-get -qq update &&

apt-get -qq install docker-ce=${DOCKER_VERSION} docker-ce-cli=${DOCKER_VERSION} containerd.io

cat > /etc/docker/daemon.json <<EOF

{

"exec-opts": ["native.cgroupdriver=systemd"],

"log-driver": "json-file",

"log-opts": {

"max-size": "100m"

},

"storage-driver": "overlay2"

}

EOF

systemctl restart docker

usermod -aG docker vagrant

## install Golang

curl --stderr /dev/null -O https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz

tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz

echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile

SCRIPT

# privileged: false (vagrant)

$git_clone = <<-SCRIPT

git clone https://github.com/kubernetes/kubernetes.git --branch ${K8S_GIT_TAG} --single-branch

## build

cd kubernetes

sudo make clean

sudo make all WHAT=cmd/kube-apiserver GOFLAGS=-v GOGCFLAGS="all=-N -l" GOLDFLAGS=""

sudo cp _output/bin/kube-apiserver /usr/local/bin/

sudo make all WHAT=cmd/kubectl GOFLAGS=-v

sudo cp _output/bin/kubectl /usr/local/bin/

cd $HOME

## install Delve

go install github.com/go-delve/delve/cmd/dlv@latest

sudo cp $HOME/go/bin/dlv /usr/local/bin

## run etcd

sudo docker container run --name etcd --rm --volume=$HOME/etcd-data:/default.etcd --detach --net=host quay.io/coreos/etcd

## run kube-apiserver

sudo nohup dlv \

--headless \

--continue \

--accept-multiclient \

--listen=:56268 \

--api-version=2 \

exec /usr/local/bin/kube-apiserver \

-- \

--etcd-servers=http://127.0.0.1:2379 \

> ./apiserver.log 2>&1 &

SCRIPT

실습준비�(m1 환경은 아래 노트참고)

## M1 (arm) 용 Vagrantfile

# -*- mode: ruby -*-

# vi: set ft=ruby :

BOX_IMAGE = "jeffnoxon/ubuntu-20.04-arm64"

BOX_VERSION = "1.0.0"

GO_VERSION = "1.17.6"

DOCKER_VERSION = "5:19.03.15~3-0"

K8S_GIT_TAG = "v1.19.16"

HOSTNAME = "kube-node"

Vagrant.configure("2") do |config|

config.vm.box = BOX_IMAGE

config.vm.box_version = BOX_VERSION

config.vm.define HOSTNAME do |subconfig|

subconfig.vm.hostname = HOSTNAME

subconfig.vm.network :private_network, ip: "192.168.100.2"

subconfig.vm.provider :parallels do |v|

v.memory = 2048

v.cpus = 2

end

subconfig.vm.provision :shell, privileged: true, env: {"GO_VERSION"=>GO_VERSION,"DOCKER_VERSION"=>DOCKER_VERSION}, inline: $pre_install

subconfig.vm.provision :shell, privileged: false, env: {"K8S_GIT_TAG"=>K8S_GIT_TAG}, inline: $git_clone

end

# privileged: true (root)

$pre_install = <<-SCRIPT

## disable swap

swapoff -a

sed -i '/swap/d' /etc/fstab

## pre-install

export DEBIAN_FRONTEND=noninteractive

echo ">>>> pre-install <<<<"

apt-get -qq update

apt-get -qq -y install gcc make pkg-config libseccomp-dev jq tree

## CPU architecture - amd64|arm64

ARCH=$(dpkg --print-architecture)

### install go

GO_PACKAGE="go${GO_VERSION}.linux-${ARCH}".tar.gz

curl --stderr /dev/null -OL https://storage.googleapis.com/golang/${GO_PACKAGE}

tar -C /usr/local -xzf ${GO_PACKAGE}

echo 'PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile

## install Docker

DOCKER_PACKAGE="${DOCKER_VERSION}~$(lsb_release -is | tr '[A-Z]' '[a-z]')-$(lsb_release -cs)"

apt-get -qq -y install apt-transport-https ca-certificates curl gnupg lsb-release &&

mkdir -p /etc/apt/keyrings

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg &&

echo \

"deb [arch=${ARCH} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \

$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null 2>&1 &&

apt-get -qq update &&

apt-get -qq -y install docker-ce=${DOCKER_PACKAGE} docker-ce-cli=${DOCKER_PACKAGE} docker-compose-plugin

## config Docker

cat > /etc/docker/daemon.json <<EOF

{

"exec-opts": ["native.cgroupdriver=systemd"],

"log-driver": "json-file",

"log-opts": {

"max-size": "100m"

},

"storage-driver": "overlay2"

}

EOF

systemctl restart docker

usermod -aG docker vagrant

SCRIPT

# privileged: false (vagrant)

$git_clone = <<-SCRIPT

git clone https://github.com/kubernetes/kubernetes.git --branch ${K8S_GIT_TAG} --single-branch

## build

cd kubernetes

sudo make clean

sudo make all WHAT=cmd/kube-apiserver GOFLAGS=-v GOGCFLAGS="all=-N -l" GOLDFLAGS=""

sudo cp _output/bin/kube-apiserver /usr/local/bin/

sudo make all WHAT=cmd/kubectl GOFLAGS=-v

sudo cp _output/bin/kubectl /usr/local/bin/

cd $HOME

## install Delve

go install github.com/go-delve/delve/cmd/dlv@latest

sudo cp $HOME/go/bin/dlv /usr/local/bin

## run etcd

sudo docker container run --name etcd --rm --volume=$HOME/etcd-data:/default.etcd --detach --net=host pachyderm/etcd:v3.5.5

## run kube-apiserver

sudo nohup dlv \

--headless \

--listen=:56268 \

--api-version=2 \

exec /usr/local/bin/kube-apiserver \

-- \

--etcd-servers=http://127.0.0.1:2379 \

> ./apiserver.log 2>&1 &

SCRIPT

3 of 30

https://kep.li/k8sdd2

실습준비

4 of 30

U

kube-apiserver

5 of 30

U

6 of 30

U

7 of 30

클라우드?

8 of 30

클라우드?

9 of 30

어떤 API 들이 있나요

https://medium.com/programming-kubernetes/building-stuff-with-the-kubernetes-api-1-cc50a3642

curl http://localhost:8080/

# .. or ..

# kubectl get --raw /

{

"paths": [

"/api",

"/api/v1",

"/apis",

...

"/ui",

"/ui/",

"/version"

]

}

U

10 of 30

어떤 API 들이 있나요 얘들이 필요로하는 API 들이 있어요

U

PATCH /api/v1/namespaces/default/pods/nginx

GET /api/v1/pods

GET /api/v1/nodes

GET /api/v1/namespaces/default/pods/nginx

DELETE /api/v1/namespaces/default/pods/busybox

POST /apis/apps/v1/namespaces/default/deployments

https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#-strong-api-overview-strong-

11 of 30

어떤 API 들이 있나요

https://cloud.redhat.com/blog/kubernetes-deep-dive-api-server-part-1

12 of 30

OpenAPI 스펙

https://kubernetes.io/docs/concepts/overview/kubernetes-api/

curl -k http://localhost:8080/openapi/v2 > kube-apiserver-openapi-spec.json

13 of 30

OpenAPI 스펙

https://kubernetes.io/docs/concepts/overview/kubernetes-api/

14 of 30

API ~ GVR

그룹 버전 리소스

/apis/batch/v1/namespaces/$NAMESPACES/pods

15 of 30

API 그룹/버전

https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/

kubectl api-versions

Print the supported API versions on the server, in the form of "group/version"

16 of 30

API 리소스

https://kubernetes.io/docs/reference/kubernetes-api/

kubectl api-resources

17 of 30

API ~ 확장

https://iximiuz.com/en/posts/kubernetes-operator-pattern/

18 of 30

kube-apiserver 구조

Aggregated Server

핸들러 체인

서버체인

KubeAPIServer

APIExtensionsServer

Aggregated Server

핸들러 체인
서버 체인
서버

19 of 30

핸들러 체인

TLS handshake

Panic Recovery

Authentication

Authorization

Timeout

…

Impersonation

20 of 30

서버 체인

https://mobile.twitter.com/hasheddan/status/1457390669937451015/photo/1

Aggregated server

21 of 30

실습

sudo nohup dlv --headless --listen=:56268 --api-version=2 exec /usr/local/bin/kube-apiserver -- --etcd-servers=http://127.0.0.1:2379

22 of 30

기동

cmd/kube-apiserver/apiserver.go

https://www.notion.so/kube-apiserver-ede702f14e0a4bdda2633f4469a275a3

Cobra ~ CLI framework

23 of 30

CreateServerChain

cmd/kube-apiserver/app/server.go

https://www.notion.so/kube-apiserver-ede702f14e0a4bdda2633f4469a275a3

24 of 30

CreateServerChain

cmd/kube-apiserver/app/server.go

https://www.notion.so/kube-apiserver-ede702f14e0a4bdda2633f4469a275a3

25 of 30

HandlerChain

src/k8s.io/apiserver/pkg/server/config.go

https://www.notion.so/kube-apiserver-ede702f14e0a4bdda2633f4469a275a3

26 of 30

Run

k8s.io/apiserver/pkg/server/genericapiserver.go

https://www.notion.so/kube-apiserver-ede702f14e0a4bdda2633f4469a275a3

27 of 30

Run

k8s.io/apiserver/pkg/server/genericapiserver.go

https://www.notion.so/kube-apiserver-ede702f14e0a4bdda2633f4469a275a3

28 of 30

Run

k8s.io/apiserver/pkg/server/secure_serving.go

https://www.notion.so/kube-apiserver-ede702f14e0a4bdda2633f4469a275a3

29 of 30

분석 토픽

기동 과정 분석
핸들러 체인 분석

Authentication “Client” 인증 처리 분석
Authorization “Client” 권한 처리 분석

서버 체인 분석

Aggregator 동작 분석

Aggregated Server와 내부 서버체인 분기 로직

리소스 핸들러 분석

Flow 분석
Conversion
Admission Webhook
API 메소드 처리 분석

일반적인 REST API
WATCH
CONNECT

ETCD I/O 분석

Aggregated Server 구현 ⇒ metrics-server

30 of 30

END