node

references:
Labels and Selectors

list node names

$ kubectl get no --no-headers -o=custom-columns=NAME:.metadata.name

# or
$ kubectl get nodes -o 'jsonpath={.items[*].metadata.name} | fmt -1

list docker images running in node

[!NOTE|label:references:]
List container images in Kubernetes cluster with SIZE (like docker image ls)

$ kubectl get node <node_name> -o json | jq -re '.status.images[] | select(.names[1]) | .names[1]'

list all Ready nodes

[!NOTE|label:references:]
* JSONPath to list all nodes in ready state except the ones which are tainted?
* list node status with jq
Kubectl Get Nodes: Why and How to Use It
Checking Kubernetes node status
Viewing and finding resources
# Check which nodes are ready
$ JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' \
  && kubectl get nodes -o jsonpath="$JSONPATH" | grep "Ready=True"
see also
* imarslo : list all ready pods

$ kubectl get nodes -o json | jq -r '.items[] | select(.spec.taints|not) | select(.status.conditions[].reason=="KubeletReady" and .status.conditions[].status=="True") | .metadata.name'
node-vm1
node1
node10
node11
node15
node4
node6
node8
node9

list status

$ kubectl get nodes -o jsonpath='{range .items[*]} {.metadata.name} {"\t"} {.status.conditions[?(@.type=="Ready")].status} {"\t"} {.spec.taints[].effect} {"\n"} {end}' | awk '$2=="True"' | awk '$3 != "NoSchedule"'
  node-vm1    True
  node1       True
  node10      True
  node11      True
  node15      True
  node4       True
  node6       True
  node8       True
  node9       True

$ kubectl get nodes -o jsonpath='{range .items[*]} {.metadata.name} {"\t"} {.status.conditions[?(@.type=="Ready")].status} {"\t"} {.spec.taints[].effect} {"\n"} {end}' | awk '$2=="True"'
  node-vm1    True
  node1       True
  node10      True
  node11      True
  node15      True
  node2       True    NoSchedule
  node3       True    NoSchedule
  node4       True
  node5       True    NoSchedule
  node6       True
  node8       True
  node9       True

list nodes metrcs

[!NOTE] references:
Need simple kubectl command to see cluster resource usage

$ kubectl get no --no-headers -o=custom-columns=NAME:.metadata.name | xargs -I {} sh -c 'echo   {} ; kubectl describe node {} | grep Allocated -A 5 | grep -ve Event -ve Allocated -ve percent -ve -- ; echo '

# alias
$ alias util='kubectl get no --no-headers -o=custom-columns=NAME:.metadata.name | fmt -1 | xargs -I {} sh -c '\''echo   {} ; kubectl describe node {} | grep Allocated -A 5 | grep -ve Event -ve Allocated -ve percent -ve -- ; echo '\'''

# simple way
$ kubectl describe nodes | grep 'Name:\|  cpu\|  memory'
# or
$ kubectl describe nodes | grep 'Name:\|Allocated' -A 5 | grep 'Name\|memory'
# or
$ kubectl describe nodes | awk '/Allocated resources/,/Events/' | grep -v "^Events:"
# or with node name
$ kubectl describe nodes | sed -n '/^Allocated /,/^Events:/ { /^  [^(]/ p; } ; /^Name: / p'

others

$ alias util='kubectl get nodes --no-headers | awk '\''{print $1}'\'' | xargs -I {} sh -c '\''echo {} ; kubectl describe node {} | grep Allocated -A 5 | grep -ve Event -ve Allocated -ve percent -ve -- ; echo '\'''

# Get CPU request total (we x20 because because each m3.large has 2 vcpus (2000m) )
alias cpualloc='util | grep % | awk '\''{print $1}'\'' | awk '\''{ sum += $1 } END { if (NR > 0) { print sum/(NR*20), "%\n" } }'\'''

# Get mem request total (we x75 because because each m3.large has 7.5G ram )
alias memalloc='util | grep % | awk '\''{print $5}'\'' | awk '\''{ sum += $1 } END { if (NR > 0) { print sum/(NR*75), "%\n" } }'\'''

cluster-wide allocation

$ kubectl get po --all-namespaces -o=jsonpath="{range .items[*]}{.metadata.namespace}:{.metadata.name}{'\n'}{range .spec.containers[*]}  {.name}:{.resources.requests.cpu}{'\n'}{end}{'\n'}{end}"

script

#!/bin/bash

set -e

KUBECTL="kubectl"
NODES=$($KUBECTL get nodes --no-headers -o custom-columns=NAME:.metadata.name)

function usage() {
  local node_count=0
  local total_percent_cpu=0
  local total_percent_mem=0
  local readonly nodes=$@

  for n in $nodes; do
    local requests=$($KUBECTL describe node $n | grep -A2 -E "^\\s*CPU Requests" | tail -n1)
    local percent_cpu=$(echo $requests | awk -F "[()%]" '{print $2}')
    local percent_mem=$(echo $requests | awk -F "[()%]" '{print $8}')
    echo "$n: ${percent_cpu}% CPU, ${percent_mem}% memory"

    node_count=$((node_count + 1))
    total_percent_cpu=$((total_percent_cpu + percent_cpu))
    total_percent_mem=$((total_percent_mem + percent_mem))
  done

  local readonly avg_percent_cpu=$((total_percent_cpu / node_count))
  local readonly avg_percent_mem=$((total_percent_mem / node_count))

  echo "Average usage: ${avg_percent_cpu}% CPU, ${avg_percent_mem}% memory."
}

usage $NODES

script

function kusage() {
    # Function returning resources usage on current kubernetes cluster
  local node_count=0
  local total_percent_cpu=0
  local total_percent_mem=0

    echo "NODE\t\t CPU_allocatable\t Memory_allocatable\t CPU_requests%\t Memory_requests%\t CPU_limits%\t Memory_limits%\t"
  for n in $(kubectl get nodes --no-headers -o custom-columns=NAME:.metadata.name); do
    local requests=$(kubectl describe node $n | grep -A2 -E "Resource" | tail -n1 | tr -d '(%)')
        local abs_cpu=$(echo $requests | awk '{print $2}')
    local percent_cpu=$(echo $requests | awk '{print $3}')
        local node_cpu=$(echo $abs_cpu $percent_cpu | tr -d 'mKi' | awk '{print int($1/$2*100)}')
        local allocatable_cpu=$(echo $node_cpu $abs_cpu | tr -d 'mKi' | awk '{print int($1 - $2)}')
        local percent_cpu_lim=$(echo $requests | awk '{print $5}')
        local requests=$(kubectl describe node $n | grep -A3 -E "Resource" | tail -n1 | tr -d '(%)')
        local abs_mem=$(echo $requests | awk '{print $2}')
    local percent_mem=$(echo $requests | awk '{print $3}')
        local node_mem=$(echo $abs_mem $percent_mem | tr -d 'mKi' | awk '{print int($1/$2*100)}')
        local allocatable_mem=$(echo $node_mem $abs_mem | tr -d 'mKi' | awk '{print int($1 - $2)}')
        local percent_mem_lim=$(echo $requests | awk '{print $5}')
    echo "$n\t ${allocatable_cpu}m\t\t\t ${allocatable_mem}Ki\t\t ${percent_cpu}%\t\t ${percent_mem}%\t\t\t ${percent_cpu_lim}%\t\t ${percent_mem_lim}%\t"

    node_count=$((node_count + 1))
    total_percent_cpu=$((total_percent_cpu + percent_cpu))
    total_percent_mem=$((total_percent_mem + percent_mem))
  done

  local avg_percent_cpu=$((total_percent_cpu / node_count))
  local avg_percent_mem=$((total_percent_mem / node_count))

  echo "Average usage (requests) : ${avg_percent_cpu}% CPU, ${avg_percent_mem}% memory."
}

with status

$ kubectl get nodes -o jsonpath='{range .items[*]} {.metadata.name} {"\t"} {.status.conditions[?(@.type=="Ready")].status} {"\t"} {.spec.taints[].effect} {"\n"} {end}'
  node-vm1    True
  node1       True
  node10      True
  node11      True
  node12      Unknown   NoSchedule
  node13      Unknown   NoSchedule
  node14      Unknown   NoSchedule
  node15      True
  node16      Unknown   NoSchedule
  node17      Unknown   NoSchedule
  node2       True      NoSchedule
  node3       True      NoSchedule
  node4       True
  node5       True      NoSchedule
  node6       True
  node7       Unknown   NoSchedule
  node8       True
  node9       True

list all status

$ kubectl get nodes -o jsonpath="{range .items[*]} {\"\n\"} {@.metadata.name} :{\"\n\"} {\"\t\"}{range @.status.conditions[*]}{@.type}={@.status} {\"\n\t\"} {end}{end}"

node1 :
  ReadonlyFilesystem=False
  KernelDeadlock=False
  OutOfDisk=False
  MemoryPressure=False
  DiskPressure=False
  PIDPressure=False
  Ready=True

node2 :
  ReadonlyFilesystem=False
  KernelDeadlock=False
  OutOfDisk=Unknown
  MemoryPressure=Unknown
  DiskPressure=Unknown
  PIDPressure=False
  Ready=Unknown

node4 :
  OutOfDisk=Unknown
  MemoryPressure=Unknown
  DiskPressure=Unknown
  PIDPressure=False
  Ready=Unknown

node-vm1 :
  OutOfDisk=False
  MemoryPressure=False
  DiskPressure=False
  PIDPressure=False
  Ready=True

list node with label

$ kubectl get node -l <label>=<value>

list node with multiple labels

[!TIP|label:tips:]
* LIST and WATCH filtering

$ kubectl get node --selector <label>=<value>,<label>=<value>

# or
$ kubectl get node -l '<label> in (<value>), <label> in (<value>)'

# or for same label, different values
$ kubectl get node -l '<label> in (<value_1>, <value_2>)'
# i.e.:
$ kubectl get pods -l 'environment in (production, qa)'

# or `notin`
$ kubectl get node -l '<label> notin (<value>)'

update label of node

$ kubectl label node <name> <label>=<value> [--overwrite]

show

show with labels

$ kubectl get node --show-labels

show particular labels

--label-columns

$ kubectl get node --label-columns <label-name>

e.g.:

$ kubectl get nodes --label-columns jenkins
NAME          STATUS    ROLES    AGE     VERSION   JENKINS
k8s-node01    Ready     worker   545d    v1.12.3
k8s-node02    Ready     worker   597d    v1.12.3
k8s-node03    Ready     worker   217d    v1.12.3
k8s-node04    Ready     worker   52d     v1.12.3
k8s-node05    Ready     worker   589d    v1.12.3
k8s-node06    Ready     master   2y33d   v1.12.3   controller
k8s-node07    Ready     master   589d    v1.12.3
k8s-node08    Ready     worker   535d    v1.12.3

with multiple label-columns

$ kubectl get nodes --selector node-role.kubernetes.io/worker=worker  \
                    --label-columns kubernetes.io/os \
                    --label-columns kubernetes.io/arch \
                    --label-columns devops/jenkins |
          grep -v -E 'SchedulingDisabled|NotReady'

NAME          STATUS   STATUS    ROLES    VERSION   OS      ARCH    JENKINS
k8s-node01    Ready    Ready     worker   v1.19.6   linux   amd64
k8s-node02    Ready    Ready     worker   v1.19.6   linux   amd64
k8s-node03    Ready    Ready     worker   v1.19.6   linux   amd64   controller
k8s-node04    Ready    Ready     worker   v1.19.6   linux   amd64
k8s-node05    Ready    Ready     worker   v1.19.6   linux   amd64
k8s-node06    Ready    Ready     master   v1.19.6   linux   amd64
k8s-node07    Ready    Ready     master   v1.19.6   linux   amd64
k8s-node08    Ready    Ready     worker   v1.19.6   linux   amd64

-l

$ kubectl get node --show-labels -l node -role.kubernetes.io/master

show with particular columns

$ kubectl get node -o custom-columns=NAME:.metadata.name,VER:.status.nodeInfo.kubeletVersion
NAME          VER
k8s-node01    v1.12.3
k8s-node02    v1.12.3
k8s-node03    v1.12.3
...

show only scheduled nodes

$ kubectl get node \
          --output 'jsonpath={range $.items[*]}{.metadata.name} {.spec.taints[*].effect}{"\n"}{end}' |
          awk '!/NoSchedule/{print $1}'

show common/diff images between nodes

common

$ comm -1 -2 \
       <(kubectl get node node-01 -o json | jq -re '.status.images[] | select(.names[1]) | .names[1]' | sort) \
       <(kubectl get node node-02 -o json | jq -re '.status.images[] | select(.names[1]) | .names[1]' | sort)

diff

[!NOTE|label:references:]
iMarslo: directory diff

$ diff --suppress-common-lines \
       --side-by-side \
       <(kubectl get node node-01 -o json | jq -re '.status.images[] | select(.names[1]) | .names[1]' | sort) \
       <(kubectl get node node-02 -o json | jq -re '.status.images[] | select(.names[1]) | .names[1]' | sort)

delete node

[!NOTE|label:references:]
How to gracefully remove a node from Kubernetes?
kubectl_drain.svg

# get info
$ kubectl get nodes

$ kubectl cordon <node_name>

$ kubectl drain <node_name>
# or
$ kubectl drain <node_name> --ignore-daemonsets --delete-local-data

$ kubectl delete node <node_name>

cleanup label

$ kubectl label node <node_name> <label>-

example

$ kubectl get node -l jenkins.master
k8s-node01    Ready    worker   1d     v1.12.3
k8s-node02    Ready    worker   1d     v1.12.3
k8s-node03    Ready    worker   1d     v1.12.3
$ kubectl label node master-01 jenkins.master-
$ kubectl get node -l jenkins.master
k8s-node02    Ready    worker   1d     v1.12.3
k8s-node03    Ready    worker   1d     v1.12.3

sort

sort via kubelet version

$ kubectl get node --sort-by={.status.nodeInfo.kubeletVersion}

$ kubectl get nodes --sort-by={.metadata.labels."kubernetes\.io\/role"}

Previoussecrets Nextcertificates

Last updated 4 months ago