kubernetes

namespace

namespace

$ cat << EOF | kubectl apply -f -
  ---
  apiVersion: v1
  kind: Namespace
  metadata:
    name: jenkins
EOF
namespace/jenkins created

quota

$ cat << EOF | kubectl apply -f -
  apiVersion: v1
  kind: ResourceQuota
  metadata:
    name: jenkins-quota
    namespace: jenkins
  spec:
    hard:
      requests.cpu: "16"
      requests.memory: 16Gi
      limits.cpu: "32"
      limits.memory: 32Gi
EOF
resourcequota/jenkins-quota edited

sa.yml

  • jenkins.io

      ---
      apiVersion: rbac.authorization.k8s.io/v1
      kind: ClusterRole
      metadata:
        name: jenkins-admin
      rules:
        - apiGroups: [""]
          resources: ["*"]
          verbs: ["*"]
      ---
      apiVersion: v1
      kind: ServiceAccount
      metadata:
        name: jenkins-admin
        namespace: devops-tools
      ---
      apiVersion: rbac.authorization.k8s.io/v1
      kind: ClusterRoleBinding
      metadata:
        name: jenkins-admin
      roleRef:
        apiGroup: rbac.authorization.k8s.io
        kind: ClusterRole
        name: jenkins-admin
      subjects:
      - kind: ServiceAccount
        name: jenkins-admin
        namespace: devops-tools
  • or

    apiVersion: v1
    kind: ServiceAccount
    metadata:
      name: jenkins-admin
      namespace: devops-tools
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: Role
    metadata:
      name: jenkins
      namespace: devops-tools
      labels:
        "app.kubernetes.io/name": 'jenkins'
    rules:
    - apiGroups: [""]
      resources: ["pods"]
      verbs: ["create","delete","get","list","patch","update","watch"]
    - apiGroups: [""]
      resources: ["pods/exec"]
      verbs: ["create","delete","get","list","patch","update","watch"]
    - apiGroups: [""]
      resources: ["pods/log"]
      verbs: ["get","list","watch"]
    - apiGroups: [""]
      resources: ["secrets"]
      verbs: ["get"]
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: RoleBinding
    metadata:
      name: jenkins-role-binding
      namespace: devops-tools
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: Role
      name: jenkins
    subjects:
    - kind: ServiceAccount
      name: jenkins-admin
      namespace: devops-tools
  • grant more permissions via RoleBinding

    apiVersion: v1
    kind: ServiceAccount
    metadata:
      name: jenkins
      namespace: devops
    ---
    kind: ClusterRole
    apiVersion: rbac.authorization.k8s.io/v1beta1
    metadata:
      name: jenkins
    rules:
      - apiGroups: ["extensions", "apps"]
        resources: ["deployments"]
        verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
      - apiGroups: [""]
        resources: ["services"]
        verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
      - apiGroups: [""]
        resources: ["pods"]
        verbs: ["create","delete","get","list","patch","update","watch"]
      - apiGroups: [""]
        resources: ["pods/exec"]
        verbs: ["create","delete","get","list","patch","update","watch"]
      - apiGroups: [""]
        resources: ["pods/log"]
        verbs: ["get","list","watch"]
      - apiGroups: [""]
        resources: ["secrets"]
        verbs: ["get"]
    ---
    apiVersion: rbac.authorization.k8s.io/v1beta1
    kind: ClusterRoleBinding
    metadata:
      name: jenkins
      namespace: devops
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: ClusterRole
      name: jenkins
    subjects:
      - kind: ServiceAccount
        name: jenkins
        namespace: devops

generate credentials for pfx

ca.crt

$ grep certificate-authority-data ~/.kube/config | awkubectl -F': ' '{print $NF}' |  base64 -d > ca.crt
# OR
$ sudo cat /etc/kubernetes/pki/ca.crt

client.crt & client.key

$ grep client-certificate-data ~/.kube/config | awkubectl -F': ' '{print $NF}' |  base64 -d > client.crt
$ grep client-key-data ~/.kube/config | awkubectl -F': ' '{print $NF}' |  base64 -d > client.key

cert.pfx

$ openssl pkcs12 -export -out cert.pfx -inkey client.key -in client.crt -certfile ca.crt
Enter Export Password:
Verifying - Enter Export Password:

$ ls
ca.crt  cert.pfx  client.crt  client.key

# or

# using password 'marslo'
$ openssl pkcs12 -export -out cert.pfx -inkey client.key -in client.crt -certfile ca.crt -password pass:marslo

full steps

$ cat ~/.kube/config \
      | grep certificate-authority-data \
      | awkubectl '{print $2}' \
      | base64 -d > ca.crt
$ cat ~/.kube/config \
      | grep client-certificate-data \
      | awkubectl '{print $2}' \
      | base64 -d > client.crt
$ cat ~/.kube/config \
      | grep client-key-data \
      | awkubectl '{print $2}' \
      | base64 -d > client.key
$ openssl pkcs12 -export \
                 -out cert.pfx \
                 -in client.crt \
                 -inkey client.key \
                 -certfile ca.crt \
                 -password pass:devops

configure in jenkins

[!NOTE]

  • url

    • by default: http://<service-name>.<namespace>.svc.cluster.local:8080

  • Manage Jenkins -> Configure System or Manage Jenkins -> Manage Nodes and Clouds -> Configure Clouds

  • Add a new Cloud -> Kuberentes

    • Name:

    • Kubernetes URL:

      • get from $ kubectl cluster-info

      • using https://kubernetes.default.svc.cluster.local

    • Kubernetes server certificate key: content of ca.crt. ($ cat ca.crt)

    • Credentials:

      • Add -> Jenkins

      • Kind: Certificate

  • setup in jenkins

using kubeconfig for remote cluster credential

get Kubernetes URL

$ kubectl config view --minify | sed -n -re 's/^.*server: (https.*)$/\1/p'
  • or

    $ kubectl config view --minify --raw --output 'jsonpath={..cluster.server}'

generate CA

the content can be also found in kubernetes-master:/etc/kubernetes/pki/ca.crt

$ kubectl -n jenkins get secret \
              $(kubectl -n jenkins get sa jenkins-admin -o jsonpath={.secrets[0].name}) \
              -o jsonpath={.data.'ca\.crt'} \
              | base64 --decode
  • or

    $ cat ~/.kube/config \
          | grep certificate-authority-data \
          | awkubectl '{print $2}' \
          | base64 -d > ca.crt

Generate token in kubernetes

$ namespace='devops'
$ serviceAccount='jenkins-admin'
$ alias k='kubectl'
  • setup sa

    $ kubectl -n jenkins create sa jenkins-admin
    $ kubectl -n jenkins create rolebinding jenkins-admin-binding \
                         --clusterrole=cluster-admin \
                         --serviceaccount=devops:jenkins-admin
  • get token

    $ kubectl -n jenkins \
           get sa jenkins-admin \
           -o go-template \
           --template='{{range .secrets}}{{.name}}{{"\n"}}{{end}}'
    jenkins-admin-token-kshsh
    
    $ kubectl -n jenkins \
           get secrets jenkins-admin-token-kshsh \
           -o go-template \
           --template '{{index .data "token"}}' \
           | base64 -d
    eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.***
    • more info

      $ kubectl get secret -n jenkins
      NAME                        TYPE                                  DATA   AGE
      default-token-8k7vj         kubernetes.io/service-account-token   3      27m
      jenkins-admin-token-9r8pt   kubernetes.io/service-account-token   3      21m
      
      $ kubectl -n jenkins get rolebinding
      NAME                    AGE
      jenkins-admin-binding   15m
      $ kubectl -n jenkins describe rolebinding jenkins-admin-binding
      Name:         jenkins-admin-binding
      Labels:       <none>
      Annotations:  <none>
      Role:
        Kind:  ClusterRole
        Name:  cluster-admin
      Subjects:
        Kind            Name           Namespace
        ----            ----           ---------
        ServiceAccount  jenkins-admin  jenkins
      
      $ kubectl describe clusterrolebindings jenkins-admin-cluster-binding
      Name:         jenkins-admin-cluster-binding
      Labels:       <none>
      Annotations:  <none>
      Role:
        Kind:  ClusterRole
        Name:  cluster-admin
      Subjects:
        Kind            Name           Namespace
        ----            ----           ---------
        ServiceAccount  jenkins-admin  jenkins
    • or

      $ kubectl -n jenkins \
          get secret \
          $(kubectl -n jenkins get sa jenkins-admin -o jsonpath={.secrets[0].name}) \
          -o jsonpath={.data.token} \
          | base64 --decode
    • or

      $ kubectl -n jenkins \
             get sa jenkins-admin \
             --template='{{range .secrets}}{{ .name }} {{end}}' \
             | xargs -n 1 kubectl -n jenkins get secret \
                            --template='{{ if .data.token }}{{ .data.token }}{{end}}' \
                            | head -n 1 \
                            | base64 -d -
    • or

      # sa='jenkins-admin'
      $ kubectl -n jenkins \
                   get secrets \
                   $(kubectl -n jenkins get sa ${sa} -o=jsonpath='{.secrets[0].name}') \
                   -o=jsonpath='{.data.token}' |
                base64 -d

setup in Jenkins

using ClusterRoleBinding

[!TIP] simplely it can be executed via commands:

$ kubectl -n kube-system create sa <service-account>
$ kubectl create clusterrolebinding <role-binding-name> --clusterrole cluster-admin --serviceaccount=<namespace>:<service-account>

[!NOTE]: <role-binding-name> can be the same as <service-account>

---
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: jenkins
  name: jenkins-admin
  namespace: jenkins
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: jenkins-admin
  labels:
    k8s-app: jenkins
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: jenkins-admin
  namespace: jenkins
  • using RoleBinding

    ---
    apiVersion: rbac.authorization.k8s.io/v1beta1
    kind: Role
    metadata:
     name: jenkins-admin
     namespace: jenkins
    rules:
    - apiGroups: [""]
     resources: ["pods"]
     verbs: ["create","delete","get","list","patch","update","watch"]
    - apiGroups: [""]
     resources: ["pods/exec"]
     verbs: ["create","delete","get","list","patch","update","watch"]
    - apiGroups: [""]
     resources: ["pods/log"]
      verbs: ["get","list","watch"]
    ---
    apiVersion: rbac.authorization.k8s.io/v1beta1
    kind: RoleBinding
    metadata:
     name: jenkins-admin
     namespace: jenkins
    roleRef:
     apiGroup: rbac.authorization.k8s.io
     kind: Role
     name: jenkins-admin
    subjects:
    - kind: ServiceAccount
     name: jenkins-admin
  • or grant more permissions via RoleBinding

    apiVersion: v1
    kind: ServiceAccount
    metadata:
      name: jenkins
      namespace: devops
    ---
    kind: ClusterRole
    apiVersion: rbac.authorization.k8s.io/v1beta1
    metadata:
      name: jenkins
    rules:
      - apiGroups: ["extensions", "apps"]
        resources: ["deployments"]
        verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
      - apiGroups: [""]
        resources: ["services"]
        verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
      - apiGroups: [""]
        resources: ["pods"]
        verbs: ["create","delete","get","list","patch","update","watch"]
      - apiGroups: [""]
        resources: ["pods/exec"]
        verbs: ["create","delete","get","list","patch","update","watch"]
      - apiGroups: [""]
        resources: ["pods/log"]
        verbs: ["get","list","watch"]
      - apiGroups: [""]
        resources: ["secrets"]
        verbs: ["get"]
    ---
    apiVersion: rbac.authorization.k8s.io/v1beta1
    kind: ClusterRoleBinding
    metadata:
      name: jenkins
      namespace: devops
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: ClusterRole
      name: jenkins
    subjects:
      - kind: ServiceAccount
        name: jenkins
        namespace: devops
  • or get all permission via RoleBinding

    # spinnaker-role-and-rolebinding-target.yml
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: Role
    metadata:
      name: spinnaker-role
      namespace: target                       # Should be namespace you are granting access to
    rules:
    - apiGroups: ["*"]
      resources: ["*"]
      verbs: ["*"]
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: RoleBinding
    metadata:
      name: spinnaker-rolebinding
      namespace: target                       # Should be namespace you are granting access to
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: Role
      name: spinnaker-role                    # Should match name of Role
    subjects:
    - namespace: source                       # Should match namespace where SA lives
      kind: ServiceAccount
      name: spinnaker-service-account         # Should match service account name, above
  • or details rules in role binding

    apiVersion: v1
    kind: ServiceAccount
    metadata:
      name: jenkins-admin
      namespace: devops-tools
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: Role
    metadata:
      name: jenkins
      namespace: devops-tools
      labels:
        "app.kubernetes.io/name": 'jenkins'
    rules:
    - apiGroups: [""]
      resources: ["pods"]
      verbs: ["create","delete","get","list","patch","update","watch"]
    - apiGroups: [""]
      resources: ["pods/exec"]
      verbs: ["create","delete","get","list","patch","update","watch"]
    - apiGroups: [""]
      resources: ["pods/log"]
      verbs: ["get","list","watch"]
    - apiGroups: [""]
      resources: ["secrets"]
      verbs: ["get"]
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: RoleBinding
    metadata:
      name: jenkins-role-binding
      namespace: devops-tools
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: Role
      name: jenkins
    subjects:
    - kind: ServiceAccount
      name: jenkins-admin
      namespace: devops-tools

pull with credentials

[!NOTE|label:references:]

  • in kubernetes

    # create secrets in namespace
    
    ## via ~/.docker/config.json
    $ kubectl -n devfops create secret generic image-pull-secrets \
              --from-file=.dockerconfigjson=.docker/config.json \
              --type=kubernetes.io/dockerconfigjson
    
    ## via cmd
    $ kubectl -n devops create secret docker-registry image-pull-secrets \
              --docker-server=artifactory.example.com \
              --docker-username=devops \
              --docker-password=password \
              --docker-email=devops@example.com
    
    # check result
    $ kubectl -n devop get secret image-pull-secrets \
              -o jsonpath="{.data.\.dockerconfigjson}" |
      base64 -d
  • copy secrets to all namespaces

    # copy
    $ kubectl get ns -o custom-columns=":metadata.name" --no-headers |
              xargs -t -i bash -c "echo -e \"\\n-- {} --\"; kubectl -n devops get secrets image-pull-secrets -o yaml --export | kubectl apply -n {} -f -"
    
    # check
    $ kubectl get secrets --all-namespaces  | grep image-pull-secrets
  • in podTemplate

    podTemplate( cloud: 'Staging Kubernetes',
                 label: env.BUILD_TAG ,
                 name: env.BUILD_TAG,
                 showRawYaml: true,
                 namespace: 'devops',
                 yaml: """
                   apiVersion: v1
                   kind: Pod
                   spec:
                     hostNetwork: true
                     nodeSelector:
                       kubernetes.io/hostname: "staging-node-1"
                     imagePullSecrets:
                     - name: "image-pull-secrets"             # secrets name here
                     containers:
                     - name: jnlp
                       image: 'artifactory.example.com/docker/sandbox:bionic'
                       workingDir: '/home/devops'
                       resources:
                         limits:
                           memory: "1024Mi"
                           cpu: "512m"
                         requests:
                           memory: "512Mi"
                           cpu: "256m"
                 """
    ) { node( env.BUILD_TAG ) { container('jnlp') {
        stage('show info') {
          sh """
            id
            whoami
            echo ${WORKSPACE}
            realpath ${WORKSPACE}
          """
          println POD_CONTAINER
        }
      }}
    }

Q&A

[!NOTE]

$ kubectl -n <namespace> create rolebinding <sa>-binding \
                                --clusterrole=cluster-admin \
                                --serviceaccount=<namespace>:<sa>
$ kubectl create clusterrolebinding jenkins-admin-cluster-binding \
                                    --clusterrole cluster-admin \
                                    --serviceaccount=<namespace>:jenkins-admin
  • thinking

    $ kubectl -n jenkins auth can-i list pods --as jenkins-admin
    no
    
    $ kubectl -n jenkins auth can-i list pods --as jenkins-admin-binding
    no
    
    $ kubectl -n jenkins auth can-i list pods --as jenkins-admin-cluster-binding
    no

Last updated