tricky

docker with proxy

[!NOTE|label:references:]

$ sudo mkdir -p /etc/systemd/system/docker.service.d
$ cat /etc/systemd/system/docker.service.d/http-proxy.conf
[Service]
Environment="HTTPS_PROXY=http://sample.proxy.com:80"
Environment="HTTP_PROXY=http://sample.proxy.com:80"
Environment="ALL_PROXY=http://sample.proxy.com:80"
systemctl daemon-reload;systemctl start docker

$ sudo systemctl daemon-reload
$ sudo systemctl restart docker

# verify
$ systemctl show docker --property Environment
Environment=HTTPS_PROXY=http://sample.proxy.com:80 HTTP_PROXY=http://sample.proxy.com:80 ALL_PROXY=http://sample.proxy.com:80
  • for socks5

    $ [ ! -d /etc/systemd/system/docker.service.d ] && sudo mkdir -p /etc/systemd/system/docker.service.d
    $ sudo bash -c "cat > /etc/systemd/system/docker.service.d/socks5-proxy.conf" << EOF
    [Service]
    Environment="ALL_PROXY=socks5://sample.proxy.com:80"
    Environment="NO_PROXY=localhost,127.0.0.1,130.147.0.0/16,130.145.0.0/16"
    EOF
    
    $ sudo systemctl daemon-reload
    $ sudo systemctl enable docker.service
    $ sudo systemctl restart docker.service
  • or

    # for rootless mode
    $ mkdir -p ~/.config/systemd/user/docker.service.d/
    # or regular mode
    $ sudo mkdir -p /etc/systemd/system/docker.service.d
    
    $ sudo bash -c "cat > /etc/systemd/system/docker.service.d" << EOF
    [Service]
    Environment="HTTP_PROXY=http://sample.proxy.com:80"
    Environment="HTTPS_PROXY=https://sample.proxy.com:443"
    Environment="NO_PROXY=localhost,127.0.0.1,sample.docker-registry.com,.corp"
    EOF
    
    $ sudo systemctl daemon-reload
    $ sudo systemctl restart docker
    
    # verify
    $ systemctl show docker --property Environment
    Environment=HTTP_PROXY=http://sample.proxy.com:80 HTTPS_PROXY=http://sample.proxy.com:443 NO_PROXY=localhost,127.0.0.1,sample.docker-registry.com,.corp

$ mkdir -p ~/.docker
$ cat > ~/.docker/config.json << EFO
{
 "proxies":
 {
   "default":
   {
     "httpProxy": "http://sample.proxy.com:80",
     "httpsProxy": "http://sample.proxy.com:80",
     "allProxy": "http://sample.proxy.com:80",
     "noProxy": "*.sample.domain1.com,.domain2.com,127.0.0.0/8"
   }
 }
}
EOF
  • or via --build-arg:

    $ docker build \
             --build-arg http_proxy=http://sample.proxy.com:80 \
             --build-arg https_proxy=http://sample.proxy.com:443 \
            .
  • details

    VARIABLEDOCKERFILE EXAMPLEDOCKER RUN EXAMPLE

    HTTP_PROXY

    ENV HTTP_PROXY="http://sample.proxy.com:80"

    --env HTTP_PROXY="http://sample.proxy.com:80"

    HTTPS_PROXY

    ENV HTTPS_PROXY="https://sample.proxy.com:80"

    --env HTTPS_PROXY="https://sample.proxy.com:80"

    FTP_PROXY

    ENV FTP_PROXY="ftp://sample.proxy.com:80"

    --env FTP_PROXY="ftp://sample.proxy.com:80"

    NO_PROXY

    ENV NO_PROXY="*.sample.domain1.com,.domain2.com"

    --env NO_PROXY="*.sample.domain1.com,.domain2.com,127.0.0.0/8"

via daemon.json

[!TIP]

$ cat /etc/docker/daemon.json
{
  "proxies": {
    "http-proxy": "http://proxy.example.com:80",
    "https-proxy": "https://proxy.example.com:443",
    "no-proxy": "*.test.example.com,.example.org"
  }
}

docker build with GPG key proxy

  • without proxy

    ...
    gpg --batch \
        --keyserver https://keyserver.ubuntu.com:80 \
        --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831E
    ...
    
    # result
    gpg: keyserver receive failed: Connection timed out
  • GPG with proxy

    ...
    apt-key adv --keyserver-options http-proxy=http://sample.proxy.com:80 \
                --keyserver hkp://keyserver.ubuntu.com:80 \
                --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF \
    ...
    
    # result
    Executing: /tmp/apt-key-gpghome.uegAG54mKu/gpg.1.sh --keyserver-options http-proxy=http://sample.proxy.com:80 --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
    gpg: key A6A19B38D3D831EF: 2 signatures not checked due to missing keys
    gpg: key A6A19B38D3D831EF: public key "Xamarin Public Jenkins (auto-signing) <releng@xamarin.com>" imported
    gpg: Total number processed: 1
    gpg:               imported: 1
    gpg: keybox '/tmp/tmp.jad0qVCQ6v/pubring.kbx' created
    gpg: WARNING: nothing exported
    Warning: apt-key output should not be parsed (stdout is not a terminal)

docker for osx

[!TIP|label:references:]

docker: command cannot be found

  • using user

    $ export $HOME/.docker/bin:$PATH
  • using system

    $ ln -sf $HOME/.docker/bin/docker /usr/local/bin/docker
    # or
    $ ln -s /Applications/Docker.app/Contents/Resources/bin/docker /usr/local/bin/docker
    
    $ sudo ln -s -f $HOME/.docker/run/docker.sock /var/run/docker.sock
    • modify ~/.docker/config.json if necessary

      {
              "auths": {},
      -        "credsStore": "desktop",
      +        "credStore": "desktop",
              "currentContext": "desktop-linux"
      }

command-line auto completion

[!TIP] reference:

Linux

$ dpkg -L docker-ce-cli | grep completion
/usr/share/bash-completion
/usr/share/bash-completion/completions
/usr/share/bash-completion/completions/docker
/usr/share/fish/vendor_completions.d
/usr/share/fish/vendor_completions.d/docker.fish
/usr/share/zsh/vendor-completions
/usr/share/zsh/vendor-completions/_docker
  • rpm

    $ rpm -ql docker-ce | grep completion
    /usr/share/bash-completion/completions/docker
    /usr/share/fish/vendor_completions.d/docker.fish
    /usr/share/zsh/vendor-completions/_docker

setup in bashrc

source /usr/share/bash-completion/completions/docker
alias d='docker'
alias dp='docker ps'

while read -r _i; do
  complete -F _docker "${_i}"
done < <(alias | grep docker | sed '/^alias /!d;s/^alias //;s/=.*$//')
  • more info

    $ source /usr/share/bash-completion/completions/docker
    $ type _docker | head
    _docker is a function
    _docker ()
    {
      local previous_extglob_setting=$(shopt -p extglob);
      shopt -s extglob;
    }

$ la '/Applications/Docker.app/Contents/Resources/etc'
total 332K
-rwxr-xr-x 1 marslo admin 124K Nov  9 21:50 docker.zsh-completion
-rwxr-xr-x 1 marslo admin  51K Nov  9 21:50 docker.fish-completion
-rwxr-xr-x 1 marslo admin 114K Nov  9 21:50 docker.bash-completion
-rw-r--r-- 1 marslo admin  18K Nov  9 21:50 docker-compose.zsh-completion
-rw-r--r-- 1 marslo admin 1.7K Nov  9 21:50 docker-compose.fish-completion
-rwxr-xr-x 1 marslo admin  13K Nov  9 21:50 docker-compose.bash-completion

$ ln -sf '/Applications/Docker.app/Contents/Resources/etc/docker.bash-completion' $(brew --prefix)/etc/bash_completion.d/docker
$ ln -sf '/Applications/Docker.app/Contents/Resources/etc/docker-compose.bash-completion' $(brew --prefix)/etc/bash_completion.d/docker-compose
  • setup in bashrc

    dockerComp="$(brew --prefix)/etc/bash_completion.d/docker"
    dockerComposeComp="$(brew --prefix)/etc/bash_completion.d/docker-compose"
    [ -f "${dockerComp}" ] && source "${dockerComp}"
    [ -f "${dockerComposeComp}" ] && source "${dockerComposeComp}"
    
    alias d='docker'
    alias dp='docker ps'
    alias dls='docker ps -l -q'
    alias dps='docker ps -l -a'
    alias di='docker images'
    alias dip="docker inspect --format '{{ .NetworkSettings.IPAddress }}'"
    
    while read -r _i; do
      complete -F _docker "${_i}"
    done < <(alias | grep docker | sed '/^alias /!d;s/^alias //;s/=.*$//')

  • _completion_loader

    _completion_loader()
    {
      . "/etc/bash_completion.d/$1.sh" >/dev/null 2>&1 && return 124
    }
    complete -D -F _completion_loader

# for Linux
$ sudo curl -sSLg https://raw.githubusercontent.com/cykerway/complete-alias/master/complete_alias \
            -o /etc/bash_completion.d/complete_alias

# for osx
$ sudo curl -sSLg https://raw.githubusercontent.com/cykerway/complete-alias/master/complete_alias \
            -o $(brew --prefix)/etc/bash_completion.d/complete_alias
  • setup in bash

    source /etc/bash_completion.d/complete_alias
    #
    source $(brew --prefix)/etc/bash_completion.d/complete_alias
    
    while read -r _i; do
      complete -F _complete_alias "${_i}"
    done < <(alias | grep -E 'docker|kubectl' | sed '/^alias /!d;s/^alias //;s/=.*$//')

get tags

  • list repos

    example: the docker registry in artifactory named docker

$ curl -sS https://artifactory.sample.com/v2/docker/_catalog |
       jq -r .repositories[]
  • or

    $ curl -sS -X GET https://artifactory.sample.com/artifactory/api/docker/docker/v2/_catalog |
           jq -r .repositories[]
  • list tags

    example: get tags from repo devops/ubuntu

    $ curl -sS https://artifactory.sample.com/artifactory/v2/docker/devops/ubuntu/tags/list [ | jq -r .tags[] ]
    • or

      $ curl -sS -X GET https://artifactory.sample.com/artifactory/api/docker/docker/v2/devops/ubuntu/tags/list

get image:tag via jf cli

$ jf rt search docker/devops/kwciagent/kw23.4-4.0.1** |
  jq -r '.[].props | select(."docker.manifest" != null) | [ ."docker.manifest"[0], ."docker.repoName"[0] ] | "\(.[1]):\(.[0])"'
devops/kwciagent:kw23.4-4.0.1-py310-jammy-dind
devops/kwciagent:kw23.4-4.0.1-py310-jammy-dind-v96-906236c7d

from docker hub

$ curl -sS 'https://hub.docker.com/v2/repositories/jenkins/jenkins/tags' |
       jq --raw-output .results[].name

jdk8-openj9-windowsservercore-1809
jdk11-hotspot-windowsservercore-1809
jdk11-openj9-windowsservercore-1809
windowsservercore-1809
jdk8-hotspot-windowsservercore-1809
2.249.3-lts-centos7
lts-centos7
centos7
2.249.3-lts-centos
lts-centos
  • get more

    $ curl -sS 'https://hub.docker.com/v2/repositories/jenkins/jenkins/tags?page_size=100&ordering=last_updated' |
           jq --raw-output .results[].name |
           sort
    • or

      $ curl -sS https://hub.docker.com/v2/repositories/jenkins/jenkins/tags?page=2 |
             jq '."results"[]["name"]' |
             sort
  • get multiple pages

    $ while read -r _i; do
        curl -sSgk "https://registry.hub.docker.com/v2/repositories/jenkins/jenkins/tags?&page=${_i}&page_size=100" |
             jq -r '.results[] | select( .name | contains("-lts-") ) | .name';
      done < <(echo {1..10} | fmt -1)
  • via docker registry tags/list API v2

    [!NOTE|label:references:]

    $ cat > docker-tags.sh << EOF
    #!/usr/bin/env bash
    set -eu -o pipefail
    docker_tags() {
      item="$1"
      case "$item" in
          */*) :                    ;; # namespace/repository syntax, leave as is
            *) item="library/$item" ;; # bare repository name (docker official image); must convert to namespace/repository syntax
      esac
      authUrl="https://auth.docker.io/token?service=registry.docker.io&scope=repository:$item:pull"
      token="$(curl -fsSL "$authUrl" | jq --raw-output '.token')"
      tagsUrl="https://registry-1.docker.io/v2/$item/tags/list"
      curl -fsSL -H "Accept: application/json" -H "Authorization: Bearer $token" "$tagsUrl" | jq --raw-output '.tags[]'
    }
    docker_tags "$@"
    EOF
    
    $ bash docker-tags.sh jenkins/jenkins | sort -r | grep --color=never -- '-lts-jdk11' | head -3
    2.440.2-lts-jdk11
    2.440.1-lts-jdk11
    2.426.3-lts-jdk11

simple script for get tags

#!/bin/sh
#
# Simple script that will display docker repository tags.
#
# Usage:
#   $ docker-show-repo-tags.sh ubuntu centos

for _r in $* ; do
  curl -sS "https://registry.hub.docker.com/v2/repositories/library/$_r/tags/" |
       sed -e 's/,/,\n/g' -e 's/\[/\[\n/g' |
       grep '"name"' |
       awk -F\" '{print $4;}' |
       sort -fu |
       sed -e "s/^/${_r}:/"
done

get current container ID

$ basename $(cat /proc/self/cpuset)
ab8c1732f1a3fdb46b9f9a477f0fbcc1d23c6787d7532648242a76d6eb1e8b84
  • or

    $ hostname
    ab8c1732f1a3

get volume from container ID

$ docker inspect -f '{{ .Mounts }}' <container ID>
  • or

    $ docker inspect <container ID> |
             grep volume
  • or get all

    $ docker ps -a --no-trunc --format "{{.ID}}\t{{.Names}}\t{{.Mounts}}"
  • or

    $ docker inspect <container ID> |
             jq --raw-output .[].Mounts
  • or

    $ docker ps -q |
             xargs docker container inspect -f '{{ .Name }} {{ .HostConfig.Binds }}'

mount volume in DinD

reference:

$ cid=$(basename $(cat /proc/self/cpuset))
$ VOLUME_OPTION="--volumes-from ${cid}:rw"
$ docker run <...> ${VOLUME_OPTION}

run inside windows docker image

add contents

> docker exec [-w 'C:\workspace'] <docker-id> powershell "Set-Content -Path '_cmd.ps1' -Value 'python --version'"
  • for multiple line contents

    > docker exec [-w 'C:\workspace'] <docker-id>' powershell "Set-Content -Path .\test.py -Value '# content of test_sample.py
    >> def func(x):
    >>     return x + 1
    >>
    >>
    >> def test_answer():
    >>     assert func(4) == 5'"

get contents

> docker exec [-w 'C:\workspace'] <docker-id> powershell "Get-Content '_cmd.ps1'"
  • equivalent tail -f

    > docker exec [-w 'C:\workspace'] <docker-id> powershell "Get-Content '_cmd.ps1' -Wait"
  • equivalent tail -10

    > docker exec [-w 'C:\workspace'] <docker-id> powershell "Get-Content '_cmd.ps1' -Tail 10"

list item

> docker exec [-w 'C:\workspace'] <docker-id> powershell Get-ChildItem .
> docker exec [-w 'C:\workspace'] <docker-id> powershell Get-ChildItem ..\

execute

> docker exec [-w 'C:\workspace'] <docker-id> powershell .\_cmd.ps1

# or
> docker exec [-w 'C:\workspace'] <docker-id> powershell "Invoke-Expression '.\_cmd.ps1' > log.txt"

# or redirect via `Out-File`
> docker exec [-w 'C:\workspace'] <docker-id> powershell "Invoke-Expression '.\_cmd.ps1' | Out-File -FilePath log.txt"

troubleshooting

permission denied while trying to connect to the Docker daemon socket

[!NOTE|label:see also:]

  • issue shows even if the account exists in docker group

    # account already been added in `docker` group
    $ id marslo
    uid=1100(marslo) gid=1100(marslo) groups=1100(marslo),994(docker)
    $ docker ps
    permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get "http://%2Fvar%2Frun%2Fdocker.sock/v1.44/containers/json": dial unix /var/run/docker.sock: connect: permission denied
    
    # group info
    $ getent group docker
    docker:x:994:devops,marslo
    $ getent group 994
    docker:x:994:devops,marslo
    
    # remote
    $ sudo gpasswd -d marslo docker
    Removing user marslo from group docker
    $ id marslo
    uid=1100(marslo) gid=1100(marslo) groups=1100(marslo)
    
    # re-added
    $ sudo usermod -aG docker marslo
    $ id marslo
    uid=1100(marslo) gid=1100(marslo) groups=1100(marslo),994(docker)
    $ docker ps
    permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get "http://%2Fvar%2Frun%2Fdocker.sock/v1.44/containers/json": dial unix /var/run/docker.sock: connect: permission denied
  • root cause

    # docker group-id was 990, and it was changed to 994; but the `/var/run/docker.sock` wasn't been changed
    $ ls -asltrh /var/run/docker.sock
    0 srw-rw---- 1 root redwillow 0 Mar  7 15:27 /var/run/docker.sock
  • solution

    $ sudo chown -R root:docker /var/run/docker.sock
    $ docker ps
    CONTAINER ID   IMAGE     COMMAND   CREATED   STATUS    PORTS     NAMES
    
    # to change all after GID changed
    $ find / -gid OLD_GID ! -type l -exec chgrp NEW_GID {} \;

Last updated