config

reference:

installation

$ python -m pip completion --bash >> ~/.bashrc
  • or

    $ python -m pip completion --bash >> ~/.profile
  • or

    $ eval "`pip completion --bash`"

[!NOTE|label:references:]

  • basic environment prepare

    # centos
    $ sudo dnf groupinstall 'development tools'
    $ sudo dnf install gcc openssl-devel bzip2-devel libffi-devel \
    $ sudo dnf install expat-devel gdbm-devel ncurses-devel \
                       readline-devel sqlite-devel tk-devel xz-devel zlib-devel
    
    # ubuntu
    $ sudo apt install build-essential checkinstall pkg-config \
                       libexpat1-dev libncursesw5-dev libssl-dev \
                       libreadline8 libreadline-dev libsqlite3-dev \
                       tk-dev libgdbm-dev libc6-dev libbz2-dev \
                       libnss3-dev zlib1g-dev
  • download source code

    $ curl -O https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tgz
    $ tar xzf Python-3.8.3.tgz
    
    # or
    $ PYTHON_VERSION='3.12.2'
    $ curl -fsSL https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz |
      tar xzf - -C /opt/python &&
      cd /opt/python/Python-${PYTHON_VERSION}
  • compile and install

    [!NOTE|label:references:]

    $ ./configure --enable-optimizations \
                  --enable-shared \                 # or export PYTHON_CONFIGURE_OPTS="--enable-shared"
                  --with-lto \
                  --with-system-expat \
                  --with-ensurepip \
                  --with-openssl=/usr/lib/ssl
    $ sudo make -j "$(nproc)"
    $ sudo make install
    
    $ export PYTHON_HOME='/opt/python/Python-3.12.2'
    # for libpython3.12.so.1.0
    $ export LD_LIBRARY_PATH=$PYTHON_HOME:$LD_LIBRARY_PATH
    
    # check
    $ which -a python3.12
    /usr/local/bin/python3.12
    $ python3 --version
    Python 3.12.2
    
    # upgrade pip
    $ sudo -H /usr/local/bin/python3.12 -m pip install --upgrade pip
    $ cd Python-3.8.3
    $ sudo ./configure --enable-optimizations
    $ sudo make -j 12
    $ sudo make altinstall
    • linker cache

      [!NOTE|label:references:]

      $ sudo ldconfig -v | grep python
              libpython3.so -> libpython3.so
              libpython3.6m.so.1.0 -> libpython3.6m.so.1.0
      
      $ sudo ldconfig -v /opt/python/Python-3.12.2 | grep python
      /opt/python/Python-3.12.2: (from <cmdline>:0)
              libpython3.so -> libpython3.so
              libpython3.12.so.1.0 -> libpython3.12.so.1.0
              libpython3.so -> libpython3.so
              libpython3.6m.so.1.0 -> libpython3.6m.so.1.0
  • setup

    $ sudo update-alternatives --install /usr/bin/python3 python3 /usr/local/bin/python3.8 99

environment in MacOS

pip.conf

pip.conf load priority

  • MacOS : /Library/Application Support/pip/pip.conf > ~/.config/pip/pip.conf > ~/.pip/pip.conf

  • user: ~/.pip/pip.conf & ~/.config/pip/pip.conf

  • global: /Library/Application Support/pip/pip.conf

  • list config:

    $ pip config list [ -v ]
    global.index-url='https://artifactory.sample.com/artifactory/api/pypi/tools/simple'
    • details

      $ pip config list -v
      For variant 'global', will try loading '/Library/Application Support/pip/pip.conf'
      For variant 'user', will try loading '/Users/marslo/.pip/pip.conf'
      For variant 'user', will try loading '/Users/marslo/.config/pip/pip.conf'
      For variant 'site', will try loading '/usr/local/opt/python@3.10/Frameworks/Python.framework/Versions/3.10/pip.conf'
      global.extra-index-url='https://artifactory.sample.com/artifactory/api/pypi/myPrivate'
      global.index-url='https://artifactory.sample.com/artifactory/api/pypi/pypi/simple'
  • PIP_CONF_FILE

    $ export PIP_CONFIG_FILE=/path/to/pip.conf
  • upgrade all outdated modules

    $ pip install --upgrade --user $(pip list --outdated | sed 1,2d | awk '{print $1}' | xargs)
    • with exclude

      $ pip3.9 install --upgrade --user $(pip3.9 list --outdated | sed 1,2d | awk '{print $1}' | grep -vw 'docker\|rich')

  • naming

    [global]
    timeout = 60
    index-url = https://download.zope.org/ppix
  • per-command section

    [global]
    timeout = 60
    
    [freeze]
    timeout = 10
  • boolean options

    [install]
    ignore-installed = true
    no-dependencies = yes
    • add

      [global]
      no-cache-dir = false
      
      [install]
      no-compile = no
      no-warn-script-location = false
  • repeatable options

    [global]
    quiet = 0
    verbose = 2
  • format: on multiple lines

    [global]
    find-links =
        http://download.example.com
    
    [install]
    find-links =
        http://mirror1.example.com
        http://mirror2.example.com
    
    trusted-host =
        mirror1.example.com
        mirror2.example.com

$ python -vv -c "import sys; print sys.path"
$ python -vvE -c "import sys; print sys.path"
$ python -vvEsS -c "import sys; print sys.path"

python libs

MacOS

  • global

 $ ls -ld /usr/local/lib/python*/
 drwxr-xr-x 3 marslo admin 96 May 17  2019 /usr/local/lib/python2.7/
 drwxr-xr-x 3 marslo admin 96 May 17  2019 /usr/local/lib/python3.7/
 drwxr-xr-x 3 marslo admin 96 Jan 13  2020 /usr/local/lib/python3.8/
 drwxr-xr-x 3 marslo admin 96 Oct 10 17:06 /usr/local/lib/python3.9/

 $ ls -ld /Library/Python/2.7/site-packages/
 drwxr-xr-x 9 root wheel 288 Aug  6 18:16 /Library/Python/2.7/site-packages/
  • or

      $ ls $(brew --prefix)/lib/python*
      /usr/local/lib/python2.7:
      site-packages
    
      /usr/local/lib/python3.7:
      site-packages
    
      /usr/local/lib/python3.8:
      site-packages
    
      /usr/local/lib/python3.9:
      site-packages
  • local

    $ ls -ld ~/Library/Python/*/
    drwx------ 4 marslo staff 128 Aug  6 17:23 /Users/marslo/Library/Python/2.7/
    drwx------ 5 marslo staff 160 Oct 12 21:17 /Users/marslo/Library/Python/3.7/
    drwx------ 5 marslo staff 160 Oct 27 19:24 /Users/marslo/Library/Python/3.8/
    drwx------ 5 marslo staff 160 Oct 27 19:24 /Users/marslo/Library/Python/3.9/
    • example:

      $ /usr/bin/python -c 'import site; print(site.USER_BASE)'
      /Users/marslo/Library/Python/2.7
      
      $ /usr/local/bin/python3.9 -c 'import site; print(site.USER_BASE)'
      /Users/marslo/Library/Python/3.9
      
      $ /usr/local/bin/python3.6 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])'
      /usr/lib/python3.6/site-packages

linux

references:

$ /usr/local/bin/python3.6 -m site --user-site
/home/marslo/.local/lib/python3.6/site-packages

$ /usr/local/bin/python3.6 -c 'import site; print(site.getsitepackages())'
['/usr/local/lib64/python3.6/site-packages', '/usr/local/lib/python3.6/site-packages', '/usr/lib64/python3.6/site-packages', '/usr/lib/python3.6/site-packages']

$ python3.6 -m site
sys.path = [
    '/home/marslo',
    '/usr/lib64/python36.zip',
    '/usr/lib64/python3.6',
    '/usr/lib64/python3.6/lib-dynload',
    '/usr/local/lib/python3.6/site-packages',
    '/usr/lib64/python3.6/site-packages',
    '/usr/lib/python3.6/site-packages',
]
USER_BASE: '/home/marslo/.local' (exists)
USER_SITE: '/home/marslo/.local/lib/python3.6/site-packages' (doesn't exist)
ENABLE_USER_SITE: True
  • check particular lib

    $ python -c "import os as _; print(_.__file__)"
    /usr/lib64/python3.6/os.py
    
    $ python -c "import setuptools as _; print(_.__path__)"
    ['/usr/lib/python3.6/site-packages/setuptools']

multiple versions

get current working version

$ CFLAGS=-I$(brew --prefix)/include LDFLAGS=-L$(brew --prefix)/lib pip --version
pip 20.2.4 from /Users/marslo/Library/Python/3.8/lib/python/site-packages/pip (python 3.8)
  • or

    $ $(brew --prefix)/opt/python/libexec/bin/python -V
    Python 3.8.6

upgrade particular modules

$ sudo -H python3.9 -m pip install --upgrade pip
Collecting pip
  Using cached pip-20.2.4-py2.py3-none-any.whl (1.5 MB)
Installing collected packages: pip
  Attempting uninstall: pip
    Found existing installation: pip 20.2.3
    Uninstalling pip-20.2.3:
      Successfully uninstalled pip-20.2.3
Successfully installed pip-20.2.4
  • upgrade to previous version

$ pip install --upgrade --no-cache-dir --pre pip

install all older version modules

$ /usr/local/bin/python3.8 -m pip freeze > pip3.8-requirements.txt
$ sudo -H /usr/local/bin/python3.9 -m pip install --pre -r pip3.8-requirements.txt
  • reference:

    $ CFLAGS=-I$(brew --prefix)/include LDFLAGS=-L$(brew --prefix)/lib pip freeze
    beautifulsoup4==4.9.1
    certifi==2020.6.20
    cffi==1.14.1
    chardet==3.0.4
    click==7.1.2
    click-config-file==0.6.0
    colorama==0.4.3
    ...
    • or

      $ pip list --outdate --format=freeze
      docker==4.2.2
      rich==3.0.5
      
      $ pip list -o --format columns
      Package Version Latest Type
      ------- ------- ------ -----
      docker  4.2.2   4.3.1  wheel
      rich    3.0.5   9.1.0  whee
      
      $ pip list --outdate --format=json
      [{"name": "docker", "version": "4.2.2", "latest_version": "4.3.1", "latest_filetype": "wheel"}, {"name": "rich", "version": "3.0.5", "latest_version": "9.1.0", "latest_filetype": "wheel"}]

version change

change default python from 3.9 to 3.10

setup default python

  • via ln

    $ unlink /usr/local/opt/python
    $ ln -sf /usr/local/Cellar/python@3.10/3.10.4 /usr/local/opt/python
    
    $ unlink /usr/local/bin/python
    $ ln -sf /usr/local/Cellar/python@3.10/3.10.4/bin/python3.10 /usr/local/bin/python3
    $ ln -sf /usr/local/Cellar/python@3.10/3.10.4/bin/python3.10 /usr/local/bin/python
    
    $ export PYTHONUSERBASE="$(/usr/local/opt/python/libexec/bin/python -c 'import site; print(site.USER_BASE)')"
    $ export PYTHON3='/usr/local/opt/python/libexec/bin'
    $ export PATH="$PYTHONUSERBASE/bin:${PYTHON3}:$PATH"
  • via brew link

    $ brew link python3 python@3.10 --overwrite
    • example

      $ python3 --version
      Python 3.9.13
      
      $ brew link python3 python@3.10 --overwrite
      Warning: Already linked: /usr/local/Cellar/python@3.9/3.9.13_1
      To relink, run:
        brew unlink python@3.9 && brew link python@3.9
      Linking /usr/local/Cellar/python@3.10/3.10.4... 24 symlinks created.
      
      If you need to have this software first in your PATH instead consider running:
        echo 'export PATH="/usr/local/opt/python@3.10/bin:$PATH"' >> /Users/marslo/.bash_profile
      
      $ python3 --version
      Python 3.10.4
  • via default

    $ defaults write com.apple.versioner.python Version 3.8
    • get default version

      $ defaults read com.apple.versioner.python Version
    • example

      $ defaults read com.apple.versioner.python Version
      3.9
      
      $ defaults write com.apple.versioner.python Version 3.10
      
      $ defaults read com.apple.versioner.python Version
      3.10

modules re-installation

$ /usr/local/bin/python3.9 -m pip freeze > pip3.9-requirements.txt
$ sudo -H /usr/local/bin/python3.10 -m pip install --pre -r pip3.9-requirements.txt

PYTHONPATH

$ export PYTHONPATH="/usr/local/lib/python3.9/site-packages"
    ⇣⇣
$ export PYTHONPATH="/usr/local/lib/python3.10/site-packages"

extension

  • download

    $ curl -o /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/idlelib/ClearWindow.py https://bugs.python.org/file14303/ClearWindow.py
  • configure

    $ cat >> /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/idlelib/config-extensions.def < EOF
    
    [ClearWindow]
    
    enable=1
    enable_editor=0
    enable_shell=1
    [ClearWindow_cfgBindings]
    clear-window=<Command-Key-l>
    EOF

python IDLE in MacOS Big Sur

IDLE quit unexpectedly

Process:               Python [53851]
Path:                  /usr/local/Cellar/python@3.9/3.9.1_5/IDLE 3.app/Contents/MacOS/Python
Identifier:            org.python.IDLE
Version:               3.9.1 (3.9.1)
Code Type:             X86-64 (Native)
Parent Process:        ??? [1]
Responsible:           Python [53851]
User ID:               501

Date/Time:             2021-01-10 15:20:06.574 +0800
OS Version:            macOS 11.1 (20C69)
Report Version:        12
Bridge OS Version:     5.1 (18P3030)
Anonymous UUID:        AB6EE819-0314-4161-9650-FFE340DF84C8

Application Specific Information:
abort() called
...
  • root cause

    $ python
    Python 3.9.1 (default, Jan  6 2021, 06:05:23)
    [Clang 12.0.0 (clang-1200.0.32.28)] on darwin
    Type "help", "copyright", "credits" or "license" for more information.
    >>> import tkinter as tk
    >>> print(tk.Tcl().eval('info patchlevel'))
    8.5.9
    >>> exit()
  • using https://www.python.org/ftp/python/3.9.1/python-3.9.1-macosx10.9.pkg

    $ brew info tcl-tk | head -1
    tcl-tk: stable 8.6.11 (bottled) [keg-only]
    
    $ python
    Python 3.9.1 (v3.9.1:1e5d33e9b9, Dec  7 2020, 12:10:52)
    [Clang 6.0 (clang-600.0.57)] on darwin
    Type "help", "copyright", "credits" or "license()" for more information.
    >>> import tkinter as tk
    >>> print(tk.Tcl().eval('info patchlevel'))
    8.6.8
  • more on tkinter

    >>> import tkinter
    >>> tkinter.TclVersion, tkinter.TkVersion
    (8.5, 8.5)
    >>> tkinter._tester()
    Traceback (most recent call last):
      File "<stdin>", line 1, in <module>
    AttributeError: module 'tkinter' has no attribute '_tester'
    >>> tkinter._test()
    macOS 11 or later required !
    Abort trap: 6

Python may not be configured for Tk

$ python
Python 3.10.4 (main, Apr 26 2022, 19:42:59) [Clang 13.1.6 (clang-1316.0.21.2)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import tkinter as tk
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/Cellar/python@3.10/3.10.4/Frameworks/Python.framework/Versions/3.10/lib/python3.10/tkinter/__init__.py", line 37, in <module>
    import _tkinter # If this fails your Python may not be configured for Tk
ModuleNotFoundError: No module named '_tkinter'
  • solution

    $ brew info python@3.10
    python@3.10: stable 3.10.4 (bottled) [keg-only]
    ... ...
    tkinter is no longer included with this formula, but it is available separately:
      brew install python-tk@3.10
    ... ...
    
    $ brew install python-tk@3.10
    ==> Downloading https://ghcr.io/v2/homebrew/core/python-tk/3.10/manifests/3.10.4

######################################################################## 100.0% ==> Downloading https://ghcr.io/v2/homebrew/core/python-tk/3.10/blobs/sha256:6a937be1fd531589ef7f9b4d971cb91ee7549d99f7f1aaf97f0fc3c0911f1c5d ==> Downloading from https://pkg-containers.githubusercontent.com/ghcr1/blobs/sha256:6a937be1fd531589ef7f9b4d971cb91ee7549d99f7f1aaf97f0fc3c0911f1c5d?s ######################################################################## 100.0% ==> Pouring python-tk@3.10--3.10.4.monterey.bottle.tar.gz ==> Caveats python-tk@3.10 is keg-only, which means it was not symlinked into /usr/local, because this is an alternate version of another formula.

==> Summary ☕️ 🐸 /usr/local/Cellar/python-tk@3.10/3.10.4: 5 files, 132.6KB ==> Running brew cleanup python-tk@3.10... Disable this behaviour by setting HOMEBREW_NO_INSTALL_CLEANUP. Hide these hints with HOMEBREW_NO_ENV_HINTS (see man brew).

$ brew reinstall python@3.10 ... ...


## venv

> [!NOTE|label:references:]
> - [12. Virtual Environments and Packages](https://docs.python.org/3/tutorial/venv.html) | [zh-cn](https://docs.python.org/zh-cn/3/tutorial/venv.html)

### PS1

> [!NOTE|label:references:]
> - [How do I change the default virtualenv prompt?](https://stackoverflow.com/a/20026992/2940319)
> - [Python venv module cannot add virtual environment name to PS1, when using PROMPT_COMMAND?](https://stackoverflow.com/a/61682159/2940319)

```bash
$ export VIRTUAL_ENV_DISABLE_PROMPT=1

$ function _venv_info() {
  local printf_format=' [%s]'
  local venv=''
  [[ $# -eq 1 ]] && printf_format="$1"
  [[ -n "${VIRTUAL_ENV}" ]] && venv="${VIRTUAL_ENV##*/}"
  if [[ -n "${venv}" ]]; then
    # shellcheck disable=SC2059
    printf -- "${printf_format}" "${venv}"
  fi
}
# usage
$ _venv_info "--%s--"
--rmk--
$ _venv_info "(\033[38;5;6;3m%s\033[0m)"
(rmk)
  • or using PROMPT_COMMAND

    $ cat ~/.bashrc
    # for venv info
    function _venv_info() {
      local printf_format=' [%s]'
      local venv=''
      [[ $# -eq 1 ]] && printf_format="$1"
      [[ -n "${VIRTUAL_ENV}" ]] && venv="${VIRTUAL_ENV##*/}"
      if [[ -n "${venv}" ]]; then
        # shellcheck disable=SC2059
        printf -- "${printf_format}" "${venv}"
      fi
    }
    
    export VIRTUAL_ENV_DISABLE_PROMPT=1
    COL_SD_PURPLE='\[\033[38;5;98;3m\]'
    COL_SD_GREEN='\[\033[32;2m\]'
    COL_NONE='\[\033[0m\]'
    COL_DEFAULT="\[\033[38;5;240m\]"
    COL_RESET='\[\033[1m\]'
    
    PS1="\\n${COL_RESET}${COL_DEFAULT}(\\u@\\h${COL_RESET} \\w${COL_RESET}${COL_DEFAULT}) "
    PS1+="\$(__git_ps1 \"- (${COL_SD_GREEN}%s${COL_NONE}${COL_DEFAULT}) \")"
    PS1+="\$(_venv_info \"- (${COL_SD_PURPLE}%s${COL_NONE}${COL_DEFAULT}) \")"
    PS1+="\\$ ${COL_NONE}"
    export PS1

init and setup

[!NOTE|label:references:]

PLATFORMSHELLHOW TO ACTIVATE

posix

bash/zsh

source <venv>/bin/activate

posix

fish

source <venv>/bin/activate.fish

posix

csh/tcsh

source <venv>/bin/activate.csh

posix

PowerShell

<venv>/Scripts/Activate.ps1

Windows

cmd.exe

<venv>\Scripts\activate.bat

Windows

PowerShell

<venv>\Scripts\Activate.ps1

# init
$ python3 -m venv ~/.venv/rmk

# enable
$ source ~/.venv/rmk/bin/activate

# disable
$ deactivate
  • tips for g:python3_host_prog in nvim

    let g:python3_host_prog = expand( substitute(system('command -v python3'), '\n\+$', '', '') )
  • check status

    $ pip config -v list
    For variant 'global', will try loading '/Library/Application Support/pip/pip.conf'
    For variant 'user', will try loading '/Users/marslo/.pip/pip.conf'
    For variant 'user', will try loading '/Users/marslo/.config/pip/pip.conf'
    For variant 'site', will try loading '/Users/marslo/.venv/rmk/pip.conf'

install packages

[!NOTE|label:references:]

Last updated