$ echo 'a b c' | xargs -n1
a
b
c
$ echo {a..c}.{1..2} | xargs -n1 | xargs -I{} echo -{}-
-a.1-
-a.2-
-b.1-
-b.2-
-c.1-
-c.2-
$ echo 'a b c' | fmt -1
a
b
c
$ echo {a..c}.{1..2} | fmt -1 | xargs -I{} echo -{}-
-a.1-
-a.2-
-b.1-
-b.2-
-c.1-
-c.2-
$ echo 'a b c' | awk '{ OFS=RS; $1=$1 }1'
a
b
c
tr
$ echo 'a b c' | tr -s ' ' '\n'
a
b
c
$ printf '%s\n' a b c
a
b
c
execute commands from file
[!TIP]
precondition
$ cat a.txt
a b c
$ echo 'a b c' | xargs -n1 -t touch
touch a
touch b
touch c
$ echo 'a b c' | xargs -n1 -p touch
touch a?...y
touch b?...y
touch c?...y
-t, --verbose
Print the command line on the standard error output before executing it.
-p, --interactive
Prompt the user about whether to run each command line and read a line from the terminal.
Only run the command line if the response starts with `y' or `Y'. Implies -t.
-I replace-str
Replace occurrences of replace-str in the initial-arguments with names read from standard in-
put. Also, unquoted blanks do not terminate input items; instead the separator is the new-
line character. Implies -x and -L 1.
$ paste <(sort a.txt) <(sort b.txt) | expand --tabs=10
a a
b b
d c
f d
e
$ pr -w 30 -m -t a.txt b.txt
a a
b b
d c
f d
e
$ paste -d ',:' file1 file2 file3
echo
[!TIP]
$ echo -e "a\t\tb"
a b
$ echo -e $(echo -e "a\t\tb")
a b
print file with ansicolor
$ command cat c.txt
get get/exec get/subresource list create create/exec get/subresource
\e[32;1myes\e[0m \e[32;1myes\e[0m \e[32;1myes\e[0m \e[32;1myes\e[0m \e[32;1myes\e[0m no no
$ echo -ne $(command cat c.txt | sed 's/$/\\n/' | sed 's/ /\\a /g')
get get/exec get/subresource list create create/exec get/subresource
yes yes yes yes yes no no
$ echo -ne $(command cat c.txt | sed 's/$/\\n/' | sed 's/ /\\033 /g')
get get/exec get/subresource list create create/exec get/subresource
yes yes yes yes yes no no
diff
[!NOTE|label:references:]
show all status
$ diff --side-by-side <(sort a.txt) <(sort b.txt)
a a
b b
> c
d d
f | e
show diff only
$ diff --suppress-common-lines --side-by-side <(sort a.txt) <(sort b.txt)
> c
f | e
show diff with --<GTYPE>-group-format
VARIABLE
APPLICABLE
old
GTYPE, LTYPE
new
GTYPE, LTYPE
unchanged
GTYPE, LTYPE
changed
GTYPE
[!NOTE|label:tips:]
[G]TYPE : [g]roup : --<GTYPE>-group-format
[L]TYPE : [l]ine : --<LTYPE>-line-format
$ diff --old-group-format="L %<" --new-group-format="R %>" --unchanged-group-format="" a.txt b.txt
R c
L f
R e
# with line number
$ diff --unchanged-line-format="" --old-line-format="< %dn: %L" --new-line-format="> %dn: %L" <(sort a.txt) <(sort b.txt)
> 3: c
< 4: f
> 5: e
# beging-end
$ diff --old-group-format='\begin{em}
-> %<\end{em}
-> ' --new-group-format='\begin{bf}
-> %>\end{bf}
-> ' --changed-group-format='\begin{em}
-> %<\end{em}
-> \begin{bf}
-> %>\end{bf}
-> ' --unchanged-group-format='%=' \
-> <(sort a.txt) <(sort b.txt)
a
b
\begin{bf}
c
\end{bf}
d
\begin{em}
f
\end{em}
\begin{bf}
e
\end{bf}
$ diff \
-> --unchanged-group-format='' \
-> --old-group-format='-------- %dn line%(n=1?:s) deleted at %df: %<' \
-> --new-group-format='-------- %dN line%(N=1?:s) added after %de: %>' \
-> --changed-group-format='-------- %dn line%(n=1?:s) changed at %df: %<-------- to: %>' \
-> <(sort a.txt) <(sort b.txt)
-------- 1 line added after 2:
c
-------- 1 line changed at 4:
f
-------- to:
e
show common
$ diff --unchanged-line-format="%L" --new-line-format="" --old-line-format="" <(sort a.txt) <(sort b.txt)
a
b
d
$ comm -3 a.txt b.txt
c
e
f
$ comm -3 <(sort a.txt) <(sort b.txt) | column -t -s $'\t' --table-columns '==== a.txt ====,==== b.txt ===='
==== a.txt ==== ==== b.txt ====
c
e
f
common
$ comm -12 <(sort a.txt) <(sort b.txt)
a
b
d
join
[!NOTE|label:references:]
function join_by {
local d=${1-} f=${2-}
if shift 2; then
printf %s "$f" "${@/#/$d}"
fi
}
$ foo=( a "b c" d )
$ printf -v joined '%s,' "${foo[@]}"
$ echo "${joined%,}"
a,b c,d
$ printf -v joined '|,%s,|' "${foo[@]}"
$ echo "${joined%,}"
|,a,||,b c,||,d,|
# join by single char
$ foo=(a "b c" d)
$ bar=$(IFS=, ; echo "${foo[*]}")
$ echo "$bar"
a,b c,d
$ echo -e 'a very long string..........\t112232432\tanotherfield\na smaller string\t123124343\tanotherfield\n' | column -t -s $'\t'
a very long string.......... 112232432 anotherfield
a smaller string 123124343 anotherfield
$ sed -n 'x;$p' <<\INPUT
a
b
c
d
INPUT
c
# or
$ echo -e 'a\nb\nc\nd' | sed -n -e '${x;1!p;};h'
c
# tac + sed
$ echo -e 'a\nb\nc\nd' | tac | sed -n '2p'
c
$ cat a.txt | sed -n '/3c/{n;p;n;p}'
4d
5e
$ cat a.txt | sed -n '/3c/{N;p;n;p}'
3c
4d
5e
to get docker registry mirrors
# exclude the pattern
$ docker system info | sed -n '/Registry Mirrors:/{n;p;}'
https://artifactory.domain.com/
# including pattern
$ docker system info | sed -n '/Registry Mirrors:/{p;n;p;}'
Registry Mirrors:
https://artifactory.domain.com/
### awk
- include pattern
```bash
$ cat a.txt | awk '/3c/,/8h/'
3c
4d
5e
6f
7g
8h
sed
[!NOTE|label:references:]
include all patterns
$ cat a.txt | sed -n '/3c/,/8h/p'
3c
4d
5e
6f
7g
8h
exclude both patterns
$ sed -n '/3c/,/8h/{//!p;}' a.txt
4d
5e
6f
7g
$ sed -n '/3c/,/8h/{/3c/!{/8h/!p}}' a.txt
4d
5e
6f
7g
# + delete from line 1 to /3c/
# | + delete /8h/ to end `$`
# +-----+ +-----+
$ cat a.txt | sed '1,/3c/d;/8h/,$d'
4d
5e
6f
7g
# + not delete since `/3c/` to `/8h`
# | + delete all the others
# +---------+ +-+
$ cat a.txt | sed '/3c/,/8h/!d;//d'
4d
5e
6f
7g
exclude single pattern
$ sed -n '/3c/,/8h/{/8h/!p}' a.txt
3c
4d
5e
6f
7g
$ sed -n '/3c/,/8h/{/3c/!p}' a.txt
4d
5e
6f
7g
8h
with empty line
[!NOTE]
use case :
$ cat a.txt
1a
2b
3c * (start)
4d
5e
6f
* (ending)
7g
8h
9i
10j
11k
$ cat a.txt | sed -n '/3c/,/^$/p'
3c
4d
5e
6f
solution: to delete / not print from first line to pattern
delete: /d
not print: -n /!p
for both CRLF and LF
# not print
# `-n` + from 1st line to line of pattern `/^\s*$/`
# ^ +------+ + not print
$ echo -e '1\n2\n\n3\n4' | sed -n '1,/^\s*$/!p'
3
4
# delete
# + from 1st line to line of pattern `/^\s*$/`
# | + delete
# +------+ |
$ echo -e '1\n2\n\n3\n4' | sed '1,/^\s*$/d'
3
4
# + not delete since `/^\s*$/` to end
# | + delete the others
# +---------+ +-+
$ echo -e '1\n\n2\n\n3\n4' | sed '/^\s*$/,$!d;//d'
2
3
4
[!TIP]
solution: with matched line number + 1 : "$(( n+1 ))"',$p'
head -n1 : for first matches pattern line number
tail -n1 : for the last matches pattern line number
$ command cat -A a | nl
1 1^M$
2 ^M$
3 2^M$
4 3^M$
5 4^M$
$ cat a | sed -n "$(( $(sed -n '/^\s*$/ =' a | head -n1 )+1 ))"' ,$p'
2
3
4
$ cat a
pattern
1
2
3
pattern
4
5
pattern * (start: the last pattern)
6
7
8
9
* (end)
10
11
sed
$ sed -ne '
/pattern/{
$d;n
:loop
s/\n$//;tdone
$bdone;N
bloop
:done
x
}
${x;/./p;}
' a
6
7
8
9
# or
$ sed -e '
/pattern/,/^$/!ba
/./!ba
H;/pattern/{z;x;}
:a
$!d;x;s/.//
' a
6
7
8
9
# or GNU sed
$ sed -Ez '
s/.*pattern\n(([^\n]+\n)+)(\n.*)?/\1/
' a
6
7
8
9
awk
$ awk '/pattern/,/^$/ { arr[NR]=$0; if (/pattern/) line1=NR; if (/^$/) line2=NR}END{ if (line1) for(i=++line1;i<line2;i++) print arr[i]}' a
6
7
8
9
# or
$ awk -v RS='' -F '\n' '$1 ~ /pattern/ { hold = $0 } END { if (hold != "") print hold }' a | sed 1d
6
7
8
9
replace the last matching pattern
# the last `boy` -> `boys`
$ printf "%s\n" boy girl boy girl boy girl | sed -z 's/.*boy/&s/'
boy
girl
boy
girl
boys
girl
$ myCommandWithDifferentQuotes=$(cat <<'EOF'
-> echo "command 1: $@"; echo 'will you do the fandango?'; echo "command 2: $@"; echo
-> EOF
-> )
$ < a.txt xargs -I @@ bash -c "$myCommandWithDifferentQuotes" -- @@
command 1: a b c
will you do the fandango?
command 2: a b c
command 1: 123
will you do the fandango?
command 2: 123
command 1: ###this is a comment
will you do the fandango?
command 2: ###this is a comment
-a file, --arg-file=file
Read items from file instead of standard input. If you use this option, stdin remains unchanged when
commands are run. Otherwise, stdin is redirected from /dev/null.
$ cat a.txt
8.8.8.8
1.1.1.1
$ xargs -L1 -a a.txt /sbin/ping -c 1
PING 8.8.8.8 (8.8.8.8): 56 data bytes
64 bytes from 8.8.8.8: icmp_seq=0 ttl=44 time=82.868 ms
--- 8.8.8.8 ping statistics ---
1 packets transmitted, 1 packets received, 0.0% packet loss
round-trip min/avg/max/stddev = 82.868/82.868/82.868/0.000 ms
PING 1.1.1.1 (1.1.1.1): 56 data bytes
64 bytes from 1.1.1.1: icmp_seq=0 ttl=63 time=1.016 ms
--- 1.1.1.1 ping statistics ---
1 packets transmitted, 1 packets received, 0.0% packet loss
round-trip min/avg/max/stddev = 1.016/1.016/1.016/0.000 ms
$ find ${JENKINS_HOME}/jobs -maxdepth 2 -name config\.xml -type f -print | tar czf ~/config.xml.tar.gz --files-from -
back build history
$ find ${JENKINS_HOME}/jobs -name builds -prune -o -type f -print | tar czf ~/m.tar.gz --files-from -
find by timestamp
[!NOTE|label:references:]
$ echo $(( ($(date +%s -d 20210131)-$(date +%s -d 20210101))/86400 )) days
30 days
# or in `%y%m%d` format
$ echo $(( ($(date --date="230301" +%s) - $(date --date="240301" +%s) )/(60*60*24) )) days
-366 days
$ find . -type f -newermt "2010-01-01" ! -newermt "2010-06-01"
via mtime
[!TIP|label:tricky on -mtime:]
+n: for greater than n
-n: for less than n
n: for exactly n
i.e.:
-mtime -14: less than 14 days, not including 14 days == 13 days ago and less
-mtime +14: more than 14 days, not including 14 days == 15 days ago and more
# get all files since 2023-10-16
$ diff=$(( ($(date --date "24-02-29" +%s) - $(date --date "231016" +%s) )/(60*60*24) ))
$ find . -type f -daystart -mtime -$((diff+1)) -printf "%T+ | %p\n" | sort | wc -l
33
# copy all files modified since 2023-10-16
$ find . -type f -daystart -mtime -$((diff+1)) -exec cp -a --parents -t /path/to/target "{}" \+
# with timezone
$ diff=$(( ($(date -d "2015-03-11 UTC" +%s) - $(date -d "2015-03-05 UTC" +%s)) / (60*60*24) ))
# find older than x days
$ find . -type f -mtime +7 -exec ls -l {} \;
# delete older than x days: https://www.commandlinefu.com/commands/view/1394/delete-files-older-than..
$ find /dir_name -mtime +5 -exec rm {} \
via newermt
[!TIP|label:tips for -newerXY]
-newerXY reference
Compares the timestamp of the current file with reference. The
reference argument is normally the name of a file (and one of
its timestamps is used for the comparison) but it may also be a
string describing an absolute time. X and Y are placeholders
for other letters, and these letters select which time belonging
to how reference is used for the comparison.
a The access time of the file reference
B The birth time of the file reference
c The inode status change time of reference
m The modification time of the file reference
t reference is interpreted directly as a time
# original
$ cal | cat -pp -A
····January·2024····␊
Su·Mo·Tu·We·Th·Fr·Sa␊
····1··2··3··4··5··6␊
·7··8··9·10·11·12·13␊
14·15·16·17·18·19·20␊
21·22·23·24·25·26·27␊
28·29·30·31·········␊
····················␊
# awk 'NF'
$ cal | awk 'NF' | cat -pp -A
····January·2024····␊
Su·Mo·Tu·We·Th·Fr·Sa␊
····1··2··3··4··5··6␊
·7··8··9·10·11·12·13␊
14·15·16·17·18·19·20␊
21·22·23·24·25·26·27␊
28·29·30·31·········␊
# sed '/^\s*$/d'
$ cal | sed '/^\s*$/d' | cat -pp -A
····January·2024····␊
Su·Mo·Tu·We·Th·Fr·Sa␊
····1··2··3··4··5··6␊
·7··8··9·10·11·12·13␊
14·15·16·17·18·19·20␊
21·22·23·24·25·26·27␊
28·29·30·31·········␊
remove empty line at the end of file
[!NOTE|label:references:]
$ cat a.txt | sed '${/^[[:space:]]*$/d;}'
a
b
c
d
remove duplicate empty lines
[!NOTE|label:references:]
# original file
$ cat a.txt --style='numbers'
1 a
2 b
3
4
5
6 c
7
8 d
9
10
$ awk 'NF || p; { p = NF }' p=1 a.txt | bat --style='numbers'
1 a
2 b
3
4 c
5
6 d
7
# or
$ awk 'NF{c=1} (c++)<3' a.txt | bat --style='numbers'
1 a
2 b
3
4 c
5
6 d
7
# or
$ awk -v RS= -v ORS='\n\n' '1' a.txt | bat --style='numbers'
1 a
2 b
3
4 c
5
6 d
7
# or
$ awk '!NF{found++} found>1 && !NF{next} NF{found=""} 1' a.txt | bat --style='numbers'
1 a
2 b
3
4 c
5
6 d
7
[!NOTE|label:reference]
sample code:
str='aa bb cc'
${variable//search/replace}
$ shopt -s extglob
$ echo ${str//+( )/|}
aa|bb|cc
$ echo "${str//+([[:blank:]])/|}"
aa|bb|cc
sed
# DO NOT USE "${str}"
$ echo ${str} | sed 's: :|:g'
aa|bb|cc
$ echo "$str" | sed 's:[ ][ ]*:|:g'
aa|bb|cc
# or
$ echo "$str" | sed 's:\s\s*:|:g'
aa|bb|cc
echo "${string:0:$(( position - 1 ))}${replacement}${string:position}"
$ echo 'abc' | od -c
0000000 a b c \n
0000004
$ echo -n 'abc' | od -c
0000000 a b c
0000003
hexdump -c
$ echo 'abc' | hexdump -c
0000000 a b c \n
0000004
$ echo -n 'abc' | hexdump -c
0000000 a b c
0000003
hexdump -C
$ echo 'abc' | hexdump -C
00000000 61 62 63 0a |abc.|
00000004
# ^
# |
# 0x0a: LF
$ echo -n 'abc' | hexdump -C
00000000 61 62 63 |abc|
00000003
$ cat a.txt | hexdump -C
# 0x0a
# v
00000000 61 61 61 61 0a |aaaa.|
00000005
$ unix2dos a.txt
unix2dos: converting file a.txt to DOS format...
$ cat a.txt | hexdump -C
# 0x0d 0x0a
# v v
00000000 61 61 61 61 0d 0a |aaaa..|
00000006
$ cat a.txt | hexdump -c
0000000 a a a a \r \n
0000006
$ file a.txt
a.txt: ASCII text, with CRLF line terminators
vim
$ vim a.txt
:%!hexdump -C
# or
$ vim -c '%!xxd' a.txt
remove the ending '\n'
[!NOTE|label:references:]
original file
$ cat foo.txt
abc
efg
$ cat -A foo.txt
abc$
efg$
$ cat foo.txt | od -c
0000000 a b c \n e f g \n
0000010
$ truncate -s -1 foo.txt
$ od -c foo.txt
0000000 a b c \n e f g
0000007
$ sed -z s/.$// foo.txt | od -c
0000000 a b c \n e f g
0000007
$ sed -z s/\\n$// foo.txt | od -c
0000000 a b c \n e f g
0000007
$ sed -z 's/\n$//' foo.txt | od -c
0000000 a b c \n e f g
0000007
$ printf %s "$(< foo.txt)" | od -c
0000000 a b c \n e f g
0000007
$ head -c -1 foo.txt | od -c
0000000 a b c \n e f g
0000007
$ od -c foo.txt
0000000 a b c \n e f g \n
0000010
$ vim -c "set binary noeol" -c "wq" foo.txt
$ od -c foo.txt
0000000 a b c \n e f g
0000007
# or : https://stackoverflow.com/a/39627416/2940319
$ vim -c "set noendofline nofixendofline" -c "wq" foo.txt
$ od -c foo.txt
0000000 a b c \n e f g
0000007
$ echo -n "$(cat ~/.ssh/id_ed25519)"$'\n' | tail -c1 | /usr/bin/xxd -u -p
0A
# or
$ cat file.txt | sed -e '$a\'
# or: https://stackoverflow.com/a/65136212/2940319
$ cat a.org.txt | sed -z 's/$/\n/g'
# or
$ echo '' >> file # fix the last line line-ending
$ sed '${/^[[:space:]]*$/d;}' -i file # remove the empty lines at end of file
fold
check the params valid
available params should be contained by 'iwfabcem'
# case insensitive
param=$( tr '[:upper:]' '[:lower:]' <<< "$1" )
for _p in $(echo "${param}" | fold -w1); do
[[ ! 'iwfabcem' =~ ${_p} ]] && exits='yes' && break
done
This type of redirection instructs the shell to read input from the current source until a line containing only delimiter (with no trailing blanks) is seen. All of the lines read up to that point are then used as the standard input for a command.
The format of here-documents is:
<<[-]word
here-document
delimiter
No parameter expansion, command substitution, arithmetic expansion, or pathname expansion is performed on word. If any characters in word are quoted, the delimiter is the result of quote removal on word, and the lines in the here-document are not expanded. If word is unquoted, all lines of the here-document are subjected to parameter expansion, command substitution, and arithmetic expansion. In the latter case, the character sequence \ is ignored, and \ must be used to quote the characters , $, and `.
cat with specific character
$ man tab
...
-T, --show-tabs
display TAB characters as ^I
other references:
$ cat -A sample.sh
LANG=C tr a-z A-Z <<- END_TEXT$
Here doc with <<$
A single space character (i.e. 0x20 ) is at the beginning of this line$
^IThis line begins with a single TAB character i.e 0x09 as does the next line$
^IEND_TEXT$
$
echo The intended end was before this line$
$ bash sample.sh
HERE DOC WITH <<-
A SINGLE SPACE CHARACTER (I.E. 0X20 ) IS AT THE BEGINNING OF THIS LINE
THIS LINE BEGINS WITH A SINGLE TAB CHARACTER I.E 0X09 AS DOES THE NEXT LINE
The intended end was before this line
$ cat -A sample.sh
LANG=C tr a-z A-Z << END_TEXT$
Here doc with <<$
A single space character (i.e. 0x20 ) is at the beginning of this line$
^IThis line begins with a single TAB character i.e 0x09 as does the next line$
^IEND_TEXT$
$
echo The intended end was before this line$
$ bash sample.sh
sample.sh: line 7: warning: here-document at line 1 delimited by end-of-file (wanted `END_TEXT')
HERE DOC WITH <<
A SINGLE SPACE CHARACTER (I.E. 0X20 ) IS AT THE BEGINNING OF THIS LINE
THIS LINE BEGINS WITH A SINGLE TAB CHARACTER I.E 0X09 AS DOES THE NEXT LINE
END_TEXT
ECHO THE INTENDED END WAS BEFORE THIS LINE
Prefixing a double-quoted string with a dollar sign ($), such as $"hello, world", will cause the string to be translated according to the current locale
[access "^refs/heads/main"]
label-Code-Review = -2..+2 group user/John Doe (jdoe)
label-Verified = -1..+1 group user/John Doe (jdoe)
push = group user/John Doe (jdoe)
submit = group user/John Doe (jdoe)
[access "refs/for/refs/heads/main"]
push = group user/John Doe (jdoe)
addPatchSet = group user/John Doe (jdoe)
abandon = group user/John Doe (jdoe)
deleteOwnChanges = group user/John Doe (jdoe)
editAssignee = group user/John Doe (jdoe)
editTopicName = group user/John Doe (jdoe)
label-Code-Review = -2..+2 group user/John Doe (jdoe)
label-Verified = -1..+1 group user/John Doe (jdoe)
owner = group user/John Doe (jdoe)
read = group user/John Doe (jdoe)
removeReviewer = group user/John Doe (jdoe)
submit = group user/John Doe (jdoe)
create = group user/John Doe (jdoe)
pushMerge = group user/John Doe (jdoe)
[access "^refs/heads/user/jdoe/.*"]
label-Code-Review = -2..+2 group user/John Doe (jdoe)
label-Verified = -1..+1 group user/John Doe (jdoe)
push = group user/John Doe (jdoe)
submit = group user/John Doe (jdoe
[access "^refs/heads/main"]
---a---
---b---
label-Code-Review = -2..+2 group user/John Doe (jdoe)
label-Verified = -1..+1 group user/John Doe (jdoe)
push = group user/John Doe (jdoe)
submit = group user/John Doe (jdoe)
[access "refs/for/refs/heads/main"]
---a---
---b---
push = group user/John Doe (jdoe)
addPatchSet = group user/John Doe (jdoe)
abandon = group user/John Doe (jdoe)
deleteOwnChanges = group user/John Doe (jdoe)
editAssignee = group user/John Doe (jdoe)
editTopicName = group user/John Doe (jdoe)
label-Code-Review = -2..+2 group user/John Doe (jdoe)
label-Verified = -1..+1 group user/John Doe (jdoe)
owner = group user/John Doe (jdoe)
read = group user/John Doe (jdoe)
removeReviewer = group user/John Doe (jdoe)
submit = group user/John Doe (jdoe)
create = group user/John Doe (jdoe)
pushMerge = group user/John Doe (jdoe)
[access "^refs/heads/user/jdoe/.*"]
label-Code-Review = -2..+2 group user/John Doe (jdoe)
label-Verified = -1..+1 group user/John Doe (jdoe)
push = group user/John Doe (jdoe)
submit = group user/John Doe (jdoe)
