AWK

Top API list from nginx log

# remove querystring
cat access.log | awk '{split($0, a, "?| "); print a[7]}' | sort | uniq -c | sort -n -r | head -n 10

# 5xx
cat access.log | grep 'HTTP/1.1" 5' |awk '{split($0, a, "?| "); print a[7]}' | sort | uniq -c | sort -n -r | head -n 10

# 4xx statistics
cat access.log | grep 'HTTP/1.1" 499' | awk '{split($0, a, "?| "); print a[7]}' | sort | uniq -c | sort -n -r | head -n 10

# 4xx by minute
cat access.log | grep 'HTTP/1.1" 499' | awk '{print $4}' | awk '{split($0, a, ":"); print a[2]a[3]}' | awk '{arr[$1]+=1} END {for (i in arr) {print arr[i], i}}' | sort -n -r | head -n 10

# top ip
cat access.log | awk '{print $1}' | sort | uniq -c | sort -n -r | head -n 10

Sum response time by API

log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                  '$status $body_bytes_sent $upstream_response_time '
                  '$upstream_connect_time $upstream_header_time "$http_referer" '
                  '"$http_user_agent" "$http_x_forwarded_for"';

cat access.log | grep -v '\- \- \-' | awk '{print $11, $7}' | awk '{split($0, a, "?"); print a[0], a[1]}' | awk '{arr[$2]+=$1} END {for (i in arr) {print arr[i], i}}' | sort -n -r | head -n 10

只留 URI 前幾層算 Top 總耗時

# 三層
cat access.log | grep -v '\- \- \-' | awk '{print $11, $7}' \
| awk '{split($0, a, "?"); print a[0], a[1]}' \
| awk '{split($0, a, "/"); print a[1]"/"a[2]"/"a[3]"/"a[4]}' \
| awk '{arr[$2]+=$1} END {for (i in arr) {print arr[i], i}}' \
| sort -n -r | head -n 10

396.319 /api/m/user
295.127 /api/get_xxx/
244.505 /api/m/member
...

# 兩層
cat access.log | grep -v '\- \- \-' | awk '{print $11, $7}' \
| awk '{split($0, a, "?"); print a[0], a[1]}' \
| awk '{split($0, a, "/"); print a[1]"/"a[2]"/"a[3]}' \
| awk '{arr[$2]+=$1} END {for (i in arr) {print arr[i], i}}' \
| sort -n -r | head -n 10

# 四層 - /api/m/user/xxxx
cat access.log | grep '/api/m/user' | awk '{print $11, $7}' \
| awk '{split($0, a, "?"); print a[0], a[1]}' \
| awk '{split($0, a, "/"); print a[1]"/"a[2]"/"a[3]"/"a[4]"/"a[5]}' \
| awk '{arr[$2]+=$1} END {for (i in arr) {print arr[i], i}}' \
| sort -n -r | head -n 10

# 四層 - /api/m/member/xxxx
cat access.log | grep '/api/m/member' | awk '{print $11, $7}' \
| awk '{split($0, a, "?"); print a[0], a[1]}' \
| awk '{split($0, a, "/"); print a[1]"/"a[2]"/"a[3]"/"a[4]"/"a[5]}' \
| awk '{arr[$2]+=$1} END {for (i in arr) {print arr[i], i}}' \
| sort -n -r | head -n 10

只留 URI 前幾層算 Top 總次數

# 三層
cat access.log | grep -v '\- \- \-' \
| awk '{split($0, a, "?| "); print a[7]}' \
| awk '{split($0, a, "/"); print a[1]"/"a[2]"/"a[3]"/"a[4]}' \
| sort | uniq -c | sort -n -r | head -n 20

# 兩層
cat access.log | grep -v '\- \- \-' \
| awk '{split($0, a, "?| "); print a[7]}' \
| awk '{split($0, a, "/"); print a[1]"/"a[2]"/"a[3]}' \
| sort | uniq -c | sort -n -r | head -n 10

# 四層 - /api/m/user/xxxx
cat access.log | grep '/api/m/user' \
| awk '{split($0, a, "?| "); print a[7]}' \
| awk '{split($0, a, "/"); print a[1]"/"a[2]"/"a[3]"/"a[4]"/"a[5]}' \
| sort | uniq -c | sort -n -r | head -n 10