[dev] refactor log.py -> logger.sh (#32)

* update.sh: fetch will prune

* log: use hostname directly

* refactor log.py to logger.sh (#31)

* logger.sh: refactor log.py to bash script

* update.sh use beacon; ci test beacon

* install.sh: install dep first

* install.sh: DFS_UPDATED_RET=85

* more beacon; use stderr

* remove py3 dep

* install.sh: remove py3

* install.sh: -s|--secure

* async log; beacon add sys.login

* update use sync beacon

* fix ci minor bug; macos: uuidgen/e2fsprogs

* logger.sh: not need to use uuid5

* bug fix

* dfs err_return

* common.sh: post_log and post_beacon will check #args

* bug fix

* zshrc: fix locale when C.UTF-8 not exists

* WIP: combine install -a and tools/ scripts

* finish combination; get_os_name; macos.sh

* test.ci: now all use -a to install deps

* bug fix

* bug fix; ci: install.sh run with -x

* install.sh: bug fix that re-run install.sh after updated needs orgin args

* format

Co-authored-by: xiongdian.me <xiongdian.me@bytedance.com>
This commit is contained in:
Dict Xiong 2022-12-03 20:37:20 +08:00 committed by GitHub
parent 56e2b217eb
commit ff8ac21424
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 251 additions and 161 deletions

View File

@ -7,12 +7,6 @@ jobs:
name: test on ubuntu
runs-on: ubuntu-latest
steps:
- name: install dependencies
run: |
sudo apt-get update
sudo apt-get install -y git python3 python3-pip zsh curl inetutils-ping
sudo pip3 install requests
- name: checkout repo
uses: actions/checkout@v3
with:
@ -22,7 +16,8 @@ jobs:
run: |
rev=`git rev-parse HEAD`
pwd
./install.sh -dl
set -x
./install.sh -adl
test `git rev-parse HEAD` = "$rev"
- name: antigen build
@ -37,6 +32,7 @@ jobs:
source tools/test.zsh
antigen reset
rm -rf $ANTIGEN $HOME/.antigen
./install.sh -dl
- name: antigen build with DFS_NO_WALL
shell: /bin/zsh -ileo PIPE_FAIL {0}
@ -54,12 +50,6 @@ jobs:
name: test on macos
runs-on: macos-12
steps:
- name: install dependencies
run: |
brew update
brew install git python3 zsh curl
sudo pip3 install requests
- name: checkout repo
uses: actions/checkout@v3
with:
@ -69,7 +59,8 @@ jobs:
run: |
rev=`git rev-parse HEAD`
pwd
./install.sh -dl
set -x
./install.sh -adl
test `git rev-parse HEAD` = "$rev"
- name: antigen build
@ -84,6 +75,7 @@ jobs:
source tools/test.zsh
antigen reset
rm -rf $ANTIGEN $HOME/.antigen
./install.sh -dl
- name: antigen build with DFS_NO_WALL
shell: /bin/zsh -ileo PIPE_FAIL {0}
@ -103,7 +95,7 @@ jobs:
steps:
- name: remove dependencies
run: |
sudo apt-get -y remove curl vim
sudo apt-get -y remove curl vim python3
- name: checkout repo
uses: actions/checkout@v3
@ -113,6 +105,7 @@ jobs:
- name: install dfs
run: |
rev=`git rev-parse HEAD`
set -x
./install.sh -dal
test `git rev-parse HEAD` = "$rev"

View File

@ -9,10 +9,10 @@ DFS_UPDATE_CHANNEL=${DFS_UPDATE_CHANNEL:-"main"}
# fetch origin
cd $DOTFILES
git fetch --all
git fetch --all --prune
if [[ -n "$(git status -s)" ]]; then
fmt_error "directory not clean"
post_log "ERROR" "$THIS_FILE" "directory not clean"
apost_beacon "dfs.dirty"
exit
fi
@ -25,18 +25,18 @@ case $DFS_UPDATE_CHANNEL in
esac
if [[ ${#DFS_COMMIT} != 40 ]]; then
fmt_error "invalid commit id"
post_log "ERROR" "$THIS_FILE" "invalid commit id: ${DFS_COMMIT}"
apost_beacon "dfs.invalid-commit"
apost_log "ERROR" "$THIS_FILE" "invalid commit id: ${DFS_COMMIT}"
exit
fi
# update
if [[ "$(git rev-parse HEAD)" == "$DFS_COMMIT" ]]; then
fmt_info "nothing to do"
post_log "INFO" "$THIS_FILE" "nothing to do"
else
fmt_info "checking out to commit $DFS_COMMIT ..."
if [[ -z "$DFS_DEV" ]]; then
post_log "INFO" "$THIS_FILE" "will check out to commit $DFS_COMMIT"
post_beacon "dfs.updated"
git -c advice.detachedHead=false checkout $DFS_COMMIT
cp ./.update.sh ./update.sh && chmod +x ./update.sh && exit $DFS_UPDATED_RET
else

18
.zshrc2
View File

@ -1,7 +1,12 @@
# env for shell
export TERM="xterm-256color"
export LC_ALL=C.UTF-8
export LANG=C.UTF-8
if ( locale -a | grep -qxE "C.UTF-8|C.utf8" ); then
export LC_ALL=C.UTF-8
export LANG=C.UTF-8
else
export LC_ALL=en_US.UTF-8
export LANG=en_US.UTF-8
fi
export DOTFILES=$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" && pwd )
export PATH="$PATH:$DOTFILES/scripts"
export EDITOR='vim'
@ -82,6 +87,13 @@ antigen theme ${ZSH_THEME:-$TMP_THEME}
antigen apply
# end of antigen config
# beacon
if [[ "$EUID" == "0" ]]; then
"$DOTFILES/tools/common.sh" apost_beacon "sys.login.root"
else
"$DOTFILES/tools/common.sh" apost_beacon "sys.login.user"
fi
# alias
alias "pls"='sudo $(fc -ln -1)'
alias "se"='sudo -sE'
@ -144,6 +156,7 @@ bindkey "\ed" delete-char # alt+d
# dfs
dfs()
{
setopt local_options err_return pipe_fail
case $1 in
cd ) cd "$DOTFILES" ;;
update ) "$DOTFILES/update.sh" ;;
@ -169,6 +182,7 @@ dfs()
echo 'Done. Please open a new shell to see the changes.'
;;
log ) "$DOTFILES/tools/common.sh" "post_log" "INFO" "dfs" "$2" ;;
beacon ) "$DOTFILES/tools/common.sh" "post_beacon" "$2" ;;
* ) echo "unknown command \"$1\". available: update, force-update, version, reset, cd, log" ;;
esac
}

View File

@ -29,55 +29,37 @@ HOME_SYMLINKS_DST[0]=".ssh/authorized_keys2"
install_dependencies()
{
fmt_note "installing dependencies ..."
case $(get_os_type) in
"linux" )
case $(get_linux_dist) in
case $(get_os_name) in
"ubuntu"|"debian" )
$SUDO apt-get update
$SUDO apt-get install -y git zsh bash tmux vim python3 python3-pip curl inetutils-ping cmake less bsdmainutils
$SUDO DFS_LITE=$DFS_LITE "$DOTFILES/tools/ubuntu.sh" apt-install
;;
"alpine" )
$SUDO apk update
$SUDO apk add zsh bash git tmux vim curl python3 py3-pip fzf iputils coreutils util-linux
;;
* ) fmt_error "dfs auto-install is not implemented on linux distribution: $(get_linux_dist)"
esac
$SUDO DFS_LITE=$DFS_LITE "$DOTFILES/tools/alpine.sh" apk-add
;;
"macos" )
$SUDO brew update
$SUDO brew install git python3 zsh curl tmux vim util-linux
DFS_LITE=$DFS_LITE "$DOTFILES/tools/macos.sh" brew-install
;;
"msys" )
pacman -Syu
pacman -S tmux git zsh bash curl vim python3 python3-pip
SUDO=""
DFS_LITE=$DFS_LITE "$DOTFILES/tools/msys2.sh" pacman-S
;;
* ) fmt_error "dfs auto-install is not implemented on OS: $(get_os_type)"
* ) fmt_error "dfs auto-install is not implemented on OS: $(get_os_name)"
esac
if [[ -x $(command -v pip3) ]]; then
$SUDO pip3 install requests
elif [[ -x $(command -v pip) ]]; then
$SUDO pip install requests
else
fmt_error "pip3 and pip not found. is pip correctly installed?"
fi
}
preinstall_check()
{
fmt_note "checking requirements ..."
local mandatory_commands=( "git" "zsh" "curl" )
local optional_commands=( "python3" "vim" "tmux" "ping" )
local mandatory_commands=( "git" "zsh" "curl" "grep" "cat" "cp" "bash" "mkdir" )
local optional_commands=( "vim" "tmux" "ping" )
for i in "${mandatory_commands[@]}"; do
if [[ ! -x "$(command -v $i)" ]]; then
if ! command -v $i 1>/dev/null; then
fmt_info "all this utils are required: ${mandatory_commands[@]}"
fmt_info "install them manually or check scripts in tools/"
fmt_fatal "\"$i\" not found. aborting ..."
fi
done
for i in "${optional_commands[@]}"; do
if [[ ! -x "$(command -v $i)" ]]; then
if ! command -v $i 1>/dev/null; then
fmt_warning "\"$i\" not found"
ask_for_Yn "continue anyway?"
if [[ "$?" == "0" ]]; then
@ -167,7 +149,8 @@ uninstall_symlink()
done
}
install_crontab(){
install_crontab()
{
if [[ -x $(command -v crontab) ]]; then
fmt_note "installing \"$CRON_JOB\" to crontab ..."
( crontab -l | grep -vxF "${CRON_JOB}" | grep -v "no crontab for"; echo "$CRON_JOB" ) | crontab -
@ -176,7 +159,8 @@ install_crontab(){
fi
}
uninstall_crontab(){
uninstall_crontab()
{
if [[ -x $(command -v crontab) ]]; then
fmt_note "removing \"$CRON_JOB\" from crontab ..."
( crontab -l | grep -vxF "$CRON_JOB" ) | crontab -
@ -185,7 +169,8 @@ uninstall_crontab(){
fi
}
install_tmux_tpm(){
install_tmux_tpm()
{
TMUX_TPM="$HOME/.tmux/plugins/tpm"
if [[ -x $(command -v tmux) && ! -d "$TMUX_TPM" ]]; then
fmt_note "installing tmux tpm ..."
@ -204,7 +189,8 @@ install_tmux_tpm(){
fi
}
install_vim_vundle(){
install_vim_vundle()
{
VIM_VUNDLE="$HOME/.vim/bundle/Vundle.vim"
if [[ -x $(command -v vim) && ! -d "$VIM_VUNDLE" ]]; then
fmt_note "installing vim vundle ..."
@ -214,37 +200,42 @@ install_vim_vundle(){
fi
}
install_update(){
install_update()
{
fmt_note "installing update.sh ..."
cp "${DOTFILES}/.update.sh" "${DOTFILES}/update.sh"
chmod +x "${DOTFILES}/update.sh"
fmt_note "running update.sh ..."
DFS_UPDATED_RET=1 ${DOTFILES}/update.sh
if [[ $? == 1 ]]; then
DFS_UPDATED_RET=85 ${DOTFILES}/update.sh
if [[ $? == 85 ]]; then
fmt_note "dfs updated. re-running install.sh ..."
"${DOTFILES}/install.sh" && exit
"${DOTFILES}/install.sh" "$ORIGIN_ARGS" && exit
fi
}
uninstall_update(){
uninstall_update()
{
fmt_note "removing update.sh ..."
rm "${DOTFILES}/update.sh"
}
install(){
install_update
install()
{
if [[ "$INSTALL_DEP" == "1" ]]; then install_dependencies; fi
install_update
preinstall_check
install_crontab
install_file_content
install_symlink
apost_beacon "dfs.installed"
# those that won't be uninstalled in the future
install_tmux_tpm
install_vim_vundle
fmt_note "done installing!"
}
uninstall(){
uninstall()
{
ask_for_yN "do you really want to uninstall?"
if [[ $? != 1 ]]; then
fmt_error "aborting this job ..."
@ -254,9 +245,11 @@ uninstall(){
uninstall_crontab
uninstall_file_content
uninstall_symlink
apost_beacon "dfs.uninstalled"
fmt_note "done uninstalling!"
}
ORIGIN_ARGS="$@"
parse_arg "$@"
FUNC=install
INSTALL_DEP=0
@ -267,7 +260,8 @@ for i in ${PARSE_ARG_RET[@]}; do
-d|--dev ) export DFS_DEV=1 ;;
-l|--lite ) export DFS_LITE=1 ;;
-a|--auto ) INSTALL_DEP=1 ;;
* ) fmt_fatal "unknown option \"$i\". available: -i, -r, -q, -d, -l, -a" ;;
-s|--secure ) export DFS_DEV=0 ;;
* ) fmt_fatal "unknown option \"$i\". available: -i, -r, -q, -d, -l, -a, -s" ;;
esac
done
$FUNC

View File

@ -1,21 +1,23 @@
#!/bin/bash
set -e
THIS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" && pwd )
source "$THIS_DIR/common.sh"
set_mirror()
{
MIRROR=${1:-"mirrors.tuna.tsinghua.edu.cn"}
MIRROR=${MIRROR//\//\\\/}
sed -i "s/dl-cdn.alpinelinux.org/$MIRROR/g" /etc/apk/repositories
sed -i "s@dl-cdn.alpinelinux.org@${MIRROR}@g" /etc/apk/repositories
}
apk_add()
{
apk update
# mass installation
apk add zsh git tmux vim curl wget bash python3 py3-pip htop gcc g++ cmake make fzf perl linux-headers bind-tools iputils man-db coreutils util-linux
#for i in {fzf,ripgrep}; do apk add $i -y; done
# lite
apk add zsh bash git tmux vim curl fzf iputils coreutils util-linux
# full
if [[ -z "$DFS_LITE" ]]; then
apk add wget python3 py3-pip htop gcc g++ cmake make perl linux-headers bind-tools man-db
fi
}
set_timezone()

View File

@ -109,7 +109,7 @@ parse_arg()
while [[ $# > 0 || -n "$ARG" ]]; do
if [[ -z "$ARG" ]]; then ARG=$1; shift; fi
case $ARG in
-q*|--quite ) DFS_QUIET=1 ;;
-q*|--quite ) export DFS_QUIET=1 ;;
--* ) PARSE_ARG_RET+=("$ARG") ;;
-* ) PARSE_ARG_RET+=("${ARG:0:2}") ;;
* ) PARSE_ARG_RET+=("$ARG") ;;
@ -148,10 +148,32 @@ ask_for_Yn()
post_log()
{
python3 "${DOTFILES}/tools/log.py" "[$1] $2: $3"
if [[ $# != 3 || -z "$1" || -z "$2" || -z "$3" ]]; then
fmt_fatal "usage: post_log <level> <section> <content>"
fi
"${DOTFILES}/tools/logger.sh" "log" "[$1][$2] $3"
}
get_os_type() {
apost_log()
{
post_log "$@" 1>/dev/null &
}
post_beacon()
{
if [[ $# != 1 || -z "$1" ]]; then
fmt_fatal "usage: post_beacon <beacon>"
fi
"${DOTFILES}/tools/logger.sh" "beacon" "$1"
}
apost_beacon()
{
post_beacon "$@" 1>/dev/null &
}
get_os_type()
{
local ans="unknown"
case "$(uname -s)" in
Darwin*) ans="MacOS";;
@ -163,7 +185,8 @@ get_os_type() {
echo $ans | tr '[:upper:]' '[:lower:]'
}
get_linux_dist() {
get_linux_dist()
{
local ans="unknown"
if [ -f /etc/os-release ]; then
. /etc/os-release
@ -185,6 +208,15 @@ get_linux_dist() {
echo $ans | tr '[:upper:]' '[:lower:]'
}
get_os_name()
{
local ans=$(get_os_type)
if [[ "$ans" == "linux" ]]; then
ans=$(get_linux_dist)
fi
echo $ans
}
# if bash-ed, else source-d
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
$1 "${@:2}"

View File

@ -1,73 +0,0 @@
#!/usr/bin/env python3
import os, uuid, socket, argparse, logging
namespace = uuid.UUID("cc23b903-1993-44eb-9c90-48bd841eeac3")
logging.basicConfig(level=logging.INFO, format="[%(filename)s:%(lineno)d][%(levelname)s] %(message)s")
try:
import requests
except ImportError:
logging.critical("Please install requests module")
exit(1)
def get_uuid_raw() -> str:
possible_uuid_files = [
"/var/lib/dbus/machine-id",
"/etc/machine-id",
os.path.join(os.path.expanduser('~'), ".config/dotfiles/uuid"),
]
for i in possible_uuid_files:
if os.path.exists(i):
with open(i, "r") as f:
return f.read().strip()
if not os.path.exists(os.path.dirname(possible_uuid_files[-1])):
os.makedirs(os.path.dirname(possible_uuid_files[-1]))
with open(possible_uuid_files[-1], 'w') as f:
ans = str(uuid.uuid4())
f.write(ans)
return ans
def get_uuid() -> str:
return str(uuid.uuid5(namespace, get_uuid_raw()))
def get_hostname() -> str:
ans = socket.gethostname()
if '-' not in ans:
ans += "-ibd-ink"
return ans
def post_log(url:str, hostname:str, uuid:str, content:str):
ans = requests.post(url, params={"hostname": hostname, "uuid": uuid}, data=content)
return ans
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="post log to server")
parser.add_argument("-u", "--url", help="url to post to", default="https://api.beardic.cn/post-log")
parser.add_argument("content")
args = parser.parse_args()
url = args.url
content = args.content
hostname = get_hostname()
uuid = get_uuid()
content=content.strip()
if not content:
logging.error("empty log content")
exit(0)
resp = post_log(url, hostname, uuid, content)
if resp.status_code == 200:
logging.info("200 ok")
exit(0)
elif resp.status_code == 403:
logging.warning("403 forbidden")
logging.info("you may need to register your hostname and uuid")
logging.info(f"hostname: {hostname}, uuid: {uuid}")
exit(0)
else:
logging.critical("unknown error")
logging.error(f"{resp.status_code}: {resp.text}")
exit(1)

100
tools/logger.sh Executable file
View File

@ -0,0 +1,100 @@
#!/bin/bash
set -e
THIS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" && pwd )
source "$THIS_DIR/common.sh"
if [[ -x $(command -v hostname) ]]; then
hostname=$(hostname)
elif [[ -x $(command -v uname) ]]; then
hostname=$(uname -n)
elif [[ -x $(command -v hostnamectl) ]]; then
hostname=$(hostnamectl --static)
elif [[ -n "$HOSTNAME" ]]; then
hostname=$HOSTNAME
elif [[ -f /proc/sys/kernel/hostname ]]; then
hostname=$(cat /proc/sys/kernel/hostname)
elif [[ -f /etc/hostname ]]; then
hostname=$(cat /etc/hostname)
else
fmt_fatal "unable to get hostname"
fi
init_uuid()
{
if [[ -f ~/.config/dotfiles/uuid ]]; then
uuid=$(cat ~/.config/dotfiles/uuid)
else
if [[ -x $(command -v uuidgen) ]]; then
uuid=$(uuidgen)
elif [[ -f /proc/sys/kernel/random/uuid ]]; then
uuid=$(cat /proc/sys/kernel/random/uuid)
else
fmt_fatal "unable to generate uuid"
fi
mkdir -p ~/.config/dotfiles
echo "$uuid" > ~/.config/dotfiles/uuid
fi
}
post_beacon()
{
local beacon_type=$1
if [[ -z "$beacon_type" ]]; then
fmt_fatal "beacon type is required"
fi
resp=$(curl -sSL -X POST "https://api.beardic.cn/post-beacon?hostname=$hostname&beacon=$beacon_type")
if grep -q "200" <<< "$resp"; then
echo $resp
else
echo $resp >&2
fmt_fatal "error posting beacon"
fi
}
post_log()
{
local log_content=$1
if [[ -z "$log_content" ]]; then
fmt_fatal "log content is required"
fi
init_uuid
resp=$(curl -sSL -X POST -H "Content-Type: text/plain" -d "$1" "https://api.beardic.cn/post-log?hostname=$hostname&uuid=$uuid")
if grep -q "200" <<< "$resp"; then
echo $resp
elif grep -q "403" <<< "$resp"; then
echo $resp >&2
fmt_error "error posting log: authentification failed"
fmt_info "try to register you hostname and uuid"
fmt_info "hostname: $hostname"
fmt_info "uuid: $uuid"
else
echo $resp >&2
fmt_fatal "error posting log"
fi
}
print_help()
{
fmt_info "usage: $0 <beacon|log> <beacon_type|log_content>"
}
if [[ $# != 2 ]]; then
print_help >&2
exit 1
fi
case "$1" in
-h|--help)
fmt_info "usage: $0 <beacon|log> <beacon_type|log_content>"
;;
beacon)
post_beacon "$2"
;;
log)
post_log "$2"
;;
*)
fmt_fatal "invalid argument"
;;
esac

20
tools/macos.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
set -e
THIS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" && pwd )
source "$THIS_DIR/common.sh"
brew_install()
{
brew update
brew install git zsh curl tmux vim util-linux
}
router()
{
case $1 in
brew-install ) brew_install ;;
* ) echo unknown command "$1". available: brew-install;;
esac
}
router $@

View File

@ -1,6 +1,7 @@
#!/bin/bash
set -e
THIS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" && pwd )
source "$THIS_DIR/common.sh"
set_mirror()
{
@ -13,7 +14,12 @@ set_mirror()
pacman_S()
{
pacman -Syu
pacman -S tmux git zsh curl vim wget base-devel mingw-w64-x86_64-toolchain make cmake gcc zip unzip python3 python3-pip man-pages-posix
# lite
pacman -S tmux git zsh bash curl vim
# full
if [[ -z "$DFS_LITE" ]]; then
pacman -S wget base-devel mingw-w64-x86_64-toolchain make cmake gcc zip unzip python3 python3-pip man-pages-posix
fi
}
router()

View File

@ -17,6 +17,7 @@ grep -q ".zshrc2" ~/.zshrc
# check scripts and functions
dfs version
dfs log 1
dfs beacon gh.ci
z ~
test ~ -ef "$(pwd)"
dogo

View File

@ -1,24 +1,25 @@
#!/bin/bash
set -e
THIS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" && pwd )
source "$THIS_DIR/common.sh"
set_mirror()
{
MIRROR=${1:-"mirrors.tuna.tsinghua.edu.cn"}
MIRROR=${MIRROR//\//\\\/}
sed -i "s@http://.*archive.ubuntu.com@https://${MIRROR}@g" /etc/apt/sources.list
sed -i "s@http://.*security.ubuntu.com@https://${MIRROR}@g" /etc/apt/sources.list
}
apt_install()
{
# basic packages
apt-get update
for i in {man-db,vim,ca-certificates}; do apt-get install $i -y; done
# mass installation
apt-get install git tmux zsh curl wget dialog net-tools dnsutils netcat traceroute sudo python3 python3-pip cron inetutils-ping openssh-client openssh-server htop gcc g++ cmake make zip less bsdmainutils
for i in {fzf,ripgrep}; do apt-get install $i -y; done
apt-get update -y
# lite
apt-get install -y git zsh bash tmux vim curl inetutils-ping less bsdmainutils
# full
if [[ -z "$DFS_LITE" ]]; then
apt-get install wget dialog net-tools dnsutils netcat traceroute sudo python3 python3-pip cron openssh-client openssh-server htop gcc g++ cmake make zip
for i in {fzf,ripgrep,man-db}; do apt-get install -y $i; done
fi
}
set_timezone()