#!/bin/bash # functions function find_directory { find "$directory_home" -maxdepth 3 -mount -type d -name "$1" 2>/dev/null } function find_remote { rclone listremotes | awk -v remote="$1" '$0 ~ remote {print $0;exit}' } function check_root { if [ "$EUID" -ne 0 ]; then echo "Please run as root" exit 0 fi } function check_not_root { if [ "$EUID" -eq 0 ]; then echo "Don't run this function as root" exit 0 fi } function check_depends { dependencies=(aria2c awk bash docker docker-compose ffmpeg getmail git gnuplot journalctl logname media-sort mp3val mpack munpack opustags phockup pip3 python3 qpdf rbw rclone sed seq sort svgo uniq vnstat we-get yt-dlp) echo "Checking dependencies..." for i in "${dependencies[@]}"; do echo -n "$i: " if [[ $(command -v "$i") ]]; then echo -e "\e[32mpresent\e[39m" else echo -e "\e[31mmissing\e[39m" fi done exit 1 } function umount_remote { if [ -z "$2" ]; then working_directory="$(find_directory "$1")" else working_directory="$(find_directory "$2")" fi umount "$working_directory" fusermount -uz "$working_directory" 2>/dev/null find "$working_directory" -maxdepth 1 -mount -type d -not -path "*/\.*" -empty -delete } function password_manager { case "$1" in addr) check_not_root && rbw get --full "$2" | awk '/URI:/ {print $2}' ;; full) check_not_root && rbw get --full "$2" ;; pass) check_not_root && rbw get "$2" ;; sync) check_not_root && rbw sync ;; user) check_not_root && rbw get --full "$2" | awk '/Username:/ {print $2}' ;; *) check_not_root && rbw get "$2" ;; esac } function duolingo_streak { # check api is installed [[ -d "$(find_directory config)/duolingo" ]] || git clone https://github.com/KartikTalwar/Duolingo.git "$(find_directory config)/duolingo" # cd to git dir to include module cd "$(find_directory config)/duolingo" || return # write script password_manager sync { printf "#!/usr/bin/env python3\\n\\n" printf "import duolingo\\n" printf "lingo = duolingo.Duolingo('%s', password='%s')\\n" "$(password_manager user duolingo)" "$(password_manager pass duolingo)" printf "lingo.buy_streak_freeze()" } >"streak-freeze.py" # run and remove script python "streak-freeze.py" rm "streak-freeze.py" } function blog_duolingo_rank { duo_username="$(awk -F'[/()]' '/Duolingo/ {print $5}' "$(find_directory blog."$domain")"/content/about.md)" rank_filename="$(find_directory blog."$domain")/content/posts/logging-duolingo-ranks-over-time.md" echo -n "Fetching data for $duo_username... " page_source="$(curl -s https://duome.eu/"$duo_username")" rank_lingot="$(printf %s "$page_source" | awk -F"[#><]" '/icon lingot/ {print $15}')" rank_streak="$(printf %s "$page_source" | awk -F"[#><]" '/icon streak/{getline;print $15}')'" echo -e "$i \e[32mdone\e[39m" echo -n "Appending ranks to page... " echo "| $(date +%F) | $(date +%H:%M) | $rank_streak | $rank_lingot |" | tr -d \' >>"$rank_filename" echo -e "$i \e[32mdone\e[39m" echo -n "Amending lastmod value... " mod_timestamp="$(date +%FT%H:%M:00)" sed -i "s/lastmod: .*/lastmod: $mod_timestamp/g" "$rank_filename" echo -e "$i \e[32mdone\e[39m" } function docker_build { cd "$directory_script" || exit # write env file, overwriting any existing password_manager sync { printf "DOMAIN=%s\\n" "$domain" printf "PUID=%s\\n" "$(id -u)" printf "PGID=%s\\n" "$(id -g)" printf "TZ=%s\\n" "$(timedatectl status | awk '/Time zone/ {print $3}')" printf "DOCKDIR=%s\\n" "$(find_directory docker)" printf "SYNCDIR=%s\\n" "$(find_directory vault)" printf "RCLONE_REMOTE_MEDIA=%s\\n" "$(find_remote media)" printf "DBPASSWORD=%s\\n" "$(password_manager pass postgresql)" printf "VPNUSER=%s\\n" "$(password_manager user transmission-openvpn)" printf "VPNPASS=%s\\n" "$(password_manager pass transmission-openvpn)" printf "HTPASSWD=%s\\n" "$(docker exec -it caddy caddy hash-password --plaintext "$(password_manager pass htpasswd)" | base64 -w0)" } >"$directory_script/.env" # make network, if not existing if ! printf "%s" "$(docker network ls)" | grep -q "proxy"; then echo Creating docker network docker network create proxy fi # start containers echo Starting docker containers docker-compose up -d --remove-orphans # delete temporary env file if [[ -f "$directory_script/.env" ]]; then echo Deleting detected env file rm "$directory_script/.env" fi # clean up existing stuff echo Cleaning up existing docker files for i in volume image system network; do docker "$i" prune -f done docker system prune -af } function media_logger { # specify directories git_directory="$(find_directory logger)" file_git_log="$git_directory/media.log" log_remote="$(find_remote media)" git_logger="git --git-dir=$git_directory/.git --work-tree=$git_directory" # git configuruation if [ ! -e "$git_directory" ]; then printf "Logger directory not found, quitting...\n" exit 1 fi if [ ! -e "$git_directory/.git" ]; then printf "Initialising blank git repo...\n" $git_logger init fi if [ -e "$file_git_log.xz" ]; then printf "Decompressing existing xz archive...\n" xz -d "$file_git_log.xz" fi if [ -e "$file_git_log" ]; then printf "Removing existing log file...\n" rm "$file_git_log" fi printf "Creating log...\n" rclone ls "$log_remote" | sort -k2 >"$file_git_log" printf "Appending size information...\n" rclone size "$log_remote" >>"$file_git_log" printf "Commiting log file to repository...\n" $git_logger add "$file_git_log" $git_logger commit -m "Update: $(date +%F)" if [ -e "$file_git_log.xz" ]; then printf "Removing xz archive...\n" rm "$file_git_log.xz" fi printf "Compressing log file...\n" xz "$file_git_log" printf "Compressing repository...\n" $git_logger config pack.windowMemory 10m $git_logger config pack.packSizeLimit 20m $git_logger gc --aggressive --prune printf "Log complete!" } function parse_magnets { # sources and destinations cd "$(find_directory vault)" || exit rclone_remote="raw-seedbox:watch/" # check for aria if [ ! -x "$(command -v aria2c)" ]; then # not installed echo "Aria doesn't seem to be installed. Exiting" && exit fi # trackers trackers_list=( "udp://9.rarbg.to:2710/announce" "udp://denis.stalker.upeer.me:6969/announce" "udp://exodus.desync.com:6969/announce" "udp://ipv6.tracker.harry.lu:80/announce" "udp://open.demonii.si:1337/announce" "udp://open.stealth.si:80/announce" "udp://p4p.arenabg.com:1337/announce" "udp://retracker.lanta-net.ru:2710/announce" "udp://tracker.coppersurfer.tk:6969/announce" "udp://tracker.cyberia.is:6969/announce" "udp://tracker.leechers-paradise.org:6969/announce" "udp://tracker.open-internet.nl:6969/announce" "udp://tracker.opentrackr.org:1337/announce" "udp://tracker.pirateparty.gr:6969/announce" "udp://tracker.tiny-vps.com:6969/announce" "udp://tracker.torrent.eu.org:451/announce" ) for i in "${trackers_list[@]}"; do trackers="$i,$trackers" done # magnet loop for j in *.magnet; do [ -f "$j" ] || break timeout 3m aria2c --bt-tracker="$trackers" --bt-metadata-only=true --bt-save-metadata=true "$(cat "$j")" && rm "$j" done # torrent loop, move to watch for k in *.torrent; do [ -f "$k" ] || break for i in *.torrent; do rclone move "$k" "$rclone_remote"; done done } function parse_payslips { # temporary directories directory_temp="$(mktemp -d)" cd "$directory_temp" || exit mkdir {cur,new,tmp} # write config file password_manager sync { printf "[retriever]\\n" printf "type = SimpleIMAPSSLRetriever\\n" printf "server = %s\\n" "$(password_manager full email | awk -F: '/Incoming/ {gsub(/ /,""); print $2}')" printf "username = %s\\n" "$(password_manager user email)" printf "port = 993\\n" printf "password = %s\\n\\n" "$(password_manager pass email)" printf "[destination]\\n" printf "type = Maildir\\n" printf "path = %s/\\n" "$directory_temp" } >getmailrc getmail --getmaildir "$directory_temp" cd new || exit grep "$(password_manager user payslip)" ./* | cut -f1 -d: | uniq | xargs munpack -f # decrypt payslip file for i in *.PDF; do fileProtected=0 qpdf "$i" --check || fileProtected=1 if [ $fileProtected == 1 ]; then parsed_name="$(printf "%s" "$i" | awk -FX '{print substr($3,5,4) "-" substr($3,3,2) "-" substr($3,1,2) ".pdf"}')" qpdf --password="$(password_manager pass payslip)" --decrypt "$i" "$parsed_name" && rm "$i" rclone move "$parsed_name" "$(find_remote paperwork)/personal/workplace/wages/" fi done # clean up afterwards rm -r "$directory_temp" } function sort_media { # check mounts mounts="seedbox media" for i in $mounts; do mount_remote mount "$i" done # main sorting process dir_import=$(find_directory seedbox)/ if [[ -d "$dir_import" ]]; then dir_tv=$(find_directory media)/videos/television dir_mov=$(find_directory media)/videos/movies temp_tv="{{ .Name }}/{{ .Name }} S{{ printf \"%02d\" .Season }}E{{ printf \"%02d\" .Episode }}{{ if ne .ExtraEpisode -1 }}-{{ printf \"%02d\" .ExtraEpisode }}{{end}}.{{ .Ext }}" temp_mov="{{ .Name }} ({{ .Year }})/{{ .Name }}.{{ .Ext }}" media-sort --action copy --concurrency 1 --accuracy-threshold 90 --tv-dir "$dir_tv" --movie-dir "$dir_mov" --tv-template "$temp_tv" --movie-template "$temp_mov" --recursive --overwrite-if-larger "$dir_import" else printf "Import directory not found.\\n" exit 0 fi for i in $mounts; do umount_remote "$i" done } function mount_remote { # check allow_other in fuse.conf if ! grep -q "^user_allow_other$" /etc/fuse.conf; then check_root printf "user_allow_other not found in fuse.conf.\\nAppending to file. Please restart the script.\\n" echo "user_allow_other" >>/etc/fuse.conf exit 0 fi if [ -n "$2" ]; then printf "Mounting specified remote...\\n" rclone_mount_process "$2" else printf "Mounting all remotes...\\n" rclone_array="$(rclone listremotes | awk -F '[-:]' '/^drive/ && !/backups/ && !/saves/ {print $2}' | xargs)" for i in $rclone_array; do rclone_mount_process "$i" done fi } function rclone_mount_process { remote="$(find_remote "$1")" mount_point="$directory_home/$1" mkdir -p "$mount_point" if [[ -f "$mount_point/.mountcheck" || -n "$(find "$mount_point" -maxdepth 1 -mindepth 1 | head -n 1)" ]]; then printf "%s already mounted.\\n" "$1" else printf "%s not mounted.\\n" "$1" printf "Re-mounting... " fusermount -uz "$mount_point" 2>/dev/null && sleep 3 rclone mount "$remote" "$mount_point" --allow-other --daemon --log-file "$(find_directory config)/logs/rclone-$1.log" printf "done\\n" fi } function blog_status { status_uptime=$(($(cut -f1 -d. 1)); then echo s; fi)" printf "* CPU Load: %s\\n" "$(cut -d" " -f1-3 "$(find_directory blog."$domain")/content/status.md" } function blog_weight { if [ "$2" = "date" ]; then printf "Writing empty dates... " weight_filename="$(find_directory blog."$domain")/content/weight.md" page_source="$(head -n -1 "$weight_filename")" previous_date="$(printf %s "$page_source" | awk -F, 'END{print $1}')" sequence_count="$((($(date --date="$(date +%F)" +%s) - $(date --date="$previous_date" +%s)) / (60 * 60 * 24)))" { printf "%s\\n" "$page_source" printf "%s" "$(for i in $(seq $sequence_count); do printf "%s,\\n" "$(date -d "$previous_date+$i day" +%F)"; done)" printf "\\n" } >"$weight_filename" printf "done\\n" exit 0 fi printf "Drawing graph... " weight_filename="$(find_directory blog."$domain")/content/weight.md" weight_rawdata="$(awk '/
/{flag=1; next} /<\/pre>/{flag=0} flag' "$weight_filename" | sort -u)" weight_dateinit="$(awk '/date:/ {print $2}' "$weight_filename")" grep "^$(date +%Y)-" <<<"$weight_rawdata" >temp.dat gnuplot <<-EOF set grid set datafile separator comma set xlabel "Month" set xdata time set timefmt "%Y-%m-%d" set xtics format "%b" set ylabel "Weight (kg)" set key off set term svg font 'sans-serif,12' set sample 50 set output "temp.svg" plot "temp.dat" using 1:2 smooth cspline with lines EOF printf "done\\nWriting page... " { printf -- "---\\ntitle: Weight\\nlayout: single\\ndate: %s\\nlastmod: %(%Y-%m-%dT%H:%M:00)T\\n---\\n\\n" "$weight_dateinit" -1 printf "%s\\n\\n" "$(svgo -i "temp.svg" --multipass -o - -q -p 0)" printf "" "$weight_rawdata" } >"$weight_filename" printf "done\\nCleaning up... " rm temp.{dat,svg} printf "done\\n" } function remotes_dedupe { dests=$(rclone listremotes | grep "gdrive" -c) for i in $(seq "$dests"); do remote=$(rclone listremotes | grep "gdrive.*$i") echo Deduplicating "$remote" rclone dedupe --dedupe-mode newest "$remote" --log-file "$(find_directory config)/logs/rclone-dupe-$(date +%F-%H%M).log" done } function remotes_tokens { echo "Refreshing rclone remote tokens" for i in $(rclone listremotes | awk -v remote="$backup_prefix" '$0 ~ remote {print $0}'); do if rclone lsd "$i" &>/dev/null; then echo -e "$i \e[32msuccess\e[39m" else echo -e "$i \e[31mfailed\e[39m" fi done } function remotes_sync { source=$(find_remote "gdrive") dests=$(rclone listremotes | grep "gdrive" -c) for i in $(seq 2 "$dests"); do dest=$(rclone listremotes | grep "gdrive.*$i") if rclone lsd "$dest" &>/dev/null; then if [ -n "$2" ]; then directory="$2" printf "Syncing %s directory to %s...\\n" "$directory" "$dest" rclone sync "$source/$directory" "$dest/$directory" --drive-server-side-across-configs --drive-stop-on-upload-limit --verbose --log-file "$(find_directory config)/logs/rclone-sync-$directory-$(date +%F-%H%M).log" else printf "Syncing %s to %s...\\n" "$source" "$dest" rclone sync "$source" "$dest" --drive-server-side-across-configs --drive-stop-on-upload-limit --verbose --log-file "$(find_directory config)/logs/rclone-sync-$(date +%F-%H%M).log" fi fi done } function parse_photos { source=$(find_directory DCIM) mount="pictures" mount_remote mount "$mount" destination=$(find_directory $mount) # main sorting process if [[ -d "$destination" ]]; then phockup "$source" "$destination/personal/photos/" -m find "$source" -maxdepth 3 -mount -type d -not -path "*/\.*" -empty -delete else printf "Import directory not found.\\n" exit 0 fi umount_remote "$mount" } function backup_docker { check_not_root password_manager sync password="$(password_manager pass 'backup archive password')" backup_final="$(find_remote backups)" cd "$(find_directory config)" || exit for i in */; do backup_file="$(basename "$i")_backup-$(date +%F-%H%M).tar.xz.gpg" echo -n Backing up "$i"... if docker ps -a | grep -q "$i"; then docker stop "$i" 1>/dev/null sudo tar -cJf - "$i" | gpg -c --batch --passphrase "$password" >"$backup_file" docker start "$i" 1>/dev/null else sudo tar -cJf - "$i" | gpg -c --batch --passphrase "$password" >"$backup_file" fi sudo chown "$username":"$username" "$backup_file" echo -e "$i \e[32mdone\e[39m" done # send to remotes, final operation is a move, removing the backup for i in *_backup-*."tar.xz.gpg"; do for j in $(rclone listremotes | awk -v remote="$backup_prefix" '$0 ~ remote {print $0}'); do echo -n Copying "$i" to "$j"... rclone copy "$i" "$j" && echo -e "\e[32mdone\e[39m" || echo -e "\e[31mfailed\e[39m" done echo -n Moving "$i" to "$backup_final"... rclone move "$i" "$backup_final" && echo -e "\e[32mdone\e[39m" || echo -e "\e[31mfailed\e[39m" done } function clean_space { space_initial="$(df / | awk 'FNR==2{ print $4}')" log_file="$(find_directory config)/logs/clean-$(date +%F-%H%M).log" check_root # journals journalctl --vacuum-size=75M >>"$log_file" # docker for i in volume image; do docker "$i" prune -f >>"$log_file" done # apt apt-get clean >>"$log_file" # temp directory rm -rf /tmp/tmp.* >>"$log_file" space_after="$(df / | awk 'FNR==2{ print $4}')" printf "Bytes freed: %s\\n" "$(("$space_after" - "$space_initial"))" } function remote_sizes { dests=$(rclone listremotes | grep "gdrive" -c) for i in $(seq "$dests"); do remote=$(rclone listremotes | grep "gdrive.*$i") echo -n Calculating "$remote"... rclone size "$remote" | xargs done } function system_update { check_root if [[ $distro =~ "Debian" ]]; then # Update Debian export DEBIAN_FRONTEND=noninteractive apt-get update apt-get dist-upgrade -y apt-get autoremove --purge -y apt-get clean if [ -x "$(command -v yt-dlp)" ]; then yt-dlp -U fi if [ -x "$(command -v rclone)" ]; then curl --silent "https://rclone.org/install.sh" | bash fi elif [[ $distro =~ "Arch" ]]; then # Update Archlinux if [ -x "$(command -v paru)" ]; then paru -Syu --noconfirm else pacman -Syu --noconfirm fi else echo "Who knows what you're running" fi # Update remaining applications find "$(find_directory config)" -maxdepth 2 -name ".git" -type d | sed 's/\/.git//' | xargs -P10 -I{} git -C {} pull if [ -x "$(command -v we-get)" ]; then echo "Updating we-get..." pip3 install --upgrade git+https://github.com/rachmadaniHaryono/we-get fi if [ -x "$(command -v media-sort)" ]; then echo "Updating media-sort..." cd "/usr/local/bin" || return curl https://i.jpillora.com/media-sort | bash fi } function main { distro="$(awk -F'"' '/^NAME/ {print $2}' /etc/os-release)" username="$(logname)" directory_home="/home/$username" backup_prefix="backup-" domain="minskio.co.uk" directory_script="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" case "$1" in backup) backup_docker ;; bookmarks) grep -P "\t\t\t\Raw data
\\n\\n%s\\n