Files
shell-scripting-templates/utilities/files.bash
Nathaniel Landau b227cf6330 Major overhaul
After working for ~6 years in private repositories, bringing my
updated BASH scripting templates back into the world.
2021-07-13 17:03:27 -04:00

652 lines
18 KiB
Bash

_listFiles_() {
# DESC: Find files in a directory. Use either glob or regex
# ARGS: $1 (Required) - 'g|glob' or 'r|regex'
# $2 (Required) - pattern to match
# $3 (Optional) - directory
# OUTS: Prints files to STDOUT
# NOTE: Searches are NOT case sensitive and MUST be quoted
# USAGE: _listFiles_ glob "*.txt" "some/backup/dir"
# _listFiles_ regex ".*\.txt" "some/backup/dir"
# readarry -t array < <(_listFiles_ g "*.txt")
[[ $# -lt 2 ]] && {
error 'Missing required argument to _listFiles_()!'
return 1
}
local t="${1}"
local p="${2}"
local d="${3:-.}"
local fileMatch e
case "$t" in
glob | Glob | g | G)
while read -r fileMatch; do
echo "${e}"
done < <(find "${d}" -iname "${p}" -type f -maxdepth 1 | sort)
;;
regex | Regex | r | R)
while read -r fileMatch; do
e="$(realpath "${fileMatch}")"
echo "${e}"
done < <(find "${d}" -iregex "${p}" -type f -maxdepth 1 | sort)
;;
*)
echo "Could not determine if search was glob or regex"
return 1
;;
esac
}
_backupFile_() {
# DESC: Creates a backup of a specified file with .bak extension or
# optionally to a specified directory
# ARGS: $1 (Required) - Source file
# $2 (Optional) - Destination dir name used only with -d flag (defaults to ./backup)
# OPTS: -d - Move files to a backup direcory
# -m - Replaces copy (default) with move, effectively removing the
# OUTS: None
# USAGE: _backupFile_ "sourcefile.txt" "some/backup/dir"
# NOTE: dotfiles have their leading '.' removed in their backup
local opt
local OPTIND=1
local useDirectory=false
local moveFile=false
while getopts ":dDmM" opt; do
case ${opt} in
d | D) useDirectory=true ;;
m | M) moveFile=true ;;
*)
{
error "Unrecognized option '$1' passed to _makeSymlink_" "${LINENO}"
return 1
}
;;
esac
done
shift $((OPTIND - 1))
[[ $# -lt 1 ]] && fatal 'Missing required argument to _backupFile_()!'
local s="${1}"
local d="${2:-backup}"
local n # New filename (created by _uniqueFilename_)
# Error handling
[ ! "$(declare -f "_execute_")" ] \
&& {
warning "need function _execute_"
return 1
}
[ ! "$(declare -f "_uniqueFileName_")" ] \
&& {
warning "need function _uniqueFileName_"
return 1
}
[ ! -e "$s" ] \
&& {
warning "Source '${s}' not found"
return 1
}
if [ ${useDirectory} == true ]; then
[ ! -d "${d}" ] \
&& _execute_ "mkdir -p \"${d}\"" "Creating backup directory"
if [ -e "$s" ]; then
n="$(basename "${s}")"
n="$(_uniqueFileName_ "${d}/${s#.}")"
if [ ${moveFile} == true ]; then
_execute_ "mv \"${s}\" \"${d}/${n##*/}\"" "Moving: '${s}' to '${d}/${n##*/}'"
else
_execute_ "cp -R \"${s}\" \"${d}/${n##*/}\"" "Backing up: '${s}' to '${d}/${n##*/}'"
fi
fi
else
n="$(_uniqueFileName_ "${s}.bak")"
if [ ${moveFile} == true ]; then
_execute_ "mv \"${s}\" \"${n}\"" "Moving '${s}' to '${n}'"
else
_execute_ "cp -R \"${s}\" \"${n}\"" "Backing up '${s}' to '${n}'"
fi
fi
}
_cleanFilename_() {
# DESC: Cleans a filename of all non-alphanumeric (or user specified)
# characters and overwrites original
# ARGS: $1 (Required) - File to be cleaned
# $2 (optional) - Additional characters to be cleaned separated by commas
# OUTS: Overwrites file with new new and prints name of new file
# USAGE: _cleanFilename_ "FILENAME.TXT" "^,&,*"
# NOTE: IMPORTANT - This will overwrite the original file
# IMPORTANT - All spaces and underscores will be replaced by dashes (-)
[[ $# -lt 1 ]] && fatal 'Missing required argument to _cleanFilename_()!'
local arrayToClean
local fileToClean="$(realpath "$1")"
local optionalUserInput="${2-}"
IFS=',' read -r -a arrayToClean <<<"$optionalUserInput"
[ ! -f "${fileToClean}" ] \
&& {
warning "_cleanFileName_ ${fileToClean}: File doesn't exist"
return 1
}
local dir="$(realpath -d "${fileToClean}")"
local extension="${fileToClean##*.}"
local baseFileName="$(basename "${fileToClean%.*}")"
for i in "${arrayToClean[@]}"; do
baseFileName="$(echo "${baseFileName}" | sed "s/$i//g")"
done
baseFileName="$(echo "${baseFileName}" | tr -dc '[:alnum:]-_ ' | sed 's/ /-/g')"
local final="${dir}/${baseFileName}.${extension}"
if [ "${fileToClean}" != "${final}" ]; then
final="$(_uniqueFileName_ "${final}")"
if ${VERBOSE}; then
_execute_ "mv \"${fileToClean}\" \"${final}\""
else
_execute_ -q "mv \"${fileToClean}\" \"${final}\""
fi
echo "${final}"
else
echo "${fileToClean}"
fi
}
_parseFilename_() {
# DESC: Break a filename into its component parts which and place them into prefixed
# variables (dir, basename, extension, full path, etc.)
# with _parseFile...
# ARGS: $1 (Required) - A file
# OUTS: $_parsedFileFull - File and its real path (ie, resolve symlinks)
# $_parseFilePath - Path to the file
# $_parseFileName - Name of the file WITH extension
# $_parseFileBase - Name of file WITHOUT extension
# $_parseFileExt - The extension of the file (from _ext_())
[[ $# -lt 1 ]] && fatal 'Missing required argument to _parseFilename_()!'
local fileToParse="${1}"
[[ -f "${fileToParse}" ]] || {
error "Can't locate good file to parse at: ${fileToParse}"
return 1
}
# Ensure we are working with a real file, not a symlink
_parsedFileFull="$(realpath "${fileToParse}")" \
&& debug "${tan}\${_parsedFileFull}: ${_parsedFileFull-}${purple}"
# use the basename of the userFile going forward since the path is now in $filePath
_parseFileName=$(basename "${fileToParse}") \
&& debug "${tan}\$_parseFileName: ${_parseFileName}${purple}"
# Grab the filename without the extension
_parseFileBase="${_parseFileName%.*}" \
&& debug "${tan}\$_parseFileBase: ${_parseFileBase}${purple}"
# Grab the extension
if [[ "${fileToParse}" =~ .*\.[a-zA-Z]{2,4}$ ]]; then
_parseFileExt="$(_ext_ "${_parseFileName}")"
else
_parseFileExt=".${_parseFileName##*.}"
fi
debug "${tan}\$_parseFileExt: ${_parseFileExt}${purple}"
# Grab the directory
_parseFilePath="${_parsedFileFull%/*}" \
&& debug "${tan}\${_parseFilePath}: ${_parseFilePath}${purple}"
}
_decryptFile_() {
# DESC: Decrypts a file with openSSL
# ARGS: $1 (Required) - File to be decrypted
# $2 (Optional) - Name of output file (defaults to $1.decrypt)
# OUTS: None
# USAGE: _decryptFile_ "somefile.txt.enc" "decrypted_somefile.txt"
# NOTE: If a variable '$PASS' has a value, we will use that as the password
# to decrypt the file. Otherwise we will ask
[[ $# -lt 1 ]] && fatal 'Missing required argument to _decryptFile_()!'
local fileToDecrypt decryptedFile defaultName
fileToDecrypt="${1:?_decryptFile_ needs a file}"
defaultName="${fileToDecrypt%.enc}"
decryptedFile="${2:-$defaultName.decrypt}"
[ ! "$(declare -f "_execute_")" ] \
&& {
echo "need function _execute_"
return 1
}
[ ! -f "$fileToDecrypt" ] && return 1
if [ -z "${PASS}" ]; then
_execute_ "openssl enc -aes-256-cbc -d -in \"${fileToDecrypt}\" -out \"${decryptedFile}\"" "Decrypt ${fileToDecrypt}"
else
_execute_ "openssl enc -aes-256-cbc -d -in \"${fileToDecrypt}\" -out \"${decryptedFile}\" -k \"${PASS}\"" "Decrypt ${fileToDecrypt}"
fi
}
_encryptFile_() {
# DESC: Encrypts a file using openSSL
# ARGS: $1 (Required) - Input file
# $2 (Optional) - Name of output file (defaults to $1.enc)
# OUTS: None
# USAGE: _encryptFile_ "somefile.txt" "encrypted_somefile.txt"
# NOTE: If a variable '$PASS' has a value, we will use that as the password
# for the encrypted file. Otherwise we will ask.
local fileToEncrypt encryptedFile defaultName
fileToEncrypt="${1:?_encodeFile_ needs a file}"
defaultName="${fileToEncrypt%.decrypt}"
encryptedFile="${2:-$defaultName.enc}"
[ ! -f "$fileToEncrypt" ] && return 1
[ ! "$(declare -f "_execute_")" ] \
&& {
echo "need function _execute_"
return 1
}
if [ -z "${PASS}" ]; then
_execute_ "openssl enc -aes-256-cbc -salt -in \"${fileToEncrypt}\" -out \"${encryptedFile}\"" "Encrypt ${fileToEncrypt}"
else
_execute_ "openssl enc -aes-256-cbc -salt -in \"${fileToEncrypt}\" -out \"${encryptedFile}\" -k \"${PASS}\"" "Encrypt ${fileToEncrypt}"
fi
}
_ext_() {
# DESC: Extract the extension from a filename
# ARGS: $1 (Required) - Input file
# OPTS: -n - optional flag for number of extension levels (Ex: -n2)
# OUTS: Print extension to STDOUT
# USAGE:
# _ext_ foo.txt #==> txt
# _ext_ -n2 foo.tar.gz #==> tar.gz
# _ext_ foo.tar.gz #==> tar.gz
# _ext_ -n1 foo.tar.gz #==> gz
[[ $# -lt 1 ]] && fatal 'Missing required argument to _ext_()!'
local levels
local option
local filename
local exts
local ext
local fn
local i
unset OPTIND
while getopts ":n:" option; do
case $option in
n) levels=$OPTARG ;;
*) continue ;;
esac
done && shift $((OPTIND - 1))
filename=${1##*/}
[[ $filename == *.* ]] || return
fn=$filename
# Detect some common multi-extensions
if [[ ! ${levels-} ]]; then
case $(tr '[:upper:]' '[:lower:]' <<<"${filename}") in
*.tar.gz | *.tar.bz2) levels=2 ;;
esac
fi
levels=${levels:-1}
for ((i = 0; i < levels; i++)); do
ext=${fn##*.}
exts=${ext}${exts-}
fn=${fn%$ext}
[[ "$exts" == "${filename}" ]] && return
done
echo "$exts"
}
_extract_() {
# DESC: Extract a compressed file
# ARGS: $1 (Required) - Input file
# $2 (optional) - Input 'v' to show verbose output
# OUTS: None
local filename
local foldername
local fullpath
local didfolderexist
local vv
[[ $# -lt 1 ]] && fatal 'Missing required argument to _extract_()!'
[[ "${2-}" == "v" ]] && vv="v"
if [ -f "$1" ]; then
case "$1" in
*.tar.bz2 | *.tbz | *.tbz2) tar "x${vv}jf" "$1" ;;
*.tar.gz | *.tgz) tar "x${vv}zf" "$1" ;;
*.tar.xz)
xz --decompress "$1"
set -- "$@" "${1:0:-3}"
;;
*.tar.Z)
uncompress "$1"
set -- "$@" "${1:0:-2}"
;;
*.bz2) bunzip2 "$1" ;;
*.deb) dpkg-deb -x${vv} "$1" "${1:0:-4}" ;;
*.pax.gz)
gunzip "$1"
set -- "$@" "${1:0:-3}"
;;
*.gz) gunzip "$1" ;;
*.pax) pax -r -f "$1" ;;
*.pkg) pkgutil --expand "$1" "${1:0:-4}" ;;
*.rar) unrar x "$1" ;;
*.rpm) rpm2cpio "$1" | cpio -idm${vv} ;;
*.tar) tar "x${vv}f" "$1" ;;
*.txz)
mv "$1" "${1:0:-4}.tar.xz"
set -- "$@" "${1:0:-4}.tar.xz"
;;
*.xz) xz --decompress "$1" ;;
*.zip | *.war | *.jar) unzip "$1" ;;
*.Z) uncompress "$1" ;;
*.7z) 7za x "$1" ;;
*) return 1 ;;
esac
else
return 1
fi
shift
}
_json2yaml_() {
# DESC: Convert JSON to YAML
# ARGS: $1 (Required) - JSON file
# OUTS: None
python -c 'import sys, yaml, json; yaml.safe_dump(json.load(sys.stdin), sys.stdout, default_flow_style=False)' <"${1:?_json2yaml_ needs a file}"
}
_makeSymlink_() {
# DESC: Creates a symlink and backs up a file which may be overwritten by the new symlink. If the
# exact same symlink already exists, nothing is done.
# Default behavior will create a backup of a file to be overwritten
# ARGS: $1 (Required) - Source file
# $2 (Required) - Destination
# $3 (Optional) - Backup directory for files which may be overwritten (defaults to 'backup')
# OPTS: -n - Do not create a backup if target already exists
# -s - Use sudo when removing old files to make way for new symlinks
# OUTS: None
# USAGE: _makeSymlink_ "/dir/someExistingFile" "/dir/aNewSymLink" "/dir/backup/location"
# NOTE: This function makes use of the _execute_ function
local opt
local OPTIND=1
local backupOriginal=true
local useSudo=false
while getopts ":nNsS" opt; do
case $opt in
n | N) backupOriginal=false ;;
s | S) useSudo=true ;;
*)
{
error "Unrecognized option '$1' passed to _makeSymlink_" "$LINENO"
return 1
}
;;
esac
done
shift $((OPTIND - 1))
if ! command -v realpath >/dev/null 2>&1; then
error "We must have 'realpath' installed and available in \$PATH to run."
if [[ "$OSTYPE" == "darwin"* ]]; then
notice "Install coreutils using homebrew and rerun this script."
info "\t$ brew install coreutils"
fi
_safeExit_ 1
fi
[[ $# -lt 2 ]] && fatal 'Missing required argument to _makeSymlink_()!'
local s="$1"
local d="$2"
local b="${3-}"
local o
# Fix files where $HOME is written as '~'
d="${d/\~/$HOME}"
s="${s/\~/$HOME}"
b="${b/\~/$HOME}"
[ ! -e "$s" ] \
&& {
error "'$s' not found"
return 1
}
[ -z "$d" ] \
&& {
error "'${d}' not specified"
return 1
}
[ ! "$(declare -f "_execute_")" ] \
&& {
echo "need function _execute_"
return 1
}
[ ! "$(declare -f "_backupFile_")" ] \
&& {
echo "need function _backupFile_"
return 1
}
# Create destination directory if needed
[ ! -d "${d%/*}" ] \
&& _execute_ "mkdir -p \"${d%/*}\""
if [ ! -e "${d}" ]; then
_execute_ "ln -fs \"${s}\" \"${d}\"" "symlink ${s}${d}"
elif [ -h "${d}" ]; then
o="$(realpath "${d}")"
[[ "${o}" == "${s}" ]] && {
if [ "${DRYRUN}" == true ]; then
dryrun "Symlink already exists: ${s}${d}"
else
info "Symlink already exists: ${s}${d}"
fi
return 0
}
if [[ "${backupOriginal}" == true ]]; then
_backupFile_ "${d}" "${b:-backup}"
fi
if [[ "${DRYRUN}" == false ]]; then
if [[ "${useSudo}" == true ]]; then
command rm -rf "${d}"
else
command rm -rf "${d}"
fi
fi
_execute_ "ln -fs \"${s}\" \"${d}\"" "symlink ${s}${d}"
elif [ -e "${d}" ]; then
if [[ "${backupOriginal}" == true ]]; then
_backupFile_ "${d}" "${b:-backup}"
fi
if [[ "${DRYRUN}" == false ]]; then
if [[ "${useSudo}" == true ]]; then
sudo command rm -rf "${d}"
else
command rm -rf "${d}"
fi
fi
_execute_ "ln -fs \"${s}\" \"${d}\"" "symlink ${s}${d}"
else
warning "Error linking: ${s}${d}"
return 1
fi
return 0
}
_parseYAML_() {
# DESC: Convert a YANML file into BASH variables for use in a shell script
# ARGS: $1 (Required) - Source YAML file
# $2 (Required) - Prefix for the variables to avoid namespace collisions
# OUTS: Prints variables and arrays derived from YAML File
# USAGE: To source into a script
# _parseYAML_ "sample.yml" "CONF_" > tmp/variables.txt
# source "tmp/variables.txt"
#
# NOTE: https://gist.github.com/DinoChiesa/3e3c3866b51290f31243
# https://gist.github.com/epiloque/8cf512c6d64641bde388
local yamlFile="${1:?_parseYAML_ needs a file}"
local prefix="${2-}"
[ ! -s "${yamlFile}" ] && return 1
local s='[[:space:]]*'
local w='[a-zA-Z0-9_]*'
local fs="$(echo @ | tr @ '\034')"
sed -ne "s|^\(${s}\)\(${w}\)${s}:${s}\"\(.*\)\"${s}\$|\1${fs}\2${fs}\3|p" \
-e "s|^\(${s}\)\(${w}\)${s}[:-]${s}\(.*\)${s}\$|\1${fs}\2${fs}\3|p" "${yamlFile}" \
| awk -F"${fs}" '{
indent = length($1)/2;
if (length($2) == 0) { conj[indent]="+";} else {conj[indent]="";}
vname[indent] = $2;
for (i in vname) {if (i > indent) {delete vname[i]}}
if (length($3) > 0) {
vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
printf("%s%s%s%s=(\"%s\")\n", "'"${prefix}"'",vn, $2, conj[indent-1],$3);
}
}' | sed 's/_=/+=/g' | sed 's/[[:space:]]*#.*"/"/g'
}
_readFile_() {
# DESC: Prints each line of a file
# ARGS: $1 (Required) - Input file
# OUTS: Prints contents of file
[[ $# -lt 1 ]] && fatal 'Missing required argument to _readFile_()!'
local result
local c="$1"
[ ! -f "$c" ] \
&& {
echo "'$c' not found"
return 1
}
while read -r result; do
echo "${result}"
done <"${c}"
}
_sourceFile_() {
# DESC: Source a file into a script
# ARGS: $1 (Required) - File to be sourced
# OUTS: None
[[ $# -lt 1 ]] && fatal 'Missing required argument to _sourceFile_()!'
local c="$1"
[ ! -f "$c" ] \
&& {
fatal "Attempted to source '$c' Not found"
return 1
}
source "$c"
return 0
}
_uniqueFileName_() {
# DESC: Ensure a file to be created has a unique filename to avoid overwriting other files
# ARGS: $1 (Required) - Name of file to be created
# $2 (Optional) - Separation characted (Defaults to a period '.')
# OUTS: Prints unique filename to STDOUT
# USAGE: _uniqueFileName_ "/some/dir/file.txt" "-"
local fullfile="${1:?_uniqueFileName_ needs a file}"
local spacer="${2:-.}"
local directory
local filename
local extension
local newfile
local n
if ! command -v realpath >/dev/null 2>&1; then
error "We must have 'realpath' installed and available in \$PATH to run."
if [[ "$OSTYPE" == "darwin"* ]]; then
notice "Install coreutils using homebrew and rerun this script."
info "\t$ brew install coreutils"
fi
_safeExit_ 1
fi
# Find directories with realpath if input is an actual file
if [ -e "${fullfile}" ]; then
fullfile="$(realpath "${fullfile}")"
fi
directory="$(dirname "${fullfile}")"
filename="$(basename "${fullfile}")"
# Extract extensions only when they exist
if [[ "${filename}" =~ \.[a-zA-Z]{2,4}$ ]]; then
extension=".${filename##*.}"
filename="${filename%.*}"
fi
newfile="${directory}/${filename}${extension-}"
if [ -e "${newfile}" ]; then
n=1
while [[ -e "${directory}/${filename}${extension-}${spacer}${n}" ]]; do
((n++))
done
newfile="${directory}/${filename}${extension-}${spacer}${n}"
fi
echo "${newfile}"
return 0
}
_yaml2json_() {
# DESC: Convert a YAML file to JSON
# ARGS: $1 (Required) - Input YAML file
# OUTS: None
python -c 'import sys, yaml, json; json.dump(yaml.load(sys.stdin), sys.stdout, indent=4)' <"${1:?_yaml2json_ needs a file}"
}