./podget-0.9.3/0000755000175000017500000000000014544355000012550 5ustar lowkeylowkey./podget-0.9.3/SCRIPTS/0000755000175000017500000000000014116456457013714 5ustar lowkeylowkey./podget-0.9.3/SCRIPTS/README0000644000175000017500000000012712620220525014553 0ustar lowkeylowkeyThis directory is for user contributed scripts that extend Podget's functionality. ./podget-0.9.3/SCRIPTS/PRE_0.7.0/0000755000175000017500000000000013344003100015033 5ustar lowkeylowkey./podget-0.9.3/SCRIPTS/PRE_0.7.0/README0000644000175000017500000000040112620220525015716 0ustar lowkeylowkeyThis folder contains older user submitted scripts that extend podget's functionality but have not be updated to reflect the new configuration file format used as of podget version 0.7.0. They are included here as inspiration to others on what can be done. ./podget-0.9.3/SCRIPTS/PRE_0.7.0/podget-add-feed0000644000175000017500000006354112620220525017710 0ustar lowkeylowkey#!/bin/bash # Copyright (C) 2009 Steven Black # Contributed to podget on 2009-11-06 to be released under the same license # as podget (currently GPL) EMPTY_GROUP="Unknown" EMPTY_TITLE="Unknown" PODGET_DIR="$HOME/.podget" config_serverlist=`sed -r -n -e 's/^[ ]*config_serverlist[ ]*=[ ]*(["]?)(.*)\1[ ]*$/\2/p' "$PODGET_DIR/podgetrc"` dir_library=`sed -r -n -e 's/^[ ]*dir_library[ ]*=[ ]*(["]?)(.*)\1[ ]*$/\2/p' "$PODGET_DIR/podgetrc"` case "$config_serverlist" in /*) SERVERLIST="$config_serverlist" ;; *) SERVERLIST="$PODGET_DIR/$config_serverlist" ;; esac case "$dir_library" in /*) LIBRARY_DIR="$dir_library" ;; *) LIBRARY_DIR="$PODGET_DIR/$dir_library" ;; esac if [ ! -f "$SERVERLIST" ] ; then exit 0 fi CURL=`which curl` fake_title() { # Originally we faked a title from the URL. # Now we only do that if we can't get one via curl and sed FAKE_TITLE="" if [ -x "$CURL" ] ; then FAKE_TITLE=`"$CURL" -A "Mozilla/4.0" -f -L -s "$1" | sed -n -e '//{ s|^.*<title>\(.*\).*$|\1|Mp q }'` fi if [ -z "$FAKE_TITLE" ] ; then FAKE_TITLE=`sed -r \ -e 's;^https?://;;' -e 's;[.](com|org|net)/;/;' \ -e 's;^(www|web|feeds?[0-9]?)?[.];;' \ -e 's;/(bookfeeds|[sS]ite)/;/;g' -e 's;/[mM][pP]3_[fF]eed;;g' \ -e 's;[.]libsyn;;' -e 's;^(me|feedburner|librivox)/;;' \ -e 's;/feeds?/;/;' -e 's;[.](xml|html)$;;' \ -e 's;/feed$;;' \ -e 's;/[?]feed=[^/.]+;;' -e 's;/rss/?$;;' -e 's;/$;;' \ -e 's;%20; ;g' -e 's;%[0-9a-fA-F][0-9a-fA-F];;' -e 'y;/;_;' \ <<< "$1"` fi } # fake_title "$1" # echo $FAKE_TITLE # exit 0 unsetcompetition() { KDIALOG="" XDIALOG="" GTKDIALOG="" WHIPTAIL="" TCDIALOG="" DIALOG="" AUTOMODE=0 } checkdialog() { USRDIALOG="$1" unsetcompetition if [ "$USRDIALOG" = "auto" ] ; then AUTOMODE=1 return fi if [ ! -z "$USRDIALOG" ] ; then case "$USRDIALOG" in */*) ;; *) USRDIALOG=`which $USRDIALOG` ;; esac if [ ! -x "$USRDIALOG" ] ; then USRDIALOG="" ; fi fi if [ ! -z "$USRDIALOG" ] ; then case "$USRDIALOG" in */kdialog*) KDIALOG="$USRDIALOG" ;; */[xX]dialog*) XDIALOG="$USRDIALOG" ;; */zenity*) ZENITY="$USRDIALOG" ;; */gtkdialog*) GTKDIALOG="$USRDIALOG" ;; */whiptail*) WHIPTAIL="$USRDIALOG" ;; */tcdialog*) TCDIALOG="$USRDIALOG" ;; */sh|*/bash*) DIALOG="" ;; *) # treat as 'dialog' if [ ! -z "$DISPLAY" ] ; then if tty -s ; then # Normally 'dialog' will only be used on a real TTY. # They have a real TTY, so we're fine. true DIALOG="$USRDIALOG" else # Woah, no TTY. Perhaps we should treat this as Xdialog. XDIALOG="$USRDIALOG" fi else DIALOG="$USRDIALOG" fi ;; esac else if [ ! -z "$DISPLAY" ] ; then KDIALOG=`which kdialog` XDIALOG=`which Xdialog` GTKDIALOG=`which gtkdialog` # Note: We *never* try to use 'zenity' unless the user specified # it. It just doesn't work the way we want -- at all. fi if tty -s ; then WHIPTAIL=`which whiptail` TCDIALOG=`which tcdialog` DIALOG=`which dialog` fi fi } dialogcheck() { local DIALOGS=0 if [ "$AUTOMODE" = "1" ] ; then if [ ! -z "$1" ] ; then echo "Running in automatic mode." fi return 0 fi if [ ! -z "$DISPLAY" ] ; then if [ ! -z "$1" ] ; then echo "X dialog services:" fi fi if [ -x "$KDIALOG" ] ; then if [ ! -z "$1" ] ; then echo "kdialog: good" fi DIALOGS=1 fi if [ -x "$GTKDIALOG" ] ; then if [ ! -z "$1" ] ; then echo "gtkdialog: good" fi DIALOGS=1 fi if [ -x "$XDIALOG" ] ; then if [ ! -z "$1" ] ; then echo "xdialog: okay" echo "---------- NOTICE regarding the use of Xdialog ----------" echo "Xdialog does not have the capacity to load in an icon or " echo "pixmap. This means it'll be uglier than the competition." echo "-----------------------------------------------------------" echo "For a prettier display, install kdialog or gtkdialog." fi DIALOGS=1 fi if [ -x "$ZENITY" ] ; then DIALOGS=1 echo "zenity: BAD (and UNSUPPORTED)" echo "---------- WARNING regarding the use of Zenity ----------" echo "'zenity' is unsupported. 'zenity' has no support for yes/no or" echo "yes/no/cancel dialogs. It has no capacity to add a button and" echo "change labels, allowing any sort of usable yes/no/cancel system." echo "zenity does not let me repurpose ok/cancel dialogs. The return" echo "code for cancel is the same as abort, so I can't even make do" echo "with changing the behavior there." echo echo "The dialogs will be confusing and missing features." echo echo "This is why it is unsupported." echo echo "Please install kdialog, Xdialog, or gtkdialog or run it from" echo "a terminal." echo echo "Really, even the automatic mode works better than zenity mode." echo "---------------------------------------------------------" echo "zenity is unsupported. Use at your own risk." if [ -z "$1" ] ; then somedialog --error "'zenity' is unsupported. 'zenity' has no support for yes/no or yes/no/cancel dialogs. It has no capacity to add a button and change labels, allowing any sort of usable yes/no/cancel system. zenity does not let me repurpose ok/cancel dialogs. The return code for cancel is the same as abort, so I can't even make do with changing the behavior there.\n\nThe dialogs will be confusing and missing features.\n\nThis is why it is unsupported.\n\nPlease install kdialog, Xdialog, or gtkdialog or run it from a terminal.\n\nReally, even the automatic mode works better than zenity mode." fi fi if [ $DIALOGS = 0 ] ; then if [ ! -z "$1" ] ; then echo "none: okay (X dialogs unavailable)" echo "------------ NOTICE regarding automatic mode -------------" echo "Will be fully automatic unless run from a terminal." echo "----------------------------------------------------------" echo "Please consider installing kdialog, Xdialog, or gtkdialog." echo fi fi if tty -s ; then if [ ! -z "$1" ] ; then echo "TTY dialog services:" fi fi if [ -x "$DIALOG" ] ; then if [ ! -z "$1" ] ; then echo "dialog: good" fi fi if [ -x "$WHIPTAIL" ] ; then if [ ! -z "$1" ] ; then echo "whiptail: okay (not as pretty as dialog)" echo "--------- NOTICE regarding the use of whiptail ---------" echo "The line-wrapping in message dialogs is fairly crude." echo "This results in the message dialogs being uglier than" echo "dialog." echo "--------------------------------------------------------" echo "For a prettier display, install dialog." echo fi fi if [ -x "$TCDIALOG" ] ; then if [ ! -z "$1" ] ; then echo "tcdialog: BAD" echo "---------- WARNING regarding the use of tcdialog ----------" echo "tcdialog requires the row and column sizes be specified, so" echo "it produces some awful looking dialogs as we do not " echo "precompute these sizes." echo echo "We've also seen cases of the cursor being missing in the" echo "input dialog with tcdialog." echo "-----------------------------------------------------------" echo "Please consider installing dialog." echo fi fi if [ ! -z "$1" ] ; then echo "bash: okay (not as pretty as dialog)" fi } askquestions() { local MY_URL="$1" local MY_GROUP="${2:-$EMPTY_GROUP}" local MY_TITLE="$3" CUR_ABORT=0 fake_title "$MY_URL" MY_TITLE="${MY_TITLE:-${FAKE_TITLE:-$EMPTY_TITLE}}" somedialog --input "Title for $MY_URL" "$MY_TITLE" if [ $? -eq 1 ] ; then CUR_ABORT=1 ; return 1 ; fi CUR_TITLE="${SOMEDIALOG:-${FAKE_TITLE:-$EMPTY_TITLE}}" somedialog --input "Genre or group for $CUR_TITLE" "$MY_GROUP" if [ $? -eq 1 ] ; then CUR_ABORT=1 ; return 1 ; fi CUR_GROUP="${SOMEDIALOG:-$EMPTY_GROUP}" somedialog --yesNOcancel "Activate <$MY_URL> in $CUR_GROUP as '$CUR_TITLE'?" CUR_ACTIVATE="$?" if [ $CUR_ACTIVATE -eq 2 ] ; then CUR_ABORT=1 ; return 1 ; fi if [ -z "$CUR_GROUP" ] ; then CUR_GROUP="$EMPTY_GROUP" fi if [ -z "$CUR_TITLE" ] ; then CUR_TITLE="${FAKE_TITLE:-$EMPTY_TITLE}" fi # Because of the backticks, the \\ needs to be escaped to \\\\ CUR_GROUP=`sed -e 'y; /\\\\;___;' <<< "$CUR_GROUP"` CUR_TITLE=`sed -e 'y;/\\\\;__;' <<< "$CUR_TITLE"` return $CUR_ABORT } somedialog() { local TYPE="$1" shift local RET=0 if [ -x "$KDIALOG" ] ; then if [ "$TYPE" = "--yesNOcancel" ] ; then TYPE="--yesnocancel" ; fi if [ "$TYPE" = "--error" ] ; then "$KDIALOG" --error "$*" elif [ "$TYPE" = "--sorry" ] ; then "$KDIALOG" --sorry "$*" elif [ "$TYPE" = "--message" ] ; then "$KDIALOG" --msgbox "$*" elif [ "$TYPE" = "--yesno" ] ; then "$KDIALOG" --yesno "$*" RET="$?" elif [ "$TYPE" = "--yesnocancel" ] ; then "$KDIALOG" --yesnocancel "$*" RET="$?" elif [ "$TYPE" = "--input" ] ; then SOMEDIALOG=`"$KDIALOG" --inputbox "$1" "$2"` RET="$?" fi elif [ -x "$XDIALOG" ] ; then if [ "$TYPE" = "--yesNOcancel" ] ; then TYPE="--yesnocancel" ; fi if [ "$TYPE" = "--error" ] ; then "$XDIALOG" --stdout --msgbox "Error: $*" 0 0 elif [ "$TYPE" = "--sorry" ] ; then "$XDIALOG" --stdout --msgbox "Sorry: $*" 0 0 elif [ "$TYPE" = "--message" ] ; then "$XDIALOG" --stdout --msgbox "$*" 0 0 elif [ "$TYPE" = "--yesno" ] ; then "$XDIALOG" --stdout --yesno "$*" 0 0 RET="$?" if [ $RET -eq 255 ] ; then RET=1 ; fi elif [ "$TYPE" = "--yesnocancel" ] ; then "$XDIALOG" --stdout --yesno "$*" 0 0 RET="$?" if [ $RET -eq 255 ] ; then RET=2 ; fi elif [ "$TYPE" = "--input" ] ; then SOMEDIALOG=`"$XDIALOG" --stdout --inputbox "$1" 0 0 "$2"` RET="$?" if [ $RET -eq 255 ] ; then RET=1 ; fi fi elif [ -x "$GTKDIALOG" ] ; then if [ "$TYPE" = "--yesNOcancel" ] ; then TYPE="--yesnocancel" ; fi if [ "$TYPE" = "--error" ] ; then GTKMSG=`echo "$*" | sed -e 's//]/g'` export GTK_DIALOG=" " "$GTKDIALOG" --program GTK_DIALOG | grep -q 'EXIT="OK"' elif [ "$TYPE" = "--sorry" ] ; then GTKMSG=`echo "$*" | sed -e 's//]/g'` export GTK_DIALOG=" " "$GTKDIALOG" --program GTK_DIALOG | grep -q 'EXIT="OK"' elif [ "$TYPE" = "--message" ] ; then GTKMSG=`echo "$*" | sed -e 's//]/g'` export GTK_DIALOG=" " "$GTKDIALOG" --program GTK_DIALOG | grep -q 'EXIT="OK"' elif [ "$TYPE" = "--yesno" ] ; then GTKMSG=`echo "$*" | sed -e 's//]/g'` export GTK_DIALOG=" " GTK_EXIT=`"$GTKDIALOG" --program GTK_DIALOG | sed -n -e 's/^EXIT="\([^"]*\)"/\1/p'` case "$GTK_EXIT" in Yes) RET=0 ;; No) RET=1 ;; *) RET=1 ;; esac elif [ "$TYPE" = "--yesnocancel" ] ; then GTKMSG=`echo "$*" | sed -e 's//]/g'` export GTK_DIALOG=" " GTK_EXIT=`"$GTKDIALOG" --program GTK_DIALOG | sed -n -e 's/^EXIT="\([^"]*\)"/\1/p'` case "$GTK_EXIT" in Yes) RET=0 ;; No) RET=1 ;; *) RET=2 ;; esac elif [ "$TYPE" = "--input" ] ; then GTKMSG=`echo "$1" | sed -e 's//]/g'` GTKDEFAULT=`echo "$2" | sed -e 's//]/g'` # This isn't what is done in gtkdialog examples, however gtkdialog # does *no escaping* of the input, so you can provide input such # as '" ; rm /etc/passwd ; true "' and what you get back is: # SOMEDIALOG="" ; rm /etc/passwd ; true "" export GTK_DIALOG=" $GTKDEFAULTSOMEDIALOG " GTK_LINES=`"$GTKDIALOG" --program GTK_DIALOG` SOMEDIALOG=`sed -n -e 's/^SOMEDIALOG="\(.*\)"$/\1/p' <<< "$GTK_LINES"` GTK_EXIT=`sed -n -e 's/^EXIT="\([^"]*\)"/\1/p' <<< "$GTK_LINES"` case "$GTK_EXIT" in OK) RET=0 ;; *) SOMEDIALOG="" ; RET=1 ;; esac fi elif [ -x "$ZENITY" ] ; then # There is *NO CANCEL* option, so it is depricated # It also has no Yes/No, so the yesno questions come out confusing if [ "$TYPE" = "--yesNOcancel" ] ; then TYPE="--yesnocancel" ; fi if [ "$TYPE" = "--error" ] ; then GTKMSG=`echo "$*" | sed -e 's//]/g'` "$ZENITY" --error --text="$GTKMSG" elif [ "$TYPE" = "--sorry" ] ; then GTKMSG=`echo "$*" | sed -e 's//]/g'` "$ZENITY" --warning --text="$GTKMSG" elif [ "$TYPE" = "--message" ] ; then GTKMSG=`echo "$*" | sed -e 's//]/g'` "$ZENITY" --info --text="$GTKMSG" elif [ "$TYPE" = "--yesno" ] ; then GTKMSG=`echo "$*" | sed -e 's//]/g'` "$ZENITY" --question --text="$GTKMSG" RET="$?" elif [ "$TYPE" = "--yesnocancel" ] ; then GTKMSG=`echo "$*" | sed -e 's//]/g'` "$ZENITY" --question --text="$GTKMSG" RET="$?" elif [ "$TYPE" = "--input" ] ; then GTKMSG=`echo "$1" | sed -e 's//]/g'` GTKDEFAULT=`echo "$2" | sed -e 's//]/g'` SOMEDIALOG=`"$ZENITY" --entry --text="$GTKMSG" --entry-text="$GTKDEFAULT"` RET="$?" fi elif tty -s && [ $AUTOMODE = 0 ] ; then TTY_ABORT=0 # trap TTY_ABORT=1 SIGINT if [ -x "$DIALOG" ] ; then if [ "$TYPE" = "--yesNOcancel" ] ; then TYPE="--yesnocancel" ; fi if [ "$TYPE" = "--error" ] ; then "$DIALOG" --msgbox "Error: $*" 0 0 elif [ "$TYPE" = "--sorry " ] ; then "$DIALOG" --msgbox "Sorry: $*" 0 0 elif [ "$TYPE" = "--message" ] ; then "$DIALOG" --msgbox "$*" 0 0 elif [ "$TYPE" = "--yesno" ] ; then "$DIALOG" --yesno "$*" 0 0 if [ $RET -eq 255 ] ; then RET=1 ; fi elif [ "$TYPE" = "--yesnocancel" ] ; then "$DIALOG" --extra-button --ok-label Yes --extra-label No --no-label Cancel --yesno "$*" 0 0 if [ $RET -eq 1 ] ; then RET=2 elif [ $RET -eq 2 ] ; then RET=1 elif [ $RET -eq 255 ] ; then RET=2 fi elif [ "$TYPE" = "--input" ] ; then "$DIALOG" --inputbox "$1" 0 0 "$2" 2>"$TMPFILE" RET="$?" SOMEDIALOG=`cat "$TMPFILE"` if [ $RET -eq 255 ] ; then RET=1 ; fi fi elif [ -x "$WHIPTAIL" ] ; then # whiptail doesn't handle 0 0 as nicely as dialog, so it # isn't preferred. if [ "$TYPE" = "--yesNOcancel" ] ; then TYPE="--yesnocancel" ; fi if [ "$TYPE" = "--error" ] ; then "$WHIPTAIL" --msgbox "Error: $*" 0 0 elif [ "$TYPE" = "--sorry " ] ; then "$WHIPTAIL" --msgbox "Sorry: $*" 0 0 elif [ "$TYPE" = "--message" ] ; then "$WHIPTAIL" --msgbox "$*" 0 0 elif [ "$TYPE" = "--yesno" ] ; then "$WHIPTAIL" --yesno "$*" 0 0 RET="$?" if [ $RET -eq 255 ] ; then RET=1 ; fi elif [ "$TYPE" = "--yesnocancel" ] ; then "$WHIPTAIL" --yesno "$* (ESC to cancel)" 0 0 RET="$?" if [ $RET -eq 255 ] ; then RET=2 ; fi elif [ "$TYPE" = "--input" ] ; then "$WHIPTAIL" --inputbox "$1" 0 0 "$2" 2>"$TMPFILE" RET="$?" SOMEDIALOG=`cat "$TMPFILE"` if [ $RET -eq 255 ] ; then RET=1 ; fi fi elif [ -x "$TCDIALOG" ] ; then # tcdialog requires the height/width values # as such, it is depricated if [ "$TYPE" = "--yesNOcancel" ] ; then TYPE="--yesnocancel" ; fi if [ "$TYPE" = "--error" ] ; then "$TCDIALOG" --msgbox "Error: $*" 20 60 elif [ "$TYPE" = "--sorry " ] ; then "$TCDIALOG" --msgbox "Sorry: $*" 20 60 elif [ "$TYPE" = "--message" ] ; then "$TCDIALOG" --msgbox "$*" 20 60 elif [ "$TYPE" = "--yesno" ] ; then "$TCDIALOG" --yesno "$*" 20 60 RET="$?" if [ $RET -eq 255 ] ; then RET=1 ; fi elif [ "$TYPE" = "--yesnocancel" ] ; then "$TCDIALOG" --yesno "$* (ESC to cancel)" 20 60 RET="$?" if [ $RET -eq 255 ] ; then RET=2 ; fi elif [ "$TYPE" = "--input" ] ; then "$TCDIALOG" --inputbox "$1" 20 60 "$2" 2>"$TMPFILE" RET="$?" SOMEDIALOG=`cat "$TMPFILE"` if [ $RET -eq 255 ] ; then RET=1 ; fi fi else if [ "$TYPE" = "--error" ] ; then echo "Error: $*" read -p "Press to continue." RET RET=0 if [ $TTY_ABORT -eq 1 ] ; then RET=1 ; fi elif [ "$TYPE" = "--sorry " ] ; then echo "Sorry: $*" read -p "Press to continue." RET RET=0 if [ $TTY_ABORT -eq 1 ] ; then RET=1 ; fi elif [ "$TYPE" = "--message" ] ; then echo "Note: $*" read -p "Press to continue." RET RET=0 if [ $TTY_ABORT -eq 1 ] ; then RET=1 ; fi elif [ "$TYPE" = "--yesno" ] ; then read -r -e -p "$* (y/N)?" RET case "$RET" in [yY]|[yY][eE][sS]) RET=0 ;; *) RET=1 ;; esac if [ $TTY_ABORT -eq 1 ] ; then RET=1 ; fi elif [ "$TYPE" = "--yesNOcancel" ] ; then read -r -e -p "$* (yes/NO/cancel):" RET if [ -z "$RET" ] ; then RET="n" ; fi case "$RET" in [yY]|[yY][eE][sS]) RET=0 ;; [nN]|[nN][oO]) RET=1 ;; *) RET=2 ;; esac if [ $TTY_ABORT -eq 1 ] ; then RET=2 ; fi elif [ "$TYPE" = "--yesnocancel" ] ; then read -r -e -p "$* (yes/no/CANCEL):" RET case "$RET" in [yY]|[yY][eE][sS]) RET=0 ;; [nN]|[nN][oO]) RET=1 ;; *) RET=2 ;; esac if [ $TTY_ABORT -eq 1 ] ; then RET=2 ; fi elif [ "$TYPE" = "--input" ] ; then read -r -e -p "$1 ($2): " SOMEDIALOG if [ -z "$SOMEDIALOG" ] ; then SOMEDIALOG="$2" fi RET=$TTY_ABORT fi fi # trap - SIGINT else RET=0 if [ "$TYPE" = "--error" ] ; then echo "Error: $*" elif [ "$TYPE" = "--sorry " ] ; then echo "Sorry: $*" elif [ "$TYPE" = "--message" ] ; then echo "Message: $*" elif [ "$TYPE" = "--yesno" ] ; then echo "Yes/No: $* (defaults to NO)" RET=1 elif [ "$TYPE" = "--yesNOcancel" ] ; then echo "Yes/No/Cancel: $* (defaults to NO)" RET=1 elif [ "$TYPE" = "--yesnocancel" ] ; then echo "Yes/No/Cancel: $* (defaults to CANCEL)" RET=2 elif [ "$TYPE" = "--input" ] ; then echo "Input: $1 (defaults to $2)" SOMEDIALOG="$2" fi fi return $RET } showversion() { # Version history: # 0.1 - (yam655) first version # 0.1.1 (2009-05-08): # fix issue with feed://somewhere/... # genre/title dialog order swap (as seen in 0.2 line) # 0.2 - (mostly aborted) added ability to run podget afterwards # 0.3 - added inactivity tag support echo "podget-add-feed Version 0.3" } showhelp() { cat << FEOF Usage: $0 {Options} Options may include: -h | --help : show this help message -V | --version : show the version -c | --check : check dialog compatibility -a | --auto : run in non-interactive mode -d | --dialog : use a specific dialog program FEOF } checkdialog TMPFILE=`tempfile -p paf.` if [ -z "$TMPFILE" ] ; then TMPFILE=`mktemp -t addfeed.XXXXXXXXXX` ; fi if [ -z "$TMPFILE" ] ; then TMPFILE="/tmp/podget-add-feed.$$.`hostname`" ; fi while [ $# -gt 0 ] ; do case "$1" in --check|-c) dialogcheck -v exit 0 ;; --help|-h) showhelp exit 0 ;; --version|-V) showversion exit 0 ;; --auto|-a) unset DISPLAY DIALOG="auto" shift continue ;; --dialog|-d) shift checkdialog "$1" dialogcheck shift continue ;; esac URL="$1" shift CUR_GROUP="$EMPTY_GROUP" CUR_TITLE="" CUR_ACTIVATE=0 # Interestingly, the body of the single-quoted segments is getting # parsed. URL=`sed -e 's@^feed://@http://@' -e 's@^feed:\([a-z]\+:\)@\1@' \ -e 's/ /%20/g' -e 's/|/%7C/g' -e 's/\\\\/%5C/g' \ -e 's/\\$/%24/g' -e 's/"/%34/g' -e "s/'/%39/g" \ <<< "$URL"` case "$URL" in http:*) ;; https:*) ;; *) somedialog --error "Unknown protocol in URL: $URL" continue ;; esac FOUNDURL=`grep -e "^\(#[a-zA-Z0-9]* \)\?$URL " "$SERVERLIST"` FURL="" FGROUP="" FTITLE="" FACTIVE="inactive" FREASON="" if [ ! -z "$FOUNDURL" ] ; then case "$FOUNDURL" in \#*) read FACTIVE FURL FGROUP FTITLE <<< "$FOUNDURL" if [ "$FACTIVE" != "#" ] ; then FREASON=" (${FACTIVE:1})" fi FACTIVE="inactive" ;; *) read FURL FGROUP FTITLE <<< "$FOUNDURL" FACTIVE="active" ;; esac if ! somedialog --yesnocancel "Found <$FURL> in $FGROUP as \"$FTITLE\". It is $FACTIVE$FREASON. Change this?" ; then continue fi fi if ! askquestions "$URL" "$FGROUP" "$FTITLE" ; then continue fi ACTIVE="" if [ $CUR_ACTIVATE -ne 0 ] ; then ACTIVE='# ' fi if [ -z "$FOUNDURL" ] ; then echo "$ACTIVE$URL $CUR_GROUP $CUR_TITLE" >> "$SERVERLIST" else OLIBDIR="$LIBRARY_DIR/$FGROUP/$FTITLE" NLIBDIR="$LIBRARY_DIR/$CUR_GROUP/$CUR_TITLE" sed -e 's|^\(# \)\?'"$URL"' .*$|'"$ACTIVE$URL $CUR_GROUP $CUR_TITLE"'|g' "$SERVERLIST" > "$SERVERLIST".new mv "$SERVERLIST" "$SERVERLIST~" mv "$SERVERLIST".new "$SERVERLIST" if [ -d "$OLIBDIR" ] ; then if [ ! -d "$LIBRARY_DIR/$CUR_GROUP" ] ; then mkdir -p "$LIBRARY_DIR/$CUR_GROUP" fi if [ -d "$NLIBDIR" ] ; then somedialog --sorry "Directory $NLIBDIR already existed. We could not move $OLIBDIR. You will need to move the files by hand." else mv -f "$OLIBDIR" "$NLIBDIR" fi fi # if [ -z "$ACTIVE" -a "$FACTIVE" != "active" ] ; then # if somedialog --yesno "Run podget now?" ; then # podget # fi # fi fi done ./podget-0.9.3/SCRIPTS/PRE_0.7.0/podget-add-feed.README0000644000175000017500000000725412620220525020643 0ustar lowkeylowkeyFrom Steven Black's posting: This is a simple shell script I created to easily add RSS feeds from within Firefox, etc. The user-interface is handled through third-party tools, falling back to BASH built-ins if on a TTY and sane defaults if not on a TTY. If you run the script from an xterm with --check it will check for the available interfaces and print the details. Usable X-based dialog tools include: kdialog, gtkdialog, Xdialog. Xdialog won't look as good as either kdialog or gtkdialog. ("zenity" is not supported as it is missing core features required.) Usable TTY-based dialog tools include: dialog (preferred), whiptail (not as good-looking or full-functioning as dialog), BASH is also well-supported. As mentioned previously, there are sane defaults and with no available user-interface it works in "automated" mode. (--help produces arguments.) For the title, it uses 'curl' to do a quick-download of the feed to try to get the proper title. (It usually works.) If it can't be determined from the feed, it tries to mangle the URL to get something of a usable default. If even that fails, it falls back to "Unknown". It supports the genre/group, but at this time it doesn't present a list of the currently in-use options. It is on my list of things to add, but I've been so lax on adding support for it that I'm making it available before I add it. It also supports both active and inactive feeds. These are handled by commenting out the feed in the ~/.podget/serverlist file. If running in automated mode, the feeds are inactive with an "Unknown" genre/group. (If it is automated, it is expected that you will need to manually tweak the title and group/genre.) If you add an existing feed, you can edit the title, genre/group, and activate/deactivate the feed. If you change the title or genre/group of a feed, it will automatically move the underlying directory if possible. (It tries to move the directory, if the directory already exists, it does nothing.) Additionally, it supports "inactivity tags". These are simple one-word reasons for why a feed is deactivated. These are handled by adding the word directly to the comment character in the serverlist. I use it to mark when a podiobook is completely downloaded ("DONE"), when I've finished reading a completed podiobook ("READ"), and when I've abandoned a feed ("QUIT"). So I have things like the following in my serverlist: #### Scott Sigler http://feeds.feedburner.com/scottsigler Horror Scott Sigler #READ http://www.podiobooks.com/title/infected/feed Horror Scott Sigler, Infected There's no user-interface to set the inactivity tag, (it needs to be set directly in the serverlist file) but if you reference the URL again it will present it to you. If I run podget-add-feed with the first URL, it will present a dialog like: Found in Horror as "Scott Sigler". It is active. Change this? For the second URL, it would say: Found in Horror as "Scott Sigler, Infected". It is inactive (READ). Change this? If the inactivity tag were not defined, the whole " (...)" section would be missing, as you would expect. Oh, and when editing an existing feed it does not move the feed within the file. It's an in-place sed-based operation. This means that if I ran the command on either of the above-mentioned URLs, they would remain in the "Scott Sigler" section of my serverlist. It's a human-readable file, so it is expected that it would have human-centered structure. I have Firefox set to display the feed within the browser. I can then choose to add it as a text feed using some other software, or I can select "podget-add-feed" and it will allow me to set it up for podget. ./podget-0.9.3/SCRIPTS/v0.8.3/0000755000175000017500000000000013344003100014517 5ustar lowkeylowkey./podget-0.9.3/SCRIPTS/v0.8.3/podget_update_playlists0000755000175000017500000004165013065252272021423 0ustar lowkeylowkey#!/usr/bin/env bash # Useful to follow command execution and determine when an extra echo outputs during silent mode. #set -x # ---------------------------------------------------------------------------------------------------------------------------------- # Filename: podget_update_playlists {{{ # Maintainer: Dave Vehrs # Copyright: (c) 2016 Dave Vehrs # # This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public # License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any # later version. # # This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied # warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # ---------------------------------------------------------------------------------------------------------------------------------- # Exit Codes {{{ # "Reserved" Exit codes # 1 General Error # 2 Misuse of shell built-ins # 126 Command invoked cannot execute # 127 Command not found # 128+n Invalid argument to exit # 130 Script terminated by Control-C (128+2) # 143 Script terminated by TERM signal (128+15) # "Our" Exit codes # Display Help (set to '0' because it is an valid exit condition, not an error.) ERR_DISPLAYHELP=0 # Library directory not defined. ERR_LIBNOTDEF=50 # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Traps {{{ # FUNCNAME is declared with a default value in case the trap is triggered while # outside a function. trap 'EXIT_ERROR ${LINENO} $? ${FUNCNAME:-"Unconfigured"}' ERR # trap to run CLEANUP function if program receives a TERM (kill) or INT (ctrl-c) signal # - CLEANUP called in line for other normal exits. trap 'CLEANUP_AND_EXIT 143' TERM trap 'CLEANUP_AND_EXIT 130' INT # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Set Shell Options to catch errors ('man bash' for details) {{{ set -o errexit set -o nounset set -o pipefail # Enable extended glob matches. shopt -s extglob # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Help text and default file formats {{{ : << HELP_STEXT -c --config Name of configuration file to use. -d --dir_config Directory that configuration files are --dir_session Directory that session files are stored in. -f --force Force download of items from each feed even if they have already been downloaded. -l --library Directory to store downloaded files in. -s --silent Run silently (for cron jobs). --verbosity Set verbosity level (0-4). -v Set verbosity to level 1. -vv Set verbosity to level 2. -vvv Set verbosity to level 3. -vvvv Set verbosity to level 4. -h --help Display help. HELP_STEXT # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Defaults {{{ ######################################################################################################################### ## Do not configure here. Run podget once to install default user configuration files ($HOME/.podget) and edit there. ## ######################################################################################################################### # Set DIR_LIBRARY, DIR_SESSION, and DIR_LOG in config file. DIR_CONFIG="${HOME}/.podget" CONFIG_CORE="podgetrc" # Default VERBOSITY # 0 == silent # 1 == Warning messages only. # 2 == Progress and Warning messages. # 3 == Debug, Progress and Warning messages. # 4 == All messages and wget set to maximum VERBOSITY. VERBOSITY=2 # Force # 0 == Only download new material. # 1 == Force download all items even those you've downloaded before. FORCE=0 # Date format for new playlist names DATE_FORMAT=+%F # ASX Playlists for Windows Media Player # 0 == do not create # 1 == create ASX_PLAYLIST=0 # Default DEBUG Disabled (Deletion of temporary files allowed) DEBUG=0 # Default DEBUG string leader DEBUG_LEADER="DEBUG --" ######################################################################################################################### ## Do not configure here. Run podget once to install default user configuration files ($HOME/.podget) and edit there. ## ######################################################################################################################### # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Functions {{{ # Function: CLEANUP_AND_EXIT # Closes session and removes lock file (if it exists) # ARGUMENTS: # ${1} == Exit Status to report. CLEANUP_AND_EXIT() { local EXITSTATUS=${1} exit "${EXITSTATUS}" } display_shelp() { echo; echo "Usage $0 [options]"; echo sed --silent -e '/HELP_STEXT$/,/^HELP_STEXT/p' "$0" | sed -e '/HELP_STEXT/d' } EXIT_ERROR() { # Name of script JOB_NAME=$(basename "$0") # The following three variables are configured with a default value in case # the function is called without options set. LINENUM="${1:-"Unconfigured"}" # Line with error EXITSTATUS="${2:-"Unconfigured"}" # exit status of error FUNCTION="${3:-"Unconfigured"}" # If error occurred in a function, its name will be listed. echo echo "Error:" echo -e "\tScript:\t\t${JOB_NAME}" # Function line only appears if it has been set to value other than the # default. Works on the assumption that "Unconfigured" is not likely to be # chosen as a function name. if [[ ${FUNCTION} != "Unconfigured" ]]; then echo -e "\tFunction:\t${FUNCTION}" fi echo -e "\tAt line:\t${LINENUM}" echo -e "\tExit Status:\t${EXITSTATUS}" echo CLEANUP_AND_EXIT 1 } PLAYLIST_ConvertToASX() { local DIR_LIBRARY=${1} local M3U_PLAYLISTNAME=${2} ASX_LOCATION="\\SD Card\\POD\\" ASX_PLAYLISTNAME=$(basename "${DIR_LIBRARY}"/"${M3U_PLAYLISTNAME}" .m3u).asx sed --silent -e '/TEXT_ASX_BEGINNING$/,/^TEXT_ASX_BEGINNING/p' "$0" | sed -e '/TEXT_ASX_BEGINNING/d' > "${DIR_LIBRARY}"/"${ASX_PLAYLISTNAME}" while read -r line ; do # local FIXED_ENTRY=$(echo "${line}" | sed 's/\//\\/g') local FIXED_ENTRY="${line//\//\\}" { echo ' ' echo " " echo " " echo ' ' } >> "${DIR_LIBRARY}"/"${ASX_PLAYLISTNAME}" done < "${DIR_LIBRARY}"/"${M3U_PLAYLISTNAME}" sed --silent -e '/TEXT_ASX_END$/,/^TEXT_ASX_END/p' "$0" | sed -e '/TEXT_ASX_END/d' >> "${DIR_LIBRARY}"/"${ASX_PLAYLISTNAME}" # Removing unix2dos dependency. Converting to sed statement with in-place editing of the file in question. # ctrl-v ctrl-m for windows line end. sed -i 's/$/ /' "${DIR_LIBRARY}"/"${ASX_PLAYLISTNAME}" } # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Version (Update with changes!) {{{ VERSION=0.1 REPORT_VERSION=0 # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Parse command line {{{ # Set defaults for command line options so variables do not conflict with 'set -o nounset' by being undeclared. CMDL_FORCE=0 # Parameter Substitution will cause script to exit with an error when 'errexit' is enabled. Disable while processing the command # line. set +o errexit # Also disable trap for reporting errors on non-zero exit statuses. trap - ERR while (( $# >= 1 )); do case ${1} in -c | --config ) CONFIG_CORE=${2:-NONE} ; shift ; shift ;; -d | --dir_config ) DIR_CONFIG=${2:-NONE} ; shift ; shift ;; --dir_session ) CMDL_SESSION=${2:-NONE} ; shift ; shift ;; -f | --force ) CMDL_FORCE=1 ; shift ;; -l | --library ) CMDL_LIBRARY=${2:-NONE} ; shift ; shift ;; -s | --silent ) VERBOSITY=0 ; shift ;; -V | --version ) VERBOSITY=2 ; REPORT_VERSION=1 ; shift ;; -v ) VERBOSITY=1 ; shift ;; -vv ) VERBOSITY=2 ; shift ;; -vvv ) VERBOSITY=3 ; shift ;; -vvvv ) VERBOSITY=4 ; shift ;; --verbosity ) VERBOSITY=${2:-NONE} ; shift ; shift ;; * ) display_shelp ; CLEANUP_AND_EXIT ${ERR_DISPLAYHELP} ;; esac done # Re-enable errexit set -o errexit # Re-enable trap trap 'EXIT_ERROR ${LINENO} $? ${FUNCNAME:-"Unconfigured"}' ERR if [[ -n ${VERBOSITY+set} ]] ; then if [[ -z ${VERBOSITY##*[!0-9]*} ]]; then echo "Verbosity is not a supported integer value" exit 1 fi fi if (( VERBOSITY >= 2 )) ; then echo "podget_update_playlists" fi if (( REPORT_VERSION == 1 )); then echo "Version: ${VERSION}" CLEANUP_AND_EXIT 0 fi if (( VERBOSITY >= 2 )) ; then echo fi if [[ ${CONFIG_CORE} == "NONE" ]]; then echo "Unset filename for configuration" CLEANUP_AND_EXIT 1 fi if [[ ${DIR_CONFIG} == "NONE" ]]; then echo "Unset directory to store configuration" CLEANUP_AND_EXIT 1 fi # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Test for existing configuration directory, if missing install it. {{{ if [[ ! -d ${DIR_CONFIG} ]] ; then echo " Configuration directory not found." CLEANUP_AND_EXIT 1 fi if [[ ! -f "${DIR_CONFIG}/${CONFIG_CORE}" ]] ; then echo "Configuration file does not exist." CLEANUP_AND_EXIT 1 fi # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Configuration {{{ # SHELLCHECK SC1090 # shellcheck source=/dev/null source "${DIR_CONFIG}"/"${CONFIG_CORE}" if (( DEBUG == 1 )) ; then echo echo "${DEBUG_LEADER} Parsing Config file." echo -e "${DEBUG_LEADER} Config directory:\t\t${DIR_CONFIG}" echo -e "${DEBUG_LEADER} Config file:\t\t\t${CONFIG_CORE}" echo fi # for testing #echo "Verbosity: ${VERBOSITY}" #echo "Debug: ${DEBUG}" #CLEANUP_AND_EXIT 0 if [[ -n ${CMDL_LIBRARY+set} ]] ; then if [[ ${CMDL_LIBRARY} == "NONE" ]]; then echo "Unset directory to store podcast library" CLEANUP_AND_EXIT 1 fi if (( VERBOSITY >= 3 )) ; then echo -e "\t\tOverriding Library Directory specified in configuration file." fi DIR_LIBRARY=${CMDL_LIBRARY} fi if (( VERBOSITY >= 3 )) ; then echo -e "Library Directory:\t\t${DIR_LIBRARY}" fi if [[ -z ${DIR_LIBRARY} ]] ; then echo "ERROR - Library directory not defined." 1>&2 CLEANUP_AND_EXIT ${ERR_LIBNOTDEF} fi if [[ -n ${CMDL_ASX+set} ]]; then ASX_PLAYLIST=${CMDL_ASX} fi if (( CMDL_FORCE != 0 )); then FORCE=${CMDL_FORCE} fi if (( VERBOSITY >= 3 )) ; then if (( DEBUG == 0 )); then echo -e "Debug:\t\t\t\tDisabled - Reduced progress messages." else echo -e "Debug:\t\t\t\tEnabled - Increased progress messages." fi fi if (( VERBOSITY >= 1 )) ; then echo fi # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Loop over files in Library to create playlists {{{ CATEGORIES=$(ls -d ${DIR_LIBRARY}/*/ | awk -F'/' '{print $(NF-1)}') while read -r CAT; do if (( VERBOSITY >= 1 )); then echo "CATEGORY: ${CAT}" fi # Test to determine if Category directory contains any Feed directories. if find "${DIR_LIBRARY}/${CAT}" -maxdepth 0 ! -empty -type d | read ;then FEEDS=$(ls -d "${DIR_LIBRARY}/${CAT}"/*/ | awk -F'/' '{print $(NF-1)}') while read -r FEED; do if (( VERBOSITY >= 2 )); then echo " FEED: ${FEED}" fi PLAYLIST_NAME="PLAYLIST_${FEED//[[:space:]]/_}.m3u" if (( VERBOSITY >= 1 )); then echo " PLAYLIST NAME: ${PLAYLIST_NAME}" fi if [ -f "${DIR_LIBRARY}/${PLAYLIST_NAME}" ]; then rm -f "${DIR_LIBRARY}/${PLAYLIST_NAME}" fi touch "${DIR_LIBRARY}/${PLAYLIST_NAME}" ITEMS=$(ls -1tr "${DIR_LIBRARY}/${CAT}/${FEED}/") while read -r ITEM; do if (( VERBOSITY >= 3 )); then echo " ITEM: ${ITEM}" fi echo "${CAT}/${FEED}/${ITEM}" >> "${DIR_LIBRARY}/${PLAYLIST_NAME}" done <<< "${ITEMS}" done <<< "${FEEDS}" else echo " NO FEEDS" fi done <<< "${CATEGORIES}" # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Close session with '0' status and clean up: {{{ # Disable extended glob matches. shopt -u extglob CLEANUP_AND_EXIT 0 # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # vim:tw=132:ts=4:sw=4 ./podget-0.9.3/SCRIPTS/v0.8.3/podget_update_playlists.README0000644000175000017500000000021513116061175022341 0ustar lowkeylowkeyThis script is not needed for versions of Podget after 0.8.4 as it was integrated to the release version. Kept here for archival purposes. ./podget-0.9.3/SCRIPTS/v0.8.5/0000755000175000017500000000000013543442076014545 5ustar lowkeylowkey./podget-0.9.3/SCRIPTS/v0.8.5/pod2player.sh0000755000175000017500000000345613543442076017175 0ustar lowkeylowkey#!/bin/bash # pod2player.sh : Move podget podcasts to a mobile player # Copyright 2017, Bruce Ingalls. See COPYING for GPL 3.0 license # WARNING! You can lose files, if you do not read, understand & edit this script! # # This script effectively does `mv ~/pod/* dest/`, while dealing with similar # subdirectory names shared between source & destination directories # # For this to work, you must first mount the filesystem of your media player device # (e.g. Android). Finally, you should have the same directory structure in the podcast # destination directory, as the ~/pod/ source podcatcher directory on your laptop # Currently, this script handles 2 levels of nesting: podcast directories in category # directories, with a default prefix of AUDIO_ #echo "Read instructions, and disable this line, before running this script"; exit # The removable media mount point #DEVICE=/Volumes/MyPlayerDevice/Podcasts # Mac uses /Volumes, Linux uses /media or /mnt #DEVICE="/Volumes/BLU\\ 1/Podcasts" DEVICE="/Volumes/BLU/Podcasts" #DEVICE="/Volumes/BLU\\ 2/Podcasts" # Default Podget Library storage directory DIR_LIBRARY=~/POD CATEGORY_PREFIX='AUDIO_' # Prefix for all your podcast categories, to filter, say, videos # End of user edited section. You should not need to modify below pushd "${DIR_LIBRARY}" # default podget storage directory for CAT in ${CATEGORY_PREFIX}*; do pushd "${DIR_LIBRARY}/${CAT}" for CHAN in *; do DEST="${DEVICE}/${CAT}/${CHAN}/" # j=$(echo ${i} | sed 's/\([ ()]\)/\\\1/g') # escape spaces, parens. Other weird podcast chars? eval mkdir -p "${DEST}" eval mv "${CHAN}/*" ${DEST} rmdir ${CHAN} # clean up podcast directory (channel) echo "moved directory ${CHAN} to ${DEST}" done popd eval rmdir "${DIR_LIBRARY}/${CAT}" # clean up category name done ./podget-0.9.3/SCRIPTS/v0.8.5/speedup.sh0000755000175000017500000000033413543442076016551 0ustar lowkeylowkey#!/bin/zsh # faster mp3 playback https://superuser.com/questions/519649/tool-to-bulk-speed-up-convert-an-audio-file for i in $*;do ffmpeg -i $i -filter:a "atempo=1.6" -c:a libmp3lame -q:a 4 -vsync 2 ~/speedup/$i done ./podget-0.9.3/SCRIPTS/v0.8.8/0000755000175000017500000000000014125415341014540 5ustar lowkeylowkey./podget-0.9.3/SCRIPTS/v0.8.8/push-new-mp3.README0000644000175000017500000000137114125415341017664 0ustar lowkeylowkeyA python3 script to list the last 48h(or what you choose) podcasts in Pushover Android app(also available for IoS and Desktops). Then can easily see what new podcasts have been downloaded. https://pushover.net/ Android : https://play.google.com/store/apps/details?id=net.superblock.pushover To use: 1) set the time period you want to scan the POD directory for recent downoads in the script 2) you need to sucbribe to Pushover with a one-time $4.99 in-app purchase and install the Android app(or IoS etc). I have no affiliation with Pushover 3) get the app and user keys from the Pushover api website 4) replace the dummy keys in the script with the real ones 5) run the python script. Optionally run it from cron after podget is run Contributed by Mike Webb ./podget-0.9.3/SCRIPTS/v0.8.8/push-new-mp3.py30000644000175000017500000000463714125415341017452 0ustar lowkeylowkey#!/usr/bin/env python3 # Copyright (c) 2021 Mike Webb # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -- #https://stackoverflow.com/questions/16953842/using-os-walk-to-recursively-traverse-directories-in-python #given a folder name, walk through its entire hierarchy import os, time, http.client, urllib now = time.time() nowLess24h = time.time() - (48*60*60) #48 hours podList = [] def recursive_walk(folder): global podList for folderName, subfolders, filenames in os.walk(folder): if subfolders: for subfolder in subfolders: recursive_walk(subfolder) created = os.path.getctime(folderName) if created <= now and created >= nowLess24h: #print(folderName, created) lenFolderName = len(folderName) folderName = (folderName[13:lenFolderName]) if folderName != ".LOG" and folderName != "m3u-tmp": #print(folderName) podList.append(folderName) recursive_walk('/home/pi/POD/') #where your podcast's are located) podsStr = "Last 48h pod Casts" + chr(int(10)) + ''.join(str(e) + chr(int(10)) for e in podList) #the font etc stuff is to give a coloured heading to the Pushover list of podcasts #print(podsStr) PUSH_MSG = podsStr #setup pushover API app_key = "xxx" user_key = "yyy" #This function sends the push message using Pushover. def sendPush(PUSH_MSG): conn = http.client.HTTPSConnection("api.pushover.net:443") conn.request("POST", "/1/messages.json", urllib.parse.urlencode({ "token": app_key, "user": user_key, "message": PUSH_MSG, "html": 1, "sound": "tugboat" }), { "Content-type": "application/x-www-form-urlencoded" }) conn.getresponse() return # Run function to send message to Pushover sendPush(PUSH_MSG) ./podget-0.9.3/README0000644000175000017500000000105013742642167013440 0ustar lowkeylowkeyPodget is a simple podcast aggregator optimized for running as a scheduled background job (i.e. cron). It features support for downloading podcasts from RSS & ATOM XML feeds, for sorting the files into folders & categories, for importing URLs from iTunes PCAST files & OPML lists automatic M3U & ASX playlist creation, and automatic cleanup of old files. It also features automatic UTF-16 conversion for podcasts hosted on MS Windows servers. Run podget once to install the users configuration files to $HOME/.podget, and then customize them there. ./podget-0.9.3/podget0000755000175000017500000061073414544355000013773 0ustar lowkeylowkey#!/usr/bin/env bash # Useful to follow command execution and determine where an extra echo outputs during silent mode. #set -x # ---------------------------------------------------------------------------------------------------------------------------------- # Filename: podget {{{ # Maintainer: Dave Vehrs # Copyright: (c) 2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017 Dave Vehrs # # This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public # License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any # later version. # # This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied # warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # # Description: Podget is a simple bash script to automate the downloading and # organizing of podcast content. # Dependencies: bash, coreutils, debianutils, findutils, grep, gawk or mawk, libc-bin (for iconv), sed, and wget. # Installation: cp podget.sh /usr/local/bin # chmod 755 /usr/local/bin/podget.sh }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Exit Codes {{{ # "Reserved" Exit codes # 1 General Error # 2 Misuse of shell built-ins # 126 Command invoked cannot execute # 127 Command not found # 128+n Invalid argument to exit # 130 Script terminated by Control-C (128+2) # 143 Script terminated by TERM signal (128+15) # "Our" Exit codes # Display Help (set to '0' because it is an valid exit condition, not an error.) ERR_DISPLAYHELP=0 # Library directory not defined. ERR_LIBNOTDEF=50 # Library directory available space below limit ERR_LIBLOWSPACE=51 # Libc6 not installed. Cannot convert UTF16 feeds. ERR_LIBC6NOTINSTALLED=60 # Another running session already exists. ERR_RUNNINGSESSION=70 # OPML import error. ERR_IMPORTOPML=80 # OPML export error. ERR_EXPORTOPML=90 # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Traps {{{ # FUNCNAME is declared with a default value in case the trap is triggered while # outside a function. trap 'EXIT_ERROR ${LINENO} ${?} ${FUNCNAME:-Unconfigured}' ERR # trap to run CLEANUP function if program receives a TERM (kill) or INT (ctrl-c) signal # - CLEANUP called in line for other normal exits. trap 'CLEANUP_AND_EXIT 143' TERM trap 'CLEANUP_AND_EXIT 130' INT # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Set Shell Options to catch errors ('man bash' for details) {{{ set -o errexit set -o nounset set -o pipefail # Enable errtrace so that the ERR trap is inherited by functions # NOTE: Causes an error about line 1841 where we try to filter various items out of # the category and name. Speculation is it is caused by our use of the expr command # that commonly gives non-zero exit status for commands that actually exited OK. # # We can get around the issue by adding a '|| true' to the end of each expr command # but that seems a hack. What about replacing expr? #set -o errtrace # Enabling inheritance of errexit by command substitution subshells, this was added in Bash 4.4 # NOTE: Test command is run within a subshell where pipefail is disabled. if (set +o pipefail && shopt inherit_errexit 2>&1 | grep -q invalid) ; then echo "Bash added the 'inherit_errexit' shell option in version 4.4, please upgrade." exit 1 fi shopt -s inherit_errexit # Enable extended glob matches. shopt -s extglob # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Help text and default file formats {{{ : << HELP_STEXT -c --config Name of configuration file to use. --create-config Exit immediately after creating configuration file. -C --cleanup Skip downloading and only run cleanup loop. --cleanup_simulate Skip downloading and simulate running cleanup loop. Display files to be deleted. --cleanup_days Number of days to retain files. Anything older will be removed. -d --dir_config Directory that configuration files are stored in. --dir_session Directory that session files are stored in. -f --force Force download of items from each feed even if they have already been downloaded. --import_opml Import servers from OPML file or HTTP/FTP URL. --export_opml Export serverlist to OPML file. --import_pcast Import servers from iTunes PCAST file or HTTP/FTP URL. -l --library Directory to store downloaded files in. -n --no-playlist Do not create M3U playlist of new items. -p --playlist-asx In addition to the default M3U playlist, create an ASX Playlist. M3U playlist must be created to convert to ASX. --playlist-per-podcast Create playlist of new items for each podcast feed. -r --recent Download only the newest items from each feed. --serverlist Serverlist to use. -s --silent Run silently (for cron jobs). --verbosity Set verbosity level (0-4). -v Set verbosity to level 1. -vv Set verbosity to level 2. -vvv Set verbosity to level 3. -vvvv Set verbosity to level 4. -h --help Display help. HELP_STEXT # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Defaults {{{ ######################################################################################################################### ## Do not configure here. Run podget once to install default user configuration files and edit there. ## ######################################################################################################################### # Set DIR_LIBRARY, DIR_SESSION, and DIR_LOG in config file. CONFIG_CORE="podgetrc" CONFIG_SERVERLIST="serverlist" # Configuration Directory: # This used to be stored by default in the base of the users home directory. #DIR_CONFIG="${HOME}/.podget" # # However we've update Podget to use three possible locations to help reduce clutter in the home directory. # For locations are: # 1. ${HOME}/.podget # 2. ${XDG_CONFIG_HOME}/podget # 3. ${HOME}/.config/podget # For existing users, when Podget runs it will test all three in that order. The first one it finds will become the # default configuration directory. # # For new users, Podget will attempt to use XDG_CONFIG_HOME but if that is not set will fallback to ${HOME}/.config/podget # # To use default configuration directory, set DIR_CONFIG to "UNSET-use-DEFAULT" # Or you can configure it to use any directory you want by configuring it here. DIR_CONFIG="UNSET-use-DEFAULT" # DEFAULT FILENAME_BADCHARS used to test the configuration filenames and then unset. The value used later in the script may be # set in the configuration file. If you have a genuine need to use one of these characters in the filenames of the serverlist or # core configuration file then you may need to remove it from this definition. For all other uses, modify the setting in your # configuration file (by default in podgetrc). # # This is a subset of the FILENAME_BADCHARS set in the default configuration file. Many of the symbols that have been removed are # because they will cause other errors when used on the command line that prevent these checks from working. FILENAME_BADCHARS="~#^=+{}[]:\"'?\\" # Filename Replace Character: Character to use to replace any/all # bad characters found. FILENAME_REPLACECHAR=_ # Default VERBOSITY # 0 == silent # 1 == Warning messages only. # 2 == Progress and Warning messages. # 3 == Debug, Progress and Warning messages. # 4 == All messages and wget set to maximum VERBOSITY. VERBOSITY=2 # Auto-Cleanup. # 0 == disabled # 1 == delete any old content CLEANUP=0 # Skip downloading and just run cleanup # 0 == disable CLEANUP_ONLY=0 # Simulate cleanup CLEANUP_SIMULATE=0 # Number of days to keep files. Cleanup will remove anything # older than this. CLEANUP_DAYS=7 # Most Recent # 0 == download all new items. # 1+ == download only the most recent MOST_RECENT=0 # Force # 0 == Only download new material. # 1 == Force download all items even those you've downloaded before. FORCE=0 # Install session. This gets called when script is first installed. INSTALL_SESSION=0 # Stop downloads if available space drops below MIN_SPACE=10000 # Date format for new playlist names DATE_FORMAT=+%F # ASX Playlists for Windows Media Player # 0 == do not create # 1 == create ASX_PLAYLIST=0 # Enable playlist creation NO_PLAYLIST=0 # Default Wget no options WGET_BASEOPTS="" # Default DEBUG Disabled (Deletion of temporary files allowed), configured # to allow its setting to be overridden from the command line. DEBUG=${DEBUG:-0} # Default DEBUG string leader DEBUG_LEADER="DEBUG --" # Order that items appear in feed. # Default: DESCENDING - Newest items appear first, oldest last. FEED_SORT_ORDER="DESCENDING" # Create or update full playlist for each feed of all available items. # 0 == Enable # 1 == Disable FEED_FULL_PLAYLIST=0 ######################################################################################################################### ## Do not configure here. Run podget once to install default user configuration files ($HOME/.podget) and edit there. ## ######################################################################################################################### # Internal script defaults NEWLINE=$'\n' # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Text for default configuration files: {{{ : << TEXT_DEFAULT_CONFIG # ---------------------------------------------------------------------------------------------------------------------------------- # Podget configuration file created by version @VERSION@ # [ NOTE: Do not delete version line as it will be used by future versions to # to test if configuration files have been updated with any required changes. # ---------------------------------------------------------------------------------------------------------------------------------- # File name and location configuration: # Name of Server List configuration file CONFIG_SERVERLIST=@SERVERLIST@ # Directory to store session files # If this option is not configured then by default podget will place the session files in the directory defined by TMPDIR/podget or # if it is not defined in the users shell then the session files will be placed in the directory /tmp/podget. # If you prefer a different location, then configure this variable. # DIR_SESSION=@HOME@/tmp/podget # Directory where to store downloaded files DIR_LIBRARY=@HOME@/POD # Directory to store logs in # By default, logs are stored in DIR_LIBRARY/.LOG # If you prefer a different location, then configure this variable. # DIR_LOG=@HOME@/POD/LOG # Set logging file names LOG_FAIL=errors LOG_COMPLETE=done # ---------------------------------------------------------------------------------------------------------------------------------- # Download Options: # Wget base options # Commonly used options: # -c Continue interupted downloads - While this flag is commonly used there are feeds that it can # cause "403 Forbidden" errors. # -nH No host directories (overrides .wgetrc defaults if necessary) # --proxy=off To disable proxy set by environmental variable http_proxy # --no-check-certificate To disable HTTPS certificate checks. Useful for sites that may be using self-signed cerficates # and not those from a trusted service authority. # --prefer-family=IPv4/IPv6 When DNS provides a choice of addresses to connect to a host, attempt to connect to the specified # address family first. If all addresses of the given family fail then the other family will be # tried. If set to 'none' then the addresses will be tried in the order provided by the server # regardless of which family they are in (this is effectively the default option). # # If you wish to force the use of IPv4 addresses only then you can use the "-4" or "--inet4-only" # options. Conversely, if you want to force the use of IPv6 addresses then you can set the "-6" # or "--inet6-only" options. # --content-disposition [EXPERIMENTAL FEATURE] Wget will look for and use "Content-Disposition" headers received from the # server. This can result in extra round-trips to the server for a "HEAD" request. This option # is useful for servers that use the "Content-Disposition" header to hold the filename of the # downloaded file rather than appending it to the URL. This has the potential to make some of # Podget's FILENAME_FORMATFIX options unneeded. # # WARNING: Enabling this flag disables any download progress information from being passed on to # the user. To debug errors that may occur during sessions with this flag enabled, it may be # necessary to enable DEBUG and then examine the temporary files that are not deleted in # DIR_SESSION. # # NOTE: This can be enable globally for all feeds here or if you want to enable it for only a few # specific feeds, you can add "OPT_CONTENT_DISPOSITION" to their line in your serverlist. # --user-agent= By default Podget will identify itself as "Podget". If that does not work or you want to # specify a custom user agent you can do so here. If the agent is a single word then it does not # need to be quoted. If it is a longer string with spaces then it will need to be quoted and have # it's quotes double escaped. # Examples: # --user-agent=Mozilla # --user-agent=Chrome # --user-agent=\\\"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36\\\" # --user-agent=\\\"Agent 007\\\" # --user-agent=\\\"Dread Pirate Roberts\\\" # # Wget options that include spaces need to be surrounded in quotes. # # WGET_BASEOPTS="-c -nH --user-agent=\\\"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36\\\"" # WGET_BASEOPTS="-c -nH --user-agent=Mozilla" # WGET_BASEOPTS="-c --proxy=off --no-check-certificate" # WGET_BASEOPTS="-nH --proxy=off --content-disposition" # WGET_BASEOPTS="-c --prefer-family=IPv4" # WGET_BASEOPTS="-c --prefer-family=IPv6" WGET_BASEOPTS="-c -nH" # Most Recent # 0 == download all new items. # 1+ == download only the most recent MOST_RECENT=0 # Force # 0 == Only download new material. # 1 == Force download all items even those you've downloaded before. FORCE=0 # Autocleanup of old playlists and the files they list. # 0 == disabled # 1 == delete any old content CLEANUP=0 # Number of days to keep files. Cleanup will remove anything # older than this. CLEANUP_DAYS=7 # Stop downloading if available space on the partition drops below value (in KB) # default: 614400 (600MB) MIN_SPACE=614400 # ---------------------------------------------------------------------------------------------------------------------------------- # Playlist Options: # Disable playlist creation [ No need to comment out other playlist variables ] # 0 == create # 1 == do not create NO_PLAYLIST=0 # Build playlists (comment out or set to a blank string to accept default format: New-). PLAYLIST_NAMEBASE=New- # Date format for new playlist names # +%F = YYYY-MM-DD like 2014-01-15 (DEFAULT) # +%m-%d-%Y = MM-DD-YYYY like 01-15-2014 # For other options 'man date' # # Date options that include spaces need to be surrounded in quotes. # DATE_FORMAT=+%F # ASX Playlists for Windows Media Player # 0 == do not create # 1 == create ASX_PLAYLIST=0 # ---------------------------------------------------------------------------------------------------------------------------------- # Filename Suffix: # Add suffix to the filename of every file downloaded to allow for subsequent scripts to detect the newly downloaded files and work # on them. Examples of this would be scripts to run id3v2 to force a standard genre for all MP3 files downloaded or to use mp3gain # to normalize files to have the same volume. # # A period (.) will automatically be added between the filename and tag like so: # filename.mp3.newtag # # Tags will not be added to filenames as they are added to the playlists. It will be necessary for any script that you run to # process the files remove the tag for the playlists to work. # # If this variable is undefined or commented out, then by default no suffix will be added. # FILENAME_SUFFIX="newtag" # ---------------------------------------------------------------------------------------------------------------------------------- # Downloaded Filename Cleanup Options: # # These options are for the filenames downloaded from the feeds. We will try to clean then up rather than interrupting the script # execution. # Filename Cleanup: For FAT32 filename compatability (Feature Request #1378956) # Tested with the following characters: !@#$%^&*()_-+=||{[}]:;"'<,>.?/ # # The \`, \" and \\ characters need to be escaped with a leading backslash. # # Bad Character definitions need to be surrounded in quotes. # # NOTE: FILENAME_BADCHARS is also used to test for characters that commonly cause errors in directory names. This can cause # FILENAME_BADCHARS to be reported as part of an error for configuration issues with DIR_SESSION, DIR_LOG, DIR_LIBRARY and podcast # FEED_NAME and FEED_CATEGORY. FILENAME_BADCHARS="\`~!#$^&=+{}*[]:;\"'<>?|\\" # Filename Replace Character: Character to use to replace any/all # bad characters found. FILENAME_REPLACECHAR=_ # When you run podget at a VERBOSITY of 3 or 4, it may appear that the filename format fixes are done out of order. That is because # they are named as they are created and as new fixes have been developed, those with more detailed exclusionary conditions have had # to be done before those with more generic conditions. Looking for improvements to fix this issue. # Filename Cleanup 2: Some RSS Feeds (like the BBC World News Bulletin) # download files with names like filename.mp3?1234567. Enable this mode # to fix the format to filename1234567.mp3. # 0 == disabled # 1 == enabled (default) FILENAME_FORMATFIX=1 # Filename Cleanup 3: Filenames of feeds hosted by LBC Plus corrupted. # Fixed per MoonUnit's feature request (#1660764) # # Takes an URL that looks like: http://lbc.audioagain.com/shared/audio/stream.mp3?guid=2007-03/14<...snip> # a7766e8ad2748269fd347eaee2b2e3f8&source=podcast.php&channel_id=88 # # Which normally creates a file named: a7766e8ad2748269fd347eaee2b2e3f8&source=podcast.php&channel_id=88 # # This fix extracts the date of the episode and changes the filename to 2007-03-14.mp3 # 0 == disabled # 1 == enabled (default) FILENAME_FORMATFIX2=1 # Filename Cleanup 4: Filenames of feeds hosted by CatRadio.cat need fixing. # Fixed per Oriol Rius's Bug Report (#1744705) # # Downloaded filenames look like: 1189153569775.mp3?programa=El+mat%ED+de+Catalunya+R%E0dio&podcast=y # This fix removes everything after the .mp3 # # NOTE: Testing in 2017 reveals changes in CatRadio's URL format that hampers this fix. # # Downloaded filenames now look like: 1487257264030.mp3&programa=Bon+dia%2C+malparits%21&var10=Neix+%2BCatR%E0dio%2C+el+dial+digital+de+Catalunya+R%E0dio&var11=video&var15=1783&var20=Podcast&var29=Audio&var3=951439&var14=951439&v25=Catalunya+R%E0dio&var19=16/02/17&var12=Tall&var18=45194 # # Two changes cause issues: # 1. Change of '?' to '&' for designating options. # 2. Use of forward slashes in date (var19) mess up some of our other filename extraction. # # However the fix for these podcasts is now simpler. If in our serverlist, we use either # the OPT_FILENAME_LOCATION or OPT_CONTENT_DISPOSTION option for these feedlists then the # filename will be correctly extracted. This leaves us with a long number as the filename, # however if we also enable the OPT_FILENAME_RENAME_MDATE option then the filename is prefixed # with the last modification date of the file which helps list the files in an order that # makes sense. # # 0 == disabled (default) # 1 == enabled FILENAME_FORMATFIX3=0 # Filename Cleanup 5: When the filename is part of the URL and the actual filename stays the same for # all items listed. # # Download URLs look like: http://feeds.theonion.com/~r/theonion/radionews/~5/213589629/podcast_redirect.mp3 # Where 213589629 is the unique filename. # # This filename change is disabled by default because it may cause unintended changes to the filename. # # 0 == disabled (default) # 1 == enabled FILENAME_FORMATFIX4=0 # Filename Cleanup 6: Remove "?referrer=rss" from the end of filenames as included in some feeds like # those from Vimcasts.org. Setup to work for MP3, M4V, OGG and OGV files. # # Feed URLs: http://vimcasts.org/feeds/ogg # http://vimcasts.org/feeds/quicktime # # In the feed, enclosure URLs look like: http://media.vimcasts.org/videos/1/show_invisibles.ogv?referrer=rss # # 0 == disabled # 1 == enabled (default) FILENAME_FORMATFIX5=1 # Filename Cleanup 7: Removes the trailing part of the filename after the '?'. # Fixed at the request of Joerg Schiermeier # # For dealing with enclosures like those formatted in the ZDF podcast. # Feed URL: http://www.zdf.de/ZDFmediathek/podcast/1193018?view=podcast # Example enclosure: # http://podfiles.zdf.de/podcast/zdf_podcasts/101103_backstage_afo_p.mp4?2010-11-03+06-42 # # 0 == disabled # 1 == enabled (default) FILENAME_FORMATFIX6=1 # Filename Cleanup 8: # This fix is for feeds that assign the same filename to be downloaded for each # enclosure and then embedded the actual filename of the object to be saved in # the media_url= parameter. This fix extracts that name and uses it for the # saved file. # # 0 == disabled # 1 == enabled (default) FILENAME_FORMATFIX7=1 # Filename Cleanup 9: # This fix is for feeds like Smodcast. It removes the "?client_id=" # from the end of each enclosure url in the feed. # # NOTE: To fully fix the filenames on feeds like Smodcast, this fix should # be used in conjunction with FILENAME_FORMATFIX4. # # Example URL: http://api.soundcloud.com/tracks/62837276/stream.mp3?client_id=a427c512429c9c90e58de7955257879c # Fixed filename: 62837276_stream.mp3 # # 0 == disabled # 1 == enabled (default) FILENAME_FORMATFIX8=1 # Filename Cleanup 10: # # This is a fix for podcast feeds formatted like those for Audioboo. Removes everything after the ? # in the filename. Attempted to make this fix generic enough to work with a variety of feeds of mp3, mp4, # ogg and ogv files. # # Feed URL: http://audioboo.fm/users/39903/boos.rss # Example URL: http://audioboo.fm/boos/1273271-mw-123-es-wird-fruhling.mp3?keyed=true&source=rss # Fixed Filename: 1273271-mw-123-es-wird-fruhling.mp3 # # NOTE: On Aug 30 2018, this fix was updated to also fix feeds formated like those from viertausendhertz.de. # # Feed URL: http://viertausendhertz.de/feed/podcast/systemfehler # Example URL: https://viertausendhertz.de/podcast-download/1538/sf04.mp3?v=1470947681&source=feed # Fixed Filename: sf04.mp3 # # 0 == disabled # 1 == enabled (default) FILENAME_FORMATFIX9=1 # Filename Cleanup 11: # # This is an attempt to fix feeds hosted on Apple ITunes. The enclosure URL from these feeds defines the # the filename as a long string of numbers and letter. It's not very descriptive. However, after the # filename and a '?', in the information passed down to the application as part of the URL, we can # extract the episode name for each podcast. It is that name that this fix will use for the filename, # with a few character replacements to insure good filenames. # # 0 == disabled # 1 == enabled (default) FILENAME_FORMATFIX10=1 # ---------------------------------------------------------------------------------------------------------------------------------- # DEBUG # # Enabling debug will: # 1. Stop podget from automatically deleting some temporary files in DIR_SESSION. # 2. Enable additional messages to track progress. # # 0 == disabled (default) # 1 == enabled # ${DEBUG:-0} == Sets DEBUG to disabled if it is not already set. This allows the user to enabled it # from the command line with "DEBUG=1 podget" # #DEBUG=${DEBUG:-0} # ---------------------------------------------------------------------------------------------------------------------------------- TEXT_DEFAULT_CONFIG : << TEXT_DEFAULT_SERVERLIST # Default Server List for podget # # Default format with category and name: # # # Alternate Formats: # 1. With a category but no name. # # 2. With a name but no category (2 ways). # No_Category # . # 3. With neither a category or name. # # # For additional formating documentation, please refer to 'man podget'. # #FEEDS: # ---------------------------------------------------------------------------------------------------------------------------------- # Using TITLES from Feed: http://thelinuxlink.net/tllts/tllts.rss LINUX The Linux Link OPT_FILENAME_RENAME_TITLETAG TEXT_DEFAULT_SERVERLIST : << TEXT_ASX_BEGINNING TEXT_ASX_BEGINNING : << TEXT_ASX_END TEXT_ASX_END # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Functions {{{ # Function: ARRAY_CONTAINS # Test if Array contains element. # Returns 0 if found, or 1 if not found. # ARGUMENTS: # ${1} == String to search for # ${2} == Array to search within (also called ${@:2}) ARRAY_CONTAINS() { local element # for element in "${@:2}"; do [[ "${element}" == "${1}" ]] && return 0; done for element in "${@:2}"; do [[ "${element}" =~ ^"${1}".* ]] && return 0; done return 1 } # Function: CLEANUP_AND_EXIT # Closes session and removes lock file (if it exists) # ARGUMENTS: # ${1} == Exit Status to report. CLEANUP_AND_EXIT() { local EXITSTATUS=${1} if (( VERBOSITY >= 2 )) ; then #echo -en "\nClosing session" printf '\n%s' "Closing session" fi if [[ -n ${DIR_SESSION+set} && -f ${DIR_SESSION}/podget.$$ ]]; then if (( DEBUG == 0 )); then if (( VERBOSITY >= 2 )) ; then printf '%s' " and removing lock file" fi if (( VERBOSITY >= 4 )) ; then echo rm -fv "${DIR_SESSION}"/podget.$$ else rm -f "${DIR_SESSION}"/podget.$$ fi else printf '\n%s\n' "${DEBUG_LEADER} Not deleting ${DIR_SESSION}/podget.$$" fi fi if (( (VERBOSITY >= 2) && (VERBOSITY <= 3) )); then printf '%s\n' "." elif (( (VERBOSITY == 1) || (VERBOSITY > 3) )); then echo fi exit "${EXITSTATUS}" } display_shelp() { echo; echo "Usage $0 [options]"; echo sed --silent -e '/HELP_STEXT$/,/^HELP_STEXT/p' "$0" | sed -e '/HELP_STEXT/d' } # Function: DIRECTORY_CHECK 'Name of Variable to be tested' # Simple function to verify that unsafe characters are not used in directory names. DIRECTORY_CHECK() { # Variables have a default value of 'UNCONFigured' because this word and combination # of capitalization is unlikely to be used. This allows us to catch improperly # formated calls to DIRECTORY_CHECK. # # Uses variable indirection, The '!' introduces indirection which can be read # to say "Get the value of the variable named this". local TEST_STRING=${!1:-"UNCONFigured"} # The second use simply reports the name of the variable to be tested. local TEST_VARIABLE=${1:-"UNCONFigured"} local TEST_FAIL=0 local OFFENDING_CHARS="" if [[ ${TEST_STRING} == "UNCONFigured" ]]; then echo "Improperly formated call to DIRECTORY_CHECK." return 1 fi # Test if FILENAME_BADCHARS is configured, if it is then check filenames to # prevent the use of disallowed characters. if [[ -n ${FILENAME_BADCHARS+set} ]]; then for (( i=0; i<${#FILENAME_BADCHARS}; i++ )); do local TEST_CHAR=${FILENAME_BADCHARS:$i:1} # Grep looks for --fixed-strings so that certain characters are not # interpreted as regular expressions (like ^ $ or /) if grep --quiet --fixed-strings "${TEST_CHAR}" <<<"${TEST_STRING}"; then OFFENDING_CHARS="${OFFENDING_CHARS}${TEST_CHAR}" # This test must come first because it will set TEST_FAIL to 1 regardless # of how many offending characters are found. Given that this test can be # reported multiple times, we do not want it to cause additional suggestions # to be given to the user below. TEST_FAIL=1 fi done unset -v i fi # consult Shellcheck SC2076 for why I choose this construct rather than using regex (=~) checks if [[ ${TEST_STRING} = *"../"* ]]; then TEST_FAIL=$((TEST_FAIL+2)) fi if [[ ${TEST_STRING} = *"*"* ]]; then TEST_FAIL=$((TEST_FAIL+4)) fi # This test will create duplicate suggestions as the back-slash character also appears in # the default FILENAME_BADCHARS. if [[ ${TEST_STRING} = *"\\000"* ]]; then TEST_FAIL=$((TEST_FAIL+8)) fi if (( TEST_FAIL != 0 )); then echo "DIRECTORY CHECK ERROR: ${TEST_VARIABLE} = ${TEST_STRING}" echo echo "Suggestion(s):" local COUNT=0 while (( TEST_FAIL != 0 )); do if (( (8<=TEST_FAIL) && (TEST_FAIL<=150) )); then COUNT=$((COUNT+1)) # Shellcheck does not like the backslash being escaped here as part of an echo statement (SC2028) # There is an open issue on github for it but some people think it is appropriate as "info". # (https://github.com/koalaman/shellcheck/issues/2486) # # Attempting to use printf here as recommended by Shellcheck. Seems unnecessary but that's how # the game is played. printf " %s. '\\000' cannot be used in directory names as mkdir expects\n" "${COUNT}" # echo " ${COUNT}. '\\000' cannot be used in directory names as mkdir expects" printf " to get a null terminated string and '\\000' is considered 'end of string'.\n" # echo " to get a null terminated string and '\\000' is considered 'end of string'." if (( TEST_FAIL >= 8 )); then TEST_FAIL=$((TEST_FAIL-8)) fi elif (( (4<=TEST_FAIL) && (TEST_FAIL<=7) )); then COUNT=$((COUNT+1)) echo " ${COUNT}. The asterisk should not be used in directories as they are commonly used" echo " to designate a wild card for expansion in Bash variables." if (( TEST_FAIL >= 4 )); then TEST_FAIL=$((TEST_FAIL-4)) fi elif (( (2<=TEST_FAIL) && (TEST_FAIL<=3) )); then COUNT=$((COUNT+1)) echo " ${COUNT}. Directories should not contain two periods and a slash in conjunction." echo " If you need to save certain files outside of the Podcast Library directory," echo " defined by this podgetrc, the proper solution is to create a second podgetrc" echo " with the new library location defined and to run podget with the --config" echo " command line option to designate the podgetrc file to use." if (( TEST_FAIL >= 2 )); then TEST_FAIL=$((TEST_FAIL-2)) fi elif ((1==TEST_FAIL)); then COUNT=$((COUNT+1)) echo " ${COUNT}. Attempts to use characters disallowed by FILENAME_BADCHARS." echo " Either remove the offending characters from the configured directory" echo " or FILENAME_BADCHARS." echo " Configured characters not allowed: ${FILENAME_BADCHARS}" echo " Offending character(s): ${OFFENDING_CHARS}" if (( TEST_FAIL >= 1 )); then TEST_FAIL=$((TEST_FAIL-1)) fi fi done if [[ ${TEST_VARIABLE} == "FEED_NAME" || ${TEST_VARIABLE} == "FEED_CATEGORY" ]]; then return 1 else CLEANUP_AND_EXIT 1 fi fi } # The commands of this function will trigger Shellcheck SC2317 (Command appears to be unreachable) # because the function is called from a trap so we can ignore this message. # shellcheck disable=SC2317 EXIT_ERROR() { # Name of script local JOB_NAME JOB_NAME=$(basename "$0") # The following three variables are configured with a default value in case # the function is called without options set. local LINENUM="${1:-"Unconfigured"}" # Line with error local EXITSTATUS="${2:-"Unconfigured"}" # exit status of error local FUNCTION="${3:-"Unconfigured"}" # If error occurred in a function, its name will be listed. printf '\n%s\n %-15s %s\n' "Error:" "Script:" "${JOB_NAME}" # Function line only appears if it has been set to value other than the # default. Works on the assumption that "Unconfigured" is not likely to be # chosen as a function name. if [[ ${FUNCTION} != "Unconfigured" ]]; then printf ' %-15s %s\n' "Function:" "${FUNCTION}" fi printf ' %-15s %s\n %-15s %s\n' "At line:" "${LINENUM}" "Exit Status:" "${EXITSTATUS}" printf '\n%s\n' "Context:" # Test is awk installed, if so use it. If not, then use tools from coreutils. if hash awk 2>/dev/null; then # This line works and adds a ">>>" to designate the offending line but adds # awk as a script dependency. awk 'NR>L-4 && NR>>":""),$0 }' L="${LINENUM}" "${0}" else # This line works and only depends on coreutils pr -tn "${0}" | tail -n+$((LINENUM - 3)) | head -n7 fi CLEANUP_AND_EXIT 1 } # Function: FILENAME_CHECK 'Name of Variable to be tested' # This function tests the filenames used by podget locally for various configuration and log files. While these checks have # some similarity to those applied to downloaded files the major difference is that violations of these rules will interrupt # the execution of podget and podget will attempt to fix the other filenames but many not always succeed. # Arguments: FILENAME_CHECK() { # Variables have a default value of 'UNCONFigured' because this word and combination # of capitalization is unlikely to be used. This allows us to catch improperly # formated calls to FILENAME_CHECK. # # Uses variable indirection, The '!' introduces indirection which can be read # to say "Get the value of the variable named this". local TEST_STRING=${!1:-"UNCONFigured"} # The second use simply reports the name of the variable to be tested. local TEST_VARIABLE=${1:-"UNCONFigured"} local TEST_FAIL=0 local OFFENDING_CHARS="" if [[ ${TEST_STRING} == "UNCONFigured" ]]; then echo "Improperly formated call to FILENAME_CHECK." return 1 fi # Test if FILENAME_BADCHARS is configured, if it is then check filenames to # prevent the use of disallowed characters. if [[ -n ${FILENAME_BADCHARS+set} ]]; then for (( i=0; i<${#FILENAME_BADCHARS}; i++ )); do local TEST_CHAR=${FILENAME_BADCHARS:$i:1} # Grep looks for --fixed-strings so that certain characters are not # interpreted as regular expressions (like ^ $ or /) if grep --quiet --fixed-strings "${TEST_CHAR}" <<<"${TEST_STRING}"; then OFFENDING_CHARS="${OFFENDING_CHARS}${TEST_CHAR}" # This test must come first because it will set TEST_FAIL to 1 regardless # of how many offending characters are found. Given that this test can be # reported multiple times, we do not want it to cause additional suggestions # to be given to the user below. TEST_FAIL=1 fi done unset -v i fi if [[ -z ${TEST_STRING##*/*} ]]; then # First test remove PATH from TEST_FILENAME variable. local TEST_DIRECTORY="${TEST_STRING%/*}" local TEST_FILENAME="${TEST_STRING##*/}" if [[ -n "${TEST_DIRECTORY}" ]]; then TEST_FAIL=$((TEST_FAIL+2)) fi # Remove directory from string to be tested for following tests. TEST_STRING=${TEST_FILENAME} fi # Configuration files should not be hidden by leading periods and trailing periods can cause issues on some file systems or # operating systems. Test if filename begins or ends with a period (.) if [[ ${TEST_STRING:0:1} == "." || ${TEST_STRING:(-1):1} == "." ]]; then TEST_FAIL=$((TEST_FAIL+4)) fi if (( TEST_FAIL != 0 )); then echo case "${TEST_VARIABLE}" in "CONFIG_CORE" ) echo "Configuration filename specified by -c or --create-config violates the following rules..." ;; "CMDL_SERVERLIST" ) echo "Serverlist filename specified by --serverlist violates the following rules..." ;; "CONFIG_SERVERLIST" ) echo "Default Serverlist filename violates the following rules..." ;; "LOG_FAIL" ) echo "LOG_FAIL defined in ${CONFIG_CORE} violates the following rules..." ;; "LOG_COMPLETE" ) echo "LOG_COMPLETE defined in ${CONFIG_CORE} violates the following rules..." ;; * ) echo "${TEST_VARIABLE} violates the following rules..." ;; esac echo echo "Suggestion(s):" COUNT=0 while (( TEST_FAIL != 0 )); do case ${TEST_FAIL} in # Included as an example of how other errors could be added with # a binary progression for the values they add to TEST_FAIL. [4-7]) COUNT=$((COUNT+1)) echo " ${COUNT}. Remove leading or trailing period from ${TEST_STRING}" if (( TEST_FAIL > 1 )); then TEST_FAIL=$((TEST_FAIL-4)) fi ;; [2-3]) COUNT=$((COUNT+1)) echo " ${COUNT}. Filenames should not include any directory configuration." echo " Remove the directory configuration." case "${TEST_VARIABLE}" in "CONFIG_CORE" | "CMDL_SERVERLIST" | "CONFIG_SERVERLIST" ) echo " If you need to specify a directory other than the default," echo " use the -d or --dir_config command line options." ;; "LOG_FAIL" | "LOG_COMPLETE" ) echo " If you wish to specify another location to store the logs," echo " then configure the DIR_LOG variable in your ${CONFIG_CORE}" ;; esac if (( TEST_FAIL >= 2 )); then TEST_FAIL=$((TEST_FAIL-2)) fi ;; 1) COUNT=$((COUNT+1)) echo " ${COUNT}. Attempts to use characters disallowed by FILENAME_BADCHARS." echo " Either remove the offending characters from the configured directory" echo " or FILENAME_BADCHARS." echo " Configured characters not allowed: ${FILENAME_BADCHARS}" echo " Offending character(s): ${OFFENDING_CHARS}" if (( TEST_FAIL >= 1 )); then TEST_FAIL=$((TEST_FAIL-1)) fi ;; esac done CLEANUP_AND_EXIT 1 fi } # Function: filenameFixFormat ${1} ${2} # Arguments: # ${1} == name of variable to hold return string # ${2} == string to fix the format of filenameFixFormat() { # variable to hold returned value. local VAR_RETURN=${1} # Filename to be modified. # Set original value for filename format fixes and character substitutions. # Set according to what is passed as the second argument to function. local MODIFIED_FILENAME=${2} # ORIGINAL and MODIFIED start out the same. local ORIGINAL_FILENAME="${MODIFIED_FILENAME}" if [[ -n ${FILENAME_FORMATFIX+set} || -n ${FILENAME_FORMATFIX2+set} || -n ${FILENAME_FORMATFIX3+set} || -n ${FILENAME_FORMATFIX4+set} || -n ${FILENAME_FORMATFIX5+set} || -n ${FILENAME_FORMATFIX6+set} || -n ${FILENAME_FORMATFIX7+set} || -n ${FILENAME_FORMATFIX8+set} || -n ${FILENAME_FORMATFIX9+set} || -n ${FILENAME_FORMATFIX10+set} ]]; then if (( VERBOSITY >= 3 )) ; then printf '%-30s %s\n' "ORIGINAL FILENAME:" "${ORIGINAL_FILENAME}" fi fi # Note: Filename format fixes that have more specific conditions come first. More generic last. This is # because a fix with too liberal a condition can prevent a more specific fix from running. Fixes are named in # the order they were created, so it may appear that they are out of order. By changing the order that they are # executed in, it is possible to have more enabled by default. # # TODO: Create exclusionary conditions for the fixes that are out of order to restore sanity to this list. # # FILENAME_FORMATFIX has been moved to the end of the order. # # FILENAME_FORMATFIX4 is not part of this function and is called immediately after this function ends. # Filename format fix for podcasts hosted on http://lbc.audioagain.com. if [[ -n ${FILENAME_FORMATFIX2+set} ]] && (( FILENAME_FORMATFIX2 > 0 )); then if (( $(expr "${MODIFIED_FILENAME}" : "[0-9a-zA-Z]\\+[&]amp;source=podcast.php[&]amp;channel_id=[0-9]\\+\$") > 0 )); then MODIFIED_FILENAME=$(echo "${MODIFIED_FILENAME}" | sed 's/.*stream.mp3[?]guid=\([0-9]\+\)-\([0-9]\+\)\/\([0-9]\+\)\/.*/\1-\2-\3.mp3/') if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} FILENAME FORMAT(2) FIXED: ${MODIFIED_FILENAME}" fi fi fi # Filename format fix for podcasts hosted on http://www.catradio.cat if [[ -n ${FILENAME_FORMATFIX3+set} ]] && (( FILENAME_FORMATFIX3 > 0 )); then if (( $(expr "${MODIFIED_FILENAME}" : "[0-9]\\+\\.mp3\\?[&]programa=[0-9a-Z+=%&;]*\$") > 0 )); then # shellcheck disable=SC2001 MODIFIED_FILENAME=$(echo "${MODIFIED_FILENAME}" | sed 's/\(.*\)\.mp3\(.*\)/\1\.mp3/g') if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} FILENAME FORMAT(3) FIXED: ${MODIFIED_FILENAME}" fi fi fi # Remove "?referrer=rss" from filename as included with some feeds like Vimcasts.org if [[ -n ${FILENAME_FORMATFIX5+set} ]] && (( FILENAME_FORMATFIX5 > 0 )); then if (( $(expr "${MODIFIED_FILENAME}" : "[-0-9a-zA-Z_]\\+\\.[agmopv34]\\+[?]referrer=rss") > 0 )); then MODIFIED_FILENAME=$(echo "${MODIFIED_FILENAME}" | sed -r 's/([-A-Za-z0-9_]+.[agmopv34]+)[?]referrer=rss/\1/g') if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} FILENAME FORMAT(5) FIXED: ${MODIFIED_FILENAME}" fi fi fi # ZDF podcast filename fix if [[ -n ${FILENAME_FORMATFIX6+set} ]] && (( FILENAME_FORMATFIX6 > 0 )); then if (( $(expr "${MODIFIED_FILENAME}" : "[-_0-9a-zA-Z]\\+\\.[agmopv34]\\+[?][-_+0-9]\\+") > 0 )); then MODIFIED_FILENAME=$(echo "${MODIFIED_FILENAME}" | sed -ru 's/([-_A-Za-z0-9]+.[agmopv34]+)[?][-+0-9]*/\1/g') if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} FILENAME FORMAT(6) FIXED: ${MODIFIED_FILENAME}" fi fi fi # media_url cleanup # This fix was inspired by the Radio France podcast feed. Each enclosure URL in the feed had the same filename # specified to be downloaded, and the actual filename of the MP3 file was appended in the media_url variable. # This fix extracts that filename and uses it for the downloaded file. # # Filename consists of: numbers, letters, dashes, underscore, plus, percent, equals, question mark, ampersand, and period # with extended regex and buffers limited # wget -O - http://radiofrance-podcast.net/podcast09/rss_12036.xml | grep enclosure | sed -ru 's/.*(media_url=.*[.][gmopv34]+)"\ .*/\1/' | sed -ru 's/.*%2F([-0-9A-Za-z_.]+[.][gmopv34]+)/\1/' if [[ -n ${FILENAME_FORMATFIX7+set} ]] && (( FILENAME_FORMATFIX7 > 0 )); then if (( $(expr "${MODIFIED_FILENAME}" : "[+_%&=?.0-9a-zA-Z]*media_url=http") > 0 )); then MODIFIED_FILENAME=$(echo "${MODIFIED_FILENAME}" | sed -ru 's/.*(media_url=http.*[.][agmopv34]+)"\ .*/\1/' | sed -ru 's/.*%2F([-0-9A-Za-z_.]+[.][agmopv34]+)/\1/') if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} FILENAME FORMAT(7) FIXED: ${MODIFIED_FILENAME}" fi fi fi # SMODCAST cleanup # Remove "?client_id=" from filename. # # Note: This is only the first part of the cleanup needed for the SMODCAST feeds. These removes the trailing portion of the # enclosure URL but every filename is left as "stream.mp3". The distinguishing part of each URL is held one segment before the # filename and so FILENAME_FORMATFIX4 must also be enabled. This can potentially affect other feeds and so it may be desirable # to separate these feeds to their own configuration and serverlist files. They can then be loaded by using the -c and # --serverlist flags on the command line. if [[ -n ${FILENAME_FORMATFIX8+set} ]] && (( FILENAME_FORMATFIX8 > 0 )); then if (( $(expr "${MODIFIED_FILENAME}" : "stream[.]mp3[?]client_id=[0-9a-zA-Z]\\+") > 0 )); then MODIFIED_FILENAME=$(echo "${MODIFIED_FILENAME}" | sed -ru 's/(stream[.]mp3)[?]client_id=[0-9A-Za-z]+/\1/') if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} FILENAME FORMAT(8) FIXED: ${MODIFIED_FILENAME}" fi fi fi # Audioboo Filename Cleanup. # Enclosure URLs have "?keyed=true&source=rss" appended to them. This fix removes that string. # It should work for Audioboo podcasts and others with similar formating. if [[ -n ${FILENAME_FORMATFIX9+set} ]] && (( FILENAME_FORMATFIX9 > 0 )); then if (( $(expr "${MODIFIED_FILENAME}" : "[-_0-9a-zA-Z]\\+[.][agmopv34]\\+[?][%&;=0-9a-zA-Z]\\+") > 0 )) ; then MODIFIED_FILENAME=$(echo "${MODIFIED_FILENAME}" | sed -ru 's/([-_A-Za-z0-9]+[.][agmopv34]+)[?][-%&#;=A-Za-z0-9]+/\1/g') if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} FILENAME FORMAT(9) FIXED: ${MODIFIED_FILENAME}" fi fi fi # MP3 on Apple ITunes # Filenames are generally long strings of numbers and letters, with the actual episode name being defined after the '?' # This extracts the episode name and uses it for the filename. if [[ -n ${FILENAME_FORMATFIX10+set} ]] && (( FILENAME_FORMATFIX10 > 0 )); then if (( $(expr "${MODIFIED_FILENAME}" : "[-0-9A-Za-z_]\\+[.][MmPp3]\\+[?][-0-9A-Za-z%=]\\+%26episodeName%3D[-0-9A-Za-z%.*]\\+%26episodeKind%3D") > 0 )); then MODIFIED_FILENAME=$(echo "${MODIFIED_FILENAME}" | sed -ru 's/.*%26episodeName%3D([-._%A-Za-z0-9*]+)%26episodeKind[-%&;=A-Za-z0-9]+/\1.mp3/g' | sed -ru 's/%2B/_/g;s/%25[0-9ACF]{2}//g;s/[*]//g') if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} FILENAME FORMAT(10) FIXED: ${MODIFIED_FILENAME}" fi fi fi # Fix improperly formated filenames (fixes filename.mp3?123456 to filename123456.mp3) if [[ -n ${FILENAME_FORMATFIX+set} ]] && (( FILENAME_FORMATFIX > 0 )); then if (( $(expr "${MODIFIED_FILENAME}" : ".*\\.mp3..*$") > 0 )); then # shellcheck disable=SC2001 MODIFIED_FILENAME=$(echo "${MODIFIED_FILENAME}" | sed 's/\.mp3\(.*\)/\1.mp3/') if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} FILENAME FORMAT FIXED: ${MODIFIED_FILENAME}" fi fi fi # Test if BADCHARS set by variable if [[ -n ${FILENAME_BADCHARS+set} ]] ; then # Test for BADCHARS in MODIFIED_FILENAME if [[ ${MODIFIED_FILENAME} =~ ["${FILENAME_BADCHARS}"] ]]; then # Two step process. First modify any BADCHARS into the REPLACECHAR and then squeeze each repetition of the REPLACECHAR # down to a single time. MODIFIED_FILENAME=$(echo "${MODIFIED_FILENAME}" | tr "${FILENAME_BADCHARS}" "${FILENAME_REPLACECHAR}" | tr -s "${FILENAME_REPLACECHAR}") if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} FILENAME FORMAT FIXED: ${MODIFIED_FILENAME}" fi fi fi if (( VERBOSITY >= 3 )) && [[ "${ORIGINAL_FILENAME}" != "${MODIFIED_FILENAME}" ]]; then printf '%-30s %s\n' "MODIFIED FILENAME:" "${MODIFIED_FILENAME}" fi # Pass the modified filename back to the calling variable. # eval "${VAR_RETURN}=${MODIFIED_FILENAME@Q}" printf -v "${VAR_RETURN}" '%s' "${MODIFIED_FILENAME}" # close without error return 0 } # Function: titleFixFormat ${1} ${2} # Arguments: # ${1} == name of variable to hold return string # ${2} == string to fix the format of titleFixFormat() { # variable to hold returned value. local VAR_RETURN=${1} # Filename to be modified. # Set original value for title format fixes and character substitutions. # Set according to what is passed as the second argument to function. local MODIFIED_TITLE=${2} # ORIGINAL and MODIFIED start out the same. local ORIGINAL_TITLE="${MODIFIED_TITLE}" if (( VERBOSITY >= 3 )) ; then printf '%-30s %s\n' "ORIGINAL TITLE:" "${ORIGINAL_TITLE}" fi if [[ "${MODIFIED_TITLE}" =~ .*" ".* ]]; then MODIFIED_TITLE="${MODIFIED_TITLE// /${FILENAME_REPLACECHAR}}" if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} TITLE FORMAT FIXED: ${MODIFIED_TITLE}" fi fi # Fix for ampersand (&) is interesting because the character will be removed by the default FILENAME_BADCHARS list. # We include it here for people who remove the & from their FILENAME_BADCHARS. if [[ "${MODIFIED_TITLE}" =~ .*"&".* ]]; then MODIFIED_TITLE="${MODIFIED_TITLE//\&/\&}" # MODIFIED_TITLE=$(echo "${MODIFIED_TITLE}" | sed -e 's/\&/\&/g') if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} TITLE FORMAT FIXED: ${MODIFIED_TITLE}" fi fi # Case insensitive BASH regex test followed by case insensitive replacement by sed. if [[ "${MODIFIED_TITLE,,}" =~ "" ]]; then MODIFIED_TITLE=$(echo "${MODIFIED_TITLE}" | sed -n -e 's//\1/Ip') if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} TITLE FORMAT FIXED: ${MODIFIED_TITLE}" fi fi # Test if BADCHARS set if [[ -n ${FILENAME_BADCHARS+set} ]] ; then # Test for BADCHARS in MODIFIED_TITLE if [[ ${MODIFIED_TITLE} =~ ["${FILENAME_BADCHARS}"] ]]; then # Two step process. First modify any BADCHARS into the REPLACECHAR and then squeeze each repetition of the REPLACECHAR # down to a single time. MODIFIED_TITLE=$(echo "${MODIFIED_TITLE}" | tr "${FILENAME_BADCHARS}" "${FILENAME_REPLACECHAR}" | tr -s "${FILENAME_REPLACECHAR}") if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} TITLE FORMAT FIXED: ${MODIFIED_TITLE}" fi fi fi if [[ ${MODIFIED_TITLE} =~ [/] ]]; then # Replace forward slashes in titles with dash as the using a forward slash in a name is not POSIX compliant MODIFIED_TITLE="${MODIFIED_TITLE//\//-}" if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} TITLE FORMAT FIXED: ${MODIFIED_TITLE}" fi fi # After replacing characters, strings may look ugly because they start or end with the FILENAME_REPLACECHAR or a dash. # Therefore we remove those here. if [[ ${MODIFIED_TITLE:0:1} =~ ["${FILENAME_REPLACECHAR}"-] ]]; then MODIFIED_TITLE="${MODIFIED_TITLE:1}" if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} TITLE FORMAT FIXED: ${MODIFIED_TITLE}" fi fi # Negative indices from the trailing end of the string require a space before the dash. if [[ ${MODIFIED_TITLE: -1} =~ ["${FILENAME_REPLACECHAR}"-] ]]; then MODIFIED_TITLE="${MODIFIED_TITLE:: -1}" if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} TITLE FORMAT FIXED: ${MODIFIED_TITLE}" fi fi if (( VERBOSITY >= 3 )) && [[ "${ORIGINAL_TITLE}" != "${MODIFIED_TITLE}" ]]; then printf '%-30s %s\n' "MODIFIED TITLE:" "${MODIFIED_TITLE}" fi # Pass the modified filename back to the calling variable. printf -v "${VAR_RETURN}" '%s' "${MODIFIED_TITLE}" # close without error return 0 } # Function: FILTER_OPTIONS ${1} # Arguments: # ${1} == name of variable to filter on (either FEED_CATEGORY or FEED_NAME) FILTER_OPTIONS() { local FILTER_ITEM="${1}" # NOTE: The regex expressions in the following checks were updated so that # they played nice on FreeBSD 12.1 with Bash 5.0.17. # # Previously the regexes included: (.*[[:space:]]|) # # This would either look for the pattern after other parts or # immediately at the beginning of the string. It was the second part # that caused issues, the part after the or but before the closing # parenthesis. The blank part worked on Linux but not on FreeBSD. # # To fix it, we changed to this: (.+[[:space:]]|[[:space:]]?) # Extract Password from FILTER_ITEM if found. if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)(PASS:)([^[:space:]]+).*$ ]]; then URL_PASSWORD="${BASH_REMATCH[3]}" # eval "${FILTER_ITEM}=\${!FILTER_ITEM/PASS:+([^[:space:]])}" printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM/PASS:+([^[:space:]])}" fi # Extract USERNAME from FILTER_ITEM if found. if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)(USER:)([^[:space:]]+).*$ ]]; then URL_USERNAME="${BASH_REMATCH[3]}" # eval "${FILTER_ITEM}=\${!FILTER_ITEM/USER:+([^[:space:]])}" printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM/USER:+([^[:space:]])}" fi if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)OPT_CONTENT_DISPOSITION.*$ ]]; then WGET_OPTION_DISPOSITION=1 # eval "${FILTER_ITEM}=\${!FILTER_ITEM/OPT_CONTENT_DISPOSITION}" printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM/OPT_CONTENT_DISPOSITION}" fi if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)OPT_DISPOSITION_FAIL.*$ ]]; then WGET_OPTION_DISPOSITION_FAIL=1 # eval "${FILTER_ITEM}=\${!FILTER_ITEM/OPT_DISPOSITION_FAIL}" printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM/OPT_DISPOSITION_FAIL}" fi if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)OPT_NO_CERT_CHECK.*$ ]]; then WGET_OPTION_NO_CHECK_CERIFICATE=1 printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM/OPT_NO_CERT_CHECK}" fi if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)OPT_PREFER_IP[vV]4.*$ ]]; then WGET_OPTION_PREFER_IP_TYPE=4 printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM//OPT_PREFER_IP[vV]4}" fi if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)OPT_PREFER_IP[vV]6.*$ ]]; then WGET_OPTION_PREFER_IP_TYPE=6 printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM//OPT_PREFER_IP[vV]6}" fi if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)OPT_FEED_ORDER_ASCENDING.*$ ]]; then FEED_SORT_ORDER="ASCENDING" # eval "${FILTER_ITEM}=\${!FILTER_ITEM/OPT_FEED_ORDER_ASCENDING}" printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM/OPT_FEED_ORDER_ASCENDING}" fi if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)OPT_FEED_PLAYLIST_NEWFIRST.*$ ]]; then FEED_FULL_PLAYLIST=1 # eval "${FILTER_ITEM}=\${!FILTER_ITEM/OPT_FEED_PLAYLIST_NEWFIRST}" printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM/OPT_FEED_PLAYLIST_NEWFIRST}" fi if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)OPT_FEED_PLAYLIST_OLDFIRST.*$ ]]; then FEED_FULL_PLAYLIST=2 # eval "${FILTER_ITEM}=\${!FILTER_ITEM/OPT_FEED_PLAYLIST_OLDFIRST}" printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM/OPT_FEED_PLAYLIST_OLDFIRST}" fi if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)OPT_WGET_DEFUSERAGENT.*$ ]]; then WGET_OPTION_DEFAULT_USERAGENT=1 printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM/OPT_WGET_DEFUSERAGENT}" fi if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)OPT_RSS_MEDIACONTENT.*$ ]]; then WGET_OPTION_RSS_MEDIACONTENT=1 printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM/OPT_RSS_MEDIACONTENT}" fi if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)OPT_FILENAME_LOCATION.*$ ]]; then WGET_OPTION_FILENAME_LOCATION=1 # eval "${FILTER_ITEM}=\${!FILTER_ITEM/OPT_FILENAME_LOCATION}" printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM/OPT_FILENAME_LOCATION}" fi if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)OPT_FILENAME_RENAME_MDATE.*$ ]]; then POST_WGET_RENAME_MDATE=1 # eval "${FILTER_ITEM}=\${!FILTER_ITEM/OPT_FILENAME_RENAME_MDATE}" printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM/OPT_FILENAME_RENAME_MDATE}" fi if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)OPT_FILENAME_RENAME_TITLETAG.*$ ]]; then POST_WGET_RENAME_TITLETAG=1 printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM/OPT_FILENAME_RENAME_TITLETAG}" fi if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)OPT_FILENAME_RENAME_REVTITLETAG.*$ ]]; then POST_WGET_RENAME_REVTITLETAG=1 printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM/OPT_FILENAME_RENAME_REVTITLETAG}" fi if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)ATOM_FILTER_SIMPLE.*$ ]]; then ATOM_FILTER_SIMPLE=1 # eval "${FILTER_ITEM}=\${!FILTER_ITEM/ATOM_FILTER_SIMPLE}" printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM/ATOM_FILTER_SIMPLE}" fi if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)(ATOM_FILTER_TYPE=\"?)([^\" ]+).*$ ]]; then ATOM_FILTER_TYPE="${BASH_REMATCH[3]}" # eval "${FILTER_ITEM}=\${!FILTER_ITEM/ATOM_FILTER_TYPE=+([^[:space:]])}" printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM/ATOM_FILTER_TYPE=+([^[:space:]])}" fi if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)(ATOM_FILTER_LANG=\"?)([^\" ]+).*$ ]]; then ATOM_FILTER_LANG="${BASH_REMATCH[3]}" # eval "${FILTER_ITEM}=\${!FILTER_ITEM/ATOM_FILTER_LANG=+([^[:space:]])}" printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM/ATOM_FILTER_LANG=+([^[:space:]])}" fi # Remove repeated spaces # eval "${FILTER_ITEM}=\$(echo \"${!FILTER_ITEM}\" | tr -s ' ')" printf -v "${FILTER_ITEM}" '%s' "$(tr -s ' ' <<< "${!FILTER_ITEM}")" # Remove any residual leading whitespace from ${!FILTER_ITEM} # eval "${FILTER_ITEM}=\${!FILTER_ITEM#\${!FILTER_ITEM%%[![:space:]]*}}" printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM#"${!FILTER_ITEM%%[![:space:]]*}"}" # Remove any residual trailing whitespace from ${!FILTER_ITEM} # eval "${FILTER_ITEM}=\${!FILTER_ITEM%\${!FILTER_ITEM##*[![:space:]]}}" printf -v "${FILTER_ITEM}" '%s' "${!FILTER_ITEM%"${!FILTER_ITEM##*[![:space:]]}"}" # No_Category, or blank item handling. if [[ "${!FILTER_ITEM}" =~ ^(.+[[:space:]]|[[:space:]]?)No_Category.*$ || -z "${!FILTER_ITEM-empty}" ]]; then # Set to single period which is a bashism for current directory. # eval "${FILTER_ITEM}=." printf -v "${FILTER_ITEM}" '%s' '.' fi if [[ "${!FILTER_ITEM}" =~ ^.*%(YY|MM|DD)%.*$ ]]; then # eval "${FILTER_ITEM}=$(echo "${!FILTER_ITEM}" | ${SED} -e "s#%YY%#$(date +%Y)#" -e "s#%MM%#$(date +%m)#" -e "s#%DD%#$(date +%d)#" )" printf -v "${FILTER_ITEM}" '%s' "$(sed -e "s#%YY%#$(date +%Y)#" -e "s#%MM%#$(date +%m)#" -e "s#%DD%#$(date +%d)#" <<< "${!FILTER_ITEM}")" fi } PLAYLIST_ConvertToASX() { local DIR_LIBRARY=${1} local M3U_PLAYLISTNAME=${2} ASX_LOCATION="\\SD Card\\POD\\" ASX_PLAYLISTNAME=$(basename "${DIR_LIBRARY}"/"${M3U_PLAYLISTNAME}" .m3u).asx sed --silent -e '/TEXT_ASX_BEGINNING$/,/^TEXT_ASX_BEGINNING/p' "$0" | sed -e '/TEXT_ASX_BEGINNING/d' > "${DIR_LIBRARY}"/"${ASX_PLAYLISTNAME}" while read -r line ; do # local FIXED_ENTRY=$(echo "${line}" | sed 's/\//\\/g') local FIXED_ENTRY="${line//\//\\}" { echo ' ' echo " " echo " " echo ' ' } >> "${DIR_LIBRARY}"/"${ASX_PLAYLISTNAME}" done < "${DIR_LIBRARY}"/"${M3U_PLAYLISTNAME}" sed --silent -e '/TEXT_ASX_END$/,/^TEXT_ASX_END/p' "$0" | sed -e '/TEXT_ASX_END/d' >> "${DIR_LIBRARY}"/"${ASX_PLAYLISTNAME}" # Removing unix2dos dependency. Converting to sed statement with in-place editing of the file in question. # ctrl-v ctrl-m for windows line end. sed -i 's/$/ /' "${DIR_LIBRARY}"/"${ASX_PLAYLISTNAME}" } PLAYLIST_Sort() { local DIR_LIBRARY=${1} local M3U_PLAYLISTNAME=${2} local REALPLAYLISTNAME="${DIR_LIBRARY}/$M3U_PLAYLISTNAME" # Sort Playlist unset -v TEMPPLAYLISTNAME local TEMPPLAYLISTNAME if hash mktemp >&2; then TEMPPLAYLISTNAME=$(mktemp 2>/dev/null) elif hash tempfile >&2; then # NOTE: The shellcheck warning for tempfile being depreciated is # disabled here because we are only using it for systems that do not # already have mktemp. So realistically, it should almost never be called. # shellcheck disable=SC2186 TEMPPLAYLISTNAME=$(tempfile 2>/dev/null) else echo "Error: Neither mktemp or tempfile found. Unable to sort playlist." unset -v REALPLAYLISTNAME TEMPPLAYLISTNAME return fi cp -p "$REALPLAYLISTNAME" "$TEMPPLAYLISTNAME" && sort -o "$REALPLAYLISTNAME" "$TEMPPLAYLISTNAME" && rm "$TEMPPLAYLISTNAME" unset -v REALPLAYLISTNAME TEMPPLAYLISTNAME } # Function: REMOVE_URL # Simple function to cleanup URLs for removal from FILE. # Arguments: # ${1} == URL to remove from file # ${2} == FILE to remove URL from REMOVE_URL() { local URL_INPUT=${1} local TARGET_FILE=${2} # Characters unlikely to appear in an URL that are suitable for use as delimiters in SED statements. # List has been pruned down to eliminate any characters that need to be escaped themselves. # Listed as "Unsafe" in RFC 1738 # Source: https://www.ietf.org/rfc/rfc1738.txt local TEST_STRING="#|^~<>[] " # Find suitable TEST_CHAR to use for sed statements below. for (( i=0; i<${#TEST_STRING}; i++ )); do local TEST_CHAR=${TEST_STRING:$i:1} # grep looks for --fixed-strings so that certain characters are not # interpreted as regular expressions (like ^ $ or /) # # We're looking for a character NOT found in the string so its ! grep if ! grep --quiet --fixed-strings "${TEST_CHAR}" <<<"${URL_INPUT}"; then # When a character is not found in the string that we can use, break out of the loop. break fi done unset -v i # Both sed statements below use TEST_CHAR as their delimiter # Escape any characters in URL that can affect the sed command below, currently: * local URL_CLEAN # URL_CLEAN as it was before trying to correct implicit escaping. # URL_CLEAN=$(echo "${URL_INPUT}" | sed -e "s${TEST_CHAR}\([^\\]\)\*${TEST_CHAR}\1\\\*${TEST_CHAR}g") # shellcheck disable=SC2001 URL_CLEAN=$(echo "${URL_INPUT}" | sed -e "s${TEST_CHAR}\\([^\\]\\)\\*${TEST_CHAR}\\1\\\\*${TEST_CHAR}g") sed -i "\\${TEST_CHAR}${URL_CLEAN}${TEST_CHAR}d" "${TARGET_FILE}" } # The commands of this function will trigger Shellcheck SC2317 (command appears to be unreachable) # because I never actually use this function but keep it here for historical reasons. # shellcheck disable=SC2317 COMPARE_StringInString() { echo "Enter Compare ..." echo "1 == ${1}" echo "2 == ${2}" case "${2}" in *"${1}" ) echo "Found!" return 0 ;; esac return 1 } # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Version (Update with changes!) {{{ VERSION=0.9.3 REPORT_VERSION=0 # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Portability code {{{ # This allows Gnu/Linux podget to also run on Mac OSX uname=$(uname) case "$uname" in "Linux") :;; "Darwin") DARWIN_EXIT=0 if ! hash gsed >/dev/null 2>&1; then echo "GNU Sed Required." echo "Try 'brew install gnu-sed' (see https://brew.sh for details)" DARWIN_EXIT=1 else hash -p "$(hash -t gsed)" sed fi if ! hash gexpr >/dev/null 2>&1; then echo "GNU Expr Required." echo "Try 'brew install coreutils' (see https://brew.sh for details)" DARWIN_EXIT=1 else hash -p "$(hash -t gexpr)" expr fi if ! hash gtr >/dev/null 2>&1; then echo "GNU tr Required." echo "Try 'brew install coreutils' (see https://brew.sh for details)" DARWIN_EXIT=1 else hash -p "$(hash -t gtr)" tr fi if ! hash gdate >/dev/null 2>&1; then echo "GNU Date Required." echo "Try 'brew install gdate' (see https://brew.sh for details)" DARWIN_EXIT=1 else hash -p "$(hash -t gdate)" date fi if ! hash gstat >/dev/null 2>&1; then echo "GNU Stat Required." echo "Try 'brew install gstat' (see https://brew.sh for details)" DARWIN_EXIT=1 else hash -p "$(hash -t gstat)" stat fi if (( DARWIN_EXIT > 0 )); then CLEANUP_AND_EXIT 1 fi unset -v DARWIN_EXIT ;; "OpenBSD"|"NetBSD"|"FreeBSD") missing_counter=0 needed_commands=( gsed gexpr gdate gtr ) if [[ $uname == FreeBSD ]]; then needed_commands+=(gnustat) else needed_commands+=(gstat) fi for needed_command in "${needed_commands[@]}"; do if ! hash "$needed_command" >/dev/null 2>&1; then printf "Command not found in PATH: %s\n" "$needed_command" >&2 ((++missing_counter)) else hash -p "$(hash -t "$needed_command")" "${needed_command##@(g|gnu)}" fi done if (( missing_counter > 0 )); then CLEANUP_AND_EXIT 1 fi unset -v needed_command{,s} missing_counter ;; esac # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Parse command line {{{ # Set defaults for command line options so variables do not conflict with 'set -o nounset' by being undeclared. CMDL_CLEANUP_SIMULATE=0 CMDL_FORCE=0 while (( $# >= 1 )); do case ${1} in -c | --config ) CONFIG_CORE=${2:-NONE} ; shift ; shift || : ;; --create-config ) CONFIG_CORE=${2:-NONE}; CMDL_CREATECONFIG=1 ; shift ; shift || : ;; -C | --cleanup ) CLEANUP_ONLY=1 ; CLEANUP=1 ; shift ;; --cleanup_days ) CMDL_CLEANUP_DAYS=${2:-NONE} ; shift ; shift || : ;; --cleanup_simulate ) CMDL_CLEANUP_SIMULATE=1 ; CLEANUP_ONLY=1 ; CLEANUP=1 ; shift ;; -d | --dir_config ) DIR_CONFIG=${2:-NONE} ; shift ; shift || : ;; --dir_session ) CMDL_SESSION=${2:-NONE} ; shift ; shift || : ;; -f | --force ) CMDL_FORCE=1 ; shift ;; --import_opml ) IMPORT_OPML=${2:-NONE} ; shift ; shift || : ;; --export_opml ) EXPORT_OPML=${2:-NONE} ; shift ; shift || : ;; --import_pcast ) IMPORT_PCAST=${2:-NONE} ; shift ; shift || : ;; -l | --library ) CMDL_LIBRARY=${2:-NONE} ; shift ; shift || : ;; -n | --no-playlist ) CMDL_NOPLAYLIST=1 ; shift ;; -p | --playlist-asx ) CMDL_ASX=1 ; shift ;; --playlist-per-podcast ) CMDL_PLAYLISTPERPODCAST=1 ; shift ;; -r | --recent ) CMDL_MOSTRECENT=${2:-NONE} ; shift ; shift || : ;; --serverlist ) CMDL_SERVERLIST=${2:-NONE} ; shift ; shift || : ;; -s | --silent ) VERBOSITY=0 ; shift ;; -V | --version ) VERBOSITY=2 ; REPORT_VERSION=1 ; shift ;; -v ) VERBOSITY=1 ; shift ;; -vv ) VERBOSITY=2 ; shift ;; -vvv ) VERBOSITY=3 ; shift ;; -vvvv ) VERBOSITY=4 ; shift ;; --verbosity ) VERBOSITY=${2:-NONE} ; shift ; shift || : ;; * ) display_shelp ; CLEANUP_AND_EXIT ${ERR_DISPLAYHELP} ;; esac done if [[ -n ${VERBOSITY+set} ]] ; then if [[ -z ${VERBOSITY##*[!0-9]*} ]]; then echo "Verbosity is not a supported integer value" exit 1 fi fi if [[ -n ${CMDL_SERVERLIST+set} ]] ; then if [[ ${CMDL_SERVERLIST} == "NONE" ]]; then echo "Unset filename for server list" CLEANUP_AND_EXIT 1 fi CONFIG_SERVERLIST=${CMDL_SERVERLIST} fi if (( VERBOSITY >= 2 )) ; then echo "podget" echo fi if (( REPORT_VERSION == 1 )); then echo "Version: ${VERSION}" CLEANUP_AND_EXIT 0 fi if [[ ${CONFIG_CORE} == "NONE" ]]; then echo "Unset filename for configuration" CLEANUP_AND_EXIT 1 fi if [[ ${DIR_CONFIG} == "NONE" ]]; then echo "Unset directory to store configuration" CLEANUP_AND_EXIT 1 fi # If DEBUG is manual set to 0 or 1 in podgetrc, then it does not have an effect # until CONFIG_CORE is read at about line 1478. On the other hand, if it is # not configured to a fixed value there, then it can be enabled from the # commandline by executing podget like so "DEBUG=1 ./podget [options]". By # default, DEBUG is set to be disabled (0). if (( DEBUG == 1 )) ; then printf '\n%s %-30s %s\n' "${DEBUG_LEADER}" "Parsing Config file." "" printf '%s %-30s %s\n' "${DEBUG_LEADER}" "Config directory:" "${DIR_CONFIG}" printf '%s %-30s %s\n' "${DEBUG_LEADER}" "Config file:" "${CONFIG_CORE}" printf '%s %-30s %s\n' "${DEBUG_LEADER}" "Server List:" "${CONFIG_SERVERLIST}" fi if [[ -n ${CMDL_NOPLAYLIST+set} ]]; then if [[ -n ${CMDL_ASX+set} ]]; then printf '%-30s %s' "Error:" "Conflicting playlist options." CLEANUP_AND_EXIT 1 fi fi # for testing #echo "Verbosity: ${VERBOSITY}" #CLEANUP_AND_EXIT 0 # Test filename for CONFIG_CORE, CMDL/CONFIG_SERVERLIST and directory for DIR_CONFIG if (( DEBUG == 1 )) ; then printf '\n%s %s\n' "${DEBUG_LEADER}" "Loading temporary FILENAME_BADCHARS to test base configuration file and directory names." fi FILENAME_CHECK CONFIG_CORE if [[ -n ${CMDL_SERVERLIST+set} ]] ; then FILENAME_CHECK CMDL_SERVERLIST else FILENAME_CHECK CONFIG_SERVERLIST fi DIRECTORY_CHECK DIR_CONFIG if (( DEBUG == 1 )) ; then printf '%s %s\n\n' "${DEBUG_LEADER}" "Clearing temporary FILENAME_BADCHARS, will read configured version from ${CONFIG_CORE}" echo fi unset -v FILENAME_BADCHARS # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # CONFIG TEST PART 1 of 4: Test for existing DIR_CONFIG, if missing create it and install CONFIG_CORE {{{ #if [[ ! -d ${DIR_CONFIG} ]] ; then # echo " Configuration directory not found. Creating \"${DIR_CONFIG}\"" # mkdir "${DIR_CONFIG}" # EXITSTATUS=$? # if (( EXITSTATUS != 0 )); then # printf '%-30s %s' "Error:" "Failed to create \"${DIR_CONFIG}\"" # CLEANUP_AND_EXIT 1 # fi # INSTALL_SESSION=1 #fi if [[ "${DIR_CONFIG}" == "UNSET-use-DEFAULT" ]]; then if (( DEBUG == 1 )) ; then printf '%s %s\n\n' "${DEBUG_LEADER}" "DIR_CONFIG is not configured, checking for other possible locations" echo fi DIR_CONFIG_NAME="podget" if [[ -d "${HOME}/.${DIR_CONFIG_NAME}" ]]; then if (( DEBUG == 1 )) ; then printf '%s %s\n\n' "${DEBUG_LEADER}" "HOME/.${DIR_CONFIG_NAME} exists!" echo fi DIR_CONFIG="${HOME}/.${DIR_CONFIG_NAME}" elif [[ -n ${XDG_CONFIG_HOME+set} && -d "${XDG_CONFIG_HOME}/${DIR_CONFIG_NAME}" ]]; then if (( DEBUG == 1 )) ; then printf '%s %s\n\n' "${DEBUG_LEADER}" "XDG_CONFIG_HOME is set and '${DIR_CONFIG_NAME}' exists within it" echo fi DIR_CONFIG="${XDG_CONFIG_HOME}/${DIR_CONFIG_NAME}" elif [[ -d "${HOME}/.config" && -d "${HOME}/.config/${DIR_CONFIG_NAME}" ]]; then if (( DEBUG == 1 )) ; then printf '%s %s\n\n' "${DEBUG_LEADER}" "HOME/.config exists and '${DIR_CONFIG_NAME}' exists within it" echo fi DIR_CONFIG="${HOME}/.config/${DIR_CONFIG_NAME}" else if (( DEBUG == 1 )) ; then printf '%s %s\n\n' "${DEBUG_LEADER}" "" echo fi if (( DEBUG == 1 )) ; then printf '%s %s\n\n' "${DEBUG_LEADER}" "'${DIR_CONFIG_NAME}' directory does not exist in any of the likely locations." echo fi if [[ -n ${XDG_CONFIG_HOME+set} ]]; then if [[ -d "${XDG_CONFIG_HOME}" ]]; then mkdir "${XDG_CONFIG_HOME}/${DIR_CONFIG_NAME}" EXITSTATUS=$? if (( EXITSTATUS != 0 )); then printf '%-30s %s' "Error:" "Error creating directory: ${XDG_CONFIG_HOME}/${DIR_CONFIG_NAME}" CLEANUP_AND_EXIT 1 fi DIR_CONFIG="${XDG_CONFIG_HOME}/${DIR_CONFIG_NAME}" else printf '%-30s %s' "Error:" "Error: XDG_CONFIG_HOME directory does not exist ('${XDG_CONFIG_HOME}')" CLEANUP_AND_EXIT 1 fi else if [[ ! -d "${HOME}/.config/${DIR_CONFIG_NAME}" ]]; then mkdir -p "${HOME}/.config/${DIR_CONFIG_NAME}" EXITSTATUS=$? if (( EXITSTATUS != 0 )); then printf '%-30s %s' "Error:" "Error creating directory: ${HOME}/.config/${DIR_CONFIG_NAME}" CLEANUP_AND_EXIT 1 fi DIR_CONFIG="${HOME}/.config/${DIR_CONFIG_NAME}" fi fi INSTALL_SESSION=1 fi else if [[ ! -d "${DIR_CONFIG}" ]]; then mkdir -p "${DIR_CONFIG}" EXITSTATUS=$? if (( EXITSTATUS != 0 )); then printf '%-30s %s' "Error:" "Error creating directory: ${DIR_CONFIG}" CLEANUP_AND_EXIT 1 fi INSTALL_SESSION=1 fi fi # Exit if set to --create-config if [[ -n ${CMDL_CREATECONFIG+set} ]]; then if (( CMDL_CREATECONFIG == 1 )); then if [[ -f "${DIR_CONFIG}/${CONFIG_CORE}" ]]; then echo " Configuration file ${DIR_CONFIG}/${CONFIG_CORE} already exists." echo " If you would like to reuse this name, then the old file needs" echo " to be deleted first. Or you need to pick a new name." CLEANUP_AND_EXIT 1 fi fi fi # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # CONFIG TEST PART 2 of 4: Test for existing CONFIG_CORE, if missing create it. {{{ if [[ ! -f "${DIR_CONFIG}/${CONFIG_CORE}" ]] ; then echo " Installing default user configuration file in ${DIR_CONFIG}/${CONFIG_CORE}" sed --silent -e '/TEXT_DEFAULT_CONFIG$/,/^TEXT_DEFAULT_CONFIG/p' "$0" | sed -e '/TEXT_DEFAULT_CONFIG/d' | sed -e "s|@HOME@|${HOME}|" -e "s/@VERSION@/${VERSION}/" -e "s/@SERVERLIST@/${CONFIG_SERVERLIST}/"> "${DIR_CONFIG}"/"${CONFIG_CORE}" EXITSTATUS=$? if (( EXITSTATUS != 0 )); then printf '%-30s %s' "Error:" "Failed to create \"${DIR_CONFIG}/${CONFIG_CORE}\"" CLEANUP_AND_EXIT 1 fi INSTALL_SESSION=1 fi # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # CONFIG TEST PART 3 of 4: Test if configuration file was created by a version that supports all required items and formats. {{{ if ! grep -F "Podget configuration file created by version" "${DIR_CONFIG}"/"${CONFIG_CORE}" >/dev/null ; then echo "${DIR_CONFIG}/${CONFIG_CORE} cannot be verified to be compatible with this version of podget." echo echo "It is missing the version line that is included in configuration files created by newer versions of podget." echo echo "Please create a new configuration file by running 'podget --create-config '," echo "and then converting your old configuration to the new format. Then move the new file" echo "in place of the old and podget will work as it used to." CLEANUP_AND_EXIT 1 else # get version VERSION=$(grep -F "Podget configuration file created by version" "${DIR_CONFIG}"/"${CONFIG_CORE}" | sed -e 's/.*by version \([0-9.]\)/\1/') # Split version string into an array by replacing '.' with 'space' IFS='.' read -r -a VERSION_ARRAY <<< "${VERSION}" # Assign values from array to named variables with a default of '0' if unset. # NOTE: BASE is currently commented out because it is currently unused. It is here if we need it. # BASE="${VERSION_ARRAY[0]:-0}" MAJOR="${VERSION_ARRAY[1]:-0}" MINOR="${VERSION_ARRAY[2]:-0}" if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} PODGETRC Version Check: (looking for newer than 0.6.18)" echo "${DEBUG_LEADER} BASE: ${BASE:-0}" echo "${DEBUG_LEADER} MAJOR: ${MAJOR}" echo "${DEBUG_LEADER} MINOR: ${MINOR}" fi if (( (MAJOR < 7) && (MINOR < 19) )); then echo "${DIR_CONFIG}/${CONFIG_CORE} was created by an older version of podget than is needed by this version." echo echo "This version of podget requires a configuration produced for version 0.7.0 or newer. This configuration" echo "was produced by version ${VERSION}" echo echo "Please update your configuration by creating a new one with the command 'podget --create-config '" echo "and then comparing its contents to your old configuration. Once the new file has been updated to reflect" echo "your desired configuration, move it in place of the old one." CLEANUP_AND_EXIT 1 fi fi # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Read core configuration options {{{ # SHELLCHECK SC1090 # shellcheck source=/dev/null source "${DIR_CONFIG}"/"${CONFIG_CORE}" # Config adjustment: Restore serverlist setting from command line if necessary if [[ -n ${CMDL_SERVERLIST+set} ]]; then if (( VERBOSITY >= 1 )) ; then echo "Serverlist set on command line, overriding value from configuration file." fi CONFIG_SERVERLIST=${CMDL_SERVERLIST} fi # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # CONFIG TEST PART 4 of 4: Test for existing DIR_SERVERLIST, if missing create it. {{{ # Note: This test needs to happen after CONFIG_CORE has been read, as it may specify a different serverlist name than the default. if [[ ! -f "${DIR_CONFIG}/${CONFIG_SERVERLIST}" ]] ; then echo " Installing default server list configuration." sed --silent -e '/TEXT_DEFAULT_SERVERLIST$/,/^TEXT_DEFAULT_SERVERLIST/p' "$0" | sed -e '/TEXT_DEFAULT_SERVERLIST/d' > "${DIR_CONFIG}"/"${CONFIG_SERVERLIST}" EXITSTATUS=$? if (( EXITSTATUS != 0 )); then echo " Failed to install \"${DIR_CONFIG}/${CONFIG_SERVERLIST}\"" CLEANUP_AND_EXIT 1 fi INSTALL_SESSION=1 fi # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Exit if set to --create-config {{{ if [[ -n ${CMDL_CREATECONFIG+set} ]]; then if (( CMDL_CREATECONFIG == 1 )); then CLEANUP_AND_EXIT 0 fi fi # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Configuration adjustments for command line options {{{ if [[ -n ${CMDL_NOPLAYLIST+set} ]]; then if (( VERBOSITY >= 1 )) ; then printf '\t\t%s\n' "NO PLAYLIST set on command line, overriding value from configuration file." fi NO_PLAYLIST=${CMDL_NOPLAYLIST} fi if (( INSTALL_SESSION > 0 )); then echo " Downloading a single item from each default server to test configuration." echo MOST_RECENT=1 VERBOSITY=3 fi if [[ -n ${CMDL_LIBRARY+set} ]] ; then if [[ ${CMDL_LIBRARY} == "NONE" ]]; then echo "Unset directory to store podcast library" CLEANUP_AND_EXIT 1 fi if (( VERBOSITY >= 3 )) ; then printf '\t\t%s\n' "Overriding Library Directory specified in configuration file." fi DIR_LIBRARY=${CMDL_LIBRARY} fi if (( VERBOSITY >= 3 )) ; then printf '%-30s %s\n' "Library Directory:" "${DIR_LIBRARY}" fi if [[ -z ${DIR_LIBRARY} ]] ; then echo "ERROR - Library directory not defined." 1>&2 CLEANUP_AND_EXIT ${ERR_LIBNOTDEF} else DIRECTORY_CHECK DIR_LIBRARY fi if [[ -n ${CMDL_SESSION+set} ]] ; then if [[ ${CMDL_SESSION} == "NONE" ]]; then echo "Unset directory to store session files." CLEANUP_AND_EXIT 1 fi DIR_SESSION=${CMDL_SESSION} fi # Added so old configuration files (that were created before this option was added) still work. if [[ -z ${DIR_SESSION+set} ]] ; then if [[ -n ${TMPDIR+set} ]]; then DIR_SESSION=${TMPDIR}/podget else DIR_SESSION=/tmp/podget fi fi if [[ -n ${DIR_SESSION+set} ]]; then DIRECTORY_CHECK DIR_SESSION fi if (( VERBOSITY >= 3 )) ; then printf '%-30s %s\n' "Session Directory:" "${DIR_SESSION}" fi if [[ -n ${CMDL_CLEANUP_DAYS+set} ]] ; then if [[ -z ${CMDL_CLEANUP_DAYS##*[!0-9]*} ]]; then echo "Cleanup Days is not a positive integer value" CLEANUP_AND_EXIT 1 fi if (( CMDL_CLEANUP_DAYS >= 0 )); then CLEANUP_DAYS=${CMDL_CLEANUP_DAYS} fi fi if [[ -n ${CMDL_CLEANUP_SIMULATE} ]] ; then CLEANUP_SIMULATE=${CMDL_CLEANUP_SIMULATE} fi if [[ -z ${DIR_LOG+set} ]] ; then DIR_LOG=${DIR_LIBRARY}/.LOG fi DIRECTORY_CHECK DIR_LOG if (( VERBOSITY >= 3 )) ; then printf '%-30s %s\n' "Log Directory:" "${DIR_LOG}" fi if [[ -n ${CMDL_ASX+set} ]]; then ASX_PLAYLIST=${CMDL_ASX} fi if [[ -n ${CMDL_PLAYLISTPERPODCAST+set} ]]; then PLAYLIST_PERPODCAST=1 fi if (( CMDL_FORCE != 0 )); then FORCE=${CMDL_FORCE} # WGET_BASEOPTS=$(echo "${WGET_BASEOPTS}" | sed -e 's/-c //') # Change to ${variable/search/replace} format. WGET_BASEOPTS="${WGET_BASEOPTS/-c /}" fi if [[ -n ${CMDL_MOSTRECENT+set} ]] ; then if [[ -z ${CMDL_MOSTRECENT##*[!0-9]*} ]]; then echo echo "Recent is not a positive integer value" CLEANUP_AND_EXIT 1 fi if (( CMDL_MOSTRECENT != 0 )); then MOST_RECENT=${CMDL_MOSTRECENT} fi fi if (( VERBOSITY <= 1 )) ; then WGET_COMMON_OPTIONS="-q ${WGET_BASEOPTS}" elif (( VERBOSITY == 2 )) ; then WGET_COMMON_OPTIONS="-nv ${WGET_BASEOPTS}" elif (( VERBOSITY == 3 )) ; then WGET_COMMON_OPTIONS="${WGET_BASEOPTS} --progress=dot:mega" else WGET_COMMON_OPTIONS="${WGET_BASEOPTS} --progress=bar" fi # Remove any residual leading spaces from WGET_COMMON_OPTIONS WGET_COMMON_OPTIONS="${WGET_COMMON_OPTIONS#[ ]*}" if (( DEBUG == 1 )) ; then # echo -e "WGet Options:\t\t\t${WGET_COMMON_OPTIONS}" printf '%-30s %s\n' "WGet Options:" "${WGET_COMMON_OPTIONS}" fi if [[ -n ${FILENAME_BADCHARS+set} ]] ; then # make sure backslash is escaped. FILENAME_BADCHARS="${FILENAME_BADCHARS/\\/\\\\}" if (( DEBUG == 1 )) ; then printf '%-30s %s\n' "Filename Bad Characters:" "${FILENAME_BADCHARS}" printf '%-30s %s\n' "Filename Replace Character:" "${FILENAME_REPLACECHAR}" fi fi if (( VERBOSITY >= 3 )) ; then if (( DEBUG == 0 )); then printf '%-30s %s\n' "Debug:" "Disabled - Delete temp files and reduced progress messages." else printf '%-30s %s\n' "Debug:" "Enabled - Do not delete temp files and increased progress messages." fi fi if (( VERBOSITY >= 1 )) ; then echo fi # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Test for another session. {{{ # Moved below the configuration reading so that the DIR_SESSION could be configurable. # Test that session directory exists (useful when placed on a tmpfs filesystem). if [[ ! -d ${DIR_SESSION} ]]; then if (( DEBUG == 1 )) ; then printf '\n%s\n' "Session directory not found, creating" fi mkdir -p "${DIR_SESSION}" fi # The cleanup portion of the following procedure may be less needed because we have gotten a lot more complete in our cleanup # procedures with traps that mean that the session file should be deleted whenever the session exits (whether good or # bad). This procedures main purpose is to prevent concurrent sessions for running on the same configuration file. TEST_SESSION=0 while read -r -d $'\0' FILE ; do if [[ $(stat -c %u "${FILE}") == "${UID}" ]]; then if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Session FILE found: ${FILE} (that my user owns)" fi TEST_SESSION=1 # shellcheck disable=SC2094 while read -r LINE ; do TEST_Index=$(expr "${LINE}" : "^[Cc]onfig\\sfile:") if [[ "A${TEST_Index}" != "A" ]]; then TEST_File=$(echo "${LINE}" | sed -n -e 's/^[^:]\+:\s\(.*\)$/\1/p') if [[ ${TEST_File} == "${CONFIG_CORE}" ]] ; then SESSION_PID=$(echo "${FILE}" | sed -n -e 's/.*podget.\([0-9]*\)$/\1/p') if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Testing PID ${SESSION_PID} to determine if its still running." fi if ps --pid "${SESSION_PID}" &>/dev/null; then echo "Another session with config file ${CONFIG_CORE} found running. Killing session." 1>&2 CLEANUP_AND_EXIT ${ERR_RUNNINGSESSION} else if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Session PID ${SESSION_PID} is not running, removing lock file" fi rm -f "${FILE}" fi fi fi done < "${FILE}" fi done < <(find "${DIR_SESSION}" -maxdepth 1 -type f -name "podget.[0-9]*" -print0) if (( VERBOSITY >= 2 )) ; then if (( DEBUG == 1 )); then echo; fi if (( TEST_SESSION > 0 )) ; then echo "Old Session file(s) found and removed. Creating new one." else echo "Session file not found. Creating podget.$$" fi fi # Set session file. Stores the configuration file used so that multiple sessions can be run per user simultaneously as # long as they each have different configuration files. # An example would be two sessions running as: # podget -c podgetrc.1 # podget -c podgetrc.2 echo -e "Config file: ${CONFIG_CORE}" > "${DIR_SESSION}"/podget.$$ # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Loop over servers on list {{{ if (( CLEANUP_ONLY == 0 )) && [[ -z ${IMPORT_OPML+set} && -z ${EXPORT_OPML+set} && -z ${IMPORT_PCAST+set} ]] ; then if (( DEBUG == 1 )) ; then printf '\n%s %s\n' "${DEBUG_LEADER}" "Main loop:" printf '%s %-30s %s\n' "${DEBUG_LEADER}" "SERVER LIST FILE:" "$CONFIG_SERVERLIST" printf '%s %-30s %s\n' "${DEBUG_LEADER}" "WGET OPTIONS:" "${WGET_COMMON_OPTIONS}" if (( NO_PLAYLIST == 0 )); then printf '%s %s\n' "${DEBUG_LEADER}" "Playlist Creation Enabled." else printf '%s %s\n' "${DEBUG_LEADER}" "Playlist Creation Disabled." fi if [[ -n ${PLAYLIST_PERPODCAST+set} ]]; then printf '%s %s\n' "${DEBUG_LEADER}" "PLAYLIST_PERPODCAST: ${PLAYLIST_PERPODCAST} (Enabled)" fi fi mkdir -p "${DIR_LOG}" FILENAME_CHECK LOG_FAIL FILENAME_CHECK LOG_COMPLETE touch "${DIR_LOG}"/"${LOG_FAIL}" "${DIR_LOG}"/"${LOG_COMPLETE}" if (( NO_PLAYLIST == 0 )) && [[ -z ${PLAYLIST_PERPODCAST+set} ]]; then SESSION_PLAYLIST_NAME="${PLAYLIST_NAMEBASE:=NEW-}$(date $DATE_FORMAT).m3u" COUNTER=2 while [[ -e ${DIR_LIBRARY}/${SESSION_PLAYLIST_NAME} ]] ; do SESSION_PLAYLIST_NAME="${PLAYLIST_NAMEBASE:=NEW-}$(date $DATE_FORMAT).r$COUNTER.m3u" COUNTER=$((COUNTER+1)) done if (( VERBOSITY >= 3 )); then echo echo "Playlist name: ${SESSION_PLAYLIST_NAME}" fi fi # UTF-8/16 handling for FILETYPE in utf8 utf16 ; do if (( DEBUG == 1 )) ; then echo case ${FILETYPE} in 'utf8') echo "${DEBUG_LEADER} UTF-8 Loop running." ;; 'utf16') echo "${DEBUG_LEADER} UTF-16 Loop running." ;; esac fi case ${FILETYPE} in 'utf8') CURRENT_SERVERLIST="${DIR_CONFIG}"/"${CONFIG_SERVERLIST}" ;; 'utf16') # Test if iconv is installed. If it is not, display error. if ! hash iconv >/dev/null 2>&1; then echo "Can't find iconv binary, is libc-bin installed?" 1>&2 echo "Exiting UTF16 loop, unable to convert file to UTF8" 1>&2 CLEANUP_AND_EXIT $ERR_LIBC6NOTINSTALLED fi CURRENT_SERVERLIST="${DIR_CONFIG}/${CONFIG_SERVERLIST}.utf16" ;; *) echo "Unknown Filetype: ${FILETYPE}" esac if [[ ! -f ${CURRENT_SERVERLIST} ]]; then if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} No config file found, exiting loop." fi continue fi while read -r FEED_URL FEED_CATEGORY FEED_NAME ; do if (( DEBUG == 1 )) ; then echo echo "${DEBUG_LEADER} ============================================================" echo "${DEBUG_LEADER} Server Loop:" echo "${DEBUG_LEADER} Serverlist Line --> ${FEED_URL:-} ${FEED_CATEGORY:-} ${FEED_NAME:-}" fi if [[ "${FEED_URL:0:1}" == "#" || -z "${FEED_URL-empty}" ]] ; then if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Discarding (comment or blank line)." echo "${DEBUG_LEADER} Clear Server Loop vars" fi unset -v FEED_URL unset -v FEED_CATEGORY unset -v FEED_NAME unset -v FEED_FULL_PLAYLIST unset -v URL_USERNAME unset -v URL_PASSWORD continue fi if (( DEBUG == 1 )) ; then echo echo "${DEBUG_LEADER} Reset options to allow loop specific options" echo "${DEBUG_LEADER} WGET_COMMON_OPTIONS: ${WGET_COMMON_OPTIONS}" fi unset -v WGET_SUBCHAR WGET_NOTCOUNT=0 # In Vim to enter digraphs that are not commonly on the US keyboards, # we use ctrl-K followed by a two character lookup code. # Examples: # !2 - double vertical line ‖ # =2 - double low line ‗ # oo - bullet • # Db - diamond bullet ◆ # '% - Greek Theta ϴ # Pd - British Pound symbol £ # Eu - Euro Symbol € # Ye - Yen symbol ¥ # Pt - Peseta symbol ₧ # W= - Won symbol ₩ # Ct - Cent symbol ¢ # Co - Copyright symbol © # Rg - Registered Trademark ® # +- - Plus Minus ± # Om - Ohm symbol Ω # AO - Angstrom symbol Å # DG - Degree Sign ° # AE - Latin Capital AE Æ # ae - Latin Small ae æ # Ca - Caret ‸ # 1R - Roman Numeral One Ⅰ # 2R - Roman Numeral Two Ⅱ # 3R - Roman Numeral Three Ⅲ # 4R - Roman Numeral Four Ⅳ # 5R - Roman Numeral Five Ⅴ # # These are not good to use because they frequently overlap with following # characters if not followed by a space. # oC - Degree Celsius ℃ # oF - Degree Fahrenheit ℉ # # Digraphs may not be included in the locale that you have configured. On my test systems, # I found that I had one that did not display the correct characters for the digraphs # but rather replaced them all with an underscore character. The function worked but # it did not display the characters correctly. # # On the systems where it did display correctly, I found that I had the following # configuration in my /etc/default/locale: # # File generated by update-locale # LANG="en_US.UTF-8" # # On the one system that did not display correctly, the configuration I had # in /etc/default/locale was: # # File generated by update-locale # LANG="en_US" # # This can be updated by modifying the file or by running: # update-locale LANG="en_US.UTF-8" # # This may or may not require a reboot to take effect but I took the easy way out and # rebooted so I would be sure it displayed as I hoped in the future. # # NOTE ON CHARACTERS TO CONSIDER NOT INCLUDING: # It is probably a good idea to not include any character that has special meaning # in Bash and needs to be escaped. That is why this string primarily starts with # digraphs and then includes a few characters that are commonly found on most # keyboards. I initially wanted to include the $ symbol with the other monetary # symbols. However I had to escape it or it could cause an "unbound variable" # error on some systems. # # NOTE ON CHARACTERS TO CONSIDER INCLUDING: # We are using this character to parse the WGET_BASEOPTS string and put it's parts # into the WGET_OPTIONS array. To make things as easy as possible, we want to use # characters that are unlikely to be used in any of the WGET options. This can be a # bit tricky for some options like passwords or user-agent that can take almost # anything. For that reason, I have included a large selection of digraphs and a # few characters that do not require escaping because it is very unlikely that # someone used them all for one of those options and we will be able to safely # find one to use. WGET_TEST_STRING="_;‖‗•◆ϴ£€¥₧₩¢ⅠⅡⅢⅣⅤΩÅ©®±°Ææ‸:|<>?~!@#%^" for (( INTERVAL=0; INTERVAL<${#WGET_TEST_STRING}; INTERVAL++ )); do WGET_TEST_CHAR=${WGET_TEST_STRING:$INTERVAL:1} # grep looks for --fixed-strings so that certain characters are not # interpreted as regular expressions. # # we're looking for a character that is not found in the string so it is # ! grep if ! grep --quiet --fixed-strings "${WGET_TEST_CHAR}" <<<"${WGET_COMMON_OPTIONS}"; then WGET_SUBCHAR="${WGET_TEST_CHAR}" break else WGET_NOTCOUNT=$((WGET_NOTCOUNT+1)) fi done unset -v INTERVAL if [[ -n "${WGET_SUBCHAR+defined}" ]] && [[ "${#WGET_SUBCHAR}" -gt 0 ]]; then if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} WGET Success finding parsing character (${WGET_SUBCHAR})" echo "${DEBUG_LEADER} Test string length: ${#WGET_TEST_STRING}" echo "${DEBUG_LEADER} Characters tested before usable found: ${WGET_NOTCOUNT}" fi else echo echo "Unable to find character to use as delimiter for WGET_BASEOPTS string." echo "Every character in our test set was also used somewhere in WGET_BASEOPTS." echo "Test Set: ${WGET_TEST_STRING}" echo "WGET_BASEOPTS: ${WGET_BASEOPTS}" CLEANUP_AND_EXIT 1 fi WGET_COMMON_OPTIONS="${WGET_COMMON_OPTIONS// -/${WGET_SUBCHAR}-}" # clear array and reimport unset -v WGET_OPTIONS # Import options from WGET_COMMON_OPTIONS string into WGET_OPTIONS array # read -r -a WGET_OPTIONS <<<"${WGET_COMMON_OPTIONS}" IFS="${WGET_SUBCHAR}" read -r -a WGET_OPTIONS <<<"${WGET_COMMON_OPTIONS}" WGET_COMMON_OPTIONS="${WGET_COMMON_OPTIONS//${WGET_SUBCHAR}-/ -}" unset -v WGET_NOTCOUNT unset -v WGET_SUBCHAR unset -v WGET_TEST_CHAR unset -v WGET_TEST_STRING if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Reset FEED_FULL_PLAYLIST=0" fi FEED_FULL_PLAYLIST=0 if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Reset FEED_SORT_ORDER=DESCENDING" fi FEED_SORT_ORDER="DESCENDING" if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Reset WGET_OPTION_DEFAULT_USERAGENT" fi WGET_OPTION_DEFAULT_USERAGENT=0 if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Reset WGET_OPTION_FILENAME_LOCATION" fi WGET_OPTION_FILENAME_LOCATION=0 if ARRAY_CONTAINS "--content-disposition" "${WGET_OPTIONS[@]}" ; then # removing actual --content-disposition option from WGET because it is not used directly # but rather to set mode. Used for COMMON_OPTIONS to allow ease of use. for index in "${!WGET_OPTIONS[@]}"; do if [[ ${WGET_OPTIONS[${index}]} == "--content-disposition" ]]; then unset -v 'WGET_OPTIONS['"${index}"']' WGET_OPTION_DISPOSITION=1 if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Reset WGET_OPTION_DISPOSITION to global setting(${WGET_OPTION_DISPOSITION})" fi fi done else WGET_OPTION_DISPOSITION=0 if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Reset WGET_OPTION_DISPOSITION to global setting(${WGET_OPTION_DISPOSITION})" fi fi if ARRAY_CONTAINS "--user-agent=" "${WGET_OPTIONS[@]}" ; then if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Custom Wget User Agent Enabled." fi else WGET_OPTIONS+=("--user-agent=Podget") if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Default Wget User Agent for Podget Enabled." fi fi if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Reset WGET_OPTION_DISPOSITION_FAIL" fi WGET_OPTION_DISPOSITION_FAIL=0 if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Reset WGET_OPTION_NO_CHECK_CERIFICATE" fi WGET_OPTION_NO_CHECK_CERIFICATE=0 if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Reset WGET_OPTION_PREFER_IP_TYPE" fi WGET_OPTION_PREFER_IP_TYPE=0 if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Reset WGET_OPTION_RSS_MEDIACONTENT" fi WGET_OPTION_RSS_MEDIACONTENT=0 if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Reset POST_WGET_RENAME_MDATE" fi POST_WGET_RENAME_MDATE=0 if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Reset POST_WGET_RENAME_TITLETAG" fi POST_WGET_RENAME_TITLETAG=0 if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Reset POST_WGET_RENAME_REVTITLETAG" fi POST_WGET_RENAME_REVTITLETAG=0 if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Reset ATOM_FILTER_SIMPLE" fi ATOM_FILTER_SIMPLE=0 if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Reset ATOM_FILTER_TYPE" fi unset -v ATOM_FILTER_TYPE if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Reset ATOM_FILTER_LANG" fi unset -v ATOM_FILTER_LANG # End resets # ----------------------------------------------------------------- # Read new configuration # Pass FEED_CATEGORY and FEED_NAME through the option filters. This allows handling of options with blank feed # categories and names. unset -v FILTER_TARGET for FILTER_TARGET in FEED_CATEGORY FEED_NAME; do FILTER_OPTIONS "${FILTER_TARGET}" done # Unset options that are not required in this while loop # End setting of FEED_CATEGORY, FEED_NAME, and OPT_items # ----------------------------------------------------------------- # Add Username and Password to wget options if found. # Options --user and --password are used for both FTP and HTTP sessions. Using --ftp-user and --http-user would require # being able to selectively use the correct one based on the URL. Same for --ftp-password and --http-password. if [[ -n ${URL_USERNAME+set} ]]; then WGET_OPTIONS+=("--user=${URL_USERNAME}") fi if [[ -n ${URL_PASSWORD+set} ]]; then WGET_OPTIONS+=("--password=${URL_PASSWORD}") fi # Reset to default WGET user-agent if (( WGET_OPTION_DEFAULT_USERAGENT==1 )); then for index in "${!WGET_OPTIONS[@]}"; do if [[ ${WGET_OPTIONS[${index}]} =~ ^"--user-agent=".* ]]; then if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Reset to Default Wget User Agent" fi unset -v 'WGET_OPTIONS['"${index}"']' fi done fi # Add --no-check-certificate to WGET_OPTIONS if (( WGET_OPTION_NO_CHECK_CERIFICATE==1 )); then for index in "${!WGET_OPTIONS[@]}"; do # if --no-check-certificate has been set before, remove it. if [[ ${WGET_OPTIONS[${index}]} == "--no-check-certificate" ]]; then unset -v 'WGET_OPTIONS['"${index}"']' fi done WGET_OPTIONS+=("--no-check-certificate") fi # Set preferred IP type for WGET_OPTIONS if (( WGET_OPTION_PREFER_IP_TYPE>0 )); then for index in "${!WGET_OPTIONS[@]}"; do # if --prefer-family has been set before, remove it. if [[ ${WGET_OPTIONS[${index}]} =~ --prefer-family=IPv[46] ]]; then unset -v 'WGET_OPTIONS['"${index}"']' fi done case "${WGET_OPTION_PREFER_IP_TYPE}" in 4 ) WGET_OPTIONS+=("--prefer-family=IPv4") ;; 6 ) WGET_OPTIONS+=("--prefer-family=IPv6") ;; esac fi if (( VERBOSITY >= 2 )) ; then printf '\n%s\n' "-------------------------------------------------" fi if (( VERBOSITY >= 1 )) ; then if [ "${FEED_CATEGORY}" == "." ]; then DISPLAY_CATEGORY="None" else DISPLAY_CATEGORY="${FEED_CATEGORY}" fi if [ "${FEED_NAME}" == "." ]; then DISPLAY_NAME="None" else DISPLAY_NAME="${FEED_NAME}" fi printf '%-30s %-50s\n' "Category: ${DISPLAY_CATEGORY}" "Name: ${DISPLAY_NAME}" fi if (( DEBUG == 1 )) ; then printf '%s %-30s %s\n' "${DEBUG_LEADER}" "Loop WGET_OPTIONS:" "${WGET_OPTIONS[*]}" fi # DIRECTORY_CHECK for FEED_NAME and FEED_CATEGORY here prior to using them. for i in FEED_CATEGORY FEED_NAME; do # FEED_NAME can be blank, if it is then we skip the DIRECTORY_CHECK if [[ ${i} == "FEED_NAME" && -z "${!i}" ]]; then # we use continue rather than break here in case the variables are in a different order. continue fi # Moved DIRECTORY_CHECK inside an IF statement so that a non-zero exit status would not trigger either the errexit # or the ERR trap conditions. if ! DIRECTORY_CHECK "${i}"; then echo echo "Skipping this feed until corrected, proceeding to next feed listed in ${CURRENT_SERVERLIST}" echo # This needs to be 'continue 2' because we are not continuing the immediate for loop but rather the # loop wrapping it. continue 2 fi done if (( FEED_FULL_PLAYLIST >= 1 )) ; then PLAYLIST_NAME=$(echo "PLAYLIST_${FEED_NAME//[[:space:]]/_}.m3u" | tr -s _) elif (( NO_PLAYLIST == 0 )) && [[ -n ${PLAYLIST_PERPODCAST+set} ]]; then # If configured to create individual playlists for each podcast, set playlist name here. # White-Space in feed name will be converted to underscores and then tr will limit their repetitions. PLAYLIST_NAME="${PLAYLIST_NAMEBASE:=NEW-}$(date $DATE_FORMAT)-$(echo "${FEED_NAME//[[:space:]]/_}" | tr -s '_').m3u" COUNTER=2 while [[ -e "${DIR_LIBRARY}/${PLAYLIST_NAME}" ]] ; do PLAYLIST_NAME="${PLAYLIST_NAMEBASE:=NEW-}$(date $DATE_FORMAT)-$(echo "${FEED_NAME//[[:space:]]/_}" | tr -s '_').r$COUNTER.m3u" COUNTER=$((COUNTER+1)) done elif (( NO_PLAYLIST == 0 )) && [[ -z ${PLAYLIST_PERPODCAST+set} ]]; then PLAYLIST_NAME="${SESSION_PLAYLIST_NAME}" fi if (( VERBOSITY >= 2 )) ; then if [[ -n ${URL_USERNAME+set} ]]; then printf '%-30s %-50s\n' "" "Username: ${URL_USERNAME}" fi if [[ -n ${URL_PASSWORD+set} ]] && (( DEBUG == 1 )); then # Password is stored in the serverlist file in plain text so displaying it here is a small risk but can be # useful for debugging connection issues. echo "${DEBUG_LEADER} Password: ${URL_PASSWORD}" fi if (( VERBOSITY >= 4 )) ; then printf '%-30s %-50s\n' "" "WGET OPTIONS: ${WGET_OPTIONS[*]}" fi if (( NO_PLAYLIST == 0 )) && [[ -n ${PLAYLIST_PERPODCAST+set} ]]; then printf '%-30s %-50s\n' "" "Individual Podcast Playlist Name: ${PLAYLIST_NAME}" fi echo echo "Downloading feed index from ${FEED_URL}" fi case ${FILETYPE} in 'utf8') INDEXFILE=$(wget "${WGET_OPTIONS[@]}" -O - "${FEED_URL}") && EXITSTATUS=0 || EXITSTATUS=1 ;; 'utf16') # Slight gymnastics to report the exit status of WGET and not ICONV. INDEXFILE=$(wget "${WGET_OPTIONS[@]}" -O - "${FEED_URL}" | iconv -f UTF-16 -t UTF-8; exit "${PIPESTATUS[0]}") && EXITSTATUS=0 || EXITSTATUS=1 ;; esac if (( EXITSTATUS != 0 )); then if (( VERBOSITY >= 1 )); then printf '%-30s %-50s\n' "" "Error Downloading Feed." else echo "ERROR Downloading Feed: ${FEED_NAME}" fi # if downloaded failed, add URL to LOG_FAIL echo "${FEED_URL}" >> "${DIR_LOG}"/"${LOG_FAIL}" continue fi INDEXFILE=$(echo "${INDEXFILE}" | tr '\n\r' ' ' | sed -e 's/<\([^/]\)/\n<\1/g') # We will look for either '/Id' # Look for podcast:liveitem tags. If found, delete. # Flags: # I Ignore case while searching # d Delete pattern space # 4. -e '/\(.*\)<[/]title>.*$/TITLE \1/Ip' # Looking for tag and grabbing contents. # 10. -e 't' if last s/// did successful substitution then branch to end of script # because no label was provided. # 11. -e '/\(<enclosure\|<title>\).*/I{N;s/\ *\n/\ /;T;ba}' # this final command is to read the next line of the file into the pattern space. # We use this for feeds where the url= portion does not appear on the same line as # the enclosure start tag or when the closing title tag is not on the same line as # the opening one. After merging, we go back to label 'a'. Might need to add # an 'I' flag to make it case insensitive for the 'enclosure' bit. # # INDEXFILE before adding TITLE bits # INDEXFILE=$(echo "${INDEXFILE}" | \ # sed -n -e :a -e 's/.*<enclosure.*url\s*=\s*"\([^"]\+\)".*/\1/Ip' -e 't' \ # -e "s/.*<enclosure.*url\\s*=\\s*'\\([^']\\+\\)'.*/\\1/Ip" \ # -e '/<enclosure\s*/{N;s/ *\n/ /;ba;}') # # INDEXFILE_TMP before adding bits to delete podcast:liveitem stuff # INDEXFILE_TMP=$(echo "${INDEXFILE}" | \ # sed -n -e :a -e 's/.*<enclosure.*url\s*=\s*"\([^"]\+\)".*/URL \1/Ip' -e t \ # -e "s/.*<enclosure.*url\\s*'=\\s*\\([^i]\\+\\)'.*/URL \\1/Ip" -e t \ # -e 's/.*<title>\(.*\)<[/]title>.*$/TITLE \1/Ip' -e t \ # -e '/\(<enclosure\|<title>\).*/I{N;s/\ *\n/\ /;T;ba}') if (( WGET_OPTION_RSS_MEDIACONTENT < 1 )); then INDEXFILE_TMP=$(echo "${INDEXFILE}" | \ sed -n -e :a -e '/<podcast:liveitem.*<\/podcast:liveitem>/Id' \ -e '/<podcast:liveitem.*/I{N;s/\ *\n/\ /;T;ba}' \ -e 's/.*<enclosure.*url\s*=\s*"\([^"]\+\)".*/URL \1/Ip' -e t \ -e "s/.*<enclosure.*url\\s*=\\s*'\\([^']\\+\\)'.*/URL \\1/Ip" -e t \ -e 's/.*<title>\(.*\)<[/]title>.*$/TITLE \1/Ip' -e t \ -e '/\(<enclosure\|<title>\).*/I{N;s/\ *\n/\ /;T;ba}') else # This section adds a couple of tests to look for <media:content ...> tags from the RSS Media Specification. # This is used in the feeds created by FreshRSS aggregator so it's use is not expected to be common but we may # find a few users that require it. These tests were added after test 8 above and before test 9. # 8.1. -e 's/.*<media:content.*type="\(video\|audio\).*url\s*=\s*"\([^"]\+\)".*/URL \2/Ip' # Look in pattern space for media:content tags that are preceded by TYPE video or audio then # extract the URL. # Flags: # I Ignore case while searching # p Print only substituted line # 8.2. -e 't' if last s/// did successful substitution then branch to end of script # because no label was provided. # 8.3. -e "s/.*<media:content.*type='\\(video\\|audio\\).*url\\s*=\\s*'\\([^']\\+\\)'.*/URL \\2/Ip" # This command is the sane as 8.1 but to catch when the HTML feed is using single quotes # rather than double quotes. # 8.4. -e 't' if last s/// did successful substitution then branch to end of script # because no label was provided. # 8.5. -e 's/.*<media:content.*url\s*=\s*"\([^"]\+\)".*type="\(video\|audio\).*/URL \1/Ip' # This is like 8.1 but TYPE option follows the URL rather than precedes it. # 8.6. -e 't' if last s/// did successful substitution then branch to end of script # because no label was provided. # 8.7. -e "s/.*<media:content.*url\\s*=\\s*'\\([^']\\+\\)'.*type=\\(video\\|audio\\).*/URL \\1/Ip" # This is like 8.3 but TYPE option follows the URL rather than precedes it. # 8.8. -e 't' if last s/// did successful substitution then branch to end of script # because no label was provided. # Then the final pattern was updated to also look for incomplete media:content tags. # 11. -e '/\(<enclosure\|<title>\|<media:content\).*/I{N;s/\ *\n/\ /;T;ba}' # this final command is to read the next line of the file into the pattern space. # We use this for feeds where the url= portion does not appear on the same line as # the enclosure start tag or when the closing title tag is not on the same line as # the opening one. After merging, we go back to label 'a'. Might need to add # an 'I' flag to make it case insensitive for the 'enclosure' bit. # INDEXFILE_TMP=$(echo "${INDEXFILE}" | \ sed -n -e :a -e '/<podcast:liveitem.*<\/podcast:liveitem>/Id' \ -e '/<podcast:liveitem.*/I{N;s/\ *\n/\ /;T;ba}' \ -e 's/.*<enclosure.*url\s*=\s*"\([^"]\+\)".*/URL \1/Ip' -e t \ -e "s/.*<enclosure.*url\\s*=\\s*'\\([^']\\+\\)'.*/URL \\1/Ip" -e t \ -e 's/.*<media:content.*type="\(video\|audio\).*url\s*=\s*"\([^"]\+\)".*/URL \2/Ip' -e t \ -e "s/.*<media:content.*type='\\(video\\|audio\\).*url\\s*=\\s*'\\([^']\\+\\)'.*/URL \\2/Ip" -e t \ -e 's/.*<media:content.*url\s*=\s*"\([^"]\+\)".*type="\(video\|audio\).*/URL \1/Ip' -e t \ -e "s/.*<media:content.*url\\s*=\\s*'\\([^']\\+\\)'.*type=\\(video\\|audio\\).*/URL \\1/Ip" -e t \ -e 's/.*<title>\(.*\)<[/]title>.*$/TITLE \1/Ip' -e t \ -e '/\(<enclosure\|<title>\|<media:content\).*/I{N;s/\ *\n/\ /;T;ba}') fi # If TITLE tag appears after the ENCLOSURE tag in the file, reverse the order of list for parsing. if (( POST_WGET_RENAME_REVTITLETAG > 0 )); then INDEXFILE_TMP=$(echo "${INDEXFILE_TMP}" | sed -n '1!G;h;$p') fi # Merge TITLE AND URL lines onto single line divided by '<Url-Title>' INDEXFILE_TMP2="" while read -r TAG DATA; do case ${TAG} in "TITLE" ) ITEM_TITLE="${DATA}" ;; "URL" ) # with cleanup for URL (used to be called FINAL below ATOM section) ITEM_URL=$(echo "${DATA}" | sed -e 's/\s/%20/g' -e "s/'/%27/g" -e 's/\'/%27/g' -e 's/\&/\&/g') # Enabled handling of unset ITEM_TITLE items by including a dash at the end of the variable name to # signify that the default is a blank value (or we could specify a value we wanted after the dash). INDEXFILE_TMP2+="${ITEM_URL}<Url-Title>${ITEM_TITLE-}${NEWLINE}" unset -v ITEM_TITLE ITEM_URL ;; esac done <<<"${INDEXFILE_TMP}" if (( POST_WGET_RENAME_REVTITLETAG > 0 )); then # Return INDEX to its default order INDEXFILE=$(echo "${INDEXFILE_TMP2}" | sed -n '1!G;h;$p') else # INDEX already in correct order. INDEXFILE="${INDEXFILE_TMP2}" fi elif grep -q '<feed ' <<< "${TESTLINES}"; then if (( DEBUG == 1 )) ; then printf '%s %s\n' "${DEBUG_LEADER}" "Atom Feed Found" echo fi # if file is Atom then we look for '<link\s.*rel="enclosure" ' # NOTE: there may be a tag between link and rel, therefore we need to add '\s.*' # Characters unlikely to appear in an XML document. Combination of "invalid" characters and those # that have worked well for my testing. TEST_STRING="^{}_~" # Find suitable TEST_CHAR to use for sed statements below. for (( i=0; i<${#TEST_STRING}; i++ )); do TEST_CHAR=${TEST_STRING:$i:1} # grep looks for --fixed-strings so that certain characters are not # interpreted as regular expressions (like ^ $ or /) # # We're looking for a character NOT found in the INDEXFILE so its ! grep if ! grep --quiet --fixed-strings "${TEST_CHAR}" <<<"${INDEXFILE}"; then # When a character is not found in the string that we can use, break out of the loop. break fi done unset -v i # Get full enclosure lines to filter on INDEXFILE=$(echo "${INDEXFILE}" | sed -n -e "\\${TEST_CHAR}<link\\s.*rel=\"enclosure\".*/>${TEST_CHAR}Ip") # NOTE: I have this section disabled because it has a high potential for confusing # users. At this stage, there is potential that they will see many enclosures # in the feed and not understand why the filtering they have selected results # in fewer or none found. However it is useful during testing so it is # available. # if (( DEBUG == 1 )) ; then # printf '%s %s\n' "${DEBUG_LEADER}" "------ Indexfile Contents --------" # while IFS= read -r LINE; do # printf '%s %s\n' "${DEBUG_LEADER}" "${LINE}" # done< <(printf "%s\n" "${INDEXFILE}") # printf '%s %s\n' "${DEBUG_LEADER}" "----------------------------------" # printf '%s %s\n' "${DEBUG_LEADER}" "----------------------------------" # echo # fi # SIMPLE FILTERING: filter down to just audio & video types. if (( ATOM_FILTER_SIMPLE > 0 )); then INDEXFILE=$(echo "${INDEXFILE}" | sed -n -e "\\${TEST_CHAR}type=\"\\(audio\\|video\\)${TEST_CHAR}p") fi if [[ -n ${ATOM_FILTER_TYPE+set} ]]; then INDEXFILE=$(echo "${INDEXFILE}" | sed -n -r -e "\\${TEST_CHAR}.*type=\"${ATOM_FILTER_TYPE}\".*${TEST_CHAR}p") fi TYPECOUNT=$(echo "${INDEXFILE}" | sed -n -e "s${TEST_CHAR}.*type=\"\\([^\"]\\+\\)\".*${TEST_CHAR}\\1${TEST_CHAR}Ip" | sort | uniq -c | awk '{printf "%7s | %s\n",$1,$2}') if (( VERBOSITY >= 2 )) ; then # Check if any filtering is already enabled. If it is, assume that is all the user wants so no need to inform # them that more is possible. if (( ATOM_FILTER_SIMPLE == 0 )) && [[ -z ${ATOM_FILTER_TYPE+set} ]]; then if (( $(echo "${TYPECOUNT}" | wc -l) > 1 )); then echo echo "This feed supports multiple types. Consider using ATOM_FILTER_SIMPLE or" echo "ATOM_FILTER_TYPE to narrow your selection." echo echo " COUNT | TYPE" echo " ------+-------------" echo "${TYPECOUNT}" echo fi fi fi if [[ -n "${ATOM_FILTER_LANG+set}" ]]; then INDEXFILE=$(echo "${INDEXFILE}" | sed -n -r -e "\\${TEST_CHAR}.*xml:lang=\"${ATOM_FILTER_LANG}\".*${TEST_CHAR}p") fi LANGCOUNT=$(echo "${INDEXFILE}" | sed -n -e "s${TEST_CHAR}.*xml:lang=\"\\([^\"]\\+\\)\".*${TEST_CHAR}\\1${TEST_CHAR}Ip" | sort | uniq -c | awk '{printf "%7s | %s\n",$1,$2}') if (( VERBOSITY >= 2 )) ; then # Check if any filtering is already enabled. If it is, assume that is all the user wants so no need to inform if [[ -z "${ATOM_FILTER_LANG+set}" ]]; then if (( $(echo "${LANGCOUNT}" | wc -l) > 1 )); then echo echo "This feed supports multiple languages. Consider using ATOM_FILTER_LANG" echo "to narrow your selection." echo echo " COUNT | LANGUAGE" echo " ------+-------------" echo "${LANGCOUNT}" echo fi fi fi # Get the URLs from each enclosure after filtering. INDEXFILE=$(echo "${INDEXFILE}" | sed -n -e "s${TEST_CHAR}.*href=\"\\([^\"]\\+\\)\".*${TEST_CHAR}\\1${TEST_CHAR}Ip") # final INDEXFILE cleanup options INDEXFILE=$(echo "${INDEXFILE}" | \ sed -e 's/\s/%20/g' -e "s/'/%27/g" -e 's/\'/%27/g' -e 's/\&/\&/g' | \ sed -e ':a;N;$!ba;s/\n/ /g') # Convert to new one item per line format. Once we figure out the best way to extract # titles from we can convert or move this section as necessary. INDEXFILE_TMP2="" for ITEM_URL in ${INDEXFILE}; do INDEXFILE_TMP2+="${ITEM_URL}<Url-Title>${NEWLINE}" done # Set back to INDEXFILE INDEXFILE="${INDEXFILE_TMP2}" else echo "Feed is neither RSS, or Atom. Skipping to next feed." # if Feed is neither RSS or Atom, add URL to LOG_FAIL echo "${FEED_URL}" >> "${DIR_LOG}"/"${LOG_FAIL}" continue fi ## Since this final cleanup is just for the URL part of the line, we need to move this before the combined url/title line is created # # final INDEXFILE cleanup options # INDEXFILE=$(echo "${INDEXFILE}" | \ # sed -e 's/\s/%20/g' -e "s/'/%27/g" -e 's/\'/%27/g' -e 's/\&/\&/g' | \ # sed -e ':a;N;$!ba;s/\n/ /g') if [[ -n ${INDEXFILE} ]]; then # Remove any error log entries for the FEED_URL if it failed before REMOVE_URL "${FEED_URL}" "${DIR_LOG}"/"${LOG_FAIL}" ### Recent will need to change when we go by lines instead of one long line with items separated by spaces. # TODO: Add choice here to get most recent from beginning or end of INDEXFILE # Possibly with -> echo ${INDEXFILE} | rev | cut -d \ -f -"${MOST_RECENT}" | rev # PROBLEM: This would require added 'util-linux' to the dependencies. # # Option just in Bash: # Temp move to array # IFS=" " read -r -a TEMP_ARRAY <<< "${INDEXFILE}" # Move back to Index # INDEXFILE="${TEMP_ARRAY[@]: -${MOST_RECENT}}" # if (( MOST_RECENT > 0 )) ; then # if [[ "${FEED_SORT_ORDER}" == "ASCENDING" ]]; then # if (( DEBUG == 1 )) ; then # echo -e "${DEBUG_LEADER} FEED SORT ORDER: ASCENDING" # fi # FULLINDEXFILE="${INDEXFILE}" # IFS=" " read -r -a TEMP_ARRAY <<< "${INDEXFILE}" # INDEXFILE="${TEMP_ARRAY[*]: -${MOST_RECENT}}" # else # if (( DEBUG == 1 )) ; then # echo -e "${DEBUG_LEADER} FEED SORT ORDER: DESCENDING" # fi # FULLINDEXFILE="${INDEXFILE}" # INDEXFILE=$(echo "${INDEXFILE}" | cut -d \ -f -"${MOST_RECENT}") # fi # fi if (( MOST_RECENT > 0 )) ; then FULLINDEXFILE="${INDEXFILE}" if [[ "${FEED_SORT_ORDER}" == "ASCENDING" ]]; then if (( DEBUG == 1 )) ; then echo -e "${DEBUG_LEADER} FEED SORT ORDER: ASCENDING" fi # Adding one to MOST_RECENT because we tend to have one blank line # at the end of our INDEXFILE # -- This was needed when using tail. Does not appear necessary when using sed. # MOST_RECENT=$((MOST_RECENT+1)) # OpenBSD Tail does not support options with two dashes (so no --lines), we have # to use the single dash option here or force OpenBSD to use GNU Tail. Given this # is a single use error, it is easier to use BSD compatible syntax even if it can be # more challenging to debug. # INDEXFILE=$(echo "${INDEXFILE}" | tail --lines="${MOST_RECENT}") # INDEXFILE=$(echo "${INDEXFILE}" | tail -n "${MOST_RECENT}") # This can also be done with sed but MOST_RECENT needs to be incremented by 2 not 1 # MOST_RECENT=$((MOST_RECENT+2)) # INDEXFILE=$(echo "${INDEXFILE}" | sed -n -e ':a' -e '$p;N;'"${MOST_RECENT}"',$D;ba') LINE_COUNT=$(echo "${FULLINDEXFILE}" | sed -n '$=;') LINE_BREAK=$((LINE_COUNT - MOST_RECENT)) INDEXFILE=$(echo "${FULLINDEXFILE}" | sed -n "${LINE_BREAK},\$ p") # SURPLUSINDEX holds all the items that are not in INDEXFILE for marking as already downloaded below. SURPLUSINDEX=$(echo "${FULLINDEXFILE}" | sed -n "${LINE_BREAK},\$ !p") else if (( DEBUG == 1 )) ; then echo -e "${DEBUG_LEADER} FEED SORT ORDER: DESCENDING" fi # Attempting to use head here results in exit condition 141, but using the sed construct # below works. Not sure why given that tail works above. # INDEXFILE=$(echo "${INDEXFILE}" | head --lines="${MOST_RECENT}") INDEXFILE=$(echo "${INDEXFILE}" | sed -n "1,${MOST_RECENT}p") # SURPLUSINDEX holds all the items that are not in INDEXFILE for marking as already downloaded below. SURPLUSINDEX=$(echo "${FULLINDEXFILE}" | sed -n "1,${MOST_RECENT}!p") fi fi # if (( DEBUG == 1 )) ; then # if (( MOST_RECENT > 0 )) ; then # printf '%s %s\n' "${DEBUG_LEADER}" "Full Index List:" # URLCOUNT=0 # for URL in ${FULLINDEXFILE}; do # printf '%s %s. %s\n' "${DEBUG_LEADER}" "${URLCOUNT}" "${URL}" # URLCOUNT=$((URLCOUNT+1)) # done # printf '%s %s\n' "${DEBUG_LEADER}" "Modified Index List:" # URLCOUNT=0 # for URL in ${INDEXFILE}; do # printf '%s %s. %s\n' "${DEBUG_LEADER}" "${URLCOUNT}" "${URL}" # URLCOUNT=$((URLCOUNT+1)) # done # unset URLCOUNT # else # printf '%s %s\n' "${DEBUG_LEADER}" "Index List:" # URLCOUNT=0 # for URL in ${INDEXFILE}; do # printf '%s %s. %s\n' "${DEBUG_LEADER}" "${URLCOUNT}" "${URL}" # URLCOUNT=$((URLCOUNT+1)) # done # unset URLCOUNT # fi # fi if (( DEBUG == 1 )) ; then if (( MOST_RECENT > 0 )) ; then printf '%s %s\n' "${DEBUG_LEADER}" "Full Index List:" URLCOUNT=0 while read -r DATA; do printf '%s %s. %s\n' "${DEBUG_LEADER}" "${URLCOUNT}" "${DATA}" URLCOUNT=$((URLCOUNT+1)) done <<<"${FULLINDEXFILE}" printf '%s %s\n' "${DEBUG_LEADER}" "Modified Index List:" URLCOUNT=0 while read -r DATA; do printf '%s %s. %s\n' "${DEBUG_LEADER}" "${URLCOUNT}" "${DATA}" URLCOUNT=$((URLCOUNT+1)) done <<<"${INDEXFILE}" unset -v URLCOUNT else printf '%s %s\n' "${DEBUG_LEADER}" "Index List:" URLCOUNT=0 while read -r DATA; do printf '%s %s. %s\n' "${DEBUG_LEADER}" "${URLCOUNT}" "${DATA}" URLCOUNT=$((URLCOUNT+1)) done <<<"${INDEXFILE}" unset -v URLCOUNT fi fi # End of this while loop is at approximately line 2980 while read -r DATA; do # Check for blank line, if found just proceed to next line. if [[ -z "${DATA}" ]]; then continue fi # Divide URL and TITLE from DATA URL="${DATA%<Url-Title>*}" TITLE="${DATA#*<Url-Title>}" # Safety check: if ${DATA} does not contain '<Url-Title>' if [[ "${URL}" == "${TITLE}" ]]; then TITLE="" fi if ! grep -F "${URL}" "${DIR_LOG}"/"${LOG_COMPLETE}" >/dev/null || (( FORCE != 0 )) ; then if (( VERBOSITY >= 2 )) ; then echo fi # Without the 'sed' statements. URL_FILENAME="${URL##*/}" URL_BASE="${URL%/*}" # replace any whitespace in the filename with underscores. URL_FILENAME=$(echo "${URL_FILENAME}" | tr '[:blank:]' '_' | tr -s '_') # Test for available space on library partition AVAIL_SPACE=$(df -kP "${DIR_LIBRARY}" | tail -n 1 | awk '{print $4}') if (( AVAIL_SPACE < MIN_SPACE )); then printf '\n%s\n%s\n' "Available space on Library partition has dropped below allowed." "Stopping Session." 1>&2 CLEANUP_AND_EXIT ${ERR_LIBLOWSPACE} fi # Filename format fixes and character substitutions. # Return value stored in variable ${MOD_FILENAME}, called here as a string to be used as the name of the # variable. filenameFixFormat MOD_FILENAME "${URL_FILENAME}" # Fix where filename part of URI is constant # This fix is known to cause issues with in filenames when it is not needed. Disabled by default. if [[ -n ${FILENAME_FORMATFIX4+set} ]] && (( FILENAME_FORMATFIX4 > 0 )); then if [[ -z ${MOD_FILENAME} ]] ; then MOD_FILENAME="${URL_FILENAME}" fi MOD_PREFIX="${URL_BASE%%/}" MOD_PREFIX="${MOD_PREFIX##*/}" MOD_FILENAME="${MOD_PREFIX##*/}_${MOD_FILENAME}" if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} FILENAME FORMAT(4) FIXED: ${MOD_FILENAME}" fi fi if (( POST_WGET_RENAME_TITLETAG == 1 ))||(( POST_WGET_RENAME_REVTITLETAG == 1 )); then # Title format fixes and character substitutions. # Return value stored in variable ${MOD_TITLE}, called here as a string to be used as the name of the # variable. if [[ -n "${TITLE}" ]]; then titleFixFormat MOD_TITLE "${TITLE}" if [[ -n "${MOD_TITLE}" ]]; then TITLE="${MOD_TITLE}" fi fi fi # If directories do not exist, then create them. if [[ ! -d "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}" ]]; then mkdir -p "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}" fi # If FORCE then remove existing files before download. if (( FORCE != 0 )); then if [[ -n ${MOD_FILENAME} ]]; then if [[ -f "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${MOD_FILENAME}" ]]; then if (( VERBOSITY == 0 )) ; then rm -f "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${MOD_FILENAME}" else rm -fv "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${MOD_FILENAME}" fi fi else if [[ -f "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${URL_FILENAME}" ]]; then if (( VERBOSITY == 0 )) ; then rm -f "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${URL_FILENAME}" else rm -fv "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${URL_FILENAME}" fi fi fi fi if (( VERBOSITY >= 2 )) ; then if [[ -n ${MOD_FILENAME} ]]; then echo -e "Downloading ${MOD_FILENAME} from ${URL_BASE}" else echo -e "Downloading ${URL_FILENAME} from ${URL_BASE}" fi fi # Added '&& WGET_EXITSTATUS=0 || WGET_EXITSTATUS=1' to each WGET call below so we could remove the disabling # of errexit and the ERR trap. if (( (WGET_OPTION_DISPOSITION == 1) || (WGET_OPTION_FILENAME_LOCATION == 1) )); then if (( VERBOSITY >= 3 )) ; then if (( WGET_OPTION_DISPOSITION == 1 )); then echo " [ Progress meters disabled while using 'wget --content-disposition' ]" fi if (( WGET_OPTION_FILENAME_LOCATION == 1 )); then echo " [ Progress meters disabled while using OPT_FILENAME_LOCATION ]" fi fi # This has been moved here to handle long filenames given by some feeds that result in a WGET Error if [[ -n ${MOD_FILENAME} ]]; then wget "${WGET_OPTIONS[@]}" -q --server-response -O "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${MOD_FILENAME}" "${URL}" 2> "${DIR_SESSION}/${URL_FILENAME}.servresp" && WGET_EXITSTATUS=0 || WGET_EXITSTATUS=1 else wget "${WGET_OPTIONS[@]}" -q --server-response -O "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${URL_FILENAME}" "${URL}" 2> "${DIR_SESSION}/${URL_FILENAME}.servresp" && WGET_EXITSTATUS=0 || WGET_EXITSTATUS=1 fi else # This has been moved here to handle long filenames given by some feeds that result in a WGET Error # ( I'm looking at you ITunes..... ) if [[ -n ${MOD_FILENAME} ]]; then wget "${WGET_OPTIONS[@]}" -O "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${MOD_FILENAME}" "${URL}" && WGET_EXITSTATUS=0 || WGET_EXITSTATUS=1 else wget "${WGET_OPTIONS[@]}" -O "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${URL_FILENAME}" "${URL}" && WGET_EXITSTATUS=0 || WGET_EXITSTATUS=1 fi fi if (( WGET_EXITSTATUS == 0 )); then # make sure CONTENT_FILENAME hasn't been used before unset -v CONTENT_FILENAME unset -v LOCATION_FILENAME unset -v FINAL_FILENAME echo "${URL}" >> "${DIR_LOG}"/"${LOG_COMPLETE}" # Remove any error log entries for the URL if it failed before REMOVE_URL "${URL}" "${DIR_LOG}"/"${LOG_FAIL}" # Before filename changes, set FINAL_FILENAME to URL_FILENAME # IF MOD_FILENAME has already been used, then set FINAL_FILENAME to MOD_FILENAME if [[ -n ${MOD_FILENAME} ]]; then FINAL_FILENAME="${MOD_FILENAME}" else FINAL_FILENAME="${URL_FILENAME}" fi # if --content-disposition get filename and remove quotes around it. if (( WGET_OPTION_DISPOSITION == 1 )); then if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} CONTENT-DISPOSITION TESTS" fi if [[ -s "${DIR_SESSION}/${URL_FILENAME}.servresp" ]]; then if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} SERVER RESPONSE FILE NONZERO SIZE" fi # Added '|| true' to catch when grep exits with a non-zero status because the # 'Content-Disposition' tag is not found. This eliminates the need to disable errexit and the # ERR trap. CONTENT_FILENAME=$(grep "^\\s\\+Content-Disposition:" "${DIR_SESSION}/${URL_FILENAME}.servresp" | grep "filename=" | tail -1 | sed -e 's/.*filename=//' -e 's/"//g') || true # Test CONTENT_FILENAME is not null or consists solely of whitespace if [[ -n ${CONTENT_FILENAME} && -n ${CONTENT_FILENAME// } ]]; then if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} CONTENT_FILENAME: '${CONTENT_FILENAME}'" fi else if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} SERVER RESPONSE FILE - No Content-Disposition Tag" fi # if variable is effectively blank, we can unset it unset -v CONTENT_FILENAME if (( WGET_OPTION_DISPOSITION_FAIL > 0 )); then if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} - Removing URL from ${DIR_LOG}/${LOG_COMPLETE} to retry next session." fi # Remove any COMPLETE log entries to allow it to retry next session. REMOVE_URL "${URL}" "${DIR_LOG}"/"${LOG_COMPLETE}" fi fi else if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} SERVER RESPONSE FILE ZERO SIZE - No Tags" fi fi # Do we need a cleanup on CONTENT_FILENAME here? # Two step process. First modify any BADCHARS into the REPLACECHAR and then squeeze each repetition of the REPLACECHAR # down to a single time. if [[ -n ${CONTENT_FILENAME+set} ]]; then CONTENT_FILENAME=$(echo "${CONTENT_FILENAME}" | tr "${FILENAME_BADCHARS}" "${FILENAME_REPLACECHAR}" | tr -s "${FILENAME_REPLACECHAR}") fi fi if (( WGET_OPTION_FILENAME_LOCATION > 0 )); then if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} FILENAME LOCATION TESTS" fi if [[ -s "${DIR_SESSION}/${URL_FILENAME}.servresp" ]]; then if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} SERVER RESPONSE FILE NONZERO SIZE" fi # Added '|| true' to catch when grep exits with non-zero status because the 'Location' tag was # not found. This eliminates the need to disable errexit and the ERR trap. LOCATION_FILENAME=$(grep "^\\s*Location:" "${DIR_SESSION}/${URL_FILENAME}.servresp" | tail -1 | sed -e 's/\s*Location:\s//' -e 's/\(.*\)?.*/\1/') || true LOCATION_FILENAME="${LOCATION_FILENAME##*/}" if (( VERBOSITY >= 2 )); then # Test if LOCATION_FILENAME is not null and does not consist solely of spaces. if [[ -n ${LOCATION_FILENAME} && -n ${LOCATION_FILENAME// } ]]; then if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} LOCATION_FILENAME: '${LOCATION_FILENAME}'" fi else if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} SERVER RESPONSE FILE - No Location Tag" fi # If LOCATION_FILENAME is effectively blank we can unset it unset -v LOCATION_FILENAME fi fi else if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} SERVER RESPONSE FILE ZERO SIZE - No Tags" fi fi # Do we need a cleanup on LOCATION_FILENAME here? # Two step process. First modify any BADCHARS into the REPLACECHAR and then squeeze each repetition of the REPLACECHAR # down to a single time. if [[ -n ${LOCATION_FILENAME+set} ]]; then LOCATION_FILENAME=$(echo "${LOCATION_FILENAME}" | tr "${FILENAME_BADCHARS}" "${FILENAME_REPLACECHAR}" | tr -s "${FILENAME_REPLACECHAR}") fi fi # Move filename to that provided by content-disposition if [[ -n ${CONTENT_FILENAME+set} && -n ${CONTENT_FILENAME} ]]; then if [[ ! -f "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${CONTENT_FILENAME}" ]]; then if (( VERBOSITY >= 2 )); then echo "FILENAME CHANGE from ${FINAL_FILENAME} to ${CONTENT_FILENAME}" fi mv "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FINAL_FILENAME}" "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${CONTENT_FILENAME}" FINAL_FILENAME="${CONTENT_FILENAME}" else if (( FORCE != 0 )); then if [[ "${FINAL_FILENAME}" == "${CONTENT_FILENAME}" ]]; then if (( VERBOSITY >= 2 )); then echo "FILENAME CHANGE PREVENTED [ Same Filenames ]" fi else if (( VERBOSITY >= 2 )); then echo "Forced FILENAME CHANGE from ${FINAL_FILENAME} to ${CONTENT_FILENAME}" fi mv -f "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FINAL_FILENAME}" "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${CONTENT_FILENAME}" FINAL_FILENAME="${CONTENT_FILENAME}" fi else if (( VERBOSITY >= 2 )); then echo "FILENAME CHANGE PREVENTED [ ${CONTENT_FILENAME} already exists ]" fi fi fi fi # Move filename to that provided by LOCATION_FILENAME if [[ -n ${LOCATION_FILENAME+set} && -n ${LOCATION_FILENAME} ]]; then if [[ ! -f "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${LOCATION_FILENAME}" ]]; then if (( VERBOSITY >= 2 )); then echo "FILENAME CHANGE from ${FINAL_FILENAME} to ${LOCATION_FILENAME}" fi mv "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FINAL_FILENAME}" "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${LOCATION_FILENAME}" FINAL_FILENAME="${LOCATION_FILENAME}" else if (( FORCE != 0 )); then if [[ "${FINAL_FILENAME}" == "${LOCATION_FILENAME}" ]]; then if (( VERBOSITY >= 2 )); then echo "FILENAME CHANGE PREVENTED [ Same Filenames ]" fi else if (( VERBOSITY >= 2 )); then echo "Forced FILENAME CHANGE from ${FINAL_FILENAME} to ${LOCATION_FILENAME}" fi mv -f "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FINAL_FILENAME}" "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${LOCATION_FILENAME}" FINAL_FILENAME="${LOCATION_FILENAME}" fi else if (( VERBOSITY >= 2 )); then echo "FILENAME CHANGE PREVENTED [ ${LOCATION_FILENAME} already exists ]" fi fi fi fi # Rename a file to title if (( POST_WGET_RENAME_TITLETAG == 1 ))||(( POST_WGET_RENAME_REVTITLETAG == 1 )); then if [[ -n "${TITLE}" ]]; then if (( DEBUG == 1 )); then echo "${DEBUG_LEADER} Title with cleanup: ${TITLE}" fi # Get extension from downloaded filename OLDEXT="${FINAL_FILENAME##*.}" TITLEEXT="${TITLE##*.}" # To protect against hidden filenames that end in a period ('.') if [[ -z "${TITLEEXT}" ]]; then TITLEEXT="${TITLE}" fi # If TITLE already has filename extension then do not add a second time, # also check that OLDEXT if [[ "${OLDEXT}" != "${TITLEEXT}" ]] && [[ -n "${OLDEXT}" ]]; then # With pattern substitution to remove a trailing period from title before # we add one for the extension. FINAL_TITLE="${TITLE/%./}.${OLDEXT}" else FINAL_TITLE="${TITLE}" fi if (( DEBUG == 1 )); then echo "${DEBUG_LEADER} Title with extension: ${FINAL_TITLE}" fi if [[ ! -f "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FINAL_TITLE}" ]]; then if (( VERBOSITY >= 2 )); then echo "FILENAME CHANGE from ${FINAL_FILENAME} to ${FINAL_TITLE}" fi mv "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FINAL_FILENAME}" "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FINAL_TITLE}" FINAL_FILENAME="${FINAL_TITLE}" else if (( FORCE != 0 )); then if (( VERBOSITY >= 2 )); then echo "Forced FILENAME CHANGE from ${FINAL_FILENAME} to ${FINAL_TITLE}" fi mv -f "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FINAL_FILENAME}" "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FINAL_TITLE}" FINAL_FILENAME="${FINAL_TITLE}" else if (( VERBOSITY >= 2 )); then echo "FILENAME CHANGE PREVENTED [ ${FINAL_TITLE} already exists ]" fi fi fi else if (( DEBUG == 1 )); then echo "${DEBUG_LEADER} Title not set for this item" fi fi fi # Rename file to start from modification date if (( POST_WGET_RENAME_MDATE == 1 )); then FILE_MDATE=$(date -r "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FINAL_FILENAME}" '+%Y%m%d_%Hh%Mm') FILENAME_MDATE="${FILE_MDATE}_${FINAL_FILENAME}" if [[ ! -f "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FILENAME_MDATE}" ]]; then if (( VERBOSITY >= 2 )); then echo "FILENAME CHANGE from ${FINAL_FILENAME} to ${FILENAME_MDATE}" fi mv "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FINAL_FILENAME}" "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FILENAME_MDATE}" FINAL_FILENAME="${FILENAME_MDATE}" else if (( FORCE != 0 )); then if (( VERBOSITY >= 2 )); then echo "Forced FILENAME CHANGE from ${FINAL_FILENAME} to ${FILENAME_MDATE}" fi mv -f "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FINAL_FILENAME}" "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FILENAME_MDATE}" FINAL_FILENAME="${FILENAME_MDATE}" else if (( VERBOSITY >= 2 )); then echo "FILENAME CHANGE PREVENTED [ ${FILENAME_MDATE} already exists ]" fi fi fi fi # Add filename.suffix if [[ -n ${FILENAME_SUFFIX+set} ]]; then if [[ ! -f "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FINAL_FILENAME}.${FILENAME_SUFFIX}" ]]; then if (( VERBOSITY >= 2 )); then echo "FILENAME CHANGE from ${FINAL_FILENAME} to ${FINAL_FILENAME}.${FILENAME_SUFFIX}" fi mv "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FINAL_FILENAME}" "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FINAL_FILENAME}.${FILENAME_SUFFIX}" FINAL_FILENAME="${FINAL_FILENAME}.${FILENAME_SUFFIX}" else if (( FORCE != 0 )); then if (( VERBOSITY >= 2 )); then echo "Forced FILENAME CHANGE from ${FINAL_FILENAME} to ${FINAL_FILENAME}.${FILENAME_SUFFIX}" fi mv -f "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FINAL_FILENAME}" "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${FINAL_FILENAME}.${FILENAME_SUFFIX}" FINAL_FILENAME="${FINAL_FILENAME}.${FILENAME_SUFFIX}" else if (( VERBOSITY >= 2 )); then echo "FILENAME CHANGE PREVENTED [ ${FINAL_FILENAME}.${FILENAME_SUFFIX} already exists ]" fi fi fi fi if (( NO_PLAYLIST == 0 )) || (( FEED_FULL_PLAYLIST >= 1 )) ; then # If creating playlists, then add to file. Added sed command to remove any './' from each item # added to a playlist or the echo statement. if [[ -n ${PLAYLIST_NAME+set} ]] ; then echo "${FEED_CATEGORY}/${FEED_NAME}/${FINAL_FILENAME}" | sed -e 's|[.]/||g' >> "${DIR_LIBRARY}/${PLAYLIST_NAME}" if (( VERBOSITY >= 2 )); then echo "PLAYLIST: Adding ${FEED_CATEGORY}/${FEED_NAME}/${FINAL_FILENAME} to ${DIR_LIBRARY}/${PLAYLIST_NAME}" | sed -e 's|[.]/||g' fi fi fi # We're done with CONTENT_FILENAME and FINAL_FILENAME, make sure they are cleared if [[ -n ${CONTENT_FILENAME+set} ]] ; then unset -v CONTENT_FILENAME fi if [[ -n ${FINAL_FILENAME+set} ]] ; then unset -v FINAL_FILENAME fi else # if downloaded failed, add URL to LOG_FAIL if (( VERBOSITY >= 2 )) ; then echo "Adding URL to error log." fi echo "${URL}" >> "${DIR_LOG}"/"${LOG_FAIL}" # If file has zero size then remove it. Files that are larger than zero size are kept so WGET's # continue function can work on later attempts. if [[ -n ${MOD_FILENAME} ]]; then REMOVE_FILENAME="${MOD_FILENAME}" else REMOVE_FILENAME="${URL_FILENAME}" fi if [[ -f "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${REMOVE_FILENAME}" ]]; then if (( VERBOSITY >= 2 )) ; then echo "Partial file exists." fi if [[ ! -s "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${REMOVE_FILENAME}" ]]; then if (( VERBOSITY >= 2 )) ; then echo "- Removing zero size file." rm -fv "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${REMOVE_FILENAME}" else rm -f "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/${REMOVE_FILENAME}" fi else if (( VERBOSITY >= 2 )) ; then echo "- Keeping for later attempts" fi fi else if (( VERBOSITY >= 2 )) ; then echo "Partial file does not exist." fi fi fi else # Remove any error log entries for the URL if it failed before REMOVE_URL "${URL}" "${DIR_LOG}"/"${LOG_FAIL}" if (( VERBOSITY >= 2 )) ; then if (( VERBOSITY >= 4 )) ; then echo ; fi echo "Already downloaded ${URL}." fi fi # Remove server response file as it's no longer needed if (( DEBUG == 0 )); then if [[ -n ${URL_FILENAME+set} && -f "${DIR_SESSION}/${URL_FILENAME}.servresp" ]]; then rm -f "${DIR_SESSION}/${URL_FILENAME}.servresp" fi else if [[ -n ${URL_FILENAME+set} && -f "${DIR_SESSION}/${URL_FILENAME}.servresp" ]] && (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Not deleting ${DIR_SESSION}/${URL_FILENAME}.servresp" fi fi if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Clear File Vars" fi unset -v URL_FILENAME done <<< "${INDEXFILE}" if (( VERBOSITY >= 3 )) ; then echo; fi if (( (MOST_RECENT != 0) && (INSTALL_SESSION == 0) )); then if (( DEBUG == 1 )); then echo "${DEBUG_LEADER} Marking additional files in Index as already downloaded (RECENT > 0)." fi # for URL in ${FULLINDEXFILE} # do while read -r DATA; do # Check for blank line, if found just proceed to next line. if [[ -z "${DATA}" ]]; then continue fi # Divide URL and TITLE from DATA URL="${DATA%<Url-Title>*}" TITLE="${DATA#*<Url-Title>}" # Safety check: if ${DATA} does not contain '<Url-Title>' if [[ "${URL}" == "${TITLE}" || "${TITLE}" == "" ]]; then unset -v TITLE fi if ! grep -F "${URL}" "${DIR_LOG}"/"${LOG_COMPLETE}" >/dev/null ; then URL_FILENAME=$(echo "${URL}" | sed -e 's/.*\/\([^\/]\+\)/\1/' -e 's/%20/ /g') if (( VERBOSITY >= 2 )) ; then if (( POST_WGET_RENAME_TITLETAG == 1 ))||(( POST_WGET_RENAME_REVTITLETAG == 1 )); then if [[ -n "${TITLE}" ]]; then echo "Marking as already downloaded ${TITLE}." else echo "Marking as already downloaded ${URL_FILENAME}." fi else echo "Marking as already downloaded ${URL_FILENAME}." fi fi echo "${URL}" >> "${DIR_LOG}"/"${LOG_COMPLETE}" fi # done <<< "${FULLINDEXFILE}" done <<< "${SURPLUSINDEX}" fi # If doing individual playlists for each podcast Sort new playlist if (( NO_PLAYLIST == 0 )) && [[ -e "${DIR_LIBRARY}/$PLAYLIST_NAME" && -n "${PLAYLIST_PERPODCAST+set}" ]]; then PLAYLIST_Sort "${DIR_LIBRARY}" "${PLAYLIST_NAME}" # If doing individual playlists for each podcast, Create ASX Playlist if (( ASX_PLAYLIST > 0 )); then PLAYLIST_ConvertToASX "${DIR_LIBRARY}" "${PLAYLIST_NAME}" fi # Done with playlist, unset name. unset -v PLAYLIST_NAME fi else if (( VERBOSITY >= 1 )) ; then echo " No enclosures in feed: ${FEED_URL}" fi echo "${FEED_URL}" >> "${DIR_LOG}"/"${LOG_FAIL}" fi if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Clear Server Loop Vars" fi unset -v FEED_URL unset -v FEED_CATEGORY unset -v FEED_NAME unset -v URL_FILENAME unset -v URL_USERNAME unset -v URL_PASSWORD done < "${CURRENT_SERVERLIST}" done # If doing one combined playlist for all podcasts, Sort new playlist if (( NO_PLAYLIST == 0 )) && [[ -n ${PLAYLIST_NAME+set} ]]; then if [[ -e "${DIR_LIBRARY}/$PLAYLIST_NAME" ]]; then PLAYLIST_Sort "${DIR_LIBRARY}" "${PLAYLIST_NAME}" # If doing one combined playlist for all podcasts, Create ASX Playlist if (( ASX_PLAYLIST > 0 )); then PLAYLIST_ConvertToASX "${DIR_LIBRARY}" "${PLAYLIST_NAME}" fi fi fi fi # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # CLEANUP loop {{{ if [[ -z ${IMPORT_OPML+set} && -z ${EXPORT_OPML+set} && -z ${IMPORT_PCAST+set} ]] ; then if (( CLEANUP != 0 || CLEANUP_ONLY != 0 )) ; then if (( VERBOSITY >= 2 )) ; then if (( CLEANUP_SIMULATE > 0 )); then echo "Simulating cleanup, the following files will be removed when you run cleanup." else printf '\n%s\n%s\n' "-------------------------------------------------" "Cleanup old tracks." fi fi FILELIST=$(find "${DIR_LIBRARY}"/ -maxdepth 1 -type f -name "*.m3u") # Convert CLEANUP_DAYS to CLEANUP_SECONDS (86400 == seconds in a day) CLEANUP_SECONDS=$((CLEANUP_DAYS*86400)) # Convert CLEANUP_SECONDS to date in seconds from unix epoch before midnight last night. # This works with uniform whole days and isn't subject to when the cleanup is run. DATE_CLEAN2=$(($(date +%s --date 0:00)-CLEANUP_SECONDS)) while IFS= read -r FILE; do # Catch for when FILELIST is blank if [[ -z "${FILE}" ]]; then continue fi if [[ "${FILE##/*/}" =~ ^${PLAYLIST_NAMEBASE}* ]]; then # Compare global playlist file modification date with DATE_CLEAN2. If file modification date is newer than # DATE_CLEAN2 then we skip it. if [ "$(stat --format %Y "${FILE}")" -gt "${DATE_CLEAN2}" ]; then continue fi fi if (( VERBOSITY >= 2 )) ; then echo "Deleting tracks from ${FILE}:" fi while read -r LINE ; do if [[ -f "${DIR_LIBRARY}/${LINE}" ]]; then # Compare each file from the playlist with file modification date. If file modification date is older than # DATE_CLEAN2 then we remove that file if [ "$(stat --format %Y "${DIR_LIBRARY}/${LINE}")" -lt "${DATE_CLEAN2}" ]; then if (( CLEANUP_SIMULATE > 0 )); then echo "File to remove: ${DIR_LIBRARY}/${LINE}" else if (( VERBOSITY >= 2 )) ; then rm -v "${DIR_LIBRARY}/${LINE}" else rm -f "${DIR_LIBRARY}/${LINE}" fi fi fi else if (( VERBOSITY >= 2 || CLEANUP_SIMULATE > 0 )) ; then echo "File not found: ${DIR_LIBRARY}/${LINE}" fi fi done < "${FILE}" if (( CLEANUP_SIMULATE > 0 )); then echo "Playlist to remove: ${FILE}" else if (( VERBOSITY <= 1 )) ; then rm -f "${FILE}" else rm -fv "${FILE}" fi fi done <<<"${FILELIST}" fi fi # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Build individual Podcast Full Playlists {{{ # # Placed here so it is run after any session whether it includes cleanup or not. Does not run before importing or exporting OPML or # PCAST feed lists. if [[ -z ${IMPORT_OPML+set} && -z ${EXPORT_OPML+set} && -z ${IMPORT_PCAST+set} ]] ; then # Test if any feeds are configured to require individual playlists. If none are configured to require them, skip this section. # NOTE: filename globbing for the grep command is somewhat clunky because expansion does not happen within quotes. Might be # nice to find a more elegant way to state this. if grep --files-with-match --quiet --no-messages "OPT_FEED_PLAYLIST_\\(NEW\\|OLD\\)FIRST" "${DIR_CONFIG}/${CONFIG_SERVERLIST}"{.utf16,}; then if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Build OPT_FEED_PLAYLISTs" fi # Set header to not be displayed unless active feed configured to require one. FEED_PLAYLIST_HEADER=0 for FILETYPE in utf8 utf16 ; do if (( DEBUG == 1 )) ; then echo case ${FILETYPE} in 'utf8') echo "${DEBUG_LEADER} UTF-8 Loop running." ;; 'utf16') echo "${DEBUG_LEADER} UTF-16 Loop running." ;; esac fi case ${FILETYPE} in 'utf8') CURRENT_SERVERLIST="${DIR_CONFIG}"/"${CONFIG_SERVERLIST}" ;; 'utf16') CURRENT_SERVERLIST="${DIR_CONFIG}/${CONFIG_SERVERLIST}.utf16" ;; *) echo "Unknown Filetype: ${FILETYPE}" esac if [[ ! -f ${CURRENT_SERVERLIST} ]]; then if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} No config file found, exiting loop." fi continue fi ### TODO: Need to find a way to shrink this. while read -r FEED_URL FEED_CATEGORY FEED_NAME ; do FEED_FULL_PLAYLIST=0 if [[ "${FEED_URL:0:1}" == "#" || -z "${FEED_URL-empty}" ]] ; then if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} Discarding (comment or blank line)." echo "${DEBUG_LEADER} Clear Loop vars" fi unset -v FEED_URL unset -v FEED_CATEGORY unset -v FEED_NAME unset -v FEED_FULL_PLAYLIST continue fi # --------------------- # Read Podcast configuration - loop to run same filters on FEED_CATEGORY and FEED_NAME unset -v FILTER_TARGET for FILTER_TARGET in FEED_CATEGORY FEED_NAME; do FILTER_OPTIONS "${FILTER_TARGET}" done # Unset options that are not required in this while loop unset -v URL_PASSWORD unset -v URL_USERNAME unset -v WGET_OPTION_DEFAULT_USERAGENT unset -v WGET_OPTION_DISPOSITION unset -v WGET_OPTION_DISPOSITION_FAIL unset -v WGET_OPTION_NO_CHECK_CERIFICATE unset -v WGET_OPTION_PREFER_IP_TYPE unset -v WGET_OPTION_FILENAME_LOCATION unset -v WGET_OPTION_RSS_MEDIACONTENT unset -v FEED_SORT_ORDER unset -v POST_WGET_RENAME_MDATE unset -v ATOM_FILTER_SIMPLE unset -v ATOM_FILTER_TYPE unset -v ATOM_FILTER_LANG if (( FEED_FULL_PLAYLIST >= 1 )) ; then if (( FEED_PLAYLIST_HEADER == 0 )); then if (( VERBOSITY == 1 )) ; then printf '\n%s\n' "Build OPT_FEED_PLAYLISTs" fi if (( VERBOSITY >= 2 )) ; then printf '\n%s\n%s\n' "-------------------------------------------------" "Build OPT_FEED_PLAYLISTs" fi # Increment header variable so it only displays once. # NOTE: If we attempt to post-increment our variable, it will cause an error because of the way that '((' # evaluates the expression. When the result is zero, it exits with '1' triggering errexit, however if # we pre-increment then the error does not happen. # More examples: https://en.wikipedia.org/wiki/Increment_and_decrement_operators (( ++FEED_PLAYLIST_HEADER )) fi if [ "${FEED_NAME}" == "." ]; then if (( VERBOSITY >= 1 )); then printf '%-30s %-50s\n' "" "No FEED_NAME, unable to create playlist for:" printf '%-30s %-50s\n' "" " ${FEED_URL}" else echo "No FEED_NAME, unable to create playlist for:" echo " ${FEED_URL}" fi continue fi # Date Substitutions FEED_CATEGORY=$(echo "${FEED_CATEGORY}" | sed -e "s#%YY%#$(date +%Y)#" -e "s#%MM%#$(date +%m)#" -e "s#%DD%#$(date +%d)#" ) if (( VERBOSITY >= 2 )) ; then printf '\n%s\n' "-------------------------------------------------" fi if (( VERBOSITY >= 1 )) ; then if [ "${FEED_CATEGORY}" == "." ]; then printf '%-30s %-50s\n' "Category: NONE" "Name: ${FEED_NAME}" else printf '%-30s %-50s\n' "Category: ${FEED_CATEGORY}" "Name: ${FEED_NAME}" fi fi # FULL_PLAYLIST_NAME="PLAYLIST_${FEED_NAME//[[:space:]]/_}.m3u" # Replace spaces with underscores and then pipe through tr to limit repetitions. FULL_PLAYLIST_NAME=$(echo "PLAYLIST_${FEED_NAME//[[:space:]]/_}.m3u" | tr -s _) # Remove old full playlist. if [ -f "${DIR_LIBRARY}/${FULL_PLAYLIST_NAME}" ]; then if (( VERBOSITY >= 2 )) ; then echo "Remove old playlist: ${FULL_PLAYLIST_NAME}" rm -v "${DIR_LIBRARY}/${FULL_PLAYLIST_NAME}" else rm -f "${DIR_LIBRARY}/${FULL_PLAYLIST_NAME}" fi fi if [[ -d "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}" ]]; then # Build new full playlist. if (( VERBOSITY >= 2 )); then echo "Build new ${DIR_LIBRARY}/${FULL_PLAYLIST_NAME}" fi if (( FEED_FULL_PLAYLIST == 1 )); then # Get list in order of newest to oldest FEED_ITEMS=$(ls -1t "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/") else # Get list in order of oldest to newest FEED_ITEMS=$(ls -1tr "${DIR_LIBRARY}/${FEED_CATEGORY}/${FEED_NAME}/") fi if [ -n "${FEED_ITEMS}" ]; then while read -r ITEM; do if (( DEBUG == 1 )) ; then echo "${DEBUG_LEADER} FEED Full Playist add ${ITEM}" fi # Added Sed statement to remove "./" (current directory) from each item added as necessary echo "${FEED_CATEGORY}/${FEED_NAME}/${ITEM}" | sed -e 's|[.]/||g' >> "${DIR_LIBRARY}/${FULL_PLAYLIST_NAME}" done <<< "${FEED_ITEMS}" else if (( VERBOSITY >= 1 )); then printf '%-30s %-50s\n' "" "No files, no playlist" fi fi else if (( VERBOSITY >= 1 )); then printf '%-30s %-50s\n' "" "No directory, no playlist" fi fi fi unset -v FEED_URL unset -v FEED_CATEGORY unset -v FEED_NAME unset -v FEED_FULL_PLAYLIST done < "${CURRENT_SERVERLIST}" done fi fi # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # OPML import loop: {{{ if [[ -n ${IMPORT_OPML+set} ]]; then if [[ ${IMPORT_OPML} == "NONE" ]]; then echo echo "unset FILE or URL to import from." CLEANUP_AND_EXIT 1 fi if (( VERBOSITY >= 2 )) ; then printf '\n%s\n' "Import servers from OPML file: ${IMPORT_OPML}" fi new_category="OPML_Import_$(date ${DATE_FORMAT})" if [[ ${IMPORT_OPML} == http:* ]] || [[ ${IMPORT_OPML} == ftp:* ]] ; then if (( VERBOSITY >= 2 )) ; then echo "Getting opml list." fi opml_list=$(wget "${WGET_OPTIONS[@]}" -O - "${IMPORT_OPML}") else opml_list=$(cat "${IMPORT_OPML}") fi new_list=$(echo "${opml_list}" | sed -e 's/\(\/>\)/\1\n/g' | sed -e :a -n -e 's/<outline\([^>]\+\)\/>/\1/Ip;/<outline/{N;s/\n\s*/ /;ba;}') if [[ -n "$new_list" ]]; then OLD_IFS=$IFS IFS=$'\n' for data in ${new_list} ; do if (( VERBOSITY >= 1 )) ; then printf '\n%s\n' "---------------" fi new_label=$(echo "${data}" | sed -n -e 's/.*text="\([^"]\+\)".*/\1/Ip' | sed -e 's/^\s*[0-9]\+\.\s\+//' -e "s/[:;'\".,!/?<>\\|]//g") new_url=$(echo "${data}" | sed -n -e 's/.*[xml]*url="\([^"]\+\)".*/\1/Ip' | sed -e 's/ /%20/g') if (( VERBOSITY >= 3 )) ; then echo "LABEL: ${new_label}" echo "URL: ${new_url}" fi # Add '|| true' to catch when grep returns a non-zero status for URLs it does not find. This replaces the need to # disable errexit and the ERR trap. test=$(grep "${new_url}" "${DIR_CONFIG}"/"${CONFIG_SERVERLIST}") || true if [[ -z "$test" ]]; then echo "${new_url} ${new_category} ${new_label}" >> "${DIR_CONFIG}"/"${CONFIG_SERVERLIST}" elif (( VERBOSITY >= 2 )) ; then echo "Feed ${new_label} is already in the serverlist" fi done IFS=$OLD_IFS else if (( VERBOSITY >= 2 )) ; then echo " OPML Import Error ${IMPORT_OPML}" 1>&2 fi echo "OPML Import Error: ${IMPORT_OPML}" >> "${DIR_LOG}"/"${LOG_FAIL}" CLEANUP_AND_EXIT ${ERR_IMPORTOPML} fi fi # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # OPML export loop: {{{ if [[ -n ${EXPORT_OPML+set} ]]; then if [[ ${EXPORT_OPML} == "NONE" ]]; then echo echo "unset FILE to import to." CLEANUP_AND_EXIT 1 fi if (( VERBOSITY >= 2 )) ; then printf '\n%s\n' "Export serverlist to OPML file: ${EXPORT_OPML}" fi if [[ ! -e ${EXPORT_OPML} ]]; then echo '<?xml version="1.0" encoding="utf-8" ?> <opml version="1.0"> <head/> <body>' > "${EXPORT_OPML}" while read -r FEED_URL FEED_CATEGORY FEED_NAME ; do if [[ "${FEED_URL:0:1}" == "#" ]] || [[ -z "${FEED_URL-empty}" ]] ; then if (( VERBOSITY >= 3 )) ; then echo " Discarding line (comment or blank line)." fi continue fi # Remove PASSWORD from FEED_NAME if found. FEED_NAME=${FEED_NAME/PASS:+([^[:space:]])} # Remove USERNAME from FEED_NAME if found. FEED_NAME=${FEED_NAME/USER:+([^[:space:]])} # Remove any residual trailing spaces from ${FEED_NAME} while (( $(expr "${FEED_NAME}" : ".*[ ]\\+$") > 0 )); do FEED_NAME=${FEED_NAME%[ ]*} done # Remove any residual leading spaces from ${FEED_NAME} while (( $(expr "${FEED_NAME}" : "[ ]\\+.*$") > 0 )); do FEED_NAME=${FEED_NAME#[ ]*} done if (( VERBOSITY >= 3 )) ; then echo " Writing out feed ${FEED_NAME} in category ${FEED_CATEGORY} with url ${FEED_URL}" fi echo '<outline text="'"${FEED_CATEGORY}"'"><outline text="'"${FEED_NAME}"'" type="rss" xmlUrl="'"${FEED_URL}"'" /></outline>' >> "${EXPORT_OPML}" done < "${DIR_CONFIG}"/"${CONFIG_SERVERLIST}" echo '</body> </opml>' >> "${EXPORT_OPML}" else echo "OPML Export Error: ${EXPORT_OPML}" >> "${DIR_LOG}"/"${LOG_FAIL}" CLEANUP_AND_EXIT ${ERR_EXPORTOPML} fi fi # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # PCAST import: {{{ if [[ -n ${IMPORT_PCAST+set} ]] ; then if [[ ${IMPORT_PCAST} == "NONE" ]]; then echo echo "unset FILE or URL to import from." CLEANUP_AND_EXIT 1 fi if (( VERBOSITY >= 2 )) ; then printf '\n%s\n' "Import server from PCAST file: ${IMPORT_PCAST}" fi if [[ ${IMPORT_PCAST} == http:* ]] || [[ ${IMPORT_PCAST} == ftp:* ]] ; then if (( VERBOSITY >= 2 )) ; then echo "Getting pcast file." fi pcast_data=$(wget "${WGET_OPTIONS[@]}" -O - "${IMPORT_PCAST}") else pcast_data=$(cat "${IMPORT_PCAST}") fi new_link=$(echo "${pcast_data}" | sed -n -e 's/.*\(href\|url\)="\([^"]\+\)".*/\2/Ip' | sed -e 's/ /%20/g') new_category=$(echo "${pcast_data}" | sed -n -e 's/.*<category>\([^<]\+\)<.*/\1/Ip' | sed -e 's/ /_/g;s/\"/\&/g;s/\&/\&/g') new_title=$(echo "${pcast_data}" | sed -n -e 's/.*<title>\([^<]\+\)<.*/\1/Ip') if (( VERBOSITY >= 2 )) ; then echo "LINK: ${new_link}" echo "CATEGORY: ${new_category}" echo "TITLE: ${new_title}" fi # Added '|| true' to catch when grep exits with a non-zero status because the link was not found. This eliminates the need to # disable errexit and the ERR trap. test=$(grep "${new_link}" "${DIR_CONFIG}"/"${CONFIG_SERVERLIST}") || true if [[ -z "$test" ]]; then echo "${new_link} ${new_category} ${new_title}" >> "${DIR_CONFIG}"/"${CONFIG_SERVERLIST}" elif (( VERBOSITY >= 2 )) ; then echo "Feed ${new_title} is already in the serverlist" fi fi # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Close session with '0' status and clean up: {{{ # Disable extended glob matches. shopt -u extglob CLEANUP_AND_EXIT 0 # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # Notes: {{{ # 1. Best viewed in Vim (http://vim.sf.net) with the Relaxedgreen colorscheme (vimscripts #791). # }}} # ---------------------------------------------------------------------------------------------------------------------------------- # vim:tw=132:ts=4:sw=4 ������������������������������������./podget-0.9.3/DOC/���������������������������������������������������������������������������������0000755�0001750�0001750�00000000000�14543630774�013172� 5����������������������������������������������������������������������������������������������������ustar �lowkey��������������������������lowkey�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������./podget-0.9.3/DOC/podget.7�������������������������������������������������������������������������0000644�0001750�0001750�00000050052�14543630774�014546� 0����������������������������������������������������������������������������������������������������ustar �lowkey��������������������������lowkey�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������.\" Hand Tweaked Man Page. .TH podget 7 "10 February 2023" "" "" .SH NAME .B Podget - Simple tool to automate downloading of podcasts. .SH SYNOPSIS .B podget .RB <options> .SH DESCRIPTION Podget is a simple podcast aggregator/downloader optimized for scheduled background jobs (i.e. cron). It features support for: .PD 0 .P - Downloading podcasts from RSS and ATOM XML feeds. .P - For sorting the files into folders and categories. .P - For importing URLs from iTunes PCAST files and OPML lists. .P - Automatic M3U & ASX playlist creation. .P - Cleanup of old files. .P - Automatic UTF-16 conversion for feeds hosted on MS Windows Servers. .PD Podget works by extracting the <enclosure> tags from the feed then downloading the specified URL. There is one exception when Podget will ignore <enclosure> tags and that is when they are within <podcast:liveItem> tags because Podget is an aggregator and not a player so has not been optimized for live content. .SH OPTIONS .TP .B -c <FILE> \fR| \fB--config <FILE> Name of configuration file. .TP .B --create-config <FILE> Create configuration file and exit. .TP .B -C \fR|\fB --cleanup Skip downloading and only run cleanup loop. .TP .B --cleanup_days <NUMBER> Cleanup files older than <NUMBER> days. .TP .B --cleanup_simulate Simulate cleanup loop to see what files would be deleted. .TP .B -d <DIRECTORY> \fR|\fB --dir_config <DIRECTORY> Directory that configuration files are stored in. .TP .B --dir_session <DIRECTORY> Directory that session files are stored in. .TP .B -f \fR|\fB --force Force download of items from each feed even if they've already been downloaded. .TP .B -h \fR|\fB --help Display condensed help dialog. .TP .B -l <DIRECTORY> \fR|\fB --library <DIRECTORY> Directory to store downloaded files in. .TP .B -n \fR|\fB --no-playlist Do not create M3U playlist of new items. .TP .B -p \fR|\fB --playlist-asx In addition to M3U playlists, create ASX playlists. .TP .B --playlist-per-podcast Create a playlist of new items for each podcast feed. .TP .B -r <COUNT> \fR|\fB --recent <COUNT> Download only the <COUNT> newest items from each feed. .TP .B --serverlist <FILE> Use <FILE> as serverlist instead of default. .TP .B -s \fR|\fB --silent Run silently (for cron jobs). .TP .B -v Set verbosity to level 1. .TP .B -vv Set verbosity to level 2. .TP .B -vvv Set verbosity to level 3. .TP .B -vvvv Set verbosity to level 4. .TP .B --verbosity <LEVEL> Set verbosity level (0-4). .TP .B -V \fR|\fB --version Display version. .TP .B OPML List Options: .RS .TP .B --import_opml <FILE or URL> Import servers from OPML file or HTTP/FTP URL. .TP .B --export_opml <FILE> Export serverlist to OPML file. .RE .TP .B PCAST List Options: .RS .TP .B --import_pcast <FILE or URL> Import server from iTunes PCAST file or HTTP/FTP URL. .RE .SH CONFIGURATION FILES By default, Podget relies on two configuration files. .RE .TP .I podgetrc This is a file with most options for how Podget should run. If it is required to run podget with different options for certain feeds, then additional configuration files can be created and used with the --config or -c option. When this option is run with a new filename that does not exist yet, the file is created with default options that can then be customized as necessary. .TP .I serverlist This is a file of all the feeds that Podget should monitor and download from. If you need to separate your feeds into multiple lists, then additional files can be created with the --serverlist option. When this option is run with a new filename that does not exist yet, the file is created with a default list of a single feed. Whenever a new list is created, Podget will download a single item from the single feed included by default to verify that everything is working. For a description of the options available for this file, please refer to the SERVER LIST CONFIGURATION section of this document. .RS .SS USER CONFIGURATION DIRECTORY The first time a user runs podget, it will create a configuration directory. In this directory, it will install the default configuration files. Where this configuration directory is automatically placed is dependent upon the version of Podget that you used when you first ran it. For version 0.8.10 and before: .RS $HOME/.podget .RE For later versions: .RS If $XDG_CONFIG_HOME is set then it will be placed in: $XDG_CONFIG_HOME/podget .RE .RS IF unset, then it will be placed in: $HOME/.config/podget .RE If a user wants to clean up their $HOME directory by moving their existing configuration directory to either of the new locations, it can be done but it is necessary to remember to remove the leading period so it is no longer a hidden directory. .RS Example: mv $HOME/.podget $HOME/.config/podget .RE These locations can be overridden by the use of the --dir_config or -d option when you run podget. .SS WHICH CONFIGURATION DIRECTORY IS USED Since there are at least three possible locations for the configuration directory then it is necessary to know which one podget will use. To keep things simple, Podget uses the first one it finds and tests in the following order: .PP .nf .fam C 1. $HOME/.podget 2. $XDG_CONFIG_HOME/podget 3. $HOME/.config/podget .fam T .fi This location testing is skipped by the use of the --dir_config or -d option. .SS AUTOMATIC CLEANUP You can enable automatic cleanup with every run by configuring it in your podgetrc file. Simply set the following options: .PP .nf .fam C # Autocleanup. # 0 == disabled # 1 == delete any old content cleanup=1 # Number of days to keep files. Cleanup will remove anything # older than this. cleanup_days=7 .fam T .fi However, some people prefer to run cleanup as a separate cron session. To do that, set the options in podgetrc to: .PP .nf .fam C # Autocleanup. # 0 == disabled # 1 == delete any old content cleanup=0 # Number of days to keep files. Cleanup will remove anything # older than this. cleanup_days=7 .fam T .fi Then add something similar to this example to your crontab: .PP .nf .fam C # Once a week on Sunday at 04:07AM 07 04 * * Sun /usr/bin/podget \-C .fam T .fi .SS MULTIPLE CONCURRENT SESSIONS Podget checks for sessions using the same core configuration file that may already be running when it starts and exits if any are found. This insures that any long running sessions are not interrupted by new ones. If you have feeds that require distinct configurations, then you can enable them to run simultaneously by using separate configuration files for each. Then if you have sufficient bandwidth, you can call them all at the same time. Example Crontab configuration: .PP .nf .fam C 00 02 * * * /usr/bin/podget -c podgetrc-group1 00 02 * * * /usr/bin/podget -c podgetrc-group2 .fam T .fi .SS SEQUENTIAL SESSIONS Sometimes, you have feed lists that use the same configuration but you wish to keep separate. There are two ways to handle this. First, run then separately from crontab with sufficient time in between so they don't interfere with each other. .PP .nf .fam C 00 02 * * * /usr/bin/podget --serverlist RSS-Feeds 00 03 * * * /usr/bin/podget --serverlist ATOM-Feeds .fam T .fi The second option is to place them into a shell script so they are called sequentially and do not interfere with each other and then add it to your crontab. .PP .nf .fam C #!/usr/bin/env bash /usr/bin/podget --serverlist RSS-Feeds /usr/bin/podget --serverlist ATOM-Feeds .fam T .fi .SS ENABLING DEBUG OUTPUT Debug output can be enabled in two ways. .PP The first way is by uncommenting the DEBUG option in your podgetrc and setting it to '1'. However this way will not enable DEBUG until just over 1400 lines of script have run and when podgetrc finally is read. This is sufficient for most issues. .PP The second way is from the command-line and enables debug as early as possible. .PP Simply execute podget like so: .PP .nf .fam C $ DEBUG=1 podget -vvvv .fam T .fi .PP You can enable other options as well if you need to but for debugging purposes, it is highly recommended that you enabled as much verbosity as possible. .SS SERVER LIST CONFIGURATION By default, Podget uses serverlist for the default list of servers to contact. However you can configure the name with the config_serverlist variable in your podgetrc file. Feeds are listed one per line in the serverlist file. .PD 0 Default format with category and name: .RS <url> <category> <name> .RE Alternate Formats: .P 1. With a category but no name. .RS <url> <category> .RE 2. With a name but no category (2 ways). .RS .P <url> No_Category <name> .P <url> . <name> .RE 3. With neither a category or name. .RS <url> .RE 1. URL Rules: .RS A. Any spaces in the URL need to be converted to %20 .RE 2. Category Rules: .RS A. Must be one word without spaces. .P B. You may use underscores and dashes. .P C. You can insert date substitutions. .RS %YY% == Year .P %MM% == Month .P %DD% == Day .RE .P D. Category disabling: .RS .P - With a name, the category must either be a single period (.) or 'No_Category'. .P - If the name is blank, the category can also be blank. .RE .RE .P 3. Name Rules: .RS .P A. If you are creating ASX playlists, make sure the feed name does not have any spaces in it and the filename cannot be blank. .P B. You can leave the feed name blank, and files will be saved in the category directory. .P C. Names with spaces are only compatible with filesystems that allow for spaces in filenames. For example, spaces in feed names are OK for feeds saved to Linux ext partitions but are not OK for those saved to Microsoft FAT partitions. .P D. Feed names can be disabled by leaving them blank. .RE .P 4. Disable the downloading of any feed by commenting it out with a leading #. .PD Example: http://www.lugradio.org/episodes.rss Linux LUG Radio Example with date substitution in the category and a blank feed name: http://downloads.bbc.co.uk/rmhttp/downloadtrial/worldservice/summary/rss.xml News-%YY%-%MM%-%DD% Example of two ways to do a feed with authentication: http://somesite.com/feed.rss CATEGORY Feed Name USER:username PASS:password http://username:password@somesite.com/feed.rss CATEGORY Feed Name .RS NOTE: The second method will fail if a colon (:) is part of the username or password. Both methods will fail if a space is part of the username or password. .RE .TP .B Common Options: .TP .I OPT_CONTENT_DISPOSITION Attempt to get filename from the Content-Disposition tag that is part of wget --server-response. .TP .I OPT_DISPOSITION_FAIL This option works in conjunction with OPT_CONTENT_DISPOSITION by removing any URLs that fail to receive a filename from the COMPLETED log. This allows them to be automatically retried the next time a session runs. If this option is added to a feed that has already been downloaded then the user will need to remove the URLs for the problematic files from the COMPLETED log manually. On one feed this allowed for the improvement of the number of filename problems from approximately 15% to under 2% over the course of 6 sessions. Those sessions can occur sequentially on one day or as part of your established cron rotation. .TP .I OPT_FEED_ORDER_ASCENDING By default, Podget assumes that items in a feed will be listed from newest to oldest (descending order). This option will modify Podget's handling of the feed for those that are listed from oldest to newest. This option will not have any noticeable effect for feeds where you want to download every item. It will have an effect for new feeds when combined with the --recent [COUNT] option. .TP .I OPT_FEED_PLAYLIST_NEWFIRST Most playlist options create lists of just the new items that are downloaded in the current session. This option creates or updates a full playlist for all items available for a feed sorted from newest to oldest based on the modification date/time of the file. .TP .I OPT_FEED_PLAYLIST_OLDFIRST Same as OPT_FEED_PLAYLIST_NEWFIRST except playlist is ordered from oldest to newest. .TP .I OPT_FILENAME_LOCATION Some feeds do not have the detailed filename listed in the FEED but rather rename the file on redirection. This option addresses that issue by attempting to grab the filename from the last 'Location:' tag in the output of 'wget --server-response'. .TP .I OPT_FILENAME_RENAME_MDATE For feeds that use a singular filename for each item that is identified by a long somewhat incomprehensible string in the URL. These feeds were previously fixed with FILENAME_FORMATFIX4 which would append the string to the common filename to produce unique filenames for each item. However this produced filenames that were not very easy to understand. This option gives us another method for dealing with these common filenames. This appends the date of the files last change (modification date) as a prefix to the filename in the format of YYYYMMDD_HHhMMm_<common-part>. This makes the filenames sortable and gives the user something that makes a moderate amount of sense. Does not work for all feeds, for some feeds the last modification time for each file is the time of download. Which may be acceptable in some situations but can cause confusion when downloading more than one item at a time from a feed. .TP .I OPT_WGET_DEFUSERAGENT Configure Wget to use it's default user-agent (normally formatted similar to "Wget/1.21.2") and to not use either Podget's default user-agent ("Podget") or a custom agent set in WGET_BASEOPTS in podgetrc. .TP .I OPT_NO_CERT_CHECK Disable wget SSL certificate verification. This is common used for feeds that are using self-signed certificates. .TP .I OPT_PREFER_IPv4 \fRor\fI OPT_PREFER_IPv6 Configure wget so that when a DNS lookup gives a choice of several addresses that it should connect to the specified family first. .P Examples: http://somesite.com/feed.rss CATEGORY Feed Name OPT_PREFER_IPv4 http://somesite.com/feed.rss CATEGORY Feed Name OPT_PREFER_IPv6 http://somesite.com/feed.rss CATEGORY Feed Name OPT_WGET_DEFUSERAGENT http://somesite.com/feed.rss CATEGORY Feed Name OPT_NO_CERT_CHECK http://somesite.com/feed.rss CATEGORY Feed Name OPT_CONTENT_DISPOSITION http://somesite.com/feed.rss CATEGORY Feed Name OPT_CONTENT_DISPOSITION OPT_DISPOSITION_FAIL http://somesite.com/feed.rss CATEGORY Feed Name OPT_FILENAME_LOCATION http://somesite.com/feed.rss CATEGORY Feed Name OPT_FILENAME_RENAME_MDATE http://somesite.com/feed.rss CATEGORY Feed Name OPT_FILENAME_LOCATION OPT_FILENAME_RENAME_MDATE http://somesite.com/feed.rss CATEGORY Feed Name OPT_FEED_ORDER_ASCENDING http://somesite.com/feed.rss CATEGORY Feed Name OPT_FEED_PLAYLIST_NEWFIRST http://somesite.com/feed.rss CATEGORY Feed Name OPT_FEED_PLAYLIST_OLDFIRST .TP .B RSS Feed Options: There are three options for RSS Feeds that are not supported for ATOM feeds. The first two are related with the renaming the downloaded files with the contents of the <TITLE> tag from the HTML and the third is to expand what tags Podget gets content from. .TP .I OPT_FILENAME_RENAME_TITLETAG This first version is for handling feeds that place the <TITLE> tag before the <ENCLOSURE> tag. The majority of tested feeds that use <TITLE> tags follow this order. .TP .I OPT_FILENAME_RENAME_REVTITLETAG The second version is for handling feeds that have the <ENCLOSURE> tag first followed by the <TITLE> tag. .TP .I OPT_RSS_MEDIACONTENT This third option will enable Podget to download content from <MEDIA:CONTENT> tags in addition to <ENCLOSURE> tags. .P Examples: http://somesite.com/feed.rss CATEGORY Feed Name OPT_FILENAME_RENAME_TITLETAG http://somesite.com/feed.rss CATEGORY Feed Name OPT_FILENAME_RENAME_TITLETAG OPT_FILENAME_RENAME_MDATE http://somesite.com/feed.rss CATEGORY Feed Name OPT_FILENAME_RENAME_REVTITLETAG http://somesite.com/feed.rss CATEGORY Feed Name OPT_RSS_MEDIACONTENT To determine if the feed uses <TITLE> tags and in which order, run the following with the URL for the feed: .PP .nf .fam C wget -O - http://somesite.com/feed.rss | sed -n -e :a -e 's/.*<enclosure.*url\\s*=\\s*"\\([^"]\+\\)".*/URL \1/Ip' -e t -e "s/.*<enclosure.*url\\s*'=\\s*\\([^i]\\+\\)'.*/URL \\1/Ip" -e t -e 's/.*<title>\\(.*\\)<[/]title>.*$/TITLE \1/Ip' -e t -e '/\\(<enclosure\\|<title>\\).*/I{N;s/\ *\n/\ /;T;ba}' .fam T .fi This will produce a list of lines that start with either TITLE or URL. The URL is from the <ENCLOSURE> tag and the TITLE is obviously from the <TITLE> tag. On many feeds the first thing you will notice is a few uses of the <TITLE> tag before the first URL is specified. In that case, Podget uses the last TITLE found, so the earlier ones are discard. The important part is when we get to the first URL, from there we need to determine if the title for that item came before or after the URL. If it comes first then we use OPT_FILENAME_RENAME_TITLETAG for it. If the title comes second then we use OPT_FILENAME_RENAME_REVTITLETAG. On some feeds, the downloaded filename will not have anything identifiable to determine which TITLE goes with it. In those cases it may be necessary to download a few items and listen to them to determine which order they use. On some feeds, it will be discovered that the downloaded filename and the TITLE are very similar. In those cases, it is left to the user to determine which they prefer. On some feeds, the TITLE will have very little to specify when it was recorded and it may be useful to use the OPT_FILENAME_RENAME_MDATE option to add a date tag to each filename as it is converted. And on some feeds, there will be a complete absence of TITLE lines. Those feeds do not use the tag so using either option will not produce any changes. .TP .B Atom Feed Options: The following options are available for advanced handling of Atom feeds. .TP .I ATOM_FILTER_SIMPLE This option will enable filtering for just audio or video files from a feed. .TP .I ATOM_FILTER_TYPE="type" This option allows more detailed filtering of the variety of types available. This can limit the files downloaded to one type (example: "audio/mpeg") or to a few types (example: "(audio|video)/.*" for all audio and video types, OR "audio/.*" for all audio types). .TP .I ATOM_FILTER_LANG="language" If an Atom feed supports multiple languages for enclosures, then you can use this option to filter to only those you desire. You can limit to one language (example: "en" for just English) or combine several supported languages to get them all (example: "(en|es|fr)" to download files in English, Spanish and French. How the languages are defined may vary from feed to feed. .P Note: If you do not enable any of the ATOM_FILTER options on a feed with multiple enclosures per item, when you run podget it will tell you the count per type or language to help you decide if you should enable the filters to reduce the number of files to be downloaded. .P Examples: http://somesite.com/feed CATEGORY Feed Name ATOM_FILTER_SIMPLE http://somesite.com/feed CATEGORY Feed Name ATOM_FILTER_TYPE="audio/mpeg" http://somesite.com/feed CATEGORY Feed Name ATOM_FILTER_TYPE="(audio|video)/.*" http://somesite.com/feed CATEGORY Feed Name ATOM_FILTER_LANG="en" http://somesite.com/feed CATEGORY Feed Name ATOM_FILTER_LANG="(en|es|fr)" http://somesite.com/feed CATEGORY Feed Name ATOM_FILTER_TYPE="audio/mpeg" ATOM_FILTER_LANG="en" .SS HANDLING UTF-16 FEEDS .PP Some servers provide their feeds in UTF-16 format rather than the more common UTF-8. .PP To automatically convert these files, create a secondary serverlist in your configuration directory: .PP .nf .fam C serverlist.utf16 .fam T .fi Remember to change the name of the serverlist to match what you set it to with config_serverlist if you changed it. .SH EXAMPLE CRON JOB Once podget\ is running correctly, it's most useful if you run it from a cron job so that the new episodes are available to play or load onto a portable player and you don't have to wait for them to download. .PP To edit your crontab, do: .PP .nf .fam C $ crontab \-e .fam T .fi Then add one line similar to this example: .PP .nf .fam C 15 04 * * * /usr/bin/podget \-s .fam T .fi This will run podget at 4:15 AM every day. In some cases, you might need to add a few directories to your PATH variable so that Podget can find everything it needs. Then the job might look like: .PP .nf .fam C 15 04 * * * PATH=/opt/local/bin:/usr/local/bin:$PATH /usr/bin/podget \-s .fam T .fi .SH AUTHORS Dave Vehrs ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������./podget-0.9.3/README.md����������������������������������������������������������������������������0000644�0001750�0001750�00000001063�13742642167�014043� 0����������������������������������������������������������������������������������������������������ustar �lowkey��������������������������lowkey�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������Podget is a simple podcast aggregator optimized for running as a scheduled background job (i.e. cron). It features support for downloading podcasts from RSS & ATOM XML feeds, for sorting the files into folders & categories, for importing URLs from iTunes PCAST files & OPML lists automatic M3U & ASX playlist creation, and automatic cleanup of old files. It also features automatic UTF-16 conversion for podcasts hosted on MS Windows servers. In version 0.8.5, we added support for Mac OS X. In version 0.8.7, we added support for FreeBSD, NetBSD and OpenBSD. �����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������./podget-0.9.3/COPYING������������������������������������������������������������������������������0000644�0001750�0001750�00000104513�12620220525�013603� 0����������������������������������������������������������������������������������������������������ustar �lowkey��������������������������lowkey����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������� GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU General Public License is a free, copyleft license for software and other kinds of works. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others. For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it. For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions. Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users. Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Use with the GNU Affero General Public License. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. <one line to give the program's name and a brief idea of what it does.> Copyright (C) <year> <name of author> This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. Also add information on how to contact you by electronic and paper mail. If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode: <program> Copyright (C) <year> <name of author> This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an "about box". You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see <http://www.gnu.org/licenses/>. The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read <http://www.gnu.org/philosophy/why-not-lgpl.html>. �������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������./podget-0.9.3/INSTALL������������������������������������������������������������������������������0000644�0001750�0001750�00000015146�14541621047�013614� 0����������������������������������������������������������������������������������������������������ustar �lowkey��������������������������lowkey����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������� General Source Install: $ make $ sudo make install After the package is installed, run it once to install default configuration files into $HOME/.podget, edit them as needed. Building and installing a Debian package: $ cd podget-<version> $ fakeroot ./debian/rules binary $ cd ../ $ sudo dpkg -i podget_<version>-all.deb Mac OSX GNU Dependencies: Podget requires GNU Date, GNU Stat, GNU Sed, Coreutils (for expr, and tr) and Bash (at least version 4.4). The easiest way to install them is to use Homebrew. For details on how to install and use Homebrew, please visit: https://brew.sh An updated version of Bash can be installed with: brew install bash GNU Date can be installed with: brew install gdate GNU Stat can be installed with: brew install gstat GNU Sed can be installed with: brew install gnu-sed And Coreutils can be installed with: brew install coreutils. Podget looks for the required binaries with a 'g' prefix. This prefix is added automatically by brew with its default options. The prefix can be overridden with the brew option '--with-default-names'. However if you do this, it may be necessary that you add symbolic links from the brew installed sed to gsed and for the others. Additionally installing packages with brew may require some changes to the PATH variable setting described below. FreeBSD Dependencies and Install: Podget requires a few things that are not part of a default installation. As root (or with doas) using 'pkg install' add: - bash (at least version 4.4) - wget - coreutils - gsed gmake - ca_root_nss NOTE: pkg will be automatically installed the first time you attempt to use it. - gnustat NOTE: When testing with FreeBSD 13.1, gnustat was part of the coreutils package and so did not require a separate install. Source Install: $ gmake $ doas gmake install OR su root -c 'gmake install' After the package is installed, run it once to install default configuration files into $HOME/.podget, edit them as needed. NetBSD Dependencies and Install: Podget requires a few things that are not part of a default installation. As root (or with doas) using pkgin install: - bash (at least version 4.4) - wget - ca-certificates - coreutils - gsed - gmake NOTE: After installing ca-certificates, run update-ca-certificates to manage the set of configured trust anchors for openssl. Source Install: $ gmake $ doas gmake install OR su root -c 'gmake install' After the package is installed, run it once to install default configuration files into $HOME/.podget, edit them as needed. OpenBSD Dependencies and Install: Podget requires a few things that are not part of a default installation. As root (or with doas) using pkg_add install: - bash (at least version 4.4) - wget - coreutils - gsed - gmake Source Install: $ gmake $ doas gmake install OR su root -c 'gmake install' After the package is installed, run it once to install default configuration files into $HOME/.podget, edit them as needed. Microsoft Windows 10+ Installation: Podget can be run on current installations of Windows by installing Debian GNU/Linux into the Windows Subsystem for Linux (WSL). Centralized Notes on how to install Debian within Windows: https://wiki.debian.org/InstallingDebianOn/Microsoft/Windows/SubsystemForLinux NOTE: You do not need to follow the 'Advanced Usage' part as Podget is not a GUI app and will not require that configuration. However you may want to enable GUI support so you can use simple GUI text editors to modify the configuration files. If you prefer to do everything in the console then you may need to learn to use a console text editor like nano, vi, vim or ed editors (just kidding, probably stay away from ed unless you want a real challenge). After Debian is installed, within it you will need to run: 1. sudo apt update 2. sudo apt upgrade 3. sudo apt install podget Once Podget is installed, the general location for its files can be found in directory like: ``` C:\Users%username%\AppData\Local\Packages\TheDebianProject.DebianGNULinux_76v4gfsz19hv4\LocalState\rootfs\home\ ``` However the string after DebianGNULinux_ may vary on each install. Crontab Potential Issue: Occasionally when you install Podget, it will run fine from the shell prompt but experiences a problem when you attempt to run it as a cron job. When this happens it may be caused by Podget not being able to find every command it needs. This can be caused by differences in the PATH variable which contains a list of directories to search for commands. This list is delimited by colons and read from left to right so if a command of the same name exists in two directories, the one you want should be read first. An example of this happened on MacOS. What we discovered was that for cronjobs the PATH variable was set to "/usr/bin:/bin" which may be a sensible default but lacked a few directories we needed. On MacOS, there are a few applications that can be useful to determine what we need to add to the PATH variable. Those applications are gexpr, gdate, gsed and gstat. We can use the which, whereis or find commands to determine where they are on our system. On the system we were having a problem with, we discovered that some of these applications were in /opt/local/bin and others were in /usr/local/bin so we needed to add those directories to our PATH variable for Podget to work. So we changed our cronjob from: 15 04 * * * /usr/bin/podget -s To: 15 04 * * * PATH=/opt/local/bin:/usr/local/bin:$PATH /usr/bin/podget -s If you wish to see the differences in how PATH is configured: FOR THE SHELL: Simply run "echo $PATH" FOR A CRONJOB: 1. Create simple shell script /tmp/test_cron.sh which contains: #!/usr/bin/env bash echo "PATH: $PATH" 2. Configure a cronjob to run the script with "crontab -e" * * * * * /tmp/test_cron.sh > /tmp/test_cron.out NOTE: This job will run every minute so disable it as soon as it has run once or twice. 3. Open the file /tmp/test_cron.out in an editor to see how it has the PATH variable configured. ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������./podget-0.9.3/Makefile�����������������������������������������������������������������������������0000644�0001750�0001750�00000001220�13116061175�014204� 0����������������������������������������������������������������������������������������������������ustar �lowkey��������������������������lowkey�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������UNAME:= $(shell uname -s) ifeq ($(UNAME), Darwin) prefix ?= /usr/local else prefix ?= /usr endif all: podget podget.7.gz changelog.gz podget.7.gz: DOC/podget.7 gzip -9 < $< > DOC/$@ changelog.gz: Changelog gzip -9 < $< > $@ install: all mkdir -p $(DESTDIR)$(prefix)/bin/ install -m 755 podget $(DESTDIR)$(prefix)/bin/podget mkdir -p $(DESTDIR)$(prefix)/share/man/man7/ install -m 644 DOC/podget.7.gz $(DESTDIR)$(prefix)/share/man/man7/podget.7.gz mkdir -p $(DESTDIR)$(prefix)/share/doc/podget install -m 444 changelog.gz $(DESTDIR)$(prefix)/share/doc/podget/changelog.gz distclean: clean clean: rm -f DOC/podget.7.gz rm -f changelog.gz ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������./podget-0.9.3/Changelog����������������������������������������������������������������������������0000644�0001750�0001750�00000100044�14544354655�014400� 0����������������������������������������������������������������������������������������������������ustar �lowkey��������������������������lowkey�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������podget (0.9.3) unstable; urgency=low * Thanks to Ludovic Bellière (xrogaan on Github) for reporting an issue with duplicate logging to the completed and errors log when using the recent option, a second issue with potential user confusion when marking files as already downloaded (also because of recent option) because we used the URL filename and not the TITLE of the item when we could, and a third issue deleting zero length files when we had modified the filename to remove some dangerous characters. All three issues have been fixed. * Thanks to Francois Marier for reporting an issue stemming from Wget's default user-agent. Turns out a few feeds are filtering for and denying downloads for anything reporting it. To address this issue, we added three options. First Podget now uses a different user-agent ("Podget") by default. Second, it is now possible to use a longer user-agent to mimic common browsers. And third, there is an option for a single feed to go back to using the default Wget user-agent. * Thanks to aab3r on Github for reporting an issue with feeds provided by the FreshRSS Feed Aggregator. When they offer a feed for content they have already downloaded the media content is not in ENCLOSURE tags but rather in MEDIA:CONTENT tags. We resolved this issue by providing a configuration option to enable both tags to be examined for URLs to download. * Skipped version 0.9.2 because of a small issue discovered while uploading Debian repositories. Changes consisted of a single character ($) being deleted in the WGET_TEST_STRING and a descriptive comment being added. -- Dave Vehrs <dvehrs@gmail.com> Fri, 29 Dec 2023 13:29:57 -0700 podget (0.9.1) unstable; urgency=low * Thanks to Joerg Schiermeier for reporting a bug in our automated cleanup loop that occurred when no playlists were found (Debian Bug# 1027121). * Added OPT_FILENAME_RENAME_TITLETAG to change name of downloaded file to what was specified in the <TITLE> tag from the feed. Included some basic cleanup so it would be friendly to most filesystems. Updated MAN page to include this option. * Updated NetBSD and FreeBSD sections of the INSTALL file. * Updated checks for MacOS, and the BSDs to check for gtr (GNU Translate). The BSD version of tr was having some issues with feeds from countries that did not use single byte character sets. The GNU version does it from a single byte perspective so is not affected by the same issue and happens to be a bit faster to run. * Thanks to Bobabadabouskie (on Github) for reporting an issue with how Podget was handling podcast liveItem tags. And thanks to Adam Curry (adamc199 on Github) for providing links to the Podcast Namespace specs detailing its format. From this report and data, added a section to the feed filtering that ignores any liveItem data that is found because Podget is not appropriate for listening to live streams. -- Dave Vehrs <dvehrs@gmail.com> Thu, 30 Mar 2023 13:48:49 -0600 podget (0.9.0) unstable; urgency=low * Thanks to Pariksheet Nanda (omsai on Github) for finding an update for our filename cleanups. They now handle m4a filetypes. * Added support for storing configuration files in either $HOME/.podget, $XDG_CONFIG_HOME/podget, or $HOME/.config/podget so that we are more in alignment with the XDG Base Directory Specification. Thanks to jeffgazso (on Github) for reporting the issue. * During command line option parsing, added "|| :" to second shift for some options to bypass error generated when option was last on a line with no value specified. * Fixed quoting of a few unset commands to address Shellcheck SC2184 * Added quotes inside a ${} variable to prevent pattern matching, addressing Shellcheck SC2295 -- Dave Vehrs <dvehrs@gmail.com> Tue, 19 Apr 2022 11:12:06 -0600 podget (0.8.10) unstable; urgency=low * SCRIPTS: Update to script provided by Mike Webb to show that it is licensed with the GPL version 3. -- Dave Vehrs <dvehrs@gmail.com> Thu, 30 Sep 2021 14:10:33 -0600 podget (0.8.9) unstable; urgency=low * SCRIPTS: Thanks to Mike Webb (theorbitofmercury on Github) for providing us with a python3 script to send a list of recent downloaded podcasts to Pushover on Android or iOS. * Thanks to Andrew Scott (scottmeup on Github) for finding a bug in our cleanup loop and working with us to find a solution. -- Dave Vehrs <dvehrs@gmail.com> Thu, 09 Sep 2021 13:16:15 -0600 podget (0.8.8) unstable; urgency=low * Mac OS Issues: Added gstat and gdate dependencies, updated man page to include a cronjob example with additional directories added to the PATH variable, updated INSTALL file with a more verbose description of what might need to be done. Thanks to Andrew Scott (scottmeup on Github) for discovering these issues and assisting in finding solutions. * Expanded description for automatic cleanup for new podgetrc files. -- Dave Vehrs <dvehrs@gmail.com> Tue, 19 Jan 2021 13:43:53 -0700 podget (0.8.7) unstable; urgency=low * Unified option filtering into single function. * Replaced 'eval' statements with 'printf' statements. * Removed FEED_FULL_PLAYLIST from Debug statements prior to it being needed. * Fixed playlist cleanup to skip global playlists newer than the cleanup date. Thanks to Kevin Wisher (kwisher on Github) for reporting this. * Added test for shopt inherit_errexit support to verify that at least Bash version 4.4 was installed. Thanks to Jude DaShiell for reporting the issue. * Thanks to Eric Cook (llua@gmx.com) for submitting a patch that updated the way required GNU applications are called simplifying script support, and added support for FreeBSD, NetBSD and OpenBSD. * On a per feed basis, added option flags to set WGET to disable SSL certificate verification and which network address family (IPv4 or IPv6) it should prefer when DNS provides a few choices. * Updated manpage and moved away from txt2man conversion. -- Dave Vehrs <dvehrs@gmail.com> Sat, 17 Oct 2020 12:14:28 -0600 podget (0.8.6) unstable; urgency=low * Improved DEBUG to allow it to be set from command line. * Fixed FILENAME_FORMATFIX9 to include a dash ('-') in removed part. * Applied patch inspired by Bob Hepple (bhepple on Github) to allow for items on server lists to have both blank categories and names. Many thanks for the great idea and help testing it. * Updated man page. * Fixed automatic creation of serverlist file so that core configuration file was read prior to testing for existence of the file. Thanks to Corin-EU for reporting this issue on Github. * Fixed Cleanup Loop to work based on individual file modification times from each playlist. Thanks to StanESmith for reporting this issue on Github. * Fixed FILENAME_FORMATFIX9 to cleanup one more formatting issue. Thanks to Gerhard Großmann (charakterziffer) for reporting this issue on Github. * Converted implicit to explicit escaping in double quoted strings (example: '\n' to '\\n') * Changed display of "Build OPT_FEED_PLAYLISTs" to only be shown if an active feed is configured to use it. * Changed display of UTF loop messages from verbosity 3 to debug. * Changed ATOM_FILTER statements to not require quotes. * Removed EXPR statements from individual playlist creation. -- Dave Vehrs <dvehrs@gmail.com> Fri, 27 Sep 2019 09:39:47 -0600 podget (0.8.5) unstable; urgency=low * Fixed OPML Import to favor xmlUrl of htmlUrl. Thanks to kuba-orlik on Github. * Added checks for GNU versions of two commands to enable support for Mac OSX. Thanks to Bruce Ingalls (bingalls on Github) for the inspiration for these changes and his assistance in making them work. * User Contributed Script: pod2player.sh by Bruce Ingalls. A BASH script to easily move media files to a player device. -- Dave Vehrs <dvehrs@gmail.com> Wed, 07 Jun 2017 14:34:22 -0600 podget (0.8.4) unstable; urgency=low * Replaced sections where errexit was disabled. * Fixed Changelog to show correct version for 0.8.3 released on October 7, 2016. * Fixed removal of leading or trailing whitespace from a Feed name to include support for removing tabs. Thanks to Corin-EU for reporting on Github. * Added OPT_FEED_PLAYLIST_NEWFIRST and OPT_FEED_PLAYLIST_OLDFIRST support to serverlist parsing to enable the creation and updating of playlists of all items (not just new) for a feed. Thanks to Capturts for requesting the feature on Github and DrAzraelTod for submitting an example for inspiration. -- Dave Vehrs <dvehrs@gmail.com> Fri, 24 Mar 2017 11:19:24 -0600 podget (0.8.3) unstable; urgency=low * Improved feed handling to allow for correct handling of feeds with either descending sort order (newest to oldest [DEFAULT]) or ascending sort order (oldest to newest). Ascending enabled with OPT_FEED_ORDER_ASCENDING tag in the serverlist file for the feed. * A few new Shellcheck fixes. -- Dave Vehrs <dvehrs@gmail.com> Fri, 07 Oct 2016 10:26:41 -0600 podget (0.8.2) unstable; urgency=low * Completed replacement of arithmetic comparison to shell built-in '(())' * Replace use of test '[]' with shell built-in new test '[[]]' * Added replacement of spaces in URLs with '%20' to address bug submitted by Piyush Purushothaman. * Thanks to Kevin Wisher for reporting an conflict between errexit being enabled and WGet exiting with a 403 error. Modified function to capture the exit status and determine how to proceed based on it. -- Dave Vehrs <dvehrs@gmail.com> Tue, 12 Jul 2016 09:10:17 -0600 podget (0.8.1) unstable; urgency=low * Kitcat490 on Github reported that version 0.8 was failing to load new configuration files because the tagged version number was not being parsed correctly. Turns out the issue was due to the new version having a two number version rather than the three that older versions commonly had. Updated the parsing strings to work with 2, 3 or more sections. * Cleanup of approximately 28 Shellcheck reported issues. * Attempt to make script more readable by replacing use of test '[]' for arithmetic comparisons with shell built-in '(())'. * Enabled Extended Pattern Matching (shopt -s extglob) * Small cleanup of output text. -- Dave Vehrs <dvehrs@gmail.com> Wed, 08 Jun 2016 11:01:53 -0600 podget (0.8) unstable; urgency=low * Thanks to hugh-m on Github for reporting Podget's lack of support for Atom feeds and for helping to test our first implementation of a few options this new feed type. * Added support for Atom feeds along with a few serverlist options (ATOM_FILTER_SIMPLE, ATOM_FILTER_TYPE, & ATOM_FILTER_LANG) -- Dave Vehrs <dvehrs@gmail.com> Sun, 05 Jun 2016 10:43:09 -0600 podget (0.7.14) unstable; urgency=low * Thanks to angoongo for submitting a report for an unbound variable error. * Thanks to Chris for reporting issue that lead to the development of a function to prefix filenames with their modification dates (OPT_FILENAME_RENAME_MDATE). * Modified filename renaming to properly respect the force option (--force), this affects OPT_FILENAME_LOCATION, OPT_CONTENT_DISPOSITION, OPT_FILENAME_RENAME_MDATE, and FILENAME_SUFFIX. * Restored modified filename handling immediately around the wget download commands. Fixed a misguided attempt at script cleanup that resulted in an error with iTunes feeds reported by Francois Marier. -- Dave Vehrs <dvehrs@gmail.com> Tue, 10 May 2016 14:02:23 -0600 podget (0.7.13) unstable; urgency=low * Thanks to Marcus Stollsteimer for submitting corrections to the man pages and help text. * Thanks to Marcus Stollsteimer for reporting a bug in the REMOVE_URL function where we failed to account for a suitable delimiter character in a sed statement. By moving the offending sed statement after our test for suitable character the issue was resolved. -- Dave Vehrs <dvehrs@gmail.com> Wed, 30 Mar 2016 16:31:11 -0600 podget (0.7.12) unstable; urgency=low * Thanks to Filip Szymański for reporting a second unbound variable error. * Thanks to Marcus Stollsteimer for submitting a couple of patches for the INSTALL and Makefile files on Github. Both have been merged. -- Dave Vehrs <dvehrs@gmail.com> Sat, 19 Mar 2016 10:45:08 -0600 podget (0.7.11) unstable; urgency=low * Thanks to Willem and our friends at Arch for reporting that we had an unbound variable error when using the --create-config functions. Fixed by setting a default for DEBUG that can be used at this stage and modified later by the value in a users configuration file. * General cleanup. -- Dave Vehrs <dvehrs@gmail.com> Sat, 19 Mar 2016 06:25:54 -0600 podget (0.7.10) unstable; urgency=low * Many thanks to Nicolae Lucian Sandor for his work on proving that we can use wget's --content-disposition and --server-response flags. With these enabled, we have been able to rename files based on values held in the Content-Disposition or Location tags. Without his inspired ideas these updates would not have been possible. * Thanks to ael for reporting an unbound variable (BINARY_ICONV). I changed how I was checking for it's installation and didn't catch all the necessary updates. * Thanks to Francois Marier for reporting a couple issues with the new DIRECTORY_CHECK function that had evaded my testing. -- Dave Vehrs <dvehrs@gmail.com> Thu, 17 Mar 2016 17:07:36 -0600 podget (0.7.9) unstable; urgency=low * Many thanks to Filip Szymański for pointing out that while I had updated the Podget script to GPLv3, I failed to update the COPYING file included in the package. This was a small but important oversight and it has now been corrected. * Fixed grep statement in REMOVE_URL to search for '--fixed-strings' so that certain characters (like ^) were not interpreted as regular expressions. * Added DIRECTORY_CHECK function to verify that none of the characters listed in FILENAME_BADCHARS are used in directory names. * Updated FILENAME_CHECK to match FILENAME_BADCHARS testing from DIRECTORY_CHECK, and made both checks work with variable indirection. -- Dave Vehrs <dvehrs@gmail.com> Sun, 08 Nov 2015 14:21:05 -0700 podget (0.7.8) unstable; urgency=low * Fixed error when changing number of days to cleanup. Thanks to Kevin Wisher for the report! * General cleanup of a few potential issues reported by shellcheck -- Dave Vehrs <dvehrs@gmail.com> Sat, 10 Oct 2015 16:08:55 -0600 podget (0.7.7) unstable; urgency=low * Fixed CLEANUP_AND_EXIT function to suppress extra blank line produced while in silent mode (verbosity=0). Thanks to Colin Turner and Brad Beyenhof for reporting this issue. * Added REMOVE_URL function to remove entries from the LOG_FAILED when they later successfully download. Thanks to Luci Sandor for reporting this and submitting an idea that grew into this function. -- Dave Vehrs <dvehrs@gmail.com> Sun, 04 Oct 2015 14:01:26 -0600 podget (0.7.6) unstable; urgency=low * Removed 'Last Modified' from podget header because of conflicts it was causing with git and building Debian packages. Also removed 'Created' because it was unneeded. * Updated parsing of Feed pages to account for newlines in enclosure tags. Thanks to Todd Harbour for the patch. * Fixed OPML and Pcast feed imports to disable 'errexit' while running grep tests. Thanks to Jude DaShiell for reporting the issue. * Added improved handling for unset --import_opml, --export_opml and --import_pcast values. Thanks again to Jude for reporting the issue. -- Dave Vehrs <dvehrs@gmail.com> Wed, 06 May 2015 11:32:57 -0600 podget (0.7.5) unstable; urgency=low * Fixes to concurrent session checking procedure. * Suppress two warning messages when command-line options override configuration file and --silent has been specified on the command-line. Thanks to Willem for pointing out this issue. * Patch submitted by Dan Rich for improved removal of extra spaces from start or end of a feed name after removing username & password. Used in two places. Thanks. * Patch submitted by Dan Rich that removed extra '=' when username was added to the wget options. Thanks again. -- Dave Vehrs <dvehrs@gmail.com> Fri, 12 Dec 2014 15:01:18 -0700 podget (0.7.4) unstable; urgency=low * Fixes to man page. Thanks again to Willem. * Fixes to recent filters. Thanks again to Willem. * Fixes to cleanup loop checks. Thanks to Andi. * Added check to remove downloaded file if download exits with error and size equals zero. -- Dave Vehrs <dvehrs@gmail.com> Mon, 27 Oct 2014 14:50:43 -0600 podget (0.7.3) unstable; urgency=low * Continued script cleanup. * Added new function to remove temporary files when script exits. * Added function to test local configuration and log file names. -- Dave Vehrs <dvehrs@gmail.com> Wed, 20 Aug 2014 15:13:56 -0600 podget (0.7.2) unstable; urgency=low * Fix for --silent. It was outputting a single black line that needed to be suppressed for verbosity < 1. Thanks to Willem for reporting it. -- Dave Vehrs <dvehrs@gmail.com> Mon, 11 Aug 2014 10:26:35 -0600 podget (0.7.1) unstable; urgency=low * Converted unix2dos statement to a sed statement to reduce dependencies. * Added exit statement after configuration files created with --create-config to allow the user to adjust the files prior to any actual downloading. (Thanks to Willem and our friends at Arch Linux). * Added check to see if --create-config was issued with a filename that is already in use. Exited with error that informs the user of their choice to delete the old file or use a different name. (Thanks again to Willem). * Updated check for PLAYLIST_NAME. (Thanks again to Willem). * Added checks for NO_PLAYLIST. * Updated MAN page. (Thanks yet again to Willem). -- Dave Vehrs <dvehrs@gmail.com> Sun, 10 Aug 2014 10:07:16 -0600 podget (0.7.0) unstable; urgency=high * Modified configuration file format. Users will need to update their configuration files before continuing use of podget. If older configuration files are detected, podget will automatically exit. * Code Cleanup. Worked to standardize variable naming, etc. * Added 'SUFFIX' option to tag all new podcast filenames with a suffix to make it easy to run subsequent scripts to modify the newly downloaded files. For example, these scripts may use id3v2 to standardize the genre of the files or use mp3gain to normalize the volumes. * Enabled shell attributes for errexit, pipefail and nounset. These should help to make a more robust and less error-prone script. -- Dave Vehrs <dvehrs@gmail.com> Fri, 8 Aug 2014 08:47:29 -0600 podget (0.6.18) unstable; urgency=low * Added new ability to handle authentication for premium podcasts by the user adding a couple tags (USER: and PASS:) to the podcast description in their serverlist file. Many thanks to Dan Rich and Stephen Michael Blott for their ideas and suggested implementations. After mixing their ideas with the need for backwards compatibility, we've come up with a solution that needs more testing but hopefully will work for all. -- Dave Vehrs <dvehrs@gmail.com> Mon, 19 May 2014 05:13:30 -0600 podget (0.6.17) unstable; urgency=low * Added default for playlist creation to automatically create playlist unless disabled on the command line or in configuration file to resolve bug reported by Yuri D'Elia (Debian Bug #742905) * Divided creation of configuration directory and files up into three parts to provide greater error reporting possibilities. Expands opportunities to use them as well. (need to expand man page to show how & when) * Fixed bug with handling of playlist creation and individual podcast feed playlists * Added header to podgetrc file that includes version string that can be checked to determine if any updates should be applied to it -- Dave Vehrs <dvehrs@gmail.com> Fri, 28 Mar 2014 15:32:39 -0600 podget (0.6.16) unstable; urgency=low * Added option for to disable playlist creation. * Added filename_formatfix10 to extract the podcast episode name from the enclosure URLs within Apple ITunes feeds. Thanks to Francois Marier for reporting the issue. * Thanks to Chris Elvidge for helping to find and fix a few bugs with date_format, and configuration file parsing. -- Dave Vehrs <dvehrs@gmail.com> Wed, 26 Mar 2014 10:18:34 -0600 podget (0.6.15) unstable; urgency=low * Moved session files to be created and stored in /tmp by default rather than the old location ${HOME}/.podget. Also created dir_session configuration file variable so the user can reassign as they desire. * Added test for ownership of session files so multiple users can use the same session directory and their sessions won't conflict with each other. * Fixed so that podget can properly handle spaces in filenames. * Include patch by Simon Rutishauser to enable export of the server list as an OPML file. -- Dave Vehrs <dvehrs@gmail.com> Tue, 17 Aug 2013 16:04:21 -0600 podget (0.6.14) unstable; urgency=low * Updated filename format fix checks so that they do not produce an error if that fix is not defined in your podgetrc. * Created filename_formatfix9 to address issues with the Audioboo feed URL format. Removes "?keyed=true&source=rss" from the enclosure URL. Tried to make the string matches as generic as possible so that it might work for other feeds. Issue submitted on Sourceforge as bug #3609600 by Markus. Many thanks! -- Dave Vehrs <dvehrs@gmail.com> Tue, 5 Apr 2013 09:27:01 -0600 podget (0.6.13) unstable; urgency=low * Update to default podgetrc contents to further explain the Smodcast fixes. -- Dave Vehrs <dvehrs@gmail.com> Tue, 29 Mar 2013 16:20:31 -0600 podget (0.6.12) unstable; urgency=low * Smodcast issue reported by Stephen (stallmer) on Sourceforge in Bug ID 3605008. Created filename format fix that works in conjunction with filename_formatfix4 to resolve the issue. * Moved all of the filename format fixes but 1 to a new function. This cleans up the main loop making it easier to understand and maintain. -- Dave Vehrs <dvehrs@gmail.com> Tue, 29 Mar 2013 14:52:10 -0600 podget (0.6.11) unstable; urgency=low * Update to filename_formatfix to correct for feeds that place the name of the file to be downloaded in the parameter media_url=. Issue reported with the Radio France feeds by tom-angoumoisin on March 6, 2013 * Experimenting with order of filename format fixes. Those with more specific exclusionary conditions go first. More generic last. -- Dave Vehrs <dvehrs@gmail.com> Tue, 16 Mar 2013 11:25:10 -0600 podget (0.6.10) unstable; urgency=low * Update to filename_formatfix to correct for trailing characters on enclosures for the zdf.de podcasts. Issue submitted by Joerg Schiermeier on Jan 20, 2013. Thanks. * Fixed to display of filenames while marking older podcasts as downloaded with the --recent option. Rather than display the current filename for each item, it would display the last modified filename repeatedly. Fixed to display the filename of the item being marked as downloaded. * Refined the filename_formatfix to correct filenames downloaded from Vimcasts.org. Filename modification is now simpler and easier to understand. -- Dave Vehrs <dvehrs@gmail.com> Tue, 30 Jan 2013 08:30:15 -0700 podget (0.6.9) unstable; urgency=low * Update to filename_formatfix to correct for "?referrer=rss" being added to all filenames downloaded from Vimcasts.org. Bug submitted by Andi (apoisel) on Dec 19, 2012. Thanks. * Update to use mktemp to create temporary file for playlist sorting. If mktemp is not available, the script falls back to trying to use tempfile. Bug submitted by Scheini-72 on Aug 20, 2012. Thanks. -- Dave Vehrs <dvehrs@gmail.com> Thu, 27 Dec 2012 14:51:20 -0700 podget (0.6.8) unstable; urgency=low * Updates for Debian Package. * Fixed version string in podget to be 0.6.8 -- Dave Vehrs <dvehrs@gmail.com> Sat, 2 Dec 2012 11:29:30 -0700 podget (0.6.7) unstable; urgency=low * Fixed Bug #3146457 reported on Sourceforge by oaolsen. Fixes urls with ampersands in them. Thanks. * Fixed Bug #3021524 reported on Sourceforge by thebreaker. Fixes running session detection. Thanks. * Fixed podgets verbosity with a patch submitted by Norman Rasmussen. And fixed my email address in the script. Thanks. -- Dave Vehrs <dvehrs@gmail.com> Sat, 14 Apr 2012 11:28:55 -0700 podget (0.6.6) unstable; urgency=low * Fixed Bug #666149 reported by Elena Grandi via the Debian reporting system. Playlist names were being created with spaces that would break some lines. Added quotes to prevent. Thanks. -- Dave Vehrs <dvehrs@gmail.com> Wed, 4 Apr 2012 15:47:55 -0700 podget (0.6.5) unstable; urgency=low * Added version reporting flags (-V or --version), so people can test to see what version they have installed. -- Dave Vehrs <dvehrs@gmail.com> Wed, 4 Apr 2012 15:30:00 -0700 podget (0.6.4) unstable; urgency=low * Fixed enclosure bug. * Fixed error checking for successful wget run. -- Dave Vehrs <dvehrs@gmail.com> Fri, 19 Nov 2011 10:44:00 -0700 podget (0.6.3) unstable; urgency=low * Fixed tempfile bug/typo. * Fixed -f and --force to redownload existing files, including fix to wget base options. -- Dave Vehrs <dvehrs@gmail.com> Fri, 18 Nov 2011 11:29:02 -0700 podget (0.6.2) unstable; urgency=low * Added use of the tempfile command to the playlist sorting per Renalt Sabitov's idea and bug report (via Debian). -- Dave Vehrs <dvehrs@gmail.com> Fri, 16 Sep 2011 18:00:19 -0700 podget (0.6.1) unstable; urgency=low * Various fixes for Debian Lintain reports. -- Dave Vehrs <dvehrs@gmail.com> Wed, 14 Sep 2011 12:56:19 -0700 podget (0.6) unstable; urgency=low * Added date substitutions to the feed category per David Willmore's feature request (#1652606). Thanks. * Reversed order of filename character modifications and format fixing. * Fixed error reports to go to stderr. * Added more custom error/exit codes to simplify debugging. * Added another filename format fix for feeds from LBC Plus per MoonUnit's feature request (#1660764). Thanks. * Fixed playlist sorting to prevent a race condition in some implementations of the sort command as suggested by Cameron Paine. Thanks. * Fixed Indexfile creation for Podcast feeds that have the enclosure URL broken over multiple lines as suggested by Colin Carter. Thanks. * Modified Session file messaging to clearer report when old session files were removed. * Fixed -r (most recent) and -f (force) command line options to override the defaults stored in the user configuration files. Thanks to Jim C. for reporting the bug and providing a potential patch. * Wrapped eval statement on line 422 in quotes to try to fix an OBSD error. * Changed install command --mode flags to -m in the Makefile so that it would be OBSD compatible. * Changed #!/bin/bash to #!/usr/bin/env bash so that its more cross platform compatible. * Moved test for configuration directory to before creation of session file so it has a place to store the session file. * Added Filename Format fix for podcasts hosted on Catradio.cat per Oriol Rius's Bug Report (#1744705). Thanks. * Fixed "Filesystem paths including spaces crash the podget script" (Bug# 1889724) submitted by Anonymous. Thanks. * Fixed "Some filenames cause podget to fail" (Patch#1845815) submitted by Anonymous. Thanks. * Fixed error with playlist loading submitted by Markus. Thanks. * Added a filename format fix as suggested by Rob Sims via the Debian Bug Reporting System (http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=459949, Bug#459949). Thanks. * Added SCRIPTS directory for user submitted scripts. First script is from Steven Black and allows for easy addition of feeds from within Firefox. Thanks. * Added patch from Steven Black to fix problem handling single apostrophes in filenames. Thanks again. -- Dave Vehrs <dvehrs@gmail.com> Sun, 25 Apr 2010 12:28:39 -0700 podget (0.5.8) unstable; urgency=low * General Cleanup. * Fixed Filename Format issue (From BBC World News Bulletin). Files download named filename.mp3?123456 and are fixed to filename123456.mp3 (Patch# 1615902 submitted by mojoandy). * Added support for creating ASX Playlists for Windows Media Player. (Patch# 1615902 submitted by mojoandy). * Changed application name from podget.sh to podget to comply with Debian Policies. * Stopped compressing the copyright file to comply with Debian Policies. * Updated Debians copyright file to point at /usr/share/common-licenses/GPL * Removed "DEBIAN" from installed changelog to comply with Debian policies. * Updated Generic COPYING file for non-Debian installs. * Fixed synopsis line in Debian Control file to comply with Debian Policies. * Updated Debian Control file to Standards-Version 3.7.2 from 3.6.2. -- Dave Vehrs <dvehrs@gmail.com> Sat, 6 Jan 2007 18:52:39 -0700 podget (0.5.7) unstable; urgency=low * Patch for avail_space bug. Thanks to Andreas Ruppen for catching it. * Added function to check for running sessions using the same core configuration file. * Added function to check for remaining space on library partition and stop if it drops below a configured level. (both these ideas were inspired by Luc's suggestions) -- Dave Vehrs <dvehrs@gmail.com> Tue, 23 May 2006 11:01:30 -0700 podget (0.5.6) unstable; urgency=low * Added simulate function to cleanup loop per marc_in_lux's request (MSG ID: 3717464). -- Dave Vehrs <dvehrs@gmail.com> Mon, 8 May 2006 10:37:30 -0700 podget (0.5.5) unstable; urgency=low * Fixed bug in cleanup loop. -- Dave Vehrs <dvehrs@gmail.com> Tue, 10 Apr 2006 09:23:00 -0700 podget (0.5.4) unstable; urgency=low * Added UTF-16 formated feed handling (Support Request# 1432993). -- Dave Vehrs <dvehrs@gmail.com> Tue, 2 Mar 2006 11:41:16 -0700 podget (0.5.3) unstable; urgency=low * Fixed enclosure url parsing (Bug# 1414282). -- Dave Vehrs <dvehrs@gmail.com> Tue, 24 Jan 2006 21:21:37 -0700 podget (0.5.2) unstable; urgency=low * Removed bit-torrent support. Rarely used and poorly implemented. For an example of a better implementation see Podracer (http://podracer.sf.net/). * Fixed cleanup_days bug (if commandline option was not set, it overrode the default with a null value). * Added support for importing servers from iTunes PCAST files. -- Dave Vehrs <dvehrs@gmail.com> Wed, 17 Jan 2006 07:21:37 -0700 podget (0.5.1) unstable; urgency=low * Suppress "no enclosures" warning while running silent (Patch# 1395582, submitted by Kai Timmer). * Test for completed file fails on filename with special chars, replace grep with fgrep (Patch# 1396969 submitted by Kai Timmer). * Standardized verbosity testing (Patch# 1397756, Submitted by Johnny Ljunggern). * Cleaned up comment output, and established clearer levels (0-4). -- Dave Vehrs <dvehrs@gmail.com> Wed, 4 Jan 2006 09:16:17 -0700 podget (0.5) unstable; urgency=low * Added sed hook to handle rss feeds that break the enclosure tag over multiple lines (LugRadio). * Initial import OPML list support from local file or download from http/ftp. * Applied patch submited by anonymous. (Bug 1380412 - unhandled environment variable). * Dropped test for interactive shell. * Fixed configuration file parsing bug (Bug 1386657). * Added filename cleanup code for FAT32 compatability (Feature Request 1378956). -- Dave Vehrs <dvehrs@gmail.com> Sun, 20 Dec 2005 18:00:00 -0700 podget (0.4.1) unstable; urgency=low * Initial Debian package release. * Moved podcast directory from /usr/local/share to $HOME/ * Improved configuration installation and testing. * Modified cleanup loop to allow user specified number of days to retain files. * Fix for RSS feeds without sufficient newlines. * Initial man page. * Installation by makefile or debian package now. * Started changelog. * Initial Debian Packages (.deb) -- Dave Vehrs <dvehrs@gmail.com> Sun, 6 Nov 2005 19:37:01 -0700 ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������