stevesBashPhotoSorter.sh

#!/bin/bash
# 
# stevesBashPhotoSorter sorts all your images and video files into a neat directory structure (YYYY/MM/DD) based on date and time of image taken
#
# Copyright (C) 2023  Stefano Longo
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.





startTime=`date +%s`

VERSION="0.7";

# trap ctrl-c and call ctrl_c()
trap ctrl_c INT

# Initialise options
exifTool_config=".ExifTool_config"
unsortedDirName="UnsortedFiles"
duplicateDirName="DuplicateFiles"
duplicateFileName="Duplicates.txt"
dupDir="";
errorDirName="ErrorFiles"
workingDir=$(pwd)
logFile=`readlink -f "./stevesBashPhotoSorter.log"`
crcFileName=".stevesPhotoSorterCRCfile.crc"
crcFile=""
resumeOp=1;
regenCRC=0;
deleteDups=0;
genConFile=0;
moveFiles=0;
copyFiles=0;
renameFiles=0;
deleteFiles=0;
interactive=0;


# REPORTING
numFilesProcessed=0;
numFilesSorted=0;
numFilesIgnored=0;
numNewerFilesReplaced=0;
numFilesUnknown=0;
numFilesDuplicate=0;
numFilesWrongDate=0;
numFilesSkipped=0;
numErrors=0;
numFilesUnsorted=0;

##########################
# Functions declarations #
##########################

function usage() {
	echo
	echo "	$0 Copyright (C) 2023 Stefano Longo"
	echo "	This program comes with ABSOLUTELY NO WARRANTY; for details type '$0 -w'."
	echo "	This is free software, and you are welcome to redistribute it"
	echo "	under certain conditions; type '$0 -z' for details."
	echo
	echo "	WARNING: THIS SCRIPT IS VERY EXPERIMENTAL. USE WITH CARE!"
	echo "	WARNING: Always make a backup of all your pictures!"
	echo "	WARNING: Use this script at your own risk!"
	echo 
    echo "USAGE: $0 -S <sourceDir> -D <destDir> [-C <ExifTool_config>]"
	echo
	echo "		-S:		Source directory: original, unsorted photos."
	echo "		-D:		Destination directory: where your sorted photos will be saved."
	echo "		-C:		The ExifTool_config file to use."
	echo "					Note: if no exif config file is specified, the script will try to look for it in your home directory."
	echo "							If no exif config file is found, the script generate one in the user's home directory."
	echo "		-E:		Files that could not be processed for some reason will be linked in this directory."
	echo "		-c:		Image files are copied across the new destination directory."
	echo "		-m:		Image files are moved across the new destination directory."
	echo "		-d:		[NOT YET IMPLEMENTED] Delete known and ignored files (ini|411|thm|htm|txt|db). Warning: can't undo this operation! Use carefully!"
	echo "		-i:		[NOT YET IMPLEMENTED] Interactive mode. Script askes user action in case of errors."
	echo "		-l:		Log file."
	echo "		-G:		Regenerate CRC file from images located in <destDir>. To be used in conjunction with -D <destDir> option."
	echo "		-u:		[NOT YET IMPLEMENTED] Delete newest duplicate files image directory located in <destDir>. Used in conjunction with -D <destDir> option."
	echo "		-X:		Generate ~/.ExifTool_config file. Not really needed, as script generates config file automatically if not found."
	echo "		-r:		[NOT YET IMPLEMENTED] Rename image files in the YYYY-MM-DD.ext format."
	echo "		-h:		This help."
	echo "		-v:		Print version."
	echo "		-w:		Print software warranty."
	echo "		-z:		Print software redistribution conditions."
	echo
	echo "Usage notes: If ExifTool_config option is not provided, it is looked for in the user ('$USER') home."
	echo "	By default, $0 will resume from last import operations using the CRC file in the <destDir> folder."
	echo "Warning: $0 does not check for corrupt image or video files."
	echo
}

function ctrl_c() {
	echo
	echo "** Trapped CTRL-C **" 2>&1 | tee -a "$logFile"
	echo
	printReport
	exit 4;
}

function version() {
	echo "$(basename $0)"
	echo "		Version: $VERSION"
	echo
	exit 0;
}

function generateExifTool_Config() {
	exifConfig="$HOME/$exifTool_config"
	echo "	INFO: Generating $exifConfig file" 2>&1 | tee -a "$logFile"
	[ -f "$exifConfig" ] && /bin/mv -v --backup=numbered "$exifConfig" "$exifConfig.bak"

	tee "$exifConfig" &>/dev/null <<EOF
# Generated by $(basename $0) on `date`  
EOF


	tee -a "$exifConfig" &>/dev/null <<'EOF'
%Image::ExifTool::UserDefined = (
    'Image::ExifTool::Composite' => {
        # Select oldest date from a number of date tags
        OldestDateTime => {
            Desire => {
                0 => 'FileModifyDate',
                1 => 'MDItemFSContentChangeDate',
                2 => 'FileCreateDate',
                3 => 'MDItemFSCreationDate',
                4 => 'ModifyDate',
                5 => 'CreateDate',
                6 => 'DateTimeCreated',
                7 => 'DateTimeOriginal',
                8 => 'CreationTime',
                9 => 'DateCreated'
            },
            ValueConv => q{
                my $oldest = undef;
                for my $date (@val) {
                    next if not defined $date or $date lt '1970:01:02';
                    $date =~ s/([+-]\d{2}:\d{2}$)|([A-Z]{3}$)|([Z]$)//; # Strip TimeZone 2000:01:01 00:00:00'+01:00' or 2000:01:01 00:00:00' IST'
					# I came across some dashed formatted date/time...
                    $date =~ s/-/:/g; # Replace dashes with colons: 2018-12-24 -> 2018:12:24
                    if ($date && (!$oldest || $date lt $oldest)) {
                       if (($date ne "0000:00:00 00:00:00") && ($date ne "    :  :     :  :  ")) {
                                $oldest = $date;
                        }
                    }
                }
                return $oldest;
            },
            PrintConv => '$self->ConvertDateTime($val)',
        },
    },
);

1;
EOF
	# TODO: CHECK FOR ERRORS 
	exit 0;
}

initLog() {
	# create log file or overwrite if already present
	printf "$(basename $0) Log File - " > "$logFile"
	# append date to log file
	date >> "$logFile"
}

initCRClog() {
	echo "	INFO: Initialising CRC file..." 2>&1 | tee -a "$logFile"
	echo "destDir='$destDir'" 2>&1 | tee -a "$logFile"
	echo "crcFileName='$crcFileName'" 2>&1 | tee -a "$logFile"
	crcFile=`readlink -f "$destDir/$crcFileName"`
	echo "crcFile='$crcFile'" 2>&1 | tee -a "$logFile"

	# If regenCRC=1, then backup old CRC file and initialise new CRC file
	if [ $regenCRC == 1 ]; then
		echo "	INFO: Regenerating new CRC file..." 2>&1 | tee -a "$logFile"

		# First make a backup of existing CRC file
		if [ -f "$crcFile" ]; then
			# Backup existing CRC file
			/bin/mv -v --backup=numbered "$crcFile" "$destDir/$crcFileName.bak" 2>&1 | tee -a "$logFile" || exit 42
		fi
		# Create new CRC log file and initialise it
		printf "$(basename $0) CRC log file" > "$crcFile"
		# Append to the file
		echo "" >> "$crcFile"
		echo "  ####################################################################" >> "$crcFile"
		echo " # WARNING: DO NOT REMOVE OR EDIT UNLESS YOU KNOW WHAT YOU ARE DOING! #" >> "$crcFile"
		echo "########################################################################" >> "$crcFile"
		echo "" >> "$crcFile"
		# append date to log file
		echo "Generated on `date`" >> "$crcFile"
		echo "" >> "$crcFile"
	else
		# It's a normal call to sort new images
#		echo "destDir=$destDir"
#		echo "crcFile=$crcFile"
#		read c
		# Check whether destDir exists of not, if not, create it?
		if [ ! -f "$crcFile" ]; then
			echo "INFO: Generating new CRC file."
			# Create new CRC log file and initialise it
			printf "$(basename $0) CRC log file" > "$crcFile"
			# Append to the file
			echo "" >> "$crcFile"
			echo "  ####################################################################" >> "$crcFile"
			echo " # WARNING: DO NOT REMOVE OR EDIT UNLESS YOU KNOW WHAT YOU ARE DOING! #" >> "$crcFile"
			echo "########################################################################" >> "$crcFile"
			echo "" >> "$crcFile"
			# append date to log file
			echo "Generated on `date`" >> "$crcFile"
			echo "" >> "$crcFile"
		else
			# Backup existing CRC file
			echo "Backing-up existing CRC file..." 2>&1 | tee -a "$logFile"
			/bin/mv -v --backup=numbered "$crcFile" "$destDir/$crcFileName.bak" 2>&1 | tee -a "$logFile" || exit 42
		fi
	fi
}


#
# copyOrMoveImageToDestination(imageFile)
#
copyOrMoveImageToDestination() {
		# Check the Date picture was taken from exif data
		OldestDateTime=`exiftool -P -OldestDateTime "$1"`
		#echo "OldestDateTime='$OldestDateTime'" 2>&1 | tee -a "$logFile"
		space_YEAR=`echo $OldestDateTime | cut -d ":" -f 2`
		#echo "spaceYear='$space_YEAR'" 2>&1 | tee -a "$logFile"
		# The above command results in the year having a leading space (eg ' 2008')
		# remove leading whitespace characters
		YEAR="${space_YEAR#"${space_YEAR%%[![:space:]]*}"}"
		#echo "year='$YEAR'" 2>&1 | tee -a "$logFile"
		MONTH=`echo $OldestDateTime | cut -d ":" -f 3`
		#echo "MONTH='$MONTH'" 2>&1 | tee -a "$logFile"
		DAY_HOUR=`echo $OldestDateTime | cut -d ":" -f 4`
		#echo "DAY_HOUR='$DAY_HOUR'" 2>&1 | tee -a "$logFile"
		DAY=`echo $DAY_HOUR | cut -d " " -f 1`
		#echo "DAY='$DAY'" 2>&1 | tee -a "$logFile"

		# Check if filename contains a date:
		fDate=$(echo "$1" | grep -Eo '[1-2]{1}[0-9]{3}[0-1]{1}[0-9]{1}[0-3]{1}[0-9]{1}')
		echo "fDate: $fDate"
		fYear=${fDate:0:4}
		fMonth=${fDate:4:2}
		fDay=${fDate:6:2}
		echo "	File Year-Month-Day: $fYear-$fMonth-$fDay"

#		[[ $fYear == +([0-9]) ]] && echo "Year is a number" || exit 2
#		[[ $fMonth == +([0-9]) ]] && echo "Month is a number" || exit 2
#		[[ $fDay == +([0-9]) ]] && echo "Day is a number" || exit 2

		if [[ $fYear == +([0-9]) ]] && [[ $fMonth == +([0-9]) ]] && [[ $fDay == +([0-9]) ]]; then
			echo "Filename contains a date..."
			if [ $fYear -ne $YEAR ] || [ $fMonth -ne $MONTH ] || [ $fDay -ne $DAY ]; then
				echo "WARNING: Filename contains Date Time in the name, but differ from exif's Date Time:"
				echo "	Exif Year-Month-Day: $YEAR-$MONTH-$DAY"
				echo "	File Year-Month-Day: $fYear-$fMonth-$fDay"
				echo "Which Date/Time accept:"

				oldestY=$fYear
				oldestM=$fMonth
				oldestD=$fDay
				if [ $fYear -gt $YEAR ]; then
					oldestY=$YEAR
					oldestM=$MONTH
					oldestD=$DAY
				elif [ $fYear -eq $YEAR ]; then
					if [ $fMonth -gt $MONTH ]; then
						oldestY=$YEAR
	                	oldestM=$MONTH
		                oldestD=$DAY
					elif [ $fMonth -eq $MONTH ]; then
						if [ $fDay -gt $DAY ]; then
							oldestY=$YEAR
		        	        oldestM=$MONTH
		            	    oldestD=$DAY
						fi
					fi
				fi

				echo "	Press '1' for $oldestY-$oldestM-$oldestD (Default)"
				echo "	Press '2' for $YEAR-$MONTH-$DAY"
				echo "	Press '3' for $fYear-$fMonth-$fDay"
				while read -p "Input: " c; do
					case $c in
						exit) break ;;
						1)
							YEAR=$oldestY
							MONTH=$oldestM
							DAY=$oldestD
							echo "Updating image with date: '$oldestY:$oldestM:$oldestD 00:00:00'"
							exiftool -AllDates='$oldestY:$oldestM:$oldestD 00:00:00' $1
							exiftool -FileModifyDate='$oldestY:$oldestM:$oldestD 00:00:00' $1
							break
						;;
						2)
							echo "No changes needed."
							break
						;;
						3)
							YEAR=$fYear
							MONTH=$fMonth
							DAY=$fDay
							echo "Updating image with date: '$fYear:$fMonth:$fDay 00:00:00'"
							`exiftool -AllDates='$fYear:$fMonth:$fDay 00:00:00' $1`
							break
						;;
						#"")
						# This code does not work: It does not stop the script to read input from user
						#	echo "empty"
						#	break
						#;;
						*)
							echo "Unknown response, press '1' or '2' or press [ENTER] to accept default option '1'."
						;;
					esac
				done
			fi
		else
			echo "Filename does not contain a date."
		fi

		# TODO: IMPROVE: Let user input valid date range
		# If ExifTool returns a date prior to 1990, move the file into errorDir
		if (($YEAR < 1990)); then
			echo "	WARNING: Possible Exif data corrupt." 2>&1 | tee -a "$logFile"
			echo "		DateTimeOriginal: $OldestDateTime"
			#echo "Press [ENTER] to continue..."
			#read c;
			let numFilesWrongDate++;
			setFileUnsorted "$1";
			echo "	Done." 2>&1 | tee -a "$logFile"
			return;
		fi

		fullDestPath="$destDir/$YEAR/$MONTH/$DAY/"
		#echo "fullDestPath=$fullDestPath" 2>&1 | tee -a "$logFile"
		#echo "copyFiles=$copyFiles" 2>&1 | tee -a "$logFile"
		# Default option (no move or copy option) is a dry run...	
		if [ $copyFiles -eq 1 ]; then
			#echo "--	Creating destination directory $fullDestPath..." 2>&1 | tee -a "$logFile"
			[ -d "$fullDestPath" ] || `/bin/mkdir -p "$fullDestPath"` 2>&1 | tee -a "$logFile" || exit 42
			#echo "--	Copying file over..." 2>&1 | tee -a "$logFile"
			# TODO: CHECK THIS... If there is a same file in the destination directory, don't copy (I assume it's the same file with same CRC!!!)
			#/bin/cp -npv "$1" "$fullDestPath" 2>&1 | tee -a "$logFile" || exit 42 
			/bin/cp -pv "$1" "$fullDestPath" 2>&1 | tee -a "$logFile" || exit 42 
			# TODO: MPG files do not have a DateTimeOriginal tag and only rely on system's create/modify date time.
			#		When a MPG file is moved, CreateDate and ModifyDate get updated to now.
			#		Use touch -a -m -t [[CC]YY]MMDDhhmm[.ss] to put the date/time back
			# Activate case insensitive so that we can match any case extension (eg: JPG|jpg|JpG|jPg ecc)
			#shopt -s nocasematch
			# extension="${file##*.}"
			# [ $extension -eq "mpg"  ] && touch -a -m -t $YEAR$MONTH$DAY0000.00 $fullDestPath/$filename
			# Deactivate case insensitive
			#shopt -u nocasematch
			# TODO: Capture any errors, save them in the variable and deal with the error later.
			#echo "--	INFO: '$1'  copied to '$fullDestPath'" 2>&1 | tee -a "$logFile"
			#echo "=========== Appending to crcFile: '$thisCRC,$fullDestPath$filename'" 2>&1 | tee -a "$logFile"
			echo "$thisCRC,$fullDestPath$filename" >> "$crcFile"
			let numFilesSorted++;
		elif [ $moveFiles -eq 1 ]; then
			#echo "--	Creating destination directory $fullDestPath..." 2>&1 | tee -a "$logFile"
			[ -d "$fullDestPath" ] || `/bin/mkdir -p "$fullDestPath"` 2>&1 | tee -a "$logFile" || exit 42
			#echo "--	Moving file over..." 2>&1 | tee -a "$logFile"
			# TODO: Capture any errors, save them in the variable and deal with the error later.
			# TODO: CHECK THIS... If there is a same file in the destination directory, don't copy (I assume it's the same file with same CRC!!!)
			/bin/mv -v "$1" "$fullDestPath" 2>&1 | tee -a "$logFile" || exit 42 
			#/bin/mv -nv "$1" "$fullDestPath" 2>&1 | tee -a "$logFile" || exit 42 
			# Capture any errors, save them in the variable and deal with the error later.
			#echo "--	INFO: '$1' moved to '$fullDestPath'" 2>&1 | tee -a "$logFile"
			#echo "=========== Appending to crcFile: '$thisCRC,$fullDestPath$filename'" 2>&1 | tee -a "$logFile"
			echo "$thisCRC,$fullDestPath$filename" >> "$crcFile"
			let numFilesSorted++;
		elif [ $regenCRC == 1 ]; then
			echo "=========== Appending to crcFile: '$thisCRC,$fullDestPath$filename'" 2>&1 | tee -a "$logFile"
			echo "$thisCRC,$fullDestPath$filename" >> "$crcFile"
		else
			# DRY RUN
			echo "--	WARNING: DRY RUN! FILE NOT COPIED OR MOVED to destination directory: $fullDestPath" 2>&1 | tee -a "$logFile"
		fi
}

#
# sortImage(imageFile)
#
# Parameter $1 is the image file to be copy or moved
#
sortImage() {
#	echo "--- file='$1'"
	filename=$(basename "$1")
#	echo "--- filename='$filename'"
#	echo "--- crcFile=$crcFile" 2>&1 | tee -a "$logFile"

	# Check for existing CRC first
	# Compute the checksum of the file N
	thisCRC=`cksum $1 | cut -d " " -f 1`
#	echo "-- thisCRC=$thisCRC" 2>&1 | tee -a "$logFile"

	# Check if we dealt with this file before...
#	echo "Just before grep"
#	read c
	foundSameFileInCRC=`grep -w -m 1 "$thisCRC" "$crcFile" | awk -F ',' '{print $2}'`

	if [ ! "$foundSameFileInCRC" == "" ]; then
		echo "	WARNING: Found same file in CRC file: $foundSameFileInCRC" 2>&1 | tee -a "$logFile"
		echo "	WARNING: CRC entry ($thisCRC) for same file exists... " 2>&1 | tee -a "$logFile"
#		read c
		# Check which file has earlier OldestDateTime and update the <destDir>.

		if [ -f "$foundSameFileInCRC" ]; then
			OldestDateTime1=`exiftool -P -OldestDateTime "$foundSameFileInCRC"`
			echo "In CRC: OldestDateTime ($foundSameFileInCRC): $OldestDateTime1" 2>&1 | tee -a "$logFile"
			OldestDateTime2=`exiftool -P -OldestDateTime "$1"`
			echo "OldestDateTime ($1): $OldestDateTime2" 2>&1 | tee -a "$logFile"

#			echo "1: Press [ENTER] to continue..."
#			read c 
			# If the file sorted and found in the CRC file is newer than the current file $1, then swap them out
			if [ $foundSameFileInCRC -nt $1 ]; then
				# $1 is the older file.
				echo "$1 is an older file." 2>&1 | tee -a "$logFile"
				echo "Removing line from CRC file..." 2>&1 | tee -a "$logFile"
#				read c 
				# ATTENTION! We found an older file of one that was already sorted!
				# We need to remove the newer file from the <destDir> and the entry in the CRC file
				# and replace with the older file that we just found and update the CRC entry for that file.
	
				# $foundSameFileInCRC is older than $1

				# Remove line from CRC file
				sed -i "/$thisCRC/d" $crcFile 2>&1 | tee -a "$logFile"

				echo "Done." 2>&1 | tee -a "$logFile"
				echo "Removing newer file from sortedDir" 2>&1 | tee -a "$logFile"
#				read c 

				# Remove the newer file ($foundSameFileInCRC) from the <destDir> and put into $errorDir
				let numFilesWrongDate++;
				#setFileUnsorted $foundSameFileInCRC

				echo "Done." 2>&1 | tee -a "$logFile"
				echo "Sorting new found file with older date..." 2>&1 | tee -a "$logFile"
				# Insert older file ($1) into place in the $destDir
				copyOrMoveImageToDestination $1
				let numNewerFilesReplaced++
				echo "Done." 2>&1 | tee -a "$logFile"
			else
				echo "	Done (Nothing to do)." 2>&1 | tee -a "$logFile"
				let numFilesSkipped++;
			fi
		else 
			echo "W:	WARNING: CRC file entry does not point to an existing file." 2>&1 | tee -a "$logFile"
			echo "E:	ERROR: Can't compare!" 2>&1 | tee -a "$logFile"
			echo "E:	ERROR: CRC File must be corrupt!" 2>&1 | tee -a "$logFile"
			echo "Press [ENTER] to continue or CTRL-c to abort..." 2>&1 | tee -a "$logFile"
#			read c
		fi
	else
		# This is a new file, we need to copy to destination.
		copyOrMoveImageToDestination $1
	fi

}

#TODO: NOT USED! CAN BE REMOVED!
handleFileError () {
	#compute the checksum of the file passes as first argument
	thisCRC=`cksum "$1" | cut -d " " -f 1`
	#echo "	Handling error of file '$1'" 2>&1 | tee -a "$logFile"

	OldestDateTime=`exiftool -P -OldestDateTime "$1"`
	#echo "DateTimeOriginal: $OldestDateTime"
	space_YEAR=`echo $OldestDateTime | cut -d ":" -f 2`
	# The above command results in the year having a leading space (eg ' 2008')
	# remove leading whitespace characters
	YEAR="${space_YEAR#"${space_YEAR%%[![:space:]]*}"}"
	MONTH=`echo $OldestDateTime | cut -d ":" -f 3`
	#echo "MONTH: $MONTH"
	DAY_HOUR=`echo $OldestDateTime | cut -d ":" -f 4`
	DAY=`echo $DAY_HOUR | cut -d " " -f 1`
	#echo "DAY: $DAY"
	HOUR=`echo $DAY_HOUR | cut -d " " -f 2`
	#echo "HOUR: $HOUR"
	MINUTES=`echo $OldestDateTime | cut -d ":" -f 5`
	#echo "MINUTES: $MINUTES"
	SECONDS=`echo $OldestDateTime | cut -d ":" -f 6`
	#echo "SECONDS: $SECONDS"
	#echo "	File name: '$1'" 2>&1 | tee -a "$logFile"

	# Check if destination file exists
	filename=$(basename "$1")
	destinationFile="$destDir/$YEAR/$MONTH/$DAY/$filename"
	if [ -f "$destinationFile" ]
	then
		# A same name destination file exists... check further
		destFileCRC=`cksum "$destinationFile" | cut -d " " -f 1`
		if cmp -s "$1" "$destDir/$YEAR/$MONTH/$DAY/$filename" && [ $destFileCRC==$thisCRC ];
		then
			# Source and destination files are the same
			#echo "	INFO: Found same file (with same CRC) at destination. Proceeding." 2>&1 | tee -a "$logFile"
			[ $moveFiles -eq "1" ] && echo "	WARNING: [TODO] IMPLEMENT DELETE ORIGINAL FILE HERE IF OPTION IS TO MOVE?" 2>&1 | tee -a "$logFile"
			setFileDuplicate "$1"
			#sortImage "$1"
			echo "	Done." 2>&1 | tee -a "$logFile"
		else
			# Source and destination files DIFFER!
			echo "	Source file -> '$1'" 2>&1 | tee -a "$logFile"
			echo "	Source file CRC: $thisCRC" 2>&1 | tee -a "$logFile"
			echo "	Destination file -> '$destDir/$YEAR/$MONTH/$DAY/$filename'" 2>&1 | tee -a "$logFile"
			echo "	Destination file CRC: $destFileCRC" 2>&1 | tee -a "$logFile"
			echo "	ERROR: Source file and destination file differ!"
			echo "	CAUGHT ExifTool $res" 2>&1 | tee -a "$logFile"
			setFileAside "$1"
		fi
	else
		# Desination file does not exists, so WHY THE ERROR? INVESTIGATE (NO SPACE LEFT?)
		echo "+++++++++++++++++" 2>&1 | tee -a "$logFile"
		echo "+	FATAL: Destination file does not exist..." 2>&1 | tee -a "$logFile"
		echo "+	Exiftool error is something else... INVESTIGATE FURTHER!" 2>&1 | tee -a "$logFile"
		echo "+	Input/output error" 2>&1 | tee -a "$logFile"
		echo "+++++++++++++++++" 2>&1 | tee -a "$logFile"
		printReport
		exit 5
	fi
}

# Create a soft link of problematic file into an error directory preserving the parent directory structure...
setFileAside() {
	#[ -d "$errorDir" ] || /bin/mkdir -p "$errorDir"
	# Following does not work across different divices
	/bin/cp -p -v --parents -l "$1" "$errorDir"
	#/bin/ln -s --backup=numbered "$1" "$errorDir"
	echo "	INFO: Placed link to file '$1' in error directory '$errorDir'" 2>&1 | tee -a "$logFile"
	let numErrors++;
}

# MOVE problematic files into an Unsorted directory preserving the parent directory structure...
# INFO: This function is also used by the function regenerateCRC, and if older duplicates files
# are found in the <destDir>, they need to be (re)MOVED from the <destDir> (or sorted directory)
setFileUnsorted() {
	# Preserve OldestDateTime (cp -p option)
	/bin/mv -v --backup=numbered "$1" "$unsortedDir" 2>&1 | tee -a "$logFile"
	echo "	INFO: Placed file '$1' in '$unsortedDir'" 2>&1 | tee -a "$logFile"
	let numFilesUnsorted++;
}

# Copy duplicate files into an Unsorted directory preserving the parent directory structure...
setFileDuplicate() {
	/bin/cp -pv --parents --backup=numbered "$1" "$dupDir"
	echo "	INFO: Placed file '$1' in '$dupDir'" 2>&1 | tee -a "$logFile"
	let numFilesDuplicate++
}

printReport() {
	echo "" 2>&1 | tee -a "$logFile";
	echo "===========================================================================" 2>&1 | tee -a "$logFile";
	echo "" 2>&1 | tee -a "$logFile";
	echo "Report (values may be inaccurate):" 2>&1 | tee -a "$logFile";
	echo "" 2>&1 | tee -a "$logFile";
	echo "Total number of files processed: $numFilesProcessed" 2>&1 | tee -a "$logFile";
	echo "Total number of files skipped: $numFilesSkipped" 2>&1 | tee -a "$logFile";
	echo "Total number of files ignored (known extension): $numFilesIgnored" 2>&1 | tee -a "$logFile";
	echo "Total number of files duplicate: $numFilesDuplicate" 2>&1 | tee -a "$logFile";
	echo "Total number of files moved or copied: $numFilesSorted" 2>&1 | tee -a "$logFile";
	echo "Total number of unsorted files: $numFilesUnsorted" 2>&1 | tee -a "$logFile";
	echo " | of which:" 2>&1 | tee -a "$logFile";
	echo " |->	Total number of newer files replaced: $numNewerFilesReplaced" 2>&1 | tee -a "$logFile";
	echo " |->	Total number of files with possible wrong date: $numFilesWrongDate" 2>&1 | tee -a "$logFile";
	echo "Total number of Errors: $numErrors" 2>&1 | tee -a "$logFile";
	echo " | of which:" 2>&1 | tee -a "$logFile";
	echo " |->	Total number of files with unknown extension: $numFilesUnknown" 2>&1 | tee -a "$logFile";

	echo "" 2>&1 | tee -a "$logFile";
	# Print running time
	endTime=`date +%s`
	seconds=$((endTime-startTime))
	echo "Total time elapsed: " $((seconds/86400))" days "$(date -d "1970-01-01 + $seconds seconds" "+%H hours %M minutes %S seconds") 2>&1 | tee -a "$logFile";
	echo "===========================================================================" 2>&1 | tee -a "$logFile";
}


# Regenerate CRC file in destDir
# NOTE: The CRC file has been already reset and initialised...
regenerateCRC() {
	echo
	echo "crcFile=$crcFile"
	echo "duplicateFileName=$duplicateFileName"
	echo
#	echo "Regenerating CRC file. Press any key to continue..."
#	read c
	echo "Generated by $0 on `date`" > $duplicateFileName

	# for loop together with find command does not handle file names with spaces
	originalIFS=$IFS
	IFS=$(echo -en "\n\b")

	count=0
	# Lets find all files in the  $sourceDir and beyond!
	# IGNORE HIDDEN FILES AND DIRECTORIES
	echo "	WARNING: Ignoring hidden files and hidden directories."
	for file in $(find "$destDir" -not -path '*/.*' -type f); do
    	echo "- Processing file #$count: '$file'" 2>&1 | tee -a "$logFile"
		

		filename=$(basename "$file")
#		echo "--- filename='$filename'"

		# Check for existing CRC first
		# Compute the checksum of the file N
		thisCRC=`cksum $file | cut -d " " -f 1`
		echo "-- thisCRC=$thisCRC" 2>&1 | tee -a "$logFile"

		# Check the CRC file to see if we dealt with this file before...
#		echo "Just before grep"
#		read c
		foundSameFileInCRC=`grep -w -m 1 "$thisCRC" "$crcFile" | awk -F ',' '{print $2}'`
#		echo "--- foundSameFileInCRC=$foundSameFileInCRC" 2>&1 | tee -a "$logFile"
#		read c

		if [ ! "$foundSameFileInCRC" == "" ]; then
			# We have already sorted this file before... Do nothing.
			# TODO: Check which file has earlier OldestDateTime and update the <destDir>.
			echo "	INFO: CRC entry for same file exists... " 2>&1 | tee -a "$logFile"

			if [ -f "$foundSameFileInCRC" ]; then
				OldestDateTime1=`exiftool -P -OldestDateTime "$foundSameFileInCRC"`
				echo "In CRC: OldestDateTime ($foundSameFileInCRC): $OldestDateTime1"
				OldestDateTime2=`exiftool -P -OldestDateTime "$file"`
				echo "OldestDateTime ($file): $OldestDateTime2"

#				echo "1"
#				read c 
				# If the file sorted and found in the CRC file is newer than the current file $1, then swap them out
				if [ $foundSameFileInCRC -nt $file ]; then
					# $file is the older file.
					echo "$file is an older file."
					echo "Removing line from CRC file..."
#					read c 
					# ATTENTION! We found an older file of one that was already sorted!
					# We need to remove the newer file from the <destDir> and the entry in the CRC file
					# and replace with the older file that we just found and update the CRC entry for that file.
	
					# $foundSameFileInCRC is older than $1

					# Remove line from CRC file
					sed -i "/$thisCRC/d" $crcFile 2>&1 | tee -a "$logFile"

					echo "Done."
					echo "Removing newer file from sortedDir"
#					read c 
	
					# Remove the newer file ($foundSameFileInCRC) from the <destDir> and put into $errorDir
					let numFilesWrongDate++;
					#setFileUnsorted $foundSameFileInCRC

					echo "Done."
					echo "Sorting new found file with older date..."
					# Insert older file ($file) into place in the $destDir
					copyOrMoveImageToDestination $file
					let numNewerFilesReplaced++
				else
					## There are two same files. We need to delete one.
					## Which one do we delete?
					## Write duplicate files in a separate file and inform the user.
					echo $thisCRC >> $duplicateFileName
					echo $file >> $duplicateFileName
					echo $foundSameFileInCRC >> $duplicateFileName
					echo "	WARNING: Duplicate files found in sorted <destDir>. They need to be removed manually." 2>&1 | tee -a "$logFile"
					let numFilesSkipped++;
				fi
			else 
				echo "W:	WARNING: CRC file entry does not point to an existing file."
				echo "E:	ERROR: Can't compare!"
				echo "E:	ERROR: CRC File must be corrupt!"
				echo "Press [ENTER] to continue or CTRL-c to abort."
#				read c
			fi
		else
			# This is a new file, we need to create an antry for this file in the CRC file
			#copyOrMoveImageToDestination $file
			echo "INSERT: '$thisCRC,$file'"
			echo "$thisCRC,$file" >> "$crcFile"
			#let numFilesSorted++;
		fi
		let count++
	done
	# Restore IFS
	IFS=$originalIFS

	echo "Number of duplicate files present: $numFilesDuplicate." 2>&1 | tee -a "$logFile"
	endTime=`date +%s`
	seconds=$((endTime-startTime))
	echo "Total time elapsed: " $((seconds/86400))" days "$(date -d "1970-01-01 + $seconds seconds" "+%H hours %M minutes %S seconds") 2>&1 | tee -a "$logFile";
	exit 0
}

delDupImages() {
	echo "	WARNING: EXPERIMENTAL FEATURE!" 2>&1 | tee -a "$logFile"
	echo "	WARNING: Delete duplicate images feature is very EXPERIMENTAL!" 2>&1 | tee -a "$logFile"
	echo "	WARNING: USE AT OWN RISK!" 2>&1 | tee -a "$logFile"
	#echo "	FATAL: Delete duplicate images feature not yet implemented." 2>&1 | tee -a "$logFile"
	# Following command gets a list of duplicate file names:

	dupCRCFile="$destDir/.dups.crc"
	awk 'BEGIN { FS="," } { c[$1]++; l[$1,c[$1]]=$0 } END { for (i in c) { if (c[i] > 1) for (j = 1; j <= c[i]; j++) print l[i,j] } }' $crcFile > $dupCRCFile
	# remove any empty lines
	sed -i '/^$/d' $dupCRCFile

#	while IFS= read -r line <&3; do {
#		# If lenght of $line>0
#		if [ -n "$line" ]; then
#			printf '%s\n' "$line" | cut -d "," -f 1 || exit
##			printf '%s\n' "$line" | cut -d "," -f 3 || exit
#			#crc=`echo "$line" | awk -F "," '{print $1}'`
#			#echo "CRC=$crc"
#		fi
#	} 3<&-
#	done 3< $dupCRCFile

	while IFS= read -r line <&3; do {
		echo "line=$line"
		# If lenght of $line>0
		if [ -n "$line" ]; then
#			file=`printf '%s\n' "$line" | cut -d "," -f 2 || exit`
			file=`echo "$line" | awk -F "," '{print $2}'`
			echo "file=$file"
#			printf '%s\n' "$line" | cut -d "," -f 3 || exit
			crc=`echo "$line" | awk -F "," '{print $1}'`
			echo "CRC=$crc"
		fi
	} 3<&-
	done 3< $dupCRCFile





	exit 38
}

######################
# PROCESS PARAMETERS #
######################
while getopts S:D:C:E:l:GXudvhmcri flag
do
	case "${flag}" in
		S) sourceDir=`readlink -f "${OPTARG%/}"`;; # '%/' removes any trailing slashes
		D) destDir=`readlink -f "${OPTARG%/}"`;; # '%/' removes any trailing slashes
		C) exifConfig=`readlink -f "${OPTARG}"`;;
		E) errorDir=`readlink -f "${OPTARG}"`;;
		l) logFile=`readlink -f "${OPTARG}"`;;
		m) moveFiles=1;;
		c) copyFiles=1;;
		#R) resumeOp=1;;
		G) regenCRC=1;;
		u) deleteDups=1;;
		X) genConFile=1;;
		d) deleteFiles=1
			echo "FATAL: Option not implemented yet. Exiting.";
			exit 125;
			;;
		i) interactive=1
			echo "FATAL: Option not implemented yet. Exiting.";
			exit 125;
			;;
		r) renameFiles=1
			echo "FATAL: Option not implemented yet. Exiting.";
			exit 125;
			;;
		v) version;;
		h) usage;
			exit 0;
			;;
		\?) usage;
			exit 1;
			;;
	esac
done

# Initialise log file
initLog

echo "" 2>&1 | tee -a "$logFile"
echo "===== Parameters passed: =====" 2>&1 | tee -a "$logFile"
echo "" 2>&1 | tee -a "$logFile"
echo "S: " "$sourceDir" 2>&1 | tee -a "$logFile"
echo "D: " "$destDir" 2>&1 | tee -a "$logFile"
echo "C: " "$exifConfig" 2>&1 | tee -a "$logFile"
echo "E: " "$errorDir" 2>&1 | tee -a "$logFile"
#echo "R: " "$resumeOp" 2>&1 | tee -a "$logFile"
echo "G: " "$regenCRC" 2>&1 | tee -a "$logFile"
echo "X: " "$genConFile" 2>&1 | tee -a "$logFile"
echo "u: " "$deleteDups" 2>&1 | tee -a "$logFile"
echo "l: " "$logFile" 2>&1 | tee -a "$logFile"
echo "m: " "$moveFiles" 2>&1 | tee -a "$logFile"
echo "c: " "$copyFiles" 2>&1 | tee -a "$logFile"
echo "d: " "$deleteFiles" 2>&1 | tee -a "$logFile"
echo "i: " "$interactive" 2>&1 | tee -a "$logFile"
echo "r: " "$renameFiles" 2>&1 | tee -a "$logFile"
echo "" 2>&1 | tee -a "$logFile"
#echo "CRC file: '$crcFile'." 2>&1 | tee -a "$logFile"
#echo "Sorted Pictures in: '$destDir'." 2>&1 | tee -a "$logFile"
echo "" 2>&1 | tee -a "$logFile"
echo "===== Starting Script =====" 2>&1 | tee -a "$logFile"


########################################################
# CHECKS IF PARAMETERS, DIRECTORIES AND FILES ARE GOOD #
########################################################

# If option to generate ~/.ExifTool_config has been passed, call generateExifTool_Config function and exit
[ $genConFile == 1 ] && generateExifTool_Config

# Check is source and destination directories are the same
[ "$sourceDir" -ef "$destDir" ] && { echo "FATAL: Source and Destination directories are the same."; usage; exit 22; }

# Check if destination directory exists. If not, create it.
([ $regenCRC == 1 ] && [ ! -d "$destDir" ]) && { echo "WARNING: Destination directory '$destDir' does not exist. Nothing to regenerate." 2>&1 | tee -a "$logFile"; exit 2; }

# Check if source directory exists
[ $regenCRC ==  0 ] && [ ! -d "$sourceDir" ] && { echo "FATAL: Source directory '$sourceDir' does not exist."; exit 2; }

# Check and initialise error directory
if [ ! $errorDir == "" ]; then
	# Check if errorDir directory exists
	[ -d "$errorDir" ] || { echo "FATAL: Error directory '$errorDir' does not exist. Creating $(readlink -f $errorDir)."; /bin/mkdir -p -v "$errorDir" 2>&1 | tee -a "$logFile"; }
else
	errorDir="$destDir/$errorDirName"
	echo "INFO: Using default error directory: '$errorDir'"
	/bin/mkdir -p -v $errorDir 2>&1 | tee -a "$logFile"
fi

# Check and initialise unsorted directory
unsortedDir="$destDir/$unsortedDirName"
[ -d "$unsortedDir" ] || /bin/mkdir -p -v "$unsortedDir" 2>&1 | tee -a "$logFile"

# Check and initialise duplicate directory
dupDir="$destDir/$duplicateDirName"
[ -d "$dupDir" ] || /bin/mkdir -p -v "$dupDir" 2>&1 | tee -a "$logFile"

#echo "Press [ENTER] to continue..."
#read c

# Initialise CRC file
initCRClog

# If option -G to regenerate CRC has been passed, call regenerateCRC function and exit
[ "$regenCRC" -eq "1" ] && regenerateCRC

# If option -u to delete duplicate images has been passed, call delDupImages function and exit
[ "$deleteDups" -eq "1" ] && delDupImages

# Check COPY or MOVE image parameters
# copy file or move files? They are two mutually exclusive options. Only one can be selected.
[ $copyFiles == 1 ] && [ $moveFiles == 1 ] && { echo "FATAL: impossible combination of copy and move files: -m or -c must be specified, but not both."; usage; exit 22; }

# Check if ExifTool_config file exists
if [ ! $exifConfig ]; then
	echo "INFO: No ExifTool config file specified."
	if [ -f "$HOME/$exifTool_config" ]; then
		echo "INFO: Home ExifTool config files exists. Using that."
	else
		generateExifTool_Config
	fi
else {
	if [ ! -f $exifConfig ]; then 
		echo "FATAL: ExifTool config file specified '$exifConfig' not found."
		exit 2
	fi
}
fi

###########################################
# Actual script functionality starts here #
###########################################

# for loop together with find command does not handle file names with spaces
originalIFS=$IFS
IFS=$(echo -en "\n\b")
#IFS=$'\n'

# Lets find all files in the  $sourceDir and beyond!
#for file in $(find "$sourceDir" -type f -iname "*.mov" -o -iname "*.mpg" -o -iname "*.avi" -o -iname "*.mp4" -o -iname "*.mkv" -o -iname "*.mpeg" -o -iname "*.3gp"); do
for file in $(find "$sourceDir" -type f); do
	let numFilesProcessed++;
	echo 2>&1 | tee -a "$logFile"
    echo "- Processing file #$numFilesProcessed: '$file'" 2>&1 | tee -a "$logFile"

	# Activate case insensitive so that we can match any case extension (eg: JPG|jpg|JpG|jPg ecc)
	shopt -s nocasematch

	extension="${file##*.}"
	case $extension in 
		# Image files
		jpg|jpeg|png|pcd|bmp|gif|cr2|tif|tiff)
			sortImage "$file";
		;;
		# Movie files
		mov|mpg|avi|mp4|mkv|mpeg|3gp)
			sortImage "$file";
		;;
		# Ignored files
		~*~|crc|bak|ini|411|thm|htm|html|json|txt|db|log|tgz|aae|xcf|zip|pdf|odt|sla|odg|svg|ora|b64|ind)
			let numFilesIgnored++;
			echo "	Ignoring file... Done." 2>&1 | tee -a "$logFile"
		;;
		# Unknown files
		*) echo "ERROR:	Unknown extension found: $extension" 2>&1 | tee -a "$logFile"
			let numFilesUnknown++;
			setFileAside $file;
			echo "	Done." 2>&1 | tee -a "$logFile"
		;;
	esac

# NO NEED FOR THIS HERE...
#	# We never encountered this file before...
#	# write this file's CRC to the CRC log file
#	# only if it's not a Dry Run...
#	if [ $copyFiles -ne $moveFiles ]; then
#		echo "$thisCRC,$file" >> "$crcFile"
#	fi

	# Deactivate case insensitive
	shopt -u nocasematch
done # done with the for loop

IFS=$originalIFS

echo 2>&1 | tee -a "$logFile"
echo "Finished!" 2>&1 | tee -a "$logFile"

# Backup CRC file in destination folder (for good measure!)
/bin/cp -pv --backup=numbered "$crcFile" "$destDir/$crcFileName.bak"

# Print the final report
printReport;

exit 0;