s3-common-functions

#! /usr/bin/env bash
cat > /dev/null << EndOfLicence
s3-bash
Copyright 2007 Raphael James Cohn

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
in compliance with the License.
You may obtain a copy of the License at
 
 	http://www.apache.org/licenses/LICENSE-2.0
 	
Unless required by applicable law or agreed to in writing, software distributed under the License
is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
or implied. See the License for the specific language governing permissions and limitations under
the License.
EndOfLicence

# Pragmas
set -u
set -e

# Constants
readonly version="0.02"
readonly userSpecifiedDataErrorExitCode=1
readonly invalidCommandLineOption=2
readonly internalErrorExitCode=3
readonly invalidEnvironmentExitCode=4
readonly ipadXorByte=0x36
readonly opadXorByte=0x5c

# Command-like aliases
readonly sha1="openssl dgst -sha1 -binary"
readonly base64encode="openssl enc -base64 -e -in"
readonly base64decode="openssl enc -base64 -d -in"

# Globals
declare -a temporaryFiles

function base64EncodedMD5
{
	openssl dgst -md5 -binary "$1" | openssl enc -e -base64
}

function printErrorMessage
{
	printf "%s: %s\n" "$1" "$2" 1>&2
}

function printErrorHelpAndExit
{
	printErrorMessage "$weAreKnownAs" "$1"
	printHelpAndExit $2
}

function checkProgramIsInEnvironment
{
	if [ ! -x "$(which $1)" ]; then
		printErrorHelpAndExit "Environment Error: $1 not found on the path or not executable" $invalidEnvironmentExitCode
	fi
}

# Do not use this from directly. Due to a bug in bash, array assignments do not work when the function is used with command substitution
function createTemporaryFile
{
	local temporaryFile="$(mktemp "$temporaryDirectory/$$.$1.XXXXXXXX")" || printErrorHelpAndExit "Environment Error: Could not create a temporary file. Please check you /tmp folder permissions allow files and folders to be created and disc space." $invalidEnvironmentExitCode
	local length="${#temporaryFiles[@]}"
	temporaryFiles[$length]="$temporaryFile"
}

function mostRecentTemporaryFile
{
	local length="${#temporaryFiles[@]}"
	local lastIndex
	((lastIndex = --length))
	echo "${temporaryFiles[$lastIndex]}"
}

function deleteTemporaryFile
{
	rm -f "$1" || printErrorHelpAndExit "Environment Error: Could not delete a temporary file ($1)." $invalidEnvironmentExitCode
}

function removeTemporaryFiles
{
	length="${#temporaryFiles[@]}"
	if [ $length -eq 0 ]; then
		return
	fi
	for temporaryFile in ${temporaryFiles[@]}; do
		deleteTemporaryFile "$temporaryFile"
	done
	temporaryFiles=()
	length="${#temporaryFiles[@]}"
}

function checkEnvironment
{
	programs=(openssl curl od dd printf sed awk sort mktemp rm grep cp ls env bash)
	for program in "${programs[@]}"; do
		checkProgramIsInEnvironment "$program"
	done
	
	local temporaryFolder="${TMPDIR:-/tmp}"
	if [ ! -x "$temporaryFolder" ]; then
		printErrorHelpAndExit "Environment Error: The temporary directory ($temporaryFolder) does not exist. Please set the TMPDIR environment variable to your temporary directory" $invalidEnvironmentExitCode
	fi
	readonly temporaryDirectory="$temporaryFolder/s3-bash/$weAreKnownAs"
	mkdir -p "$temporaryDirectory" || printErrorHelpAndExit "Environment Error: Could not create a temporary directory ($temporaryDiectory). Please check you /tmp folder permissions allow files and folders to be created and you have sufficient disc space" $invalidEnvironmentExitCode
	
	#Check we can create and delete temporary files
	createTemporaryFile "check"
	temporaryFileCheck="$(mostRecentTemporaryFile)"
	echo "Checking we can write to temporary files. If this is still here then we could not delete temporary files." > "$temporaryFileCheck"
	removeTemporaryFiles
}

function setErrorTraps
{
	trap "removeTemporaryFiles; exit $internalErrorExitCode" INT TERM EXIT
}

function unsetErrorTraps
{
	trap - INT TERM EXIT
}

function verifyUrl
{
	if [ -z "$url" ]; then
		printErrorHelpAndExit "URL not specified" $userSpecifiedDataErrorExitCode
	elif echo $url | grep -q http://; then
		printErrorHelpAndExit "URL starts with http://" $userSpecifiedDataErrorExitCode
	elif echo $url | grep -q https://; then
		printErrorHelpAndExit "URL starts with https://" $userSpecifiedDataErrorExitCode
	elif echo $url | grep -v ^/; then
		printErrorHelpAndExit "URL does not start with /" $userSpecifiedDataErrorExitCode
	fi
}

function appendHash
{
	local fileToHash="$1"
	local fileToWriteTo="$2"
	$sha1 "$fileToHash" >> "$fileToWriteTo"
}

function writeHash
{
	local fileToHash="$1"
	local fileToWriteTo="$2"
	$sha1 -out "$fileToWriteTo" "$fileToHash"
}

function checkAwsKey
{
	local originalKeyFile="$1"
	local keySize="$(ls -l "$originalKeyFile" | awk '{ print $5 }')"
	if [ ! $keySize -eq 40 ]; then
		printErrorHelpAndExit "We do not understand Amazon AWS secret keys which are not 40 bytes long. Have you included a carriage return or line feed by mistake at the end of the secret key file?" $userSpecifiedDataErrorExitCode
	fi
}

function padDecodedKeyTo
{
	local originalKeyFile="$1"
	local keyFile="$2"
	cp "$originalKeyFile" "$keyFile"
	
	local keySize=$(ls -l "$keyFile" | awk '{ print $5 }')
	if [ $keySize -lt 64 ]; then
		local zerosToWrite=$((64 - $keySize))
		dd if=/dev/zero of=$keyFile bs=1 count=$zerosToWrite seek=$keySize 2> /dev/null
	elif [ $keySize -gt 64 ]; then
		echo "Warning: Support for hashing keys bigger than the SHA1 block size of 64 bytes is untested" 1>&2
		writeHash "$originalKeyFile" "$keyFile"
		local keySize=$(ls -l "$keyFile" | awk '{ print $5 }')
		if [ $keySize -lt 64 ]; then
			local zerosToWrite=$((64 - $keySize))
			dd if=/dev/zero of=$keyFile bs=1 count=$zerosToWrite seek=$keySize 2> /dev/null
		fi
		exit 1
	else
		:
	fi
}

function writeLongAsByte
{
	local byte="$1"
	local file="$2"
	printf "\\$(printf "%o" $byte)" >> "$file"
}

function readBytesAndXorAndWriteAsBytesTo
{
	local inputFile="$1"
	local xorByte=$2
	local outputFile="$3"
	
	od -v -A n -t uC "$inputFile" | awk '{ OFS="\n"; for (i = 1; i <= NF; i++) print $i }' |
	while read byte; do
		((xord = byte ^ xorByte))
		writeLongAsByte $xord "$outputFile"
	done
}

function writeHexByte
{
	local byte="$1"
	local file="$2"
	printf "\\$(printf "%o" 0x$byte)" >> "$file"
}

function writeHexString
{
	local hexString="$1"
	for byte in $(echo $hexString | sed 's/../& /g'); do
		writeHexByte "$byte" "$2"
	done
}

function writeStringToSign
{
	local outputFile="$1"
	echo $verb >> "$outputFile"
	echo "$contentMD5" >> "$outputFile"
	echo "$contentType" >> "$outputFile"
	echo "$currentDateTime" >> "$outputFile"
	
	writeStringToSignAmazonHeaders "$outputFile"
	
	urlPath="/$bucket$(echo "$url" | awk 'BEGIN { FS="[?]"} { print $1 }')"
	urlQueryString="$(echo "$url" | awk 'BEGIN { FS="[?]"} { print $2 }')"
	printf "$urlPath" >> "$outputFile"
	if [ "$urlQueryString" = "acl" ] || [ "$urlQueryString" = "torrent" ]; then
		printf "?" >> "$outputFile"
		printf "$urlQueryString" >> "$outputFile"
	fi
}

function writeStringToSignAmazonHeaders()
{
	local outputFile="$1"
	
	#Convert all headers to lower case
	#sort
	#Strip ": " to ":"
	#Add LF to each header
        awk 'BEGIN { FS=": " } NF == 2 { print tolower($1) ":" $2 }' "$amazonHeaderFile" | grep ^x-amz-* | sort >> "$outputFile"
	#TODO: RFC 2616, section 4.2 (combine repeated headers' values)
	#TODO: Unfold long lines (not supported elsewhere)
}

function computeAwsAuthorizationHeader
{
	checkAwsKey "$awsAccessSecretKeyIdFile"
	
	createTemporaryFile "key"
	local tempKeyFile="$(mostRecentTemporaryFile)"
	
	createTemporaryFile "ipad"
	local ipadHashingFile="$(mostRecentTemporaryFile)"
	
	createTemporaryFile "opad"
	local opadHashingFile="$(mostRecentTemporaryFile)"
	
	createTemporaryFile "HMAC-SHA1"
	local hmacSha1File="$(mostRecentTemporaryFile)"

	padDecodedKeyTo "$awsAccessSecretKeyIdFile" "$tempKeyFile"
	readBytesAndXorAndWriteAsBytesTo "$tempKeyFile" ipadXorByte "$ipadHashingFile"

	writeStringToSign "$ipadHashingFile"

	readBytesAndXorAndWriteAsBytesTo "$tempKeyFile" opadXorByte "$opadHashingFile"
	appendHash "$ipadHashingFile" "$opadHashingFile"
	writeHash "$opadHashingFile" "$hmacSha1File"

	local signature="$($base64encode "$hmacSha1File")"

	echo "Authorization: AWS $awsAccessKeyId:$signature"
}

function writeAmazonHeadersForCurl
{
	if [ ! -e "$amazonHeaderFile" ]; then
		printErrorHelpAndExit "Amazon Header file does not exist" $userSpecifiedDataErrorExitCode
	elif grep -q ^X-Amz-Date: "$amazonHeaderFile"; then
		printErrorHelpAndExit "X-Amz-Date header not allowed" $userSpecifiedDataErrorExitCode
	fi
	# Consider using sed...
        awk 'BEGIN { ORS=" "; FS="\n" } { print "--header \"" $1 "\""}' "$amazonHeaderFile" >> "$1"
}

function runCurl
{
	local verbAndAnyData="$1"
	local fullUrl="$protocol://$bucket.s3.amazonaws.com$url"
	createTemporaryFile "curl"
	local tempCurlCommand="$(mostRecentTemporaryFile)"
	local cleanUpCommand="rm -f "$tempCurlCommand""

	echo "#! /usr/bin/env bash" >> "$tempCurlCommand"
	printf "curl %s %s --dump-header \"%s\" " "$verbose" "$verbAndAnyData" "$dumpHeaderFile" >> "$tempCurlCommand"
	writeAmazonHeadersForCurl "$tempCurlCommand"
	printf " --header \"%s\"" "Date: $currentDateTime" >> "$tempCurlCommand"
	printf " --header \"%s\"" "$authorizationHeader" >> "$tempCurlCommand"
	if [ ! -z "$contentType" ]; then
		printf " --header \"Content-Type: %s\"" "$contentType" >> "$tempCurlCommand"
	fi
	if [ ! -z "$contentMD5" ]; then
		printf " --header \"Content-MD5: %s\"" "$contentMD5" >> "$tempCurlCommand"
	fi
	printf " \"%s\"\n" "$fullUrl" >> "$tempCurlCommand"
	
	unsetErrorTraps
	exec env bash "$tempCurlCommand"
}

function initialise
{
	setErrorTraps
	checkEnvironment
}

function main
{
	initialise
	parseOptions "$@"
	readonly currentDateTime="$(LC_TIME=C TZ=utc date "+%a, %d %h %Y %T %z")"
	prepareToRunCurl
	readonly authorizationHeader="$(computeAwsAuthorizationHeader)"
	runCurl "$verbToPass"
}