Skip to content

Commit

Permalink
[BUGFIX]: Handle special chars in filenames | Resolve #156
Browse files Browse the repository at this point in the history 
try to follow https://tools.ietf.org/html/rfc7159#section-7

use _json_escape function to handle it

also use parsed var to properly handle pretty printing
  • Loading branch information
@Akianonymus
Akianonymus committed May 16, 2021
1 parent 4b1aa41 commit 55201c7
Show file tree
Hide file tree
Showing 8 changed files with 268 additions and 219 deletions.
31 changes: 17 additions & 14 deletions bash/common-utils.bash
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,24 +192,27 @@ _get_latest_sha() {
###################################################
# Encode the given string to parse properly as json
# Globals: None
# Arguments: 1
# ${1} = string
# Result: print encoded string
# Arguments: 2
# ${1} = json or something else
# ${2} = input
# Result: if ${1} is j, then escape all chars, else only special chars
# Reference:
# https://tools.ietf.org/html/rfc7159#section-7
###################################################
_json_escape() {
declare input="${1:?Provide Input}"
# \ and /
: "${input//\\/\\\\}"
: "${_//\//\\\/}"
: "${_//\'/\\\'}" # ' (not strictly needed ?)
: "${_//\"/\\\"}" # "
: "${_//$'\t'/\\\t}" # \t (tab)
: "${_//$'\n'/\\\n}" # \n (newline)
: "${_//$'\r'/\\\r}" # \r (carriage return)
: "${_//$'\f'/\\\f}" # \f (form feed)
: "${_//$'\b'/\\\b}" # \b (backspace)
declare mode="${1:?}" input="${2:?Provide Input}"
[[ ${mode} = "j" ]] && {
# \ and /
: "${input//\\/\\\\}"
: "${_//\//\\\/}"
# : "${_//\'/\\\'}" # ' (not strictly needed ?)
input="${_//\"/\\\"}" # "
}
: "${input//$'\t'/\\\t}" # \t (tab)
: "${_//$'\n'/\\\n}" # \n (newline)
: "${_//$'\r'/\\\r}" # \r (carriage return)
: "${_//$'\f'/\\\f}" # \f (form feed)
: "${_//$'\b'/\\\b}" # \b (backspace)
printf "%s" "${_}"
}

Expand Down
73 changes: 37 additions & 36 deletions bash/drive-utils.bash
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ _check_existing_file() {
declare name="${1##*/}" rootdir="${2}" query search_response id

"${EXTRA_LOG}" "justify" "Checking if file" " exists on gdrive.." "-" 1>&2
query="$(_url_encode "name='${name}' and '${rootdir}' in parents and trashed=false")"
query="$(_url_encode "name=\"${name}\" and '${rootdir}' in parents and trashed=false")"

search_response="$(_api_request "${CURL_PROGRESS_EXTRA}" \
"${API_URL}/drive/${API_VERSION}/files?q=${query}&fields=files(id,name,mimeType)&supportsAllDrives=true&includeItemsFromAllDrives=true" || :)" && _clear_line 1 1>&2
Expand All @@ -29,9 +29,9 @@ _check_existing_file() {

###################################################
# Copy/Clone a public gdrive file/folder from another/same gdrive account
# Globals: 6 variables, 2 functions
# Globals: 6 variables, 6 functions
# Variables - API_URL, API_VERSION, CURL_PROGRESS, LOG_FILE_ID, QUIET, ACCESS_TOKEN, DESCRIPTION_FILE
# Functions - _print_center, _check_existing_file, _json_value, _bytes_to_human, _clear_line
# Functions - _print_center, _check_existing_file, _json_value, _json_escape _bytes_to_human, _clear_line
# Arguments: 5
# ${1} = update or upload ( upload type )
# ${2} = file id to upload
Expand All @@ -47,32 +47,32 @@ _check_existing_file() {
_clone_file() {
[[ $# -lt 5 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1
declare job="${1}" file_id="${2}" file_root_id="${3}" name="${4}" size="${5}"
declare clone_file_post_data clone_file_response readable_size _file_id description && STRING="Cloned"
readable_size="$(_bytes_to_human "${size}")"
declare clone_file_post_data clone_file_response readable_size _file_id description escaped_name && STRING="Cloned"
escaped_name="$(_json_escape j "${name}")" print_name="$(_json_escape p "${name}")" readable_size="$(_bytes_to_human "${size}")"

# create description data
[[ -n ${DESCRIPTION_FILE} ]] && {
: "${DESCRIPTION_FILE//%f/${name}}" && : "${_//%s/${readable_size}}"
description="$(_json_escape "${_}")" # escape for json
description="$(_json_escape j "${_}")" # escape for json
}

clone_file_post_data="{\"parents\": [\"${file_root_id}\"]${description:+,\"description\":\"${description}\"}}"

_print_center "justify" "${name} " "| ${readable_size}" "="
_print_center "justify" "${print_name} " "| ${readable_size}" "="

if [[ ${job} = update ]]; then
declare file_check_json
# Check if file actually exists.
if file_check_json="$(_check_existing_file "${name}" "${file_root_id}")"; then
if file_check_json="$(_check_existing_file "${escaped_name}" "${file_root_id}")"; then
if [[ -n ${SKIP_DUPLICATES} ]]; then
_collect_file_info "${file_check_json}" || return 1
_clear_line 1
"${QUIET:-_print_center}" "justify" "${name}" " already exists." "=" && return 0
"${QUIET:-_print_center}" "justify" "${print_name}" " already exists." "=" && return 0
else
_print_center "justify" "Overwriting file.." "-"
{ _file_id="$(_json_value id 1 1 <<< "${file_check_json}")" &&
clone_file_post_data="$(_drive_info "${_file_id}" "parents,writersCanShare")"; } ||
{ _error_logging_upload "${name}" "${post_data:-${file_check_json}}" || return 1; }
{ _error_logging_upload "${print_name}" "${post_data:-${file_check_json}}" || return 1; }
if [[ ${_file_id} != "${file_id}" ]]; then
_api_request -s \
-X DELETE \
Expand All @@ -97,15 +97,15 @@ _clone_file() {
"${API_URL}/drive/${API_VERSION}/files/${file_id}/copy?supportsAllDrives=true&includeItemsFromAllDrives=true" || :)"
for _ in 1 2 3; do _clear_line 1; done
_collect_file_info "${clone_file_response}" || return 1
"${QUIET:-_print_center}" "justify" "${name} " "| ${readable_size} | ${STRING}" "="
"${QUIET:-_print_center}" "justify" "${print_name} " "| ${readable_size} | ${STRING}" "="
return 0
}

###################################################
# Create/Check directory in google drive.
# Globals: 3 variables, 2 functions
# Globals: 3 variables, 3 functions
# Variables - API_URL, API_VERSION, ACCESS_TOKEN
# Functions - _url_encode, _json_value
# Functions - _url_encode, _json_value, _json_escape
# Arguments: 2
# ${1} = dir name
# ${2} = root dir id of given dir
Expand All @@ -115,17 +115,18 @@ _clone_file() {
###################################################
_create_directory() {
[[ $# -lt 2 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1
declare dirname="${1##*/}" rootdir="${2}" query search_response folder_id
declare dirname="${1##*/}" escaped_dirname rootdir="${2}" query search_response folder_id
escaped_dirname="$(_json_escape j "${dirname}")" print_dirname="$(_json_escape p "${dirname}")"

"${EXTRA_LOG}" "justify" "Creating gdrive folder:" " ${dirname}" "-" 1>&2
query="$(_url_encode "mimeType='application/vnd.google-apps.folder' and name='${dirname}' and trashed=false and '${rootdir}' in parents")"
"${EXTRA_LOG}" "justify" "Creating gdrive folder:" " ${print_dirname}" "-" 1>&2
query="$(_url_encode "mimeType='application/vnd.google-apps.folder' and name=\"${escaped_dirname}\" and trashed=false and '${rootdir}' in parents")"

search_response="$(_api_request "${CURL_PROGRESS_EXTRA}" \
"${API_URL}/drive/${API_VERSION}/files?q=${query}&fields=files(id)&supportsAllDrives=true&includeItemsFromAllDrives=true" || :)" && _clear_line 1 1>&2

if ! folder_id="$(printf "%s\n" "${search_response}" | _json_value id 1 1)"; then
declare create_folder_post_data create_folder_response
create_folder_post_data="{\"mimeType\": \"application/vnd.google-apps.folder\",\"name\": \"${dirname}\",\"parents\": [\"${rootdir}\"]}"
create_folder_post_data="{\"mimeType\": \"application/vnd.google-apps.folder\",\"name\": \"${escaped_dirname}\",\"parents\": [\"${rootdir}\"]}"
create_folder_response="$(_api_request "${CURL_PROGRESS_EXTRA}" \
-X POST \
-H "Content-Type: application/json; charset=UTF-8" \
Expand Down Expand Up @@ -187,9 +188,9 @@ _extract_id() {
###################################################
# Upload ( Create/Update ) files on gdrive.
# Interrupted uploads can be resumed.
# Globals: 8 variables, 10 functions
# Globals: 8 variables, 11 functions
# Variables - API_URL, API_VERSION, QUIET, VERBOSE, VERBOSE_PROGRESS, CURL_PROGRESS, LOG_FILE_ID, ACCESS_TOKEN, DESCRIPTION_FILE
# Functions - _url_encode, _json_value, _print_center, _bytes_to_human
# Functions - _url_encode, _json_value, _json_escape _print_center, _bytes_to_human
# _generate_upload_link, _upload_file_from_uri, _log_upload_session, _remove_upload_session
# _full_upload, _collect_file_info
# Arguments: 3
Expand All @@ -207,10 +208,10 @@ _extract_id() {
_upload_file() {
[[ $# -lt 3 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1
declare job="${1}" input="${2}" folder_id="${3}" \
slug inputname extension inputsize readable_size request_method url postdata uploadlink upload_body mime_type description \
slug escaped_slug inputname extension inputsize readable_size request_method url postdata uploadlink upload_body mime_type description \
resume_args1 resume_args2 resume_args3

slug="${input##*/}"
slug="${input##*/}" escaped_slug="$(_json_escape j "${slug}")" print_slug="$(_json_escape p "${slug}")"
inputname="${slug%.*}"
extension="${slug##*.}"
inputsize="$(($(wc -c < "${input}")))" && content_length="${inputsize}"
Expand All @@ -227,28 +228,28 @@ _upload_file() {
# create description data
[[ -n ${DESCRIPTION_FILE} ]] && {
: "${DESCRIPTION_FILE//%f/${slug}}" && : "${_//%s/${inputsize}}" && : "${_//%m/${mime_type}}"
description="$(_json_escape "${_}")" # escape for json
description="$(_json_escape j "${_}")" # escape for json
}

_print_center "justify" "${input##*/}" " | ${readable_size}" "="
_print_center "justify" "${print_slug}" " | ${readable_size}" "="

# Set proper variables for overwriting files
[[ ${job} = update ]] && {
declare file_check_json
# Check if file actually exists, and create if not.
if file_check_json="$(_check_existing_file "${slug}" "${folder_id}")"; then
if file_check_json="$(_check_existing_file "${escaped_slug}" "${folder_id}")"; then
if [[ -n ${SKIP_DUPLICATES} ]]; then
# Stop upload if already exists ( -d/--skip-duplicates )
_collect_file_info "${file_check_json}" "${slug}" || return 1
_collect_file_info "${file_check_json}" "${escaped_slug}" || return 1
_clear_line 1
"${QUIET:-_print_center}" "justify" "${slug}" " already exists." "=" && return 0
"${QUIET:-_print_center}" "justify" "${print_slug}" " already exists." "=" && return 0
else
request_method="PATCH"
_file_id="$(_json_value id 1 1 <<< "${file_check_json}")" ||
{ _error_logging_upload "${slug}" "${file_check_json}" || return 1; }
{ _error_logging_upload "${print_slug}" "${file_check_json}" || return 1; }
url="${API_URL}/upload/drive/${API_VERSION}/files/${_file_id}?uploadType=resumable&supportsAllDrives=true&includeItemsFromAllDrives=true"
# JSON post data to specify the file name and folder under while the file to be updated
postdata="{\"mimeType\": \"${mime_type}\",\"name\": \"${slug}\",\"addParents\": [\"${folder_id}\"]${description:+,\"description\":\"${description}\"}}"
postdata="{\"mimeType\": \"${mime_type}\",\"name\": \"${escaped_slug}\",\"addParents\": [\"${folder_id}\"]${description:+,\"description\":\"${description}\"}}"
STRING="Updated"
fi
else
Expand All @@ -261,11 +262,11 @@ _upload_file() {
url="${API_URL}/upload/drive/${API_VERSION}/files?uploadType=resumable&supportsAllDrives=true&includeItemsFromAllDrives=true"
request_method="POST"
# JSON post data to specify the file name and folder under while the file to be created
postdata="{\"mimeType\": \"${mime_type}\",\"name\": \"${slug}\",\"parents\": [\"${folder_id}\"]${description:+,\"description\":\"${description}\"}}"
postdata="{\"mimeType\": \"${mime_type}\",\"name\": \"${escaped_slug}\",\"parents\": [\"${folder_id}\"]${description:+,\"description\":\"${description}\"}}"
STRING="Uploaded"
}

__file="${HOME}/.google-drive-upload/${slug}__::__${folder_id}__::__${inputsize}"
__file="${HOME}/.google-drive-upload/${print_slug}__::__${folder_id}__::__${inputsize}"
# https://developers.google.com/drive/api/v3/manage-uploads
if [[ -r "${__file}" ]]; then
uploadlink="$(< "${__file}")"
Expand All @@ -283,7 +284,7 @@ _upload_file() {
# Resuming interrupted uploads needs http1.1
resume_args1='-s' resume_args2='--http1.1' resume_args3="Content-Range: ${content_range}"
_upload_file_from_uri _clear_line
_collect_file_info "${upload_body}" "${slug}" || return 1
_collect_file_info "${upload_body}" "${print_slug}" || return 1
_normal_logging_upload
_remove_upload_session
else
Expand All @@ -292,7 +293,7 @@ _upload_file() {
;;
201 | 200) # Completed Resumable URI give 20* status
upload_body="${http_code}"
_collect_file_info "${upload_body}" "${slug}" || return 1
_collect_file_info "${upload_body}" "${print_slug}" || return 1
_normal_logging_upload
_remove_upload_session
;;
Expand Down Expand Up @@ -337,7 +338,7 @@ _upload_file_from_uri() {
-X PUT \
-H "Content-Type: ${mime_type}" \
-H "Content-Length: ${content_length}" \
-H "Slug: ${slug}" \
-H "Slug: ${print_slug}" \
-T "${input}" \
-o- \
--url "${uploadlink}" \
Expand All @@ -351,7 +352,7 @@ _upload_file_from_uri() {
# logging in case of successful upload
_normal_logging_upload() {
[[ -z ${VERBOSE_PROGRESS} ]] && _clear_line 1
"${QUIET:-_print_center}" "justify" "${slug} " "| ${readable_size} | ${STRING}" "="
"${QUIET:-_print_center}" "justify" "${print_slug} " "| ${readable_size} | ${STRING}" "="
return 0
}

Expand All @@ -369,10 +370,10 @@ _remove_upload_session() {

# wrapper to fully upload a file from scratch
_full_upload() {
_generate_upload_link || { _error_logging_upload "${slug}" "${uploadlink}" || return 1; }
_generate_upload_link || { _error_logging_upload "${print_slug}" "${uploadlink}" || return 1; }
_log_upload_session
_upload_file_from_uri
_collect_file_info "${upload_body}" "${slug}" || return 1
_collect_file_info "${upload_body}" "${print_slug}" || return 1
_normal_logging_upload
_remove_upload_session
return 0
Expand Down
31 changes: 17 additions & 14 deletions bash/release/gsync
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,24 +193,27 @@ _get_latest_sha() {
###################################################
# Encode the given string to parse properly as json
# Globals: None
# Arguments: 1
# ${1} = string
# Result: print encoded string
# Arguments: 2
# ${1} = json or something else
# ${2} = input
# Result: if ${1} is j, then escape all chars, else only special chars
# Reference:
# https://tools.ietf.org/html/rfc7159#section-7
###################################################
_json_escape() {
declare input="${1:?Provide Input}"
# \ and /
: "${input//\\/\\\\}"
: "${_//\//\\\/}"
: "${_//\'/\\\'}" # ' (not strictly needed ?)
: "${_//\"/\\\"}" # "
: "${_//$'\t'/\\\t}" # \t (tab)
: "${_//$'\n'/\\\n}" # \n (newline)
: "${_//$'\r'/\\\r}" # \r (carriage return)
: "${_//$'\f'/\\\f}" # \f (form feed)
: "${_//$'\b'/\\\b}" # \b (backspace)
declare mode="${1:?}" input="${2:?Provide Input}"
[[ ${mode} = "j" ]] && {
# \ and /
: "${input//\\/\\\\}"
: "${_//\//\\\/}"
# : "${_//\'/\\\'}" # ' (not strictly needed ?)
input="${_//\"/\\\"}" # "
}
: "${input//$'\t'/\\\t}" # \t (tab)
: "${_//$'\n'/\\\n}" # \n (newline)
: "${_//$'\r'/\\\r}" # \r (carriage return)
: "${_//$'\f'/\\\f}" # \f (form feed)
: "${_//$'\b'/\\\b}" # \b (backspace)
printf "%s" "${_}"
}

Expand Down
Loading

0 comments on commit 55201c7

Please sign in to comment.