gupload: Rewrite logic for access token usage

launch a background service to check access token and update it

checks ACCESS_TOKEN_EXPIRY, try to update before 5 mins of expiry, a fresh token gets 60 mins ( 3600 seconds )

process will be killed when script exits

create a temp file where updated access token will be stored by the bg service

every function that uses access token will source it on every call

make a new function named _api_request for all oauth network calls

Fix labbots#113

README.md

@@ -114,6 +114,7 @@ This repo contains two types of scripts, posix compatible and bash compatible.
 | sed              | Miscellaneous                                          |
 | mktemp           | To generate temporary files ( optional )               |
 | sleep            | Self explanatory                                       |
+| ps               | To manage different processes                          |
 
 <strong>If BASH is not available or BASH is available but version is less tham 4.x, then below programs are also required:</strong>
 
@@ -124,11 +125,10 @@ This repo contains two types of scripts, posix compatible and bash compatible.
 | cat                 | Miscellaneous                              |
 | stty or zsh or tput | To determine column size ( optional )      |
 
-<strong>These programs are needed for synchronisation script:</strong>
+<strong>These are the additional programs needed for synchronisation script:</strong>
 
 | Program       | Role In Script            |
 | ------------- | ------------------------- |
-| ps            | To manage background jobs |
 | tail          | To show indefinite logs   |
 
 ### Installation

bash/drive-utils.bash

@@ -1,4 +1,16 @@
 #!/usr/bin/env bash
+# shellcheck source=/dev/null
+
+###################################################
+# A simple wrapper to check tempfile for access token and make authorized oauth requests to drive api
+###################################################
+_api_request() {
+    . "${TMPFILE}_ACCESS_TOKEN"
+
+    curl --compressed \
+        -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+        "${@}"
+}
 
 ###################################################
 # Method to regenerate access_token ( also updates in config ).
@@ -57,27 +69,24 @@ _error_logging_upload() {
 
 ###################################################
 # Get information for a gdrive folder/file.
-# Globals: 2 variables, 1 function
-#   Variables - API_URL, API_VERSION
+# Globals: 3 variables, 1 function
+#   Variables - API_URL, API_VERSION, ACCESS_TOKEN
 #   Functions - _json_value
-# Arguments: 3
+# Arguments: 2
 #   ${1} = folder/file gdrive id
 #   ${2} = information to fetch, e.g name, id
-#   ${3} = Access Token
 # Result: On
 #   Success - print fetched value
 #   Error   - print "message" field from the json
 # Reference:
 #   https://developers.google.com/drive/api/v3/search-files
 ###################################################
 _drive_info() {
-    [[ $# -lt 3 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1
-    declare folder_id="${1}" fetch="${2}" token="${3}"
-    declare search_response
+    [[ $# -lt 2 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1
+    declare folder_id="${1}" fetch="${2}" search_response
 
     "${EXTRA_LOG}" "justify" "Fetching info.." "-" 1>&2
-    search_response="$(curl --compressed "${CURL_PROGRESS_EXTRA}" \
-        -H "Authorization: Bearer ${token}" \
+    search_response="$(_api_request "${CURL_PROGRESS_EXTRA}" \
         "${API_URL}/drive/${API_VERSION}/files/${folder_id}?fields=${fetch}&supportsAllDrives=true&includeItemsFromAllDrives=true" || :)" && _clear_line 1 1>&2
     _clear_line 1 1>&2
 
@@ -87,27 +96,24 @@ _drive_info() {
 
 ###################################################
 # Search for an existing file on gdrive with write permission.
-# Globals: 2 variables, 2 functions
-#   Variables - API_URL, API_VERSION
+# Globals: 3 variables, 2 functions
+#   Variables - API_URL, API_VERSION, ACCESS_TOKEN
 #   Functions - _url_encode, _json_value
-# Arguments: 3
+# Arguments: 2
 #   ${1} = file name
 #   ${2} = root dir id of file
-#   ${3} = Access Token
 # Result: print file id else blank
 # Reference:
 #   https://developers.google.com/drive/api/v3/search-files
 ###################################################
 _check_existing_file() {
-    [[ $# -lt 3 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1
-    declare name="${1##*/}" rootdir="${2}" token="${3}"
-    declare query search_response id
+    [[ $# -lt 2 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1
+    declare name="${1##*/}" rootdir="${2}" query search_response id
 
     "${EXTRA_LOG}" "justify" "Checking if file" " exists on gdrive.." "-" 1>&2
     query="$(_url_encode "name='${name}' and '${rootdir}' in parents and trashed=false")"
 
-    search_response="$(curl --compressed "${CURL_PROGRESS_EXTRA}" \
-        -H "Authorization: Bearer ${token}" \
+    search_response="$(_api_request "${CURL_PROGRESS_EXTRA}" \
         "${API_URL}/drive/${API_VERSION}/files?q=${query}&fields=files(id,name,mimeType)&supportsAllDrives=true&includeItemsFromAllDrives=true" || :)" && _clear_line 1 1>&2
     _clear_line 1 1>&2
 
@@ -117,35 +123,31 @@ _check_existing_file() {
 
 ###################################################
 # Create/Check directory in google drive.
-# Globals: 2 variables, 2 functions
-#   Variables - API_URL, API_VERSION
+# Globals: 3 variables, 2 functions
+#   Variables - API_URL, API_VERSION, ACCESS_TOKEN
 #   Functions - _url_encode, _json_value
-# Arguments: 3
+# Arguments: 2
 #   ${1} = dir name
 #   ${2} = root dir id of given dir
-#   ${3} = Access Token
 # Result: print folder id
 # Reference:
 #   https://developers.google.com/drive/api/v3/folder
 ###################################################
 _create_directory() {
-    [[ $# -lt 3 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1
-    declare dirname="${1##*/}" rootdir="${2}" token="${3}"
-    declare query search_response folder_id
+    [[ $# -lt 2 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1
+    declare dirname="${1##*/}" rootdir="${2}" query search_response folder_id
 
     "${EXTRA_LOG}" "justify" "Creating gdrive folder:" " ${dirname}" "-" 1>&2
     query="$(_url_encode "mimeType='application/vnd.google-apps.folder' and name='${dirname}' and trashed=false and '${rootdir}' in parents")"
 
-    search_response="$(curl --compressed "${CURL_PROGRESS_EXTRA}" \
-        -H "Authorization: Bearer ${token}" \
+    search_response="$(_api_request "${CURL_PROGRESS_EXTRA}" \
         "${API_URL}/drive/${API_VERSION}/files?q=${query}&fields=files(id)&supportsAllDrives=true&includeItemsFromAllDrives=true" || :)" && _clear_line 1 1>&2
 
     if ! folder_id="$(printf "%s\n" "${search_response}" | _json_value id 1 1)"; then
         declare create_folder_post_data create_folder_response
         create_folder_post_data="{\"mimeType\": \"application/vnd.google-apps.folder\",\"name\": \"${dirname}\",\"parents\": [\"${rootdir}\"]}"
-        create_folder_response="$(curl --compressed "${CURL_PROGRESS_EXTRA}" \
+        create_folder_response="$(_api_request "${CURL_PROGRESS_EXTRA}" \
             -X POST \
-            -H "Authorization: Bearer ${token}" \
             -H "Content-Type: application/json; charset=UTF-8" \
             -d "${create_folder_post_data}" \
             "${API_URL}/drive/${API_VERSION}/files?fields=id&supportsAllDrives=true&includeItemsFromAllDrives=true" || :)" && _clear_line 1 1>&2
@@ -162,9 +164,8 @@ _create_directory() {
 # generate resumable upload link
 _generate_upload_link() {
     "${EXTRA_LOG}" "justify" "Generating upload link.." "-" 1>&2
-    uploadlink="$(curl --compressed "${CURL_PROGRESS_EXTRA}" \
+    uploadlink="$(_api_request "${CURL_PROGRESS_EXTRA}" \
         -X "${request_method}" \
-        -H "Authorization: Bearer ${token}" \
         -H "Content-Type: application/json; charset=UTF-8" \
         -H "X-Upload-Content-Type: ${mime_type}" \
         -H "X-Upload-Content-Length: ${inputsize}" \
@@ -185,9 +186,8 @@ _generate_upload_link() {
 _upload_file_from_uri() {
     _print_center "justify" "Uploading.." "-"
     # shellcheck disable=SC2086 # Because unnecessary to another check because ${CURL_PROGRESS} won't be anything problematic.
-    upload_body="$(curl --compressed ${CURL_PROGRESS} \
+    upload_body="$(_api_request ${CURL_PROGRESS} \
         -X PUT \
-        -H "Authorization: Bearer ${token}" \
         -H "Content-Type: ${mime_type}" \
         -H "Content-Length: ${content_length}" \
         -H "Slug: ${slug}" \
@@ -235,16 +235,15 @@ _full_upload() {
 ###################################################
 # Upload ( Create/Update ) files on gdrive.
 # Interrupted uploads can be resumed.
-# Globals: 7 variables, 10 functions
-#   Variables - API_URL, API_VERSION, QUIET, VERBOSE, VERBOSE_PROGRESS, CURL_PROGRESS, LOG_FILE_ID
+# Globals: 8 variables, 10 functions
+#   Variables - API_URL, API_VERSION, QUIET, VERBOSE, VERBOSE_PROGRESS, CURL_PROGRESS, LOG_FILE_ID, ACCESS_TOKEN
 #   Functions - _url_encode, _json_value, _print_center, _bytes_to_human
 #               _generate_upload_link, _upload_file_from_uri, _log_upload_session, _remove_upload_session
 #               _full_upload, _collect_file_info
-# Arguments: 5
+# Arguments: 3
 #   ${1} = update or upload ( upload type )
 #   ${2} = file to upload
 #   ${3} = root dir id for file
-#   ${4} = Access Token
 # Result: On
 #   Success - Upload/Update file and export FILE_ID
 #   Error - return 1
@@ -254,9 +253,9 @@ _full_upload() {
 #   https://developers.google.com/drive/api/v3/reference/files/update
 ###################################################
 _upload_file() {
-    [[ $# -lt 4 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1
-    declare job="${1}" input="${2}" folder_id="${3}" token="${4}"
-    declare slug inputname extension inputsize readable_size request_method url postdata uploadlink upload_body mime_type resume_args1 resume_args2 resume_args3
+    [[ $# -lt 3 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1
+    declare job="${1}" input="${2}" folder_id="${3}" \
+        slug inputname extension inputsize readable_size request_method url postdata uploadlink upload_body mime_type resume_args1 resume_args2 resume_args3
 
     slug="${input##*/}"
     inputname="${slug%.*}"
@@ -278,7 +277,7 @@ _upload_file() {
     [[ ${job} = update ]] && {
         declare file_check_json
         # Check if file actually exists, and create if not.
-        if file_check_json="$(_check_existing_file "${slug}" "${folder_id}" "${token}")"; then
+        if file_check_json="$(_check_existing_file "${slug}" "${folder_id}")"; then
             if [[ -n ${SKIP_DUPLICATES} ]]; then
                 # Stop upload if already exists ( -d/--skip-duplicates )
                 _collect_file_info "${file_check_json}" "${slug}" || return 1
@@ -350,8 +349,8 @@ _upload_file() {
 ###################################################
 # A extra wrapper for _upload_file function to properly handle retries
 # also handle uploads in case uploading from folder
-# Globals: 3 variables, 1 function
-#   Variables - RETRY, UPLOAD_MODE and ACCESS_TOKEN
+# Globals: 2 variables, 1 function
+#   Variables - RETRY, UPLOAD_MODE
 #   Functions - _upload_file
 # Arguments: 3
 #   ${1} = parse or norparse
@@ -367,9 +366,9 @@ _upload_file_main() {
     retry="${RETRY:-0}" && unset RETURN_STATUS
     until [[ ${retry} -le 0 ]] && [[ -n ${RETURN_STATUS} ]]; do
         if [[ -n ${4} ]]; then
-            _upload_file "${UPLOAD_MODE:-create}" "${file}" "${dirid}" "${ACCESS_TOKEN}" 2>| /dev/null 1>&2 && RETURN_STATUS=1 && break
+            _upload_file "${UPLOAD_MODE:-create}" "${file}" "${dirid}" 2>| /dev/null 1>&2 && RETURN_STATUS=1 && break
         else
-            _upload_file "${UPLOAD_MODE:-create}" "${file}" "${dirid}" "${ACCESS_TOKEN}" && RETURN_STATUS=1 && break
+            _upload_file "${UPLOAD_MODE:-create}" "${file}" "${dirid}" && RETURN_STATUS=1 && break
         fi
         RETURN_STATUS=2 retry="$((retry - 1))" && continue
     done
@@ -379,8 +378,8 @@ _upload_file_main() {
 
 ###################################################
 # Upload all files in the given folder, parallelly or non-parallely and show progress
-# Globals: 2 variables, 3 functions
-#   Variables - VERBOSE and VERBOSE_PROGRESS, NO_OF_PARALLEL_JOBS, NO_OF_FILES, TMPFILE, UTILS_FOLDER and QUIET
+# Globals: 7 variables, 3 functions
+#   Variables - VERBOSE, VERBOSE_PROGRESS, NO_OF_PARALLEL_JOBS, NO_OF_FILES, TMPFILE, UTILS_FOLDER and QUIET
 #   Functions - _clear_line, _newline, _print_center and _upload_file_main
 # Arguments: 4
 #   ${1} = parallel or normal
@@ -439,25 +438,24 @@ _upload_folder() {
 
 ###################################################
 # Copy/Clone a public gdrive file/folder from another/same gdrive account
-# Globals: 2 variables, 2 functions
-#   Variables - API_URL, API_VERSION, CURL_PROGRESS, LOG_FILE_ID, QUIET
+# Globals: 6 variables, 2 functions
+#   Variables - API_URL, API_VERSION, CURL_PROGRESS, LOG_FILE_ID, QUIET, ACCESS_TOKEN
 #   Functions - _print_center, _check_existing_file, _json_value, _bytes_to_human, _clear_line
 # Arguments: 5
 #   ${1} = update or upload ( upload type )
 #   ${2} = file id to upload
 #   ${3} = root dir id for file
-#   ${4} = Access Token
-#   ${5} = name of file
-#   ${6} = size of file
+#   ${4} = name of file
+#   ${5} = size of file
 # Result: On
 #   Success - Upload/Update file and export FILE_ID
 #   Error - return 1
 # Reference:
 #   https://developers.google.com/drive/api/v2/reference/files/copy
 ###################################################
 _clone_file() {
-    [[ $# -lt 4 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1
-    declare job="${1}" file_id="${2}" file_root_id="${3}" token="${4}" name="${5}" size="${6}"
+    [[ $# -lt 5 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1
+    declare job="${1}" file_id="${2}" file_root_id="${3}" name="${4}" size="${5}"
     declare clone_file_post_data clone_file_response readable_size _file_id && STRING="Cloned"
     clone_file_post_data="{\"parents\": [\"${file_root_id}\"]}"
     readable_size="$(_bytes_to_human "${size}")"
@@ -467,20 +465,19 @@ _clone_file() {
     if [[ ${job} = update ]]; then
         declare file_check_json
         # Check if file actually exists.
-        if file_check_json="$(_check_existing_file "${name}" "${file_root_id}" "${token}")"; then
+        if file_check_json="$(_check_existing_file "${name}" "${file_root_id}")"; then
             if [[ -n ${SKIP_DUPLICATES} ]]; then
                 _collect_file_info "${file_check_json}" || return 1
                 _clear_line 1
                 "${QUIET:-_print_center}" "justify" "${name}" " already exists." "=" && return 0
             else
                 _print_center "justify" "Overwriting file.." "-"
                 { _file_id="$(_json_value id 1 1 <<< "${file_check_json}")" &&
-                    clone_file_post_data="$(_drive_info "${_file_id}" "parents,writersCanShare" "${token}")"; } ||
+                    clone_file_post_data="$(_drive_info "${_file_id}" "parents,writersCanShare")"; } ||
                     { _error_logging_upload "${name}" "${post_data:-${file_check_json}}" && return 1; }
                 if [[ ${_file_id} != "${file_id}" ]]; then
-                    curl --compressed -s \
+                    _api_request -s \
                         -X DELETE \
-                        -H "Authorization: Bearer ${token}" \
                         "${API_URL}/drive/${API_VERSION}/files/${_file_id}?supportsAllDrives=true&includeItemsFromAllDrives=true" 2>| /dev/null 1>&2 || :
                     STRING="Updated"
                 else
@@ -495,9 +492,8 @@ _clone_file() {
     fi
 
     # shellcheck disable=SC2086 # Because unnecessary to another check because ${CURL_PROGRESS} won't be anything problematic.
-    clone_file_response="$(curl --compressed ${CURL_PROGRESS} \
+    clone_file_response="$(_api_request ${CURL_PROGRESS} \
         -X POST \
-        -H "Authorization: Bearer ${token}" \
         -H "Content-Type: application/json; charset=UTF-8" \
         -d "${clone_file_post_data}" \
         "${API_URL}/drive/${API_VERSION}/files/${file_id}/copy?supportsAllDrives=true&includeItemsFromAllDrives=true" || :)"
@@ -509,28 +505,26 @@ _clone_file() {
 
 ###################################################
 # Share a gdrive file/folder
-# Globals: 2 variables, 4 functions
-#   Variables - API_URL and API_VERSION
+# Globals: 3 variables, 4 functions
+#   Variables - API_URL, API_VERSION, ACCESS_TOKEN
 #   Functions - _url_encode, _json_value, _print_center, _clear_line
-# Arguments: 3
+# Arguments: 2
 #   ${1} = gdrive ID of folder/file
-#   ${2} = Access Token
-#   ${3} = Email to which file will be shared ( optional )
+#   ${2} = Email to which file will be shared ( optional )
 # Result: read description
 # Reference:
 #   https://developers.google.com/drive/api/v3/manage-sharing
 ###################################################
 _share_id() {
     [[ $# -lt 2 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1
-    declare id="${1}" token="${2}" share_email="${3}" role="reader" type="${share_email:+user}"
+    declare id="${1}" share_email="${2}" role="reader" type="${share_email:+user}"
     declare type share_post_data share_post_data share_response
 
     "${EXTRA_LOG}" "justify" "Sharing.." "-" 1>&2
     share_post_data="{\"role\":\"${role}\",\"type\":\"${type:-anyone}\"${share_email:+,\\\"emailAddress\\\":\\\"${share_email}\\\"}}"
 
-    share_response="$(curl --compressed "${CURL_PROGRESS_EXTRA}" \
+    share_response="$(_api_request "${CURL_PROGRESS_EXTRA}" \
         -X POST \
-        -H "Authorization: Bearer ${token}" \
         -H "Content-Type: application/json; charset=UTF-8" \
         -d "${share_post_data}" \
         "${API_URL}/drive/${API_VERSION}/files/${id}/permissions?supportsAllDrives=true&includeItemsFromAllDrives=true" || :)" && _clear_line 1 1>&2