From 1c1d7ba63f1db511b0929e06a017c317fa50b536 Mon Sep 17 00:00:00 2001 From: Akianonymus Date: Fri, 30 Oct 2020 13:20:25 +0530 Subject: [PATCH] Add support for service accounts | Resolve #122 * Add guide in readme on how to generate * Add -sa/--service-account flag * access tokens created by sa will be saved in config, when the sa is given, then it will try to reuse the access token if valid --- README.md | 70 +++++++++-- bash/drive-utils.bash | 17 ++- bash/google-oauth2.bash | 4 +- bash/release/gupload | 264 ++++++++++++++++++++++++++------------- bash/upload-utils.bash | 43 +++++++ bash/upload.bash | 206 +++++++++++++++++------------- install.sh | 28 +++-- sh/drive-utils.sh | 19 ++- sh/google-oauth2.sh | 4 +- sh/release/gupload | 270 +++++++++++++++++++++++++++------------- sh/upload-utils.sh | 51 ++++++++ sh/upload.sh | 202 +++++++++++++++++------------- 12 files changed, 804 insertions(+), 374 deletions(-) diff --git a/README.md b/README.md index 2351b09..6395286 100755 --- a/README.md +++ b/README.md @@ -18,6 +18,9 @@ > It utilizes google drive api v3 and google OAuth2.0 to generate access tokens and to authorize application for uploading files/folders to your google drive. - Minimal +- Two modes of authentication + - Oauth credentials + - Service account credentials - Upload or Update files/folders - Recursive folder uploading - Sync your folders @@ -50,6 +53,7 @@ - [Updation](#updation) - [Usage](#usage) - [Generating Oauth Credentials](#generating-oauth-credentials) + - [Generating service account credentials](#generating-service-account-credentials) - [Enable Drive API](#enable-drive-api) - [First Run](#first-run) - [Config file](#config) @@ -116,6 +120,9 @@ This repo contains two types of scripts, posix compatible and bash compatible. | mktemp | To generate temporary files ( optional ) | | sleep | Self explanatory | | ps | To manage different processes | +| openssl | For service account usage ( optional ) | + +Note: If openssl if not installed, then `-sa | --service-account` flag won't work, but script will install successfully. If BASH is not available or BASH is available but version is less tham 4.x, then below programs are also required: @@ -296,10 +303,14 @@ There are two methods: ## Usage -First, we need to obtain our oauth credentials, here's how to do it: +First, we have to authenticate. + +There are two ways to authenticate, oauth credentials or service accounts. Use any one. ### Generating Oauth Credentials +To obtain oauth credentials, follow below steps: + - Follow [Enable Drive API](#enable-drive-api) section. - Open [google console](https://console.developers.google.com/). - Click on "Credentials". @@ -311,6 +322,24 @@ First, we need to obtain our oauth credentials, here's how to do it: Now, we have obtained our credentials, move to the [First run](#first-run) section to use those credentials: +### Generating service account credentials + +To obtain service account credentials, follow below steps: + +- Follow [Enable Drive API](#enable-drive-api) section. +- Open [google console](https://console.developers.google.com/). +- Click on "Credentials". +- Click "Create credentials" and select "Service account". +- Provide name for service account and click on create. If successful, it should be on step 2. +- Now tap on role and select owner. Click on continue. If successful, it should be on step 3. +- Click on done. +- Now click on manage service accounts. +- Click on the service account name you created. +- Click on add key, then tap on create new key. Choose json and tap on create. +- If successful, a file should download in the .json format. + +Now, we have obtained our service account json, move to the [First run](#first-run) section to use those credentials: + ### Enable Drive API - Log into google developer console at [google console](https://console.developers.google.com/). @@ -331,28 +360,47 @@ By this, a side bar is opened. At there, select "API & Services" -> "Library". A [Go back to oauth credentials setup](#generating-oauth-credentials) +[Go back to service account generation](#generating-service-account-credentials) + ### First Run +On first run, there are two possibilities, either using oauth credentials or service account credentials. + +#### For Oauth + On first run, the script asks for all the required credentials, which we have obtained in the previous section. Execute the script: `gupload filename` Now, it will ask for following credentials: -**Client ID:** Copy and paste from credentials.json +- **Client ID:** Copy and paste from credentials.json -**Client Secret:** Copy and paste from credentials.json +- **Client Secret:** Copy and paste from credentials.json -**Refresh Token:** If you have previously generated a refresh token authenticated to your account, then enter it, otherwise leave blank. -If you don't have refresh token, script outputs a URL on the terminal script, open that url in a web browser and tap on allow. Copy the code and paste in the terminal. +- **Refresh Token:** If you have previously generated a refresh token authenticated to your account, then enter it, otherwise leave blank. -**Root Folder:** Gdrive folder url/id from your account which you want to set as root folder. You can leave it blank and it takes `root` folder as default. + If you don't have refresh token, script outputs a URL on the terminal script, open that url in a web browser and tap on allow. Copy the code and paste in the terminal. + +- **Root Folder:** Gdrive folder url/id from your account which you want to set as root folder. You can leave it blank and it takes `root` folder as default. If everything went fine, all the required credentials have been set, read the next section on how to upload a file/folder. +#### For service accounts + +For using service account, use `-sa | --service-account` flag. + +Execute the script: `gupload filename -sa "service account json file path"` + +Note: For service accounts it is necessary to use the `-sa | --service-account` flag everytime. + +For more info, see `-sa | --service-account` flag in [Upload Script Custom Flags](#upload-script-custom-flags). + ### Config -After first run, the credentials are saved in config file. By default, the config file is `${HOME}/.googledrive.conf`. +After first run, if oauth credentials are used, then credentials are saved in config file, otherwise for service accounts it is necessary to use the `-sa | --service-account` flag everytime. + +By default, the config file is `${HOME}/.googledrive.conf`. To change the default config file or use a different one temporarily, see `-z / --config` custom in [Upload Script Custom Flags](#upload-script-custom-flags). @@ -410,6 +458,14 @@ Apart from basic usage, this script provides many flags for custom usecases, lik These are the custom flags that are currently implemented: +- --sa | --service-accounts 'service account json file path' + + Use a service account. Should be in proper json format. + + To generate service accounts, see [service account generation](#generating-service-account-credentials) section. + + --- + - -z | --config Override default config file with custom config file. diff --git a/bash/drive-utils.bash b/bash/drive-utils.bash index 4e4a44a..e1c6fd3 100755 --- a/bash/drive-utils.bash +++ b/bash/drive-utils.bash @@ -186,11 +186,22 @@ _extract_id() { # Result: Update access_token and expiry else print error ################################################### _get_access_token_and_update() { - RESPONSE="${1:-$(curl --compressed -s -X POST --data "client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&refresh_token=${REFRESH_TOKEN}&grant_type=refresh_token" "${TOKEN_URL}")}" || : + case "${1:?Error: sa or normal}" in + normal) + RESPONSE="${2:-$(curl --compressed -s -X POST --data "client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&refresh_token=${REFRESH_TOKEN}&grant_type=refresh_token" "${TOKEN_URL}")}" || : + ;; + sa) + declare assertion_data="${2:?2lError: Missing assertion data.}" + RESPONSE="$(curl --compressed -s --data "grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=${assertion_data}" "${TOKEN_URL}")" || : + # sa token jsons are not pretty printed + RESPONSE="${RESPONSE//,\"/$'\n'\"}" + ;; + esac + if ACCESS_TOKEN="$(_json_value access_token 1 1 <<< "${RESPONSE}")"; then ACCESS_TOKEN_EXPIRY="$(($(printf "%(%s)T\\n" "-1") + $(_json_value expires_in 1 1 <<< "${RESPONSE}") - 1))" - _update_config ACCESS_TOKEN "${ACCESS_TOKEN}" "${CONFIG}" - _update_config ACCESS_TOKEN_EXPIRY "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" + _update_config "${SERVICE_ACCOUNT:+${SERVICE_ACCOUNT}_}ACCESS_TOKEN" "${ACCESS_TOKEN}" "${CONFIG}" + _update_config "${SERVICE_ACCOUNT:+${SERVICE_ACCOUNT}_}ACCESS_TOKEN_EXPIRY" "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" else "${QUIET:-_print_center}" "justify" "Error: Something went wrong" ", printing error." "=" 1>&2 printf "%s\n" "${RESPONSE}" 1>&2 diff --git a/bash/google-oauth2.bash b/bash/google-oauth2.bash index bd0e7c7..1fa34a7 100755 --- a/bash/google-oauth2.bash +++ b/bash/google-oauth2.bash @@ -113,7 +113,7 @@ if [[ ${1} = create ]]; then _clear_line 1 1>&2 REFRESH_TOKEN="$(_json_value refresh_token 1 1 <<< "${RESPONSE}" || :)" - if _get_access_token_and_update "${RESPONSE}"; then + if _get_access_token_and_update normal "${RESPONSE}"; then _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}" printf "Access Token: %s\n" "${ACCESS_TOKEN}" printf "Refresh Token: %s\n" "${REFRESH_TOKEN}" @@ -123,7 +123,7 @@ if [[ ${1} = create ]]; then elif [[ ${1} = refresh ]]; then if [[ -n ${REFRESH_TOKEN} ]]; then _print_center "justify" "Required credentials set." "=" - { _get_access_token_and_update && _clear_line 1; } || return 1 + { _get_access_token_and_update normal && _clear_line 1; } || return 1 printf "Access Token: %s\n" "${ACCESS_TOKEN}" else "${QUIET:-_print_center}" "normal" "Refresh Token not set" ", use ${0##*/} create to generate one." "=" diff --git a/bash/release/gupload b/bash/release/gupload index cf17fc5..d10002c 100755 --- a/bash/release/gupload +++ b/bash/release/gupload @@ -539,11 +539,22 @@ _extract_id() { # Result: Update access_token and expiry else print error ################################################### _get_access_token_and_update() { - RESPONSE="${1:-$(curl --compressed -s -X POST --data "client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&refresh_token=${REFRESH_TOKEN}&grant_type=refresh_token" "${TOKEN_URL}")}" || : + case "${1:?Error: sa or normal}" in + normal) + RESPONSE="${2:-$(curl --compressed -s -X POST --data "client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&refresh_token=${REFRESH_TOKEN}&grant_type=refresh_token" "${TOKEN_URL}")}" || : + ;; + sa) + declare assertion_data="${2:?2lError: Missing assertion data.}" + RESPONSE="$(curl --compressed -s --data "grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=${assertion_data}" "${TOKEN_URL}")" || : + # sa token jsons are not pretty printed + RESPONSE="${RESPONSE//,\"/$'\n'\"}" + ;; + esac + if ACCESS_TOKEN="$(_json_value access_token 1 1 <<< "${RESPONSE}")"; then ACCESS_TOKEN_EXPIRY="$(($(printf "%(%s)T\\n" "-1") + $(_json_value expires_in 1 1 <<< "${RESPONSE}") - 1))" - _update_config ACCESS_TOKEN "${ACCESS_TOKEN}" "${CONFIG}" - _update_config ACCESS_TOKEN_EXPIRY "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" + _update_config "${SERVICE_ACCOUNT:+${SERVICE_ACCOUNT}_}ACCESS_TOKEN" "${ACCESS_TOKEN}" "${CONFIG}" + _update_config "${SERVICE_ACCOUNT:+${SERVICE_ACCOUNT}_}ACCESS_TOKEN_EXPIRY" "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" else "${QUIET:-_print_center}" "justify" "Error: Something went wrong" ", printing error." "=" 1>&2 printf "%s\n" "${RESPONSE}" 1>&2 @@ -825,6 +836,49 @@ _error_logging_upload() { return 1 } +################################################### +# Generate rs256 jwt just with cli commands and shell +# Specifically for gdrive service accounts usage +# Globals: 1 +# SCOPE ( optional ) +# Arguments: 2 +# ${1} = service account json file +# ${2} = SCOPE for gdrive +# Result: print jwt +# Refrences: +# https://stackoverflow.com/questions/46657001/how-do-you-create-an-rs256-jwt-assertion-with-bash-shell-scripting +# Inspired by implementation by Will Haley at: +# http://willhaley.com/blog/generate-jwt-with-bash/ +################################################### +_generate_jwt() { + declare json_file="${1:?Error: Give service json file name}" \ + scope="${2:-${SCOPE:?Error: Missing scope}}" \ + aud="https://oauth2.googleapis.com/token" \ + header='{"alg":"RS256","typ":"JWT"}' \ + algo="256" payload_data iss exp iat rsa_secret signed_content sign + + if iss="$(_json_value client_email 1 1 < "${json_file}")" && + rsa_secret="$(_json_value private_key 1 1 < "${json_file}")"; then + rsa_secret="$(printf "%b\n" "${rsa_secret}")" + else + "${QUIET:-_print_center}" 'normal' "Error: Invalid service account file." "=" && return 1 + fi + + iat="$(printf "%(%s)T\\n" "-1")" exp="$((iat + 3400))" + + b64enc() { : "$(openssl enc -base64 -A)" && : "${_//+/-}" && : "${_//\//_}" && printf "%s\n" "${_//=/}"; } + + payload_data='{"iss":"'${iss}'","scope":"'${scope}'","aud":"'${aud}'","exp":'${exp}',"iat":'${iat}'}' + + { + signed_content="$(b64enc <<< "${header}").$(b64enc <<< "${payload_data}")" + sign="$(printf %s "${signed_content}" | openssl dgst -binary -sha"${algo}" -sign <(printf '%s\n' "${rsa_secret}") | b64enc)" + } || return 1 + + printf '%s.%s\n' "${signed_content}" "${sign}" + return 0 +} + ################################################### # A small function to get rootdir id for files in sub folder uploads # Globals: 1 variable, 1 function @@ -942,6 +996,7 @@ Usage:\n ${0##*/} [options.. ] \n Foldername argument is optional. If not provided, the file will be uploaded to preconfigured google drive.\n File name argument is optional if create directory option is used.\n Options:\n + -sa | --service-accounts 'service account json file path' - Use a bot service account. Should be in proper json format.\n -c | -C | --create-dir - option to create directory. Will provide folder id. Can be used to provide input folder, see README.\n -r | --root-dir or - google folder ID/URL to which the file/directory is going to upload. If you want to change the default value, then use this format, -r/--root-dir default=root_folder_id/root_folder_url\n @@ -1044,14 +1099,15 @@ _setup_arguments() { # De-initialize if any variables set already. unset FIRST_INPUT FOLDER_INPUT FOLDERNAME LOCAL_INPUT_ARRAY ID_INPUT_ARRAY unset PARALLEL NO_OF_PARALLEL_JOBS SHARE SHARE_EMAIL OVERWRITE SKIP_DUPLICATES SKIP_SUBDIRS ROOTDIR QUIET - unset VERBOSE VERBOSE_PROGRESS DEBUG LOG_FILE_ID CURL_SPEED RETRY + unset VERBOSE VERBOSE_PROGRESS DEBUG LOG_FILE_ID CURL_SPEED RETRY OPENSSL_ERROR CURL_PROGRESS="-s" EXTRA_LOG=":" CURL_PROGRESS_EXTRA="-s" INFO_PATH="${HOME}/.google-drive-upload" CONFIG_INFO="${INFO_PATH}/google-drive-upload.configpath" [[ -f ${CONFIG_INFO} ]] && . "${CONFIG_INFO}" CONFIG="${CONFIG:-${HOME}/.googledrive.conf}" # Configuration variables # Remote gDrive variables - unset ROOT_FOLDER CLIENT_ID CLIENT_SECRET REFRESH_TOKEN ACCESS_TOKEN + unset ROOT_FOLDER CLIENT_ID CLIENT_SECRET REFRESH_TOKEN ACCESS_TOKEN \ + SERVICE_ACCOUNT SERVICE_ACCOUNT_ACCESS_TOKEN SERVICE_ACCOUNT_ACCESS_TOKEN_EXPIRY API_URL="https://www.googleapis.com" API_VERSION="v3" SCOPE="${API_URL}/auth/drive" @@ -1078,6 +1134,11 @@ _setup_arguments() { -h | --help) _usage ;; -D | --debug) DEBUG="true" && export DEBUG ;; --info) _version_info ;; + -sa | --service-account) + _check_longoptions "${1}" "${2}" + SERVICE_ACCOUNT_FILE="${2}" && shift + ! [[ -f ${SERVICE_ACCOUNT_FILE} ]] && printf "%s\n" "Error: Service account json file exist ( ${SERVICE_ACCOUNT_FILE} )." 1>&2 && exit 1 + ;; -c | -C | --create-dir) _check_longoptions "${1}" "${2}" FOLDERNAME="${2}" && shift @@ -1229,107 +1290,134 @@ _check_credentials() { printf "%s\n" "Add in config manually if terminal is not accessible. CLIENT_ID, CLIENT_SECRET and REFRESH_TOKEN is required." && return 1 } - # Following https://developers.google.com/identity/protocols/oauth2#size - CLIENT_ID_REGEX='[0-9]+-[0-9A-Za-z_]{32}\.apps\.googleusercontent\.com' - CLIENT_SECRET_REGEX='[0-9A-Za-z_-]+' - REFRESH_TOKEN_REGEX='[0-9]//[0-9A-Za-z_-]+' # 512 bytes - ACCESS_TOKEN_REGEX='ya29\.[0-9A-Za-z_-]+' # 2048 bytes - AUTHORIZATION_CODE_REGEX='[0-9]/[0-9A-Za-z_-]+' # 256 bytes - - until [[ -n ${CLIENT_ID} && -n ${CLIENT_ID_VALID} ]]; do - [[ -n ${CLIENT_ID} ]] && { - if [[ ${CLIENT_ID} =~ ${CLIENT_ID_REGEX} ]]; then - [[ -n ${client_id} ]] && _update_config CLIENT_ID "${CLIENT_ID}" "${CONFIG}" - CLIENT_ID_VALID="true" && continue - else - { [[ -n ${client_id} ]] && message="- Try again"; } || message="in config ( ${CONFIG} )" - "${QUIET:-_print_center}" "normal" " Invalid Client ID ${message} " "-" && unset CLIENT_ID client_id - fi + ACCESS_TOKEN_REGEX='ya29\.[0-9A-Za-z_-]+' # 2048 bytes + + if [[ -z ${SERVICE_ACCOUNT_FILE} ]]; then + # Following https://developers.google.com/identity/protocols/oauth2#size + CLIENT_ID_REGEX='[0-9]+-[0-9A-Za-z_]{32}\.apps\.googleusercontent\.com' + CLIENT_SECRET_REGEX='[0-9A-Za-z_-]+' + REFRESH_TOKEN_REGEX='[0-9]//[0-9A-Za-z_-]+' # 512 bytes + AUTHORIZATION_CODE_REGEX='[0-9]/[0-9A-Za-z_-]+' # 256 bytes + + until [[ -n ${CLIENT_ID} && -n ${CLIENT_ID_VALID} ]]; do + [[ -n ${CLIENT_ID} ]] && { + if [[ ${CLIENT_ID} =~ ${CLIENT_ID_REGEX} ]]; then + [[ -n ${client_id} ]] && _update_config CLIENT_ID "${CLIENT_ID}" "${CONFIG}" + CLIENT_ID_VALID="true" && continue + else + { [[ -n ${client_id} ]] && message="- Try again"; } || message="in config ( ${CONFIG} )" + "${QUIET:-_print_center}" "normal" " Invalid Client ID ${message} " "-" && unset CLIENT_ID client_id + fi + } + [[ -z ${client_id} ]] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter Client ID " "-" + [[ -n ${client_id} ]] && _clear_line 1 + printf -- "-> " + read -r CLIENT_ID && client_id=1 + done + + until [[ -n ${CLIENT_SECRET} && -n ${CLIENT_SECRET_VALID} ]]; do + [[ -n ${CLIENT_SECRET} ]] && { + if [[ ${CLIENT_SECRET} =~ ${CLIENT_SECRET_REGEX} ]]; then + [[ -n ${client_secret} ]] && _update_config CLIENT_SECRET "${CLIENT_SECRET}" "${CONFIG}" + CLIENT_SECRET_VALID="true" && continue + else + { [[ -n ${client_secret} ]] && message="- Try again"; } || message="in config ( ${CONFIG} )" + "${QUIET:-_print_center}" "normal" " Invalid Client Secret ${message} " "-" && unset CLIENT_SECRET client_secret + fi + } + [[ -z ${client_secret} ]] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter Client Secret " "-" + [[ -n ${client_secret} ]] && _clear_line 1 + printf -- "-> " + read -r CLIENT_SECRET && client_secret=1 + done + + [[ -n ${REFRESH_TOKEN} ]] && { + ! [[ ${REFRESH_TOKEN} =~ ${REFRESH_TOKEN_REGEX} ]] && + "${QUIET:-_print_center}" "normal" " Error: Invalid Refresh token in config file, follow below steps.. " "-" && unset REFRESH_TOKEN } - [[ -z ${client_id} ]] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter Client ID " "-" - [[ -n ${client_id} ]] && _clear_line 1 - printf -- "-> " - read -r CLIENT_ID && client_id=1 - done - until [[ -n ${CLIENT_SECRET} && -n ${CLIENT_SECRET_VALID} ]]; do - [[ -n ${CLIENT_SECRET} ]] && { - if [[ ${CLIENT_SECRET} =~ ${CLIENT_SECRET_REGEX} ]]; then - [[ -n ${client_secret} ]] && _update_config CLIENT_SECRET "${CLIENT_SECRET}" "${CONFIG}" - CLIENT_SECRET_VALID="true" && continue + [[ -z ${REFRESH_TOKEN} ]] && { + printf "\n" && "${QUIET:-_print_center}" "normal" "If you have a refresh token generated, then type the token, else leave blank and press return key.." " " + printf "\n" && "${QUIET:-_print_center}" "normal" " Refresh Token " "-" && printf -- "-> " + read -r REFRESH_TOKEN + if [[ -n ${REFRESH_TOKEN} ]]; then + "${QUIET:-_print_center}" "normal" " Checking refresh token.. " "-" + if [[ ${REFRESH_TOKEN} =~ ${REFRESH_TOKEN_REGEX} ]]; then + { _get_access_token_and_update normal && _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}"; } || check_error=true + else + check_error=true + fi + [[ -n ${check_error} ]] && "${QUIET:-_print_center}" "normal" " Error: Invalid Refresh token given, follow below steps to generate.. " "-" && unset REFRESH_TOKEN else - { [[ -n ${client_secret} ]] && message="- Try again"; } || message="in config ( ${CONFIG} )" - "${QUIET:-_print_center}" "normal" " Invalid Client Secret ${message} " "-" && unset CLIENT_SECRET client_secret + "${QUIET:-_print_center}" "normal" " No Refresh token given, follow below steps to generate.. " "-" fi + + [[ -z ${REFRESH_TOKEN} ]] && { + printf "\n" && "${QUIET:-_print_center}" "normal" "Visit the below URL, tap on allow and then enter the code obtained" " " + URL="https://accounts.google.com/o/oauth2/auth?client_id=${CLIENT_ID}&redirect_uri=${REDIRECT_URI}&scope=${SCOPE}&response_type=code&prompt=consent" + printf "\n%s\n" "${URL}" + until [[ -n ${AUTHORIZATION_CODE} && -n ${AUTHORIZATION_CODE_VALID} ]]; do + [[ -n ${AUTHORIZATION_CODE} ]] && { + if [[ ${AUTHORIZATION_CODE} =~ ${AUTHORIZATION_CODE_REGEX} ]]; then + AUTHORIZATION_CODE_VALID="true" && continue + else + "${QUIET:-_print_center}" "normal" " Invalid CODE given, try again.. " "-" && unset AUTHORIZATION_CODE authorization_code + fi + } + { [[ -z ${authorization_code} ]] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter the authorization code " "-"; } || _clear_line 1 + printf -- "-> " + read -r AUTHORIZATION_CODE && authorization_code=1 + done + RESPONSE="$(curl --compressed "${CURL_PROGRESS}" -X POST \ + --data "code=${AUTHORIZATION_CODE}&client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&redirect_uri=${REDIRECT_URI}&grant_type=authorization_code" "${TOKEN_URL}")" || : + _clear_line 1 1>&2 + + REFRESH_TOKEN="$(_json_value refresh_token 1 1 <<< "${RESPONSE}" || :)" + { _get_access_token_and_update normal "${RESPONSE}" && _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}"; } || return 1 + } + printf "\n" } - [[ -z ${client_secret} ]] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter Client Secret " "-" - [[ -n ${client_secret} ]] && _clear_line 1 - printf -- "-> " - read -r CLIENT_SECRET && client_secret=1 - done + [[ -z ${ACCESS_TOKEN} || ${ACCESS_TOKEN_EXPIRY:-0} -lt "$(printf "%(%s)T\\n" "-1")" ]] || ! [[ ${ACCESS_TOKEN} =~ ${ACCESS_TOKEN_REGEX} ]] && + { _get_access_token_and_update normal || return 1; } + printf "%b\n" "ACCESS_TOKEN=\"${ACCESS_TOKEN}\"\nACCESS_TOKEN_EXPIRY=\"${ACCESS_TOKEN_EXPIRY}\"" >| "${TMPFILE}_ACCESS_TOKEN" + else + command -v openssl 2>| /dev/null 1>&2 || + { "${QUIET:-_print_center}" 'normal' "Error: openssl not installed, install openssl to use '-sa | --service-account' flag." "=" 1>&2 && return 1; } - [[ -n ${REFRESH_TOKEN} ]] && { - ! [[ ${REFRESH_TOKEN} =~ ${REFRESH_TOKEN_REGEX} ]] && - "${QUIET:-_print_center}" "normal" " Error: Invalid Refresh token in config file, follow below steps.. " "-" && unset REFRESH_TOKEN - } + SERVICE_ACCOUNT="SA_$(_json_value private_key_id 1 1 < "${SERVICE_ACCOUNT_FILE}")_SA" || + { "${QUIET:-_print_center}" 'normal' "Error: Invalid service account file." "=" 1>&2 && return 1; } - [[ -z ${REFRESH_TOKEN} ]] && { - printf "\n" && "${QUIET:-_print_center}" "normal" "If you have a refresh token generated, then type the token, else leave blank and press return key.." " " - printf "\n" && "${QUIET:-_print_center}" "normal" " Refresh Token " "-" && printf -- "-> " - read -r REFRESH_TOKEN - if [[ -n ${REFRESH_TOKEN} ]]; then - "${QUIET:-_print_center}" "normal" " Checking refresh token.. " "-" - if [[ ${REFRESH_TOKEN} =~ ${REFRESH_TOKEN_REGEX} ]]; then - { _get_access_token_and_update && _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}"; } || check_error=true - else - check_error=true - fi - [[ -n ${check_error} ]] && "${QUIET:-_print_center}" "normal" " Error: Invalid Refresh token given, follow below steps to generate.. " "-" && unset REFRESH_TOKEN - else - "${QUIET:-_print_center}" "normal" " No Refresh token given, follow below steps to generate.. " "-" - fi + SA_ACCESS_TOKEN_NAME="${SERVICE_ACCOUNT}_ACCESS_TOKEN" \ + SA_ACCESS_TOKEN_EXPIRY_NAME="${SA_ACCESS_TOKEN_NAME}_EXPIRY" - [[ -z ${REFRESH_TOKEN} ]] && { - printf "\n" && "${QUIET:-_print_center}" "normal" "Visit the below URL, tap on allow and then enter the code obtained" " " - URL="https://accounts.google.com/o/oauth2/auth?client_id=${CLIENT_ID}&redirect_uri=${REDIRECT_URI}&scope=${SCOPE}&response_type=code&prompt=consent" - printf "\n%s\n" "${URL}" - until [[ -n ${AUTHORIZATION_CODE} && -n ${AUTHORIZATION_CODE_VALID} ]]; do - [[ -n ${AUTHORIZATION_CODE} ]] && { - if [[ ${AUTHORIZATION_CODE} =~ ${AUTHORIZATION_CODE_REGEX} ]]; then - AUTHORIZATION_CODE_VALID="true" && continue - else - "${QUIET:-_print_center}" "normal" " Invalid CODE given, try again.. " "-" && unset AUTHORIZATION_CODE authorization_code - fi - } - { [[ -z ${authorization_code} ]] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter the authorization code " "-"; } || _clear_line 1 - printf -- "-> " - read -r AUTHORIZATION_CODE && authorization_code=1 - done - RESPONSE="$(curl --compressed "${CURL_PROGRESS}" -X POST \ - --data "code=${AUTHORIZATION_CODE}&client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&redirect_uri=${REDIRECT_URI}&grant_type=authorization_code" "${TOKEN_URL}")" || : - _clear_line 1 1>&2 + SERVICE_ACCOUNT_ACCESS_TOKEN="${!SA_ACCESS_TOKEN_NAME}" + SERVICE_ACCOUNT_ACCESS_TOKEN_EXPIRY="${!SA_ACCESS_TOKEN_EXPIRY_NAME}" - REFRESH_TOKEN="$(_json_value refresh_token 1 1 <<< "${RESPONSE}" || :)" - { _get_access_token_and_update "${RESPONSE}" && _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}"; } || return 1 + [[ -z ${SERVICE_ACCOUNT_ACCESS_TOKEN} || ${SERVICE_ACCOUNT_ACCESS_TOKEN_EXPIRY:-0} -lt "$(printf "%(%s)T\\n" "-1")" ]] || ! [[ ${SERVICE_ACCOUNT_ACCESS_TOKEN} =~ ${ACCESS_TOKEN_REGEX} ]] && { + ASSERTION_DATA="$(_generate_jwt "${SERVICE_ACCOUNT_FILE}" "${SCOPE}")" || { printf "%s\n" "${ASSERTION_DATA}" 1>&2 && return 1; } + _get_access_token_and_update sa "${ASSERTION_DATA}" || return 1 } - printf "\n" - } - [[ -z ${ACCESS_TOKEN} || ${ACCESS_TOKEN_EXPIRY:-0} -lt "$(printf "%(%s)T\\n" "-1")" ]] || ! [[ ${ACCESS_TOKEN} =~ ${ACCESS_TOKEN_REGEX} ]] && - { _get_access_token_and_update || return 1; } - printf "%b\n" "ACCESS_TOKEN=\"${ACCESS_TOKEN}\"\nACCESS_TOKEN_EXPIRY=\"${ACCESS_TOKEN_EXPIRY}\"" >| "${TMPFILE}_ACCESS_TOKEN" + printf "%s\n%s\n" "ACCESS_TOKEN=\"${!SA_ACCESS_TOKEN_NAME}\"" \ + "ACCESS_TOKEN_EXPIRY=\"${!SA_ACCESS_TOKEN_EXPIRY_NAME}\"" >| "${TMPFILE}_ACCESS_TOKEN" + fi # launch a background service to check access token and update it # checks ACCESS_TOKEN_EXPIRY, try to update before 5 mins of expiry, a fresh token gets 60 mins # process will be killed when script exits or "${MAIN_PID}" is killed { until ! kill -0 "${MAIN_PID}" 2>| /dev/null 1>&2; do + unset ASSERTION_DATA MODE . "${TMPFILE}_ACCESS_TOKEN" CURRENT_TIME="$(printf "%(%s)T\\n" "-1")" REMAINING_TOKEN_TIME="$((ACCESS_TOKEN_EXPIRY - CURRENT_TIME))" if [[ ${REMAINING_TOKEN_TIME} -le 300 ]]; then + [[ -n ${SERVICE_ACCOUNT_FILE} ]] && { + ASSERTION_DATA="$(_generate_jwt "${SERVICE_ACCOUNT_FILE}" "${SCOPE}")" || : + MODE="sa" + } # timeout after 30 seconds, it shouldn't take too long anyway, and update tmp config - CONFIG="${TMPFILE}_ACCESS_TOKEN" _timeout 30 _get_access_token_and_update || : + SERVICE_ACCOUNT="" CONFIG="${TMPFILE}_ACCESS_TOKEN" _timeout 30 _get_access_token_and_update "${MODE:-normal}" "${ASSERTION_DATA:-}" || : else TOKEN_PROCESS_TIME_TO_SLEEP="$(if [[ ${REMAINING_TOKEN_TIME} -le 301 ]]; then printf "0\n" @@ -1340,7 +1428,7 @@ _check_credentials() { fi sleep 1 done - } & + } 2>| /dev/null 1>&2 & ACCESS_TOKEN_SERVICE_PID="${!}" return 0 @@ -1439,7 +1527,7 @@ _process_arguments() { export -f _bytes_to_human _dirname _json_value _url_encode _support_ansi_escapes _newline _print_center_quiet _print_center _clear_line \ _api_request _get_access_token_and_update _check_existing_file _upload_file _upload_file_main _clone_file _collect_file_info _generate_upload_link _upload_file_from_uri _full_upload \ - _normal_logging_upload _error_logging_upload _log_upload_session _remove_upload_session _upload_folder _share_id _get_rootdir_id + _normal_logging_upload _error_logging_upload _log_upload_session _remove_upload_session _upload_folder _share_id _get_rootdir_id _generate_jwt # on successful uploads _share_and_print_link() { @@ -1613,8 +1701,8 @@ main() { # update the config with latest ACCESS_TOKEN and ACCESS_TOKEN_EXPIRY only if changed . "${TMPFILE}_ACCESS_TOKEN" [[ ${INITIAL_ACCESS_TOKEN} = "${ACCESS_TOKEN}" ]] || { - _update_config ACCESS_TOKEN "${ACCESS_TOKEN}" "${CONFIG}" - _update_config ACCESS_TOKEN_EXPIRY "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" + _update_config "${SERVICE_ACCOUNT:+${SERVICE_ACCOUNT}_}ACCESS_TOKEN" "${ACCESS_TOKEN}" "${CONFIG}" + _update_config "${SERVICE_ACCOUNT:+${SERVICE_ACCOUNT}_}ACCESS_TOKEN_EXPIRY" "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" } } 1>| /dev/null diff --git a/bash/upload-utils.bash b/bash/upload-utils.bash index fb56906..a835b4b 100755 --- a/bash/upload-utils.bash +++ b/bash/upload-utils.bash @@ -54,6 +54,49 @@ _error_logging_upload() { return 1 } +################################################### +# Generate rs256 jwt just with cli commands and shell +# Specifically for gdrive service accounts usage +# Globals: 1 +# SCOPE ( optional ) +# Arguments: 2 +# ${1} = service account json file +# ${2} = SCOPE for gdrive +# Result: print jwt +# Refrences: +# https://stackoverflow.com/questions/46657001/how-do-you-create-an-rs256-jwt-assertion-with-bash-shell-scripting +# Inspired by implementation by Will Haley at: +# http://willhaley.com/blog/generate-jwt-with-bash/ +################################################### +_generate_jwt() { + declare json_file="${1:?Error: Give service json file name}" \ + scope="${2:-${SCOPE:?Error: Missing scope}}" \ + aud="https://oauth2.googleapis.com/token" \ + header='{"alg":"RS256","typ":"JWT"}' \ + algo="256" payload_data iss exp iat rsa_secret signed_content sign + + if iss="$(_json_value client_email 1 1 < "${json_file}")" && + rsa_secret="$(_json_value private_key 1 1 < "${json_file}")"; then + rsa_secret="$(printf "%b\n" "${rsa_secret}")" + else + "${QUIET:-_print_center}" 'normal' "Error: Invalid service account file." "=" && return 1 + fi + + iat="$(printf "%(%s)T\\n" "-1")" exp="$((iat + 3400))" + + b64enc() { : "$(openssl enc -base64 -A)" && : "${_//+/-}" && : "${_//\//_}" && printf "%s\n" "${_//=/}"; } + + payload_data='{"iss":"'${iss}'","scope":"'${scope}'","aud":"'${aud}'","exp":'${exp}',"iat":'${iat}'}' + + { + signed_content="$(b64enc <<< "${header}").$(b64enc <<< "${payload_data}")" + sign="$(printf %s "${signed_content}" | openssl dgst -binary -sha"${algo}" -sign <(printf '%s\n' "${rsa_secret}") | b64enc)" + } || return 1 + + printf '%s.%s\n' "${signed_content}" "${sign}" + return 0 +} + ################################################### # A small function to get rootdir id for files in sub folder uploads # Globals: 1 variable, 1 function diff --git a/bash/upload.bash b/bash/upload.bash index bc6d589..c3b3e1b 100755 --- a/bash/upload.bash +++ b/bash/upload.bash @@ -9,6 +9,7 @@ Usage:\n ${0##*/} [options.. ] \n Foldername argument is optional. If not provided, the file will be uploaded to preconfigured google drive.\n File name argument is optional if create directory option is used.\n Options:\n + -sa | --service-accounts 'service account json file path' - Use a bot service account. Should be in proper json format.\n -c | -C | --create-dir - option to create directory. Will provide folder id. Can be used to provide input folder, see README.\n -r | --root-dir or - google folder ID/URL to which the file/directory is going to upload. If you want to change the default value, then use this format, -r/--root-dir default=root_folder_id/root_folder_url\n @@ -111,14 +112,15 @@ _setup_arguments() { # De-initialize if any variables set already. unset FIRST_INPUT FOLDER_INPUT FOLDERNAME LOCAL_INPUT_ARRAY ID_INPUT_ARRAY unset PARALLEL NO_OF_PARALLEL_JOBS SHARE SHARE_EMAIL OVERWRITE SKIP_DUPLICATES SKIP_SUBDIRS ROOTDIR QUIET - unset VERBOSE VERBOSE_PROGRESS DEBUG LOG_FILE_ID CURL_SPEED RETRY + unset VERBOSE VERBOSE_PROGRESS DEBUG LOG_FILE_ID CURL_SPEED RETRY OPENSSL_ERROR CURL_PROGRESS="-s" EXTRA_LOG=":" CURL_PROGRESS_EXTRA="-s" INFO_PATH="${HOME}/.google-drive-upload" CONFIG_INFO="${INFO_PATH}/google-drive-upload.configpath" [[ -f ${CONFIG_INFO} ]] && . "${CONFIG_INFO}" CONFIG="${CONFIG:-${HOME}/.googledrive.conf}" # Configuration variables # Remote gDrive variables - unset ROOT_FOLDER CLIENT_ID CLIENT_SECRET REFRESH_TOKEN ACCESS_TOKEN + unset ROOT_FOLDER CLIENT_ID CLIENT_SECRET REFRESH_TOKEN ACCESS_TOKEN \ + SERVICE_ACCOUNT SERVICE_ACCOUNT_ACCESS_TOKEN SERVICE_ACCOUNT_ACCESS_TOKEN_EXPIRY API_URL="https://www.googleapis.com" API_VERSION="v3" SCOPE="${API_URL}/auth/drive" @@ -145,6 +147,11 @@ _setup_arguments() { -h | --help) _usage ;; -D | --debug) DEBUG="true" && export DEBUG ;; --info) _version_info ;; + -sa | --service-account) + _check_longoptions "${1}" "${2}" + SERVICE_ACCOUNT_FILE="${2}" && shift + ! [[ -f ${SERVICE_ACCOUNT_FILE} ]] && printf "%s\n" "Error: Service account json file exist ( ${SERVICE_ACCOUNT_FILE} )." 1>&2 && exit 1 + ;; -c | -C | --create-dir) _check_longoptions "${1}" "${2}" FOLDERNAME="${2}" && shift @@ -296,107 +303,134 @@ _check_credentials() { printf "%s\n" "Add in config manually if terminal is not accessible. CLIENT_ID, CLIENT_SECRET and REFRESH_TOKEN is required." && return 1 } - # Following https://developers.google.com/identity/protocols/oauth2#size - CLIENT_ID_REGEX='[0-9]+-[0-9A-Za-z_]{32}\.apps\.googleusercontent\.com' - CLIENT_SECRET_REGEX='[0-9A-Za-z_-]+' - REFRESH_TOKEN_REGEX='[0-9]//[0-9A-Za-z_-]+' # 512 bytes - ACCESS_TOKEN_REGEX='ya29\.[0-9A-Za-z_-]+' # 2048 bytes - AUTHORIZATION_CODE_REGEX='[0-9]/[0-9A-Za-z_-]+' # 256 bytes - - until [[ -n ${CLIENT_ID} && -n ${CLIENT_ID_VALID} ]]; do - [[ -n ${CLIENT_ID} ]] && { - if [[ ${CLIENT_ID} =~ ${CLIENT_ID_REGEX} ]]; then - [[ -n ${client_id} ]] && _update_config CLIENT_ID "${CLIENT_ID}" "${CONFIG}" - CLIENT_ID_VALID="true" && continue - else - { [[ -n ${client_id} ]] && message="- Try again"; } || message="in config ( ${CONFIG} )" - "${QUIET:-_print_center}" "normal" " Invalid Client ID ${message} " "-" && unset CLIENT_ID client_id - fi + ACCESS_TOKEN_REGEX='ya29\.[0-9A-Za-z_-]+' # 2048 bytes + + if [[ -z ${SERVICE_ACCOUNT_FILE} ]]; then + # Following https://developers.google.com/identity/protocols/oauth2#size + CLIENT_ID_REGEX='[0-9]+-[0-9A-Za-z_]{32}\.apps\.googleusercontent\.com' + CLIENT_SECRET_REGEX='[0-9A-Za-z_-]+' + REFRESH_TOKEN_REGEX='[0-9]//[0-9A-Za-z_-]+' # 512 bytes + AUTHORIZATION_CODE_REGEX='[0-9]/[0-9A-Za-z_-]+' # 256 bytes + + until [[ -n ${CLIENT_ID} && -n ${CLIENT_ID_VALID} ]]; do + [[ -n ${CLIENT_ID} ]] && { + if [[ ${CLIENT_ID} =~ ${CLIENT_ID_REGEX} ]]; then + [[ -n ${client_id} ]] && _update_config CLIENT_ID "${CLIENT_ID}" "${CONFIG}" + CLIENT_ID_VALID="true" && continue + else + { [[ -n ${client_id} ]] && message="- Try again"; } || message="in config ( ${CONFIG} )" + "${QUIET:-_print_center}" "normal" " Invalid Client ID ${message} " "-" && unset CLIENT_ID client_id + fi + } + [[ -z ${client_id} ]] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter Client ID " "-" + [[ -n ${client_id} ]] && _clear_line 1 + printf -- "-> " + read -r CLIENT_ID && client_id=1 + done + + until [[ -n ${CLIENT_SECRET} && -n ${CLIENT_SECRET_VALID} ]]; do + [[ -n ${CLIENT_SECRET} ]] && { + if [[ ${CLIENT_SECRET} =~ ${CLIENT_SECRET_REGEX} ]]; then + [[ -n ${client_secret} ]] && _update_config CLIENT_SECRET "${CLIENT_SECRET}" "${CONFIG}" + CLIENT_SECRET_VALID="true" && continue + else + { [[ -n ${client_secret} ]] && message="- Try again"; } || message="in config ( ${CONFIG} )" + "${QUIET:-_print_center}" "normal" " Invalid Client Secret ${message} " "-" && unset CLIENT_SECRET client_secret + fi + } + [[ -z ${client_secret} ]] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter Client Secret " "-" + [[ -n ${client_secret} ]] && _clear_line 1 + printf -- "-> " + read -r CLIENT_SECRET && client_secret=1 + done + + [[ -n ${REFRESH_TOKEN} ]] && { + ! [[ ${REFRESH_TOKEN} =~ ${REFRESH_TOKEN_REGEX} ]] && + "${QUIET:-_print_center}" "normal" " Error: Invalid Refresh token in config file, follow below steps.. " "-" && unset REFRESH_TOKEN } - [[ -z ${client_id} ]] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter Client ID " "-" - [[ -n ${client_id} ]] && _clear_line 1 - printf -- "-> " - read -r CLIENT_ID && client_id=1 - done - until [[ -n ${CLIENT_SECRET} && -n ${CLIENT_SECRET_VALID} ]]; do - [[ -n ${CLIENT_SECRET} ]] && { - if [[ ${CLIENT_SECRET} =~ ${CLIENT_SECRET_REGEX} ]]; then - [[ -n ${client_secret} ]] && _update_config CLIENT_SECRET "${CLIENT_SECRET}" "${CONFIG}" - CLIENT_SECRET_VALID="true" && continue + [[ -z ${REFRESH_TOKEN} ]] && { + printf "\n" && "${QUIET:-_print_center}" "normal" "If you have a refresh token generated, then type the token, else leave blank and press return key.." " " + printf "\n" && "${QUIET:-_print_center}" "normal" " Refresh Token " "-" && printf -- "-> " + read -r REFRESH_TOKEN + if [[ -n ${REFRESH_TOKEN} ]]; then + "${QUIET:-_print_center}" "normal" " Checking refresh token.. " "-" + if [[ ${REFRESH_TOKEN} =~ ${REFRESH_TOKEN_REGEX} ]]; then + { _get_access_token_and_update normal && _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}"; } || check_error=true + else + check_error=true + fi + [[ -n ${check_error} ]] && "${QUIET:-_print_center}" "normal" " Error: Invalid Refresh token given, follow below steps to generate.. " "-" && unset REFRESH_TOKEN else - { [[ -n ${client_secret} ]] && message="- Try again"; } || message="in config ( ${CONFIG} )" - "${QUIET:-_print_center}" "normal" " Invalid Client Secret ${message} " "-" && unset CLIENT_SECRET client_secret + "${QUIET:-_print_center}" "normal" " No Refresh token given, follow below steps to generate.. " "-" fi + + [[ -z ${REFRESH_TOKEN} ]] && { + printf "\n" && "${QUIET:-_print_center}" "normal" "Visit the below URL, tap on allow and then enter the code obtained" " " + URL="https://accounts.google.com/o/oauth2/auth?client_id=${CLIENT_ID}&redirect_uri=${REDIRECT_URI}&scope=${SCOPE}&response_type=code&prompt=consent" + printf "\n%s\n" "${URL}" + until [[ -n ${AUTHORIZATION_CODE} && -n ${AUTHORIZATION_CODE_VALID} ]]; do + [[ -n ${AUTHORIZATION_CODE} ]] && { + if [[ ${AUTHORIZATION_CODE} =~ ${AUTHORIZATION_CODE_REGEX} ]]; then + AUTHORIZATION_CODE_VALID="true" && continue + else + "${QUIET:-_print_center}" "normal" " Invalid CODE given, try again.. " "-" && unset AUTHORIZATION_CODE authorization_code + fi + } + { [[ -z ${authorization_code} ]] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter the authorization code " "-"; } || _clear_line 1 + printf -- "-> " + read -r AUTHORIZATION_CODE && authorization_code=1 + done + RESPONSE="$(curl --compressed "${CURL_PROGRESS}" -X POST \ + --data "code=${AUTHORIZATION_CODE}&client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&redirect_uri=${REDIRECT_URI}&grant_type=authorization_code" "${TOKEN_URL}")" || : + _clear_line 1 1>&2 + + REFRESH_TOKEN="$(_json_value refresh_token 1 1 <<< "${RESPONSE}" || :)" + { _get_access_token_and_update normal "${RESPONSE}" && _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}"; } || return 1 + } + printf "\n" } - [[ -z ${client_secret} ]] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter Client Secret " "-" - [[ -n ${client_secret} ]] && _clear_line 1 - printf -- "-> " - read -r CLIENT_SECRET && client_secret=1 - done + [[ -z ${ACCESS_TOKEN} || ${ACCESS_TOKEN_EXPIRY:-0} -lt "$(printf "%(%s)T\\n" "-1")" ]] || ! [[ ${ACCESS_TOKEN} =~ ${ACCESS_TOKEN_REGEX} ]] && + { _get_access_token_and_update normal || return 1; } + printf "%b\n" "ACCESS_TOKEN=\"${ACCESS_TOKEN}\"\nACCESS_TOKEN_EXPIRY=\"${ACCESS_TOKEN_EXPIRY}\"" >| "${TMPFILE}_ACCESS_TOKEN" + else + command -v openssl 2>| /dev/null 1>&2 || + { "${QUIET:-_print_center}" 'normal' "Error: openssl not installed, install openssl to use '-sa | --service-account' flag." "=" 1>&2 && return 1; } - [[ -n ${REFRESH_TOKEN} ]] && { - ! [[ ${REFRESH_TOKEN} =~ ${REFRESH_TOKEN_REGEX} ]] && - "${QUIET:-_print_center}" "normal" " Error: Invalid Refresh token in config file, follow below steps.. " "-" && unset REFRESH_TOKEN - } + SERVICE_ACCOUNT="SA_$(_json_value private_key_id 1 1 < "${SERVICE_ACCOUNT_FILE}")_SA" || + { "${QUIET:-_print_center}" 'normal' "Error: Invalid service account file." "=" 1>&2 && return 1; } - [[ -z ${REFRESH_TOKEN} ]] && { - printf "\n" && "${QUIET:-_print_center}" "normal" "If you have a refresh token generated, then type the token, else leave blank and press return key.." " " - printf "\n" && "${QUIET:-_print_center}" "normal" " Refresh Token " "-" && printf -- "-> " - read -r REFRESH_TOKEN - if [[ -n ${REFRESH_TOKEN} ]]; then - "${QUIET:-_print_center}" "normal" " Checking refresh token.. " "-" - if [[ ${REFRESH_TOKEN} =~ ${REFRESH_TOKEN_REGEX} ]]; then - { _get_access_token_and_update && _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}"; } || check_error=true - else - check_error=true - fi - [[ -n ${check_error} ]] && "${QUIET:-_print_center}" "normal" " Error: Invalid Refresh token given, follow below steps to generate.. " "-" && unset REFRESH_TOKEN - else - "${QUIET:-_print_center}" "normal" " No Refresh token given, follow below steps to generate.. " "-" - fi + SA_ACCESS_TOKEN_NAME="${SERVICE_ACCOUNT}_ACCESS_TOKEN" \ + SA_ACCESS_TOKEN_EXPIRY_NAME="${SA_ACCESS_TOKEN_NAME}_EXPIRY" - [[ -z ${REFRESH_TOKEN} ]] && { - printf "\n" && "${QUIET:-_print_center}" "normal" "Visit the below URL, tap on allow and then enter the code obtained" " " - URL="https://accounts.google.com/o/oauth2/auth?client_id=${CLIENT_ID}&redirect_uri=${REDIRECT_URI}&scope=${SCOPE}&response_type=code&prompt=consent" - printf "\n%s\n" "${URL}" - until [[ -n ${AUTHORIZATION_CODE} && -n ${AUTHORIZATION_CODE_VALID} ]]; do - [[ -n ${AUTHORIZATION_CODE} ]] && { - if [[ ${AUTHORIZATION_CODE} =~ ${AUTHORIZATION_CODE_REGEX} ]]; then - AUTHORIZATION_CODE_VALID="true" && continue - else - "${QUIET:-_print_center}" "normal" " Invalid CODE given, try again.. " "-" && unset AUTHORIZATION_CODE authorization_code - fi - } - { [[ -z ${authorization_code} ]] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter the authorization code " "-"; } || _clear_line 1 - printf -- "-> " - read -r AUTHORIZATION_CODE && authorization_code=1 - done - RESPONSE="$(curl --compressed "${CURL_PROGRESS}" -X POST \ - --data "code=${AUTHORIZATION_CODE}&client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&redirect_uri=${REDIRECT_URI}&grant_type=authorization_code" "${TOKEN_URL}")" || : - _clear_line 1 1>&2 - - REFRESH_TOKEN="$(_json_value refresh_token 1 1 <<< "${RESPONSE}" || :)" - { _get_access_token_and_update "${RESPONSE}" && _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}"; } || return 1 + SERVICE_ACCOUNT_ACCESS_TOKEN="${!SA_ACCESS_TOKEN_NAME}" + SERVICE_ACCOUNT_ACCESS_TOKEN_EXPIRY="${!SA_ACCESS_TOKEN_EXPIRY_NAME}" + + [[ -z ${SERVICE_ACCOUNT_ACCESS_TOKEN} || ${SERVICE_ACCOUNT_ACCESS_TOKEN_EXPIRY:-0} -lt "$(printf "%(%s)T\\n" "-1")" ]] || ! [[ ${SERVICE_ACCOUNT_ACCESS_TOKEN} =~ ${ACCESS_TOKEN_REGEX} ]] && { + ASSERTION_DATA="$(_generate_jwt "${SERVICE_ACCOUNT_FILE}" "${SCOPE}")" || { printf "%s\n" "${ASSERTION_DATA}" 1>&2 && return 1; } + _get_access_token_and_update sa "${ASSERTION_DATA}" || return 1 } - printf "\n" - } - [[ -z ${ACCESS_TOKEN} || ${ACCESS_TOKEN_EXPIRY:-0} -lt "$(printf "%(%s)T\\n" "-1")" ]] || ! [[ ${ACCESS_TOKEN} =~ ${ACCESS_TOKEN_REGEX} ]] && - { _get_access_token_and_update || return 1; } - printf "%b\n" "ACCESS_TOKEN=\"${ACCESS_TOKEN}\"\nACCESS_TOKEN_EXPIRY=\"${ACCESS_TOKEN_EXPIRY}\"" >| "${TMPFILE}_ACCESS_TOKEN" + printf "%s\n%s\n" "ACCESS_TOKEN=\"${!SA_ACCESS_TOKEN_NAME}\"" \ + "ACCESS_TOKEN_EXPIRY=\"${!SA_ACCESS_TOKEN_EXPIRY_NAME}\"" >| "${TMPFILE}_ACCESS_TOKEN" + fi # launch a background service to check access token and update it # checks ACCESS_TOKEN_EXPIRY, try to update before 5 mins of expiry, a fresh token gets 60 mins # process will be killed when script exits or "${MAIN_PID}" is killed { until ! kill -0 "${MAIN_PID}" 2>| /dev/null 1>&2; do + unset ASSERTION_DATA MODE . "${TMPFILE}_ACCESS_TOKEN" CURRENT_TIME="$(printf "%(%s)T\\n" "-1")" REMAINING_TOKEN_TIME="$((ACCESS_TOKEN_EXPIRY - CURRENT_TIME))" if [[ ${REMAINING_TOKEN_TIME} -le 300 ]]; then + [[ -n ${SERVICE_ACCOUNT_FILE} ]] && { + ASSERTION_DATA="$(_generate_jwt "${SERVICE_ACCOUNT_FILE}" "${SCOPE}")" || : + MODE="sa" + } # timeout after 30 seconds, it shouldn't take too long anyway, and update tmp config - CONFIG="${TMPFILE}_ACCESS_TOKEN" _timeout 30 _get_access_token_and_update || : + SERVICE_ACCOUNT="" CONFIG="${TMPFILE}_ACCESS_TOKEN" _timeout 30 _get_access_token_and_update "${MODE:-normal}" "${ASSERTION_DATA:-}" || : else TOKEN_PROCESS_TIME_TO_SLEEP="$(if [[ ${REMAINING_TOKEN_TIME} -le 301 ]]; then printf "0\n" @@ -407,7 +441,7 @@ _check_credentials() { fi sleep 1 done - } & + } 2>| /dev/null 1>&2 & ACCESS_TOKEN_SERVICE_PID="${!}" return 0 @@ -506,7 +540,7 @@ _process_arguments() { export -f _bytes_to_human _dirname _json_value _url_encode _support_ansi_escapes _newline _print_center_quiet _print_center _clear_line \ _api_request _get_access_token_and_update _check_existing_file _upload_file _upload_file_main _clone_file _collect_file_info _generate_upload_link _upload_file_from_uri _full_upload \ - _normal_logging_upload _error_logging_upload _log_upload_session _remove_upload_session _upload_folder _share_id _get_rootdir_id + _normal_logging_upload _error_logging_upload _log_upload_session _remove_upload_session _upload_folder _share_id _get_rootdir_id _generate_jwt # on successful uploads _share_and_print_link() { @@ -680,8 +714,8 @@ main() { # update the config with latest ACCESS_TOKEN and ACCESS_TOKEN_EXPIRY only if changed . "${TMPFILE}_ACCESS_TOKEN" [[ ${INITIAL_ACCESS_TOKEN} = "${ACCESS_TOKEN}" ]] || { - _update_config ACCESS_TOKEN "${ACCESS_TOKEN}" "${CONFIG}" - _update_config ACCESS_TOKEN_EXPIRY "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" + _update_config "${SERVICE_ACCOUNT:+${SERVICE_ACCOUNT}_}ACCESS_TOKEN" "${ACCESS_TOKEN}" "${CONFIG}" + _update_config "${SERVICE_ACCOUNT:+${SERVICE_ACCOUNT}_}ACCESS_TOKEN_EXPIRY" "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" } } 1>| /dev/null diff --git a/install.sh b/install.sh index 1167675..da7fd35 100755 --- a/install.sh +++ b/install.sh @@ -92,22 +92,28 @@ _check_dependencies() { command -v "${program}" 2>| /dev/null 1>&2 || error_list="${error_list}\n${program}" done - command -v tail 2>| /dev/null 1>&2 || warning_list="${warning_list}\ntail" + command -v tail 2>| /dev/null 1>&2 || sync_error="tail" - [ -n "${warning_list}" ] && { - [ -z "${UNINSTALL}" ] && { + command -v openssl 2>| /dev/null 1>&2 || + openssl_error="openssl not installed. If not installed, service accounts ( '-sa | --service-account' flag ) will not work." + + [ -z "${UNINSTALL}" ] && { + + [ -n "${sync_error}" ] && { printf "Warning: " - printf "%b, " "${error_list}" + printf "%b, " "${sync_error}" printf "%b" "not found, sync script will be not installed/updated.\n" + SKIP_SYNC="true" } - SKIP_SYNC="true" - } - [ -n "${error_list}" ] && [ -z "${UNINSTALL}" ] && { - printf "Error: " - printf "%b, " "${error_list}" - printf "%b" "not found, install before proceeding.\n" - exit 1 + [ -n "${openssl_error}" ] && printf "%s\n" "Warning: ${openssl_error}" + + [ -n "${error_list}" ] && { + printf "Error: " + printf "%b, " "${error_list}" + printf "%b" "not found, install before proceeding.\n" + exit 1 + } } return 0 } diff --git a/sh/drive-utils.sh b/sh/drive-utils.sh index 736c0db..3e98918 100755 --- a/sh/drive-utils.sh +++ b/sh/drive-utils.sh @@ -189,11 +189,24 @@ _extract_id() { # Result: Update access_token and expiry else print error ################################################### _get_access_token_and_update() { - RESPONSE="${1:-$(curl --compressed -s -X POST --data "client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&refresh_token=${REFRESH_TOKEN}&grant_type=refresh_token" "${TOKEN_URL}")}" || : + case "${1:?Error: sa or normal}" in + normal) + RESPONSE="${2:-$(curl --compressed -s -X POST --data "client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&refresh_token=${REFRESH_TOKEN}&grant_type=refresh_token" "${TOKEN_URL}")}" || : + ;; + sa) + assertion_data="${2:?Error: Missing assertion data.}" + RESPONSE="$(curl --compressed -s --data "grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=${assertion_data}" "${TOKEN_URL}")" || : + # sa token jsons are not pretty printed + # shellcheck disable=SC1004 + RESPONSE="$(printf "%s\n" "${RESPONSE}" | sed -e 's/,"/\ +"/g')" + ;; + esac + if ACCESS_TOKEN="$(printf "%s\n" "${RESPONSE}" | _json_value access_token 1 1)"; then ACCESS_TOKEN_EXPIRY="$(($(date +"%s") + $(printf "%s\n" "${RESPONSE}" | _json_value expires_in 1 1) - 1))" - _update_config ACCESS_TOKEN "${ACCESS_TOKEN}" "${CONFIG}" - _update_config ACCESS_TOKEN_EXPIRY "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" + _update_config "${SERVICE_ACCOUNT:+${SERVICE_ACCOUNT}_}ACCESS_TOKEN" "${ACCESS_TOKEN}" "${CONFIG}" + _update_config "${SERVICE_ACCOUNT:+${SERVICE_ACCOUNT}_}ACCESS_TOKEN_EXPIRY" "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" else "${QUIET:-_print_center}" "justify" "Error: Something went wrong" ", printing error." "=" 1>&2 printf "%s\n" "${RESPONSE}" 1>&2 diff --git a/sh/google-oauth2.sh b/sh/google-oauth2.sh index c373c8e..7d16f02 100755 --- a/sh/google-oauth2.sh +++ b/sh/google-oauth2.sh @@ -113,7 +113,7 @@ if [ "${1}" = create ]; then _clear_line 1 1>&2 REFRESH_TOKEN="$(printf "%s\n" "${RESPONSE}" | _json_value refresh_token 1 1 || :)" - if _get_access_token_and_update "${RESPONSE}"; then + if _get_access_token_and_update normal "${RESPONSE}"; then _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}" printf "Access Token: %s\n" "${ACCESS_TOKEN}" printf "Refresh Token: %s\n" "${REFRESH_TOKEN}" @@ -123,7 +123,7 @@ if [ "${1}" = create ]; then elif [ "${1}" = refresh ]; then if [ -n "${REFRESH_TOKEN}" ]; then "${QUIET:-_print_center}" "justify" "Required credentials set." "=" - { _get_access_token_and_update && _clear_line 1; } || return 1 + { _get_access_token_and_update normal && _clear_line 1; } || return 1 printf "Access Token: %s\n" "${ACCESS_TOKEN}" else "${QUIET:-_print_center}" "normal" "Refresh Token not set" ", use ${0##*/} create to generate one." "=" diff --git a/sh/release/gupload b/sh/release/gupload index 27d432b..1e448ec 100755 --- a/sh/release/gupload +++ b/sh/release/gupload @@ -520,11 +520,24 @@ _extract_id() { # Result: Update access_token and expiry else print error ################################################### _get_access_token_and_update() { - RESPONSE="${1:-$(curl --compressed -s -X POST --data "client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&refresh_token=${REFRESH_TOKEN}&grant_type=refresh_token" "${TOKEN_URL}")}" || : + case "${1:?Error: sa or normal}" in + normal) + RESPONSE="${2:-$(curl --compressed -s -X POST --data "client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&refresh_token=${REFRESH_TOKEN}&grant_type=refresh_token" "${TOKEN_URL}")}" || : + ;; + sa) + assertion_data="${2:?Error: Missing assertion data.}" + RESPONSE="$(curl --compressed -s --data "grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=${assertion_data}" "${TOKEN_URL}")" || : + # sa token jsons are not pretty printed + # shellcheck disable=SC1004 + RESPONSE="$(printf "%s\n" "${RESPONSE}" | sed -e 's/,"/\ +"/g')" + ;; + esac + if ACCESS_TOKEN="$(printf "%s\n" "${RESPONSE}" | _json_value access_token 1 1)"; then ACCESS_TOKEN_EXPIRY="$(($(date +"%s") + $(printf "%s\n" "${RESPONSE}" | _json_value expires_in 1 1) - 1))" - _update_config ACCESS_TOKEN "${ACCESS_TOKEN}" "${CONFIG}" - _update_config ACCESS_TOKEN_EXPIRY "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" + _update_config "${SERVICE_ACCOUNT:+${SERVICE_ACCOUNT}_}ACCESS_TOKEN" "${ACCESS_TOKEN}" "${CONFIG}" + _update_config "${SERVICE_ACCOUNT:+${SERVICE_ACCOUNT}_}ACCESS_TOKEN_EXPIRY" "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" else "${QUIET:-_print_center}" "justify" "Error: Something went wrong" ", printing error." "=" 1>&2 printf "%s\n" "${RESPONSE}" 1>&2 @@ -807,6 +820,57 @@ _error_logging_upload() { return 1 } +################################################### +# Generate rs256 jwt just with cli commands and shell +# Specifically for gdrive service accounts usage +# Globals: 1 +# SCOPE ( optional ) +# Arguments: 2 +# ${1} = service account json file +# ${2} = SCOPE for gdrive +# Result: print jwt +# Refrences: +# https://stackoverflow.com/questions/46657001/how-do-you-create-an-rs256-jwt-assertion-with-bash-shell-scripting +# Inspired by implementation by Will Haley at: +# http://willhaley.com/blog/generate-jwt-with-bash/ +################################################### +_generate_jwt() { + json_file_generate_jwt="${1:?Error: Give service json file name}" + scope_generate_jwt="${2:-${SCOPE:?Error: Missing scope}}" + aud_generate_jwt="https://oauth2.googleapis.com/token" + header_generate_jwt='{"alg":"RS256","typ":"JWT"}' + algo_generate_jwt="256" + unset payload_data_generate_jwt iss_generate_jwt exp_generate_jwt iat_generate_jwt rsa_secret_generate_jwt \ + signed_content_generate_jwt sign_generate_jwt + + if iss_generate_jwt="$(_json_value client_email 1 1 < "${json_file_generate_jwt}")" && + rsa_secret_generate_jwt="$(_json_value private_key 1 1 < "${json_file_generate_jwt}")"; then + rsa_secret_generate_jwt="$(printf "%b\n" "${rsa_secret_generate_jwt}")" + else + "${QUIET:-_print_center}" 'normal' "Error: Invalid service account file." "=" && return 1 + fi + + iat_generate_jwt="$(date +"%s")" exp_generate_jwt="$((iat_generate_jwt + 3400))" + + b64enc() { openssl enc -base64 -A | sed -e "s|+|-|g" -e "s|/|_|g" -e "s|=||g"; } + + payload_data_generate_jwt='{"iss":"'${iss_generate_jwt}'","scope":"'${scope_generate_jwt}'","aud":"'${aud_generate_jwt}'","exp":'${exp_generate_jwt}',"iat":'${iat_generate_jwt}'}' + + { + signed_content_generate_jwt="$(printf "%s\n" "${header_generate_jwt}" | b64enc).$(printf "%s\n" "${payload_data_generate_jwt}" | b64enc)" + # Open file discriptor for rsa_secret + exec 5<< EOF +$(printf "%s\n" "${rsa_secret_generate_jwt}") +EOF + sign_generate_jwt="$(printf %s "${signed_content_generate_jwt}" | openssl dgst -binary -sha"${algo_generate_jwt}" -sign /dev/fd/5 | b64enc)" + # close file discriptor + exec 5<&- + } || return 1 + + printf '%s.%s\n' "${signed_content_generate_jwt}" "${sign_generate_jwt}" + return 0 +} + ################################################### # A small function to get rootdir id for files in sub folder uploads # Globals: 1 variable, 1 function @@ -927,6 +991,7 @@ Usage:\n ${0##*/} [options.. ] \n Foldername argument is optional. If not provided, the file will be uploaded to preconfigured google drive.\n File name argument is optional if create directory option is used.\n Options:\n + -sa | --service-accounts 'service account json file path' - Use a bot service account. Should be in proper json format.\n -c | -C | --create-dir - option to create directory. Will provide folder id. Can be used to provide input folder, see README.\n -r | --root-dir or - google folder ID/URL to which the file/directory is going to upload. If you want to change the default value, then use this format, -r/--root-dir default=root_folder_id/root_folder_url\n @@ -1031,14 +1096,15 @@ _setup_arguments() { # De-initialize if any variables set already. unset FIRST_INPUT FOLDER_INPUT FOLDERNAME FINAL_LOCAL_INPUT_ARRAY FINAL_ID_INPUT_ARRAY unset PARALLEL NO_OF_PARALLEL_JOBS SHARE SHARE_EMAIL OVERWRITE SKIP_DUPLICATES SKIP_SUBDIRS ROOTDIR QUIET - unset VERBOSE VERBOSE_PROGRESS DEBUG LOG_FILE_ID CURL_SPEED RETRY + unset VERBOSE VERBOSE_PROGRESS DEBUG LOG_FILE_ID CURL_SPEED RETRY OPENSSL_ERROR CURL_PROGRESS="-s" EXTRA_LOG=":" CURL_PROGRESS_EXTRA="-s" INFO_PATH="${HOME}/.google-drive-upload" CONFIG_INFO="${INFO_PATH}/google-drive-upload.configpath" [ -f "${CONFIG_INFO}" ] && . "${CONFIG_INFO}" CONFIG="${CONFIG:-${HOME}/.googledrive.conf}" # Configuration variables # Remote gDrive variables - unset ROOT_FOLDER CLIENT_ID CLIENT_SECRET REFRESH_TOKEN ACCESS_TOKEN + unset ROOT_FOLDER CLIENT_ID CLIENT_SECRET REFRESH_TOKEN ACCESS_TOKEN \ + SERVICE_ACCOUNT SERVICE_ACCOUNT_ACCESS_TOKEN SERVICE_ACCOUNT_ACCESS_TOKEN_EXPIRY API_URL="https://www.googleapis.com" API_VERSION="v3" SCOPE="${API_URL}/auth/drive" @@ -1065,6 +1131,11 @@ _setup_arguments() { -h | --help) _usage ;; -D | --debug) DEBUG="true" && export DEBUG ;; --info) _version_info ;; + -sa | --service-account) + _check_longoptions "${1}" "${2}" + SERVICE_ACCOUNT_FILE="${2}" + ! [ -f "${SERVICE_ACCOUNT_FILE}" ] && printf "%s\n" "Error: Service account json file exist ( ${SERVICE_ACCOUNT_FILE} )." 1>&2 && exit 1 + ;; -c | -C | --create-dir) _check_longoptions "${1}" "${2}" FOLDERNAME="${2}" && shift @@ -1225,107 +1296,132 @@ _check_credentials() { printf "%s\n" "Add in config manually if terminal is not accessible. CLIENT_ID, CLIENT_SECRET and REFRESH_TOKEN is required." && return 1 } - # Following https://developers.google.com/identity/protocols/oauth2#size - CLIENT_ID_REGEX='[0-9]+-[0-9A-Za-z_]{32}\.apps\.googleusercontent\.com' - CLIENT_SECRET_REGEX='[0-9A-Za-z_-]+' - REFRESH_TOKEN_REGEX='[0-9]//[0-9A-Za-z_-]+' # 512 bytes - ACCESS_TOKEN_REGEX='ya29\.[0-9A-Za-z_-]+' # 2048 bytes - AUTHORIZATION_CODE_REGEX='[0-9]/[0-9A-Za-z_-]+' # 256 bytes - - until [ -n "${CLIENT_ID}" ] && [ -n "${CLIENT_ID_VALID}" ]; do - [ -n "${CLIENT_ID}" ] && { - if printf "%s\n" "${CLIENT_ID}" | grep -qE "${CLIENT_ID_REGEX}"; then - [ -n "${client_id}" ] && _update_config CLIENT_ID "${CLIENT_ID}" "${CONFIG}" - CLIENT_ID_VALID="true" && continue - else - { [ -n "${client_id}" ] && message="- Try again"; } || message="in config ( ${CONFIG} )" - "${QUIET:-_print_center}" "normal" " Invalid Client ID ${message} " "-" && unset CLIENT_ID client_id - fi + ACCESS_TOKEN_REGEX='ya29\.[0-9A-Za-z_-]+' # 2048 bytes + + if [ -z "${SERVICE_ACCOUNT_FILE}" ]; then + # Following https://developers.google.com/identity/protocols/oauth2#size + CLIENT_ID_REGEX='[0-9]+-[0-9A-Za-z_]{32}\.apps\.googleusercontent\.com' + CLIENT_SECRET_REGEX='[0-9A-Za-z_-]+' + REFRESH_TOKEN_REGEX='[0-9]//[0-9A-Za-z_-]+' # 512 bytes + AUTHORIZATION_CODE_REGEX='[0-9]/[0-9A-Za-z_-]+' # 256 bytes + + until [ -n "${CLIENT_ID}" ] && [ -n "${CLIENT_ID_VALID}" ]; do + [ -n "${CLIENT_ID}" ] && { + if printf "%s\n" "${CLIENT_ID}" | grep -qE "${CLIENT_ID_REGEX}"; then + [ -n "${client_id}" ] && _update_config CLIENT_ID "${CLIENT_ID}" "${CONFIG}" + CLIENT_ID_VALID="true" && continue + else + { [ -n "${client_id}" ] && message="- Try again"; } || message="in config ( ${CONFIG} )" + "${QUIET:-_print_center}" "normal" " Invalid Client ID ${message} " "-" && unset CLIENT_ID client_id + fi + } + [ -z "${client_id}" ] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter Client ID " "-" + [ -n "${client_id}" ] && _clear_line 1 + printf -- "-> " + read -r CLIENT_ID && client_id=1 + done + + until [ -n "${CLIENT_SECRET}" ] && [ -n "${CLIENT_SECRET_VALID}" ]; do + [ -n "${CLIENT_SECRET}" ] && { + if printf "%s\n" "${CLIENT_SECRET}" | grep -qE "${CLIENT_SECRET_REGEX}"; then + [ -n "${client_secret}" ] && _update_config CLIENT_SECRET "${CLIENT_SECRET}" "${CONFIG}" + CLIENT_SECRET_VALID="true" && continue + else + { [ -n "${client_secret}" ] && message="- Try again"; } || message="in config ( ${CONFIG} )" + "${QUIET:-_print_center}" "normal" " Invalid Client Secret ${message} " "-" && unset CLIENT_SECRET client_secret + fi + } + [ -z "${client_secret}" ] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter Client Secret " "-" + [ -n "${client_secret}" ] && _clear_line 1 + printf -- "-> " + read -r CLIENT_SECRET && client_secret=1 + done + + [ -n "${REFRESH_TOKEN}" ] && { + ! printf "%s\n" "${REFRESH_TOKEN}" | grep -qE "${REFRESH_TOKEN_REGEX}" && + "${QUIET:-_print_center}" "normal" " Error: Invalid Refresh token in config file, follow below steps.. " "-" && unset REFRESH_TOKEN } - [ -z "${client_id}" ] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter Client ID " "-" - [ -n "${client_id}" ] && _clear_line 1 - printf -- "-> " - read -r CLIENT_ID && client_id=1 - done - until [ -n "${CLIENT_SECRET}" ] && [ -n "${CLIENT_SECRET_VALID}" ]; do - [ -n "${CLIENT_SECRET}" ] && { - if printf "%s\n" "${CLIENT_SECRET}" | grep -qE "${CLIENT_SECRET_REGEX}"; then - [ -n "${client_secret}" ] && _update_config CLIENT_SECRET "${CLIENT_SECRET}" "${CONFIG}" - CLIENT_SECRET_VALID="true" && continue + [ -z "${REFRESH_TOKEN}" ] && { + printf "\n" && "${QUIET:-_print_center}" "normal" "If you have a refresh token generated, then type the token, else leave blank and press return key.." " " + printf "\n" && "${QUIET:-_print_center}" "normal" " Refresh Token " "-" && printf -- "-> " + read -r REFRESH_TOKEN + if [ -n "${REFRESH_TOKEN}" ]; then + "${QUIET:-_print_center}" "normal" " Checking refresh token.. " "-" + if ! printf "%s\n" "${REFRESH_TOKEN}" | grep -qE "${REFRESH_TOKEN_REGEX}"; then + { _get_access_token_and_update normal && _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}"; } || check_error=1 + else + check_error=true + fi + [ -n "${check_error}" ] && "${QUIET:-_print_center}" "normal" " Error: Invalid Refresh token given, follow below steps to generate.. " "-" && unset REFRESH_TOKEN else - { [ -n "${client_secret}" ] && message="- Try again"; } || message="in config ( ${CONFIG} )" - "${QUIET:-_print_center}" "normal" " Invalid Client Secret ${message} " "-" && unset CLIENT_SECRET client_secret + "${QUIET:-_print_center}" "normal" " No Refresh token given, follow below steps to generate.. " "-" fi + + [ -z "${REFRESH_TOKEN}" ] && { + printf "\n" && "${QUIET:-_print_center}" "normal" "Visit the below URL, tap on allow and then enter the code obtained" " " + URL="https://accounts.google.com/o/oauth2/auth?client_id=${CLIENT_ID}&redirect_uri=${REDIRECT_URI}&scope=${SCOPE}&response_type=code&prompt=consent" + printf "\n%s\n" "${URL}" + until [ -n "${AUTHORIZATION_CODE}" ] && [ -n "${AUTHORIZATION_CODE_VALID}" ]; do + [ -n "${AUTHORIZATION_CODE}" ] && { + if printf "%s\n" "${AUTHORIZATION_CODE}" | grep -qE "${AUTHORIZATION_CODE_REGEX}"; then + AUTHORIZATION_CODE_VALID="true" && continue + else + "${QUIET:-_print_center}" "normal" " Invalid CODE given, try again.. " "-" && unset AUTHORIZATION_CODE authorization_code + fi + } + { [ -z "${authorization_code}" ] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter the authorization code " "-"; } || _clear_line 1 + printf -- "-> " + read -r AUTHORIZATION_CODE && authorization_code=1 + done + RESPONSE="$(curl --compressed "${CURL_PROGRESS}" -X POST \ + --data "code=${AUTHORIZATION_CODE}&client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&redirect_uri=${REDIRECT_URI}&grant_type=authorization_code" "${TOKEN_URL}")" || : + _clear_line 1 1>&2 + + REFRESH_TOKEN="$(printf "%s\n" "${RESPONSE}" | _json_value refresh_token 1 1 || :)" + { _get_access_token_and_update normal "${RESPONSE}" && _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}"; } || return 1 + } + printf "\n" } - [ -z "${client_secret}" ] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter Client Secret " "-" - [ -n "${client_secret}" ] && _clear_line 1 - printf -- "-> " - read -r CLIENT_SECRET && client_secret=1 - done - [ -n "${REFRESH_TOKEN}" ] && { - ! printf "%s\n" "${REFRESH_TOKEN}" | grep -qE "${REFRESH_TOKEN_REGEX}" && - "${QUIET:-_print_center}" "normal" " Error: Invalid Refresh token in config file, follow below steps.. " "-" && unset REFRESH_TOKEN - } + { [ -z "${ACCESS_TOKEN}" ] || ! printf "%s\n" "${ACCESS_TOKEN}" | grep -qE "${ACCESS_TOKEN_REGEX}" || [ "${ACCESS_TOKEN_EXPIRY:-0}" -lt "$(date +'%s')" ]; } && + { _get_access_token_and_update normal || return 1; } + printf "%b\n" "ACCESS_TOKEN=\"${ACCESS_TOKEN}\"\nACCESS_TOKEN_EXPIRY=\"${ACCESS_TOKEN_EXPIRY}\"" >| "${TMPFILE}_ACCESS_TOKEN" + else + command -v openssl 2>| /dev/null 1>&2 || + { "${QUIET:-_print_center}" 'normal' "Error: openssl not installed, install openssl to use '-sa | --service-account' flag." "=" 1>&2 && return 1; } - [ -z "${REFRESH_TOKEN}" ] && { - printf "\n" && "${QUIET:-_print_center}" "normal" "If you have a refresh token generated, then type the token, else leave blank and press return key.." " " - printf "\n" && "${QUIET:-_print_center}" "normal" " Refresh Token " "-" && printf -- "-> " - read -r REFRESH_TOKEN - if [ -n "${REFRESH_TOKEN}" ]; then - "${QUIET:-_print_center}" "normal" " Checking refresh token.. " "-" - if ! printf "%s\n" "${REFRESH_TOKEN}" | grep -qE "${REFRESH_TOKEN_REGEX}"; then - { _get_access_token_and_update && _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}"; } || check_error=1 - else - check_error=true - fi - [ -n "${check_error}" ] && "${QUIET:-_print_center}" "normal" " Error: Invalid Refresh token given, follow below steps to generate.. " "-" && unset REFRESH_TOKEN - else - "${QUIET:-_print_center}" "normal" " No Refresh token given, follow below steps to generate.. " "-" - fi + SERVICE_ACCOUNT="SA_$(_json_value private_key_id 1 1 < "${SERVICE_ACCOUNT_FILE}")_SA" || + { "${QUIET:-_print_center}" 'normal' "Error: Invalid service account file." "=" 1>&2 && return 1; } - [ -z "${REFRESH_TOKEN}" ] && { - printf "\n" && "${QUIET:-_print_center}" "normal" "Visit the below URL, tap on allow and then enter the code obtained" " " - URL="https://accounts.google.com/o/oauth2/auth?client_id=${CLIENT_ID}&redirect_uri=${REDIRECT_URI}&scope=${SCOPE}&response_type=code&prompt=consent" - printf "\n%s\n" "${URL}" - until [ -n "${AUTHORIZATION_CODE}" ] && [ -n "${AUTHORIZATION_CODE_VALID}" ]; do - [ -n "${AUTHORIZATION_CODE}" ] && { - if printf "%s\n" "${AUTHORIZATION_CODE}" | grep -qE "${AUTHORIZATION_CODE_REGEX}"; then - AUTHORIZATION_CODE_VALID="true" && continue - else - "${QUIET:-_print_center}" "normal" " Invalid CODE given, try again.. " "-" && unset AUTHORIZATION_CODE authorization_code - fi - } - { [ -z "${authorization_code}" ] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter the authorization code " "-"; } || _clear_line 1 - printf -- "-> " - read -r AUTHORIZATION_CODE && authorization_code=1 - done - RESPONSE="$(curl --compressed "${CURL_PROGRESS}" -X POST \ - --data "code=${AUTHORIZATION_CODE}&client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&redirect_uri=${REDIRECT_URI}&grant_type=authorization_code" "${TOKEN_URL}")" || : - _clear_line 1 1>&2 + SERVICE_ACCOUNT_ACCESS_TOKEN="$(eval printf "%s" \"\$"${SERVICE_ACCOUNT}_ACCESS_TOKEN"\")" + SERVICE_ACCOUNT_ACCESS_TOKEN_EXPIRY="$(eval printf "%s" \"\$"${SERVICE_ACCOUNT}_ACCESS_TOKEN_EXPIRY"\")" - REFRESH_TOKEN="$(printf "%s\n" "${RESPONSE}" | _json_value refresh_token 1 1 || :)" - { _get_access_token_and_update "${RESPONSE}" && _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}"; } || return 1 + { [ -z "${SERVICE_ACCOUNT_ACCESS_TOKEN}" ] || ! printf "%s\n" "${SERVICE_ACCOUNT_ACCESS_TOKEN}" | grep -qE "${ACCESS_TOKEN_REGEX}" || [ "${SERVICE_ACCOUNT_ACCESS_TOKEN_EXPIRY:-0}" -lt "$(date +'%s')" ]; } && { + ASSERTION_DATA="$(_generate_jwt "${SERVICE_ACCOUNT_FILE}" "${SCOPE}")" || { printf "%s\n" "${ASSERTION_DATA}" 1>&2 && return 1; } + _get_access_token_and_update sa "${ASSERTION_DATA}" || return 1 } - printf "\n" - } - { [ -z "${ACCESS_TOKEN}" ] || ! printf "%s\n" "${ACCESS_TOKEN}" | grep -qE "${ACCESS_TOKEN_REGEX}" || [ "${ACCESS_TOKEN_EXPIRY:-0}" -lt "$(date +'%s')" ]; } && - { _get_access_token_and_update || return 1; } - printf "%b\n" "ACCESS_TOKEN=\"${ACCESS_TOKEN}\"\nACCESS_TOKEN_EXPIRY=\"${ACCESS_TOKEN_EXPIRY}\"" >| "${TMPFILE}_ACCESS_TOKEN" + printf "%s\n%s\n" "ACCESS_TOKEN=\"$(eval printf "%s" \"\$"${SERVICE_ACCOUNT}_ACCESS_TOKEN"\")"\" \ + "ACCESS_TOKEN_EXPIRY=\"$(eval printf "%s" \"\$"${SERVICE_ACCOUNT}_ACCESS_TOKEN_EXPIRY"\")"\" >| "${TMPFILE}_ACCESS_TOKEN" + fi # launch a background service to check access token and update it # checks ACCESS_TOKEN_EXPIRY, try to update before 5 mins of expiry, a fresh token gets 60 mins # process will be killed when script exits or "${MAIN_PID}" is killed { until ! kill -0 "${MAIN_PID}" 2>| /dev/null 1>&2; do + unset ASSERTION_DATA MODE . "${TMPFILE}_ACCESS_TOKEN" CURRENT_TIME="$(date +'%s')" REMAINING_TOKEN_TIME="$((CURRENT_TIME - ACCESS_TOKEN_EXPIRY))" if [ "${REMAINING_TOKEN_TIME}" -le 300 ]; then + [ -n "${SERVICE_ACCOUNT_FILE}" ] && { + ASSERTION_DATA="$(_generate_jwt "${SERVICE_ACCOUNT_FILE}" "${SCOPE}")" || : + MODE="sa" + } # timeout after 30 seconds, it shouldn't take too long anyway, and update tmp config - CONFIG="${TMPFILE}_ACCESS_TOKEN" _timeout 30 _get_access_token_and_update || : + SERVICE_ACCOUNT="" CONFIG="${TMPFILE}_ACCESS_TOKEN" _timeout 30 _get_access_token_and_update "${MODE:-normal}" "${ASSERTION_DATA:-}" || : else TOKEN_PROCESS_TIME_TO_SLEEP="$(if [ "${REMAINING_TOKEN_TIME}" -le 301 ]; then printf "0\n" @@ -1336,7 +1432,7 @@ _check_credentials() { fi sleep 1 done - } & + } 2>| /dev/null 1>&2 & ACCESS_TOKEN_SERVICE_PID="${!}" return 0 @@ -1624,8 +1720,8 @@ main() { [ -f "${TMPFILE}_ACCESS_TOKEN" ] && { . "${TMPFILE}_ACCESS_TOKEN" [ "${INITIAL_ACCESS_TOKEN}" = "${ACCESS_TOKEN}" ] || { - _update_config ACCESS_TOKEN "${ACCESS_TOKEN}" "${CONFIG}" - _update_config ACCESS_TOKEN_EXPIRY "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" + _update_config "${SERVICE_ACCOUNT:+${SERVICE_ACCOUNT}_}ACCESS_TOKEN" "${ACCESS_TOKEN}" "${CONFIG}" + _update_config "${SERVICE_ACCOUNT:+${SERVICE_ACCOUNT}_}ACCESS_TOKEN_EXPIRY" "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" } } 1>| /dev/null diff --git a/sh/upload-utils.sh b/sh/upload-utils.sh index ddf864c..252208e 100755 --- a/sh/upload-utils.sh +++ b/sh/upload-utils.sh @@ -54,6 +54,57 @@ _error_logging_upload() { return 1 } +################################################### +# Generate rs256 jwt just with cli commands and shell +# Specifically for gdrive service accounts usage +# Globals: 1 +# SCOPE ( optional ) +# Arguments: 2 +# ${1} = service account json file +# ${2} = SCOPE for gdrive +# Result: print jwt +# Refrences: +# https://stackoverflow.com/questions/46657001/how-do-you-create-an-rs256-jwt-assertion-with-bash-shell-scripting +# Inspired by implementation by Will Haley at: +# http://willhaley.com/blog/generate-jwt-with-bash/ +################################################### +_generate_jwt() { + json_file_generate_jwt="${1:?Error: Give service json file name}" + scope_generate_jwt="${2:-${SCOPE:?Error: Missing scope}}" + aud_generate_jwt="https://oauth2.googleapis.com/token" + header_generate_jwt='{"alg":"RS256","typ":"JWT"}' + algo_generate_jwt="256" + unset payload_data_generate_jwt iss_generate_jwt exp_generate_jwt iat_generate_jwt rsa_secret_generate_jwt \ + signed_content_generate_jwt sign_generate_jwt + + if iss_generate_jwt="$(_json_value client_email 1 1 < "${json_file_generate_jwt}")" && + rsa_secret_generate_jwt="$(_json_value private_key 1 1 < "${json_file_generate_jwt}")"; then + rsa_secret_generate_jwt="$(printf "%b\n" "${rsa_secret_generate_jwt}")" + else + "${QUIET:-_print_center}" 'normal' "Error: Invalid service account file." "=" && return 1 + fi + + iat_generate_jwt="$(date +"%s")" exp_generate_jwt="$((iat_generate_jwt + 3400))" + + b64enc() { openssl enc -base64 -A | sed -e "s|+|-|g" -e "s|/|_|g" -e "s|=||g"; } + + payload_data_generate_jwt='{"iss":"'${iss_generate_jwt}'","scope":"'${scope_generate_jwt}'","aud":"'${aud_generate_jwt}'","exp":'${exp_generate_jwt}',"iat":'${iat_generate_jwt}'}' + + { + signed_content_generate_jwt="$(printf "%s\n" "${header_generate_jwt}" | b64enc).$(printf "%s\n" "${payload_data_generate_jwt}" | b64enc)" + # Open file discriptor for rsa_secret + exec 5<< EOF +$(printf "%s\n" "${rsa_secret_generate_jwt}") +EOF + sign_generate_jwt="$(printf %s "${signed_content_generate_jwt}" | openssl dgst -binary -sha"${algo_generate_jwt}" -sign /dev/fd/5 | b64enc)" + # close file discriptor + exec 5<&- + } || return 1 + + printf '%s.%s\n' "${signed_content_generate_jwt}" "${sign_generate_jwt}" + return 0 +} + ################################################### # A small function to get rootdir id for files in sub folder uploads # Globals: 1 variable, 1 function diff --git a/sh/upload.sh b/sh/upload.sh index 61d598d..ce65362 100755 --- a/sh/upload.sh +++ b/sh/upload.sh @@ -9,6 +9,7 @@ Usage:\n ${0##*/} [options.. ] \n Foldername argument is optional. If not provided, the file will be uploaded to preconfigured google drive.\n File name argument is optional if create directory option is used.\n Options:\n + -sa | --service-accounts 'service account json file path' - Use a bot service account. Should be in proper json format.\n -c | -C | --create-dir - option to create directory. Will provide folder id. Can be used to provide input folder, see README.\n -r | --root-dir or - google folder ID/URL to which the file/directory is going to upload. If you want to change the default value, then use this format, -r/--root-dir default=root_folder_id/root_folder_url\n @@ -113,14 +114,15 @@ _setup_arguments() { # De-initialize if any variables set already. unset FIRST_INPUT FOLDER_INPUT FOLDERNAME FINAL_LOCAL_INPUT_ARRAY FINAL_ID_INPUT_ARRAY unset PARALLEL NO_OF_PARALLEL_JOBS SHARE SHARE_EMAIL OVERWRITE SKIP_DUPLICATES SKIP_SUBDIRS ROOTDIR QUIET - unset VERBOSE VERBOSE_PROGRESS DEBUG LOG_FILE_ID CURL_SPEED RETRY + unset VERBOSE VERBOSE_PROGRESS DEBUG LOG_FILE_ID CURL_SPEED RETRY OPENSSL_ERROR CURL_PROGRESS="-s" EXTRA_LOG=":" CURL_PROGRESS_EXTRA="-s" INFO_PATH="${HOME}/.google-drive-upload" CONFIG_INFO="${INFO_PATH}/google-drive-upload.configpath" [ -f "${CONFIG_INFO}" ] && . "${CONFIG_INFO}" CONFIG="${CONFIG:-${HOME}/.googledrive.conf}" # Configuration variables # Remote gDrive variables - unset ROOT_FOLDER CLIENT_ID CLIENT_SECRET REFRESH_TOKEN ACCESS_TOKEN + unset ROOT_FOLDER CLIENT_ID CLIENT_SECRET REFRESH_TOKEN ACCESS_TOKEN \ + SERVICE_ACCOUNT SERVICE_ACCOUNT_ACCESS_TOKEN SERVICE_ACCOUNT_ACCESS_TOKEN_EXPIRY API_URL="https://www.googleapis.com" API_VERSION="v3" SCOPE="${API_URL}/auth/drive" @@ -147,6 +149,11 @@ _setup_arguments() { -h | --help) _usage ;; -D | --debug) DEBUG="true" && export DEBUG ;; --info) _version_info ;; + -sa | --service-account) + _check_longoptions "${1}" "${2}" + SERVICE_ACCOUNT_FILE="${2}" + ! [ -f "${SERVICE_ACCOUNT_FILE}" ] && printf "%s\n" "Error: Service account json file exist ( ${SERVICE_ACCOUNT_FILE} )." 1>&2 && exit 1 + ;; -c | -C | --create-dir) _check_longoptions "${1}" "${2}" FOLDERNAME="${2}" && shift @@ -307,107 +314,132 @@ _check_credentials() { printf "%s\n" "Add in config manually if terminal is not accessible. CLIENT_ID, CLIENT_SECRET and REFRESH_TOKEN is required." && return 1 } - # Following https://developers.google.com/identity/protocols/oauth2#size - CLIENT_ID_REGEX='[0-9]+-[0-9A-Za-z_]{32}\.apps\.googleusercontent\.com' - CLIENT_SECRET_REGEX='[0-9A-Za-z_-]+' - REFRESH_TOKEN_REGEX='[0-9]//[0-9A-Za-z_-]+' # 512 bytes - ACCESS_TOKEN_REGEX='ya29\.[0-9A-Za-z_-]+' # 2048 bytes - AUTHORIZATION_CODE_REGEX='[0-9]/[0-9A-Za-z_-]+' # 256 bytes - - until [ -n "${CLIENT_ID}" ] && [ -n "${CLIENT_ID_VALID}" ]; do - [ -n "${CLIENT_ID}" ] && { - if printf "%s\n" "${CLIENT_ID}" | grep -qE "${CLIENT_ID_REGEX}"; then - [ -n "${client_id}" ] && _update_config CLIENT_ID "${CLIENT_ID}" "${CONFIG}" - CLIENT_ID_VALID="true" && continue - else - { [ -n "${client_id}" ] && message="- Try again"; } || message="in config ( ${CONFIG} )" - "${QUIET:-_print_center}" "normal" " Invalid Client ID ${message} " "-" && unset CLIENT_ID client_id - fi + ACCESS_TOKEN_REGEX='ya29\.[0-9A-Za-z_-]+' # 2048 bytes + + if [ -z "${SERVICE_ACCOUNT_FILE}" ]; then + # Following https://developers.google.com/identity/protocols/oauth2#size + CLIENT_ID_REGEX='[0-9]+-[0-9A-Za-z_]{32}\.apps\.googleusercontent\.com' + CLIENT_SECRET_REGEX='[0-9A-Za-z_-]+' + REFRESH_TOKEN_REGEX='[0-9]//[0-9A-Za-z_-]+' # 512 bytes + AUTHORIZATION_CODE_REGEX='[0-9]/[0-9A-Za-z_-]+' # 256 bytes + + until [ -n "${CLIENT_ID}" ] && [ -n "${CLIENT_ID_VALID}" ]; do + [ -n "${CLIENT_ID}" ] && { + if printf "%s\n" "${CLIENT_ID}" | grep -qE "${CLIENT_ID_REGEX}"; then + [ -n "${client_id}" ] && _update_config CLIENT_ID "${CLIENT_ID}" "${CONFIG}" + CLIENT_ID_VALID="true" && continue + else + { [ -n "${client_id}" ] && message="- Try again"; } || message="in config ( ${CONFIG} )" + "${QUIET:-_print_center}" "normal" " Invalid Client ID ${message} " "-" && unset CLIENT_ID client_id + fi + } + [ -z "${client_id}" ] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter Client ID " "-" + [ -n "${client_id}" ] && _clear_line 1 + printf -- "-> " + read -r CLIENT_ID && client_id=1 + done + + until [ -n "${CLIENT_SECRET}" ] && [ -n "${CLIENT_SECRET_VALID}" ]; do + [ -n "${CLIENT_SECRET}" ] && { + if printf "%s\n" "${CLIENT_SECRET}" | grep -qE "${CLIENT_SECRET_REGEX}"; then + [ -n "${client_secret}" ] && _update_config CLIENT_SECRET "${CLIENT_SECRET}" "${CONFIG}" + CLIENT_SECRET_VALID="true" && continue + else + { [ -n "${client_secret}" ] && message="- Try again"; } || message="in config ( ${CONFIG} )" + "${QUIET:-_print_center}" "normal" " Invalid Client Secret ${message} " "-" && unset CLIENT_SECRET client_secret + fi + } + [ -z "${client_secret}" ] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter Client Secret " "-" + [ -n "${client_secret}" ] && _clear_line 1 + printf -- "-> " + read -r CLIENT_SECRET && client_secret=1 + done + + [ -n "${REFRESH_TOKEN}" ] && { + ! printf "%s\n" "${REFRESH_TOKEN}" | grep -qE "${REFRESH_TOKEN_REGEX}" && + "${QUIET:-_print_center}" "normal" " Error: Invalid Refresh token in config file, follow below steps.. " "-" && unset REFRESH_TOKEN } - [ -z "${client_id}" ] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter Client ID " "-" - [ -n "${client_id}" ] && _clear_line 1 - printf -- "-> " - read -r CLIENT_ID && client_id=1 - done - until [ -n "${CLIENT_SECRET}" ] && [ -n "${CLIENT_SECRET_VALID}" ]; do - [ -n "${CLIENT_SECRET}" ] && { - if printf "%s\n" "${CLIENT_SECRET}" | grep -qE "${CLIENT_SECRET_REGEX}"; then - [ -n "${client_secret}" ] && _update_config CLIENT_SECRET "${CLIENT_SECRET}" "${CONFIG}" - CLIENT_SECRET_VALID="true" && continue + [ -z "${REFRESH_TOKEN}" ] && { + printf "\n" && "${QUIET:-_print_center}" "normal" "If you have a refresh token generated, then type the token, else leave blank and press return key.." " " + printf "\n" && "${QUIET:-_print_center}" "normal" " Refresh Token " "-" && printf -- "-> " + read -r REFRESH_TOKEN + if [ -n "${REFRESH_TOKEN}" ]; then + "${QUIET:-_print_center}" "normal" " Checking refresh token.. " "-" + if ! printf "%s\n" "${REFRESH_TOKEN}" | grep -qE "${REFRESH_TOKEN_REGEX}"; then + { _get_access_token_and_update normal && _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}"; } || check_error=1 + else + check_error=true + fi + [ -n "${check_error}" ] && "${QUIET:-_print_center}" "normal" " Error: Invalid Refresh token given, follow below steps to generate.. " "-" && unset REFRESH_TOKEN else - { [ -n "${client_secret}" ] && message="- Try again"; } || message="in config ( ${CONFIG} )" - "${QUIET:-_print_center}" "normal" " Invalid Client Secret ${message} " "-" && unset CLIENT_SECRET client_secret + "${QUIET:-_print_center}" "normal" " No Refresh token given, follow below steps to generate.. " "-" fi + + [ -z "${REFRESH_TOKEN}" ] && { + printf "\n" && "${QUIET:-_print_center}" "normal" "Visit the below URL, tap on allow and then enter the code obtained" " " + URL="https://accounts.google.com/o/oauth2/auth?client_id=${CLIENT_ID}&redirect_uri=${REDIRECT_URI}&scope=${SCOPE}&response_type=code&prompt=consent" + printf "\n%s\n" "${URL}" + until [ -n "${AUTHORIZATION_CODE}" ] && [ -n "${AUTHORIZATION_CODE_VALID}" ]; do + [ -n "${AUTHORIZATION_CODE}" ] && { + if printf "%s\n" "${AUTHORIZATION_CODE}" | grep -qE "${AUTHORIZATION_CODE_REGEX}"; then + AUTHORIZATION_CODE_VALID="true" && continue + else + "${QUIET:-_print_center}" "normal" " Invalid CODE given, try again.. " "-" && unset AUTHORIZATION_CODE authorization_code + fi + } + { [ -z "${authorization_code}" ] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter the authorization code " "-"; } || _clear_line 1 + printf -- "-> " + read -r AUTHORIZATION_CODE && authorization_code=1 + done + RESPONSE="$(curl --compressed "${CURL_PROGRESS}" -X POST \ + --data "code=${AUTHORIZATION_CODE}&client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&redirect_uri=${REDIRECT_URI}&grant_type=authorization_code" "${TOKEN_URL}")" || : + _clear_line 1 1>&2 + + REFRESH_TOKEN="$(printf "%s\n" "${RESPONSE}" | _json_value refresh_token 1 1 || :)" + { _get_access_token_and_update normal "${RESPONSE}" && _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}"; } || return 1 + } + printf "\n" } - [ -z "${client_secret}" ] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter Client Secret " "-" - [ -n "${client_secret}" ] && _clear_line 1 - printf -- "-> " - read -r CLIENT_SECRET && client_secret=1 - done - [ -n "${REFRESH_TOKEN}" ] && { - ! printf "%s\n" "${REFRESH_TOKEN}" | grep -qE "${REFRESH_TOKEN_REGEX}" && - "${QUIET:-_print_center}" "normal" " Error: Invalid Refresh token in config file, follow below steps.. " "-" && unset REFRESH_TOKEN - } + { [ -z "${ACCESS_TOKEN}" ] || ! printf "%s\n" "${ACCESS_TOKEN}" | grep -qE "${ACCESS_TOKEN_REGEX}" || [ "${ACCESS_TOKEN_EXPIRY:-0}" -lt "$(date +'%s')" ]; } && + { _get_access_token_and_update normal || return 1; } + printf "%b\n" "ACCESS_TOKEN=\"${ACCESS_TOKEN}\"\nACCESS_TOKEN_EXPIRY=\"${ACCESS_TOKEN_EXPIRY}\"" >| "${TMPFILE}_ACCESS_TOKEN" + else + command -v openssl 2>| /dev/null 1>&2 || + { "${QUIET:-_print_center}" 'normal' "Error: openssl not installed, install openssl to use '-sa | --service-account' flag." "=" 1>&2 && return 1; } - [ -z "${REFRESH_TOKEN}" ] && { - printf "\n" && "${QUIET:-_print_center}" "normal" "If you have a refresh token generated, then type the token, else leave blank and press return key.." " " - printf "\n" && "${QUIET:-_print_center}" "normal" " Refresh Token " "-" && printf -- "-> " - read -r REFRESH_TOKEN - if [ -n "${REFRESH_TOKEN}" ]; then - "${QUIET:-_print_center}" "normal" " Checking refresh token.. " "-" - if ! printf "%s\n" "${REFRESH_TOKEN}" | grep -qE "${REFRESH_TOKEN_REGEX}"; then - { _get_access_token_and_update && _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}"; } || check_error=1 - else - check_error=true - fi - [ -n "${check_error}" ] && "${QUIET:-_print_center}" "normal" " Error: Invalid Refresh token given, follow below steps to generate.. " "-" && unset REFRESH_TOKEN - else - "${QUIET:-_print_center}" "normal" " No Refresh token given, follow below steps to generate.. " "-" - fi + SERVICE_ACCOUNT="SA_$(_json_value private_key_id 1 1 < "${SERVICE_ACCOUNT_FILE}")_SA" || + { "${QUIET:-_print_center}" 'normal' "Error: Invalid service account file." "=" 1>&2 && return 1; } - [ -z "${REFRESH_TOKEN}" ] && { - printf "\n" && "${QUIET:-_print_center}" "normal" "Visit the below URL, tap on allow and then enter the code obtained" " " - URL="https://accounts.google.com/o/oauth2/auth?client_id=${CLIENT_ID}&redirect_uri=${REDIRECT_URI}&scope=${SCOPE}&response_type=code&prompt=consent" - printf "\n%s\n" "${URL}" - until [ -n "${AUTHORIZATION_CODE}" ] && [ -n "${AUTHORIZATION_CODE_VALID}" ]; do - [ -n "${AUTHORIZATION_CODE}" ] && { - if printf "%s\n" "${AUTHORIZATION_CODE}" | grep -qE "${AUTHORIZATION_CODE_REGEX}"; then - AUTHORIZATION_CODE_VALID="true" && continue - else - "${QUIET:-_print_center}" "normal" " Invalid CODE given, try again.. " "-" && unset AUTHORIZATION_CODE authorization_code - fi - } - { [ -z "${authorization_code}" ] && printf "\n" && "${QUIET:-_print_center}" "normal" " Enter the authorization code " "-"; } || _clear_line 1 - printf -- "-> " - read -r AUTHORIZATION_CODE && authorization_code=1 - done - RESPONSE="$(curl --compressed "${CURL_PROGRESS}" -X POST \ - --data "code=${AUTHORIZATION_CODE}&client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&redirect_uri=${REDIRECT_URI}&grant_type=authorization_code" "${TOKEN_URL}")" || : - _clear_line 1 1>&2 - - REFRESH_TOKEN="$(printf "%s\n" "${RESPONSE}" | _json_value refresh_token 1 1 || :)" - { _get_access_token_and_update "${RESPONSE}" && _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}"; } || return 1 + SERVICE_ACCOUNT_ACCESS_TOKEN="$(eval printf "%s" \"\$"${SERVICE_ACCOUNT}_ACCESS_TOKEN"\")" + SERVICE_ACCOUNT_ACCESS_TOKEN_EXPIRY="$(eval printf "%s" \"\$"${SERVICE_ACCOUNT}_ACCESS_TOKEN_EXPIRY"\")" + + { [ -z "${SERVICE_ACCOUNT_ACCESS_TOKEN}" ] || ! printf "%s\n" "${SERVICE_ACCOUNT_ACCESS_TOKEN}" | grep -qE "${ACCESS_TOKEN_REGEX}" || [ "${SERVICE_ACCOUNT_ACCESS_TOKEN_EXPIRY:-0}" -lt "$(date +'%s')" ]; } && { + ASSERTION_DATA="$(_generate_jwt "${SERVICE_ACCOUNT_FILE}" "${SCOPE}")" || { printf "%s\n" "${ASSERTION_DATA}" 1>&2 && return 1; } + _get_access_token_and_update sa "${ASSERTION_DATA}" || return 1 } - printf "\n" - } - { [ -z "${ACCESS_TOKEN}" ] || ! printf "%s\n" "${ACCESS_TOKEN}" | grep -qE "${ACCESS_TOKEN_REGEX}" || [ "${ACCESS_TOKEN_EXPIRY:-0}" -lt "$(date +'%s')" ]; } && - { _get_access_token_and_update || return 1; } - printf "%b\n" "ACCESS_TOKEN=\"${ACCESS_TOKEN}\"\nACCESS_TOKEN_EXPIRY=\"${ACCESS_TOKEN_EXPIRY}\"" >| "${TMPFILE}_ACCESS_TOKEN" + printf "%s\n%s\n" "ACCESS_TOKEN=\"$(eval printf "%s" \"\$"${SERVICE_ACCOUNT}_ACCESS_TOKEN"\")"\" \ + "ACCESS_TOKEN_EXPIRY=\"$(eval printf "%s" \"\$"${SERVICE_ACCOUNT}_ACCESS_TOKEN_EXPIRY"\")"\" >| "${TMPFILE}_ACCESS_TOKEN" + fi # launch a background service to check access token and update it # checks ACCESS_TOKEN_EXPIRY, try to update before 5 mins of expiry, a fresh token gets 60 mins # process will be killed when script exits or "${MAIN_PID}" is killed { until ! kill -0 "${MAIN_PID}" 2>| /dev/null 1>&2; do + unset ASSERTION_DATA MODE . "${TMPFILE}_ACCESS_TOKEN" CURRENT_TIME="$(date +'%s')" REMAINING_TOKEN_TIME="$((CURRENT_TIME - ACCESS_TOKEN_EXPIRY))" if [ "${REMAINING_TOKEN_TIME}" -le 300 ]; then + [ -n "${SERVICE_ACCOUNT_FILE}" ] && { + ASSERTION_DATA="$(_generate_jwt "${SERVICE_ACCOUNT_FILE}" "${SCOPE}")" || : + MODE="sa" + } # timeout after 30 seconds, it shouldn't take too long anyway, and update tmp config - CONFIG="${TMPFILE}_ACCESS_TOKEN" _timeout 30 _get_access_token_and_update || : + SERVICE_ACCOUNT="" CONFIG="${TMPFILE}_ACCESS_TOKEN" _timeout 30 _get_access_token_and_update "${MODE:-normal}" "${ASSERTION_DATA:-}" || : else TOKEN_PROCESS_TIME_TO_SLEEP="$(if [ "${REMAINING_TOKEN_TIME}" -le 301 ]; then printf "0\n" @@ -418,7 +450,7 @@ _check_credentials() { fi sleep 1 done - } & + } 2>| /dev/null 1>&2 & ACCESS_TOKEN_SERVICE_PID="${!}" return 0 @@ -706,8 +738,8 @@ main() { [ -f "${TMPFILE}_ACCESS_TOKEN" ] && { . "${TMPFILE}_ACCESS_TOKEN" [ "${INITIAL_ACCESS_TOKEN}" = "${ACCESS_TOKEN}" ] || { - _update_config ACCESS_TOKEN "${ACCESS_TOKEN}" "${CONFIG}" - _update_config ACCESS_TOKEN_EXPIRY "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" + _update_config "${SERVICE_ACCOUNT:+${SERVICE_ACCOUNT}_}ACCESS_TOKEN" "${ACCESS_TOKEN}" "${CONFIG}" + _update_config "${SERVICE_ACCOUNT:+${SERVICE_ACCOUNT}_}ACCESS_TOKEN_EXPIRY" "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" } } 1>| /dev/null