From 3a3690acc4487d0b7224332ab362039ffdeb1f1d Mon Sep 17 00:00:00 2001 From: Simon Baars Date: Wed, 27 Sep 2023 14:38:58 +0100 Subject: [PATCH 01/10] Use a Garth OAuth implementation --- gcexport.py | 104 ++++++++++++----------------------------------- requirements.txt | 1 + 2 files changed, 26 insertions(+), 79 deletions(-) create mode 100644 requirements.txt diff --git a/gcexport.py b/gcexport.py index 8772615..3884407 100644 --- a/gcexport.py +++ b/gcexport.py @@ -34,6 +34,7 @@ import unicodedata import urllib.request import zipfile +import garth from datetime import datetime, timedelta, tzinfo from getpass import getpass from math import floor @@ -135,15 +136,15 @@ URL_GC_LOGIN = 'https://sso.garmin.com/sso/signin?' + urlencode(DATA) URL_GC_POST_AUTH = 'https://connect.garmin.com/modern/activities?' URL_GC_PROFILE = 'https://connect.garmin.com/modern/profile' -URL_GC_USERSTATS = 'https://connect.garmin.com/modern/proxy/userstats-service/statistics/' -URL_GC_LIST = 'https://connect.garmin.com/modern/proxy/activitylist-service/activities/search/activities?' -URL_GC_ACTIVITY = 'https://connect.garmin.com/modern/proxy/activity-service/activity/' -URL_GC_DEVICE = 'https://connect.garmin.com/modern/proxy/device-service/deviceservice/app-info/' -URL_GC_GEAR = 'https://connect.garmin.com/modern/proxy/gear-service/gear/filterGear?activityId=' +URL_GC_USERSTATS = 'https://connect.garmin.com/userstats-service/statistics/' +URL_GC_LIST = 'https://connect.garmin.com/activitylist-service/activities/search/activities?' +URL_GC_ACTIVITY = 'https://connect.garmin.com/activity-service/activity/' +URL_GC_DEVICE = 'https://connect.garmin.com/device-service/deviceservice/app-info/' +URL_GC_GEAR = 'https://connect.garmin.com/gear-service/gear/filterGear?activityId=' URL_GC_ACT_PROPS = 'https://connect.garmin.com/modern/main/js/properties/activity_types/activity_types.properties' URL_GC_EVT_PROPS = 'https://connect.garmin.com/modern/main/js/properties/event_types/event_types.properties' -URL_GC_GPX_ACTIVITY = 'https://connect.garmin.com/modern/proxy/download-service/export/gpx/activity/' -URL_GC_TCX_ACTIVITY = 'https://connect.garmin.com/modern/proxy/download-service/export/tcx/activity/' +URL_GC_GPX_ACTIVITY = 'https://connect.garmin.com/download-service/export/gpx/activity/' +URL_GC_TCX_ACTIVITY = 'https://connect.garmin.com/download-service/export/tcx/activity/' URL_GC_ORIGINAL_ACTIVITY = 'http://connect.garmin.com/proxy/download-service/files/activity/' @@ -232,6 +233,8 @@ def http_req(url, post=None, headers=None): 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2816.0 Safari/537.36', ) request.add_header('nk', 'NT') # necessary since 2021-02-23 to avoid http error code 402 + request.add_header('authorization', garth.client.oauth2_token.__str__()) + request.add_header('di-backend', 'connectapi.garmin.com') if headers: for header_key, header_value in headers.items(): request.add_header(header_key, header_value) @@ -513,52 +516,15 @@ def login_to_garmin_connect(args): username = args.username if args.username else input('Username: ') password = args.password if args.password else getpass() - logging.debug("Login params: %s", urlencode(DATA)) - - # Initially, we need to get a valid session cookie, so we pull the login page. - print('Connecting to Garmin Connect...', end='') - logging.info('Connecting to %s', URL_GC_LOGIN) - connect_response = http_req_as_string(URL_GC_LOGIN) - if args.verbosity > 0: - write_to_file(os.path.join(args.directory, 'connect_response.html'), connect_response, 'w') - for cookie in COOKIE_JAR: - logging.debug("Cookie %s : %s", cookie.name, cookie.value) + print('Authenticating using OAuth...', end=' ') + garth.login(username, password) print(' Done.') - # Now we'll actually login. - # Fields that are passed in a typical Garmin login. - post_data = { - 'username': username, - 'password': password, - 'embed': 'false', - 'rememberme': 'on', - } - - headers = {'referer': URL_GC_LOGIN} - - print('Requesting Login ticket...', end='') - logging.info('Requesting Login ticket') - login_response = http_req_as_string(f'{URL_GC_LOGIN}#', post_data, headers) - - for cookie in COOKIE_JAR: - logging.debug("Cookie %s : %s", cookie.name, cookie.value) - if args.verbosity > 0: - write_to_file(os.path.join(args.directory, 'login_response.html'), login_response, 'w') - - # extract the ticket from the login response - pattern = re.compile(r".*\?ticket=([-\w]+)\";.*", re.MULTILINE | re.DOTALL) - match = pattern.match(login_response) - if not match: - raise GarminException( - 'Couldn\'t find ticket in the login response. Cannot log in. Did you enter the correct username and password?' - ) - login_ticket = match.group(1) - print(' Done. Ticket=', login_ticket, sep='') + token = garth.client.oauth2_token + if not token: + raise Exception("Could not get token") - print("Authenticating...", end='') - logging.info('Authentication URL %s', f'{URL_GC_POST_AUTH}ticket={login_ticket}') - http_req(f'{URL_GC_POST_AUTH}ticket={login_ticket}') - print(' Done.') + return garth.client.oauth2_token def csv_write_record(csv_filter, extract, actvty, details, activity_type_name, event_type_name): @@ -979,7 +945,7 @@ def extract_display_name(profile_page): return display_name -def fetch_activity_list(args, total_to_download): +def fetch_activity_list(args): """ Fetch the first 'total_to_download' activity summaries; as a side effect save them in json format. :param args: command-line arguments (for args.directory etc) @@ -987,26 +953,15 @@ def fetch_activity_list(args, total_to_download): :return: List of activity summaries """ - # This while loop will download data from the server in multiple chunks, if necessary. activities = [] - - total_downloaded = 0 - while total_downloaded < total_to_download: - # Maximum chunk size 'LIMIT_MAXIMUM' ... 400 return status if over maximum. So download - # maximum or whatever remains if less than maximum. - # As of 2018-03-06 I get return status 500 if over maximum - if total_to_download - total_downloaded > LIMIT_MAXIMUM: - num_to_download = LIMIT_MAXIMUM - else: - num_to_download = total_to_download - total_downloaded - - chunk = fetch_activity_chunk(args, num_to_download, total_downloaded) + # This while loop will download data from the server in multiple chunks, if necessary. + # The absolute limit for activities is 10.000 on server side + while len(activities) <= 10000 - LIMIT_MAXIMUM: + chunk = fetch_activity_chunk(args, LIMIT_MAXIMUM, len(activities)) activities.extend(chunk) - total_downloaded += num_to_download + if len(chunk) != LIMIT_MAXIMUM: + break - # it seems that parent multisport activities are not counted in userstats - if len(activities) != total_to_download: - logging.info('Expected %s activities, got %s.', total_to_download, len(activities)) return activities @@ -1294,16 +1249,7 @@ def main(argv): else: os.mkdir(args.directory) - login_to_garmin_connect(args) - - # Query the userstats (activities totals on the profile page). Needed for - # filtering and for downloading 'all' to know how many activities are available - userstats_json = fetch_userstats(args) - - if args.count == 'all': - total_to_download = int(userstats_json['userMetrics'][0]['totalActivities']) - else: - total_to_download = int(args.count) + login_to_garmin_connect(args).__str__() device_dict = {} @@ -1317,7 +1263,7 @@ def main(argv): write_to_file(os.path.join(args.directory, 'event_types.properties'), event_type_props, 'w') event_type_name = load_properties(event_type_props) - activities = fetch_activity_list(args, total_to_download) + activities = fetch_activity_list(args) action_list = annotate_activity_list(activities, args.start_activity_no, exclude_list) csv_filename = os.path.join(args.directory, 'activities.csv') diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..fe7f630 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +garth==0.4.28 From 0cddb3dbfa4c961ab23cfa37268e3b167efcb405 Mon Sep 17 00:00:00 2001 From: Simon Baars Date: Wed, 27 Sep 2023 14:48:53 +0100 Subject: [PATCH 02/10] Improve Garth error handling --- gcexport.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/gcexport.py b/gcexport.py index 3884407..fac0a6e 100644 --- a/gcexport.py +++ b/gcexport.py @@ -517,13 +517,11 @@ def login_to_garmin_connect(args): password = args.password if args.password else getpass() print('Authenticating using OAuth...', end=' ') - garth.login(username, password) + try: + garth.login(username, password) + except Exception as ex: + raise GarminException(f'Authentication failure ({ex}). Did you enter correct credentials?') print(' Done.') - - token = garth.client.oauth2_token - if not token: - raise Exception("Could not get token") - return garth.client.oauth2_token From ce8084e1aee1c9abec81edfb9b5476db90203ab0 Mon Sep 17 00:00:00 2001 From: Simon Baars Date: Wed, 27 Sep 2023 14:50:01 +0100 Subject: [PATCH 03/10] Elaborate on installation instructions --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a37241e..1f9aec8 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ If you have many activities, you may find that this script crashes with an "Oper - If you're comfortable using Git, just clone the repo from github - Otherwise get the latest `zip` (or `tar.gz`) from the [releases page](https://github.com/pe-st/garmin-connect-export/releases) and unpack it where it suits you. +- Install the dependencies: `pip install -r requirements.txt` ## Usage From e69e6f0ffef2ca006b334ab90fb1dc9eda1d4d2e Mon Sep 17 00:00:00 2001 From: Simon Baars Date: Wed, 27 Sep 2023 16:53:22 +0100 Subject: [PATCH 04/10] Enforce activity limit --- gcexport.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/gcexport.py b/gcexport.py index fac0a6e..f601c5d 100644 --- a/gcexport.py +++ b/gcexport.py @@ -89,7 +89,9 @@ # Maximum number of activities you can request at once. # Used to be 100 and enforced by Garmin for older endpoints; for the current endpoint 'URL_GC_LIST' # the limit is not known (I have less than 1000 activities and could get them all in one go) -LIMIT_MAXIMUM = 1000 +SINGLE_REQUEST_LIMIT = 1000 +# The absolute limit for activities is 10.000 on server side +TOTAL_ACTIVITY_LIMIT = 10_000 MAX_TRIES = 3 @@ -947,17 +949,22 @@ def fetch_activity_list(args): """ Fetch the first 'total_to_download' activity summaries; as a side effect save them in json format. :param args: command-line arguments (for args.directory etc) - :param total_to_download: number of activities to download :return: List of activity summaries """ + if args.count == 'all': + total_to_download = TOTAL_ACTIVITY_LIMIT + else: + total_to_download = min(int(args.count), TOTAL_ACTIVITY_LIMIT) + activities = [] # This while loop will download data from the server in multiple chunks, if necessary. - # The absolute limit for activities is 10.000 on server side - while len(activities) <= 10000 - LIMIT_MAXIMUM: - chunk = fetch_activity_chunk(args, LIMIT_MAXIMUM, len(activities)) + while len(activities) <= TOTAL_ACTIVITY_LIMIT - SINGLE_REQUEST_LIMIT: + num_downloaded = len(activities) + num_to_download = min(SINGLE_REQUEST_LIMIT, total_to_download - num_downloaded) + chunk = fetch_activity_chunk(args, num_to_download, num_downloaded) activities.extend(chunk) - if len(chunk) != LIMIT_MAXIMUM: + if len(chunk) != SINGLE_REQUEST_LIMIT: break return activities From e8b914a914b96ae6fab51eb26de3b92db0051e90 Mon Sep 17 00:00:00 2001 From: Simon Baars Date: Thu, 28 Sep 2023 10:52:14 +0100 Subject: [PATCH 05/10] Revive userstats --- gcexport.py | 93 ++++++++++++----------------------------------------- 1 file changed, 21 insertions(+), 72 deletions(-) diff --git a/gcexport.py b/gcexport.py index f601c5d..6e14108 100644 --- a/gcexport.py +++ b/gcexport.py @@ -97,57 +97,20 @@ CSV_TEMPLATE = os.path.join(os.path.dirname(os.path.realpath(__file__)), "csv_header_default.properties") -WEBHOST = "https://connect.garmin.com" -REDIRECT = "https://connect.garmin.com/modern/" -BASE_URL = "https://connect.garmin.com/en-US/signin" -SSO = "https://sso.garmin.com/sso" -CSS = "https://static.garmincdn.com/com.garmin.connect/ui/css/gauth-custom-v1.2-min.css" - -DATA = { - 'service': REDIRECT, - 'webhost': WEBHOST, - 'source': BASE_URL, - 'redirectAfterAccountLoginUrl': REDIRECT, - 'redirectAfterAccountCreationUrl': REDIRECT, - 'gauthHost': SSO, - 'locale': 'en_US', - 'id': 'gauth-widget', - 'cssUrl': CSS, - 'clientId': 'GarminConnect', - 'rememberMeShown': 'true', - 'rememberMeChecked': 'false', - 'createAccountShown': 'true', - 'openCreateAccount': 'false', - 'displayNameShown': 'false', - 'consumeServiceTicket': 'false', - 'initialFocus': 'true', - 'embedWidget': 'false', - 'generateExtraServiceTicket': 'true', - 'generateTwoExtraServiceTickets': 'false', - 'generateNoServiceTicket': 'false', - 'globalOptInShown': 'true', - 'globalOptInChecked': 'false', - 'mobile': 'false', - 'connectLegalTerms': 'true', - 'locationPromptShown': 'true', - 'showPassword': 'true', -} +GARMIN_BASE_URL = "https://connect.garmin.com" # URLs for various services. - -URL_GC_LOGIN = 'https://sso.garmin.com/sso/signin?' + urlencode(DATA) -URL_GC_POST_AUTH = 'https://connect.garmin.com/modern/activities?' -URL_GC_PROFILE = 'https://connect.garmin.com/modern/profile' -URL_GC_USERSTATS = 'https://connect.garmin.com/userstats-service/statistics/' -URL_GC_LIST = 'https://connect.garmin.com/activitylist-service/activities/search/activities?' -URL_GC_ACTIVITY = 'https://connect.garmin.com/activity-service/activity/' -URL_GC_DEVICE = 'https://connect.garmin.com/device-service/deviceservice/app-info/' -URL_GC_GEAR = 'https://connect.garmin.com/gear-service/gear/filterGear?activityId=' -URL_GC_ACT_PROPS = 'https://connect.garmin.com/modern/main/js/properties/activity_types/activity_types.properties' -URL_GC_EVT_PROPS = 'https://connect.garmin.com/modern/main/js/properties/event_types/event_types.properties' -URL_GC_GPX_ACTIVITY = 'https://connect.garmin.com/download-service/export/gpx/activity/' -URL_GC_TCX_ACTIVITY = 'https://connect.garmin.com/download-service/export/tcx/activity/' -URL_GC_ORIGINAL_ACTIVITY = 'http://connect.garmin.com/proxy/download-service/files/activity/' +URL_GC_USER = f'{GARMIN_BASE_URL}/userprofile-service/socialProfile' +URL_GC_USERSTATS = f'{GARMIN_BASE_URL}/userstats-service/statistics/' +URL_GC_LIST = f'{GARMIN_BASE_URL}/activitylist-service/activities/search/activities?' +URL_GC_ACTIVITY = f'{GARMIN_BASE_URL}/activity-service/activity/' +URL_GC_DEVICE = f'{GARMIN_BASE_URL}/device-service/deviceservice/app-info/' +URL_GC_GEAR = f'{GARMIN_BASE_URL}/gear-service/gear/filterGear?activityId=' +URL_GC_ACT_PROPS = f'{GARMIN_BASE_URL}/modern/main/js/properties/activity_types/activity_types.properties' +URL_GC_EVT_PROPS = f'{GARMIN_BASE_URL}/modern/main/js/properties/event_types/event_types.properties' +URL_GC_GPX_ACTIVITY = f'{GARMIN_BASE_URL}/download-service/export/gpx/activity/' +URL_GC_TCX_ACTIVITY = f'{GARMIN_BASE_URL}/download-service/export/tcx/activity/' +URL_GC_ORIGINAL_ACTIVITY = f'{GARMIN_BASE_URL}/download-service/files/activity/' class GarminException(Exception): @@ -548,7 +511,7 @@ def csv_write_record(csv_filter, extract, actvty, details, activity_type_name, e # fmt: off csv_filter.set_column('id', str(actvty['activityId'])) - csv_filter.set_column('url', 'https://connect.garmin.com/modern/activity/' + str(actvty['activityId'])) + csv_filter.set_column('url', f'{GARMIN_BASE_URL}/modern/activity/' + str(actvty['activityId'])) csv_filter.set_column('activityName', actvty['activityName'] if present('activityName', actvty) else None) csv_filter.set_column('description', actvty['description'] if present('description', actvty) else None) csv_filter.set_column('startTimeIso', extract['start_time_with_offset'].isoformat()) @@ -910,12 +873,12 @@ def fetch_userstats(args): :return: json with user statistics """ print('Getting display name...', end='') - logging.info('Profile page %s', URL_GC_PROFILE) - profile_page = http_req_as_string(URL_GC_PROFILE) + logging.info('Profile page %s', URL_GC_USER) + profile_page = http_req_as_string(URL_GC_USER) if args.verbosity > 0: - write_to_file(os.path.join(args.directory, 'profile.html'), profile_page, 'w') + write_to_file(os.path.join(args.directory, 'user.json'), profile_page, 'w') - display_name = extract_display_name(profile_page) + display_name = json.loads(profile_page)['displayName'] print(' Done. displayName=', display_name, sep='') print('Fetching user stats...', end='') @@ -929,22 +892,6 @@ def fetch_userstats(args): return json.loads(result) -def extract_display_name(profile_page): - """ - Extract the display name from the profile page HTML document - :param profile_page: HTML document - :return: the display name - """ - # the display name should be in the HTML document as - # "displayName":"John.Doe" - pattern = re.compile(r".*\"displayName\":\"(.+?)\".*", re.MULTILINE | re.DOTALL) - match = pattern.match(profile_page) - if not match: - raise GarminException('Did not find the display name in the profile page.') - display_name = match.group(1) - return display_name - - def fetch_activity_list(args): """ Fetch the first 'total_to_download' activity summaries; as a side effect save them in json format. @@ -1256,9 +1203,10 @@ def main(argv): login_to_garmin_connect(args).__str__() - device_dict = {} + # Get user stats + fetch_userstats(args) - # load some dictionaries with lookup data from REST services + # Load some dictionaries with lookup data from REST services activity_type_props = http_req_as_string(URL_GC_ACT_PROPS) if args.verbosity > 0: write_to_file(os.path.join(args.directory, 'activity_types.properties'), activity_type_props, 'w') @@ -1274,6 +1222,7 @@ def main(argv): csv_filename = os.path.join(args.directory, 'activities.csv') csv_existed = os.path.isfile(csv_filename) + device_dict = {} with open(csv_filename, mode='a', encoding='utf-8') as csv_file: csv_filter = CsvFilter(csv_file, args.template) From e94d05273f629c00a4cc8e60d1b67574d33c4645 Mon Sep 17 00:00:00 2001 From: Simon Baars Date: Thu, 28 Sep 2023 11:53:37 +0100 Subject: [PATCH 06/10] Address pylint issues --- gcexport.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/gcexport.py b/gcexport.py index 6e14108..dc31e2a 100644 --- a/gcexport.py +++ b/gcexport.py @@ -34,7 +34,6 @@ import unicodedata import urllib.request import zipfile -import garth from datetime import datetime, timedelta, tzinfo from getpass import getpass from math import floor @@ -44,6 +43,7 @@ from urllib.error import HTTPError, URLError from urllib.parse import urlencode from urllib.request import Request +import garth # Local application/library specific imports from filtering import read_exclude, update_download_stats @@ -198,7 +198,7 @@ def http_req(url, post=None, headers=None): 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2816.0 Safari/537.36', ) request.add_header('nk', 'NT') # necessary since 2021-02-23 to avoid http error code 402 - request.add_header('authorization', garth.client.oauth2_token.__str__()) + request.add_header('authorization', str(garth.client.oauth2_token)) request.add_header('di-backend', 'connectapi.garmin.com') if headers: for header_key, header_value in headers.items(): @@ -485,9 +485,8 @@ def login_to_garmin_connect(args): try: garth.login(username, password) except Exception as ex: - raise GarminException(f'Authentication failure ({ex}). Did you enter correct credentials?') + raise GarminException(f'Authentication failure ({ex}). Did you enter correct credentials?') from ex print(' Done.') - return garth.client.oauth2_token def csv_write_record(csv_filter, extract, actvty, details, activity_type_name, event_type_name): @@ -1201,7 +1200,7 @@ def main(argv): else: os.mkdir(args.directory) - login_to_garmin_connect(args).__str__() + login_to_garmin_connect(args) # Get user stats fetch_userstats(args) From 6b7881e241fdc49277f19e2e80dd73ebb09d50e0 Mon Sep 17 00:00:00 2001 From: Simon Baars Date: Wed, 4 Oct 2023 22:02:10 +0100 Subject: [PATCH 07/10] Use any 0.4 version of garth --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fe7f630..e410a65 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -garth==0.4.28 +garth>=0.4.0,<0.5.0 From 068deadf1e9a7a2116e1364a3d8e06087a9d6201 Mon Sep 17 00:00:00 2001 From: Simon Baars Date: Wed, 4 Oct 2023 22:02:36 +0100 Subject: [PATCH 08/10] Update `README` with python3 pip version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1f9aec8..8fe3266 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ If you have many activities, you may find that this script crashes with an "Oper - If you're comfortable using Git, just clone the repo from github - Otherwise get the latest `zip` (or `tar.gz`) from the [releases page](https://github.com/pe-st/garmin-connect-export/releases) and unpack it where it suits you. -- Install the dependencies: `pip install -r requirements.txt` +- Install the dependencies: `python3 -m pip install -r requirements.txt` ## Usage From e93ffeba8c135891dffba5943d4113644291198c Mon Sep 17 00:00:00 2001 From: Simon Baars Date: Wed, 4 Oct 2023 22:09:46 +0100 Subject: [PATCH 09/10] Revert fetch_activity_list to old version --- gcexport.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/gcexport.py b/gcexport.py index dc31e2a..57f7dd5 100644 --- a/gcexport.py +++ b/gcexport.py @@ -89,9 +89,7 @@ # Maximum number of activities you can request at once. # Used to be 100 and enforced by Garmin for older endpoints; for the current endpoint 'URL_GC_LIST' # the limit is not known (I have less than 1000 activities and could get them all in one go) -SINGLE_REQUEST_LIMIT = 1000 -# The absolute limit for activities is 10.000 on server side -TOTAL_ACTIVITY_LIMIT = 10_000 +LIMIT_MAXIMUM = 1000 MAX_TRIES = 3 @@ -891,28 +889,34 @@ def fetch_userstats(args): return json.loads(result) -def fetch_activity_list(args): +def fetch_activity_list(args, total_to_download): """ Fetch the first 'total_to_download' activity summaries; as a side effect save them in json format. :param args: command-line arguments (for args.directory etc) + :param total_to_download: number of activities to download :return: List of activity summaries """ - if args.count == 'all': - total_to_download = TOTAL_ACTIVITY_LIMIT - else: - total_to_download = min(int(args.count), TOTAL_ACTIVITY_LIMIT) - - activities = [] # This while loop will download data from the server in multiple chunks, if necessary. - while len(activities) <= TOTAL_ACTIVITY_LIMIT - SINGLE_REQUEST_LIMIT: - num_downloaded = len(activities) - num_to_download = min(SINGLE_REQUEST_LIMIT, total_to_download - num_downloaded) - chunk = fetch_activity_chunk(args, num_to_download, num_downloaded) + activities = [] + + total_downloaded = 0 + while total_downloaded < total_to_download: + # Maximum chunk size 'LIMIT_MAXIMUM' ... 400 return status if over maximum. So download + # maximum or whatever remains if less than maximum. + # As of 2018-03-06 I get return status 500 if over maximum + if total_to_download - total_downloaded > LIMIT_MAXIMUM: + num_to_download = LIMIT_MAXIMUM + else: + num_to_download = total_to_download - total_downloaded + + chunk = fetch_activity_chunk(args, num_to_download, total_downloaded) activities.extend(chunk) - if len(chunk) != SINGLE_REQUEST_LIMIT: - break + total_downloaded += num_to_download + # it seems that parent multisport activities are not counted in userstats + if len(activities) != total_to_download: + logging.info('Expected %s activities, got %s.', total_to_download, len(activities)) return activities @@ -1203,7 +1207,7 @@ def main(argv): login_to_garmin_connect(args) # Get user stats - fetch_userstats(args) + userstats = fetch_userstats(args) # Load some dictionaries with lookup data from REST services activity_type_props = http_req_as_string(URL_GC_ACT_PROPS) @@ -1215,7 +1219,7 @@ def main(argv): write_to_file(os.path.join(args.directory, 'event_types.properties'), event_type_props, 'w') event_type_name = load_properties(event_type_props) - activities = fetch_activity_list(args) + activities = fetch_activity_list(args, userstats['userMetrics'][0]['totalActivities']) action_list = annotate_activity_list(activities, args.start_activity_no, exclude_list) csv_filename = os.path.join(args.directory, 'activities.csv') From c3f7cbc7cbcd0c6fec727ddf14b095a247e2c9c9 Mon Sep 17 00:00:00 2001 From: Simon Baars Date: Wed, 4 Oct 2023 22:14:11 +0100 Subject: [PATCH 10/10] Remove extract display name test --- gcexport_test.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/gcexport_test.py b/gcexport_test.py index 656a59d..c6c5064 100644 --- a/gcexport_test.py +++ b/gcexport_test.py @@ -174,23 +174,6 @@ def test_load_zones(): assert 2462.848 == zones[0]['secsInZone'] -def test_extract_display_name(): - with open('html/profile_simple.html') as html: - profile_page = html.read() - assert 'John.Doe' == extract_display_name(profile_page) - - # some users reported (issue #65) to have an email address as display name - with open('html/profile_email.html') as html: - profile_page = html.read() - assert 'john.doe@email.org' == extract_display_name(profile_page) - - # some users reported to have a UUID as display name: - # https://github.com/moderation/garmin-connect-export/issues/31 - with open('html/profile_uuid.html') as html: - profile_page = html.read() - assert '36e29d65-715c-456b-9115-84f0b9a0c0ba' == extract_display_name(profile_page) - - def test_resolve_path(): assert resolve_path('root', 'sub/{YYYY}', '2018-03-08 12:23:22') == 'root/sub/2018' assert resolve_path('root', 'sub/{MM}', '2018-03-08 12:23:22') == 'root/sub/03'