Skip to content

Commit

Permalink
initial, partial changes to enable different cmr and edl maturity levels
Browse files Browse the repository at this point in the history
  • Loading branch information
@danielfromearth
danielfromearth committed Jan 11, 2024
1 parent a48df1a commit 9a1c938
Show file tree
Hide file tree
Showing 4 changed files with 81 additions and 34 deletions.
32 changes: 20 additions & 12 deletions earthaccess/api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import earthaccess

from .auth import Auth
from .auth import Auth, Maturity
from .results import DataGranule
from .search import CollectionQuery, DataCollections, DataGranules, GranuleQuery
from .store import Store
Expand Down Expand Up @@ -71,9 +71,11 @@ def search_datasets(
)
return []
if earthaccess.__auth__.authenticated:
query = DataCollections(auth=earthaccess.__auth__).parameters(**kwargs)
query = DataCollections(auth=earthaccess.__auth__,
cmr_and_edl_maturity=earthaccess.__auth__.cmr_and_edl_maturity).parameters(**kwargs)
else:
query = DataCollections().parameters(**kwargs)
query = DataCollections(
cmr_and_edl_maturity=earthaccess.__auth__.cmr_and_edl_maturity).parameters(**kwargs)
datasets_found = query.hits()
print(f"Datasets found: {datasets_found}")
if count > 0:
Expand Down Expand Up @@ -120,17 +122,19 @@ def search_data(
```
"""
if earthaccess.__auth__.authenticated:
query = DataGranules(earthaccess.__auth__).parameters(**kwargs)
query = DataGranules(earthaccess.__auth__,
cmr_and_edl_maturity=earthaccess.__auth__.cmr_and_edl_maturity).parameters(**kwargs)
else:
query = DataGranules().parameters(**kwargs)
query = DataGranules(
cmr_and_edl_maturity=earthaccess.__auth__.cmr_and_edl_maturity).parameters(**kwargs)
granules_found = query.hits()
print(f"Granules found: {granules_found}")
if count > 0:
return query.get(count)
return query.get_all()


def login(strategy: str = "all", persist: bool = False) -> Auth:
def login(strategy: str = "all", persist: bool = False, cmr_and_edl_maturity=Maturity.PROD) -> Auth:
"""Authenticate with Earthdata login (https://urs.earthdata.nasa.gov/)
Parameters:
Expand All @@ -145,21 +149,22 @@ def login(strategy: str = "all", persist: bool = False) -> Auth:
"environment": retrieve username and password from $EARTHDATA_USERNAME and $EARTHDATA_PASSWORD.
persist (Boolean): will persist credentials in a .netrc file
cmr_and_edl_maturity (Maturity): the CMR endpoint to log in to Earthdata, defaults to PROD
Returns:
an instance of Auth.
"""
if strategy == "all":
for strategy in ["environment", "netrc", "interactive"]:
try:
earthaccess.__auth__.login(strategy=strategy, persist=persist)
earthaccess.__auth__.login(strategy=strategy, persist=persist, cmr_and_edl_maturity=cmr_and_edl_maturity)
except Exception:
pass

if earthaccess.__auth__.authenticated:
earthaccess.__store__ = Store(earthaccess.__auth__)
break
else:
earthaccess.__auth__.login(strategy=strategy, persist=persist)
earthaccess.__auth__.login(strategy=strategy, persist=persist, cmr_and_edl_maturity=cmr_and_edl_maturity)
if earthaccess.__auth__.authenticated:
earthaccess.__store__ = Store(earthaccess.__auth__)

Expand Down Expand Up @@ -252,9 +257,11 @@ def collection_query() -> Type[CollectionQuery]:
class earthaccess.DataCollections: a query builder instance for data collections.
"""
if earthaccess.__auth__.authenticated:
query_builder = DataCollections(earthaccess.__auth__)
query_builder = DataCollections(earthaccess.__auth__,
cmr_and_edl_maturity=earthaccess.__auth__.cmr_and_edl_maturity)
else:
query_builder = DataCollections()
query_builder = DataCollections(
cmr_and_edl_maturity=earthaccess.__auth__.cmr_and_edl_maturity)
return query_builder


Expand All @@ -268,9 +275,10 @@ def granule_query() -> Type[GranuleQuery]:
class earthaccess.DataGranules: a query builder instance for data granules.
"""
if earthaccess.__auth__.authenticated:
query_builder = DataGranules(earthaccess.__auth__)
query_builder = DataGranules(earthaccess.__auth__,
cmr_and_edl_maturity=earthaccess.__auth__.cmr_and_edl_maturity)
else:
query_builder = DataGranules()
query_builder = DataGranules(cmr_and_edl_maturity=earthaccess.__auth__.cmr_and_edl_maturity)
return query_builder


Expand Down
48 changes: 30 additions & 18 deletions earthaccess/auth.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,15 +63,12 @@ class Auth(object):
"""

def __init__(self) -> None:
# Maybe all these predefined URLs should be in a constants.py file
self.authenticated = False
self.tokens: List = []
self.EDL_GET_TOKENS_URL = "https://urs.earthdata.nasa.gov/api/users/tokens"
self.EDL_GET_PROFILE = "https://urs.earthdata.nasa.gov/api/users/<USERNAME>?client_id=ntD0YGC_SM3Bjs-Tnxd7bg"
self.EDL_GENERATE_TOKENS_URL = "https://urs.earthdata.nasa.gov/api/users/token"
self.EDL_REVOKE_TOKEN = "https://urs.earthdata.nasa.gov/api/users/revoke_token"
self._set_cmr_and_edl_maturity(Maturity.PROD)

def login(self, strategy: str = "netrc", persist: bool = False) -> Any:
def login(self, strategy: str = "netrc", persist: bool = False,
cmr_and_edl_maturity: Optional[Maturity] = None) -> Any:
"""Authenticate with Earthdata login
Parameters:
Expand All @@ -84,10 +81,11 @@ def login(self, strategy: str = "netrc", persist: bool = False) -> Any:
"environment": retrieve username and password from $EARTHDATA_USERNAME and $EARTHDATA_PASSWORD.
persist (Boolean): will persist credentials in a .netrc file
cmr_and_edl_maturity (Maturity): the CMR endpoint to log in to Earthdata, defaults to PROD
Returns:
an instance of Auth.
"""
if self.authenticated:
if self.authenticated and (cmr_and_edl_maturity == self.cmr_and_edl_maturity):
logger.debug("We are already authenticated with NASA EDL")
return self
if strategy == "interactive":
Expand All @@ -96,8 +94,24 @@ def login(self, strategy: str = "netrc", persist: bool = False) -> Any:
self._netrc()
if strategy == "environment":
self._environment()

if cmr_and_edl_maturity is not None:
self._set_cmr_and_edl_maturity(cmr_and_edl_maturity)

return self

def _set_cmr_and_edl_maturity(self, cmr_and_edl_maturity: Maturity) -> None:
self.cmr_and_edl_maturity = cmr_and_edl_maturity

# Maybe all these predefined URLs should be in a constants.py file
self.EDL_GET_TOKENS_URL = f"https://{self.cmr_and_edl_maturity.value}/api/users/tokens"
self.EDL_GET_PROFILE = f"https://{self.cmr_and_edl_maturity.value}/api/users/<USERNAME>?client_id=ntD0YGC_SM3Bjs-Tnxd7bg"
self.EDL_GENERATE_TOKENS_URL = f"https://{self.cmr_and_edl_maturity.value}/api/users/token"
self.EDL_REVOKE_TOKEN = f"https://{self.cmr_and_edl_maturity.value}/api/users/revoke_token"

self._eula_url = f"https://{self.cmr_and_edl_maturity.value}/users/earthaccess/unaccepted_eulas"
self._apps_url = f"https://{self.cmr_and_edl_maturity.value}/application_search"

def refresh_tokens(self) -> bool:
"""Refresh CMR tokens
Tokens are used to do authenticated queries on CMR for restricted and early access datastes
Expand Down Expand Up @@ -168,7 +182,7 @@ def get_s3_credentials(
"""
if self.authenticated:
session = SessionWithHeaderRedirection(self.username, self.password)
session = SessionWithHeaderRedirection(self.username, self.password, self.cmr_and_edl_maturity)
if endpoint is None:
auth_url = self._get_cloud_auth_url(
daac_shortname=daac, provider=provider
Expand All @@ -193,10 +207,8 @@ def get_s3_credentials(
print(
f"Authentication with Earthdata Login failed with:\n{auth_resp.text[0:1000]}"
)
eula_url = "https://urs.earthdata.nasa.gov/users/earthaccess/unaccepted_eulas"
apps_url = "https://urs.earthdata.nasa.gov/application_search"
print(
f"Consider accepting the EULAs available at {eula_url} and applications at {apps_url}"
f"Consider accepting the EULAs available at {self._eula_url} and applications at {self._apps_url}"
)
return {}

Expand Down Expand Up @@ -257,9 +269,9 @@ def _netrc(self) -> bool:
) from err
except NetrcParseError as err:
raise NetrcParseError("Unable to parse .netrc") from err
if my_netrc["urs.earthdata.nasa.gov"] is not None:
username = my_netrc["urs.earthdata.nasa.gov"]["login"]
password = my_netrc["urs.earthdata.nasa.gov"]["password"]
if my_netrc[self.cmr_and_edl_maturity.value] is not None:
username = my_netrc[self.cmr_and_edl_maturity.value]["login"]
password = my_netrc[self.cmr_and_edl_maturity.value]["password"]
else:
return False
authenticated = self._get_credentials(username, password)
Expand Down Expand Up @@ -319,7 +331,7 @@ def _get_credentials(
return self.authenticated

def _get_user_tokens(self, username: str, password: str) -> Any:
session = SessionWithHeaderRedirection(username, password)
session = SessionWithHeaderRedirection(username, password, self.cmr_and_edl_maturity)
auth_resp = session.get(
self.EDL_GET_TOKENS_URL,
headers={
Expand All @@ -330,7 +342,7 @@ def _get_user_tokens(self, username: str, password: str) -> Any:
return auth_resp

def _generate_user_token(self, username: str, password: str) -> Any:
session = SessionWithHeaderRedirection(username, password)
session = SessionWithHeaderRedirection(username, password, self.cmr_and_edl_maturity)
auth_resp = session.post(
self.EDL_GENERATE_TOKENS_URL,
headers={
Expand All @@ -342,7 +354,7 @@ def _generate_user_token(self, username: str, password: str) -> Any:

def _revoke_user_token(self, token: str) -> bool:
if self.authenticated:
session = SessionWithHeaderRedirection(self.username, self.password)
session = SessionWithHeaderRedirection(self.username, self.password, self.cmr_and_edl_maturity)
auth_resp = session.post(
self.EDL_REVOKE_TOKEN,
params={"token": token},
Expand All @@ -365,7 +377,7 @@ def _persist_user_credentials(self, username: str, password: str) -> bool:
print(e)
return False
my_netrc = Netrc(str(netrc_path))
my_netrc["urs.earthdata.nasa.gov"] = {"login": username, "password": password}
my_netrc[self.cmr_and_edl_maturity.value] = {"login": username, "password": password}
my_netrc.save()
return True

Expand Down
33 changes: 30 additions & 3 deletions earthaccess/search.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@

import dateutil.parser as parser # type: ignore
from cmr import CollectionQuery, GranuleQuery # type: ignore
from cmr import CMR_OPS, CMR_UAT, CMR_SIT
from requests import exceptions, session

from .auth import Auth
from .auth import Auth, Maturity
from .daac import find_provider, find_provider_by_shortname
from .results import DataCollection, DataGranule

Expand All @@ -33,7 +34,7 @@ class DataCollections(CollectionQuery):
"umm_json",
]

def __init__(self, auth: Optional[Auth] = None, *args: Any, **kwargs: Any) -> None:
def __init__(self, auth: Optional[Auth] = None, cmr_and_edl_maturity: Optional[Maturity] = None, *args: Any, **kwargs: Any) -> None:
"""Builds an instance of DataCollections to query CMR
Parameters:
Expand All @@ -42,6 +43,19 @@ def __init__(self, auth: Optional[Auth] = None, *args: Any, **kwargs: Any) -> No
"""
super().__init__(*args, **kwargs)
self.session = session()
if auth is not None:
cmr_and_edl_maturity = auth.cmr_and_edl_maturity

if (cmr_and_edl_maturity is None) or (cmr_and_edl_maturity == Maturity.PROD):
self.mode(CMR_OPS)
elif cmr_and_edl_maturity == Maturity.UAT:
self.mode(CMR_UAT)
elif cmr_and_edl_maturity == Maturity.SIT:
self.mode(CMR_SIT)

print(f"[in DataCollections] CMR and EDL maturity: {cmr_and_edl_maturity}")
print(f"[in DataCollections] cmr_and_edl_maturity == Maturity.PROD -----> {cmr_and_edl_maturity == Maturity.PROD}")

if auth is not None and auth.authenticated:
# To search we need the new bearer tokens from NASA Earthdata
self.session = auth.get_session(bearer_token=True)
Expand Down Expand Up @@ -311,10 +325,23 @@ class DataGranules(GranuleQuery):
"umm_json",
]

def __init__(self, auth: Any = None, *args: Any, **kwargs: Any) -> None:
def __init__(self, auth: Any = None, cmr_and_edl_maturity: Optional[Maturity] = None, *args: Any, **kwargs: Any) -> None:
"""Base class for Granule and Collection CMR queries."""
super().__init__(*args, **kwargs)
self.session = session()
if auth is not None:
cmr_and_edl_maturity = auth.cmr_and_edl_maturity

if (cmr_and_edl_maturity is None) or (cmr_and_edl_maturity == Maturity.PROD):
self.mode(CMR_OPS)
elif cmr_and_edl_maturity == Maturity.UAT:
self.mode(CMR_UAT)
elif cmr_and_edl_maturity == Maturity.SIT:
self.mode(CMR_SIT)

print(f"[in DataGranules] CMR and EDL maturity: {cmr_and_edl_maturity}")
print(f"[in DataGranules] cmr_and_edl_maturity == Maturity.PROD -----> {cmr_and_edl_maturity == Maturity.PROD}")

if auth is not None and auth.authenticated:
# To search we need the new bearer tokens from NASA Earthdata
self.session = auth.get_session(bearer_token=True)
Expand Down
2 changes: 1 addition & 1 deletion earthaccess/store.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ def __init__(self, auth: Any, pre_authorize: bool = False) -> None:
self._s3_credentials: Dict[
Tuple, Tuple[datetime.datetime, Dict[str, str]]
] = {}
oauth_profile = "https://urs.earthdata.nasa.gov/profile"
oauth_profile = f"https://{auth.cmr_and_edl_maturity.value}/profile"
# sets the initial URS cookie
self._requests_cookies: Dict[str, Any] = {}
self.set_requests_session(oauth_profile)
Expand Down

0 comments on commit 9a1c938

Please sign in to comment.