diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b28b07a3627de909302dee59f261711c0643994e..e1bcbe90c0b008dfb3d42077ab69bcb6b90afee1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,7 +41,7 @@ pip_install: .static_analysis_base: stage: Static Analysis - allow_failure: true + allow_failure: false flake8: needs: diff --git a/README.md b/README.md index d80f17b72e3fc7ce14a2aad2eb430d8244d5a34c..c1842f66b346b5c9597d05c729f5cc9c21cac4d1 100644 --- a/README.md +++ b/README.md @@ -27,3 +27,5 @@ api = pystac_client.Client.open( For more information read the [documentation](https://cdos-pub.pages.mia.inra.fr/dinamis-sdk). + +For `R` users, check [rstactheia](https://cdos-pub.pages.mia.inra.fr/rstactheia). diff --git a/dinamis_sdk/__init__.py b/dinamis_sdk/__init__.py index b1178182904368c4e1acc0046741e5c8bbb2985e..74ef43269c1ceb0655dd26185dfc29491bee646c 100644 --- a/dinamis_sdk/__init__.py +++ b/dinamis_sdk/__init__.py @@ -2,7 +2,7 @@ # flake8: noqa -__version__ = "0.4.0" +__version__ = "0.4.1" from dinamis_sdk.signing import ( sign, sign_inplace, diff --git a/dinamis_sdk/cli.py b/dinamis_sdk/cli.py index 2d013d427e7bc018924fa6858192d4da47af08d6..57469f7aad1024368e78820a12dc9c871c7c893f 100644 --- a/dinamis_sdk/cli.py +++ b/dinamis_sdk/cli.py @@ -17,7 +17,7 @@ def app() -> None: """Click group for dinamis sdk subcommands.""" -def http(route: str): +def _http(route: str): """Perform an HTTP request.""" session = create_session() ret = session.get( @@ -31,17 +31,17 @@ def http(route: str): def create_key() -> Dict[str, str]: """Create an API key.""" - return http("create_api_key").json() + return _http("create_api_key").json() def list_keys() -> List[str]: """List all generated API keys.""" - return http("list_api_keys").json() + return _http("list_api_keys").json() def revoke_key(key: str): """Revoke an API key.""" - http(f"revoke_api_key?access_key={key}") + _http(f"revoke_api_key?access_key={key}") log.info(f"API key {key} revoked") diff --git a/dinamis_sdk/signing.py b/dinamis_sdk/signing.py index 152d57a44944c6bf373628f7fb13d04353d63629..9621314c3292fd06b0e8dcdc49b87e3c9ac38019 100644 --- a/dinamis_sdk/signing.py +++ b/dinamis_sdk/signing.py @@ -3,7 +3,6 @@ S3 Module. Revamp of Microsoft Planetary Computer SAS, using S3 and custom URL signing endpoint instead. - """ import collections.abc @@ -14,6 +13,7 @@ from copy import deepcopy from datetime import datetime, timezone from functools import singledispatch from typing import Any, Dict, List, Mapping, TypeVar, cast +from enum import Enum from urllib.parse import parse_qs, urlparse import packaging.version @@ -153,7 +153,14 @@ def sign_string(url: str, copy: bool = True) -> str: return sign_urls(urls=[url])[url] -def _generic_sign_urls(urls: List[str], route: str) -> Dict[str, str]: +class SignURLRoute(Enum): + """Different routes used for sign_urls.""" + + SIGN_URLS_GET = "sign_urls" + SIGN_URLS_PUT = "sign_urls_put" + + +def _generic_sign_urls(urls: List[str], route: SignURLRoute) -> Dict[str, str]: """Sign URLs with a S3 Token. Signing URL allows read access to files in storage. @@ -204,12 +211,12 @@ def _generic_sign_urls(urls: List[str], route: str) -> Dict[str, str]: def sign_urls(urls: List[str]) -> Dict[str, str]: """Sign multiple URLs for GET.""" - return _generic_sign_urls(urls=urls, route="sign_urls") + return _generic_sign_urls(urls=urls, route=SignURLRoute(SignURLRoute.SIGN_URLS_GET)) def sign_urls_put(urls: List[str]) -> Dict[str, str]: """Sign multiple URLs for PUT.""" - return _generic_sign_urls(urls=urls, route="sign_urls_put") + return _generic_sign_urls(urls=urls, route=SignURLRoute(SignURLRoute.SIGN_URLS_PUT)) def sign_url_put(url: str) -> str: @@ -436,7 +443,7 @@ sign_reference_file = sign_mapping def _generic_get_signed_urls( urls: List[str], - route: str, + route: SignURLRoute, ) -> Dict[str, SignedURL]: """ Get multiple signed URLs. @@ -460,7 +467,7 @@ def _generic_get_signed_urls( signed_urls = {} for url in urls: signed_url_in_cache = CACHE.get(url) - if signed_url_in_cache: + if signed_url_in_cache and route == SignURLRoute.SIGN_URLS_GET: log.debug("URL %s already in cache", url) ttl = signed_url_in_cache.ttl() log.debug("URL %s TTL is %s", url, ttl) @@ -491,7 +498,7 @@ def _generic_get_signed_urls( params: Dict[str, Any] = {"urls": not_signed_urls_chunk} if ENV.dinamis_sdk_url_duration: params["duration_seconds"] = ENV.dinamis_sdk_url_duration - response = session.post(route=route, params=params) + response = session.post(route=route.value, params=params) signed_url_batch = SignedURLBatch(**response.json()) if not signed_url_batch: raise ValueError( @@ -505,7 +512,7 @@ def _generic_get_signed_urls( ) for url, href in signed_url_batch.hrefs.items(): signed_url = SignedURL(expiry=signed_url_batch.expiry, href=href) - if route == "sign_urls": + if route == SignURLRoute.SIGN_URLS_GET: # Only put GET urls in cache CACHE[url] = signed_url signed_urls[url] = signed_url diff --git a/tests/test_push.py b/tests/test_push.py index d009e307740f0b67d9587e2f6ea3856db3e25921..f61c71958382036e7e07dec1c39aa1c7e05ee008 100755 --- a/tests/test_push.py +++ b/tests/test_push.py @@ -1,14 +1,25 @@ #!/usr/bin/env python +import requests + import dinamis_sdk -local_filename = "/tmp/toto.txt" +LOCAL_FILENAME = "/tmp/toto.txt" -with open(local_filename, "w") as f: +with open(LOCAL_FILENAME, "w") as f: f.write("hello world") -pushed = dinamis_sdk.push( - local_filename=local_filename, - target_url="https://s3-data.meso.umontpellier.fr/sm1-gdc-tests/titi.txt", -) +TARGET_URL = "https://s3-data.meso.umontpellier.fr/sm1-gdc-tests/titi.txt" + +dinamis_sdk.push(local_filename=LOCAL_FILENAME, target_url=TARGET_URL) +print("push OK") + +signed_url = dinamis_sdk.sign(TARGET_URL) +print("sign OK") + +res = requests.get(signed_url, stream=True, timeout=10) +assert res.status_code == 200, "Get NOK" +print("get OK") + + print("Done")