diff --git a/.gitignore b/.gitignore
index 7dc1985e4fe41ddf8b2098bd75c80181685c186b..ff863056e2369e053c70fad93ae84af3b6ca5619 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,4 @@
dinamis_sdk/test
build
dist
+*venv/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 300962ee0fd7b5b2cf85a46e61dc4df8b36ff269..6f9da0865b79a10714fcdd8da266877a6d36a833 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,5 +1,19 @@
default:
- image: python:3.8-slim
+ image: python:3.12-slim
+
+variables:
+ PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+
+cache:
+ paths:
+ - .cache/pip
+ - .venv
+
+before_script:
+ - pip install virtualenv
+ - virtualenv .venv
+ - source .venv/bin/activate
+ - pip install -e .
workflow:
rules:
@@ -7,12 +21,22 @@ workflow:
- if: $CI_COMMIT_BRANCH == 'main'
stages:
- - Static Analysis
- Install
+ - Static Analysis
- Test
- Ship
- Documentation
+# ------------------------------- Install -------------------------------------
+
+pip_install:
+ stage: Install
+ before_script:
+ - python --version ; pip --version
+ script:
+ - pip install .
+ - pip list -v
+
# ------------------------------ Static analysis ------------------------------
.static_analysis_base:
@@ -20,35 +44,45 @@ stages:
allow_failure: true
flake8:
+ needs:
+ - pip_install
extends: .static_analysis_base
script:
- pip install flake8
- - flake8 $PWD/dinamis_sdk
+ - flake8 --ignore E501 ./dinamis_sdk
pylint:
+ needs:
+ - pip_install
extends: .static_analysis_base
script:
- - pip install pylint appdirs requests pystac pystac-client pydantic urllib3 qrcode
- - pylint $PWD/dinamis_sdk --disable=W0718,W0603,R0914,C0415 --ignore=examples
+ - pip install pylint
+ - pylint ./dinamis_sdk
codespell:
+ needs:
+ - pip_install
extends: .static_analysis_base
script:
- pip install codespell
- - codespell dinamis_sdk docs README.md
+ - codespell ./dinamis_sdk docs README.md
pydocstyle:
+ needs:
+ - pip_install
extends: .static_analysis_base
script:
- pip install pydocstyle
- - pydocstyle $PWD/dinamis_sdk
-
-# ------------------------------- Install -------------------------------------
+ - pydocstyle ./dinamis_sdk
-pip_install:
- stage: Install
+mypy:
+ needs:
+ - pip_install
+ extends: .static_analysis_base
script:
- - pip install .
+ - pip install mypy
+ - pip list -v
+ - mypy --install-types --non-interactive .
# --------------------------------- Doc ---------------------------------------
@@ -78,13 +112,11 @@ pages:
# --------------------------------- Test --------------------------------------
.tests_base:
+ needs:
+ - pip_install
stage: Test
except:
- main
- before_script:
- - pip install .
- - pip install pystac-client
-
OAuth2 Tests:
extends: .tests_base
@@ -116,10 +148,9 @@ API key Tests:
pypi:
stage: Ship
only:
- - main
- before_script:
- - python3 -m pip install --upgrade build twine
+ - main
script:
- - python3 -m build
+ - pip install --upgrade build twine
+ - python3 -m build
after_script:
- - python3 -m twine upload --repository-url https://upload.pypi.org/legacy/ --non-interactive --verbose -u __token__ -p $pypi_token dist/*
+ - python3 -m twine upload --repository-url https://upload.pypi.org/legacy/ --non-interactive --verbose -u __token__ -p $pypi_token dist/*
diff --git a/dinamis_sdk/__init__.py b/dinamis_sdk/__init__.py
index d062092cd2d27ae2fa6bcf97a1cc879ea5f2501b..bf723aa3836a291e33b9f1dbece62f92a6cf7ae3 100644
--- a/dinamis_sdk/__init__.py
+++ b/dinamis_sdk/__init__.py
@@ -1,6 +1,8 @@
"""Dinamis SDK module."""
+
# flake8: noqa
import pkg_resources
+
__version__ = pkg_resources.require("dinamis-sdk")[0].version
from dinamis_sdk.s3 import (
sign,
@@ -9,7 +11,7 @@ from dinamis_sdk.s3 import (
sign_item,
sign_asset,
sign_item_collection,
- sign_url_put
+ sign_url_put,
) # noqa
from dinamis_sdk import auth # noqa
from dinamis_sdk.upload import push
diff --git a/dinamis_sdk/auth.py b/dinamis_sdk/auth.py
index 1bcdb40ddef82e804433c7f3c20201c76eb61903..9e938f1e87a64f6ddbe977408f9fde29c9c95d57 100644
--- a/dinamis_sdk/auth.py
+++ b/dinamis_sdk/auth.py
@@ -1,4 +1,5 @@
"""Module dedicated to OAuth2 device flow."""
+
import datetime
import io
import json
@@ -8,9 +9,14 @@ from abc import abstractmethod
from typing import Dict
import requests
from pydantic import BaseModel # pylint: disable = no-name-in-module
-import qrcode
+import qrcode # type: ignore
from .utils import (
- log, JWT_FILE, TOKEN_ENDPOINT, AUTH_BASE_URL, settings, create_session
+ log,
+ JWT_FILE,
+ TOKEN_ENDPOINT,
+ AUTH_BASE_URL,
+ settings,
+ create_session,
)
@@ -36,9 +42,7 @@ class DeviceGrantResponse(BaseModel): # pylint: disable = R0903
class GrantMethodBase:
"""Base class for grant methods."""
- headers: Dict[str, str] = {
- "Content-Type": "application/x-www-form-urlencoded"
- }
+ headers: Dict[str, str] = {"Content-Type": "application/x-www-form-urlencoded"}
token_endpoint = TOKEN_ENDPOINT
base_url = AUTH_BASE_URL
keycloak_realm = "dinamis"
@@ -60,7 +64,7 @@ class GrantMethodBase:
"""Base payload."""
return {
"client_id": self.client_id,
- "scope": "openid offline_access"
+ "scope": "openid offline_access",
}
def refresh_token(self, old_jwt: JWT) -> JWT:
@@ -77,15 +81,17 @@ class GrantMethodBase:
log.debug("Refreshing token")
assert old_jwt, "JWT is empty"
data = self.data_base.copy()
- data.update({
- "refresh_token": old_jwt.refresh_token,
- "grant_type": "refresh_token"
- })
+ data.update(
+ {
+ "refresh_token": old_jwt.refresh_token,
+ "grant_type": "refresh_token",
+ }
+ )
ret = requests.post(
self.token_endpoint,
headers=self.headers,
data=data,
- timeout=10
+ timeout=10,
)
if ret.status_code == 200:
log.debug(ret.text)
@@ -113,7 +119,7 @@ class DeviceGrant(GrantMethodBase):
device_endpoint,
headers=self.headers,
data=self.data_base,
- timeout=10
+ timeout=10,
)
if ret.status_code == 200:
response = DeviceGrantResponse(**ret.json())
@@ -132,19 +138,26 @@ class DeviceGrant(GrantMethodBase):
log.info("Waiting for authentication...")
start = time.time()
data = self.data_base.copy()
- data.update({
- "device_code": response.device_code,
- "grant_type": "urn:ietf:params:oauth:grant-type:device_code"
- })
+ grant_type = "urn:ietf:params:oauth:grant-type:device_code"
+ data.update(
+ {
+ "device_code": response.device_code,
+ "grant_type": grant_type,
+ }
+ )
while True:
ret = requests.post(
self.token_endpoint,
headers=self.headers,
data=data,
- timeout=10
+ timeout=10,
)
elapsed = time.time() - start
- log.debug("Elapsed: %s, status: %s", elapsed, ret.status_code)
+ log.debug(
+ "Elapsed: %s, status: %s",
+ elapsed,
+ ret.status_code,
+ )
if elapsed > response.expires_in:
raise ConnectionError("Authentication link has expired.")
if ret.status_code != 200:
@@ -157,7 +170,7 @@ class DeviceGrant(GrantMethodBase):
class OAuth2Session:
"""Class to start an OAuth2 session."""
- def __init__(self, grant_type: GrantMethodBase = DeviceGrant):
+ def __init__(self, grant_type: type[GrantMethodBase] = DeviceGrant):
"""
Initialize.
@@ -168,7 +181,7 @@ class OAuth2Session:
self.grant = grant_type()
self.jwt_ttl_margin_seconds = 60
self.jwt_issuance = datetime.datetime(year=1, month=1, day=1)
- self.jwt = None
+ self.jwt: JWT | None = None
def save_token(self, now: datetime.datetime):
"""
@@ -181,8 +194,9 @@ class OAuth2Session:
self.jwt_issuance = now
if JWT_FILE:
try:
- with open(JWT_FILE, 'w', encoding='UTF-8') as file:
- json.dump(self.jwt.dict(), file)
+ if self.jwt:
+ with open(JWT_FILE, "w", encoding="UTF-8") as file:
+ json.dump(self.jwt.dict(), file)
log.debug("Token saved in %s", JWT_FILE)
except IOError as io_err:
log.warning("Unable to save token (%s)", io_err)
@@ -197,9 +211,7 @@ class OAuth2Session:
log.debug("access_token_ttl is %s", access_token_ttl_seconds)
if access_token_ttl_seconds >= ttl_margin_seconds:
# Token is still valid
- log.debug(
- "Credentials from %s still valid", JWT_FILE
- )
+ log.debug("Credentials from %s still valid", JWT_FILE)
return
if access_token_ttl_seconds < ttl_margin_seconds:
# Access token in not valid, but refresh might be
@@ -209,7 +221,7 @@ class OAuth2Session:
log.warning(
"Unable to refresh token (reason: %s). "
"Renewing initial authentication.",
- con_err
+ con_err,
)
self.jwt = self.grant.get_first_token()
else:
@@ -223,13 +235,13 @@ class OAuth2Session:
if JWT_FILE and os.path.isfile(JWT_FILE):
log.debug("Trying to grab credentials from %s", JWT_FILE)
try:
- with open(JWT_FILE, encoding='UTF-8') as json_file:
+ with open(JWT_FILE, encoding="UTF-8") as json_file:
self.jwt = JWT(**json.load(json_file))
except FileNotFoundError as error:
log.warning(
"Warning: can't use token from file %s (%s)",
JWT_FILE,
- error
+ error,
)
if not self.jwt:
# if JWT is still None, we use the grant method
@@ -249,7 +261,7 @@ class TokenServer:
self.endpoint = endpoint
self.session = create_session(
retry_total=retry_total,
- retry_backoff_factor=retry_backoff_factor
+ retry_backoff_factor=retry_backoff_factor,
)
log.info("Using Token Server: %s", self.endpoint)
@@ -258,8 +270,11 @@ class TokenServer:
return self.session.get(self.endpoint, timeout=10).json()
-session = TokenServer(settings.dinamis_sdk_token_server) \
- if settings.dinamis_sdk_token_server else OAuth2Session()
+session = (
+ TokenServer(settings.dinamis_sdk_token_server)
+ if settings.dinamis_sdk_token_server
+ else OAuth2Session()
+)
def _get_access_token():
@@ -278,5 +293,5 @@ def set_access_token_fn(func):
func: access token accessor
"""
- global get_access_token
+ global get_access_token # pylint: disable=W0603
get_access_token = func
diff --git a/dinamis_sdk/cli.py b/dinamis_sdk/cli.py
index dd2582af525f697d8118040103847b16d56a1af0..f47981776dcac43c89052a59afe7d5501aae07ef 100644
--- a/dinamis_sdk/cli.py
+++ b/dinamis_sdk/cli.py
@@ -1,16 +1,23 @@
"""Dinamis Command Line Interface."""
+
+import json
+import os
+from typing import Dict, List
+
import click
-from .utils import APIKEY_FILE, create_session, S3_SIGNING_ENDPOINT, log
+
from .auth import get_access_token
-import os
-import json
-from typing import List, Dict
+from .utils import (
+ APIKEY_FILE,
+ S3_SIGNING_ENDPOINT,
+ create_session,
+ log,
+)
@click.group(help="Dinamis CLI")
def app() -> None:
"""Click group for dinamis sdk subcommands."""
- pass
def http(route: str):
@@ -19,7 +26,7 @@ def http(route: str):
ret = session.get(
f"{S3_SIGNING_ENDPOINT}{route}",
timeout=5,
- headers={"authorization": f"bearer {get_access_token()}"}
+ headers={"authorization": f"bearer {get_access_token()}"},
)
ret.raise_for_status()
return ret
@@ -48,7 +55,7 @@ def create():
@app.command(help="List all API keys")
-def list():
+def list(): # [redefined-builtin]
"""List all API keys."""
log.info(f"All generated API keys: {list_keys()}")
@@ -73,7 +80,7 @@ def revoke(access_key: str):
@app.command(help="Get and store an API key")
def register():
"""Get and store an API key."""
- with open(APIKEY_FILE, 'w') as f:
+ with open(APIKEY_FILE, "w", encoding="utf-8") as f:
json.dump(create_key(), f)
log.info(f"API key successfully created and stored in {APIKEY_FILE}")
@@ -84,7 +91,7 @@ def delete(dont_revoke):
"""Delete the stored API key."""
if os.path.isfile(APIKEY_FILE):
if not dont_revoke:
- with open(APIKEY_FILE, encoding='UTF-8') as json_file:
+ with open(APIKEY_FILE, encoding="UTF-8") as json_file:
api_key = json.load(json_file)
if "access-key" in api_key:
revoke_key(api_key["access-key"])
diff --git a/dinamis_sdk/examples/pyotb_ndvi_loss.py b/dinamis_sdk/examples/pyotb_ndvi_loss.py
index 770830bbb5b3626e088a5492d3fc57dc7afb31c2..935c5cb05f5ca257a8ea21e1c642f39a4a057a13 100644
--- a/dinamis_sdk/examples/pyotb_ndvi_loss.py
+++ b/dinamis_sdk/examples/pyotb_ndvi_loss.py
@@ -1,11 +1,12 @@
"""NDVI loss example with pyotb."""
+
from pystac_client import Client
-import pyotb
+import pyotb # type: ignore
from dinamis_sdk import sign_inplace
api = Client.open(
- 'https://stacapi-cdos.apps.okd.crocc.meso.umontpellier.fr',
- modifier=sign_inplace
+ "https://stacapi-cdos.apps.okd.crocc.meso.umontpellier.fr",
+ modifier=sign_inplace,
)
@@ -14,7 +15,7 @@ def mosa(year):
res = api.search(
bbox=[4, 42.99, 5, 44.05],
datetime=[f"{year}-01-01", f"{year}-12-25"],
- collections=["spot-6-7-drs"]
+ collections=["spot-6-7-drs"],
)
urls = [f"/vsicurl/{r.assets['src_xs'].href}" for r in res.items()]
return pyotb.Mosaic({"il": urls})
diff --git a/dinamis_sdk/examples/pyotb_toa_mosaic.py b/dinamis_sdk/examples/pyotb_toa_mosaic.py
index 9eccf0fd4ab77d4a3893b503a9c291d292a77e5c..596a1e05e7f2bc5d8c7ff3789130fcc4c5e6e878 100644
--- a/dinamis_sdk/examples/pyotb_toa_mosaic.py
+++ b/dinamis_sdk/examples/pyotb_toa_mosaic.py
@@ -1,17 +1,20 @@
"""TOA images mosaicing with pyotb."""
+
+# pylint: disable=duplicate-code
+
from pystac_client import Client
-import pyotb
+import pyotb # type: ignore
from dinamis_sdk import sign_inplace
api = Client.open(
- 'https://stacapi-cdos.apps.okd.crocc.meso.umontpellier.fr',
- modifier=sign_inplace
+ "https://stacapi-cdos.apps.okd.crocc.meso.umontpellier.fr",
+ modifier=sign_inplace,
)
res = api.search(
bbox=[4, 42.99, 5, 44.05],
datetime=["2022-01-01", "2022-12-25"],
- collections=["spot-6-7-drs"]
+ collections=["spot-6-7-drs"],
)
urls = [f"/vsicurl/{r.assets['src_xs'].href}" for r in res.items()]
diff --git a/dinamis_sdk/examples/rio_metadata.py b/dinamis_sdk/examples/rio_metadata.py
index 3f75e7d044913ec42cfc37b7a2741bca640b1e8f..4d5a30d692ca72ef91bb87af8f5a25ea22798f00 100644
--- a/dinamis_sdk/examples/rio_metadata.py
+++ b/dinamis_sdk/examples/rio_metadata.py
@@ -1,34 +1,31 @@
"""Metadata extraction example with rasterio."""
+
from pystac_client import Client
-import rasterio.features
-import rasterio.warp
+import rasterio.features # type: ignore
+import rasterio.warp # type: ignore
from dinamis_sdk import sign_inplace
api = Client.open(
- 'https://stacapi-cdos.apps.okd.crocc.meso.umontpellier.fr',
- modifier=sign_inplace
+ "https://stacapi-cdos.apps.okd.crocc.meso.umontpellier.fr",
+ modifier=sign_inplace,
)
-year = 2022
+YEAR = 2022
bbox = [4, 42.99, 5, 44.05]
-res = api.search(bbox=bbox, datetime=[f'{year}-01-01', f'{year}-12-25'])
+res = api.search(bbox=bbox, datetime=[f"{YEAR}-01-01", f"{YEAR}-12-25"])
for item in res.items():
url = item.assets["src_xs"].href
with rasterio.open(url) as dataset:
-
# Read the dataset's valid data mask as a ndarray.
mask = dataset.dataset_mask()
# Extract feature shapes and values from the array.
- for geom, val in rasterio.features.shapes(
- mask, transform=dataset.transform
- ):
-
+ for geom, val in rasterio.features.shapes(mask, transform=dataset.transform):
# Transform shapes from the dataset's own coordinate
# reference system to CRS84 (EPSG:4326).
geom = rasterio.warp.transform_geom(
- dataset.crs, 'EPSG:4326', geom, precision=6
+ dataset.crs, "EPSG:4326", geom, precision=6
)
# Print GeoJSON shapes to stdout.
diff --git a/dinamis_sdk/py.typed b/dinamis_sdk/py.typed
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/dinamis_sdk/s3.py b/dinamis_sdk/s3.py
index cc0cad7b456ef9090400af85361b72779143c8b0..69c933b4687eaa7cb4d35d778f05e26adb2de06d 100644
--- a/dinamis_sdk/s3.py
+++ b/dinamis_sdk/s3.py
@@ -5,32 +5,42 @@ Revamp of Microsoft Planetary Computer SAS, using S3 and custom URL signing
endpoint instead.
"""
+
import collections.abc
+import math
import re
import time
+from ast import literal_eval
from copy import deepcopy
from datetime import datetime, timezone
from functools import singledispatch
-from typing import Any, Dict, Mapping, TypeVar, cast, List
-from urllib.parse import urlparse, parse_qs
-import math
+from typing import Any, Dict, List, Mapping, TypeVar, cast
+from urllib.parse import parse_qs, urlparse
+
+import packaging.version
+import pydantic
+import pystac_client
from pydantic import BaseModel # pylint: disable = no-name-in-module
-from pystac import Asset, Item, ItemCollection, STACObjectType, Collection
+from pystac import (
+ Asset,
+ Collection,
+ Item,
+ ItemCollection,
+ STACObjectType,
+)
from pystac.serialization.identify import identify_stac_object_type
from pystac.utils import datetime_to_str
-import pystac_client
from pystac_client import ItemSearch
-import packaging.version
-import pydantic
+from .auth import get_access_token
from .utils import (
- log,
- settings,
+ APIKEY,
MAX_URLS,
S3_SIGNING_ENDPOINT,
S3_STORAGE_DOMAIN,
create_session,
- APIKEY
+ log,
+ settings,
)
_PYDANTIC_2_0 = packaging.version.parse(
@@ -41,7 +51,7 @@ AssetLike = TypeVar("AssetLike", Asset, Dict[str, Any])
asset_xpr = re.compile(
r"https:\/\/(?P<account>[A-z0-9-.]+?)"
- r"\.meso\.umontpellier\.fr\/(?P<blob>[^<]+)"
+ r"\.meso\.umontpellier\.fr\/(?P<blob>[^<]+)" # ignore
)
@@ -115,8 +125,9 @@ def sign_inplace(obj: Any) -> Any:
def is_vrt_string(string: str) -> bool:
"""Check whether a string looks like a VRT."""
- return string.strip().startswith("<VRTDataset") and \
- string.strip().endswith("</VRTDataset>")
+ return string.strip().startswith("<VRTDataset") and string.strip().endswith(
+ "</VRTDataset>"
+ )
@sign.register(str)
@@ -147,10 +158,7 @@ def sign_string(url: str, copy: bool = True) -> str:
return sign_urls(urls=[url])[url]
-def _generic_sign_urls(
- urls: List[str],
- route: str
-) -> Dict[str, str]:
+def _generic_sign_urls(urls: List[str], route: str) -> Dict[str, str]:
"""Sign URLs with a S3 Token.
Signing URL allows read access to files in storage.
@@ -182,19 +190,20 @@ def _generic_sign_urls(
if set(parsed_qs) & {
"X-Amz-Security-Token",
"X-Amz-Signature",
- "X-Amz-Credential"
+ "X-Amz-Credential",
}:
# looks like we've already signed it
signed_urls[url] = url
not_signed_urls = [url for url in urls if url not in signed_urls]
- signed_urls.update({
- url: signed_url.href
- for url, signed_url in _generic_get_signed_urls(
- urls=not_signed_urls,
- route=route
- ).items()
- })
+ signed_urls.update(
+ {
+ url: signed_url.href
+ for url, signed_url in _generic_get_signed_urls(
+ urls=not_signed_urls, route=route
+ ).items()
+ }
+ )
return signed_urls
@@ -214,8 +223,7 @@ def sign_url_put(url: str) -> str:
return urls[url]
-def sign_vrt_string(vrt: str,
- copy: bool = True) -> str: # pylint: disable = W0613 # noqa: E501
+def sign_vrt_string(vrt: str, copy: bool = True) -> str: # pylint: disable = W0613 # noqa: E501
"""Sign a VRT-like string containing URLs from the storage.
Signing URLs allows read access to files in storage.
@@ -235,7 +243,7 @@ def sign_vrt_string(vrt: str,
"""
urls = []
- def _repl_vrt(m: re.Match) -> str:
+ def _repl_vrt(m: re.Match):
urls.append(m.string[slice(*m.span())])
asset_xpr.sub(_repl_vrt, vrt)
@@ -296,7 +304,7 @@ def sign_asset(asset: Asset, copy: bool = True) -> Asset:
@sign.register(ItemCollection)
def sign_item_collection(
- item_collection: ItemCollection, copy: bool = True
+ item_collection: ItemCollection, copy: bool = True
) -> ItemCollection:
"""Sign a PySTAC item collection.
@@ -315,11 +323,7 @@ def sign_item_collection(
"""
if copy:
item_collection = item_collection.clone()
- urls = [
- asset.href
- for item in item_collection
- for asset in item.assets.values()
- ]
+ urls = [asset.href for item in item_collection for asset in item.assets.values()]
signed_urls = sign_urls(urls=urls)
for item in item_collection:
for key, asset in item.assets.items():
@@ -371,10 +375,7 @@ def sign_collection(collection: Collection, copy: bool = True) -> Collection:
if assets and not collection.assets:
collection.assets = deepcopy(assets)
- urls = [
- collection.assets[key].href
- for key in collection.assets
- ]
+ urls = [collection.assets[key].href for key in collection.assets]
signed_urls = sign_urls(urls=urls)
for key, asset in collection.assets.items():
collection.assets[key].href = signed_urls[asset.href]
@@ -420,8 +421,7 @@ def sign_mapping(mapping: Mapping, copy: bool = True) -> Mapping:
url = val["href"]
val["href"] = signed_urls[url]
- elif mapping.get("type") == "FeatureCollection" and \
- mapping.get("features"):
+ elif mapping.get("type") == "FeatureCollection" and mapping.get("features"):
urls = [
val["href"]
for feat in mapping["features"]
@@ -440,10 +440,10 @@ sign_reference_file = sign_mapping
def _generic_get_signed_urls(
- urls: List[str],
- route: str,
- retry_total: int = 10,
- retry_backoff_factor: float = .8
+ urls: List[str],
+ route: str,
+ retry_total: int = 10,
+ retry_backoff_factor: float = 0.8,
) -> Dict[str, SignedURL]:
"""
Get multiple signed URLs.
@@ -468,11 +468,14 @@ def _generic_get_signed_urls(
Returns:
SignedURL: the signed URL
"""
+ # pylint: disable=too-many-statements
+ # pylint: disable=too-many-locals
+ # pylint: disable=too-many-branches
log.debug("Get signed URLs for %s", urls)
start_time = time.time()
headers = {
"Content-Type": "application/json",
- "Accept": "application/json"
+ "Accept": "application/json",
}
if APIKEY:
headers.update(APIKEY)
@@ -480,11 +483,12 @@ def _generic_get_signed_urls(
elif settings.dinamis_sdk_bypass_api:
log.debug("Using bypass API %s", settings.dinamis_sdk_bypass_api)
else:
- from .auth import get_access_token
access_token = get_access_token()
headers.update({"Authorization": f"Bearer {access_token}"})
log.debug(
- "Access token: %s...%s", access_token[:8], access_token[-8:]
+ "Access token: %s...%s",
+ access_token[:8],
+ access_token[-8:],
)
signed_urls = {}
for url in urls:
@@ -497,7 +501,7 @@ def _generic_get_signed_urls(
log.debug(
"Using cache (%s > %s)",
ttl,
- settings.dinamis_sdk_ttl_margin
+ settings.dinamis_sdk_ttl_margin,
)
signed_urls[url] = signed_url_in_cache
not_signed_urls = [url for url in urls if url not in signed_urls]
@@ -510,7 +514,7 @@ def _generic_get_signed_urls(
# give a small amount of time to do stuff with the url
session = create_session(
retry_backoff_factor=retry_backoff_factor,
- retry_total=retry_total
+ retry_total=retry_total,
)
n_urls = len(not_signed_urls)
log.debug("Number of URLs to sign: %s", n_urls)
@@ -521,40 +525,33 @@ def _generic_get_signed_urls(
chunk_start = i_chunk * MAX_URLS
chunk_end = min(chunk_start + MAX_URLS, n_urls)
not_signed_urls_chunk = not_signed_urls[chunk_start:chunk_end]
- params = {"urls": not_signed_urls_chunk}
+ params: Dict[str, Any] = {"urls": not_signed_urls_chunk}
if settings.dinamis_sdk_url_duration:
params["duration_seconds"] = settings.dinamis_sdk_url_duration
post_url = f"{S3_SIGNING_ENDPOINT}{route}"
log.debug("POST %s", post_url)
response = session.post(
- post_url,
- params=params,
- headers=headers,
- timeout=10
+ post_url, params=params, headers=headers, timeout=10
)
try:
response.raise_for_status()
except Exception as e:
- log.error(eval(response.content))
- raise(e)
+ log.error(literal_eval(response.content))
+ raise e
signed_url_batch = SignedURLBatch(**response.json())
if not signed_url_batch:
raise ValueError(
f"No signed url batch found in response: {response.json()}"
)
- if not all(key in signed_url_batch.hrefs
- for key in not_signed_urls_chunk):
+ if not all(key in signed_url_batch.hrefs for key in not_signed_urls_chunk):
raise ValueError(
f"URLs to sign are {not_signed_urls_chunk} but returned "
f"signed URLs"
f"are for {signed_url_batch.hrefs.keys()}"
)
for url, href in signed_url_batch.hrefs.items():
- signed_url = SignedURL(
- expiry=signed_url_batch.expiry,
- href=href
- )
+ signed_url = SignedURL(expiry=signed_url_batch.expiry, href=href)
if route == "sign_urls":
# Only put GET urls in cache
CACHE[url] = signed_url
@@ -562,7 +559,7 @@ def _generic_get_signed_urls(
log.debug(
"Got signed urls %s in %s seconds",
signed_urls,
- f"{time.time() - start_time:.2f}"
+ f"{time.time() - start_time:.2f}",
)
return signed_urls
diff --git a/dinamis_sdk/settings.py b/dinamis_sdk/settings.py
index 4940795ed0c6853f1e8c14fe38b2b958e4f15f37..b237b196d21e0ab4a737d2060030426bc492979e 100644
--- a/dinamis_sdk/settings.py
+++ b/dinamis_sdk/settings.py
@@ -1,4 +1,5 @@
"""Settings from environment variables."""
+
from pydantic_settings import BaseSettings
diff --git a/dinamis_sdk/upload.py b/dinamis_sdk/upload.py
index ec5bc4f9f124639294e807a1233dd17aa5744e04..f971e1a762607d833fd7496509133a0cd29077a0 100644
--- a/dinamis_sdk/upload.py
+++ b/dinamis_sdk/upload.py
@@ -1,23 +1,24 @@
"""This module is used to upload files using HTTP requests."""
+
from .s3 import sign_url_put
from .utils import create_session
def push(
- local_filename: str,
- target_url: str,
- retry_total: int = 5,
- retry_backoff_factor: float = .8
+ local_filename: str,
+ target_url: str,
+ retry_total: int = 5,
+ retry_backoff_factor: float = 0.8,
):
"""Publish a local file to the cloud."""
remote_presigned_url = sign_url_put(target_url)
session = create_session(
retry_total=retry_total,
- retry_backoff_factor=retry_backoff_factor
+ retry_backoff_factor=retry_backoff_factor,
)
- with open(local_filename, 'rb') as f:
+ with open(local_filename, "rb") as f:
ret = session.put(remote_presigned_url, data=f)
if ret.status_code == 200:
diff --git a/dinamis_sdk/utils.py b/dinamis_sdk/utils.py
index 439eba0635eedd960809ab12af7c45b1369f371a..9bb6fd154c4dace1aa8cf44b1b11a5dc91ff86ae 100644
--- a/dinamis_sdk/utils.py
+++ b/dinamis_sdk/utils.py
@@ -1,12 +1,16 @@
"""Some helpers."""
+
import json
import logging
import os
-import appdirs
+import sys
+
+import appdirs # type: ignore
+
import requests
import urllib3.util.retry
+
from .settings import Settings
-import sys
# Settings
settings = Settings()
@@ -14,17 +18,17 @@ settings = Settings()
# Logger
logging.basicConfig(stream=sys.stdout)
log = logging.getLogger("dinamis_sdk")
-log.setLevel(level=os.environ.get('LOGLEVEL', 'INFO').upper())
+log.setLevel(level=os.environ.get("LOGLEVEL", "INFO").upper())
# Constants
MAX_URLS = 64
S3_STORAGE_DOMAIN = "meso.umontpellier.fr"
-S3_SIGNING_ENDPOINT = \
- "https://s3-signing-cdos.apps.okd.crocc.meso.umontpellier.fr/"
+S3_SIGNING_ENDPOINT = "https://s3-signing-cdos.apps.okd.crocc.meso.umontpellier.fr/"
# Config path
-CFG_PTH = settings.dinamis_sdk_settings_dir or \
- appdirs.user_config_dir(appname='dinamis_sdk_auth')
+CFG_PTH = settings.dinamis_sdk_settings_dir or appdirs.user_config_dir(
+ appname="dinamis_sdk_auth"
+)
if not os.path.exists(CFG_PTH):
try:
os.makedirs(CFG_PTH)
@@ -47,7 +51,7 @@ APIKEY = None
if APIKEY_FILE and os.path.isfile(APIKEY_FILE):
try:
log.debug("Found a stored API key")
- with open(APIKEY_FILE, encoding='UTF-8') as json_file:
+ with open(APIKEY_FILE, encoding="UTF-8") as json_file:
APIKEY = json.load(json_file)
log.debug("API key successfully loaded")
except json.decoder.JSONDecodeError:
@@ -56,20 +60,17 @@ if APIKEY_FILE and os.path.isfile(APIKEY_FILE):
if settings.dinamis_sdk_access_key and settings.dinamis_sdk_secret_key:
APIKEY = {
"access-key": settings.dinamis_sdk_access_key,
- "secret-key": settings.dinamis_sdk_secret_key
+ "secret-key": settings.dinamis_sdk_secret_key,
}
-def create_session(
- retry_total: int = 5,
- retry_backoff_factor: float = .8
-):
+
+def create_session(retry_total: int = 5, retry_backoff_factor: float = 0.8):
"""Create a session for requests."""
session = requests.Session()
retry = urllib3.util.retry.Retry(
total=retry_total,
backoff_factor=retry_backoff_factor,
status_forcelist=[404, 429, 500, 502, 503, 504],
- allowed_methods=False,
)
adapter = requests.adapters.HTTPAdapter(max_retries=retry)
session.mount("http://", adapter)
@@ -78,7 +79,9 @@ def create_session(
return session
-def retrieve_token_endpoint(s3_signing_endpoint: str = S3_SIGNING_ENDPOINT):
+def retrieve_token_endpoint(
+ s3_signing_endpoint: str = S3_SIGNING_ENDPOINT,
+):
"""Retrieve the token endpoint from the s3 signing endpoint."""
openapi_url = s3_signing_endpoint + "openapi.json"
log.debug("Fetching OAuth2 endpoint from openapi url %s", openapi_url)
@@ -98,9 +101,9 @@ if settings.dinamis_sdk_bypass_api:
# Token endpoint is typically something like: https://keycloak-dinamis.apps.okd
# .crocc.meso.umontpellier.fr/auth/realms/dinamis/protocol/openid-connect/token
-TOKEN_ENDPOINT = None if settings.dinamis_sdk_bypass_api \
- else retrieve_token_endpoint()
+TOKEN_ENDPOINT = " " if settings.dinamis_sdk_bypass_api else retrieve_token_endpoint()
# Auth base URL is typically something like: https://keycloak-dinamis.apps.okd.
# crocc.meso.umontpellier.fr/auth/realms/dinamis/protocol/openid-connect
-AUTH_BASE_URL = None if settings.dinamis_sdk_bypass_api \
- else TOKEN_ENDPOINT.rsplit('/', 1)[0]
+AUTH_BASE_URL = (
+ "" if settings.dinamis_sdk_bypass_api else TOKEN_ENDPOINT.rsplit("/", 1)[0]
+)
diff --git a/doc/gen_ref_pages.py b/doc/gen_ref_pages.py
index a35c15e96461b7bf6f741eb4e3ee35c35e4badce..f427d1633f9251c76df8f0a5cd6cd870d2585724 100755
--- a/doc/gen_ref_pages.py
+++ b/doc/gen_ref_pages.py
@@ -22,4 +22,3 @@ for path in sorted(Path("dinamis_sdk").rglob("*.py")): #
print("::: " + identifier, file=fd) #
mkdocs_gen_files.set_edit_path(full_doc_path, path)
-
diff --git a/pyproject.toml b/pyproject.toml
index 502f757d2376653008fd8bc7e90f0c4fba1621f3..8df4e7852fd69b1bd8e64c4063e253e9f5ab66e2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -5,10 +5,11 @@ build-backend = "setuptools.build_meta"
[project]
name = "dinamis-sdk"
authors = [{name = "inrae", email = "remi.cresson@inrae.fr"}]
-version = "0.3.3"
+version = "0.3.4"
description = "DINAMIS SDK for Python"
requires-python = ">=3.7"
dependencies = [
+ "setuptools>=61.2",
"click>=7.1",
"pydantic>=1.7.3",
"pystac>=1.0.0",
@@ -25,5 +26,13 @@ license = {file = "LICENSE"}
[project.scripts]
dinamis_cli = "dinamis_sdk.cli:app"
+[tool.mypy]
+show_error_codes = true
+pretty = true
+exclude = ["doc", "venv", ".venv"]
+
+[tool.pylint]
+disable = "W1203,R0903,E0401,W0622"
+
[tool.setuptools]
-include-package-data = false
+packages = ["dinamis_sdk"]
diff --git a/tests/test_push.py b/tests/test_push.py
old mode 100644
new mode 100755
index d7623ea8116fab84071eb093d837f29ab824f77d..252e2728540abd4f50ded8b5518ecbd07a775ca3
--- a/tests/test_push.py
+++ b/tests/test_push.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
import dinamis_sdk
import time
@@ -8,6 +10,6 @@ with open(local_filename, "w") as f:
pushed = dinamis_sdk.push(
local_filename=local_filename,
- target_url="https://s3-data.meso.umontpellier.fr/sm1-gdc-tests/titi.txt"
+ target_url="https://s3-data.meso.umontpellier.fr/sm1-gdc-tests/titi.txt",
)
print("Done")
diff --git a/tests/test_spot-6-7-drs.py b/tests/test_spot-6-7-drs.py
old mode 100644
new mode 100755
index eb4f285adfb0fc9e3409817777989b4c2de6c6fb..7cf85577b8488bd016ceaeba251f8329f66b4baf
--- a/tests/test_spot-6-7-drs.py
+++ b/tests/test_spot-6-7-drs.py
@@ -1,16 +1,17 @@
+#!/usr/bin/env python
+
import dinamis_sdk
import pystac_client
api = pystac_client.Client.open(
- 'https://stacapi-cdos.apps.okd.crocc.meso.umontpellier.fr',
- modifier=dinamis_sdk.sign_inplace,
+ "https://stacapi-cdos.apps.okd.crocc.meso.umontpellier.fr",
+ modifier=dinamis_sdk.sign_inplace,
)
res = api.search(
bbox=[-3.75, 30, 10, 60],
datetime=["2017-01-01", "2022-12-31"],
- collections=["spot-6-7-drs"]
+ collections=["spot-6-7-drs"],
)
-urls = [item.assets['src_xs'].href for item in res.items()]
+urls = [item.assets["src_xs"].href for item in res.items()]
print(len(urls))
assert len(urls) > 1000
-
diff --git a/tests/test_super-s2.py b/tests/test_super-s2.py
old mode 100644
new mode 100755
index 28966b6aeee228f0009a3a7a3e57dfc5c56d14ec..eb54991b2381a3ddda4e317d3c1bde052d104f2b
--- a/tests/test_super-s2.py
+++ b/tests/test_super-s2.py
@@ -1,19 +1,29 @@
-import dinamis_sdk
-import pystac_client
+#!/usr/bin/env python
+
import pystac
+import pystac_client
+import requests
+
+import dinamis_sdk
api = pystac_client.Client.open(
- 'https://stacapi-cdos.apps.okd.crocc.meso.umontpellier.fr',
- modifier=dinamis_sdk.sign_inplace,
+ "https://stacapi-cdos.apps.okd.crocc.meso.umontpellier.fr",
+ modifier=dinamis_sdk.sign_inplace,
)
res = api.search(
bbox=[3.75, 43.58, 3.95, 43.67],
datetime=["2017-01-01", "2022-12-31"],
- collections=["super-sentinel-2-l2a"]
+ collections=["super-sentinel-2-l2a"],
)
-urls = [item.assets['img'].href for item in res.items()]
+urls = [item.assets["img"].href for item in res.items()]
assert len(urls) == 672
# ItemCollection (bug #17)
-ic = pystac.item_collection.ItemCollection(res.items())
-dinamis_sdk.sign_inplace(ic)
\ No newline at end of file
+ic: pystac.ItemCollection = pystac.item_collection.ItemCollection(res.items())
+dinamis_sdk.sign_inplace(ic)
+
+item = ic.items[0]
+_, asset = next(iter(item.get_assets().items()))
+
+response = requests.get(asset.href, timeout=5)
+response.raise_for_status()