diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 78c2dc4c8dfbc9a7d8e1258c458498af83298307..3fc17e890d63af8e156089091984b5f3147b3392 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -10,7 +10,7 @@ stages:
- Static Analysis
- Install
- Test
-# - Ship
+ - Ship
# ------------------------------ Static analysis ------------------------------
@@ -64,15 +64,15 @@ Tests:
script:
- python tests/all.py
-## --------------------------------- Ship --------------------------------------
-#
-#pypi:
-# stage: Ship
-# only:
-# - main
-# before_script:
-# - python3 -m pip install --upgrade build twine
-# script:
-# - python3 -m build
-# after_script:
-# - python3 -m twine upload --repository-url https://upload.pypi.org/legacy/ --non-interactive --verbose -u __token__ -p $pypi_token dist/*
+# --------------------------------- Ship --------------------------------------
+
+pypi:
+ stage: Ship
+ only:
+ - main
+ before_script:
+ - python3 -m pip install --upgrade build twine
+ script:
+ - python3 -m build
+ after_script:
+ - python3 -m twine upload --repository-url https://upload.pypi.org/legacy/ --non-interactive --verbose -u __token__ -p $pypi_token dist/*
diff --git a/tests/all.py b/tests/all.py
index 604c10664e86401c5b01f6f7e172a81b203717aa..cc75766611fe24d5f8d9b30000b1a7cd29233d01 100644
--- a/tests/all.py
+++ b/tests/all.py
@@ -19,7 +19,7 @@ image_href = (
"otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
)
-
+col_id = "collection-for-theia-dumper-tests"
items_ids = [
"item_1",
"item_2"
@@ -33,6 +33,12 @@ with open(raster_file1, 'wb') as f:
shutil.copyfile(raster_file1, raster_file2)
+def clear():
+ for item_id in items_ids:
+ handler.delete(col_id=col_id, item_id=item_id)
+ handler.delete(col_id=col_id)
+
+
def create_item(item_id: str):
"""Create a STAC item."""
item = pystac.Item(
@@ -66,7 +72,7 @@ def create_collection():
intervals=[(None, None)]
)
col = pystac.Collection(
- id="collection-for-theia-dumper-tests",
+ id=col_id,
extent=pystac.Extent(spat_extent, temp_extent),
description="Some description",
href="http://hello.fr/collections/collection-for-tests",
@@ -117,23 +123,35 @@ def generate_item_collection(file_pth, relative=True):
def test_item_collection():
"""Test item collection."""
for relative in [True, False]:
+ print(f"Relative: {relative}")
+
+ # we need to create an empty collection before
+ col = create_collection()
+ handler.publish_collection(collection=col)
+
with tempfile.NamedTemporaryFile() as tmp:
generate_item_collection(tmp.name, relative=relative)
- handler.publish(tmp.name)
+ handler.load_and_publish(tmp.name)
+ clear()
def test_collection():
"""Test collection."""
for relative in [True, False]:
+ print(f"\nRelative: {relative}")
with tempfile.TemporaryDirectory() as tmpdir:
generate_collection(tmpdir, relative=relative)
- handler.publish(os.path.join(tmpdir, "collection.json"))
+ handler.load_and_publish(os.path.join(tmpdir, "collection.json"))
+ clear()
def test_all():
"""Test all."""
- test_item_collection()
+ # test collection
test_collection()
+ # test item collection
+ test_item_collection()
+
test_all()
diff --git a/theia_dumper/logger.py b/theia_dumper/logger.py
index 0f8c7a008407d3a9ef9ae5970f3fbce3dd4abf52..0021812bd9b44ae3fab4e4fa01e4e2c4472c0c77 100644
--- a/theia_dumper/logger.py
+++ b/theia_dumper/logger.py
@@ -1,3 +1,4 @@
+"""Logging stuff."""
import logging
logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)
logger = logging.getLogger(__name__)
diff --git a/theia_dumper/stac.py b/theia_dumper/stac.py
index 52b218748539e1f26642c9142147e0f720054a40..41a7809dc25bd3efcb35e5a542d5f709f937396b 100644
--- a/theia_dumper/stac.py
+++ b/theia_dumper/stac.py
@@ -1,14 +1,16 @@
+"""STAC stuff."""
+
import pystac
-from pystac import Collection, ItemCollection, Item, Asset
+from pystac import Collection, ItemCollection, Item
+
from .logger import logger
-from urllib.parse import urlparse, urljoin
-from typing import List, Any
+from urllib.parse import urljoin
+from typing import List
import os
import dinamis_sdk
import requests
from requests.adapters import HTTPAdapter, Retry
from dataclasses import dataclass
-from functools import singledispatch
class STACObjectUnresolved(Exception):
@@ -25,7 +27,7 @@ def create_session():
retries = Retry(
total=5,
backoff_factor=1,
- status_forcelist=[400, 401, 403, 408, 410, 419, 421, 422,
+ status_forcelist=[400, 403, 408, 410, 419, 421, 422,
424, 425, 429, 500, 502, 503, 504, 505],
allowed_methods=frozenset(["PUT", "POST"])
)
@@ -63,10 +65,11 @@ def load(obj_pth):
"item collection": ItemCollection,
"item": Item
}.items():
+ logger.debug("Try to read file %s", obj_pth)
try:
obj = getattr(cls, "from_file")(obj_pth)
- logger.info("Loaded %s", obj_name)
- logger.info(obj.to_dict())
+ logger.info("Loaded %s from file %s", obj_name, obj_pth)
+ logger.debug(obj.to_dict())
return obj
except pystac.errors.STACTypeError:
pass
@@ -76,11 +79,14 @@ def load(obj_pth):
def get_assets_root_dir(items: List[Item]) -> str:
"""Get the common prefix of all items assets paths."""
- return os.path.commonprefix([
+ prefix = os.path.commonprefix([
asset.href
for item in items
for asset in item.assets.values()
])
+ if os.path.isdir(prefix):
+ return prefix
+ return os.path.dirname(prefix)
def check_items_collection_id(items: List[Item]):
@@ -91,10 +97,23 @@ def check_items_collection_id(items: List[Item]):
)
-def get_col_items(col: pystac.Collection) -> List[Item]:
+def get_col_href(col: Collection):
+ """Retrieve collection href."""
+ for link in col.links:
+ if link.rel == "self":
+ return link.href
+
+
+def get_col_items(col: Collection) -> List[Item]:
"""Retrieve collection items."""
+ col_href = get_col_href(col=col)
return [
- load(link.href)
+ load(
+ os.path.join(
+ os.path.dirname(col_href),
+ link.href[2:]
+ ) if link.href.startswith("./") else link.href
+ )
for link in col.links
if link.rel == "item"
]
@@ -108,13 +127,6 @@ class TransactionsHandler:
storage_bucket: str
assets_overwrite: bool
- @singledispatch
- def publish(self, obj: Any) -> Any:
- """Publish a STAC object and its items/assets."""
- raise TypeError(
- "Invalid type, must be ItemCollection or Collection"
- )
-
def publish_item(self, item: Item, assets_root_dir: str):
"""Publish an item and all its assets"""
col_id = item.collection_id
@@ -127,10 +139,12 @@ class TransactionsHandler:
# Upload assets files
for local_filename in local_assets_files:
+ logger.debug("Local file: %s", local_filename)
target_url = local_filename.replace(
assets_root_dir,
target_root_dir
)
+ logger.debug("Target file: %s", target_url)
# Skip when target file exists and overwrite is not enabled
if not self.assets_overwrite:
@@ -153,43 +167,79 @@ class TransactionsHandler:
)
# Update assets hrefs
+ logger.debug("Updating assets HREFs ...")
for asset_name in item.assets:
item.assets[asset_name].href = target_url
# Push item
+ logger.info(
+ "Publishing item \"%s\" in collection \"%s\"",
+ item.id,
+ col_id
+ )
post_or_put(
urljoin(
self.stac_endpoint,
f"collections/{col_id}/items"
),
- item.to_dict()
+ item.to_dict(transform_hrefs=False)
)
def publish_items(self, items: List[Item]):
"""Publish items."""
check_items_collection_id(items=items)
+ assets_root_dir = get_assets_root_dir(items=items)
+ logger.debug("Assets root directory: %s", assets_root_dir)
for item in items:
self.publish_item(
item=item,
- assets_root_dir=get_assets_root_dir(items=items)
+ assets_root_dir=assets_root_dir
)
- @publish.register(Collection)
def publish_collection(self, collection: Collection):
- """Publish a collection and all its items."""
- items = get_col_items(col=collection)
- check_items_collection_id(items)
+ """Publish an empty collection"""
post_or_put(
url=urljoin(self.stac_endpoint, "/collections"),
data=collection.to_dict()
)
+
+ def publish_collection_with_items(self, collection: Collection):
+ """Publish a collection and all its items."""
+ items = get_col_items(col=collection)
+ check_items_collection_id(items)
+ self.publish_collection(collection=collection)
self.publish_items(items=items)
- @publish.register(ItemCollection)
def publish_item_collection(self, item_collection: ItemCollection):
"""Publish an item collection and all of its items."""
self.publish_items(items=item_collection.items)
def load_and_publish(self, obj_pth: str):
"""Load and publish the serialized STAC object"""
- self.publish(obj=load(obj_pth=obj_pth))
+ obj = load(obj_pth=obj_pth)
+ if isinstance(obj, Collection):
+ self.publish_collection_with_items(collection=obj)
+ elif isinstance(obj, ItemCollection):
+ self.publish_item_collection(item_collection=obj)
+ else:
+ raise TypeError(
+ "Invalid type, must be ItemCollection or Collection "
+ f"(got {type(obj)})"
+ )
+
+ def delete(self, col_id: str, item_id: str = None):
+ """Delete an item or a collection."""
+ logger.info("Delete item %s", item_id)
+ if item_id:
+ url = f"{self.stac_endpoint}/collections/{col_id}/items/{item_id}"
+ else:
+ url = f"{self.stac_endpoint}/collections/{col_id}"
+ resp = requests.delete(
+ url,
+ headers={
+ "Authorization":
+ f"Bearer {dinamis_sdk.auth.get_access_token()}"
+ }
+ )
+ if resp.status_code != 200:
+ logger.warning("Deletion failed")