Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 18 additions & 7 deletions eodag/plugins/download/aws.py
Original file line number Diff line number Diff line change
Expand Up @@ -267,23 +267,30 @@ def download(
)
progress_callback = ProgressCallback(disable=True)

product_conf = getattr(self.config, "products", {}).get(product.collection, {})
asset_filter = kwargs.get("asset")
ignore_assets = (
False if asset_filter else getattr(self.config, "ignore_assets", False)
)
should_download_assets = len(product.assets) > 0 and (
not ignore_assets or bool(asset_filter)
)

# prepare download & create dirs (before updating metadata)
product_local_path, record_filename = self._download_preparation(
product, progress_callback=progress_callback, **kwargs
product,
progress_callback=progress_callback,
download_assets=should_download_assets,
**kwargs,
)
if not record_filename or not product_local_path:
return product_local_path

product_conf = getattr(self.config, "products", {}).get(product.collection, {})

# do not try to build SAFE if asset filter is used
asset_filter = kwargs.get("asset")
if asset_filter:
build_safe = False
ignore_assets = False
else:
build_safe = product_conf.get("build_safe", False)
ignore_assets = getattr(self.config, "ignore_assets", False)

# product conf overrides provider conf for "flatten_top_dirs"
flatten_top_dirs = product_conf.get(
Expand Down Expand Up @@ -556,6 +563,7 @@ def _download_preparation(
self,
product: EOProduct,
progress_callback: ProgressCallback,
download_assets: bool = False,
**kwargs: Unpack[DownloadConf],
) -> tuple[str | None, str | None]:
"""Prepare the download.
Expand All @@ -571,7 +579,10 @@ def _download_preparation(
:return: local path and file name
"""
product_local_path, record_filename = self._prepare_download(
product, progress_callback=progress_callback, **kwargs
product,
progress_callback=progress_callback,
download_assets=download_assets,
**kwargs,
)
if not product_local_path or not record_filename:
if product_local_path:
Expand Down
9 changes: 6 additions & 3 deletions eodag/plugins/download/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,14 +183,14 @@ def _prepare_download(
self,
product: EOProduct,
progress_callback: ProgressCallback | None = None,
allow_missing_url: bool = False,
download_assets: bool = False,
**kwargs: Unpack[DownloadConf],
) -> tuple[str | None, str | None]:
"""Check if file has already been downloaded, and prepare product download

:param product: The EO product to download
:param progress_callback: (optional) A progress callback
:param allow_missing_url: (optional) Allow preparation when downloading assets without a product URL
:param download_assets: (optional) Allow preparation without a product URL when downloadable assets exist
:returns: fs_path, record_filename
"""
if product.location != product.remote_location:
Expand All @@ -204,7 +204,10 @@ def _prepare_download(
return fs_path, None

url = product.remote_location
if not url and not allow_missing_url:
has_downloadable_assets = download_assets and any(
"href" in asset for asset in product.assets.values()
)
if not url and not has_downloadable_assets:
logger.debug(
f"Unable to get download url for {product}, skipping download",
)
Expand Down
5 changes: 1 addition & 4 deletions eodag/plugins/download/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -647,13 +647,10 @@ def download(
should_download_assets = len(product.assets) > 0 and (
not self._should_ignore_assets(product) or kwargs.get("asset") is not None
)
has_downloadable_assets = should_download_assets and any(
"href" in asset for asset in product.assets.values()
)
fs_path, record_filename = self._prepare_download(
product,
progress_callback=progress_callback,
allow_missing_url=has_downloadable_assets,
download_assets=should_download_assets,
**kwargs,
)
if not fs_path or not record_filename:
Expand Down
42 changes: 42 additions & 0 deletions tests/units/test_download_plugins.py
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,31 @@ def test_plugins_download_base_prepare_download_no_url(self):
self.assertIsNone(fs_path)
self.assertIsNone(record_filename)

def test_plugins_download_base_prepare_download_assets_without_url(self):
"""Download._prepare_download allows selected assets without a product URL."""
self.product.assets.update({"foo": {"href": "s3://bucket/key"}})
plugin = self.get_download_plugin(self.product)

with TemporaryDirectory() as output_dir:
fs_path, record_filename = plugin._prepare_download(
self.product, download_assets=True, output_dir=output_dir
)

self.assertIsNotNone(fs_path)
self.assertIsNotNone(record_filename)

def test_plugins_download_base_prepare_download_assets_without_href(self):
"""Download._prepare_download still requires a URL if selected assets lack hrefs."""
self.product.assets.update({"foo": {"type": "application/octet-stream"}})
plugin = self.get_download_plugin(self.product)

fs_path, record_filename = plugin._prepare_download(
self.product, download_assets=True
)

self.assertIsNone(fs_path)
self.assertIsNone(record_filename)

def test_plugins_download_base_prepare_download_collision_avoidance(self):
"""Download._prepare_download must use collision avoidance suffix."""
self.product.properties["title"] = "needs sanitïze"
Expand Down Expand Up @@ -2257,6 +2282,23 @@ def setUp(self):
"http://somebucket.somehost.com/path/to/some/product"
)

def test_plugins_download_aws_prepare_download_assets_without_product_url(self):
"""AwsDownload preparation allows asset downloads when the product has no URL."""
plugin = self.get_download_plugin(self.product)
self.product.properties.pop("eodag:download_link", None)
self.product.location = self.product.remote_location = ""
self.product.assets.update({"foo": {"href": "s3://bucket/key"}})

with TemporaryDirectory() as output_dir:
product_path, record_filename = plugin._download_preparation(
self.product,
progress_callback=ProgressCallback(disable=True),
download_assets=True,
output_dir=output_dir,
)
self.assertTrue(os.path.isdir(product_path))
self.assertIsNotNone(record_filename)

def test_plugins_download_aws_get_bucket_prefix(self):
"""AwsDownload.get_product_bucket_name_and_prefix() must extract bucket & prefix from location."""
plugin = self.get_download_plugin(self.product)
Expand Down
Loading