Product bundles

packages/gnss-product-management/src/gnss_product_management/client/gnss_client.py

@@ -282,6 +282,8 @@ def resolve_dependencies(
         *,
         sink_id: str,
         force_download: bool = False,
+        centers: list[str] | None = None,
+        bundle_centers: list[str] | None = None,
     ) -> tuple[DependencyResolution, AnyPath | None]:
         """Resolve all dependencies in a spec for the given date.
 
@@ -298,6 +300,8 @@ def resolve_dependencies(
             date: Target date (timezone-aware datetime, midnight UTC).
             sink_id: Local resource alias for storing resolved files
                 (e.g. ``"local"``).
+            centers: Optional remote resource IDs to search.  When omitted,
+                all configured centers are eligible.
 
         Returns:
             A ``(DependencyResolution, lockfile_path)`` tuple.
@@ -318,4 +322,6 @@ def resolve_dependencies(
             date,
             sink_id=sink_id,
             force_download=force_download,
+            centers=centers,
+            bundle_centers=bundle_centers,
         )

packages/gnss-product-management/src/gnss_product_management/client/product_query.py

@@ -12,7 +12,6 @@
 from gnss_product_management.factories.models import FoundResource
 from gnss_product_management.factories.ranking import (
     sort_by_preferences,
-    sort_by_protocol,
 )
 from gnss_product_management.factories.remote_transport import WormHole
 from gnss_product_management.factories.search_planner import SearchPlanner
@@ -226,13 +225,12 @@ def _ranked_targets(self) -> list[SearchTarget]:
         """Return sorted :class:`SearchTarget` candidates before deduplication.
 
         Builds queries via :class:`SearchPlanner`, expands them through
-        :class:`WormHole`, then applies preference and protocol sorting.
+        :class:`WormHole`, then applies configured preference sorting.
         The result is the pre-deduplication list used both by :meth:`search`
         and by :class:`ResolvePipeline`.
 
         Returns:
-            Sorted list of :class:`SearchTarget` objects, local/file first,
-            then ordered by *preferences* within each protocol tier.
+            Sorted list of :class:`SearchTarget` objects ordered by *preferences*.
 
         Raises:
             ValueError: If :meth:`for_product` or :meth:`on` have not been called.
@@ -266,7 +264,7 @@ def _ranked_targets(self) -> list[SearchTarget]:
         )
         if self._preferences:
             expanded = sort_by_preferences(expanded, self._preferences)
-        return sort_by_protocol(expanded)
+        return expanded
 
     def _search_range(self) -> list[FoundResource]:
         """Execute a search for every date in the configured range.
@@ -311,8 +309,7 @@ def search(self) -> list[FoundResource]:
 
         Returns:
             Ranked list of :class:`FoundResource` objects, best first.
-            Local/file results precede remote ones; within each protocol
-            tier results are ordered by *preferences*.
+            Results are ordered by *preferences*, independently of server protocol.
             When :meth:`on_range` was used, results from all dates are
             returned as a flat combined list.
 
@@ -348,6 +345,13 @@ def search(self) -> list[FoundResource]:
                 continue
             seen[key] = True
             params = {p.name: p.value for p in rq.product.parameters if p.value is not None}
+            # Search targets may retain wildcard regexes for unconstrained
+            # filename axes (for example TTT/PPP on Wuhan ORBIT).  Replace
+            # those patterns with metadata parsed from the actual match so
+            # callers can group candidates by their real product family.
+            classified = self._search_planner._product_registry.classify(filename)
+            if classified and classified.get("product") == rq.product.name:
+                params.update(classified.get("parameters", {}))
             protocol = (rq.server.protocol or "").upper()
             is_local = protocol in ("FILE", "LOCAL")
             if is_local:

packages/gnss-product-management/src/gnss_product_management/factories/connection_pool.py

@@ -2,21 +2,22 @@
 
 import logging
 import os
+import random
 import re
+import shutil
 import threading
+import time
 from contextlib import contextmanager
 from pathlib import Path
 from typing import Optional
 from urllib.parse import quote, urlparse
+from urllib.request import Request, urlopen
 
 import fsspec
 import fsspec.utils
 
 logger = logging.getLogger(__name__)
 
-_AIUB_DOWNLOAD_ROOT = "https://www.aiub.unibe.ch/download"
-_AIUB_LISTING_ENDPOINT = "https://code.aiub.unibe.ch/s3_script/aiub_s3_bucket_listing.php?path="
-
 
 class ConnectionPool:
     """Thread-safe pool of fsspec filesystem instances for a single host.
@@ -27,16 +28,24 @@ class ConnectionPool:
         max_connections: Maximum number of concurrent connections.
     """
 
-    def __init__(self, hostname: str, max_connections: int = 4):
+    def __init__(
+        self,
+        hostname: str,
+        max_connections: int = 4,
+        listing_url: str | None = None,
+    ):
         """Initialise a connection pool for *hostname*.
 
         Args:
             hostname: Server address or local path.
             max_connections: Maximum number of concurrent connections.
+            listing_url: Optional URL template for a separate HTML directory
+                listing. ``{directory}`` is replaced with the quoted path.
         """
         self.hostname = hostname
         self.protocol = fsspec.utils.get_protocol(hostname) or "file"
         self.max_connections = max_connections
+        self.listing_url = listing_url
         self._pool: list[fsspec.AbstractFileSystem] = []
         self._semaphore: threading.Semaphore | None = None
         self._pool_lock = threading.Lock()
@@ -197,15 +206,22 @@ def __init__(self, max_connections: int = 4):
         self._listing_cache: dict[str, list[str]] = {}
         self._listing_cache_lock = threading.Lock()
 
-    def add_connection(self, hostname: str):
+    def add_connection(self, hostname: str, *, listing_url: str | None = None):
         """Ensure a connection pool exists for *hostname*.
 
         Args:
             hostname: Server address to pool.
+            listing_url: Optional separate HTML directory-listing URL template.
         """
         with self._factory_lock:
             if hostname not in self._pools:
-                self._pools[hostname] = ConnectionPool(hostname, self.max_connections)
+                self._pools[hostname] = ConnectionPool(
+                    hostname,
+                    self.max_connections,
+                    listing_url=listing_url,
+                )
+            elif self._pools[hostname].listing_url != listing_url:
+                raise ValueError(f"Conflicting listing configuration for: {hostname}")
 
     @contextmanager
     def get_connection(self, hostname: str):
@@ -253,12 +269,13 @@ def list_directory(self, hostname: str, directory: str) -> list[str]:
         full_path = pool.full_path(directory)
 
         def _ls(conn: "fsspec.AbstractFileSystem") -> list[str]:
-            if hostname.rstrip("/") == _AIUB_DOWNLOAD_ROOT:
-                listing_url = _AIUB_LISTING_ENDPOINT + quote(directory.strip("/"), safe="/")
+            if pool.listing_url:
+                quoted_directory = quote(directory.strip("/"), safe="/")
+                listing_url = pool.listing_url.format(directory=quoted_directory)
                 html = conn.cat_file(listing_url)
                 if isinstance(html, bytes):
                     html = html.decode("utf-8", errors="replace")
-                download_prefix = re.escape(f"{_AIUB_DOWNLOAD_ROOT}/{directory.strip('/')}/")
+                download_prefix = re.escape(f"{hostname.rstrip('/')}/{directory.strip('/')}/")
                 return sorted(set(re.findall(rf'href="{download_prefix}([^"/]+)"', html)))
             raw = conn.ls(full_path, detail=False)
             return [Path(p).name for p in raw]
@@ -346,6 +363,30 @@ def download_file(self, hostname: str, remote_path: str, target_dir: str) -> Pat
         filename = Path(remote_path).name
         local_path = Path(target_dir) / filename
 
+        # GithubFileSystem's Contents API path can return only the first 4 MiB
+        # of larger blobs.  Use GitHub's immutable raw-content endpoint for
+        # the actual transfer while retaining GithubFileSystem for listings.
+        if pool.protocol == "github":
+            match = re.fullmatch(r"github://([^:]+):([^@]+)@(.+)", hostname)
+            if match:
+                owner, repository, revision = match.groups()
+                raw_url = (
+                    f"https://raw.githubusercontent.com/{owner}/{repository}/"
+                    f"{revision}/{remote_path.lstrip('/')}"
+                )
+                try:
+                    request = Request(raw_url, headers={"Accept-Encoding": "identity"})
+                    with (
+                        urlopen(request, timeout=60) as source,
+                        local_path.open("wb") as destination,
+                    ):
+                        shutil.copyfileobj(source, destination, length=1024 * 1024)
+                    return local_path if local_path.stat().st_size > 0 else None
+                except Exception as exc:
+                    logger.warning("Raw GitHub download failed for %s: %s", raw_url, exc)
+                    local_path.unlink(missing_ok=True)
+                    return None
+
         def _get(conn: "fsspec.AbstractFileSystem") -> Path | None:
             conn.get(full_path, str(local_path))
             if local_path.exists() and local_path.stat().st_size > 0:
@@ -359,6 +400,8 @@ def _get(conn: "fsspec.AbstractFileSystem") -> Path | None:
                 return _get(conn)
             except (BrokenPipeError, ConnectionError, EOFError, OSError) as e:
                 logger.debug("Stale connection for %s, reconnecting: %s", hostname, e)
+                if pool.protocol != "file":
+                    time.sleep(random.uniform(1.0, 3.0))
                 fresh = pool.replace_connection(conn)
                 if fresh is None:
                     return None