Source code for brainglobe_atlasapi.remotes

"""Work out which remote an atlas comes from and where it is cached.

Each remote is mirrored in its own directory under the brainglobe
directory, holding a record of the remote it mirrors.
"""

from collections.abc import Callable
from pathlib import Path
from typing import Optional

from brainglobe_atlasapi import descriptors
from brainglobe_atlasapi.descriptors import V3_ATLAS_ROOTDIR


[docs] def cache_key(remote: str) -> str: """Return the local cache directory name for a remote. The BrainGlobe remote keeps its historical directory; every other remote is cached under the last segment of its URL, e.g. ``s3://bucket/prefix/allen-atlas-assets-rc11`` under ``allen-atlas-assets-rc11``. Parameters ---------- remote : str Remote, without a trailing slash. Returns ------- str Directory name, relative to the brainglobe directory. Raises ------ ValueError If a non-default remote would take the default remote's directory. """ if remote == descriptors.DEFAULT_REMOTE: return descriptors.DEFAULT_REMOTE_KEY key = remote.removeprefix("s3://").rsplit("/", 1)[-1] if key == descriptors.DEFAULT_REMOTE_KEY: raise ValueError( f"{remote} would share the cache directory reserved " f"for {descriptors.DEFAULT_REMOTE}." ) return key
[docs] def check_remote(cache_dir: Path, remote: str) -> None: """Check a cache directory does not mirror a different remote. Two remotes sharing a last URL segment map to the same directory; the record written by ``record_remote`` keeps them from mixing. Parameters ---------- cache_dir : Path Local cache directory for ``remote``. remote : str Remote, without a trailing slash. Raises ------ ValueError If the directory already records a different remote. """ record = cache_dir / descriptors.REMOTE_FILENAME if record.exists(): recorded = record.read_text().strip() if recorded != remote: raise ValueError( f"{cache_dir} holds atlases from {recorded}, not " f"{remote}. Remotes sharing a last URL segment " "cannot share a cache directory." )
[docs] def record_remote(cache_dir: Path, remote: str) -> None: """Record which remote a cache directory mirrors, if unrecorded. Called only once the directory holds an atlas, so a failed lookup (a typo'd remote or atlas name) never claims the directory. Parameters ---------- cache_dir : Path Local cache directory for ``remote``. remote : str Remote, without a trailing slash. """ record = cache_dir / descriptors.REMOTE_FILENAME if record.exists(): return try: record.write_text(f"{remote}\n") except OSError: # A read-only shared cache still loads; it just goes unrecorded. pass
[docs] def resolve_remote( fs, atlas_name: str, remotes, is_reachable: Callable[[str], bool] ) -> Optional[str]: """Find which configured remote holds an atlas. Remotes are searched in declaration order, so the first-declared remote wins when a name exists in more than one bucket. Each remote is probed for reachability only when its turn comes, and unreachable remotes are skipped, so finding an atlas on the first remote costs two requests. Parameters ---------- fs : s3fs.S3FileSystem Filesystem used to list the atlas directory. atlas_name : str Name of the atlas to locate. remotes : dict Mapping of remote key to remote, in resolution order. is_reachable : callable Returns whether a remote answers. Returns ------- str or None The winning remote, or None if no remote is reachable. Raises ------ ValueError If the atlas name is present in none of the reachable remotes. """ searched = [] for remote in remotes.values(): if not is_reachable(remote): continue searched.append(remote) try: if fs.ls(f"{remote}/{V3_ATLAS_ROOTDIR}/{atlas_name}"): return remote except FileNotFoundError: # Not on this remote (ls may raise or return []): try the next. pass if not searched: return None raise ValueError( f"{atlas_name} is not a valid atlas name! " f"Searched remotes: {', '.join(searched)}" )
[docs] def known_remotes(base_dir: Path, remotes: dict) -> list: """List the remotes whose local cache may hold an atlas. Parameters ---------- base_dir : Path Directory holding one cache directory per remote. remotes : dict Mapping of remote key to configured remote. Returns ------- list of str Configured remotes in config order, then any other remote recorded under ``base_dir``, without duplicates. """ records = sorted(base_dir.glob(f"*/{descriptors.REMOTE_FILENAME}")) recorded_remotes = [record.read_text().strip() for record in records] return list(dict.fromkeys([*remotes.values(), *recorded_remotes]))