From 5be0f84cb6b3f25d770470d07033a7d9336847c1 Mon Sep 17 00:00:00 2001 From: Kevin Barnard Date: Thu, 19 Nov 2026 12:29:03 -0800 Subject: [PATCH 1/5] feat: decouple the Dial proxy from MOLARS types; add lcmtypes extra - The Dial proxy no longer imports senlcm/stdlcm. It resolves the image type by LCM name (--image-type, default senlcm.image_t) from the discovered packages, so it works with prefixed packages such as compas_lcmtypes or with unprefixed lcm-gen output. - Both proxies accept --lcm-packages. The Dial proxy defaults to compas_lcmtypes,molars_lcmtypes; the JSON proxy keeps its positional argument for compatibility. - New 'lcmtypes' extra installs compas-lcmtypes (PyPI) and molars-lcmtypes (CoMPASLab/molars_lcmtypes GitHub release). - Configuration errors (no types found, unknown image type, failed workers) now exit non-zero. - Tests and benchmarks use compas-lcmtypes instead of the vendored MOLARS copies, which are removed; benchmarks no longer need PYTHONPATH. Co-Authored-By: Claude Opus 5.5 --- .pre-commit-config.yaml | 2 +- benchmarks/encode.py | 12 +- benchmarks/load.py | 7 +- lcm_websocket_server/apps/dial_proxy.py | 62 +++++----- lcm_websocket_server/apps/json_proxy.py | 11 +- lcm_websocket_server/lib/cli.py | 15 ++- lcm_websocket_server/lib/lcm_utils/types.py | 16 +++ pyproject.toml | 13 +- tests/conftest.py | 10 +- tests/lcmtypes/README.md | 8 -- tests/lcmtypes/senlcm/__init__.py | 6 - tests/lcmtypes/senlcm/image_metadata_t.py | 74 ------------ tests/lcmtypes/senlcm/image_t.py | 126 -------------------- tests/lcmtypes/stdlcm/__init__.py | 5 - tests/lcmtypes/stdlcm/header_t.py | 72 ----------- tests/test_cli.py | 97 +++++++++++---- tests/test_dial_handler.py | 11 +- tests/test_json_handler.py | 2 +- tests/test_pubsub.py | 2 +- tests/test_server.py | 2 +- tests/test_types.py | 21 +++- uv.lock | 35 +++++- 22 files changed, 228 insertions(+), 381 deletions(-) delete mode 100644 tests/lcmtypes/README.md delete mode 100644 tests/lcmtypes/senlcm/__init__.py delete mode 100644 tests/lcmtypes/senlcm/image_metadata_t.py delete mode 100644 tests/lcmtypes/senlcm/image_t.py delete mode 100644 tests/lcmtypes/stdlcm/__init__.py delete mode 100644 tests/lcmtypes/stdlcm/header_t.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d88f97c..99b829c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -exclude: ^(tests/lcmtypes/|lcm_websocket_server/lib/lcm_utils/channel_stats(_list)?\.py$) +exclude: ^(lcm_websocket_server/lib/lcm_utils/channel_stats(_list)?\.py$) repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.17.6 diff --git a/benchmarks/encode.py b/benchmarks/encode.py index 0dd454f..2755714 100644 --- a/benchmarks/encode.py +++ b/benchmarks/encode.py @@ -2,22 +2,22 @@ Microbenchmark of JSONHandler.handle (decode + JSON encode) on the stdlib and orjson paths. -Usage (requires stdlcm/senlcm/navlcm/geolcm on PYTHONPATH): +Usage (requires the lcmtypes extra: uv sync --all-extras): uv run python benchmarks/encode.py """ import time -from geolcm import point_t +from compas_lcmtypes.geolcm import point_t +from compas_lcmtypes.navlcm import odometry_t +from compas_lcmtypes.senlcm import multibeam_euclidean_t +from compas_lcmtypes.stdlcm import header_t from lcmutils import LCMTypeRegistry -from navlcm import odometry_t -from senlcm import multibeam_euclidean_t -from stdlcm import header_t from lcm_websocket_server.apps.json_proxy import JSONHandler from lcm_websocket_server.lib.lcm_utils import types as T reg = LCMTypeRegistry() -for p in ("stdlcm", "senlcm", "navlcm", "geolcm"): reg.discover(p) +reg.discover("compas_lcmtypes") hdr = header_t(); hdr.frame_id = "base" odo = odometry_t(); odo.header = hdr mb = multibeam_euclidean_t(); mb.header = hdr; mb.n_beams = 512 diff --git a/benchmarks/load.py b/benchmarks/load.py index 0c4bc5c..b6489ae 100644 --- a/benchmarks/load.py +++ b/benchmarks/load.py @@ -9,7 +9,7 @@ end-to-end latency (from the header_t timestamp stamped at publish) and the spy channel's reported rates. -Usage (requires stdlcm/senlcm/navlcm/geolcm on PYTHONPATH): +Usage (requires the lcmtypes extra: uv sync --all-extras): uv run python benchmarks/load.py SUITE [--duration S] Suites: latency, json-scale, json-rate, dial, slow, periodic, all @@ -92,6 +92,7 @@ def run_server(port, mode, server_kwargs, streams, ready, go, window_end, stop, import logging import cv2 + from compas_lcmtypes.senlcm import image_t from lcm_websocket_server.apps.dial_proxy import ( DialHandler, @@ -113,11 +114,11 @@ def emit(self, record): pubsub._logger.addHandler(DropCounter()) - registry = build_registry(["stdlcm", "senlcm", "navlcm", "geolcm"], logging.getLogger("bench")) + registry = build_registry(["compas_lcmtypes"], logging.getLogger("bench")) json_handler = JSONHandler(registry) if mode == "dial": encoder = DownsamplingMJPEGEncoder(scale=1.0, params=[cv2.IMWRITE_JPEG_QUALITY, 75]) - inner = DialHandler(ImageMessageToJPEGHandler(encoder), json_handler) + inner = DialHandler(ImageMessageToJPEGHandler(encoder, image_t), json_handler) else: inner = json_handler diff --git a/lcm_websocket_server/apps/dial_proxy.py b/lcm_websocket_server/apps/dial_proxy.py index b3eda7e..b00af62 100644 --- a/lcm_websocket_server/apps/dial_proxy.py +++ b/lcm_websocket_server/apps/dial_proxy.py @@ -10,8 +10,6 @@ import cv2 from lcmlog.event import Header from numpy import ndarray -from senlcm import image_t -from stdlcm import header_t from lcm_websocket_server.apps.json_proxy import JSONHandler from lcm_websocket_server.lib.cli import ( @@ -31,6 +29,7 @@ get_decoder, ) from lcm_websocket_server.lib.lcm_utils.pubsub import LCMRepublisher +from lcm_websocket_server.lib.lcm_utils.types import find_lcm_type from lcm_websocket_server.lib.log import LogMixin, get_logger from lcm_websocket_server.lib.server import DEFAULT_HEAVY_WORKERS, LCMWebSocketServer @@ -39,14 +38,20 @@ class ImageMessageToJPEGHandler(LCMWebSocketHandler, LogMixin): """ - Handler that converts image_t LCM messages to JPEG. + Handler that converts image LCM messages (senlcm.image_t layout) to JPEG. Decoder instances are cached by (pixelformat, width, height) so a new object is not allocated on every frame. """ - def __init__(self, encoder: MJPEGEncoder): + def __init__(self, encoder: MJPEGEncoder, image_type: type): + """ + Args: + encoder: JPEG encoder. + image_type: The LCM image class (e.g. compas_lcmtypes.senlcm.image_t). + """ self._encoder = encoder + self.image_type = image_type self._decoder_cache: dict[tuple, object] = {} def _get_decoder(self, pixelformat: int, width: int, height: int): @@ -62,13 +67,13 @@ def _get_decoder(self, pixelformat: int, width: int, height: int): def is_heavy(self, channel: str, data: bytes) -> bool: return True - def handle(self, channel: str, data: bytes | image_t) -> bytes | None: - # Accept either raw bytes or a pre-decoded image_t object. - if isinstance(data, image_t): + def handle(self, channel: str, data) -> bytes | None: + # Accept either raw bytes or a pre-decoded image object. + if isinstance(data, self.image_type): image_event = data else: try: - image_event = image_t.decode(data) + image_event = self.image_type.decode(data) except Exception as e: # noqa: BLE001 - malformed input must not propagate self.logger.warning(f"Failed to decode image_t event from channel {channel}: {e}") return None @@ -125,8 +130,6 @@ class DialHandler(LCMWebSocketHandler, LogMixin): The JSON handler generates UTF-8 JSON from the LCM message, and sends the result as a text frame over the WebSocket. """ - IMAGE_T_FINGERPRINT = image_t._get_packed_fingerprint() - def __init__( self, image_handler: ImageMessageToJPEGHandler, @@ -144,6 +147,7 @@ def __init__( raise ValueError("max_image_fps must be positive") self._image_handler = image_handler self._json_handler = json_handler + self._image_fingerprint = image_handler.image_type._get_packed_fingerprint() self._image_interval_ns = int(1e9 / max_image_fps) if max_image_fps else 0 self._next_image_ns: dict[str, int] = {} self._rate_lock = threading.Lock() @@ -177,8 +181,8 @@ def _encode_image_t(self, channel: str, data: bytes) -> bytes | None: The encoded binary frame, or None if the message could not be encoded. """ # Decode the image_t to get the payload header timestamp. - image_event = image_t.decode(data) - payload_header: header_t = image_event.header + image_event = self._image_handler.image_type.decode(data) + payload_header = image_event.header # Encode the image as JPEG (pass pre-decoded object to skip a second decode). jpeg_bytes = self._image_handler.handle(channel, image_event) @@ -201,26 +205,19 @@ def _encode_image_t(self, channel: str, data: bytes) -> bytes | None: return header_bytes + channel_name_utf8 + jpeg_bytes def is_heavy(self, channel: str, data: bytes) -> bool: - return data[:8] == DialHandler.IMAGE_T_FINGERPRINT + return data[:8] == self._image_fingerprint def handle(self, channel: str, data: bytes) -> bytes | str | None: - if data[:8] == DialHandler.IMAGE_T_FINGERPRINT: + if data[:8] == self._image_fingerprint: if not self._image_due(channel, monotonic_ns()): return None return self._encode_image_t(channel, data) return self._json_handler.handle(channel, data) -DIAL_LCM_PACKAGES = ( - "stdlcm", - "senlcm", - "navlcm", - "geolcm", - "lcm_action_server", - "mission_server", - "nav_server", - "control_server", -) +# LCM packages discovered by default (MOLARS types plus the CoMPAS types they build on) +DEFAULT_LCM_PACKAGES = ("compas_lcmtypes", "molars_lcmtypes") +DEFAULT_IMAGE_TYPE = "senlcm.image_t" async def run( @@ -229,6 +226,7 @@ async def run( quality: int = 75, image_workers: int = DEFAULT_HEAVY_WORKERS, max_fps: float | None = None, + image_type: str = DEFAULT_IMAGE_TYPE, ) -> None: """ Run the LCM WebSocket Dial proxy server. @@ -239,14 +237,20 @@ async def run( quality: The JPEG quality level. Clamped to the range [0, 100]. image_workers: Threads for image transcoding. max_fps: Optional per-channel cap on transcoded image frames per second. + image_type: LCM name of the image type to transcode to JPEG. """ - registry = build_registry(DIAL_LCM_PACKAGES, logger) + registry = build_registry(options.lcm_packages, logger) if registry is None: - return + raise SystemExit(1) + try: + image_cls = find_lcm_type(registry.types, image_type) + except LookupError as e: + logger.critical(f"{e}. Check --lcm-packages and --image-type.") + raise SystemExit(1) from None quality = round(max(0, min(quality, 100))) jpeg_encoder = DownsamplingMJPEGEncoder(scale=scale, params=[cv2.IMWRITE_JPEG_QUALITY, quality]) - handler = DialHandler(ImageMessageToJPEGHandler(jpeg_encoder), JSONHandler(registry), max_image_fps=max_fps) + handler = DialHandler(ImageMessageToJPEGHandler(jpeg_encoder, image_cls), JSONHandler(registry), max_image_fps=max_fps) logger.debug(f"Creating LCM republisher for channel '{options.channel}'") republisher = LCMRepublisher(options.channel) @@ -267,10 +271,11 @@ def main(): Entry point for the LCM WebSocket Dial proxy server. """ parser = argparse.ArgumentParser(description=__doc__) - add_server_args(parser, compression_default="none") + add_server_args(parser, compression_default="none", lcm_packages_default=",".join(DEFAULT_LCM_PACKAGES)) parser.add_argument("--scale", type=float, default=1.0, help="The scale factor to resize the image by. Default: %(default)s") parser.add_argument("--quality", type=int, default=75, help="The JPEG quality level, 0-100. Default: %(default)s") parser.add_argument("--max-fps", type=float, default=None, help="Cap on transcoded frames per second per image channel. Default: no cap") + parser.add_argument("--image-type", type=str, default=DEFAULT_IMAGE_TYPE, help="LCM name of the image type to transcode to JPEG. Default: %(default)s") parser.add_argument("--image-workers", type=int, default=DEFAULT_HEAVY_WORKERS, help="Threads for image transcoding. Default: %(default)s") args = parser.parse_args() @@ -283,6 +288,7 @@ def main(): quality=args.quality, image_workers=args.image_workers, max_fps=args.max_fps, + image_type=args.image_type, ), options, logger, diff --git a/lcm_websocket_server/apps/json_proxy.py b/lcm_websocket_server/apps/json_proxy.py index 59fb8bf..e7c216f 100644 --- a/lcm_websocket_server/apps/json_proxy.py +++ b/lcm_websocket_server/apps/json_proxy.py @@ -69,17 +69,16 @@ def handle(self, channel: str, data: bytes) -> TextPayload | None: return TextPayload(encode_event_json_bytes(channel, fingerprint_hex, event)) -async def run(options: ServerOptions, lcm_packages: list[str]) -> None: +async def run(options: ServerOptions) -> None: """ Run the LCM WebSocket JSON proxy server. Args: options: Common server options - lcm_packages: LCM packages to discover types from """ - registry = build_registry(lcm_packages, logger) + registry = build_registry(options.lcm_packages, logger) if registry is None: - return + raise SystemExit(1) logger.debug(f"Creating LCM republisher for channel '{options.channel}'") republisher = LCMRepublisher(options.channel) @@ -100,11 +99,11 @@ def main(): """ parser = argparse.ArgumentParser(description=__doc__) add_server_args(parser, compression_default="deflate") - parser.add_argument("lcm_packages", type=str, help="The LCM packages to discover LCM types from. Separate multiple packages with a comma.") + parser.add_argument("lcm_packages_positional", nargs="?", metavar="LCM_PACKAGES", help="Same as --lcm-packages (kept for compatibility).") args = parser.parse_args() options = parse_server_options(args, logger) - run_app(app_factory(run, options, args.lcm_packages.split(",")), options, logger) + run_app(app_factory(run, options), options, logger) if __name__ == "__main__": diff --git a/lcm_websocket_server/lib/cli.py b/lcm_websocket_server/lib/cli.py index 2d19e2c..b6a8ad7 100644 --- a/lcm_websocket_server/lib/cli.py +++ b/lcm_websocket_server/lib/cli.py @@ -30,6 +30,7 @@ class ServerOptions: """Options common to all proxy apps.""" + lcm_packages: tuple[str, ...] = () host: str = "localhost" port: int = 8765 channel: str = ".*" @@ -54,8 +55,11 @@ def server_kwargs(self) -> dict[str, Any]: } -def add_server_args(parser: argparse.ArgumentParser, compression_default: str) -> None: +def add_server_args( + parser: argparse.ArgumentParser, compression_default: str, lcm_packages_default: str | None = None +) -> None: """Add the common server arguments to a parser.""" + parser.add_argument("--lcm-packages", type=str, default=lcm_packages_default, help="Comma-separated Python packages to discover LCM types from (searched recursively). Default: %(default)s") parser.add_argument("--host", type=str, default="localhost", help="The host to listen on. Default: %(default)s") parser.add_argument("--port", type=int, default=8765, help="The port to listen on. Default: %(default)s") parser.add_argument("--channel", type=str, default=".*", help="The LCM channel to subscribe to. Use '.*' to subscribe to all channels.") @@ -77,7 +81,12 @@ def parse_server_options(args: argparse.Namespace, logger: logging.Logger) -> Se logger.warning("--poll-interval is deprecated and ignored; delivery is now event-driven") if args.processes < 1: raise SystemExit("--processes must be >= 1") + # An app may also accept the packages positionally (lcm_packages_positional) + packages = getattr(args, "lcm_packages_positional", None) or args.lcm_packages + if not packages: + raise SystemExit("No LCM packages given; pass --lcm-packages PKG[,PKG...]") return ServerOptions( + lcm_packages=tuple(p.strip() for p in packages.split(",") if p.strip()), host=args.host, port=args.port, channel=args.channel, @@ -187,6 +196,10 @@ def terminate(*_): terminate() for p in procs: p.join(timeout=5) + # Negative exit codes are signals (e.g. our own SIGTERM on shutdown) + failed = [p.name for p in procs if p.exitcode is not None and p.exitcode > 0] + if failed: + raise SystemExit(f"Worker(s) failed: {', '.join(failed)}") logger.info("Stopped") diff --git a/lcm_websocket_server/lib/lcm_utils/types.py b/lcm_websocket_server/lib/lcm_utils/types.py index e588000..19c7572 100644 --- a/lcm_websocket_server/lib/lcm_utils/types.py +++ b/lcm_websocket_server/lib/lcm_utils/types.py @@ -7,6 +7,7 @@ """ import json import math +from collections.abc import Iterable from typing import Any try: @@ -30,6 +31,21 @@ def _lcm_slots(tp: type) -> tuple[str, ...] | None: return slots +def find_lcm_type(types: Iterable[type], name: str) -> type: + """ + Find a type by its LCM name (e.g. "senlcm.image_t"), regardless of any + Python package prefix it was generated with (e.g. compas_lcmtypes.senlcm.image_t). + + Raises: + LookupError: If no type, or more than one type, matches. + """ + matches = [t for t in types if t.__module__ == name or t.__module__.endswith("." + name)] + if len(matches) != 1: + found = ", ".join(f"{t.__module__}" for t in matches) or "none" + raise LookupError(f"Expected exactly one LCM type named {name!r}, found: {found}") + return matches[0] + + def _encode_sequence(values) -> Any: if not values: return [] diff --git a/pyproject.toml b/pyproject.toml index c25dcfe..899f642 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,11 @@ orjson = [ uvloop = [ "uvloop>=0.21", ] +# LCM types used by the Docker image and the Dial proxy defaults +lcmtypes = [ + "compas-lcmtypes>=0.4,<0.5", + "molars-lcmtypes @ https://github.com/CoMPASLab/molars_lcmtypes/releases/download/v0.1.0/molars_lcmtypes-0.1.0-py3-none-any.whl", +] [project.scripts] lcm-websocket-server = "lcm_websocket_server.apps.json_proxy:main" @@ -36,11 +41,16 @@ lcm-websocket-dial-proxy = "lcm_websocket_server.apps.dial_proxy:main" requires = ["hatchling"] build-backend = "hatchling.build" +[tool.hatch.metadata] +# molars-lcmtypes is published as GitHub release assets, not on PyPI +allow-direct-references = true + [tool.hatch.build.targets.wheel] packages = ["lcm_websocket_server"] [dependency-groups] dev = [ + "compas-lcmtypes>=0.4,<0.5", "pre-commit>=4.2.0", "pytest>=8.0", "pytest-asyncio>=0.23", @@ -53,7 +63,7 @@ release = [ [tool.pytest.ini_options] asyncio_mode = "auto" testpaths = ["tests"] -pythonpath = ["tests/lcmtypes", "."] +pythonpath = ["."] asyncio_default_fixture_loop_scope = "function" [tool.ruff] @@ -62,7 +72,6 @@ target-version = "py311" extend-exclude = [ "lcm_websocket_server/lib/lcm_utils/channel_stats.py", "lcm_websocket_server/lib/lcm_utils/channel_stats_list.py", - "tests/lcmtypes", ] [tool.semantic_release] diff --git a/tests/conftest.py b/tests/conftest.py index 16c192d..611445a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,9 +19,9 @@ import numpy as np import pytest import pytest_asyncio +from compas_lcmtypes.senlcm import image_t +from compas_lcmtypes.stdlcm import header_t from lcmutils import LCMTypeRegistry -from senlcm import image_t -from stdlcm import header_t from lcm_websocket_server.apps.dial_proxy import ( DialHandler, @@ -99,9 +99,9 @@ def make_bgr_image_t(width: int = 8, height: int = 6, timestamp: int = 0) -> byt @pytest.fixture(scope="session") def registry() -> LCMTypeRegistry: - """Shared LCMTypeRegistry with stdlcm and senlcm types (plus internal spy types).""" + """Shared LCMTypeRegistry with the CoMPAS types (plus internal spy types).""" reg = LCMTypeRegistry() - for pkg in ("stdlcm", "senlcm"): + for pkg in ("compas_lcmtypes",): try: reg.discover(pkg) except ModuleNotFoundError: @@ -132,7 +132,7 @@ def json_handler(registry: LCMTypeRegistry) -> JSONHandler: @pytest.fixture def image_handler(jpeg_encoder: DownsamplingMJPEGEncoder) -> ImageMessageToJPEGHandler: """Fresh ImageMessageToJPEGHandler per test (fresh decoder cache).""" - return ImageMessageToJPEGHandler(jpeg_encoder) + return ImageMessageToJPEGHandler(jpeg_encoder, image_t) @pytest.fixture diff --git a/tests/lcmtypes/README.md b/tests/lcmtypes/README.md deleted file mode 100644 index bcf7249..0000000 --- a/tests/lcmtypes/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Test LCM types - -Python LCM types used by the test suite, copied unmodified from the lcm-gen output of -[MolaRS](https://github.com/CoMPASLab/molars) (`stdlcm.header_t`, `senlcm.image_t`, -`senlcm.image_metadata_t`), so the tests run without MolaRS installed. - -pytest puts this directory first on `sys.path` (see `pythonpath` in `pyproject.toml`). -Regenerate from MolaRS if those message definitions change. diff --git a/tests/lcmtypes/senlcm/__init__.py b/tests/lcmtypes/senlcm/__init__.py deleted file mode 100644 index 464fbc7..0000000 --- a/tests/lcmtypes/senlcm/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Subset of the MolaRS senlcm types used by the tests (generated by lcm-gen).""" - -from .image_metadata_t import image_metadata_t -from .image_t import image_t - -__all__ = ["image_metadata_t", "image_t"] diff --git a/tests/lcmtypes/senlcm/image_metadata_t.py b/tests/lcmtypes/senlcm/image_metadata_t.py deleted file mode 100644 index 6297e79..0000000 --- a/tests/lcmtypes/senlcm/image_metadata_t.py +++ /dev/null @@ -1,74 +0,0 @@ -"""LCM type definitions -This file automatically generated by lcm. -DO NOT MODIFY BY HAND!!!! -""" - -try: - import cStringIO.StringIO as BytesIO -except ImportError: - from io import BytesIO -import struct - -class image_metadata_t(object): - __slots__ = ["key", "n", "value"] - - __typenames__ = ["string", "int32_t", "byte"] - - __dimensions__ = [None, None, ["n"]] - - def __init__(self): - self.key = "" - self.n = 0 - self.value = b"" - - def encode(self): - buf = BytesIO() - buf.write(image_metadata_t._get_packed_fingerprint()) - self._encode_one(buf) - return buf.getvalue() - - def _encode_one(self, buf): - __key_encoded = self.key.encode('utf-8') - buf.write(struct.pack('>I', len(__key_encoded)+1)) - buf.write(__key_encoded) - buf.write(b"\0") - buf.write(struct.pack(">i", self.n)) - buf.write(bytearray(self.value[:self.n])) - - def decode(data): - if hasattr(data, 'read'): - buf = data - else: - buf = BytesIO(data) - if buf.read(8) != image_metadata_t._get_packed_fingerprint(): - raise ValueError("Decode error") - return image_metadata_t._decode_one(buf) - decode = staticmethod(decode) - - def _decode_one(buf): - self = image_metadata_t() - __key_len = struct.unpack('>I', buf.read(4))[0] - self.key = buf.read(__key_len)[:-1].decode('utf-8', 'replace') - self.n = struct.unpack(">i", buf.read(4))[0] - self.value = buf.read(self.n) - return self - _decode_one = staticmethod(_decode_one) - - def _get_hash_recursive(parents): - if image_metadata_t in parents: return 0 - tmphash = (0x4d25b1a682bbfdc7) & 0xffffffffffffffff - tmphash = (((tmphash<<1)&0xffffffffffffffff) + (tmphash>>63)) & 0xffffffffffffffff - return tmphash - _get_hash_recursive = staticmethod(_get_hash_recursive) - _packed_fingerprint = None - - def _get_packed_fingerprint(): - if image_metadata_t._packed_fingerprint is None: - image_metadata_t._packed_fingerprint = struct.pack(">Q", image_metadata_t._get_hash_recursive([])) - return image_metadata_t._packed_fingerprint - _get_packed_fingerprint = staticmethod(_get_packed_fingerprint) - - def get_hash(self): - """Get the LCM hash of the struct""" - return struct.unpack(">Q", image_metadata_t._get_packed_fingerprint())[0] - diff --git a/tests/lcmtypes/senlcm/image_t.py b/tests/lcmtypes/senlcm/image_t.py deleted file mode 100644 index 1ca795e..0000000 --- a/tests/lcmtypes/senlcm/image_t.py +++ /dev/null @@ -1,126 +0,0 @@ -"""LCM type definitions -This file automatically generated by lcm. -DO NOT MODIFY BY HAND!!!! -""" - -try: - import cStringIO.StringIO as BytesIO -except ImportError: - from io import BytesIO -import struct - -import stdlcm.header_t - -import senlcm.image_metadata_t - -class image_t(object): - __slots__ = ["header", "width", "height", "row_stride", "pixelformat", "size", "data", "nmetadata", "metadata"] - - __typenames__ = ["stdlcm.header_t", "int32_t", "int32_t", "int32_t", "int32_t", "int32_t", "byte", "int32_t", "senlcm.image_metadata_t"] - - __dimensions__ = [None, None, None, None, None, None, ["size"], None, ["nmetadata"]] - - PIXEL_FORMAT_UYVY = 1498831189 - PIXEL_FORMAT_YUYV = 1448695129 - PIXEL_FORMAT_IYU1 = 827677001 - PIXEL_FORMAT_IYU2 = 844454217 - PIXEL_FORMAT_YUV420 = 842093913 - PIXEL_FORMAT_YUV411P = 1345401140 - PIXEL_FORMAT_I420 = 808596553 - PIXEL_FORMAT_NV12 = 842094158 - PIXEL_FORMAT_GRAY = 1497715271 - PIXEL_FORMAT_RGB = 859981650 - PIXEL_FORMAT_BGR = 861030210 - PIXEL_FORMAT_RGBA = 876758866 - PIXEL_FORMAT_BGRA = 877807426 - PIXEL_FORMAT_BAYER_BGGR = 825770306 - PIXEL_FORMAT_BAYER_GBRG = 844650584 - PIXEL_FORMAT_BAYER_GRBG = 861427800 - PIXEL_FORMAT_BAYER_RGGB = 878205016 - PIXEL_FORMAT_BE_BAYER16_BGGR = 826360386 - PIXEL_FORMAT_BE_BAYER16_GBRG = 843137602 - PIXEL_FORMAT_BE_BAYER16_GRBG = 859914818 - PIXEL_FORMAT_BE_BAYER16_RGGB = 876692034 - PIXEL_FORMAT_LE_BAYER16_BGGR = 826360396 - PIXEL_FORMAT_LE_BAYER16_GBRG = 843137612 - PIXEL_FORMAT_LE_BAYER16_GRBG = 859914828 - PIXEL_FORMAT_LE_BAYER16_RGGB = 876692044 - PIXEL_FORMAT_MJPEG = 1196444237 - PIXEL_FORMAT_BE_GRAY16 = 357 - PIXEL_FORMAT_LE_GRAY16 = 909199180 - PIXEL_FORMAT_BE_RGB16 = 358 - PIXEL_FORMAT_LE_RGB16 = 1279412050 - PIXEL_FORMAT_BE_SIGNED_GRAY16 = 359 - PIXEL_FORMAT_BE_SIGNED_RGB16 = 360 - PIXEL_FORMAT_FLOAT_GRAY32 = 842221382 - PIXEL_FORMAT_INVALID = -2 - PIXEL_FORMAT_ANY = -1 - - def __init__(self): - self.header = stdlcm.header_t() - self.width = 0 - self.height = 0 - self.row_stride = 0 - self.pixelformat = 0 - self.size = 0 - self.data = b"" - self.nmetadata = 0 - self.metadata = [] - - def encode(self): - buf = BytesIO() - buf.write(image_t._get_packed_fingerprint()) - self._encode_one(buf) - return buf.getvalue() - - def _encode_one(self, buf): - assert self.header._get_packed_fingerprint() == stdlcm.header_t._get_packed_fingerprint() - self.header._encode_one(buf) - buf.write(struct.pack(">iiiii", self.width, self.height, self.row_stride, self.pixelformat, self.size)) - buf.write(bytearray(self.data[:self.size])) - buf.write(struct.pack(">i", self.nmetadata)) - for i0 in range(self.nmetadata): - assert self.metadata[i0]._get_packed_fingerprint() == senlcm.image_metadata_t._get_packed_fingerprint() - self.metadata[i0]._encode_one(buf) - - def decode(data): - if hasattr(data, 'read'): - buf = data - else: - buf = BytesIO(data) - if buf.read(8) != image_t._get_packed_fingerprint(): - raise ValueError("Decode error") - return image_t._decode_one(buf) - decode = staticmethod(decode) - - def _decode_one(buf): - self = image_t() - self.header = stdlcm.header_t._decode_one(buf) - self.width, self.height, self.row_stride, self.pixelformat, self.size = struct.unpack(">iiiii", buf.read(20)) - self.data = buf.read(self.size) - self.nmetadata = struct.unpack(">i", buf.read(4))[0] - self.metadata = [] - for i0 in range(self.nmetadata): - self.metadata.append(senlcm.image_metadata_t._decode_one(buf)) - return self - _decode_one = staticmethod(_decode_one) - - def _get_hash_recursive(parents): - if image_t in parents: return 0 - newparents = parents + [image_t] - tmphash = (0x5f0325a1392627aa+ stdlcm.header_t._get_hash_recursive(newparents)+ senlcm.image_metadata_t._get_hash_recursive(newparents)) & 0xffffffffffffffff - tmphash = (((tmphash<<1)&0xffffffffffffffff) + (tmphash>>63)) & 0xffffffffffffffff - return tmphash - _get_hash_recursive = staticmethod(_get_hash_recursive) - _packed_fingerprint = None - - def _get_packed_fingerprint(): - if image_t._packed_fingerprint is None: - image_t._packed_fingerprint = struct.pack(">Q", image_t._get_hash_recursive([])) - return image_t._packed_fingerprint - _get_packed_fingerprint = staticmethod(_get_packed_fingerprint) - - def get_hash(self): - """Get the LCM hash of the struct""" - return struct.unpack(">Q", image_t._get_packed_fingerprint())[0] - diff --git a/tests/lcmtypes/stdlcm/__init__.py b/tests/lcmtypes/stdlcm/__init__.py deleted file mode 100644 index d982849..0000000 --- a/tests/lcmtypes/stdlcm/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Subset of the MolaRS stdlcm types used by the tests (generated by lcm-gen).""" - -from .header_t import header_t - -__all__ = ["header_t"] diff --git a/tests/lcmtypes/stdlcm/header_t.py b/tests/lcmtypes/stdlcm/header_t.py deleted file mode 100644 index 64996d6..0000000 --- a/tests/lcmtypes/stdlcm/header_t.py +++ /dev/null @@ -1,72 +0,0 @@ -"""LCM type definitions -This file automatically generated by lcm. -DO NOT MODIFY BY HAND!!!! -""" - -try: - import cStringIO.StringIO as BytesIO -except ImportError: - from io import BytesIO -import struct - -class header_t(object): - __slots__ = ["sequence", "timestamp", "frame_id"] - - __typenames__ = ["int32_t", "int64_t", "string"] - - __dimensions__ = [None, None, None] - - def __init__(self): - self.sequence = 0 - self.timestamp = 0 - self.frame_id = "" - - def encode(self): - buf = BytesIO() - buf.write(header_t._get_packed_fingerprint()) - self._encode_one(buf) - return buf.getvalue() - - def _encode_one(self, buf): - buf.write(struct.pack(">iq", self.sequence, self.timestamp)) - __frame_id_encoded = self.frame_id.encode('utf-8') - buf.write(struct.pack('>I', len(__frame_id_encoded)+1)) - buf.write(__frame_id_encoded) - buf.write(b"\0") - - def decode(data): - if hasattr(data, 'read'): - buf = data - else: - buf = BytesIO(data) - if buf.read(8) != header_t._get_packed_fingerprint(): - raise ValueError("Decode error") - return header_t._decode_one(buf) - decode = staticmethod(decode) - - def _decode_one(buf): - self = header_t() - self.sequence, self.timestamp = struct.unpack(">iq", buf.read(12)) - __frame_id_len = struct.unpack('>I', buf.read(4))[0] - self.frame_id = buf.read(__frame_id_len)[:-1].decode('utf-8', 'replace') - return self - _decode_one = staticmethod(_decode_one) - - def _get_hash_recursive(parents): - if header_t in parents: return 0 - tmphash = (0xa2111e7853888e66) & 0xffffffffffffffff - tmphash = (((tmphash<<1)&0xffffffffffffffff) + (tmphash>>63)) & 0xffffffffffffffff - return tmphash - _get_hash_recursive = staticmethod(_get_hash_recursive) - _packed_fingerprint = None - - def _get_packed_fingerprint(): - if header_t._packed_fingerprint is None: - header_t._packed_fingerprint = struct.pack(">Q", header_t._get_hash_recursive([])) - return header_t._packed_fingerprint - _get_packed_fingerprint = staticmethod(_get_packed_fingerprint) - - def get_hash(self): - """Get the LCM hash of the struct""" - return struct.unpack(">Q", header_t._get_packed_fingerprint())[0] - diff --git a/tests/test_cli.py b/tests/test_cli.py index 4043166..6394554 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -25,15 +25,26 @@ logger = logging.getLogger("test-cli") -def parse(argv, compression_default="deflate"): +def parse(argv, compression_default="deflate", lcm_packages_default="pkg"): parser = argparse.ArgumentParser() - add_server_args(parser, compression_default) + add_server_args(parser, compression_default, lcm_packages_default) + parser.add_argument("lcm_packages_positional", nargs="?") return parse_server_options(parser.parse_args(argv), logger) class TestServerOptions: def test_defaults(self): - assert parse([]) == ServerOptions() + assert parse([]) == ServerOptions(lcm_packages=("pkg",)) + + def test_lcm_packages_flag(self): + assert parse(["--lcm-packages", "a, b"]).lcm_packages == ("a", "b") + + def test_positional_lcm_packages_take_precedence(self): + assert parse(["--lcm-packages", "a", "c,d"]).lcm_packages == ("c", "d") + + def test_missing_lcm_packages_exits(self): + with pytest.raises(SystemExit): + parse([], lcm_packages_default=None) def test_app_specific_compression_default(self): assert parse([], compression_default="none").compression is None @@ -43,7 +54,10 @@ def test_flags(self): "--host", "0.0.0.0", "--port", "1", "--channel", "NAV.*", "--queue-size", "8", "--spy-interval", "0.5", "--compression", "none", "--processes", "3", "-vv", ]) - assert opts == ServerOptions("0.0.0.0", 1, "NAV.*", 8, 500_000_000, None, False, 3, 2) + assert opts == ServerOptions( + lcm_packages=("pkg",), host="0.0.0.0", port=1, channel="NAV.*", queue_size=8, + spy_interval_ns=500_000_000, compression=None, uvloop=False, processes=3, verbosity=2, + ) def test_server_kwargs(self): kwargs = ServerOptions(processes=2, compression=None, queue_size=8).server_kwargs() @@ -72,7 +86,7 @@ def test_invalid_processes(self): class TestBuildRegistry: def test_registers_spy_types(self): - registry = build_registry(["stdlcm"], logger) + registry = build_registry(["compas_lcmtypes"], logger) assert channel_stats_list in registry.types def test_returns_none_when_nothing_discovered(self): @@ -102,28 +116,59 @@ def test_rejects_unknown_compression(self, republisher, json_handler): LCMWebSocketServer("localhost", 0, json_handler, republisher, compression="brotli") -class TestMultiProcess: +async def spy_message_from_cli(module: str, *args: str) -> dict: + """Start a proxy CLI as a subprocess and return its first spy message.""" + port = free_port() + env = {**os.environ, "PYTHONPATH": os.pathsep.join(sys.path)} + proc = await asyncio.create_subprocess_exec( + sys.executable, "-m", module, "--port", str(port), "--spy-interval", "0.1", + "--channel", "LWS_TEST_NO_TRAFFIC", *args, + env=env, + ) + try: + for _ in range(100): + try: + ws = await websockets.connect(f"ws://localhost:{port}/LWS_LCM_SPY") + break + except OSError: + await asyncio.sleep(0.1) + else: + pytest.fail("server did not start") + async with ws: + return json.loads(await asyncio.wait_for(ws.recv(), 5)) + finally: + proc.terminate() + await asyncio.wait_for(proc.wait(), 10) + + +class TestCliEndToEnd: async def test_json_proxy_serves_with_multiple_processes(self): - """End to end: the CLI starts 2 workers sharing a port; each serves the spy channel.""" - port = free_port() + """The CLI starts 2 workers sharing a port; each serves the spy channel.""" + msg = await spy_message_from_cli( + "lcm_websocket_server.apps.json_proxy", "--processes", "2", "compas_lcmtypes" + ) + assert msg["channel"] == "LWS_LCM_SPY" + + async def test_dial_proxy_starts_with_default_type_packages(self): + """Defaults discover compas_lcmtypes + molars_lcmtypes and resolve senlcm.image_t.""" + pytest.importorskip("molars_lcmtypes") + msg = await spy_message_from_cli("lcm_websocket_server.apps.dial_proxy") + assert msg["channel"] == "LWS_LCM_SPY" + + +class TestExitCodes: + @pytest.mark.parametrize( + "args", + [ + ["lcm_websocket_server.apps.dial_proxy", "--image-type", "nope.image_t"], + ["lcm_websocket_server.apps.json_proxy", "no_such_package_xyz"], + ["lcm_websocket_server.apps.json_proxy", "--processes", "2", "no_such_package_xyz"], + ], + ) + async def test_configuration_errors_exit_nonzero(self, args): env = {**os.environ, "PYTHONPATH": os.pathsep.join(sys.path)} proc = await asyncio.create_subprocess_exec( - sys.executable, "-m", "lcm_websocket_server.apps.json_proxy", "--port", str(port), - "--processes", "2", "--spy-interval", "0.1", "--channel", "LWS_TEST_NO_TRAFFIC", "stdlcm", - env=env, + sys.executable, "-m", *args, env=env, + stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL, ) - try: - for _ in range(100): - try: - ws = await websockets.connect(f"ws://localhost:{port}/LWS_LCM_SPY") - break - except OSError: - await asyncio.sleep(0.1) - else: - pytest.fail("server did not start") - async with ws: - msg = json.loads(await asyncio.wait_for(ws.recv(), 5)) - assert msg["channel"] == "LWS_LCM_SPY" - finally: - proc.terminate() - await asyncio.wait_for(proc.wait(), 10) + assert await asyncio.wait_for(proc.wait(), 60) != 0 diff --git a/tests/test_dial_handler.py b/tests/test_dial_handler.py index d6c443a..a8c4c95 100644 --- a/tests/test_dial_handler.py +++ b/tests/test_dial_handler.py @@ -8,7 +8,7 @@ import cv2 import numpy as np import pytest -from senlcm import image_t +from compas_lcmtypes.senlcm import image_t from lcm_websocket_server.apps.dial_proxy import ( DialHandler, @@ -61,7 +61,7 @@ def test_returned_jpeg_is_decodable(self, image_handler): def test_downsampling_encoder_reduces_dimensions(self, registry): enc = DownsamplingMJPEGEncoder(scale=0.5, params=[cv2.IMWRITE_JPEG_QUALITY, 75]) - handler = ImageMessageToJPEGHandler(enc) + handler = ImageMessageToJPEGHandler(enc, image_t) jpeg = handler.handle("CAM", make_bgr_image_t(width=16, height=12)) arr = cv2.imdecode(np.frombuffer(jpeg, np.uint8), cv2.IMREAD_COLOR) assert arr.shape == (6, 8, 3) @@ -109,11 +109,10 @@ def test_unknown_type_returns_none(self, dial_handler): result = dial_handler.handle("CH", garbage) assert result is None - def test_fingerprint_routing_matches_image_t(self, dial_handler): - """Verify DialHandler.IMAGE_T_FINGERPRINT matches the real type.""" + def test_routes_by_configured_image_type_fingerprint(self, dial_handler): raw = make_bgr_image_t() - fingerprint_in_data = raw[:8] - assert fingerprint_in_data == DialHandler.IMAGE_T_FINGERPRINT + assert raw[:8] == image_t._get_packed_fingerprint() + assert dial_handler.is_heavy("CAM", raw) # --------------------------------------------------------------------------- diff --git a/tests/test_json_handler.py b/tests/test_json_handler.py index 4bd08d4..6cc1d70 100644 --- a/tests/test_json_handler.py +++ b/tests/test_json_handler.py @@ -5,7 +5,7 @@ import inspect import json -from stdlcm import header_t +from compas_lcmtypes.stdlcm import header_t from lcm_websocket_server.lib.handler import TextPayload from lcm_websocket_server.lib.lcm_utils.pubsub import LCMObserver, LCMRepublisher diff --git a/tests/test_pubsub.py b/tests/test_pubsub.py index 9e7d378..e5e138a 100644 --- a/tests/test_pubsub.py +++ b/tests/test_pubsub.py @@ -8,7 +8,7 @@ import time import pytest -from stdlcm import header_t +from compas_lcmtypes.stdlcm import header_t from lcm_websocket_server.lib.lcm_utils.pubsub import ( MAX_CHANNEL_REGEX_LENGTH, diff --git a/tests/test_server.py b/tests/test_server.py index 2eacf32..c6d123b 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -13,7 +13,7 @@ import pytest import websockets -from stdlcm import header_t +from compas_lcmtypes.stdlcm import header_t from lcm_websocket_server.lib.handler import LCMWebSocketHandler from lcm_websocket_server.lib.lcm_utils.pubsub import ( diff --git a/tests/test_types.py b/tests/test_types.py index 652041b..ff84890 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -6,7 +6,7 @@ import math import pytest -from stdlcm import header_t +from compas_lcmtypes.stdlcm import header_t from lcm_websocket_server.lib.lcm_utils import types as T @@ -103,3 +103,22 @@ def test_output_is_strict_json(self, backend): def test_none_event(self, backend): assert json.loads(T.encode_event_json_bytes("CH", "00", None))["event"] == {} + + +class TestFindLcmType: + def test_finds_prefixed_type_by_lcm_name(self): + from compas_lcmtypes.senlcm import image_t + + assert T.find_lcm_type([header_t, image_t], "senlcm.image_t") is image_t + + def test_exact_module_match(self): + assert T.find_lcm_type([header_t], header_t.__module__) is header_t + + def test_does_not_match_partial_names(self): + with pytest.raises(LookupError): + T.find_lcm_type([header_t], "dlcm.header_t") + + def test_ambiguous_match_raises(self): + clone = type("header_t", (), {"__module__": "other_prefix.stdlcm.header_t"}) + with pytest.raises(LookupError, match="exactly one"): + T.find_lcm_type([header_t, clone], "stdlcm.header_t") diff --git a/uv.lock b/uv.lock index 4c63a76..44aeda6 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.12'", @@ -210,6 +210,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "compas-lcmtypes" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/fd/161554aab889e12b611d86c81049371704a501b4145825eef32faf1d0827/compas_lcmtypes-0.4.0.tar.gz", hash = "sha256:5b1149e0ddaefd36720b7dc4df512a0e666f4b47186cda95a51613bdb9d8455b", size = 22057, upload-time = "2026-03-23T23:01:17.816Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/ab/05729ec3ce692bae2a4c5f1ed281a32b76caa176d3aa55a7d1650d891da8/compas_lcmtypes-0.4.0-py3-none-any.whl", hash = "sha256:1f74c278a1e7e6698058c6fa6448065564b770728dba80ed0423e00144701df9", size = 52245, upload-time = "2026-03-23T23:01:16.66Z" }, +] + [[package]] name = "deprecated" version = "1.3.1" @@ -386,6 +395,10 @@ image = [ { name = "numpy", version = "2.5.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "opencv-python-headless" }, ] +lcmtypes = [ + { name = "compas-lcmtypes" }, + { name = "molars-lcmtypes" }, +] orjson = [ { name = "orjson" }, ] @@ -395,6 +408,7 @@ uvloop = [ [package.dev-dependencies] dev = [ + { name = "compas-lcmtypes" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -406,19 +420,22 @@ release = [ [package.metadata] requires-dist = [ + { name = "compas-lcmtypes", marker = "extra == 'lcmtypes'", specifier = ">=0.4,<0.5" }, { name = "lcm", specifier = ">=1.5.0" }, { name = "lcmlog-py", specifier = ">=0.1.0" }, { name = "lcmutils", specifier = ">=0.1.1" }, + { name = "molars-lcmtypes", marker = "extra == 'lcmtypes'", url = "https://github.com/CoMPASLab/molars_lcmtypes/releases/download/v0.1.0/molars_lcmtypes-0.1.0-py3-none-any.whl" }, { name = "numpy", marker = "extra == 'image'", specifier = ">=1.26.4" }, { name = "opencv-python-headless", marker = "extra == 'image'", specifier = ">=4.9.0.80" }, { name = "orjson", marker = "extra == 'orjson'", specifier = ">=3.10" }, { name = "uvloop", marker = "extra == 'uvloop'", specifier = ">=0.21" }, { name = "websockets", specifier = ">=16.0" }, ] -provides-extras = ["image", "orjson", "uvloop"] +provides-extras = ["image", "orjson", "uvloop", "lcmtypes"] [package.metadata.requires-dev] dev = [ + { name = "compas-lcmtypes", specifier = ">=0.4,<0.5" }, { name = "pre-commit", specifier = ">=4.2.0" }, { name = "pytest", specifier = ">=8.0" }, { name = "pytest-asyncio", specifier = ">=0.23" }, @@ -596,6 +613,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "molars-lcmtypes" +version = "0.1.0" +source = { url = "https://github.com/CoMPASLab/molars_lcmtypes/releases/download/v0.1.0/molars_lcmtypes-0.1.0-py3-none-any.whl" } +dependencies = [ + { name = "compas-lcmtypes" }, +] +wheels = [ + { url = "https://github.com/CoMPASLab/molars_lcmtypes/releases/download/v0.1.0/molars_lcmtypes-0.1.0-py3-none-any.whl", hash = "sha256:49e0ee147b110699b44fc84ab8a34edf25c519151a868806c49b0c07e57a275a" }, +] + +[package.metadata] +requires-dist = [{ name = "compas-lcmtypes", specifier = ">=0.4,<0.5" }] + [[package]] name = "nodeenv" version = "1.12.0" From f7f95ed87ffb750d3e4ad232c0c8cf6b5cd42a3b Mon Sep 17 00:00:00 2001 From: Kevin Barnard Date: Thu, 19 Nov 2026 12:30:57 -0800 Subject: [PATCH 2/5] build(docker): one generic image with bundled LCM types, bake definition - A single image serves both proxies (PROXY=json|dial), replacing Dockerfile.dial and the -dial tags. It bundles compas-lcmtypes and molars-lcmtypes, so no MOLARS wheel needs to be copied into the repo. - Installs from uv.lock (uv sync --locked) into a venv copied onto a slim Python base; libglib comes from the distro package instead of a copied x86_64 .so. - New entrypoint configures the proxy from HOST, PORT, CHANNEL, LCM_PACKAGES and ARGS; extra flags may also be passed as arguments, and other commands run as given. - docker-bake.hcl is the shared build definition for CI and local builds; docker/smoke_test.sh checks both proxies become healthy. - Removes the docker_* shell scripts. Co-Authored-By: Claude Opus 5.5 --- .dockerignore | 3 -- docker-bake.hcl | 22 +++++++++++ docker/Dockerfile | 59 +++++++++++++++++----------- docker/Dockerfile.dial | 15 ------- docker/entrypoint.sh | 25 ++++++++++++ docker/smoke_test.sh | 35 +++++++++++++++++ scripts/docker_build.sh | 30 -------------- scripts/docker_build_all.sh | 11 ------ scripts/docker_push.sh | 14 ------- scripts/docker_push_all.sh | 11 ------ scripts/docker_update_latest_tags.sh | 11 ------ 11 files changed, 117 insertions(+), 119 deletions(-) create mode 100644 docker-bake.hcl delete mode 100644 docker/Dockerfile.dial create mode 100755 docker/entrypoint.sh create mode 100755 docker/smoke_test.sh delete mode 100755 scripts/docker_build.sh delete mode 100755 scripts/docker_build_all.sh delete mode 100755 scripts/docker_push.sh delete mode 100755 scripts/docker_push_all.sh delete mode 100755 scripts/docker_update_latest_tags.sh diff --git a/.dockerignore b/.dockerignore index 9512f3e..d82f745 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,13 +1,10 @@ .dockerignore -docker/ .git .gitignore poetry.lock -uv.lock dist/ -scripts/ *__pycache__/ .venv/ diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 0000000..b7b9ec7 --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,22 @@ +# Image build definition shared by CI and local builds. +# +# docker buildx bake # build mbari/lcm-websocket-server:local +# docker/smoke_test.sh mbari/lcm-websocket-server:local +# +# In CI, docker/metadata-action overrides the tags and labels of the +# "docker-metadata-action" target. + +target "docker-metadata-action" { + tags = ["mbari/lcm-websocket-server:local"] +} + +target "image" { + inherits = ["docker-metadata-action"] + context = "." + dockerfile = "docker/Dockerfile" + platforms = ["linux/amd64"] +} + +group "default" { + targets = ["image"] +} diff --git a/docker/Dockerfile b/docker/Dockerfile index 321e0dd..0a76552 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,36 +1,47 @@ +# syntax=docker/dockerfile:1 + +# Build a virtual environment from the locked dependencies FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS builder -ENV DEBIAN_FRONTEND=noninteractive +ENV UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=0 +WORKDIR /app +ARG EXTRAS="--extra image --extra orjson --extra uvloop --extra lcmtypes" -# Install dependencies -RUN apt update && \ - apt install -y --no-install-recommends libglib2.0-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* +# Dependencies first so they are cached across source changes +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=uv.lock,target=uv.lock \ + --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ + uv sync --locked --no-dev --no-install-project $EXTRAS -# Copy app into container -COPY . /app -WORKDIR /app +COPY . /src +RUN --mount=type=cache,target=/root/.cache/uv \ + cd /src && UV_PROJECT_ENVIRONMENT=/app/.venv uv sync --locked --no-dev --no-editable $EXTRAS -# Build lcm_websocket_server*.whl -RUN uv build --wheel +# Runtime image: same Python as the builder, without uv or sources +FROM python:3.13-slim-bookworm -FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim +# libglib is required by the lcm Python bindings +RUN apt-get update && \ + apt-get install -y --no-install-recommends libglib2.0-0 && \ + rm -rf /var/lib/apt/lists/* -# Copy glib-2.0 shared libraries -COPY --from=builder /usr/lib/x86_64-linux-gnu/libglib-2.0.so.0 /usr/lib/x86_64-linux-gnu/libglib-2.0.so.0 +COPY --from=builder /app/.venv /app/.venv +COPY docker/entrypoint.sh /usr/local/bin/lws-entrypoint -# Install lcm-websocket-server with the performance extras -COPY --from=builder /app/dist/*.whl /tmp/ -RUN uv pip install --system --no-cache "$(ls /tmp/lcm_websocket_server-*.whl)[orjson,uvloop]" +# Packages mounted under /app (e.g. -v ./my_types:/app/my_types) are importable +ENV PATH="/app/.venv/bin:$PATH" \ + PYTHONPATH=/app \ + PROXY=json \ + HOST=0.0.0.0 \ + PORT=8765 \ + CHANNEL=.* \ + LCM_PACKAGES=compas_lcmtypes,molars_lcmtypes \ + ARGS= -ENV HOST=0.0.0.0 -ENV PORT=8765 -ENV CHANNEL=.* -ENV LCM_PACKAGES= -# Extra command-line flags, e.g. "--uvloop --processes 4 -vv" -ENV ARGS= +EXPOSE 8765 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s \ CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/healthz' % os.environ['PORT'], timeout=3)"] -CMD ["sh", "-c", "PYTHONPATH=$PYTHONPATH:/app lcm-websocket-json-proxy --host \"$HOST\" --port \"$PORT\" --channel \"$CHANNEL\" $ARGS \"$LCM_PACKAGES\""] +ENTRYPOINT ["lws-entrypoint"] diff --git a/docker/Dockerfile.dial b/docker/Dockerfile.dial deleted file mode 100644 index e397228..0000000 --- a/docker/Dockerfile.dial +++ /dev/null @@ -1,15 +0,0 @@ -ARG VERSION=latest -FROM mbari/lcm-websocket-server:$VERSION - -# Install lcm-websocket-server with the image extra -RUN uv pip install --system --no-cache "$(ls /tmp/lcm_websocket_server-*.whl | head -n 1)[image,orjson,uvloop]" - -# Install the molars-lcmtypes wheel from the host -COPY molars_lcmtypes-0.0.0-py3-none-any.whl /tmp/molars_lcmtypes-0.0.0-py3-none-any.whl -RUN uv pip install --system --no-cache /tmp/molars_lcmtypes-0.0.0-py3-none-any.whl - -ENV JPEG_QUALITY=75 -ENV JPEG_SCALE=1.0 - -# Run lcm-websocket-dial-proxy (HEALTHCHECK is inherited from the base image) -CMD ["sh", "-c", "lcm-websocket-dial-proxy -vv --host \"$HOST\" --port \"$PORT\" --channel \"$CHANNEL\" --quality \"$JPEG_QUALITY\" --scale \"$JPEG_SCALE\" $ARGS"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..90fad16 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Start the proxy selected by $PROXY (json or dial), configured from the +# environment. Any other command (e.g. "bash") is executed as given. +set -e + +if [ "$#" -gt 0 ] && [ "${1#-}" = "$1" ]; then + exec "$@" +fi + +case "$PROXY" in + json | dial) ;; + *) + echo "Unknown PROXY '$PROXY' (expected 'json' or 'dial')" >&2 + exit 2 + ;; +esac + +# ARGS is intentionally unquoted so it splits into separate flags +# shellcheck disable=SC2086 +exec "lcm-websocket-$PROXY-proxy" \ + --host "$HOST" \ + --port "$PORT" \ + --channel "$CHANNEL" \ + --lcm-packages "$LCM_PACKAGES" \ + $ARGS "$@" diff --git a/docker/smoke_test.sh b/docker/smoke_test.sh new file mode 100755 index 0000000..0c0ce13 --- /dev/null +++ b/docker/smoke_test.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Smoke-test an image: start each proxy, wait for /healthz, check /metrics. +# Usage: docker/smoke_test.sh IMAGE +set -euo pipefail + +IMAGE=${1:?usage: $0 IMAGE} + +for proxy in json dial; do + name="lws-smoke-$proxy-$$" + # memq:// keeps LCM in-process, so no multicast route is needed + docker run -d --rm --name "$name" -p 127.0.0.1::8765 \ + -e PROXY="$proxy" -e LCM_DEFAULT_URL=memq:// -e ARGS="-v" "$IMAGE" >/dev/null + trap 'docker rm -f "$name" >/dev/null 2>&1 || true' EXIT + port=$(docker port "$name" 8765/tcp | head -n1 | cut -d: -f2) + + ok="" + for _ in $(seq 1 30); do + if curl -fsS "http://127.0.0.1:$port/healthz" >/dev/null 2>&1; then + ok=1 + break + fi + if ! docker inspect -f '{{.State.Running}}' "$name" 2>/dev/null | grep -q true; then + break + fi + sleep 1 + done + if [ -z "$ok" ]; then + echo "FAIL: $proxy proxy did not become healthy" >&2 + docker logs "$name" >&2 || true + exit 1 + fi + curl -fsS "http://127.0.0.1:$port/metrics" | grep -q '^lws_clients 0$' + echo "OK: $proxy proxy healthy on port $port" + docker rm -f "$name" >/dev/null +done diff --git a/scripts/docker_build.sh b/scripts/docker_build.sh deleted file mode 100755 index 95f4722..0000000 --- a/scripts/docker_build.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -# Read the first argument from the command line. This will specify which image we are building: -# - empty: docker/Dockerfile -# - x: docker/Dockerfile.x - -# If no argument is provided, use the default Dockerfile - -DOCKEROPT=${1:-""} -DOCKERFILE="Dockerfile" -if [ -n "$DOCKEROPT" ]; then - DOCKERFILE="Dockerfile.$DOCKEROPT" -fi -# Check that the Dockerfile exists -if [ ! -f "docker/$DOCKERFILE" ]; then - echo "Dockerfile $DOCKERFILE does not exist" - echo "Usage: $0 [DOCKERFILE EXTENSION]" - exit 1 -fi - -VERSION=$(grep '^version =' pyproject.toml | awk -F'"' '{print $2}') -DOCKERTAG="$VERSION" -if [ -n "$DOCKEROPT" ]; then - DOCKERTAG="$DOCKERTAG-$DOCKEROPT" -fi - -DOCKERIMAGE="mbari/lcm-websocket-server:$DOCKERTAG" - -echo "Building docker image $DOCKERIMAGE" -docker build -t $DOCKERIMAGE -f docker/$DOCKERFILE --build-arg VERSION=$VERSION . diff --git a/scripts/docker_build_all.sh b/scripts/docker_build_all.sh deleted file mode 100755 index 54a56fd..0000000 --- a/scripts/docker_build_all.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -# Build all of the docker images - -EXTS=( "dial" ) - -./scripts/docker_build.sh - -for ext in "${EXTS[@]}"; do - ./scripts/docker_build.sh $ext -done diff --git a/scripts/docker_push.sh b/scripts/docker_push.sh deleted file mode 100755 index 289d244..0000000 --- a/scripts/docker_push.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -# Read the first argument from the command line. This will specify which image we are pushing. - -DOCKEROPT=${1:-""} -VERSION=$(grep '^version =' pyproject.toml | awk -F'"' '{print $2}') -DOCKERTAG="$VERSION" -if [ -n "$DOCKEROPT" ]; then - DOCKERTAG="$DOCKERTAG-$DOCKEROPT" -fi -DOCKERIMAGE="mbari/lcm-websocket-server:$DOCKERTAG" - -echo "Pushing docker image $DOCKERIMAGE" -docker push $DOCKERIMAGE diff --git a/scripts/docker_push_all.sh b/scripts/docker_push_all.sh deleted file mode 100755 index 5f3ad09..0000000 --- a/scripts/docker_push_all.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -# Push all of the docker images - -EXTS=( "dial" ) - -./scripts/docker_push.sh - -for ext in "${EXTS[@]}"; do - ./scripts/docker_push.sh $ext -done diff --git a/scripts/docker_update_latest_tags.sh b/scripts/docker_update_latest_tags.sh deleted file mode 100755 index c79b29d..0000000 --- a/scripts/docker_update_latest_tags.sh +++ /dev/null @@ -1,11 +0,0 @@ -VERSION=$(grep '^version =' pyproject.toml | awk -F'"' '{print $2}') - -EXTS=( "dial" ) - -docker tag mbari/lcm-websocket-server:$VERSION mbari/lcm-websocket-server:latest -docker push mbari/lcm-websocket-server:latest - -for EXT in "${EXTS[@]}"; do - docker tag mbari/lcm-websocket-server:$VERSION-$EXT mbari/lcm-websocket-server:latest-$EXT - docker push mbari/lcm-websocket-server:latest-$EXT -done From 6e5766a0eb2dbc2084fbd66dfda9059956ee604d Mon Sep 17 00:00:00 2001 From: Kevin Barnard Date: Thu, 19 Nov 2026 12:30:57 -0800 Subject: [PATCH 3/5] ci: upload release assets, smoke-test and publish Docker images - Run semantic-release publish so GitHub releases get the wheel and sdist (version alone does not upload them). - Expose released/version/prerelease from the release job. - New docker job: on pull requests the image is built and smoke-tested; after a release it is built from the release tag, smoke-tested and pushed to Docker Hub as X.Y.Z, X.Y and latest (main) or X.Y.Z-rc.N and develop (develop). Requires DOCKERHUB_USERNAME and DOCKERHUB_TOKEN. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 91 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6dc2f87..95bee6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,10 @@ jobs: concurrency: release permissions: contents: write + outputs: + released: ${{ steps.release.outputs.released }} + version: ${{ steps.release.outputs.version }} + prerelease: ${{ steps.release.outputs.prerelease }} steps: - uses: actions/checkout@v7 with: @@ -66,7 +70,90 @@ jobs: with: enable-cache: true - - name: Release - run: uv run --group release semantic-release version + - name: Version, tag and GitHub release + id: release + run: | + BEFORE=$(git tag -l | sort) + uv run --group release semantic-release version + AFTER=$(git tag -l | sort) + NEW_TAG=$(comm -13 <(echo "$BEFORE") <(echo "$AFTER") | head -1) + if [ -n "$NEW_TAG" ]; then + VERSION=${NEW_TAG#v} + echo "released=true" >> "$GITHUB_OUTPUT" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + if [[ "$VERSION" == *-* ]]; then PRE=true; else PRE=false; fi + echo "prerelease=$PRE" >> "$GITHUB_OUTPUT" + else + echo "released=false" >> "$GITHUB_OUTPUT" + fi env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload release assets + if: steps.release.outputs.released == 'true' + run: uv run --group release semantic-release publish + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Builds and smoke-tests the Docker image. On pull requests the image is + # only tested. After a release it is built from the release tag and pushed + # to Docker Hub: + # release on main: X.Y.Z, X.Y, latest + # prerelease on develop: X.Y.Z-rc.N, develop + # (Tags pushed by GITHUB_TOKEN do not trigger workflows, so this runs here + # rather than in a separate on-tag workflow.) + docker: + needs: [build, release] + if: >- + !cancelled() && needs.build.result == 'success' && + (github.event_name == 'pull_request' || needs.release.outputs.released == 'true') + runs-on: ubuntu-latest + env: + PUBLISH: ${{ needs.release.outputs.released == 'true' }} + VERSION: ${{ needs.release.outputs.version }} + PRERELEASE: ${{ needs.release.outputs.prerelease == 'true' }} + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ env.PUBLISH == 'true' && format('v{0}', env.VERSION) || '' }} + - uses: docker/setup-buildx-action@v3 + + - name: Image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: mbari/lcm-websocket-server + flavor: latest=false + tags: | + type=semver,pattern={{version}},value=${{ env.VERSION }},enable=${{ env.PUBLISH }} + type=semver,pattern={{major}}.{{minor}},value=${{ env.VERSION }},enable=${{ env.PUBLISH == 'true' && env.PRERELEASE == 'false' }} + type=raw,value=latest,enable=${{ env.PUBLISH == 'true' && env.PRERELEASE == 'false' }} + type=raw,value=develop,enable=${{ env.PUBLISH == 'true' && env.PRERELEASE == 'true' }} + type=raw,value=ci,enable=${{ env.PUBLISH != 'true' }} + + - name: Build + uses: docker/bake-action@v6 + with: + source: . + files: | + ./docker-bake.hcl + cwd://${{ steps.meta.outputs.bake-file }} + targets: image + load: true + set: | + *.cache-from=type=gha + *.cache-to=type=gha,mode=max + + - name: Smoke test + run: docker/smoke_test.sh "mbari/lcm-websocket-server:${{ env.PUBLISH == 'true' && env.VERSION || 'ci' }}" + + - name: Log in to Docker Hub + if: env.PUBLISH == 'true' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push + if: env.PUBLISH == 'true' + run: echo "${{ steps.meta.outputs.tags }}" | xargs -n1 docker push From 25f41308bbfd88e9b31aa625c3e56567820c1cf5 Mon Sep 17 00:00:00 2001 From: Kevin Barnard Date: Thu, 19 Nov 2026 12:35:09 -0800 Subject: [PATCH 4/5] fix(benchmarks): import LCM types from compas_lcmtypes Co-Authored-By: Claude Opus 5.5 --- benchmarks/README.md | 4 ++-- benchmarks/load.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 329c309..0d073ef 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -2,10 +2,10 @@ Load and microbenchmarks used to characterize the server under load. -Both scripts need the CoMPAS LCM Python types (`stdlcm`, `senlcm`, `navlcm`, `geolcm`) on `PYTHONPATH`. +Both scripts use the CoMPAS LCM types from the `lcmtypes` extra. ```bash -export PYTHONPATH=/path/to/lcmtypes/python +uv sync --all-extras uv run python benchmarks/load.py all # or: latency, json-scale, json-rate, dial, slow, periodic uv run python benchmarks/encode.py ``` diff --git a/benchmarks/load.py b/benchmarks/load.py index b6489ae..415a1c3 100644 --- a/benchmarks/load.py +++ b/benchmarks/load.py @@ -37,10 +37,10 @@ def _cpu_seconds() -> float: def _message_builders(): import numpy as np - from geolcm import point_t - from navlcm import odometry_t - from senlcm import image_t, multibeam_euclidean_t - from stdlcm import header_t + from compas_lcmtypes.geolcm import point_t + from compas_lcmtypes.navlcm import odometry_t + from compas_lcmtypes.senlcm import image_t, multibeam_euclidean_t + from compas_lcmtypes.stdlcm import header_t from lcm_websocket_server.lib.image import PixelFormat From 485bd069ccd1b9f0980fd1a4a4cad458d5b92870 Mon Sep 17 00:00:00 2001 From: Kevin Barnard Date: Thu, 19 Nov 2026 12:35:09 -0800 Subject: [PATCH 5/5] docs: document the generic Docker image, lcmtypes extra and releases Co-Authored-By: Claude Opus 5.5 --- README.md | 77 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index aba334c..da1546e 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,11 @@ Or with optional extras: ```bash pip install ".[image]" # Dial proxy (OpenCV) pip install ".[orjson,uvloop]" # faster JSON encoding and event loop +pip install ".[lcmtypes]" # CoMPAS and MOLARS LCM types ``` +The `lcmtypes` extra installs [compas-lcmtypes](https://github.com/CoMPASLab/compas_lcmtypes) and [molars-lcmtypes](https://github.com/CoMPASLab/molars_lcmtypes), the default types for the Dial proxy and the Docker image. Any other Python package of lcm-gen types works as well (see `--lcm-packages`). + See [Development](#wrench-development) for working on the project itself. ## :rocket: Usage @@ -111,9 +114,7 @@ To run the server locally on port 8765 and republish messages on all channels (J lcm-websocket-dial-proxy --host localhost --port 8765 --channel '.*' --quality 75 --scale 1.0 ``` -The Dial proxy depends on the `molars-lcmtypes` Python package to be installed. This package is not available on PyPI, so it must be built and installed manually; see the [MolaRS repository](https://github.com/CoMPASLab/molars/) for more info. - -The `Dockerfile.dial` file can be used to build the image with the `molars-lcmtypes` package installed. For this, the Python wheel `molars_lcmtypes-0.0.0-py3-none-any.whl` must be placed at the repository root before building the image. +By default the Dial proxy discovers types from `compas_lcmtypes` and `molars_lcmtypes` (install the `lcmtypes` extra) and transcodes messages of the LCM type `senlcm.image_t`. Use `--lcm-packages` and `--image-type` to change either; the image type is matched by its LCM name regardless of the Python package prefix it was generated with. ### Connecting to the Server @@ -228,41 +229,61 @@ The server has no authentication or encryption. When it is reachable beyond a tr ## :whale: Docker -### Build +One image, [`mbari/lcm-websocket-server`](https://hub.docker.com/r/mbari/lcm-websocket-server), runs either proxy. It includes the `image`, `orjson`, `uvloop` and `lcmtypes` extras, so the CoMPAS and MOLARS types are available without mounting anything. + +### Tags + +Images are built and pushed by CI after each release: -A Docker image to run the `lcm-websocket-server` can be built with: +| Tag | Source | +|-----|--------| +| `X.Y.Z`, `X.Y`, `latest` | Releases from `main` | +| `X.Y.Z-rc.N`, `develop` | Prereleases from `develop` | + +### Run ```bash -./scripts/docker_build.sh +# JSON proxy for all channels +docker run -d --name lcm-websocket-server --network=host mbari/lcm-websocket-server + +# Dial proxy, downscaled JPEGs capped at 5 fps per camera +docker run -d --name lcm-websocket-dial --network=host \ + -e PROXY=dial -e ARGS="--scale 0.5 --quality 60 --max-fps 5" \ + mbari/lcm-websocket-server ``` -This will create the `mbari/lcm-websocket-server` image. +Run with `--network=host` so LCM's UDP multicast reaches the container. -### Run +| Variable | Default | Description | +|----------|---------|-------------| +| `PROXY` | `json` | `json` or `dial` | +| `HOST` | `0.0.0.0` | Address to listen on | +| `PORT` | `8765` | Port to listen on | +| `CHANNEL` | `.*` | LCM channel regex to subscribe to | +| `LCM_PACKAGES` | `compas_lcmtypes,molars_lcmtypes` | Python packages to discover LCM types from | +| `ARGS` | | Extra flags, e.g. `--uvloop --processes 4 -vv` | + +Flags can also be passed as container arguments (`docker run ... mbari/lcm-websocket-server --uvloop`), and any other command runs as given (`docker run -it ... mbari/lcm-websocket-server bash`). -The container can be run with: +To use other LCM types, mount the package under `/app`, which is on `PYTHONPATH`, and list it in `LCM_PACKAGES`: ```bash -docker run \ - --name lcm-websocket-server \ - --rm \ - -e HOST=0.0.0.0 \ - -e PORT=8765 \ - -e CHANNEL=".*" \ - -v /path/to/your_lcm_types_package:/app/your_lcm_types_package \ - -e LCM_PACKAGES=your_lcm_types_package \ - --network=host \ - -d \ +docker run -d --network=host \ + -v /path/to/my_lcmtypes:/app/my_lcmtypes:ro \ + -e LCM_PACKAGES=my_lcmtypes \ mbari/lcm-websocket-server ``` -Note that the `HOST`, `PORT`, and `CHANNEL` environment variables specified above are the defaults for the `mbari/lcm-websocket-server` image. These can be omitted if the defaults are acceptable. +The image has a `HEALTHCHECK` on `/healthz`; it cannot reach the server if `HOST` is set to an address other than `0.0.0.0` or `127.0.0.1`. -Extra command-line flags can be passed with the `ARGS` environment variable, e.g. `-e ARGS="--uvloop --processes 4 -vv"`. The image includes the `orjson` and `uvloop` extras and a `HEALTHCHECK` that polls `/healthz`; if `HOST` is set to an address other than `0.0.0.0` or `127.0.0.1`, the health check cannot reach it. +### Build -The `LCM_PACKAGES` environment variable should be set to the name of the package (or comma-separated list of packages) that contains the LCM Python message definitions. The `/app` directory is included in the `PYTHONPATH` so that any packages mounted there (as shown with `-v` above) can be imported. +The build is defined in `docker-bake.hcl` and installs exactly the dependencies in `uv.lock`: -It's recommended to run with `--network=host` to avoid issues with LCM over UDP. This will allow the container to use the host's network stack. +```bash +docker buildx bake # builds mbari/lcm-websocket-server:local +docker/smoke_test.sh mbari/lcm-websocket-server:local # checks both proxies start and report healthy +``` ## :wrench: Development @@ -273,10 +294,16 @@ uv run pytest uv run ruff check . ``` -The tests are self-contained: they use LCM types vendored in `tests/lcmtypes` and the in-process `memq://` LCM provider, so no LCM network or MolaRS install is needed. CI runs lint and tests on Python 3.11 to 3.14. +The tests use the `compas-lcmtypes` package and the in-process `memq://` LCM provider, so no LCM network is needed. CI runs lint and tests on Python 3.11 to 3.14 and smoke-tests the Docker image. -Before a release, run the load benchmarks and compare against [benchmarks/README.md](benchmarks/README.md) to catch performance regressions. They need the full CoMPAS LCM types on `PYTHONPATH`: +Before a release, run the load benchmarks and compare against [benchmarks/README.md](benchmarks/README.md) to catch performance regressions: ```bash uv run python benchmarks/load.py all ``` + +### Releases + +[python-semantic-release](https://python-semantic-release.readthedocs.io/) runs in CI after lint, tests and build pass. It derives the version from [conventional commits](https://www.conventionalcommits.org/), updates `pyproject.toml`, `uv.lock` and `CHANGELOG.md`, tags the release, and creates a GitHub release with the wheel and sdist. Pushes to `main` make releases and pushes to `develop` make `-rc.N` prereleases; each release also publishes the Docker image. Nothing is published to PyPI. + +Publishing the image needs two repository secrets: `DOCKERHUB_USERNAME`, and `DOCKERHUB_TOKEN`, a Docker Hub access token with read and write access to `mbari/lcm-websocket-server`.