{html.escape(note.summary)}
" + "{html.escape(note.method_text)}
" + "Formula: "
+ f"{html.escape(note.formula)}
{html.escape(source)}" + f"{VMT_EXPORT_DROPDOWN_NOTE}" + "
" + ), + } + if content.get("kind") == "container": + children = list(content.get("children", [])) + insert_at = 1 if children else 0 + return { + **content, + "children": [*children[:insert_at], note_node, *children[insert_at:]], + "child_count": len(children) + 1, + } + return { + "kind": "container", + "layout": "column", + "child_count": 2, + "children": [content, note_node], + "styles": {}, + "css_classes": ["export-page-note-wrapper"], + } + + +def build_widget_metadata( + page: Any, + *, + selector_metadata_by_id: dict[str, SelectorMetadataPayload], +) -> dict[int, tuple[str | None, SelectorMetadataPayload | None]]: + """Build selector metadata keyed by widget identity for serialization.""" + metadata: dict[int, tuple[str | None, SelectorMetadataPayload | None]] = {} + for selector_def in traversal.page_selectors(page): + selector_id = traversal.selector_id(selector_def) + selector_meta = selector_metadata_by_id[selector_id] + widget = traversal.selector_widget(selector_def) + if widget is not None: + metadata[id(widget)] = (selector_id, selector_meta) + return metadata + + +def build_region_nodes( + page: Any, + *, + page_def: DashboardPageDefinition, + widget_metadata: dict[int, tuple[str | None, SelectorMetadataPayload | None]], + selector_metadata_by_id: dict[str, SelectorMetadataPayload], + page_diagnostics: dict[str, Any], +) -> dict[int, dict[str, Any]]: + """Build serialized region nodes keyed by region-root object id.""" + if page.view is None: + return {} + + resolved_parts = traversal.resolve_page_parts(page, page_def=page_def) + if not resolved_parts: + return {} + + interactive_selector_ids = { + selector_id + for selector_id, selector_meta in selector_metadata_by_id.items() + if selector_meta["export_enabled"] + } + referenced_selector_ids = { + selector_id + for part_def, _ in resolved_parts + for selector_id in part_def.selector_ids + } + unused_selector_ids = sorted(interactive_selector_ids - referenced_selector_ids) + if unused_selector_ids: + raise ValueError( + f"Dashboard page {page_def.page_id!r} does not assign export regions to selector ids: " + + ", ".join(repr(selector_id) for selector_id in unused_selector_ids) + ) + + selector_widgets = { + traversal.selector_id(selector_def): traversal.selector_widget(selector_def) + for selector_def in traversal.page_selectors(page) + } + region_nodes: dict[int, dict[str, Any]] = {} + for part_def, part_view in resolved_parts: + region_start = perf_counter() + region_label = f"{page_def.page_id}.{part_def.part_id}" + active_selector_ids = [ + selector_id + for selector_id in part_def.selector_ids + if selector_metadata_by_id.get(selector_id, {}).get("export_enabled") + ] + LOGGER.info( + "Export region %s: start (%s).", + region_label, + ( + "selectors: " + ", ".join(active_selector_ids) + if active_selector_ids + else "static/no exported selectors" + ), + ) + default_values = [ + selector_metadata_by_id[selector_id]["default_value"] + for selector_id in active_selector_ids + ] + default_key = variant_key(default_values) + variants: dict[str, Any] = {} + variant_aliases: dict[str, str] = {} + + if active_selector_ids: + state_specs, variant_aliases = selector_states.resolve_export_section_states( + page, + page_def=page_def, + part_def=part_def, + active_selector_ids=active_selector_ids, + selector_widgets=selector_widgets, + selector_metadata_by_id=selector_metadata_by_id, + ) + selector_counts = { + selector_id: len(selector_metadata_by_id[selector_id]["resolved_values"]) + for selector_id in active_selector_ids + } + raw_state_count = 1 + for count in selector_counts.values(): + raw_state_count *= count + diagnostics_key = f"export_region:{part_def.part_id}" + page_diagnostics[diagnostics_key] = { + "selector_ids": list(active_selector_ids), + "selector_counts": selector_counts, + "raw_state_count": raw_state_count, + "valid_state_count": len(state_specs), + "alias_count": len(variant_aliases), + "pruned_state_count": max( + raw_state_count - len(state_specs) - len(variant_aliases), + 0, + ), + } + diagnostic_message = ( + f"Export region {region_label}: " + f"{len(state_specs)} valid states from {raw_state_count} raw selector combinations " + f"({len(variant_aliases)} aliases, " + f"{max(raw_state_count - len(state_specs) - len(variant_aliases), 0)} pruned)." + ) + if len(state_specs) > EXPORT_SECTION_VARIANT_WARNING_COUNT: + LOGGER.warning("Warning: " + diagnostic_message) + else: + LOGGER.info(diagnostic_message) + last_progress_log = perf_counter() + total_states = len(state_specs) + for state_index, state_spec in enumerate(state_specs, start=1): + state_values = [ + state_spec[selector_id] for selector_id in active_selector_ids + ] + state_key = variant_key(state_values) + with ( + selector_states.suppress_page_selector_refresh(page), + selector_states.scoped_widget_values( + selector_widgets, + state_spec, + ), + ): + refreshed_part_view = _refresh_page_part_view( + page, + part_def, + page_id=page_def.page_id, + context_label="during variant serialization", + ) + page_diagnostics[ + f"region:{part_def.part_id}:{state_key}" + ] = _serialize_page_diagnostics(page) + variants[state_key] = serialize_viewable( + refreshed_part_view, + disable_widgets=False, + widget_metadata=widget_metadata, + ) + now = perf_counter() + if ( + state_index == total_states + or state_index % EXPORT_REGION_PROGRESS_VARIANT_INTERVAL == 0 + or now - last_progress_log >= EXPORT_REGION_PROGRESS_INTERVAL_SECONDS + ): + LOGGER.info( + "Export region %s: serialized %s/%s variants in %.1fs.", + region_label, + state_index, + total_states, + now - region_start, + ) + last_progress_log = now + else: + LOGGER.info( + "Export region %s: static region, serializing default content.", + region_label, + ) + + with ( + selector_states.suppress_page_selector_refresh(page), + selector_states.scoped_widget_values( + selector_widgets, + dict(zip(active_selector_ids, default_values)), + ), + ): + refreshed_part_view = _refresh_page_part_view( + page, + part_def, + page_id=page_def.page_id, + context_label="after restoring defaults", + ) + default_content = serialize_viewable( + refreshed_part_view, + disable_widgets=False, + widget_metadata=widget_metadata, + ) + region_nodes[id(part_view)] = { + "kind": "region", + "region_id": part_def.part_id, + "selector_ids": active_selector_ids, + "content_mode": "snapshot", + "default_key": default_key, + "default_content": default_content, + "variants": variants, + "variant_aliases": variant_aliases, + } + LOGGER.info( + "Export region %s: finished in %.1fs (variants: %s, aliases: %s).", + region_label, + perf_counter() - region_start, + len(variants), + len(variant_aliases), + ) + + return region_nodes + + +def _refresh_page_part_view( + page: Any, + part_def: Any, + *, + page_id: str, + context_label: str, +) -> pn.viewable.Viewable: + """Refresh one export section and resolve its current subtree.""" + if hasattr(page, "clear_query_cache"): + page.clear_query_cache() + if hasattr(page, "mark_section_stale"): + page.mark_section_stale(part_def.part_id) + page.refresh(force=False) + else: + page.refresh(force=True) + refreshed_part_view = part_def.view_for(page) + if refreshed_part_view is None: + raise ValueError( + f"Dashboard page {page_id!r} export region {part_def.part_id!r} " + f"resolved to no view {context_label}." + ) + return refreshed_part_view + + +def _serialize_page_diagnostics(page: Any) -> list[dict[str, Any]]: + diagnostics = getattr(page, "visualization_diagnostics", []) + serialized: list[dict[str, Any]] = [] + for diagnostic in diagnostics: + serialized.append( + { + "visualization_id": diagnostic.visualization_id, + "render_state": diagnostic.render_state, + "input_kind": diagnostic.input_kind, + "input_ids": list(diagnostic.input_ids), + "usable_run_labels": list(diagnostic.usable_run_labels), + "excluded_runs": [ + { + "label": issue.label, + "status": issue.status, + "detail": issue.detail, + "source_kind": issue.source_kind, + "source_id": issue.source_id, + "missing_columns": list(issue.missing_columns), + } + for issue in diagnostic.excluded_runs + ], + } + ) + return serialized diff --git a/dashboard/export/payload.py b/dashboard/export/payload.py index 184ec95..3673810 100644 --- a/dashboard/export/payload.py +++ b/dashboard/export/payload.py @@ -2,30 +2,22 @@ from __future__ import annotations -from contextlib import contextmanager -from itertools import product import json from typing import Any -from activitysim_viz_logging import get_logger +from runtime.logging import get_logger import panel as pn -from dashboard.components import ( - build_run_legend_entries, - set_percent_mode, - set_run_colors, - set_run_label_order, -) +from dashboard.rendering import RenderContext, run_legend_entries from dashboard import DashboardState from dashboard.export.context import ExportBuildContext from dashboard.export.protocols import validate_export_page from dashboard.export.serializer import ( json_default, page_definition_for_page, - sanitize_export_payload, - serialize_viewable, - variant_key, + sanitize_export_payload_in_place, ) +from dashboard.export import page_serializer, selector_states, traversal from dashboard.export.types import ( EXPORT_CLIENT_RUNTIME, EXPORT_PAGE_SELECTOR_RUNTIME, @@ -42,14 +34,13 @@ from dashboard.page_registry import ( build_export_prepared_run_provider, build_registered_export_pages, - effective_export_parts, group_definition_by_id, page_definition_by_id, resolve_export_navigation_entries, ) from processor.models import RunData -from processor.summarize.cache import SummaryRun -from runtime.config import Config, ExportSelectorRequest +from processor.summarize.cache_types import SummaryRun +from runtime.config import Config LOGGER = get_logger("dashboard.export") @@ -58,100 +49,6 @@ PAGE_WARNING_BYTES = 10 * 1024 * 1024 STATIC_REGION_WARNING_BYTES = 5 * 1024 * 1024 SELECTOR_REGION_WARNING_BYTES = 1 * 1024 * 1024 - - -class _RuntimeExportPart: - def __init__( - self, - *, - part_id: str, - selector_ids: tuple[str, ...], - export_data_mode: str, - view: pn.viewable.Viewable, - ) -> None: - self.part_id = part_id - self.selector_ids = selector_ids - self.export_data_mode = export_data_mode - self._view = view - - def view_for(self, page: Any) -> pn.viewable.Viewable | None: - return self._view - - -def _page_selector_defs( - page: Any, page_def: DashboardPageDefinition -) -> tuple[Any, ...]: - runtime_selectors = tuple(getattr(page, "registered_selectors", ())) - if runtime_selectors: - return runtime_selectors - return tuple(page_def.selectors) - - -def _selector_id(selector_def: Any) -> str: - return str(selector_def.selector_id) - - -def _selector_label(selector_def: Any) -> str: - return str(selector_def.label) - - -def _selector_exportable(selector_def: Any) -> bool: - return bool(getattr(selector_def, "exportable", True)) - - -def _selector_widget(selector_def: Any, page: Any) -> pn.widgets.Widget | None: - widget = getattr(selector_def, "widget", None) - if isinstance(widget, pn.widgets.Widget): - return widget - if hasattr(selector_def, "widget_for"): - return selector_def.widget_for(page) - return None - - -def _selector_available(selector_def: Any, page: Any, config: Config) -> bool: - widget = _selector_widget(selector_def, page) - if widget is None: - return False - if hasattr(selector_def, "available_for"): - return bool(selector_def.available_for(page, config)) - return True - - -def _selector_options(widget: pn.widgets.Widget) -> list[str]: - raw_options = getattr(widget, "options", None) - if raw_options is None: - return [] - return [str(option) for option in raw_options] - - -def _selector_supports_option_enumeration(widget: pn.widgets.Widget) -> bool: - return hasattr(widget, "options") - - -def _page_export_parts(page: Any, page_def: DashboardPageDefinition) -> tuple[Any, ...]: - runtime_sections = tuple(getattr(page, "registered_sections", ())) - if runtime_sections: - return tuple( - _RuntimeExportPart( - part_id=section.section_id, - selector_ids=section.selector_ids, - export_data_mode=section.export_data_mode, - view=section.container, - ) - for section in runtime_sections - if section.export - ) - return effective_export_parts(page_def) - - -def _part_uses_prepared_data(part_def: Any) -> bool: - return str(getattr(part_def, "export_data_mode", "none")) != "none" - - -def _include_part_in_export(part_def: Any) -> bool: - return not _part_uses_prepared_data(part_def) - - def _build_validation_page( page_def: DashboardPageDefinition, config: Config, @@ -178,7 +75,6 @@ def build_export_artifacts( summary_runs: list[SummaryRun] | None = None, ) -> tuple[ExportPayload, dict[str, Any]]: """Build export payload plus sidecar diagnostics.""" - set_run_colors(config.run_colors) validate_page_export_config(config) export_weight_values = config.export_html.panel_weighting_values() export_value_values = config.export_html.panel_value_values() @@ -188,7 +84,6 @@ def build_export_artifacts( prepared_run_provider=build_export_prepared_run_provider(runs, config), ) chrome_state = context.build_dashboard_state() - set_run_label_order(chrome_state.run_labels) state_payloads: dict[str, dict[str, Any]] = {} diagnostics_by_state: dict[str, Any] = {} @@ -208,7 +103,9 @@ def build_export_artifacts( payload: ExportPayload = { "schema_version": EXPORT_SCHEMA_VERSION, "title": config.dashboard_title, - "runs_loaded": build_run_legend_entries(chrome_state.run_labels), + "runs_loaded": run_legend_entries( + RenderContext.from_dashboard(config, chrome_state) + ), "chrome": { "layout": "left_rail", "rail_sections": ["runs_loaded", "display_options"], @@ -406,7 +303,7 @@ def emit_export_size_warnings(size_analysis: dict[str, Any] | None) -> None: def _estimate_json_bytes(value: Any) -> int: - sanitized = sanitize_export_payload(value) + sanitized = sanitize_export_payload_in_place(value) return len( json.dumps( sanitized, @@ -461,7 +358,7 @@ def _export_part_disable_hint( part_id: str, ) -> str: page_key = _page_config_key(page_def) - return f"visualizer.export_html.pages.{page_key}.parts.{part_id}.enabled: false" + return f"dashboard.export.pages.{page_key}.parts.{part_id}.enabled: false" def serialize_dashboard_state( @@ -475,7 +372,6 @@ def serialize_dashboard_state( state = context.build_dashboard_state() state.weight_mode = weight_mode state.value_mode = value_mode - set_percent_mode(value_mode == "Percent") pages = build_registered_export_pages(state, context.config) leaf_page_defs: list[PageDescriptorPayload] = [] @@ -488,16 +384,16 @@ def serialize_dashboard_state( if page.view is None: continue page_def = page_definition_for_page(page) - enabled_part_defs = resolve_enabled_export_parts( + enabled_part_defs = traversal.enabled_export_parts( page, page_def, page.config.export_html, ) - if _page_export_parts(page, page_def) and not enabled_part_defs: + if traversal.page_export_parts(page) and not enabled_part_defs: continue - selector_defs = _page_selector_defs(page, page_def) + selector_defs = traversal.page_selectors(page) selector_metadata_by_id = { - _selector_id(selector_def): resolve_selector_metadata( + traversal.selector_id(selector_def): resolve_selector_metadata( page_def, selector_def, page, @@ -505,8 +401,8 @@ def serialize_dashboard_state( ) for selector_def in selector_defs } - widget_metadata = build_widget_metadata( - page_def, + selector_states.apply_selector_dependencies(page, selector_metadata_by_id) + widget_metadata = page_serializer.build_widget_metadata( page, selector_metadata_by_id=selector_metadata_by_id, ) @@ -525,7 +421,7 @@ def serialize_dashboard_state( } leaf_page_defs.append(descriptor) page_by_id[page_def.page_id] = descriptor - content_by_page[page_def.page_id] = serialize_page_content( + content_by_page[page_def.page_id] = page_serializer.serialize_page_content( page, page_def=page_def, widget_metadata=widget_metadata, @@ -583,338 +479,6 @@ def _resolve_group_default_child_page_id( return page_definitions[0].page_id -def serialize_page_content( - page: Any, - *, - page_def: DashboardPageDefinition, - widget_metadata: dict[int, tuple[str | None, SelectorMetadataPayload | None]], - selector_metadata_by_id: dict[str, SelectorMetadataPayload], - diagnostics_for_state: dict[str, Any] | None = None, -) -> PageContentPayload: - """Serialize one page shell with explicit region nodes.""" - page_diagnostics: dict[str, Any] = {"default": _serialize_page_diagnostics(page)} - enabled_part_defs = resolve_enabled_export_parts( - page, - page_def, - page.config.export_html, - ) - enabled_part_ids = {part_def.part_id for part_def in enabled_part_defs} - enabled_selector_ids = { - selector_id - for part_def in enabled_part_defs - for selector_id in part_def.selector_ids - } - interactive_selectors = [ - selector_meta - for selector_meta in selector_metadata_by_id.values() - if selector_meta["export_enabled"] - ] - if interactive_selectors and not _page_export_parts(page, page_def): - raise ValueError( - f"Dashboard page {page_def.page_id!r} declares exportable selectors but no export parts." - ) - - region_nodes_by_id = build_region_nodes( - page, - page_def=page_def, - widget_metadata=widget_metadata, - selector_metadata_by_id=selector_metadata_by_id, - page_diagnostics=page_diagnostics, - ) - content = serialize_viewable( - page.view, - disable_widgets=False, - widget_metadata=widget_metadata, - region_nodes_by_id=region_nodes_by_id, - hidden_widget_ids={ - id(widget) - for selector_def in _page_selector_defs(page, page_def) - for widget in [_selector_widget(selector_def, page)] - if _selector_id(selector_def) not in enabled_selector_ids - and widget is not None - }, - hidden_view_ids={ - id(part_def.view_for(page)) - for part_def in _page_export_parts(page, page_def) - if part_def.part_id not in enabled_part_ids - and part_def.view_for(page) is not None - }, - ) - if diagnostics_for_state is not None: - diagnostics_for_state[page_def.page_id] = page_diagnostics - return {"kind": "page", "content": content} - - -def build_widget_metadata( - page_def: DashboardPageDefinition, - page: Any, - *, - selector_metadata_by_id: dict[str, SelectorMetadataPayload], -) -> dict[int, tuple[str | None, SelectorMetadataPayload | None]]: - """Build selector metadata keyed by widget identity for serialization.""" - metadata: dict[int, tuple[str | None, SelectorMetadataPayload | None]] = {} - for selector_def in _page_selector_defs(page, page_def): - selector_id = _selector_id(selector_def) - selector_meta = selector_metadata_by_id[selector_id] - widget = _selector_widget(selector_def, page) - if widget is not None: - metadata[id(widget)] = (selector_id, selector_meta) - return metadata - - -def build_region_nodes( - page: Any, - *, - page_def: DashboardPageDefinition, - widget_metadata: dict[int, tuple[str | None, SelectorMetadataPayload | None]], - selector_metadata_by_id: dict[str, SelectorMetadataPayload], - page_diagnostics: dict[str, Any], -) -> dict[int, dict[str, Any]]: - """Build serialized region nodes keyed by region-root object id.""" - if page.view is None: - return {} - - resolved_parts = resolve_page_parts(page, page_def=page_def) - if not resolved_parts: - return {} - - interactive_selector_ids = { - selector_id - for selector_id, selector_meta in selector_metadata_by_id.items() - if selector_meta["export_enabled"] - } - referenced_selector_ids = { - selector_id - for part_def, _ in resolved_parts - for selector_id in part_def.selector_ids - } - unused_selector_ids = sorted(interactive_selector_ids - referenced_selector_ids) - if unused_selector_ids: - raise ValueError( - f"Dashboard page {page_def.page_id!r} does not assign export regions to selector ids: " - + ", ".join(repr(selector_id) for selector_id in unused_selector_ids) - ) - - selector_widgets = { - _selector_id(selector_def): _selector_widget(selector_def, page) - for selector_def in _page_selector_defs(page, page_def) - } - region_nodes: dict[int, dict[str, Any]] = {} - for part_def, part_view in resolved_parts: - active_selector_ids = [ - selector_id - for selector_id in part_def.selector_ids - if selector_metadata_by_id.get(selector_id, {}).get("export_enabled") - ] - default_values = [ - selector_metadata_by_id[selector_id]["default_value"] - for selector_id in active_selector_ids - ] - default_key = variant_key(default_values) - variants: dict[str, Any] = {} - - if active_selector_ids: - selector_values = [ - selector_metadata_by_id[selector_id]["resolved_values"] - for selector_id in active_selector_ids - ] - for combination in product(*selector_values): - with temporary_widget_values( - selector_widgets, - dict(zip(active_selector_ids, combination)), - ): - refreshed_part_view = _refresh_page_part_view( - page, - part_def, - page_id=page_def.page_id, - context_label="during variant serialization", - ) - page_diagnostics[ - f"region:{part_def.part_id}:{variant_key(combination)}" - ] = _serialize_page_diagnostics(page) - variants[variant_key(combination)] = serialize_viewable( - refreshed_part_view, - disable_widgets=False, - widget_metadata=widget_metadata, - ) - - with temporary_widget_values( - selector_widgets, - dict(zip(active_selector_ids, default_values)), - ): - refreshed_part_view = _refresh_page_part_view( - page, - part_def, - page_id=page_def.page_id, - context_label="after restoring defaults", - ) - default_content = serialize_viewable( - refreshed_part_view, - disable_widgets=False, - widget_metadata=widget_metadata, - ) - region_nodes[id(part_view)] = { - "kind": "region", - "region_id": part_def.part_id, - "selector_ids": active_selector_ids, - "content_mode": "snapshot", - "default_key": default_key, - "default_content": default_content, - "variants": variants, - } - - return region_nodes - - -def resolve_page_parts( - page: Any, - *, - page_def: DashboardPageDefinition, -) -> list[tuple[Any, pn.viewable.Viewable]]: - """Resolve and validate explicit export parts for one page instance.""" - if page.view is None: - return [] - - root_paths = _view_paths_by_id(page.view) - enabled_part_defs = resolve_enabled_export_parts( - page, page_def, page.config.export_html - ) - resolved: list[tuple[Any, pn.viewable.Viewable]] = [] - for part_def in enabled_part_defs: - part_view = part_def.view_for(page) - if part_view is None: - raise ValueError( - f"Dashboard page {page_def.page_id!r} export region {part_def.part_id!r} " - "could not resolve its registered view." - ) - part_path = root_paths.get(id(part_view)) - if part_path is None: - raise ValueError( - f"Dashboard page {page_def.page_id!r} export region {part_def.part_id!r} " - "does not belong to the page view tree." - ) - resolved.append((part_def, part_view)) - - for index, (part_def, part_view) in enumerate(resolved): - part_path = root_paths[id(part_view)] - for other_def, other_view in resolved[index + 1 :]: - other_path = root_paths[id(other_view)] - if part_path == other_path: - raise ValueError( - f"Dashboard page {page_def.page_id!r} export regions {part_def.part_id!r} " - f"and {other_def.part_id!r} resolve to the same subtree." - ) - if _is_prefix(part_path, other_path) or _is_prefix(other_path, part_path): - raise ValueError( - f"Dashboard page {page_def.page_id!r} export regions {part_def.part_id!r} " - f"and {other_def.part_id!r} must not overlap or nest." - ) - - return resolved - - -@contextmanager -def temporary_widget_values( - selector_widgets: dict[str, pn.widgets.Widget | None], - values_by_selector_id: dict[str, Any], -): - """Temporarily set selector widget values and always restore the originals. - - Export region serialization needs to mutate live widget instances so each - selector combination can be rendered offline. This helper keeps that - mutation boundary explicit and guarantees restoration on both success and - failure paths. - """ - original_values = { - selector_id: widget.value - for selector_id, widget in selector_widgets.items() - if widget is not None - } - try: - for selector_id, value in values_by_selector_id.items(): - widget = selector_widgets.get(selector_id) - if widget is not None: - widget.value = value - yield - finally: - for selector_id, original_value in original_values.items(): - widget = selector_widgets.get(selector_id) - if widget is not None: - widget.value = original_value - - -def _refresh_page_part_view( - page: Any, - part_def: Any, - *, - page_id: str, - context_label: str, -) -> pn.viewable.Viewable: - """Refresh one page and resolve the current export-part subtree.""" - if hasattr(page, "clear_filtered_view_cache"): - page.clear_filtered_view_cache() - page.refresh(force=True) - refreshed_part_view = part_def.view_for(page) - if refreshed_part_view is None: - raise ValueError( - f"Dashboard page {page_id!r} export region {part_def.part_id!r} " - f"resolved to no view {context_label}." - ) - return refreshed_part_view - - -def resolve_enabled_export_parts( - page: Any, - page_def: DashboardPageDefinition, - export_html: Any, -) -> tuple[Any, ...]: - """Return the enabled export parts for one page definition.""" - override = export_html.page_override( - page_def.page_id, - group_id=page_def.group_id, - ) - enabled_parts = [] - for part_def in _page_export_parts(page, page_def): - if not _include_part_in_export(part_def): - continue - part_override = override.parts.get(part_def.part_id) - if part_override is not None and part_override.enabled is False: - continue - enabled_parts.append(part_def) - return tuple(enabled_parts) - - -def _view_paths_by_id( - root: pn.viewable.Viewable, - path: tuple[int, ...] = (), - result: dict[int, tuple[int, ...]] | None = None, -) -> dict[int, tuple[int, ...]]: - """Return stable DFS paths for supported viewables in one page tree.""" - result = result or {} - result[id(root)] = path - for index, child in enumerate(_child_viewables(root)): - _view_paths_by_id(child, path + (index,), result) - return result - - -def _child_viewables(view: pn.viewable.Viewable) -> list[pn.viewable.Viewable]: - """Return supported child viewables for export-region path validation.""" - if isinstance(view, (pn.Column, pn.Row, pn.Card)): - return [ - child for child in view.objects if isinstance(child, pn.viewable.Viewable) - ] - if isinstance(view, pn.Tabs): - return [ - child for child in view.objects if isinstance(child, pn.viewable.Viewable) - ] - return [] - - -def _is_prefix(left: tuple[int, ...], right: tuple[int, ...]) -> bool: - """Return whether one view path is an ancestor-prefix of another.""" - return len(left) < len(right) and right[: len(left)] == left - - def resolve_selector_metadata( page_def: DashboardPageDefinition, selector_def: Any, @@ -936,9 +500,9 @@ def resolve_selector_metadata( group_id=page_def.group_id, ).selector_requests ) - widget = _selector_widget(selector_def, page) - available = _selector_available(selector_def, page, context.config) - export_enabled = _selector_exportable(selector_def) + widget = traversal.selector_widget(selector_def) + available = traversal.selector_available(selector_def) + export_enabled = traversal.selector_exportable(selector_def) if not available or widget is None: if configured: @@ -953,7 +517,7 @@ def resolve_selector_metadata( context.warned_unavailable_selectors.add(warning_key) return { "id": selector_id, - "label": _selector_label(selector_def), + "label": traversal.selector_label(selector_def), "available": False, "request_mode": request.mode, "requested_values": list(request.values), @@ -963,15 +527,15 @@ def resolve_selector_metadata( "export_enabled": False, } - options = _selector_options(widget) + options = selector_states.selector_options(widget) default_value = str(widget.value) - enabled_part_defs = resolve_enabled_export_parts( + enabled_part_defs = traversal.enabled_export_parts( page, page_def, context.config.export_html ) selector_used_by_enabled_part = any( selector_id in part_def.selector_ids for part_def in enabled_part_defs ) - supports_option_enumeration = _selector_supports_option_enumeration(widget) + supports_option_enumeration = selector_states.supports_option_enumeration(widget) if configured and not selector_used_by_enabled_part: warning_key = (page_id, selector_id, "unused") if warning_key not in context.warned_unavailable_selectors: @@ -983,7 +547,7 @@ def resolve_selector_metadata( context.warned_unavailable_selectors.add(warning_key) return { "id": selector_id, - "label": _selector_label(selector_def), + "label": traversal.selector_label(selector_def), "available": True, "request_mode": request.mode, "requested_values": list(request.values), @@ -1006,7 +570,7 @@ def resolve_selector_metadata( context.warned_unavailable_selectors.add(warning_key) return { "id": selector_id, - "label": _selector_label(selector_def), + "label": traversal.selector_label(selector_def), "available": True, "request_mode": request.mode, "requested_values": list(request.values), @@ -1015,16 +579,16 @@ def resolve_selector_metadata( "options": options, "export_enabled": False, } - resolved_values = resolve_selector_values( + resolved_values = selector_states.resolve_selector_values( request=request, options=options, default_value=default_value, field_name=_selector_field_name(page_def, selector_id), ) - export_enabled = _selector_exportable(selector_def) and len(resolved_values) > 1 + export_enabled = traversal.selector_exportable(selector_def) and len(resolved_values) > 1 return { "id": selector_id, - "label": _selector_label(selector_def), + "label": traversal.selector_label(selector_def), "available": True, "request_mode": request.mode, "requested_values": list(request.values), @@ -1035,51 +599,17 @@ def resolve_selector_metadata( } -def resolve_selector_values( - *, - request: ExportSelectorRequest, - options: list[str], - default_value: str, - field_name: str, -) -> list[str]: - """Resolve configured selector requests against actual widget options.""" - if request.mode == "default": - return [default_value] - if request.mode == "all": - if not options: - raise ValueError(f"{field_name} resolved to no values.") - return list(options) - - option_lookup = {option.strip().lower(): option for option in options} - resolved: list[str] = [] - invalid: list[str] = [] - for token in request.values: - option = option_lookup.get(token) - if option is None: - invalid.append(token) - continue - if option not in resolved: - resolved.append(option) - if invalid: - raise ValueError( - f"Unsupported {field_name} values: " - + ", ".join(repr(token) for token in invalid) - ) - if not resolved: - raise ValueError(f"{field_name} resolved to no values.") - return resolved - - def validate_page_export_config(config: Config) -> None: """Validate export page and selector ids against the live registry.""" unknown_pages = sorted( page_id for page_id in config.export_html.pages if _page_definition_for_export_override(page_id) is None + and group_definition_by_id(page_id) is None ) if unknown_pages: raise ValueError( - "Unsupported visualizer.export_html.pages entries: " + "Unsupported dashboard.export.pages entries: " + ", ".join(repr(page_id) for page_id in unknown_pages) ) @@ -1090,7 +620,7 @@ def validate_page_export_config(config: Config) -> None: ) if unknown_excluded_pages: raise ValueError( - "Unsupported visualizer.export_html.exclude_pages entries: " + "Unsupported dashboard.export.exclude_pages entries: " + ", ".join(repr(page_id) for page_id in unknown_excluded_pages) ) unknown_excluded_groups = sorted( @@ -1100,29 +630,31 @@ def validate_page_export_config(config: Config) -> None: ) if unknown_excluded_groups: raise ValueError( - "Unsupported visualizer.export_html.exclude_groups entries: " + "Unsupported dashboard.export.exclude_groups entries: " + ", ".join(repr(group_id) for group_id in unknown_excluded_groups) ) for page_id, override in config.export_html.pages.items(): page_def = _page_definition_for_export_override(page_id) if page_def is None: + # A group-level empty mapping is a valid no-op override and has no + # leaf selectors or parts to validate. continue page = _build_validation_page(page_def, config) - selector_defs = _page_selector_defs(page, page_def) + selector_defs = traversal.page_selectors(page) unknown_selectors = sorted( selector_id for selector_id in override.selector_requests if selector_defs and selector_id - not in {_selector_id(selector_def) for selector_def in selector_defs} + not in {traversal.selector_id(selector_def) for selector_def in selector_defs} ) if unknown_selectors: raise ValueError( - f"Unsupported visualizer.export_html.pages.{page_id} entries: " + f"Unsupported dashboard.export.pages.{page_id} entries: " + ", ".join(repr(selector_id) for selector_id in unknown_selectors) ) - export_parts = _page_export_parts(page, page_def) + export_parts = traversal.page_export_parts(page) unknown_parts = sorted( part_id for part_id in override.parts @@ -1130,7 +662,7 @@ def validate_page_export_config(config: Config) -> None: ) if unknown_parts: raise ValueError( - f"Unsupported visualizer.export_html.pages.{_page_config_key(page_def)}.parts entries: " + f"Unsupported dashboard.export.pages.{_page_config_key(page_def)}.parts entries: " + ", ".join(repr(part_id) for part_id in unknown_parts) ) @@ -1159,7 +691,7 @@ def _selector_field_name( page_def: DashboardPageDefinition, selector_id: str, ) -> str: - return f"visualizer.export_html.pages.{_page_config_key(page_def)}.{selector_id}" + return f"dashboard.export.pages.{_page_config_key(page_def)}.{selector_id}" def _page_config_key(page_def: DashboardPageDefinition | None) -> str: @@ -1188,30 +720,3 @@ def _page_definition_for_export_override( def state_key(weight_mode: str, value_mode: str) -> str: """Return the stable key for one dashboard-level state combination.""" return f"{weight_mode}||{value_mode}" - - -def _serialize_page_diagnostics(page: Any) -> list[dict[str, Any]]: - diagnostics = getattr(page, "visualization_diagnostics", []) - serialized: list[dict[str, Any]] = [] - for diagnostic in diagnostics: - serialized.append( - { - "visualization_id": diagnostic.visualization_id, - "render_state": diagnostic.render_state, - "input_kind": diagnostic.input_kind, - "input_ids": list(diagnostic.input_ids), - "usable_run_labels": list(diagnostic.usable_run_labels), - "excluded_runs": [ - { - "label": issue.label, - "status": issue.status, - "detail": issue.detail, - "source_kind": issue.source_kind, - "source_id": issue.source_id, - "missing_columns": list(issue.missing_columns), - } - for issue in diagnostic.excluded_runs - ], - } - ) - return serialized diff --git a/dashboard/export/runtime_assets.py b/dashboard/export/runtime_assets.py index 2578717..4ca179f 100644 --- a/dashboard/export/runtime_assets.py +++ b/dashboard/export/runtime_assets.py @@ -30,12 +30,12 @@ def load_export_runtime_js() -> str: ) -def build_export_html_shell(*, title: str, payload_json: str, plotly_js: str) -> str: - """Assemble the final self-contained HTML document.""" +def build_export_html_shell_parts(*, title: str, plotly_js: str) -> tuple[str, str]: + """Build the shell surrounding the export payload JSON.""" export_css = load_export_css() runtime_js = load_export_runtime_js() - return f""" + prefix = f""" @@ -50,10 +50,19 @@ def build_export_html_shell(*, title: str, payload_json: str, plotly_js: str) -> - + """ + return prefix, suffix + + +def build_export_html_shell(*, title: str, payload_json: str, plotly_js: str) -> str: + """Assemble the final self-contained HTML document.""" + + prefix, suffix = build_export_html_shell_parts(title=title, plotly_js=plotly_js) + return f"{prefix}{payload_json}{suffix}" diff --git a/dashboard/export/selector_states.py b/dashboard/export/selector_states.py new file mode 100644 index 0000000..a61e9dc --- /dev/null +++ b/dashboard/export/selector_states.py @@ -0,0 +1,350 @@ +"""Resolve and safely enumerate page-selector states for offline export.""" + +from __future__ import annotations + +from contextlib import contextmanager +from typing import Any, Iterator + +import panel as pn + +from dashboard.export.serializer import variant_key +from dashboard.export.types import SelectorMetadataPayload +from dashboard.page_definitions import DashboardPageDefinition +from runtime.config import ExportSelectorRequest + + +def selector_options(widget: pn.widgets.Widget) -> list[str]: + if isinstance(widget, pn.widgets.Checkbox): + return ["False", "True"] + options = getattr(widget, "options", None) + return [] if options is None else [str(option) for option in options] + + +def supports_option_enumeration(widget: pn.widgets.Widget) -> bool: + return isinstance(widget, pn.widgets.Checkbox) or hasattr(widget, "options") + + +def resolve_selector_values( + *, + request: ExportSelectorRequest, + options: list[str], + default_value: str, + field_name: str, +) -> list[str]: + """Resolve a configured selector request against current widget options.""" + if request.mode == "default": + return [default_value] + if request.mode == "all": + if not options: + raise ValueError(f"{field_name} resolved to no values.") + return list(options) + + option_lookup = {option.strip().lower(): option for option in options} + resolved: list[str] = [] + invalid: list[str] = [] + for token in request.values: + option = option_lookup.get(token) + if option is None and token == "all": + option = next( + ( + candidate + for candidate in options + if candidate.strip().lower() == "all" + or candidate.strip().lower().startswith("all ") + ), + None, + ) + if option is None: + invalid.append(token) + elif option not in resolved: + resolved.append(option) + if invalid: + raise ValueError( + f"Unsupported {field_name} values: " + + ", ".join(repr(token) for token in invalid) + + ". Supported values: " + + ", ".join(repr(option) for option in options) + ) + if not resolved: + raise ValueError(f"{field_name} resolved to no values.") + return resolved + + +def apply_selector_dependencies( + page: Any, + selector_metadata_by_id: dict[str, SelectorMetadataPayload], +) -> None: + """Attach page-declared dependent selector domains to export metadata.""" + dependency_provider = getattr(page, "export_selector_dependencies", None) + if not callable(dependency_provider): + return + + dependencies = dependency_provider() or {} + for selector_id, dependency in dependencies.items(): + selector_meta = selector_metadata_by_id.get(str(selector_id)) + parent_selector_id = str(dependency.get("parent_selector_id", "")) + parent_meta = selector_metadata_by_id.get(parent_selector_id) + if selector_meta is None or parent_meta is None: + continue + + raw_options_by_parent = dependency.get("options_by_parent_value", {}) + if not isinstance(raw_options_by_parent, dict): + continue + parent_values = [str(value) for value in parent_meta["resolved_values"]] + options_by_parent_value = { + parent_value: [ + str(option) + for option in raw_options_by_parent.get(parent_value, []) + ] + for parent_value in parent_values + } + allowed_options = list( + dict.fromkeys( + option + for parent_value in parent_values + for option in options_by_parent_value[parent_value] + ) + ) + if not allowed_options: + continue + + original_resolved = { + str(value) for value in selector_meta["resolved_values"] + } + resolved_values = ( + allowed_options + if selector_meta["request_mode"] == "all" + else [ + option for option in allowed_options if option in original_resolved + ] + ) + if not resolved_values: + resolved_values = [allowed_options[0]] + + selector_meta["options"] = allowed_options + selector_meta["resolved_values"] = resolved_values + if str(selector_meta["default_value"]) not in allowed_options: + selector_meta["default_value"] = allowed_options[0] + selector_meta["export_enabled"] = bool( + selector_meta["available"] and len(resolved_values) > 1 + ) + selector_meta["parent_selector_id"] = parent_selector_id + selector_meta["options_by_parent_value"] = options_by_parent_value + selector_meta["disabled_parent_values"] = [ + str(value) + for value in dependency.get("disabled_parent_values", []) + if str(value) in parent_values + ] + + +def resolve_export_section_states( + page: Any, + *, + page_def: DashboardPageDefinition, + part_def: Any, + active_selector_ids: list[str], + selector_widgets: dict[str, pn.widgets.Widget | None], + selector_metadata_by_id: dict[str, SelectorMetadataPayload], +) -> tuple[list[dict[str, str]], dict[str, str]]: + """Return canonical section states and aliases for collapsed raw states.""" + if not active_selector_ids: + return [], {} + + states_by_key: dict[str, dict[str, str]] = {} + aliases: dict[str, str] = {} + + def visit( + index: int, + canonical_values: dict[str, str], + raw_values: dict[str, str], + ) -> None: + with suppress_page_selector_refresh(page), scoped_widget_values( + selector_widgets, canonical_values + ): + sync_page_controls(page) + if index >= len(active_selector_ids): + effective_values = effective_selector_values( + active_selector_ids, selector_widgets + ) + state_key = variant_key( + [effective_values[item] for item in active_selector_ids] + ) + raw_key = variant_key( + [ + raw_values.get(item, effective_values[item]) + for item in active_selector_ids + ] + ) + states_by_key.setdefault(state_key, effective_values) + if raw_key != state_key: + aliases[raw_key] = state_key + return + + selector_id = active_selector_ids[index] + widget = selector_widgets.get(selector_id) + if widget is None: + return + ignored = export_ignored_selectors(page, part_def.part_id, canonical_values) + candidates = selector_values_for_current_state( + selector_id=selector_id, + widget=widget, + selector_metadata=selector_metadata_by_id[selector_id], + selected_values=canonical_values, + ) + if not candidates: + return + collapsed = ( + bool(getattr(widget, "disabled", False)) or selector_id in ignored + ) + for raw_value in candidates: + canonical_value = export_canonical_selector_value( + page, + part_def.part_id, + selector_id, + raw_value, + canonical_values, + ) + if collapsed and canonical_value == raw_value: + canonical_value = str(widget.value) + visit( + index + 1, + {**canonical_values, selector_id: canonical_value}, + {**raw_values, selector_id: raw_value}, + ) + + visit(0, {}, {}) + aliases = { + raw_key: canonical_key + for raw_key, canonical_key in aliases.items() + if canonical_key in states_by_key and raw_key != canonical_key + } + if not states_by_key: + raise ValueError( + f"Dashboard page {page_def.page_id!r} export region " + f"{part_def.part_id!r} resolved to no valid selector states." + ) + return list(states_by_key.values()), aliases + + +def sync_page_controls(page: Any) -> None: + sync_declared = getattr(page, "_sync_declared_selectors", None) + if callable(sync_declared): + sync_declared() + sync_controls = getattr(page, "sync_controls", None) + if callable(sync_controls): + sync_controls() + + +@contextmanager +def suppress_page_selector_refresh(page: Any) -> Iterator[None]: + if not hasattr(page, "_is_refreshing"): + yield + return + previous_refreshing = bool(getattr(page, "_is_refreshing")) + previous_queue = set(getattr(page, "_queued_selector_ids", set())) + page._is_refreshing = True + try: + yield + finally: + page._is_refreshing = previous_refreshing + if hasattr(page, "_queued_selector_ids"): + page._queued_selector_ids = previous_queue + + +@contextmanager +def scoped_widget_values( + selector_widgets: dict[str, pn.widgets.Widget | None], + values_by_selector_id: dict[str, Any], +) -> Iterator[None]: + """Set selector widgets temporarily and restore them on every exit path.""" + original_values = { + selector_id: widget.value + for selector_id, widget in selector_widgets.items() + if widget is not None + } + try: + for selector_id, value in values_by_selector_id.items(): + widget = selector_widgets.get(selector_id) + if widget is not None: + widget.value = coerce_widget_value(widget, value) + yield + finally: + for selector_id, original_value in original_values.items(): + widget = selector_widgets.get(selector_id) + if widget is not None: + widget.value = original_value + + +def coerce_widget_value(widget: pn.widgets.Widget, value: Any) -> Any: + if isinstance(widget, pn.widgets.Checkbox) and isinstance(value, str): + normalized = value.strip().lower() + if normalized == "true": + return True + if normalized == "false": + return False + return value + + +def effective_selector_values( + active_selector_ids: list[str], + selector_widgets: dict[str, pn.widgets.Widget | None], +) -> dict[str, str]: + return { + selector_id: str(widget.value) + for selector_id in active_selector_ids + if (widget := selector_widgets.get(selector_id)) is not None + } + + +def export_ignored_selectors( + page: Any, + section_id: str, + selected_values: dict[str, str], +) -> set[str]: + ignored = getattr(page, "export_ignored_selectors", None) + if not callable(ignored): + return set() + return set(ignored(section_id, dict(selected_values)) or set()) + + +def export_canonical_selector_value( + page: Any, + section_id: str, + selector_id: str, + value: str, + selected_values: dict[str, str], +) -> str: + canonical = getattr(page, "export_canonical_selector_value", None) + if not callable(canonical): + return value + return str(canonical(section_id, selector_id, value, dict(selected_values))) + + +def selector_values_for_current_state( + *, + selector_id: str, + widget: pn.widgets.Widget, + selector_metadata: SelectorMetadataPayload, + selected_values: dict[str, str] | None = None, +) -> list[str]: + options = selector_options(widget) + parent_selector_id = selector_metadata.get("parent_selector_id") + if parent_selector_id and selected_values is not None: + parent_value = selected_values.get(parent_selector_id) + options_by_parent = selector_metadata.get("options_by_parent_value", {}) + if parent_value in options_by_parent: + options = list(options_by_parent[parent_value]) + default_value = str(widget.value) + request_mode = selector_metadata["request_mode"] + if request_mode == "default": + return [default_value] + if request_mode == "all": + return options or [default_value] + + option_lookup = {option.strip().lower(): option for option in options} + resolved: list[str] = [] + for value in selector_metadata["resolved_values"]: + option = option_lookup.get(str(value).strip().lower()) + if option is not None and option not in resolved: + resolved.append(option) + return resolved diff --git a/dashboard/export/serializer.py b/dashboard/export/serializer.py index 435294f..f952361 100644 --- a/dashboard/export/serializer.py +++ b/dashboard/export/serializer.py @@ -12,7 +12,7 @@ import pandas as pd import panel as pn -from dashboard.components import format_numeric_for_display +from dashboard.rendering import format_numeric from dashboard.export.types import ExportNode from dashboard.page_definitions import DashboardPageDefinition @@ -24,7 +24,7 @@ def _serialize_table_cell(value: Any) -> Any: Tabulator, so we normalize numeric display here using the same significant-digit rule as live dashboard tables. """ - return format_numeric_for_display(sanitize_export_payload(value), precision=2) + return format_numeric(sanitize_export_payload(value), precision=2) def serialize_viewable( @@ -41,8 +41,37 @@ def serialize_viewable( region_nodes_by_id = region_nodes_by_id or {} hidden_widget_ids = hidden_widget_ids or set() hidden_view_ids = hidden_view_ids or set() + + def _is_hidden_view(viewable: Any) -> bool: + if id(viewable) in hidden_view_ids: + return True + note_target_id = getattr(viewable, "_calculation_note_target_id", None) + if note_target_id in hidden_view_ids: + return True + css_classes = set(getattr(viewable, "css_classes", []) or []) + if not css_classes.intersection( + {"calculation-note-section", "calculation-note-view"} + ): + return False + return any( + id(child) in hidden_view_ids + for child in getattr(viewable, "objects", []) + ) + if id(obj) in region_nodes_by_id: return region_nodes_by_id[id(obj)] + + def _container_styles(viewable: Any) -> dict[str, Any]: + styles = getattr(viewable, "styles", None) + if not isinstance(styles, dict): + return {} + return {str(key): value for key, value in styles.items() if value is not None} + + def _container_css_classes(viewable: Any) -> list[str]: + css_classes = getattr(viewable, "css_classes", None) + if not css_classes: + return [] + return [str(css_class) for css_class in css_classes if css_class] if isinstance(obj, pn.Card): children = [ serialize_viewable( @@ -54,7 +83,7 @@ def serialize_viewable( hidden_view_ids=hidden_view_ids, ) for child in obj.objects - if id(child) not in hidden_view_ids + if not _is_hidden_view(child) and not ( isinstance(child, pn.widgets.Widget) and id(child) in hidden_widget_ids ) @@ -75,7 +104,7 @@ def serialize_viewable( hidden_view_ids=hidden_view_ids, ) for child in obj.objects - if id(child) not in hidden_view_ids + if not _is_hidden_view(child) and not ( isinstance(child, pn.widgets.Widget) and id(child) in hidden_widget_ids ) @@ -85,6 +114,8 @@ def serialize_viewable( "layout": "column", "child_count": len(children), "children": children, + "styles": _container_styles(obj), + "css_classes": _container_css_classes(obj), } if isinstance(obj, pn.Row): children = [ @@ -97,7 +128,7 @@ def serialize_viewable( hidden_view_ids=hidden_view_ids, ) for child in obj.objects - if id(child) not in hidden_view_ids + if not _is_hidden_view(child) and not ( isinstance(child, pn.widgets.Widget) and id(child) in hidden_widget_ids ) @@ -107,6 +138,8 @@ def serialize_viewable( "layout": "row", "child_count": len(children), "children": children, + "styles": _container_styles(obj), + "css_classes": _container_css_classes(obj), } if isinstance(obj, pn.Tabs): return { @@ -124,7 +157,7 @@ def serialize_viewable( ), } for title, child in iter_tabs(obj) - if id(child) not in hidden_view_ids + if not _is_hidden_view(child) ], } if isinstance(obj, pn.pane.Plotly): @@ -134,13 +167,20 @@ def serialize_viewable( return {"kind": "plotly", "figure": figure, "height": height} if isinstance(obj, pn.widgets.Tabulator): frame = obj.value + title_map = { + str(column): str(title) + for column, title in (obj.titles or {}).items() + if title is not None + } + columns = [str(column) for column in frame.columns] + display_columns = [title_map.get(column, column) for column in columns] return { "kind": "table", - "columns": [str(column) for column in frame.columns], + "columns": display_columns, "rows": [ { - str(column): _serialize_table_cell(value) - for column, value in row.items() + display_column: _serialize_table_cell(row.get(column)) + for column, display_column in zip(columns, display_columns) } for row in frame.to_dict(orient="records") ], @@ -189,7 +229,7 @@ def serialize_viewable( disabled = False else: disabled = True - return { + payload = { "kind": "widget", "widget_type": "select", "name": widget_name, @@ -199,6 +239,46 @@ def serialize_viewable( "selector_id": selector_id, "export_enabled": bool(selector_meta and selector_meta["export_enabled"]), } + if selector_meta and selector_meta.get("parent_selector_id"): + payload.update( + { + "parent_selector_id": selector_meta["parent_selector_id"], + "options_by_parent_value": selector_meta.get( + "options_by_parent_value", {} + ), + "disabled_parent_values": selector_meta.get( + "disabled_parent_values", [] + ), + } + ) + return payload + if isinstance(obj, pn.widgets.Checkbox): + if id(obj) in hidden_widget_ids: + return {"kind": "spacer", "height": 0, "width": 0} + selector_id, selector_meta = widget_metadata.get(id(obj), (None, None)) + widget_name = ( + str(selector_meta.get("label")) + if selector_meta and selector_meta.get("label") + else obj.name or "" + ) + options = ["False", "True"] + disabled = True if disable_widgets else bool(obj.disabled) + if selector_meta: + if selector_meta["export_enabled"]: + options = list(selector_meta["resolved_values"]) + disabled = False + else: + disabled = True + return { + "kind": "widget", + "widget_type": "checkbox", + "name": widget_name, + "value": "True" if obj.value else "False", + "options": options, + "disabled": disabled, + "selector_id": selector_id, + "export_enabled": bool(selector_meta and selector_meta["export_enabled"]), + } if isinstance(obj, pn.widgets.FloatInput): if id(obj) in hidden_widget_ids: return {"kind": "spacer", "height": 0, "width": 0} @@ -222,6 +302,18 @@ def serialize_viewable( "selector_id": selector_id, "export_enabled": bool(selector_meta and selector_meta["export_enabled"]), } + if isinstance(obj, pn.widgets.Button): + return { + "kind": "widget", + "widget_type": "button", + "name": obj.name or "", + "value": obj.name or "", + "options": [], + "step": None, + "disabled": True if disable_widgets else bool(obj.disabled), + "selector_id": None, + "export_enabled": False, + } if isinstance(obj, pn.pane.Markdown): source = ( obj.object if isinstance(obj.object, str) else obj.object._repr_markdown_() @@ -264,7 +356,7 @@ def page_definition_for_page(page: Any) -> DashboardPageDefinition: if isinstance(page_def, DashboardPageDefinition): return page_def raise ValueError( - f"Dashboard page {type(page).__name__} is missing its registered PAGE definition." + f"Dashboard page {type(page).__name__} is missing its @dashboard_page definition." ) @@ -298,6 +390,41 @@ def sanitize_export_payload(value: Any) -> Any: return value +def sanitize_export_payload_in_place(value: Any) -> Any: + """Replace JSON-unsafe values while retaining existing containers. + + Export payloads are transient and can be very large. Mutating their + dictionaries and lists avoids constructing a second complete object graph + immediately before JSON serialization. + """ + + if isinstance(value, dict): + for key, item in value.items(): + value[key] = sanitize_export_payload_in_place(item) + return value + if isinstance(value, list): + for index, item in enumerate(value): + value[index] = sanitize_export_payload_in_place(item) + return value + if isinstance(value, tuple): + return [sanitize_export_payload_in_place(item) for item in value] + if isinstance(value, np.integer): + return int(value) + if isinstance(value, np.floating): + if np.isnan(value) or np.isinf(value): + return None + return float(value) + if isinstance(value, float): + if np.isnan(value) or np.isinf(value): + return None + return value + if isinstance(value, pd.Timestamp): + return value.isoformat() + if pd.isna(value): + return None + return value + + def json_default(value: Any) -> Any: """Serialize pandas/numpy scalars that standard ``json`` cannot handle.""" if isinstance(value, np.integer): diff --git a/dashboard/export/traversal.py b/dashboard/export/traversal.py new file mode 100644 index 0000000..e2c4606 --- /dev/null +++ b/dashboard/export/traversal.py @@ -0,0 +1,154 @@ +"""Traverse dashboard pages and resolve their exportable section subtrees.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import panel as pn + +from dashboard.page_definitions import DashboardPageDefinition + + +@dataclass(frozen=True) +class RuntimeExportPart: + """Export-facing view of one section registered by a dashboard page.""" + + part_id: str + selector_ids: tuple[str, ...] + export_data_mode: str + view: pn.viewable.Viewable + + def view_for(self, page: Any) -> pn.viewable.Viewable: + return self.view + + +def page_selectors(page: Any) -> tuple[Any, ...]: + return tuple(getattr(page, "registered_selectors", ())) + + +def selector_id(selector: Any) -> str: + return str(selector.selector_id) + + +def selector_label(selector: Any) -> str: + return str(selector.label) + + +def selector_exportable(selector: Any) -> bool: + return bool(getattr(selector, "exportable", True)) + + +def selector_widget(selector: Any) -> pn.widgets.Widget | None: + widget = getattr(selector, "widget", None) + return widget if isinstance(widget, pn.widgets.Widget) else None + + +def selector_available(selector: Any) -> bool: + return selector_widget(selector) is not None + + +def page_export_parts(page: Any) -> tuple[RuntimeExportPart, ...]: + """Project registered page sections onto the export traversal contract.""" + exportable_selector_ids = { + selector_id(selector) + for selector in page_selectors(page) + if selector_exportable(selector) + } + return tuple( + RuntimeExportPart( + part_id=section.section_id, + selector_ids=tuple( + item + for item in section.selector_ids + if item in exportable_selector_ids + ), + export_data_mode=section.export_data_mode, + view=section.container, + ) + for section in tuple(getattr(page, "registered_sections", ())) + if section.export + ) + + +def enabled_export_parts( + page: Any, + page_definition: DashboardPageDefinition, + export_config: Any, +) -> tuple[RuntimeExportPart, ...]: + """Return sections enabled by both their page contract and export config.""" + override = export_config.page_override( + page_definition.page_id, + group_id=page_definition.group_id, + ) + enabled: list[RuntimeExportPart] = [] + for part in page_export_parts(page): + if part.export_data_mode != "none": + continue + part_override = override.parts.get(part.part_id) + if part_override is not None and part_override.enabled is False: + continue + enabled.append(part) + return tuple(enabled) + + +def resolve_page_parts( + page: Any, + *, + page_def: DashboardPageDefinition, +) -> list[tuple[RuntimeExportPart, pn.viewable.Viewable]]: + """Resolve non-overlapping export section roots inside one page tree.""" + if page.view is None: + return [] + + paths = view_paths_by_id(page.view) + resolved: list[tuple[RuntimeExportPart, pn.viewable.Viewable]] = [] + for part in enabled_export_parts(page, page_def, page.config.export_html): + part_view = part.view_for(page) + part_path = paths.get(id(part_view)) + if part_path is None: + raise ValueError( + f"Dashboard page {page_def.page_id!r} export region " + f"{part.part_id!r} does not belong to the page view tree." + ) + resolved.append((part, part_view)) + + for index, (part, part_view) in enumerate(resolved): + part_path = paths[id(part_view)] + for other, other_view in resolved[index + 1 :]: + other_path = paths[id(other_view)] + if part_path == other_path: + raise ValueError( + f"Dashboard page {page_def.page_id!r} export regions " + f"{part.part_id!r} and {other.part_id!r} resolve to the same subtree." + ) + if is_prefix(part_path, other_path) or is_prefix(other_path, part_path): + raise ValueError( + f"Dashboard page {page_def.page_id!r} export regions " + f"{part.part_id!r} and {other.part_id!r} must not overlap or nest." + ) + return resolved + + +def view_paths_by_id( + root: pn.viewable.Viewable, + path: tuple[int, ...] = (), + result: dict[int, tuple[int, ...]] | None = None, +) -> dict[int, tuple[int, ...]]: + result = result or {} + result[id(root)] = path + for index, child in enumerate(child_viewables(root)): + view_paths_by_id(child, path + (index,), result) + return result + + +def child_viewables(view: pn.viewable.Viewable) -> list[pn.viewable.Viewable]: + if isinstance(view, (pn.Column, pn.Row, pn.Card, pn.Tabs)): + return [ + child for child in view.objects if isinstance(child, pn.viewable.Viewable) + ] + return [] + + +def is_prefix(left: tuple[int, ...], right: tuple[int, ...]) -> bool: + return len(left) < len(right) and right[: len(left)] == left diff --git a/dashboard/export/types.py b/dashboard/export/types.py index 03e1748..efa4f24 100644 --- a/dashboard/export/types.py +++ b/dashboard/export/types.py @@ -20,11 +20,17 @@ "region", ] ContainerLayout = Literal["row", "column"] -WidgetType = Literal["radio_button_group", "select", "float_input"] +WidgetType = Literal["radio_button_group", "select", "float_input", "checkbox", "button"] RegionContentMode = Literal["snapshot"] -class SelectorMetadataPayload(TypedDict): +class DependentSelectorPayload(TypedDict, total=False): + parent_selector_id: str + options_by_parent_value: dict[str, list[str]] + disabled_parent_values: list[str] + + +class SelectorMetadataPayload(DependentSelectorPayload): id: str label: str available: bool @@ -74,6 +80,9 @@ class ContainerNode(TypedDict): kind: Literal["container"] layout: ContainerLayout children: list["ExportNode"] + child_count: int + styles: dict[str, Any] + css_classes: list[str] class CardNode(TypedDict): @@ -103,7 +112,7 @@ class TableNode(TypedDict): rows: list[dict[str, Any]] -class WidgetNode(TypedDict): +class WidgetNode(DependentSelectorPayload): kind: Literal["widget"] widget_type: WidgetType name: str @@ -132,6 +141,7 @@ class RegionNode(TypedDict): default_key: str default_content: "ExportNode" variants: dict[str, "ExportNode"] + variant_aliases: dict[str, str] ExportNode = ( diff --git a/dashboard/helpers/category_helpers.py b/dashboard/helpers/category_helpers.py index 75b2037..667d73a 100644 --- a/dashboard/helpers/category_helpers.py +++ b/dashboard/helpers/category_helpers.py @@ -135,6 +135,34 @@ def ordered_category_values( return config.ordered_values(category_id, values) +def category_label_matches( + config: Config, + category_id: str, + raw_value: object, + label: str, +) -> bool: + """Return whether one raw category value resolves to the target display label.""" + return ( + config.label_value(category_id, raw_value).strip().casefold() + == str(label).strip().casefold() + ) + + +def exclude_category_values_by_label( + raw_values: Iterable[str], + *, + category_id: str, + config: Config, + label: str, +) -> list[str]: + """Return raw category values whose display label does not match ``label``.""" + return [ + str(raw_value) + for raw_value in raw_values + if not category_label_matches(config, category_id, raw_value, label) + ] + + def selector_options_from_values( raw_values: Iterable[str], *, @@ -320,10 +348,117 @@ def numeric_like_sort_expr(column: str) -> pl.Expr: return pl.when(base_value.is_null()).then(pl.lit(float("inf"))).otherwise(base_value) +def capped_numeric_category_expr( + column: str, + cap_value: int, + *, + target_col: str | None = None, +) -> pl.Expr: + """Return a string category expression with values at or above ``cap_value`` capped.""" + numeric_value = pl.col(column).cast(pl.Int64, strict=False) + return ( + pl.when(numeric_value >= cap_value) + .then(pl.lit(f"{cap_value}+")) + .otherwise(pl.col(column).cast(pl.Int64, strict=False).cast(pl.Utf8)) + .alias(target_col or column) + ) + + +def cap_numeric_category_frame( + df: pl.DataFrame, + *, + category: str, + cap_value: int, + value_cols: tuple[str, ...], + target_col: str | None = None, +) -> pl.DataFrame: + """Aggregate numeric categories at or above ``cap_value`` into one ``N+`` bucket.""" + output_col = target_col or category + if category not in df.columns: + return df + available_value_cols = [column for column in value_cols if column in df.columns] + if not available_value_cols: + return df.with_columns( + capped_numeric_category_expr( + category, + cap_value, + target_col=output_col, + ) + ) + return ( + df.with_columns( + capped_numeric_category_expr( + category, + cap_value, + target_col=output_col, + ) + ) + .group_by(output_col) + .agg([pl.col(column).sum().alias(column) for column in available_value_cols]) + .sort(numeric_like_sort_expr(output_col)) + ) + + +def cap_numeric_category_data( + data_list: list[tuple[str, pl.DataFrame]], + *, + category: str, + cap_value: int, + value_cols: tuple[str, ...], + target_col: str | None = None, +) -> list[tuple[str, pl.DataFrame]]: + """Apply numeric category capping across a run-indexed data list.""" + return [ + ( + label, + cap_numeric_category_frame( + df, + category=category, + cap_value=cap_value, + value_cols=value_cols, + target_col=target_col, + ), + ) + for label, df in nonempty(data_list) + ] + + +def capped_numeric_category_values( + data_list: list[tuple[str, pl.DataFrame]], + column: str, + *, + cap_value: int, + minimum: int | None = None, +) -> list[str]: + """Return observed numeric category labels after applying an ``N+`` cap.""" + values: set[int] = set() + saw_capped = False + for _, df in nonempty(data_list): + if column not in df.columns: + continue + for value in ( + df.select(pl.col(column).cast(pl.Int64, strict=False)) + .to_series() + .to_list() + ): + if value is None: + continue + if minimum is not None and value < minimum: + continue + if value >= cap_value: + saw_capped = True + else: + values.add(int(value)) + labels = [str(value) for value in sorted(values)] + if saw_capped: + labels.append(f"{cap_value}+") + return labels + + def complete_category_counts( data_list: list[tuple[str, pl.DataFrame]], *, - category_col: str, + category: str, category_values: list[str], value_cols: tuple[str, ...], extra_fill_values: dict[str, Any] | None = None, @@ -331,19 +466,19 @@ def complete_category_counts( """Ensure each run has one row per category value with zero-filled metric columns.""" if not category_values: return data_list - base = pl.DataFrame({category_col: category_values}, schema={category_col: pl.Utf8}) + base = pl.DataFrame({category: category_values}, schema={category: pl.Utf8}) fill_values = dict(extra_fill_values or {}) out: list[tuple[str, pl.DataFrame]] = [] for label, df in data_list: if df is None: completed = base else: - available_cols = [column for column in df.columns if column != category_col] + available_cols = [column for column in df.columns if column != category] completed = base.join( - df.with_columns(pl.col(category_col).cast(pl.Utf8)).select( - category_col, *available_cols + df.with_columns(pl.col(category).cast(pl.Utf8)).select( + category, *available_cols ), - on=category_col, + on=category, how="left", ) fill_exprs = [] @@ -359,3 +494,25 @@ def complete_category_counts( fill_exprs.append(pl.lit(fill_value).alias(column)) out.append((label, completed.with_columns(fill_exprs))) return out + + +def add_percent_of_total( + data_list: list[tuple[str, pl.DataFrame]], + *, + value_col: str, + percent_col: str, +) -> list[tuple[str, pl.DataFrame]]: + """Add a percent column using each run table's full value-column total.""" + out: list[tuple[str, pl.DataFrame]] = [] + for label, df in data_list: + if df is None or value_col not in df.columns: + out.append((label, df)) + continue + denominator = float(df[value_col].sum() or 0.0) + percent_expr = ( + (pl.col(value_col).cast(pl.Float64) / denominator * 100.0) + if denominator > 0 + else pl.lit(0.0) + ) + out.append((label, df.with_columns(percent_expr.alias(percent_col)))) + return out diff --git a/dashboard/helpers/comparison_helpers.py b/dashboard/helpers/comparison_helpers.py index 31683ef..7db242d 100644 --- a/dashboard/helpers/comparison_helpers.py +++ b/dashboard/helpers/comparison_helpers.py @@ -3,7 +3,7 @@ from __future__ import annotations import math -from typing import Any +from typing import Any, Mapping import polars as pl @@ -50,6 +50,95 @@ def percent_difference_string( return f"{pct_diff:.{precision}f}%" +def _finite_float(value: float | int | None) -> float | None: + if value is None: + return None + try: + numeric = float(value) + except (TypeError, ValueError): + return None + return numeric if math.isfinite(numeric) else None + + +def ab_percent_difference_string( + quantity_a: float | int | None, + quantity_b: float | int | None, + *, + precision: int = 2, +) -> str: + """Return formatted percent difference using the second value as denominator.""" + a_value = _finite_float(quantity_a) + b_value = _finite_float(quantity_b) + if a_value is None or b_value is None or b_value == 0.0: + return "" + pct_diff = ((a_value - b_value) / b_value) * 100.0 + return f"{pct_diff:.{precision}f}%" + + +def ab_difference_value( + quantity_a: float | int | None, + quantity_b: float | int | None, +) -> float | None: + """Return absolute difference for a single A/B comparison.""" + a_value = _finite_float(quantity_a) + b_value = _finite_float(quantity_b) + if a_value is None or b_value is None: + return None + return math.sqrt((a_value - b_value) ** 2) + + +def build_ab_comparison_row( + *, + keys: Mapping[str, Any], + quantity_a: float | int | None, + quantity_b: float | int | None, + quantity_a_column: str, + quantity_b_column: str, + precision: int = 2, +) -> dict[str, Any]: + """Build one long-form A/B comparison row with caller-supplied value labels.""" + a_value = _finite_float(quantity_a) + b_value = _finite_float(quantity_b) + return { + **dict(keys), + quantity_a_column: a_value, + quantity_b_column: b_value, + "Difference": ab_difference_value(a_value, b_value), + "% Difference": ab_percent_difference_string( + a_value, + b_value, + precision=precision, + ), + } + + +def build_ab_comparison_table( + rows: list[Mapping[str, Any]], + *, + key_columns: list[str], + quantity_a_column: str, + quantity_b_column: str, +) -> pl.DataFrame: + """Return rows with stable key/A/B/difference/% difference column ordering.""" + columns = [ + *key_columns, + quantity_a_column, + quantity_b_column, + "Difference", + "% Difference", + ] + if not rows: + schema = { + **{column: pl.Utf8 for column in key_columns}, + quantity_a_column: pl.Float64, + quantity_b_column: pl.Float64, + "Difference": pl.Float64, + "% Difference": pl.Utf8, + } + return pl.DataFrame(schema=schema).select(columns) + return pl.DataFrame(rows).select(columns) + + def build_base_run_percent_difference_table( *, run_labels: list[str], @@ -91,7 +180,7 @@ def build_base_run_percent_difference_table( def weighted_average_lookup( df: pl.DataFrame, *, - category_col: str, + category: str, average_col: str, weight_col: str, ) -> dict[str, float]: @@ -103,21 +192,21 @@ def weighted_average_lookup( """ if df.is_empty(): return {} - if category_col not in df.columns or average_col not in df.columns: + if category not in df.columns or average_col not in df.columns: return {} if weight_col not in df.columns: aggregated = ( - df.group_by(category_col) + df.group_by(category) .agg(pl.col(average_col).mean().alias(average_col)) - .select(category_col, average_col) + .select(category, average_col) ) return { - str(row[category_col]): float(row[average_col]) + str(row[category]): float(row[average_col]) for row in aggregated.to_dicts() - if row.get(category_col) is not None and row.get(average_col) is not None + if row.get(category) is not None and row.get(average_col) is not None } aggregated = ( - df.group_by(category_col) + df.group_by(category) .agg( pl.col(weight_col).sum().alias(weight_col), (pl.col(average_col) * pl.col(weight_col)).sum().alias("_weighted_value"), @@ -128,10 +217,10 @@ def weighted_average_lookup( .otherwise(None) .alias(average_col) ) - .select(category_col, average_col) + .select(category, average_col) ) return { - str(row[category_col]): float(row[average_col]) + str(row[category]): float(row[average_col]) for row in aggregated.to_dicts() - if row.get(category_col) is not None and row.get(average_col) is not None + if row.get(category) is not None and row.get(average_col) is not None } diff --git a/dashboard/helpers/distance_range.py b/dashboard/helpers/distance_range.py new file mode 100644 index 0000000..3d4c09a --- /dev/null +++ b/dashboard/helpers/distance_range.py @@ -0,0 +1,267 @@ +"""Live-only distance range controls for distance distribution charts.""" + +from __future__ import annotations + +from dataclasses import dataclass +from math import isfinite +import re +from typing import Any + +import panel as pn +import polars as pl + +from dashboard.rendering import control_row +from dashboard.helpers.category_helpers import nonempty + + +_DISTANCE_NUMBER_RE = re.compile(r"-?\d+(?:\.\d+)?") + + +def distance_bin_value(value: object) -> float | None: + """Return the numeric position for a distance-bin label such as ``40+``.""" + if value is None: + return None + if isinstance(value, int | float): + numeric = float(value) + return numeric if isfinite(numeric) else None + match = _DISTANCE_NUMBER_RE.search(str(value).strip()) + if match is None: + return None + numeric = float(match.group(0)) + return numeric if isfinite(numeric) else None + + +def with_distance_axis( + data_list: list[tuple[str, pl.DataFrame]], + *, + source_col: str = "distance_bin", + axis_col: str = "_distance_axis", +) -> list[tuple[str, pl.DataFrame]]: + """Add a numeric distance-axis column while preserving distance-bin labels.""" + out: list[tuple[str, pl.DataFrame]] = [] + for label, df in nonempty(data_list): + if source_col not in df.columns: + out.append((label, df)) + continue + axis_values = [distance_bin_value(value) for value in df[source_col].to_list()] + out.append((label, df.with_columns(pl.Series(axis_col, axis_values)))) + return out + + +def distance_axis_bounds( + data_list: list[tuple[str, pl.DataFrame]], + *, + source_col: str = "distance_bin", +) -> tuple[float, float] | None: + """Return finite min/max distance-axis bounds across chart-ready data.""" + values: list[float] = [] + for _, df in nonempty(data_list): + if source_col not in df.columns: + continue + values.extend( + numeric + for numeric in (distance_bin_value(value) for value in df[source_col].to_list()) + if numeric is not None + ) + if not values: + return None + lower = min(values) + upper = max(values) + if lower == upper: + upper = lower + 1.0 + return (float(lower), float(upper)) + + +def distance_axis_ticks( + data_list: list[tuple[str, pl.DataFrame]], + *, + source_col: str = "distance_bin", +) -> tuple[list[float], list[str]]: + """Return stable numeric tick positions and original distance-bin labels.""" + tick_labels: dict[float, str] = {} + for _, df in nonempty(data_list): + if source_col not in df.columns: + continue + for value in df[source_col].to_list(): + numeric = distance_bin_value(value) + if numeric is not None and numeric not in tick_labels: + tick_labels[numeric] = str(value) + tickvals = sorted(tick_labels) + return (tickvals, [tick_labels[value] for value in tickvals]) + + +def fixed_distance_axis_ticks( + *, + max_value: int = 40, + step: int = 2, + plus_label: bool = True, +) -> tuple[list[float], list[str]]: + """Return fixed whole-mile distance ticks, labeling the max as ``40+``.""" + tickvals = [float(value) for value in range(0, max_value + 1, step)] + ticktext = [str(int(value)) for value in tickvals] + if plus_label and ticktext: + ticktext[-1] = f"{int(tickvals[-1])}+" + return tickvals, ticktext + + +def capped_distance_max_options(max_value: int = 40) -> list[str]: + """Return max-range dropdown options ending with a capped ``40+`` label.""" + return [str(value) for value in range(1, max_value)] + [f"{max_value}+"] + + +def resolve_distance_range( + min_value: object, + max_value: object, +) -> tuple[float, float] | None: + """Return a valid finite distance range, or None.""" + lower = distance_bin_value(min_value) + upper = distance_bin_value(max_value) + if lower is None or upper is None: + return None + if not isfinite(lower) or not isfinite(upper) or lower >= upper: + return None + return (lower, upper) + + +def _range_widget_value(value: float, widget: pn.widgets.Widget) -> object: + """Return a range-bound value compatible with the target widget.""" + if isinstance(widget, pn.widgets.Select): + if abs(value - 40.0) < 1e-9: + return "40+" + if abs(value - round(value)) < 1e-9: + return str(int(round(value))) + return str(value) + return float(value) + + +def _ranges_equal( + left: tuple[float, float] | None, + right: tuple[float, float] | None, +) -> bool: + if left is None or right is None: + return False + return abs(left[0] - right[0]) < 1e-9 and abs(left[1] - right[1]) < 1e-9 + + +@dataclass +class DistanceRangeControls: + """A pair of min/max widgets plus reset behavior for one distance chart group.""" + + page: Any + prefix: str + min_widget: pn.widgets.FloatInput + max_widget: pn.widgets.Widget + reset_button: pn.widgets.Button + selector_ids: tuple[str, ...] + + @classmethod + def create( + cls, + page: Any, + prefix: str, + *, + min_label: str = "Distance Min", + max_label: str = "Distance Max", + reset_label: str = "Reset distance range", + step: float = 1.0, + max_options: list[object] | None = None, + ) -> "DistanceRangeControls": + """Create registered live controls, or disabled export-only clones.""" + min_widget = pn.widgets.FloatInput(name=min_label, step=step, value=0.0) + max_widget: pn.widgets.Widget + if max_options: + max_widget = pn.widgets.Select( + name=max_label, + options=list(max_options), + value=list(max_options)[-1], + ) + else: + max_widget = pn.widgets.FloatInput(name=max_label, step=step, value=1.0) + selector_ids: tuple[str, ...] = () + if page.state.export_mode: + min_widget.disabled = True + max_widget.disabled = True + else: + min_widget = page.selector( + f"{prefix}_min", + widget=min_widget, + label=min_label, + exportable=False, + ) + max_widget = page.selector( + f"{prefix}_max", + widget=max_widget, + label=max_label, + exportable=False, + ) + selector_ids = (f"{prefix}_min", f"{prefix}_max") + reset_button = pn.widgets.Button( + name=reset_label, + button_type="default", + width=175, + disabled=page.state.export_mode, + ) + controls = cls( + page=page, + prefix=prefix, + min_widget=min_widget, + max_widget=max_widget, + reset_button=reset_button, + selector_ids=selector_ids, + ) + if not page.state.export_mode: + reset_button.on_click(lambda event: controls.reset()) + return controls + + def row(self) -> pn.Row: + """Return a standard controls row for this range group.""" + return control_row(self.min_widget, self.max_widget, self.reset_button) + + def current_range(self) -> tuple[float, float] | None: + """Return the currently selected valid range, or None.""" + return resolve_distance_range(self.min_widget.value, self.max_widget.value) + + def sync( + self, + context_key: object, + bounds: tuple[float, float] | None, + ) -> None: + """Initialize or update controls for the current chart context.""" + if bounds is None: + self.min_widget.disabled = True + self.max_widget.disabled = True + self.reset_button.disabled = True + return + + if self.page.state.export_mode: + self.min_widget.value = float(bounds[0]) + self.max_widget.value = _range_widget_value(float(bounds[1]), self.max_widget) + self.min_widget.disabled = True + self.max_widget.disabled = True + self.reset_button.disabled = True + return + + self.min_widget.disabled = False + self.max_widget.disabled = False + self.reset_button.disabled = False + state_key = f"{self.prefix}_range_context" + auto_key = f"{self.prefix}_auto_range" + last_context = self.page._page_state.get(state_key) + last_auto_range = self.page._page_state.get(auto_key) + current_range = self.current_range() + should_reset = last_auto_range is None or _ranges_equal( + current_range, last_auto_range + ) + if should_reset: + self.min_widget.value = float(bounds[0]) + self.max_widget.value = _range_widget_value(float(bounds[1]), self.max_widget) + self.page._page_state[state_key] = context_key + self.page._page_state[auto_key] = tuple(bounds) + + def reset(self) -> None: + """Restore the controls to the last observed auto range.""" + auto_range = self.page._page_state.get(f"{self.prefix}_auto_range") + if not auto_range: + return + self.min_widget.value = float(auto_range[0]) + self.max_widget.value = _range_widget_value(float(auto_range[1]), self.max_widget) diff --git a/dashboard/helpers/geography_helpers.py b/dashboard/helpers/geography_helpers.py index 829461a..829b916 100644 --- a/dashboard/helpers/geography_helpers.py +++ b/dashboard/helpers/geography_helpers.py @@ -6,7 +6,11 @@ import polars as pl -from dashboard.helpers.category_helpers import first_nonempty_frame, nonempty +from dashboard.helpers.category_helpers import ( + first_nonempty_frame, + nonempty, + raw_display_options, +) if TYPE_CHECKING: from runtime.config import Config @@ -20,7 +24,11 @@ ] AGGREGATE_GEOGRAPHY_LEVEL = "all_geographies" ALL_WITHIN_LEVEL_VALUE = "All" +ALL_GEOGRAPHY_TYPES_VALUE = AGGREGATE_GEOGRAPHY_LEVEL +ALL_GEOGRAPHY_TYPES_LABEL = "All Geography Types" ALL_GEOGRAPHIES_LABEL = "All Geographies" +GEOGRAPHY_TYPE_SELECTOR_LABEL = "Geography Type" +GEOGRAPHY_NAME_SELECTOR_LABEL = "Geography Name" DEFAULT_GEO_LEVEL_COL = "geography_level" DEFAULT_GEO_COL = "geography" DEFAULT_GEO_TYPE_COL = "geography_type" @@ -29,7 +37,10 @@ def is_all_geographies(value: str | None) -> bool: """Return whether a selector value targets the cross-level aggregate geography.""" - return str(value) in {AGGREGATE_GEOGRAPHY_LEVEL, ALL_GEOGRAPHIES_LABEL} + return str(value) in { + AGGREGATE_GEOGRAPHY_LEVEL, + ALL_GEOGRAPHIES_LABEL, + } def is_all_within_level(value: str | None) -> bool: @@ -37,6 +48,140 @@ def is_all_within_level(value: str | None) -> bool: return str(value) in {ALL_WITHIN_LEVEL_VALUE, "All", "Total"} +def geography_type_label(value: str | None, *, config: Config) -> str: + """Return the display label for one raw geography type/level value.""" + value_str = str(value) + if value_str in {ALL_GEOGRAPHY_TYPES_VALUE, "Total"}: + return ALL_GEOGRAPHY_TYPES_LABEL + configured = config.label_value("geography", value_str) + if configured != value_str: + return configured + uppercase_labels = {"taz": "TAZ", "maz": "MAZ", "mpo": "MPO"} + if value_str.lower() in uppercase_labels: + return uppercase_labels[value_str.lower()] + return value_str.replace("_", " ").title() + + +def geography_name_label(value: str | None, *, config: Config) -> str: + """Return a display label for one geography id/name value.""" + value_str = str(value) + if is_all_geographies(value_str) or value_str in aggregate_geography_level_values(): + configured = config.label_value("geography", AGGREGATE_GEOGRAPHY_LEVEL) + return configured if configured != AGGREGATE_GEOGRAPHY_LEVEL else ALL_GEOGRAPHIES_LABEL + configured = config.label_value("geography", value_str) + if configured != value_str: + return configured + return value_str.replace("_", " ").title() + + +def with_display_geography_columns( + df: pl.DataFrame, + *, + config: Config, + geography_level_col: str = DEFAULT_GEO_LEVEL_COL, + geography_col: str = DEFAULT_GEO_COL, + geography_type_col: str = DEFAULT_GEO_TYPE_COL, + geography_id_col: str = DEFAULT_GEO_ID_COL, + type_display_col: str = GEOGRAPHY_TYPE_SELECTOR_LABEL, + name_display_col: str = GEOGRAPHY_NAME_SELECTOR_LABEL, +) -> pl.DataFrame: + """Add friendly geography type/name columns for dashboard tables.""" + level_col = ( + geography_level_col + if geography_level_col in df.columns + else geography_type_col + if geography_type_col in df.columns + else None + ) + name_col = ( + geography_col + if geography_col in df.columns + else geography_id_col + if geography_id_col in df.columns + else None + ) + exprs: list[pl.Expr] = [] + if level_col is not None: + exprs.append( + pl.col(level_col) + .cast(pl.Utf8) + .map_elements( + lambda value: ( + geography_name_label(value, config=config) + if is_all_geographies(str(value)) + else geography_type_label(value, config=config) + ), + return_dtype=pl.Utf8, + ) + .alias(type_display_col) + ) + if name_col is not None: + exprs.append( + pl.col(name_col) + .cast(pl.Utf8) + .map_elements( + lambda value: geography_name_label(value, config=config), + return_dtype=pl.Utf8, + ) + .alias(name_display_col) + ) + return df.with_columns(exprs) if exprs else df + + +def normalize_geography_level_value(value: str | None) -> str: + """Normalize accepted aggregate geography tokens to the canonical raw value.""" + value_str = str(value) + if value_str in {ALL_GEOGRAPHIES_LABEL, "All", "Total"}: + return AGGREGATE_GEOGRAPHY_LEVEL + return value_str + + +def pluralize_geography_label(label: str) -> str: + """Return a compact plural display label for a geography type label.""" + label = str(label) + lower = label.lower() + explicit = { + "county": "Counties", + "geography": "Geographies", + "all geography": "All Geographies", + } + if lower in explicit: + return explicit[lower] + if label.isupper(): + return f"{label}s" + if lower.endswith("y") and (len(lower) < 2 or lower[-2] not in "aeiou"): + return f"{label[:-1]}ies" + if lower.endswith(("s", "x", "z")) or lower.endswith(("ch", "sh")): + return f"{label}es" + return f"{label}s" + + +def all_within_geography_type_label( + geography_type: str | None, + *, + config: Config, +) -> str: + """Return the display label for all geography names within a selected type.""" + if geography_type in {None, ALL_GEOGRAPHY_TYPES_VALUE, "Total"}: + return config.label_value("geography", AGGREGATE_GEOGRAPHY_LEVEL) + if is_all_geographies(geography_type): + return config.label_value("geography", AGGREGATE_GEOGRAPHY_LEVEL) + return f"All {pluralize_geography_label(geography_type_label(geography_type, config=config))}" + + +def geography_name_selector_label( + geography_type: str | None, + *, + config: Config, +) -> str: + """Return the selector label for geography names under one geography type.""" + if geography_type in {None, ALL_GEOGRAPHY_TYPES_VALUE, "Total"}: + return GEOGRAPHY_NAME_SELECTOR_LABEL + if is_all_geographies(geography_type): + return GEOGRAPHY_NAME_SELECTOR_LABEL + return f"{geography_type_label(geography_type, config=config)} Name" + + def visible_geography_levels( values: list[str] | set[str] | tuple[str, ...], *, @@ -71,9 +216,16 @@ def detail_geography_levels( values: list[str] | set[str] | tuple[str, ...], *, config: Config, + include_disabled_maz: bool = False, ) -> list[str]: """Return selector geography levels, hiding only MAZ when disabled.""" - return ordered_visible_geography_levels(values, config=config) + if include_disabled_maz: + visible = [str(value).strip() for value in values if str(value).strip()] + ordered = [value for value in PREFERRED_TYPED_GEO_ORDER if value in visible] + ordered.extend(sorted(value for value in visible if value not in ordered)) + else: + ordered = ordered_visible_geography_levels(values, config=config) + return config.ordered_values("geography", ordered) def rename_present(df: pl.DataFrame, mapping: dict[str, str]) -> pl.DataFrame: @@ -152,7 +304,7 @@ def geography_level_option_set( .cast(pl.Utf8) .to_list() ) - available.update(values) + available.update(normalize_geography_level_value(value) for value in values) elif geography_type_col in df.columns: values = ( df.select(geography_type_col) @@ -162,13 +314,19 @@ def geography_level_option_set( .cast(pl.Utf8) .to_list() ) - available.update(values) + available.update(normalize_geography_level_value(value) for value in values) elif {origin_level_col, destination_level_col}.issubset(df.columns): origin_values = set( - df[origin_level_col].cast(pl.Utf8).drop_nulls().unique().to_list() + normalize_geography_level_value(value) + for value in df[origin_level_col].cast(pl.Utf8).drop_nulls().unique().to_list() ) destination_values = set( - df[destination_level_col].cast(pl.Utf8).drop_nulls().unique().to_list() + normalize_geography_level_value(value) + for value in df[destination_level_col] + .cast(pl.Utf8) + .drop_nulls() + .unique() + .to_list() ) available.update(origin_values & destination_values) return available @@ -189,10 +347,45 @@ def geography_level_options( if not available_sets: return [total_label] ordered = detail_geography_levels(set().union(*available_sets), config=config) - return [ - config.label_value("geography", value) if is_all_geographies(value) else value - for value in (ordered or [total_label]) + return [geography_type_label(value, config=config) for value in ordered] if ordered else [total_label] + + +def geography_type_options( + *summary_lists: list[tuple[str, pl.DataFrame]] | None, + config: Config, + include_all_types: bool = False, + include_disabled_maz: bool = False, + all_types_label: str = ALL_GEOGRAPHY_TYPES_LABEL, + fallback_raw: str = AGGREGATE_GEOGRAPHY_LEVEL, +) -> tuple[list[str], dict[str, str | None]]: + """Return geography type display options plus display-to-raw lookup.""" + available_sets = [ + geography_level_option_set(summary) + for summary in summary_lists + if summary is not None ] + available_sets = [available for available in available_sets if available] + if not available_sets: + raw_values = [fallback_raw] + else: + raw_values = detail_geography_levels( + set().union(*available_sets), + config=config, + include_disabled_maz=include_disabled_maz, + ) + raw_by_label: dict[str, str | None] = {} + if include_all_types: + raw_by_label[all_types_label] = ALL_GEOGRAPHY_TYPES_VALUE + for raw_value in raw_values: + if include_all_types and raw_value == ALL_GEOGRAPHY_TYPES_VALUE: + continue + raw_by_label[geography_type_label(raw_value, config=config)] = raw_value + return list(raw_by_label), raw_by_label + + +def aggregate_geography_level_values() -> set[str]: + """Return accepted raw values for aggregate geography levels.""" + return {AGGREGATE_GEOGRAPHY_LEVEL, "All", "Total", ALL_GEOGRAPHIES_LABEL} def geography_id_option_set( @@ -282,6 +475,39 @@ def geography_options_for_level( return [all_within_level_label] + ordered if ordered else [all_within_level_label] +def geography_name_options_for_type( + geography_type: str, + *summary_lists: list[tuple[str, pl.DataFrame]] | None, + config: Config, +) -> tuple[list[str], dict[str, str | None]]: + """Return geography name display options plus display-to-raw lookup.""" + if geography_type in {ALL_GEOGRAPHY_TYPES_VALUE, "Total"}: + display = config.label_value("geography", AGGREGATE_GEOGRAPHY_LEVEL) + return [display], {display: ALL_WITHIN_LEVEL_VALUE} + if is_all_geographies(geography_type): + display = config.label_value("geography", AGGREGATE_GEOGRAPHY_LEVEL) + return [display], {display: AGGREGATE_GEOGRAPHY_LEVEL} + + available_sets = [ + geography_id_option_set(summary, geography_type) + for summary in summary_lists + if summary is not None + ] + available_sets = [available for available in available_sets if available] + all_label = all_within_geography_type_label(geography_type, config=config) + if not available_sets: + return [all_label], {all_label: ALL_WITHIN_LEVEL_VALUE} + + raw_values = config.ordered_values("geography", sorted(set().union(*available_sets))) + return raw_display_options( + raw_values, + category_id="geography", + config=config, + total_raw=ALL_WITHIN_LEVEL_VALUE, + total_label=all_label, + ) + + def export_geography_options( geography_opts_by_level: dict[str, list[str]], *, @@ -300,6 +526,33 @@ def export_geography_options( return [all_within_level_label] + ordered if ordered else [all_within_level_label] +def export_geography_name_options( + geography_opts_by_level: dict[str, tuple[list[str], dict[str, str | None]]], + *, + config: Config, + all_within_level_label: str = ALL_WITHIN_LEVEL_VALUE, +) -> tuple[list[str], dict[str, str | None]]: + """Flatten per-level geography display options for export mode.""" + raw_values: set[str] = set() + for options, raw_by_label in geography_opts_by_level.values(): + for option in options: + raw_value = raw_by_label.get(str(option), str(option)) + if raw_value is None: + continue + raw_value_str = str(raw_value) + if raw_value_str == all_within_level_label or is_all_geographies(raw_value_str): + continue + raw_values.add(raw_value_str) + ordered = config.ordered_values("geography", sorted(raw_values)) + return raw_display_options( + ordered, + category_id="geography", + config=config, + total_raw=all_within_level_label, + total_label=all_within_level_label, + ) + + def filter_geography_level( data_list: list[tuple[str, pl.DataFrame]] | None, geo_level: str, @@ -312,8 +565,8 @@ def filter_geography_level( """Filter a run-indexed summary list to one geography level.""" if not data_list: return [] - if is_all_geographies(geo_level): - geo_level = AGGREGATE_GEOGRAPHY_LEVEL + aggregate_values = aggregate_geography_level_values() + match_aggregate = is_all_geographies(geo_level) out: list[tuple[str, pl.DataFrame]] = [] for label, df in nonempty(data_list): filtered = df @@ -321,19 +574,34 @@ def filter_geography_level( if geography_level_col in filtered.columns: filtered = filtered.with_columns( pl.col(geography_level_col).cast(pl.Utf8) - ).filter(pl.col(geography_level_col) == geo_level) + ).filter( + pl.col(geography_level_col).is_in(aggregate_values) + if match_aggregate + else pl.col(geography_level_col) == geo_level + ) elif geography_type_col in filtered.columns: filtered = filtered.with_columns( pl.col(geography_type_col).cast(pl.Utf8) - ).filter(pl.col(geography_type_col) == geo_level) + ).filter( + pl.col(geography_type_col).is_in(aggregate_values) + if match_aggregate + else pl.col(geography_type_col) == geo_level + ) elif {origin_level_col, destination_level_col}.issubset(filtered.columns): filtered = filtered.with_columns( pl.col(origin_level_col).cast(pl.Utf8), pl.col(destination_level_col).cast(pl.Utf8), - ).filter( - (pl.col(origin_level_col) == geo_level) - & (pl.col(destination_level_col) == geo_level) ) + if match_aggregate: + filtered = filtered.filter( + pl.col(origin_level_col).is_in(aggregate_values) + & pl.col(destination_level_col).is_in(aggregate_values) + ) + else: + filtered = filtered.filter( + (pl.col(origin_level_col) == geo_level) + & (pl.col(destination_level_col) == geo_level) + ) out.append((label, filtered)) return out @@ -350,20 +618,26 @@ def filter_geography( return [] if is_all_within_level(geography): return nonempty(data_list) - if is_all_geographies(geography): - geography = AGGREGATE_GEOGRAPHY_LEVEL + aggregate_values = aggregate_geography_level_values() + match_aggregate = is_all_geographies(geography) out: list[tuple[str, pl.DataFrame]] = [] for label, df in nonempty(data_list): filtered = df if geography_col in filtered.columns: filtered = filtered.with_columns(pl.col(geography_col).cast(pl.Utf8)).filter( - pl.col(geography_col) == geography + pl.col(geography_col).is_in(aggregate_values) + if match_aggregate + else pl.col(geography_col) == geography ) elif geography_id_col in filtered.columns: filtered = filtered.with_columns( pl.col(geography_id_col).cast(pl.Utf8) - ).filter(pl.col(geography_id_col) == geography) + ).filter( + pl.col(geography_id_col).is_in(aggregate_values) + if match_aggregate + else pl.col(geography_id_col) == geography + ) out.append((label, filtered)) return out @@ -381,24 +655,32 @@ def filter_origin_geography( return [] if is_all_within_level(geography): return nonempty(data_list) - if is_all_geographies(geography): - geography = AGGREGATE_GEOGRAPHY_LEVEL + aggregate_values = aggregate_geography_level_values() + match_aggregate = is_all_geographies(geography) out: list[tuple[str, pl.DataFrame]] = [] for label, df in nonempty(data_list): filtered = df if origin_id_col in filtered.columns: filtered = filtered.with_columns(pl.col(origin_id_col).cast(pl.Utf8)).filter( - pl.col(origin_id_col) == geography + pl.col(origin_id_col).is_in(aggregate_values) + if match_aggregate + else pl.col(origin_id_col) == geography ) elif geography_col in filtered.columns: filtered = filtered.with_columns(pl.col(geography_col).cast(pl.Utf8)).filter( - pl.col(geography_col) == geography + pl.col(geography_col).is_in(aggregate_values) + if match_aggregate + else pl.col(geography_col) == geography ) elif geography_id_col in filtered.columns: filtered = filtered.with_columns( pl.col(geography_id_col).cast(pl.Utf8) - ).filter(pl.col(geography_id_col) == geography) + ).filter( + pl.col(geography_id_col).is_in(aggregate_values) + if match_aggregate + else pl.col(geography_id_col) == geography + ) out.append((label, filtered)) return out diff --git a/dashboard/page_base.py b/dashboard/page_base.py index 976f5c2..116dcd7 100644 --- a/dashboard/page_base.py +++ b/dashboard/page_base.py @@ -1,825 +1,21 @@ -"""Base classes for persistent Panel dashboard pages.""" +"""Small public facade for dashboard page authors.""" -from __future__ import annotations - -from collections import defaultdict -from dataclasses import dataclass -from typing import Any, Callable, TYPE_CHECKING - -import panel as pn - -from activitysim_viz_logging import get_logger -from dashboard import DashboardState -from dashboard.components import data_unavailable_card -from dashboard.data_access import ( - DashboardDataSelection, - VisualizationDiagnostic, - VisualizationInputResult, - VisualizationRunAvailability, +from dashboard.page_declarations import ( + PAGE_SELECTOR_STYLESHEET, + RegisteredPageSection, + RegisteredPageSelector, + SectionContent, ) -from runtime.config import Config - -if TYPE_CHECKING: - from dashboard.page_definitions import DashboardPageDefinition, PreparedDataMode - -LOGGER = get_logger("dashboard.page") - -SectionContent = ( - pn.viewable.Viewable | list[pn.viewable.Viewable] | tuple[pn.viewable.Viewable, ...] -) - -PAGE_SELECTOR_STYLESHEET = """ -:host(.page-selector-widget) { - max-width: 300px; -} - -:host(.page-selector-widget) .bk-input-group { - width: auto; -} - -:host(.page-selector-widget) .bk-input-group-label, -:host(.page-selector-widget) label { - font-size: 15px; - font-weight: 600; - color: #1f2937; - margin-bottom: 6px; -} - -:host(.page-selector-widget) select, -:host(.page-selector-widget) input { - font-size: 13px; - font-weight: 500; -} -""" - - -@dataclass(frozen=True) -class RegisteredPageSelector: - selector_id: str - widget: pn.widgets.Widget - label: str - exportable: bool = True - - -@dataclass -class RegisteredPageSection: - section_id: str - container: pn.Column - selector_ids: tuple[str, ...] - export: bool - export_data_mode: "PreparedDataMode" - render: Callable[[], SectionContent] - dirty: bool = True - - -class DashboardPage: - """Persistent controller for one dashboard page. - - Pages own widget instances, page-local cached views, and the summary/prepared-run - lookups needed to refresh their visible Panel layout. - """ - - definition: DashboardPageDefinition | None = None - - def __init__(self, *args) -> None: - if len(args) == 2: - state, config = args - definition = self.definition - derived_name = ( - definition.title if definition is not None else type(self).__name__ - ) - elif len(args) == 3: - derived_name, state, config = args - else: - raise TypeError( - "DashboardPage expects (state, config) or legacy (name, state, config)." - ) - if not isinstance(state, DashboardState): - raise TypeError("DashboardPage requires a DashboardState instance.") - if not isinstance(config, Config): - raise TypeError("DashboardPage requires a Config instance.") - - name = str(derived_name) - self.name = name - self.state = state - self.config = config - self._page_state = state.get_page_state(name) - self.view: pn.viewable.Viewable | None = None - self._registered_selectors: dict[str, RegisteredPageSelector] = {} - self._registered_sections: dict[str, RegisteredPageSection] = {} - self._selector_ids_by_widget_id: dict[int, str] = {} - self._is_refreshing = False - self._queued_selector_ids: set[str] = set() - self._active_section_id: str | None = None - - if type(self).build_page is not DashboardPage.build_page: - self.view = self.build_page() - self._validate_registered_components() - - def refresh_if_needed(self) -> None: - """Refresh the page when its rendered global state is stale.""" - if self._page_state.get("last_rendered_state") != self.state.global_state_key(): - self.refresh(force=True) - - def refresh(self, force: bool = False) -> None: - """Refresh the page content.""" - current_state_key = self.state.global_state_key() - last_state_key = self._page_state.get("last_rendered_state") - global_state_changed = force or last_state_key != current_state_key - if not force and not global_state_changed and not self._dirty_sections(): - return - self._page_state["visualization_diagnostics"] = [] - if self._registered_sections: - if global_state_changed: - self.on_global_state_changed() - for section in self._registered_sections.values(): - section.dirty = True - self._refresh_registered_sections() - else: - self._active_section_id = None - self._refresh() - self._page_state["last_rendered_state"] = current_state_key - - def mark_stale(self) -> None: - """Mark the page stale so the next activation refreshes it.""" - self._page_state["last_rendered_state"] = None - for section in self._registered_sections.values(): - section.dirty = True - - def mark_section_stale(self, *section_ids: str) -> None: - """Mark one or more registered sections stale.""" - for section_id in section_ids: - section = self._registered_sections.get(section_id) - if section is None: - raise KeyError( - f"Unknown section id {section_id!r} on page {self.name!r}." - ) - section.dirty = True - - def build_page(self) -> pn.viewable.Viewable: - raise NotImplementedError - - def sync_controls(self) -> None: - """Update selector options/values before rendering dirty sections.""" - - def on_global_state_changed(self) -> None: - """Hook for page-local cache invalidation on global dashboard state changes.""" - - def selector( - self, - selector_id: str, - *, - widget: pn.widgets.Widget, - label: str, - exportable: bool = True, - ) -> pn.widgets.Widget: - """Register one page-local selector widget.""" - if selector_id in self._registered_selectors: - raise ValueError( - f"Dashboard page {self.name!r} declares duplicate selector id {selector_id!r}." - ) - if hasattr(widget, "name"): - widget.name = label - css_classes = list(getattr(widget, "css_classes", []) or []) - if "page-selector-widget" not in css_classes: - css_classes.append("page-selector-widget") - widget.css_classes = css_classes - stylesheets = list(getattr(widget, "stylesheets", []) or []) - if PAGE_SELECTOR_STYLESHEET not in stylesheets: - stylesheets.append(PAGE_SELECTOR_STYLESHEET) - widget.stylesheets = stylesheets - selector = RegisteredPageSelector( - selector_id=selector_id, - widget=widget, - label=label, - exportable=exportable, - ) - self._registered_selectors[selector_id] = selector - self._selector_ids_by_widget_id[id(widget)] = selector_id - widget.param.watch( - lambda event, sid=selector_id: self._handle_selector_change(sid), - "value", - ) - return widget - - def section( - self, - section_id: str, - *, - selectors: tuple[str, ...] = (), - export: bool = True, - export_data_mode: "PreparedDataMode" = "none", - render: Callable[[], SectionContent], - ) -> pn.Column: - """Register one stable page section.""" - if section_id in self._registered_sections: - raise ValueError( - f"Dashboard page {self.name!r} declares duplicate section id {section_id!r}." - ) - unknown_selectors = [ - selector_id - for selector_id in selectors - if selector_id not in self._registered_selectors - ] - if unknown_selectors: - raise ValueError( - f"Dashboard page {self.name!r} section {section_id!r} references unknown selectors: " - + ", ".join(repr(selector_id) for selector_id in unknown_selectors) - ) - container = self.new_section() - self._registered_sections[section_id] = RegisteredPageSection( - section_id=section_id, - container=container, - selector_ids=tuple(selectors), - export=export, - export_data_mode=export_data_mode, - render=render, - ) - return container - - def section_view(self, section_id: str) -> pn.Column: - section = self._registered_sections.get(section_id) - if section is None: - raise KeyError(f"Unknown section id {section_id!r} on page {self.name!r}.") - return section.container - - @property - def registered_selectors(self) -> tuple[RegisteredPageSelector, ...]: - return tuple(self._registered_selectors.values()) - - @property - def registered_sections(self) -> tuple[RegisteredPageSection, ...]: - return tuple(self._registered_sections.values()) - - def _dirty_sections(self) -> tuple[RegisteredPageSection, ...]: - return tuple( - section for section in self._registered_sections.values() if section.dirty - ) - - def _handle_selector_change(self, selector_id: str) -> None: - if self._is_refreshing: - self._queued_selector_ids.add(selector_id) - return - self._mark_sections_for_selectors({selector_id}) - self.refresh(force=False) - - def _mark_sections_for_selectors(self, selector_ids: set[str]) -> None: - for section in self._registered_sections.values(): - if selector_ids.intersection(section.selector_ids): - section.dirty = True - - def _refresh_registered_sections(self) -> None: - self._is_refreshing = True - try: - rerun_requested = False - for _ in range(2): - self._queued_selector_ids.clear() - self.sync_controls() - dirty_sections = list(self._dirty_sections()) - if not dirty_sections and not self._queued_selector_ids: - break - for section in dirty_sections: - self._render_section(section) - section.dirty = False - if not self._queued_selector_ids: - break - self._mark_sections_for_selectors(set(self._queued_selector_ids)) - rerun_requested = True - if rerun_requested: - self._queued_selector_ids.clear() - finally: - self._is_refreshing = False - - def _render_section(self, section: RegisteredPageSection) -> None: - previous_section_id = self._active_section_id - self._active_section_id = section.section_id - try: - rendered = section.render() - finally: - self._active_section_id = previous_section_id - if isinstance(rendered, pn.viewable.Viewable): - objects = [rendered] - else: - objects = list(rendered) - section.container.objects = objects - - def _validate_registered_components(self) -> None: - if self.view is None: - raise ValueError( - f"Dashboard page {self.name!r} build_page() returned no view." - ) - if not isinstance(self.view, pn.viewable.Viewable): - raise TypeError( - f"Dashboard page {self.name!r} build_page() must return a Panel viewable." - ) - - def _watch_widget(self, widget: pn.widgets.Widget) -> None: - """Legacy helper for pages that have not migrated to selector().""" - widget.param.watch(lambda event: self.refresh(force=True), "value") - - def new_section(self, *objects, **kwargs) -> pn.Column: - """Create a stable page section container that can be refreshed in place.""" - kwargs.setdefault("sizing_mode", "stretch_width") - return pn.Column(*objects, **kwargs) - - @property - def as_percent(self) -> bool: - """Return whether the current display mode should show percentages.""" - return self.state.value_mode == "Percent" - - @property - def weighting_key(self) -> str: - """Return the current weighting key used for summary-table lookup.""" - return self.state.weighting_key() - - @classmethod - def page_id(cls) -> str | None: - """Return the registered page id when one has been assigned.""" - return cls.definition.page_id if cls.definition is not None else None - - @classmethod - def page_title(cls) -> str | None: - """Return the registered page title when one has been assigned.""" - return cls.definition.title if cls.definition is not None else None - - @property - def required_summary_ids(self) -> tuple[str, ...]: - if self.definition is None: - return () - return self.definition.required_summary_ids - - @property - def prepared_data_mode(self) -> str: - if self.definition is None: - return "none" - return self.definition.prepared_data_mode - - def _warn_once(self, key: str, message: str) -> None: - warnings = self._page_state.setdefault("warnings_emitted", set()) - if key in warnings: - return - LOGGER.warning(message) - warnings.add(key) - - def data_not_available_card( - self, - *, - detail: str, - missing_items: list[str] | tuple[str, ...] | None = None, - title: str = "Data Not Available", - ) -> pn.Card: - effective_title = title - effective_detail = detail - if missing_items: - effective_detail = self._augment_missing_data_detail(detail, missing_items) - if title == "Data Not Available": - statuses = self._missing_item_statuses(missing_items) - if statuses and statuses == {"empty"}: - effective_title = "Data Empty" - elif statuses and statuses == {"schema_mismatch"}: - effective_title = "Schema Mismatch" - elif statuses and statuses == {"failed"}: - effective_title = "Data Failed" - return data_unavailable_card( - effective_title, - effective_detail, - missing_items=missing_items, - ) - - def no_runs_message(self) -> pn.pane.Markdown: - """Return the standard no-runs-loaded placeholder used across pages.""" - return pn.pane.Markdown("No runs loaded.") - - def summary_only_unavailable_card( - self, - *, - summary_ids: list[str] | tuple[str, ...] | None = None, - detail: str = "This page only renders from precomputed summary tables.", - title: str = "Data Not Available", - ) -> pn.Card: - """Return the standard summary-only unavailable card for this page.""" - missing_items = list(summary_ids or self.required_summary_ids) - return self.data_not_available_card( - detail=detail, - missing_items=missing_items, - title=title, - ) - - @property - def missing_data_display(self) -> str: - return self.config.missing_data_display - - @property - def visualization_diagnostics(self) -> list[VisualizationDiagnostic]: - return list(self._page_state.get("visualization_diagnostics", [])) - - def _record_visualization_diagnostic( - self, - result: VisualizationInputResult, - ) -> None: - render_state = "rendered" - if not result.has_usable_runs: - render_state = "skipped" - elif result.excluded_runs: - render_state = "partial" - diagnostics = self._page_state.setdefault("visualization_diagnostics", []) - diagnostics.append( - VisualizationDiagnostic( - visualization_id=result.visualization_id, - render_state=render_state, - input_kind=result.input_kind, - input_ids=result.input_ids, - usable_run_labels=tuple( - label - for label, _ in next(iter(result.usable_by_input.values()), []) - ), - excluded_runs=tuple(result.excluded_runs), - ) - ) - - def _combine_selections( - self, - visualization_id: str, - selections: dict[str, DashboardDataSelection], - ) -> VisualizationInputResult: - usable_labels_by_input = { - input_id: {label for label, _ in selection.usable_runs} - for input_id, selection in selections.items() - } - common_labels = ( - set.intersection(*usable_labels_by_input.values()) - if usable_labels_by_input - else set() - ) - usable_by_input = { - input_id: [ - (label, value) - for label, value in selection.usable_runs - if label in common_labels - ] - for input_id, selection in selections.items() - } - excluded_by_key: dict[tuple[str, str, str], VisualizationRunAvailability] = {} - for selection in selections.values(): - for issue in selection.excluded_runs: - excluded_by_key[(issue.label, issue.source_kind, issue.source_id)] = ( - issue - ) - for label, _ in selection.usable_runs: - if label in common_labels: - continue - excluded_by_key[(label, selection.source_kind, selection.source_id)] = ( - VisualizationRunAvailability( - label=label, - status="missing", - detail="required alongside another unavailable input for this visualization", - source_kind=selection.source_kind, - source_id=selection.source_id, - ) - ) - input_kinds = {selection.source_kind for selection in selections.values()} - input_kind = next(iter(input_kinds)) if len(input_kinds) == 1 else "mixed" - result = VisualizationInputResult( - visualization_id=visualization_id, - input_kind=input_kind, - usable_by_input=usable_by_input, - excluded_runs=list(excluded_by_key.values()), - input_ids=tuple(selections), - ) - self._record_visualization_diagnostic(result) - return result - - def resolve_summary_visualization( - self, - visualization_id: str, - *, - summary_requirements: dict[str, tuple[str, ...]], - weighting_key: str | None = None, - ) -> VisualizationInputResult: - selections = { - summary_name: self.state.inspect_summary_table( - summary_name, - weighting_key=weighting_key or self.weighting_key, - required_columns=required_columns, - ) - for summary_name, required_columns in summary_requirements.items() - } - return self._combine_selections(visualization_id, selections) - - def resolve_prepared_visualization( - self, - visualization_id: str, - *, - table_requirements: dict[str, tuple[str, ...]], - weighted: bool | None = None, - ) -> VisualizationInputResult: - selections = { - table_name: self.state.inspect_prepared_table( - table_name, - weighted=weighted, - required_columns=required_columns, - ) - for table_name, required_columns in table_requirements.items() - } - return self._combine_selections(visualization_id, selections) - - def unavailable_visualization( - self, - result: VisualizationInputResult, - *, - detail: str, - title: str = "Data Not Available", - ) -> pn.viewable.Viewable: - if self.missing_data_display == "blank": - return pn.Spacer(height=0) - missing_items = list(result.input_ids) - if result.excluded_runs: - detail = self._format_issue_detail(detail, result.excluded_runs) - return self.data_not_available_card( - detail=detail, - missing_items=missing_items, - title=title, - ) - - def get_summary(self, summary_name: str): - """Return one summary table per run for the current weighting mode.""" - return self.state.get_summary_table_set(summary_name, self.weighting_key) - - def has_summary(self, summary_name: str) -> bool: - return self.state.has_summary_table_set(summary_name, self.weighting_key) - - def require_summary(self, summary_name: str): - """Return one summary table per run, warning once when unavailable.""" - selection = self.state.inspect_summary_table( - summary_name, - weighting_key=self.weighting_key, - ) - self._page_state.setdefault("required_summary_selections", {})[summary_name] = ( - selection - ) - if not selection.has_usable_runs: - self._warn_once( - f"missing-summary:{summary_name}", - ( - f"Warning: dashboard page '{self.name}' requires summary " - f"'{summary_name}' for weighting mode '{self.weighting_key}', " - "but no usable data was available." - ), - ) - return None - return [(label, table) for label, table in selection.usable_runs] - - def inspect_summary( - self, - summary_name: str, - *, - required_columns: tuple[str, ...] = (), - ): - """Inspect one summary table and store its availability for page diagnostics.""" - selection = self.state.inspect_summary_table( - summary_name, - weighting_key=self.weighting_key, - required_columns=required_columns, - ) - self._page_state.setdefault("required_summary_selections", {})[summary_name] = ( - selection - ) - return selection - - def optional_summary( - self, - summary_name: str, - *, - required_columns: tuple[str, ...] = (), - ): - """Return usable rows for one summary or ``None`` when unavailable.""" - selection = self.inspect_summary( - summary_name, - required_columns=required_columns, - ) - if not selection.has_usable_runs: - return None - return [(label, table) for label, table in selection.usable_runs] - - def optional_summaries_dict( - self, - *summary_names: str, - required_columns_by_summary: dict[str, tuple[str, ...]] | None = None, - ) -> dict[str, Any]: - """Return optional summaries keyed by summary id without failing the whole page.""" - required_columns_by_summary = required_columns_by_summary or {} - return { - summary_name: self.optional_summary( - summary_name, - required_columns=required_columns_by_summary.get(summary_name, ()), - ) - for summary_name in summary_names - } - - def require_summaries(self, *summary_names: str) -> dict[str, Any] | None: - """Return multiple summary tables or ``None`` when any are missing.""" - selections = { - summary_name: self.state.inspect_summary_table( - summary_name, - weighting_key=self.weighting_key, - ) - for summary_name in summary_names - } - self._page_state["required_summary_selections"] = selections - missing = [ - summary_name - for summary_name in summary_names - if not selections[summary_name].has_usable_runs - ] - if missing: - for summary_name in missing: - self._warn_once( - f"missing-summary:{summary_name}", - ( - f"Warning: dashboard page '{self.name}' requires summary " - f"'{summary_name}' for weighting mode '{self.weighting_key}', " - "but no usable data was available." - ), - ) - return None - return { - summary_name: [ - (label, table) for label, table in selections[summary_name].usable_runs - ] - for summary_name in summary_names - } - - def _missing_item_statuses( - self, - missing_items: list[str] | tuple[str, ...], - ) -> set[str]: - selections = self._page_state.get("required_summary_selections", {}) - statuses: set[str] = set() - for item in missing_items: - selection = selections.get(item) - if selection is None: - continue - statuses.update(issue.status for issue in selection.excluded_runs) - return statuses - - def _augment_missing_data_detail( - self, - detail: str, - missing_items: list[str] | tuple[str, ...], - ) -> str: - selections = self._page_state.get("required_summary_selections", {}) - excluded_runs: list[VisualizationRunAvailability] = [] - for item in missing_items: - selection = selections.get(item) - if selection is None: - continue - excluded_runs.extend(selection.excluded_runs) - if not excluded_runs: - return detail - return self._format_issue_detail(detail, excluded_runs) - - def _format_issue_detail( - self, - detail: str, - issues: ( - list[VisualizationRunAvailability] - | tuple[VisualizationRunAvailability, ...] - ), - ) -> str: - lines = [detail, "", "Availability details:"] - by_source: dict[str, list[VisualizationRunAvailability]] = defaultdict(list) - for issue in issues: - by_source[issue.source_id].append(issue) - - status_order = { - "empty": 0, - "schema_mismatch": 1, - "unavailable": 2, - "failed": 3, - "missing": 4, - } - for source_id in sorted(by_source): - source_issues = by_source[source_id] - grouped: dict[tuple[str, str], list[str]] = defaultdict(list) - for issue in source_issues: - grouped[(issue.status, issue.detail)].append(issue.label) - for (status, issue_detail), labels in sorted( - grouped.items(), - key=lambda item: ( - status_order.get(item[0][0], 99), - item[0][0], - item[0][1], - ), - ): - label_list = ", ".join(f"`{label}`" for label in sorted(labels)) - lines.append( - f"- `{source_id}` is {status.replace('_', ' ')} for {label_list}: {issue_detail}" - ) - return "\n".join(lines) - - def get_prepared_runs(self, *, weighted: bool | None = None): - """Return prepared runs when this dashboard session has loaded them explicitly.""" - return self.state.get_prepared_runs_if_loaded(weighted=weighted) - - def require_prepared_runs(self, *, weighted: bool | None = None): - """Return prepared runs or warn once when this session does not have them.""" - prepared_runs = self.get_prepared_runs(weighted=weighted) - if prepared_runs is not None: - return prepared_runs - - availability = self.state.prepared_run_availability - reason = ( - "prepared run data was not requested for this dashboard session" - if availability == "not_requested" - else "prepared run data is unavailable" - ) - self._warn_once( - f"missing-prepared-runs:{availability}", - ( - f"Warning: dashboard page '{self.name}' requires prepared run data, " - f"but {reason}." - ), - ) - return None - - def get_filtered_view(self, view_name: str, *filters: Any, factory): - """Return a cached chart-ready filtered view for the current page state.""" - page_cache_id = self.page_id() or self.name - return self.state.get_or_create_cached( - "filtered_view", - page_cache_id, - self._active_section_id or "*", - self.weighting_key, - view_name, - *filters, - factory=factory, - ) - - def clear_filtered_view_cache(self) -> None: - """Clear cached filtered views for this page and weighting mode.""" - page_cache_id = self.page_id() or self.name - cache = self.state.get_cache("filtered_view") - prefix = (page_cache_id, self.weighting_key) - stale_keys = [ - key - for key in cache - if key[0] == page_cache_id and key[2] == self.weighting_key - ] - for key in stale_keys: - cache.pop(key, None) - - def _refresh(self) -> None: - raise NotImplementedError - - -class GroupedDashboardPage: - """Top-level navigation wrapper that renders child dashboard pages as tabs.""" - - def __init__( - self, - group_id: str, - title: str, - pages: list[DashboardPage], - default_child_page_id: str | None = None, - ) -> None: - if not pages: - raise ValueError("GroupedDashboardPage requires at least one child page.") - self._group_id = group_id - self.name = title - self.pages = pages - self.view = pn.Tabs( - *[(page.name, page.view) for page in pages], - dynamic=False, - ) - self._active_child = self._default_child_index(default_child_page_id) - self.view.active = self._active_child - self.view.param.watch(self._on_child_tab_change, "active") - - def _default_child_index(self, default_child_page_id: str | None) -> int: - if default_child_page_id is None: - return 0 - for index, page in enumerate(self.pages): - if page.page_id() == default_child_page_id: - return index - return 0 - - def _on_child_tab_change(self, event) -> None: - self._active_child = int(event.new) - self.refresh_if_needed() - - def page_id(self) -> str: - return self._group_id - - @property - def active_child(self) -> DashboardPage: - return self.pages[self._active_child] - - def refresh_if_needed(self) -> None: - self.active_child.refresh_if_needed() - - def refresh(self, force: bool = False) -> None: - self.active_child.refresh(force=force) - - def mark_stale(self) -> None: - for page in self.pages: - page.mark_stale() +from dashboard.page_features import PageFeature +from dashboard.page_lifecycle import DashboardPage +from dashboard.page_navigation import GroupedDashboardPage + +__all__ = [ + "DashboardPage", + "GroupedDashboardPage", + "PageFeature", + "PAGE_SELECTOR_STYLESHEET", + "RegisteredPageSection", + "RegisteredPageSelector", + "SectionContent", +] diff --git a/dashboard/page_declarations.py b/dashboard/page_declarations.py new file mode 100644 index 0000000..9f3ea84 --- /dev/null +++ b/dashboard/page_declarations.py @@ -0,0 +1,81 @@ +"""Author-facing selector and section declarations for dashboard pages.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Callable, Literal, TypeAlias + +import panel as pn + +from dashboard.page_definitions import PreparedDataMode + +SectionContent: TypeAlias = ( + pn.viewable.Viewable | list[pn.viewable.Viewable] | tuple[pn.viewable.Viewable, ...] +) +SelectorOptions: TypeAlias = list | tuple | dict +OptionProvider: TypeAlias = Callable[[], SelectorOptions] +DefaultPolicy: TypeAlias = ( + Literal["first", "last"] | Callable[[SelectorOptions], object] +) + +PAGE_SELECTOR_STYLESHEET = """ +:host(.page-selector-widget) { max-width: 300px; } +:host(.page-selector-widget) .bk-input-group { width: auto; } +:host(.page-selector-widget) .bk-input-group-label, +:host(.page-selector-widget) label { + font-size: 15px; + font-weight: 600; + color: #1f2937; + margin-bottom: 6px; +} +:host(.page-selector-widget) select, +:host(.page-selector-widget) input { font-size: 13px; font-weight: 500; } +""" + +UNSET = object() + + +@dataclass(frozen=True) +class RegisteredPageSelector: + selector_id: str + widget: pn.widgets.Widget + label: str + exportable: bool = True + options: OptionProvider | None = None + default: DefaultPolicy = "first" + + +@dataclass +class RegisteredPageSection: + section_id: str + container: pn.Column + selector_ids: tuple[str, ...] + export: bool + export_data_mode: PreparedDataMode + render: Callable[[], SectionContent] + dirty: bool = True + + +def option_values(options: SelectorOptions) -> list[object]: + """Return the selectable values represented by Panel options.""" + return list(options.values()) if isinstance(options, dict) else list(options) + + +def default_value(options: SelectorOptions, policy: DefaultPolicy) -> object: + """Resolve a selector's value after its previous value becomes stale.""" + values = option_values(options) + if callable(policy): + return policy(options) + if not values: + return None + return values[-1] if policy == "last" else values[0] + + +__all__ = [ + "DefaultPolicy", + "OptionProvider", + "RegisteredPageSection", + "RegisteredPageSelector", + "SectionContent", + "SelectorOptions", +] diff --git a/dashboard/page_definitions.py b/dashboard/page_definitions.py index 00300d4..c0a7897 100644 --- a/dashboard/page_definitions.py +++ b/dashboard/page_definitions.py @@ -3,81 +3,16 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Callable, Literal - -import panel as pn +from typing import TYPE_CHECKING, Literal, TypeVar from processor.models import PreparedTableName if TYPE_CHECKING: from dashboard.page_base import DashboardPage - from runtime.config import Config - -PreparedDataMode = Literal["none", "optional", "required"] -DashboardPageSelectionMode = Literal["default", "all", "explicit"] - - -@dataclass(frozen=True) -class PageSelectorDefinition: - """Describe one page-local widget that may participate in HTML export. - - Attributes: - selector_id: Stable config-facing selector name used under - ``visualizer.export_html.pages.Uses A & B.
" in rendered + assert "How <this> works" in rendered + assert "A < B" in rendered
+ assert "Join <records> by their shared key." in rendered
+ assert "Generic grouping text" not in rendered
+ assert "Only A & B." in rendered
+ assert "source_one" in rendered
+ assert "Summary Tables Used:" in rendered
+ assert "Prepared summaries used:" not in rendered
+ assert "One <record>" not in rendered
+ assert rendered.endswith("Still visible
"}] + + +def test_export_hides_adjacent_note_when_its_visualization_part_is_disabled() -> None: + visualization = pn.Column(pn.pane.Markdown("A plot or table")) + note = calculation_note("trip_skims.summary_table") + note._calculation_note_target_id = id(visualization) + page = pn.Column(visualization, note, pn.pane.Markdown("Still visible")) + + payload = serialize_viewable( + page, + disable_widgets=False, + hidden_view_ids={id(visualization)}, + ) + + assert payload["child_count"] == 1 + assert payload["children"] == [{"kind": "html", "html": "Still visible
"}] diff --git a/tests/test_config_refactor_phase1.py b/tests/test_config_refactor_phase1.py index acb2bb2..79778c8 100644 --- a/tests/test_config_refactor_phase1.py +++ b/tests/test_config_refactor_phase1.py @@ -44,6 +44,42 @@ def _write_config(tmp_path: Path, lines: list[str]) -> Config: return Config.from_yaml(config_path) +def test_summary_failure_policy_defaults_to_record_and_accepts_error( + tmp_path: Path, +) -> None: + default_config = _write_config(tmp_path / "default", []) + strict_config = _write_config( + tmp_path / "strict", + ["summarize:", " failure_policy: error"], + ) + + assert default_config.summary_failure_policy == "record" + assert strict_config.summary_failure_policy == "error" + + +def test_summary_failure_policy_rejects_unknown_values(tmp_path: Path) -> None: + with pytest.raises( + ValueError, + match="summarize.failure_policy must be either 'record' or 'error'", + ): + _write_config( + tmp_path, + ["summarize:", " failure_policy: keep-going"], + ) + + +def test_skimjoin_failure_policy_is_normalized_without_enabling_skimjoin( + tmp_path: Path, +) -> None: + config = _write_config( + tmp_path, + ["skimjoin:", " failure_policy: error"], + ) + + assert config.skimjoin.enabled is False + assert config.skimjoin.failure_policy == "error" + + def test_new_config_layout_normalizes_to_existing_runtime_fields(tmp_path: Path) -> None: skim_path = tmp_path / "override.omx" network_los_path = tmp_path / "network_los.yaml" @@ -177,340 +213,93 @@ def test_new_config_layout_normalizes_to_existing_runtime_fields(tmp_path: Path) assert config.segmentation.dashboard.visibility == "segments_only" assert config.skim_file == str(skim_path.name) assert config.skim_matrix == "SOV_TIME__EA" - assert config.skimjoin.enabled is True + assert config.skimjoin_step_enabled() is True assert config.skimjoin.config_path == str(skimjoin_config_path.resolve()) assert config.skimjoin.resolved_skim_files == (str(skim_path.resolve()),) assert config.skimjoin.resolved_network_los_file == str(network_los_path.resolve()) -def test_legacy_distance_skim_locations_still_normalize_to_prepare_distance_skim( - tmp_path: Path, - caplog: pytest.LogCaptureFixture, - capsys: pytest.CaptureFixture[str], -) -> None: - skim_path = tmp_path / "legacy.omx" - network_los_path = tmp_path / "network_los.yaml" - skimjoin_config_path = tmp_path / "skimjoin.yaml" - _write_omx(skim_path, matrix_name="SOV_DIST__MD") - _write_network_los(network_los_path) - skimjoin_config_path.write_text( - "\n".join( - [ - "project:", - f" skim_files: [{skim_path.name}]", - f" network_los_file: {network_los_path.name}", - "activitysim:", - " trip_mode_column: trip_mode", - " trip_id_column: trip_id", - " tour_id_column: tour_id", - " outbound_column: outbound", - "dimensions:", - " PERIOD:", - " source_columns:", - " trip_source_column: depart", - " outbound_tour_source_column: start", - " inbound_tour_source_column: first_inbound_trip_depart", - " values_from_network_los: true", - "modes:", - " SOV:", - " distance:", - " matrix: SOV_DIST__{PERIOD}", - ] - ), - encoding="utf-8", - ) - - config = _write_config( - tmp_path, - [ - 'name: "Legacy Distance Skim"', - "skimjoin:", - " defaults:", - f" config_path: {skimjoin_config_path.name}", - f" skim_files: [{skim_path.name}]", - f" network_los_file: {network_los_path.name}", - " distance_skim:", - f" file: {skim_path.name}", - " matrix: SOV_DIST__MD", - "runs: []", - ], - ) - - assert config.skim_file == str(skim_path.name) - assert config.skim_matrix == "SOV_DIST__MD" - captured = capsys.readouterr() - combined_output = caplog.text + captured.err + captured.out - assert "skimjoin.distance_skim" in combined_output - assert "prepare.distance_skim" in combined_output - - -def test_new_keys_take_precedence_over_legacy_equivalents_and_warn( - tmp_path: Path, - caplog: pytest.LogCaptureFixture, - capsys: pytest.CaptureFixture[str], -) -> None: - config = _write_config( - tmp_path, - [ - 'name: "Precedence Test"', - "root: new_root", - "log_level: error", - "processor:", - " root: old_root", - " summaries:", - " weighting_modes: [unweighted]", - "summaries:", - " root: older_root", - " weighting_modes: [weighted, unweighted]", - "visualizer:", - ' dashboard_title: "Legacy Dashboard"', - " log_level: info", - " run_colors: ['#aaaaaa']", - "dashboard:", - ' title: "New Dashboard"', - "display:", - " run_colors:", - ' - "#222222"', - "summarize:", - " weighting_modes: [weighted]", - "runs: []", - ], - ) - - assert config.summary_root == str((tmp_path / "new_root").resolve()) - assert config.log_level == "ERROR" - assert config.dashboard_title == "New Dashboard" - assert config.weighting_modes == ["weighted"] - assert config.run_colors == ["#222222"] - captured = capsys.readouterr() - combined_output = caplog.text + captured.err + captured.out - assert "processor.root" in combined_output - assert "summaries.root" in combined_output - assert "visualizer.log_level" in combined_output - assert "visualizer.dashboard_title" in combined_output - assert "processor.summaries.weighting_modes" in combined_output - assert "visualizer.run_colors" in combined_output - assert "Deprecated config keys were detected. Prefer the canonical schema:" in combined_output - - @pytest.mark.parametrize( ("lines", "message"), [ + (["summaries: {}"], "summaries: Use root and summarize"), + (["visualizer: {}"], "visualizer: Use dashboard and display"), + (["processor: {}"], "processor: Use root, prepare, and summarize"), + (["dashboard:", " pages: []"], "dashboard.pages: Use dashboard.live.pages"), + (["segment:", " enabled: true"], "segment.enabled: Use pipeline.steps"), ( - [ - 'name: "Invalid Pipeline"', - "pipeline:", - " steps: [prepare, prepare]", - "runs: []", - ], - "duplicate step", - ), - ( - [ - 'name: "Invalid Pipeline"', - "pipeline:", - " steps: [prepare, publish]", - "runs: []", - ], - "unsupported step", - ), - ( - [ - 'name: "Invalid Pipeline"', - "pipeline:", - " steps: [skimjoin, summarize]", - "runs: []", - ], - "without 'prepare'", - ), - ( - [ - 'name: "Invalid Pipeline"', - "pipeline:", - " steps: [segment, dashboard]", - "runs: []", - ], - "without 'summarize'", - ), - ( - [ - 'name: "Invalid Pipeline"', - "pipeline:", - " steps: [dashboard, summarize]", - "runs: []", - ], - "place 'dashboard' last", - ), - ( - [ - 'name: "Invalid Pipeline"', - "pipeline:", - " steps: [summarize]", - " dashboard_mode: deploy", - "runs: []", - ], - "dashboard_mode", - ), - ( - [ - 'name: "Invalid Pipeline"', - "pipeline:", - " steps: [summarize]", - " overwrite: maybe", - "runs: []", - ], - "pipeline.overwrite", + ["skimjoin:", " config_path: skimjoin.yaml"], + "skimjoin.config_path: Use skimjoin.defaults.config_path", ), ], ) -def test_pipeline_validation_rejects_invalid_configurations( +def test_removed_config_keys_name_the_canonical_replacement( tmp_path: Path, lines: list[str], message: str, ) -> None: with pytest.raises(ValueError, match=message): - _write_config(tmp_path, lines) - + _write_config(tmp_path, [*lines, "runs: []"]) -def test_equivalent_old_and_new_configs_produce_matching_runtime_values_and_digests( - tmp_path: Path, -) -> None: - shared_root = str((tmp_path / "shared_cache").resolve()).replace("\\", "/") - old_config = _write_config( - tmp_path / "old", - [ - 'name: "Equivalent Config"', - "summaries:", - f" root: {shared_root}", - " weighting_modes: [weighted]", - "visualizer:", - ' dashboard_title: "Equivalent Dashboard"', - " log_level: warning", - " dashboard_pages:", - " - overview", - " run_colors:", - ' - "#333333"', - "dashboard_labels:", - " mode:", - " mapping:", - " WALK: Walk", - "runs: []", - ], - ) - new_config = _write_config( - tmp_path / "new", - [ - 'name: "Equivalent Config"', - f"root: {shared_root}", - "log_level: warning", - "dashboard:", - ' title: "Equivalent Dashboard"', - " live:", - " pages:", - " - overview", - "display:", - " labels:", - " mode:", - " mapping:", - " WALK: Walk", - " run_colors:", - ' - "#333333"', - "summarize:", - " weighting_modes: [weighted]", - "runs: []", - ], - ) - assert old_config.summary_root == new_config.summary_root - assert old_config.log_level == new_config.log_level - assert old_config.dashboard_title == new_config.dashboard_title - assert [entry.page_id for entry in old_config.dashboard_pages or []] == [ - entry.page_id for entry in new_config.dashboard_pages or [] - ] - assert old_config.run_colors == new_config.run_colors - assert old_config.weighting_modes == new_config.weighting_modes - assert old_config.prepare_config_digest == new_config.prepare_config_digest - assert old_config.summary_config_digest == new_config.summary_config_digest - assert ( - old_config.presentation_config_digest - == new_config.presentation_config_digest - ) +def test_unknown_top_level_config_key_is_rejected(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="Unknown top-level config keys: 'dashbaord'"): + _write_config(tmp_path, ["dashbaord: {}", "runs: []"]) -def test_dashboard_live_pages_take_precedence_over_dashboard_pages_and_warn( - tmp_path: Path, - caplog: pytest.LogCaptureFixture, - capsys: pytest.CaptureFixture[str], -) -> None: +def test_dashboard_host_placeholder_is_validated_and_ignored(tmp_path: Path) -> None: config = _write_config( tmp_path, [ - 'name: "Dashboard Live Pages Precedence"', + "pipeline:", + " steps: [dashboard]", + " dashboard_mode: live", "dashboard:", - " pages:", - " - overview", - " live:", - " pages:", - " - trip_mode", + " host:", + " account: example-account", + " app_id: 12345", + " title: Example Dashboard", + " verify: true", "runs: []", ], ) - assert [entry.page_id for entry in config.dashboard_pages or []] == ["trip_mode"] - captured = capsys.readouterr() - combined_output = caplog.text + captured.err + captured.out - assert "dashboard.pages" in combined_output - assert "dashboard.live.pages" in combined_output - + assert config.pipeline.dashboard_mode == "live" + assert not hasattr(config, "host") -def test_summarize_grouping_flags_take_precedence_over_top_level_and_warn( - tmp_path: Path, - caplog: pytest.LogCaptureFixture, - capsys: pytest.CaptureFixture[str], -) -> None: - config = _write_config( - tmp_path, - [ - 'name: "Summarize Grouping Precedence"', - "group_joint_tour_purposes: false", - "group_atwork_tour_purposes: false", - "group_school_tour_purposes: false", - "summarize:", - " group_joint_tour_purposes: true", - " group_atwork_tour_purposes: true", - " group_school_tour_purposes: true", - "runs: []", - ], - ) - assert config.group_joint_tour_purposes is True - assert config.group_atwork_tour_purposes is True - assert config.group_school_tour_purposes is True - captured = capsys.readouterr() - combined_output = caplog.text + captured.err + captured.out - assert "group_joint_tour_purposes" in combined_output - assert "summarize.group_joint_tour_purposes" in combined_output +def test_dashboard_host_placeholder_rejects_unknown_fields(tmp_path: Path) -> None: + with pytest.raises( + ValueError, + match="Unknown dashboard.host config keys: 'acount'", + ): + _write_config( + tmp_path, + [ + "dashboard:", + " host:", + " acount: typo", + "runs: []", + ], + ) -def test_summarize_pnr_tour_modes_take_precedence_over_legacy_modes_and_warn( +@pytest.mark.parametrize( + ("lines", "message"), + [ + (["pipeline:", " steps: [prepare, prepare]"], "duplicate step"), + (["pipeline:", " steps: [prepare, publish]"], "unsupported step"), + (["pipeline:", " steps: [skimjoin, summarize]"], "without 'prepare'"), + (["pipeline:", " steps: [segment, dashboard]"], "without 'summarize'"), + (["pipeline:", " steps: [dashboard, summarize]"], "place 'dashboard' last"), + (["pipeline:", " dashboard_mode: deploy"], "dashboard_mode"), + (["pipeline:", " overwrite: maybe"], "pipeline.overwrite"), + ], +) +def test_pipeline_validation_rejects_invalid_configurations( tmp_path: Path, - caplog: pytest.LogCaptureFixture, - capsys: pytest.CaptureFixture[str], + lines: list[str], + message: str, ) -> None: - config = _write_config( - tmp_path, - [ - 'name: "PNR Mode Precedence"', - "modes:", - " pnr_tour_modes: [PNR_LOCAL]", - "summarize:", - " pnr_tour_modes: [PNR_PREMIUM]", - "runs: []", - ], - ) - - assert config.pnr_tour_modes == ["PNR_PREMIUM"] - captured = capsys.readouterr() - combined_output = caplog.text + captured.err + captured.out - assert "modes.pnr_tour_modes" in combined_output - assert "summarize.pnr_tour_modes" in combined_output + with pytest.raises(ValueError, match=message): + _write_config(tmp_path, [*lines, "runs: []"]) diff --git a/tests/test_dashboard_helpers_phase1.py b/tests/test_dashboard_helpers_phase1.py index c48b1ab..fa9da15 100644 --- a/tests/test_dashboard_helpers_phase1.py +++ b/tests/test_dashboard_helpers_phase1.py @@ -5,11 +5,16 @@ import panel as pn import polars as pl +import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[1])) sys.path.insert(0, str(Path(__file__).resolve().parent)) -from dashboard.components import bar_chart +from dashboard.rendering import ( + column_titles, + data_table, +) +from dashboard.data_access import RunTables from dashboard.helpers.category_helpers import ( column_value_intersection, common_column_options, @@ -18,13 +23,20 @@ numeric_like_sort_expr, ) from dashboard.helpers.comparison_helpers import ( + build_ab_comparison_row, + build_ab_comparison_table, build_base_run_percent_difference_table, format_percent_error_table, weighted_average_lookup, ) from dashboard.helpers.geography_helpers import ( + all_within_geography_type_label, + export_geography_name_options, geography_level_options, + geography_name_options_for_type, + geography_name_selector_label, geography_options_for_level, + geography_type_options, normalize_geography_columns, export_geography_options, filter_geography_level, @@ -43,10 +55,99 @@ timebin_label, ) from dashboard.page_base import DashboardPage +from dashboard.pages.trip_summaries.parking_location import parking_scatter_data from dashboard.state import DashboardState +from processor.models import RunData from test_export_html import _full_summary_run, _write_config +def test_run_table_view_filters_transforms_and_joins_by_run_label() -> None: + counts = RunTables.from_runs( + [ + ("Base", pl.DataFrame({"direction": ["outbound", "inbound"], "n": [2, 3]})), + ("Build", pl.DataFrame({"direction": ["outbound"], "n": [5]})), + ] + ) + totals = RunTables.from_runs( + [ + ("Base", pl.DataFrame({"direction": ["outbound"], "total": [10]})), + ("Build", pl.DataFrame({"direction": ["outbound"], "total": [20]})), + ] + ) + + result = ( + counts.where(direction="outbound") + .join(totals, on="direction") + .with_columns((pl.col("n") / pl.col("total") * 100).alias("pct")) + .select("direction", "pct") + ) + + assert result.values("direction") == ["outbound"] + assert [frame["pct"][0] for _, frame in result] == [20.0, 25.0] + + empty_build = counts.where(direction="inbound") + assert [label for label, _ in empty_build] == ["Base", "Build"] + assert empty_build[1][1].is_empty() + + complete = RunTables.from_runs( + [ + ("Base", pl.DataFrame({"id": [1], "value": [2]})), + ("Build", pl.DataFrame({"id": [2]})), + ] + ).requiring("id", "value") + assert [label for label, _ in complete] == ["Base"] + assert [label for label, _ in empty_build.drop_empty()] == ["Base"] + + outer = RunTables.from_runs( + [("Base", pl.DataFrame({"id": [1], "left": [10]}))] + ).join( + RunTables.from_runs( + [("Base", pl.DataFrame({"id": [2], "right": [20]}))] + ), + on="id", + how="full", + coalesce=True, + ) + assert outer[0][1].sort("id")["id"].to_list() == [1, 2] + + +def test_parking_query_joins_summary_and_prepared_tables_by_run() -> None: + empty = pl.DataFrame() + prepared = RunData( + label="Base", + run_dir="base", + skim_file=None, + hh=empty, + per=empty, + tours=empty, + trips=empty, + joint_participants=empty, + land_use=pl.DataFrame({"MAZ": [1, 2], "PRKSPACES": [10, 20]}), + skim_matrix=None, + ) + summaries = [ + ( + "Base", + pl.DataFrame( + { + "geography_type": ["maz", "maz"], + "geography_id": ["1", "3"], + "trip_count": [4, 6], + } + ), + ) + ] + + result = parking_scatter_data(summaries, [("Base", prepared.land_use)]) + + assert [label for label, _ in result] == ["Base"] + assert result[0][1].to_dict(as_series=False) == { + "geography_id": ["1", "2", "3"], + "parking_capacity": [10.0, 20.0, 0.0], + "trip_count": [4.0, 0.0, 6.0], + } + + def test_category_helpers_support_intersection_normalization_and_numeric_sort( tmp_path: Path, ) -> None: @@ -68,7 +169,7 @@ def test_category_helpers_support_intersection_normalization_and_numeric_sort( ) completed = complete_category_counts( [("Base", pl.DataFrame({"bin": ["10", "40+"], "count": [2, 1]}))], - category_col="bin", + category="bin", category_values=["2", "10", "40+"], value_cols=("count",), ) @@ -92,14 +193,35 @@ def test_geography_helpers_normalize_and_build_options(tmp_path: Path) -> None: config = _write_config( tmp_path, geography_lines=["enabled: true", "landuse_col: COUNTY"], + extra_lines=[ + "display:", + " labels:", + " geography:", + " mapping:", + " all_geographies: All Geographies", + " home_county: County", + " district: District", + ], ) summary = [ ( "Base", pl.DataFrame( { - "geography_type": ["all_geographies", "district", "district"], - "geography_id": ["all_geographies", "Urban", "Suburban"], + "geography_type": [ + "all_geographies", + "district", + "district", + "home_county", + "home_county", + ], + "geography_id": [ + "all_geographies", + "Urban", + "Suburban", + "Wake", + "Durham", + ], } ), ) @@ -118,17 +240,48 @@ def test_geography_helpers_normalize_and_build_options(tmp_path: Path) -> None: normalized = normalize_geography_columns(summary[0][1]) geo_levels = geography_level_options(summary, flow_summary, config=config) + geo_type_options, geo_type_raw_by_label = geography_type_options( + summary, + flow_summary, + config=config, + include_all_types=True, + ) + county_options, county_raw_by_label = geography_name_options_for_type( + "home_county", + summary, + config=config, + ) district_options = geography_options_for_level("district", summary, config=config) flattened = export_geography_options( {"district": district_options, "all_geographies": ["All Geographies"]}, config=config, ) + flattened_display, flattened_raw_by_label = export_geography_name_options( + { + "home_county": (county_options, county_raw_by_label), + }, + config=config, + ) filtered = filter_geography_level(summary, "district") assert {"geography_level", "geography"}.issubset(normalized.columns) - assert geo_levels == ["All Geographies", "district"] + assert geo_levels == ["All Geography Types", "County", "District"] + assert geo_type_options == [ + "All Geography Types", + "County", + "District", + ] + assert geo_type_raw_by_label["All Geography Types"] == "all_geographies" + assert geo_type_raw_by_label["County"] == "home_county" + assert all_within_geography_type_label("home_county", config=config) == "All Counties" + assert geography_name_selector_label("home_county", config=config) == "County Name" + assert county_options == ["All Counties", "Durham", "Wake"] + assert county_raw_by_label["All Counties"] == "All" + assert county_raw_by_label["Wake"] == "Wake" assert district_options == ["All", "Suburban", "Urban"] assert flattened == ["All", "Suburban", "Urban"] + assert flattened_display == ["All", "Durham", "Wake"] + assert flattened_raw_by_label["Wake"] == "Wake" assert filtered[0][1]["geography_id"].to_list() == ["Urban", "Suburban"] @@ -214,7 +367,7 @@ def test_comparison_helpers_format_and_build_comparison_tables() -> None: "tour_count": [1.0, 3.0, 2.0], } ), - category_col="purpose", + category="purpose", average_col="average_distance", weight_col="tour_count", ) @@ -243,30 +396,104 @@ def test_comparison_helpers_format_and_build_comparison_tables() -> None: assert renamed_table.columns == ["Metric", "Reference (Base Run)", "Build"] -def test_bar_chart_omits_pct_hover_lines() -> None: - chart = bar_chart( +def test_ab_comparison_helper_formats_difference_columns() -> None: + table = build_ab_comparison_table( + [ + build_ab_comparison_row( + keys={"Metric": "Tours"}, + quantity_a=110.0, + quantity_b=100.0, + quantity_a_column="Build Value", + quantity_b_column="Base Value", + ), + build_ab_comparison_row( + keys={"Metric": "Trips"}, + quantity_a=45.0, + quantity_b=0.0, + quantity_a_column="Build Value", + quantity_b_column="Base Value", + ), + build_ab_comparison_row( + keys={"Metric": "Distance"}, + quantity_a=None, + quantity_b=10.0, + quantity_a_column="Build Value", + quantity_b_column="Base Value", + ), + ], + key_columns=["Metric"], + quantity_a_column="Build Value", + quantity_b_column="Base Value", + ) + + assert table.to_dicts() == [ + { + "Metric": "Tours", + "Build Value": 110.0, + "Base Value": 100.0, + "Difference": 10.0, + "% Difference": "10.00%", + }, + { + "Metric": "Trips", + "Build Value": 45.0, + "Base Value": 0.0, + "Difference": 45.0, + "% Difference": "", + }, + { + "Metric": "Distance", + "Build Value": None, + "Base Value": 10.0, + "Difference": None, + "% Difference": "", + }, + ] + + +def test_data_table_drops_index_columns_and_hides_pandas_index() -> None: + table = data_table( [ ( - "Base", + "Run", pl.DataFrame( { - "mode": ["Walk"], - "trip_count": [5.0], - "pct": [100.0], + "index": [0], + "__index_level_0__": [99], + "metric": ["Tours"], + "value": [10.0], } ), ) - ], - x_col="mode", - y_col="trip_count", - pct_col="pct", - xaxis_categoryarray=["Walk", "Bike"], + ] ) + tabulator = table.objects[0] + + assert tabulator.show_index is False + assert tabulator.value.columns.tolist() == ["metric", "value"] + assert tabulator.titles == {"metric": "Metric", "value": "Value"} - hover = list(chart.object.data[0].customdata) - assert "Pct:" not in hover[0] - assert "Pct:" not in hover[1] +def test_column_titles_for_display_humanizes_machine_column_names() -> None: + titles = column_titles( + [ + "id", + "facility_type", + "From_Node", + "auto_vmt", + "pm_vol", + "nonmandatory_tour_purpose", + ] + ) + + assert titles == { + "id": "ID", + "facility_type": "Facility Type", + "From_Node": "From Node", + "auto_vmt": "Auto VMT", + "pm_vol": "PM Volume", + "nonmandatory_tour_purpose": "Non-Mandatory Tour Purpose", + } def test_dashboard_page_phase1_convenience_helpers(tmp_path: Path) -> None: @@ -278,11 +505,11 @@ def test_dashboard_page_phase1_convenience_helpers(tmp_path: Path) -> None: class ProbePage(DashboardPage): def __init__(self) -> None: - super().__init__("Probe", state, config) + super().__init__(state, config) self.view = pn.Column() def _refresh(self) -> None: - self.summary_dict = self.optional_summaries_dict( + self.summary_dict = self.data.summaries( "population_totals", "missing_summary", ) @@ -296,5 +523,5 @@ def _refresh(self) -> None: assert isinstance(no_runs, pn.pane.Markdown) assert no_runs.object == "No runs loaded." assert unavailable.title == "Data Not Available" - assert page.summary_dict["population_totals"] is not None - assert page.summary_dict["missing_summary"] is None + assert page.summary_dict["population_totals"] + assert not page.summary_dict["missing_summary"] diff --git a/tests/test_dashboard_live.py b/tests/test_dashboard_live.py index 5a182a7..583d098 100644 --- a/tests/test_dashboard_live.py +++ b/tests/test_dashboard_live.py @@ -1,7 +1,6 @@ from __future__ import annotations import importlib -import logging import pkgutil from pathlib import Path import sys @@ -22,109 +21,2032 @@ from test_export_html import _full_summary_run, _write_config from dashboard.app import build_dashboard from dashboard.data_access import DashboardPreparedRunProvider +from dashboard.export.selector_states import resolve_export_section_states from dashboard.page_base import DashboardPage from dashboard.page_base import PAGE_SELECTOR_STYLESHEET from dashboard.page_definitions import DashboardPageDefinition from dashboard.pages.skim_summaries.trip_skims import TripSkimsPage from dashboard.pages.skim_summaries.tour_skims import TourSkimsPage +from dashboard.pages.long_term_choices.mandatory_location_choice import ( + MandatoryLocationChoicePage, +) +from dashboard.pages.tour_summaries.tour_distance import TourDistancePage from dashboard.pages.trip_summaries.trip_mode import TripModePage +from dashboard.pages.validation.regional import ( + RegionalValidationPage, + flow_comparison_data, + flow_heatmap, + normalize_flow_matrix, +) +from dashboard.pages.validation.traffic import ( + demo_count_scatter_data, + demo_count_scatter_data_from_sources, + demo_facility_comparison_table, + demo_count_fit_line_data, + demo_link_aggregate_data, + demo_volume_comparison_table, +) +from dashboard.pages.validation.vmt import ( + NON_MOTORIZED_VMT_SUMMARY_ID, + PERSONAL_AUTO_VMT_SUMMARY_ID, + VMTValidationPage, + demo_commercial_filter_options, + demo_commercial_vehicle_chart_data, + external_travel_chart_data, + external_travel_filter_options, + personal_auto_vmt_chart_data, + wide_tod_chart_data, +) import dashboard.pages as dashboard_pages_package from dashboard.page_registry import ( + _validate_page_definition, all_page_definitions, data_requirements_for_pages, default_page_definitions, enabled_prepared_data_mode, page_definition_by_id, - resolve_page_definitions, + resolve_live_page_definitions, ) from dashboard.state import DashboardState from processor.models import RunData -from processor.summarize.cache import SUMMARY_SPEC_BY_ID +from processor.summarize.cache_types import create_summary_run +from processor.summarize.catalog import SUMMARY_BY_ID + + +def _collect_tabulators(viewable) -> list[pn.widgets.Tabulator]: + tables: list[pn.widgets.Tabulator] = [] + if isinstance(viewable, pn.widgets.Tabulator): + return [viewable] + for child in getattr(viewable, "objects", []): + tables.extend(_collect_tabulators(child)) + return tables + + +def _raw_trip_run() -> RunData: + return RunData( + label="Base", + run_dir="C:/runs/base", + skim_file=None, + hh=pl.DataFrame({"household_id": [1], "finalweight": [2.0]}), + per=pl.DataFrame({"person_id": [1], "household_id": [1], "finalweight": [3.0]}), + tours=pl.DataFrame({"tour_id": [10], "finalweight": [4.0]}), + trips=pl.DataFrame( + { + "trip_id": [100, 101, 102], + "tour_id": [10, 10, 10], + "trip_mode": ["DRIVEALONE", "WALK", "WALK"], + "finalweight": [5.0, 2.0, 1.0], + } + ), + joint_participants=pl.DataFrame(), + land_use=pl.DataFrame(), + skim_matrix=None, + skim_zone_map=None, + ) + + +def test_page_registry_exposes_expected_default_definitions() -> None: + definitions = default_page_definitions() + + assert [ + definition.page_id for definition in definitions + ] == EXPECTED_DEFAULT_LEAF_PAGE_IDS + assert [ + definition.title for definition in definitions + ] == EXPECTED_DEFAULT_LEAF_PAGE_TITLES + assert page_definition_by_id("daily_activity_pattern") is not None + assert ( + page_definition_by_id("daily_activity_pattern").title + == "Daily Activity Pattern" + ) + assert page_definition_by_id("daily_activity_pattern").group_id == "daily_travel" + assert not hasattr(page_definition_by_id("daily_activity_pattern"), "child_id") + assert page_definition_by_id("trip_mode").page_cls is not None + assert page_definition_by_id("raw_trip_demo") is not None + assert page_definition_by_id("raw_trip_demo").default_enabled is False + assert page_definition_by_id("raw_trip_demo").title == "Prepared Trip Demo" + assert page_definition_by_id("raw_trip_demo").prepared_data_mode == "required" + assert page_definition_by_id("raw_trip_demo").required_prepared_tables == ("trips",) + + +def test_discovered_page_modules_declare_decorated_page_classes() -> None: + discovered_modules = [] + for module_info in pkgutil.iter_modules(dashboard_pages_package.__path__): + if module_info.name.startswith("_"): + continue + module = importlib.import_module( + f"{dashboard_pages_package.__name__}.{module_info.name}" + ) + discovered_modules.append(module) + if module_info.ispkg: + discovered_modules.extend( + importlib.import_module(f"{module.__name__}.{child_info.name}") + for child_info in pkgutil.iter_modules(module.__path__) + if not child_info.name.startswith("_") + ) + + assert discovered_modules + assert any(hasattr(module, "GROUP") for module in discovered_modules) + assert all( + hasattr(module, "GROUP") + or any( + isinstance(value, type) + and issubclass(value, DashboardPage) + and value is not DashboardPage + and value.__module__ == module.__name__ + and isinstance(value.definition, DashboardPageDefinition) + for value in vars(module).values() + ) + for module in discovered_modules + ) + assert all(not hasattr(module, "PAGE") for module in discovered_modules) + assert all(not hasattr(module, "build") for module in discovered_modules) + + +def test_page_registry_smoke_checks_metadata_and_class_attachment() -> None: + definitions = all_page_definitions() + + assert all(definition.page_id for definition in definitions) + assert all(definition.title for definition in definitions) + assert len({definition.page_id for definition in definitions}) == len(definitions) + assert all(definition.page_cls is not None for definition in definitions) + + for definition in definitions: + assert definition.page_cls.definition is definition + assert definition.prepared_data_mode in {"none", "optional", "required"} + assert len(set(definition.required_summary_ids)) == len( + definition.required_summary_ids + ) + assert all( + summary_id in SUMMARY_BY_ID + for summary_id in definition.required_summary_ids + ) + assert len(set(definition.optional_summary_ids)) == len( + definition.optional_summary_ids + ) + assert all( + summary_id in SUMMARY_BY_ID + for summary_id in definition.optional_summary_ids + ) + + +def test_page_registry_accepts_non_default_registered_summary_id() -> None: + class ValidationAutoVmtPage(DashboardPage): + pass + + definition = DashboardPageDefinition( + page_id="auto_vmt_validation", + title="Auto VMT Validation", + page_cls=ValidationAutoVmtPage, + required_summary_ids=("auto_vmt_validation_summary",), + default_enabled=False, + ) + + _validate_page_definition(definition) + requirements = data_requirements_for_pages([definition]) + + assert requirements.required_summary_ids == ("auto_vmt_validation_summary",) + + +def test_dashboard_state_reports_missing_non_default_summary_as_diagnostic() -> None: + summary_run = create_summary_run( + label="Base", + run_key="base", + summaries_by_mode={ + "weighted": {"population_totals": pl.DataFrame({"population": [1.0]})}, + "unweighted": {"population_totals": pl.DataFrame({"population": [1.0]})}, + }, + ) + state = DashboardState( + summary_runs=[summary_run], + weighting_modes=["weighted", "unweighted"], + ) + + selection = state.inspect_summary_table("auto_vmt_validation_summary") + + assert selection.usable_runs == [] + assert [excluded.status for excluded in selection.excluded_runs] == ["missing"] + assert selection.excluded_runs[0].source_id == "auto_vmt_validation_summary" + + +def test_external_traffic_helpers_filter_period_and_facility_type( + tmp_path: Path, +) -> None: + config = _write_config(tmp_path) + counts = [ + ( + "Run", + pl.DataFrame( + { + "id": [1, 2], + "FACTYPE": [3, 4], + "am_vol": [10.0, 20.0], + "day_vol": [100.0, 200.0], + } + ), + ) + ] + volumes = [ + ( + "Run", + pl.DataFrame( + { + "id": [1, 2], + "FACTYPE": [3, 4], + "am_vol": [11.0, 21.0], + "day_vol": [110.0, 210.0], + } + ), + ) + ] + links = [ + ( + "Run", + pl.DataFrame( + { + "id": [10, 11], + "From_Node": [100, 101], + "To_Node": [200, 201], + "FACTYPE": [3, 4], + "day_vol": [5.0, 15.0], + } + ), + ) + ] + + scatter = demo_count_scatter_data_from_sources( + counts, + volumes, + volume_col="day_vol", + facility_type="4", + ) + aggregate = demo_link_aggregate_data( + links, + volume_col="day_vol", + facility_type="All", + ) + + assert scatter[0][1].to_dicts() == [ + { + "id": 2, + "facility_type": "4", + "observed_volume": 200.0, + "modeled_volume": 210.0, + } + ] + assert aggregate[0][1].to_dicts() == [ + {"FACTYPE": "3", "volume": 5.0}, + {"FACTYPE": "4", "volume": 15.0}, + ] + comparison = demo_volume_comparison_table( + counts, + volumes, + link_list=[ + ( + "Run", + pl.DataFrame( + { + "id": [1, 2], + "From_Node": [100, 101], + "To_Node": [200, 201], + } + ), + ) + ], + volume_col="day_vol", + facility_type="4", + top_n=10, + ) + assert comparison[0][1].to_dicts() == [ + { + "link_id": 2, + "facility_type": "4", + "From_Node": 101, + "To_Node": 201, + "Observed Link Volume": 200.0, + "Modeled Link Volume": 210.0, + "Difference": 10.0, + "% Difference": "5.00%", + } + ] + comparison_without_metadata = demo_volume_comparison_table( + counts, + volumes, + volume_col="day_vol", + facility_type="4", + top_n=10, + ) + assert comparison_without_metadata[0][1].columns == [ + "link_id", + "facility_type", + "Observed Link Volume", + "Modeled Link Volume", + "Difference", + "% Difference", + ] + comparison_with_empty_metadata = demo_volume_comparison_table( + counts, + volumes, + link_list=[ + ( + "Run", + pl.DataFrame( + { + "id": [1, 2], + "From_Node": [None, None], + "To_Node": [None, None], + } + ), + ) + ], + volume_col="day_vol", + facility_type="4", + top_n=10, + ) + assert comparison_with_empty_metadata[0][1].columns == [ + "link_id", + "facility_type", + "Observed Link Volume", + "Modeled Link Volume", + "Difference", + "% Difference", + ] + comparison_top_modeled = demo_volume_comparison_table( + [ + ( + "Run", + pl.DataFrame( + { + "id": [1, 2], + "FACTYPE": [4, 4], + "day_vol": [999.0, 100.0], + } + ), + ) + ], + [ + ( + "Run", + pl.DataFrame( + { + "id": [1, 2], + "FACTYPE": [4, 4], + "day_vol": [50.0, 200.0], + } + ), + ) + ], + volume_col="day_vol", + facility_type="All", + top_n=1, + ) + assert comparison_top_modeled[0][1]["link_id"].to_list() == [2] + + derived_scatter = demo_count_scatter_data( + [ + ( + "Run", + pl.DataFrame( + { + "id": [1, 2, 3], + "facility_type": ["3", "4", "4"], + "period": ["Day", "AM", "Day"], + "observed_volume": [100.0, 20.0, 300.0], + "modeled_volume": [110.0, 21.0, 310.0], + } + ), + ) + ], + period="Day", + facility_type="4", + ) + assert derived_scatter[0][1].to_dicts() == [ + { + "id": 3, + "facility_type": "4", + "period": "Day", + "observed_volume": 300.0, + "modeled_volume": 310.0, + } + ] + facility_comparison = demo_facility_comparison_table( + [ + ( + "Run", + pl.DataFrame( + { + "id": [1, 2, 3], + "facility_type": ["4", "4", "3"], + "period": ["Day", "Day", "Day"], + "observed_volume": [100.0, 300.0, 50.0], + "modeled_volume": [110.0, 330.0, 75.0], + } + ), + ) + ], + [ + ( + "Run", + pl.DataFrame( + { + "facility_type": ["4"], + "period": ["Day"], + "r_squared": [0.875], + } + ), + ) + ], + period="Day", + facility_type="4", + config=config, + ) + + assert facility_comparison[0][1].to_dicts() == [ + { + "Facility Type": "4", + "n": 2, + "Total Observed Count": 400.0, + "Total Modeled Count": 440.0, + "% Difference": "10.00%", + "RMSE": 22.360679774997898, + "R^2": 0.875, + } + ] + + +def test_demo_count_fit_line_helper_builds_plot_data() -> None: + fit_lines = demo_count_fit_line_data( + [ + ( + "Run", + pl.DataFrame( + { + "facility_type": ["All", "4"], + "period": ["Day", "Day"], + "slope": [2.0, 3.0], + "intercept": [5.0, 7.0], + "r_squared": [1.0, 0.9], + "n_locations": [3, 2], + "observed_min": [10.0, 20.0], + "observed_max": [30.0, 40.0], + "equation_label": ["y = 2.00x + 5.00", "y = 3.00x + 7.00"], + "r_squared_label": ["R^2 = 1.00", "R^2 = 0.90"], + } + ), + ) + ], + period="Day", + facility_type="4", + ) + + assert fit_lines[0][1].select("observed_volume", "modeled_volume").to_dicts() == [ + {"observed_volume": 20.0, "modeled_volume": 67.0}, + {"observed_volume": 40.0, "modeled_volume": 127.0}, + ] + assert "y = 3.00x + 7.00" in fit_lines[0][1]["annotation"][0] + + +def test_external_vmt_helper_reshapes_wide_tod_table() -> None: + data = [ + ( + "Run", + pl.DataFrame( + { + "TOD": ["AM", "Daily"], + "SOV": [10.0, 20.0], + "HOV2": [2.0, 4.0], + "Total": [12.0, 24.0], + } + ), + ) + ] + + chart_data = wide_tod_chart_data( + data, + tod_col="TOD", + value_columns=["SOV", "HOV2"], + ) + + assert chart_data[0][1].to_dicts() == [ + {"tod": "AM", "category": "SOV", "value": 10.0}, + {"tod": "AM", "category": "HOV2", "value": 2.0}, + ] + + +def test_demo_commercial_vehicle_helper_aggregates_selected_breakdown( + tmp_path: Path, +) -> None: + config = _write_config( + tmp_path, + extra_lines=[ + "display:", + " labels:", + " commercial_vehicle_type:", + " mapping:", + " car: Car", + " su: Single-Unit Truck", + " mu: Multi-Unit Truck", + ], + ) + data = [ + ( + "Run", + pl.DataFrame( + { + "tod": ["AM", "PM", "Daily"], + "car": [10.0, 20.0, 100.0], + "mu": [30.0, 10.0, 200.0], + "su": [5.0, 15.0, 300.0], + } + ), + ) + ] + + by_period = demo_commercial_vehicle_chart_data( + data, + breakdown="Time Period", + ) + by_type = demo_commercial_vehicle_chart_data( + data, + breakdown="Commercial Vehicle Type", + time_period="AM", + ) + by_type_daily = demo_commercial_vehicle_chart_data( + data, + breakdown="Commercial Vehicle Type", + time_period="Daily", + ) + time_period_options, (vehicle_type_options, raw_by_label) = ( + demo_commercial_filter_options(data, config=config) + ) + + assert by_period[0][1].to_dicts() == [ + { + "category": "AM", + "value": 45.0, + "value_percent": pytest.approx(7.5), + }, + { + "category": "PM", + "value": 45.0, + "value_percent": pytest.approx(7.5), + }, + { + "category": "Daily", + "value": 600.0, + "value_percent": 100.0, + }, + ] + assert by_period[0][1].select("category", "value_percent").to_dicts() == [ + {"category": "AM", "value_percent": pytest.approx(7.5)}, + {"category": "PM", "value_percent": pytest.approx(7.5)}, + {"category": "Daily", "value_percent": 100.0}, + ] + assert by_type[0][1].to_dicts() == [ + {"category": "car", "value": 10.0}, + {"category": "mu", "value": 30.0}, + {"category": "su", "value": 5.0}, + ] + assert by_type_daily[0][1].to_dicts() == [ + {"category": "car", "value": 100.0}, + {"category": "mu", "value": 200.0}, + {"category": "su", "value": 300.0}, + ] + assert time_period_options == ["Daily", "AM", "PM"] + assert vehicle_type_options == [ + "All", + "Car", + "Single-Unit Truck", + "Multi-Unit Truck", + ] + assert raw_by_label["Car"] == "car" + assert raw_by_label["Multi-Unit Truck"] == "mu" + + +def test_external_travel_helper_aggregates_selected_breakdown( + tmp_path: Path, +) -> None: + config = _write_config( + tmp_path, + extra_lines=[ + "display:", + " labels:", + " trip_purpose:", + " mapping:", + " hbw: Work", + " nhbw: Non-Home-Based Work", + ], + ) + data = [ + ( + "Run", + pl.DataFrame( + { + "tod": ["AM", "PM", "Daily"], + "hbw": [10.0, 20.0, 100.0], + "nhbw": [30.0, 10.0, 200.0], + "truck": [5.0, 15.0, 300.0], + "Total": [45.0, 45.0, 600.0], + } + ), + ) + ] + + by_period = external_travel_chart_data( + data, + breakdown="Time Period", + ) + by_purpose = external_travel_chart_data( + data, + breakdown="Trip Purpose", + time_period="AM", + ) + time_period_options, (purpose_options, raw_by_label) = ( + external_travel_filter_options( + data, + config=config, + ) + ) + + assert by_period[0][1].to_dicts() == [ + { + "category": "AM", + "value": 45.0, + "value_percent": pytest.approx(7.5), + }, + { + "category": "PM", + "value": 45.0, + "value_percent": pytest.approx(7.5), + }, + { + "category": "Daily", + "value": 600.0, + "value_percent": 100.0, + }, + ] + assert by_purpose[0][1].to_dicts() == [ + {"category": "hbw", "value": 10.0}, + {"category": "nhbw", "value": 30.0}, + {"category": "truck", "value": 5.0}, + ] + assert time_period_options == ["Daily", "AM", "PM"] + assert purpose_options[:3] == ["All", "Work", "Non-Home-Based Work"] + assert raw_by_label["Work"] == "hbw" + + +def test_personal_auto_vmt_helper_aggregates_time_period_with_filters() -> None: + data = [ + ( + "Period Run", + pl.DataFrame( + { + "geography_type": ["all_geographies"] * 4, + "geography_id": ["all_geographies"] * 4, + "income_segment": ["low", "low", "high", "low"], + "household_size": ["1", "2", "1", "1"], + "time_period": ["AM", "PM", "AM", "EA"], + "auto_vmt": [10.0, 5.0, 99.0, 3.0], + "trip_count": [2.0, 1.0, 9.0, 1.0], + } + ), + ), + ( + "Daily Run", + pl.DataFrame( + { + "geography_type": ["all_geographies"], + "geography_id": ["all_geographies"], + "income_segment": ["low"], + "household_size": ["1"], + "time_period": ["Daily"], + "auto_vmt": [7.0], + "trip_count": [2.0], + } + ), + ), + ] + + chart_data = personal_auto_vmt_chart_data( + data, + breakdown="Time Period", + geography_type="all_geographies", + geography_id="all_geographies", + time_period="All", + income_segment="low", + household_size="All", + ) + + assert chart_data[0][1].select("category", "auto_vmt", "trip_count").to_dicts() == [ + {"category": "EA", "auto_vmt": 3.0, "trip_count": 1.0}, + {"category": "AM", "auto_vmt": 10.0, "trip_count": 2.0}, + {"category": "PM", "auto_vmt": 5.0, "trip_count": 1.0}, + ] + assert chart_data[1][1].select("category", "auto_vmt", "trip_count").to_dicts() == [ + {"category": "Daily", "auto_vmt": 7.0, "trip_count": 2.0}, + ] + + +def test_personal_auto_vmt_time_period_percent_uses_daily_total() -> None: + chart_data = personal_auto_vmt_chart_data( + [ + ( + "Run", + pl.DataFrame( + { + "geography_type": ["all_geographies"] * 3, + "geography_id": ["all_geographies"] * 3, + "income_segment": ["low"] * 3, + "household_size": ["1"] * 3, + "time_period": ["AM", "PM", "Daily"], + "auto_vmt": [10.0, 5.0, 15.0], + "trip_count": [2.0, 1.0, 3.0], + } + ), + ) + ], + breakdown="Time Period", + geography_type="all_geographies", + geography_id="all_geographies", + time_period="All", + income_segment="low", + household_size="1", + ) + + rows = chart_data[0][1].select("category", "auto_vmt_percent").to_dicts() + assert rows == [ + {"category": "AM", "auto_vmt_percent": pytest.approx(66.6666666667)}, + {"category": "PM", "auto_vmt_percent": pytest.approx(33.3333333333)}, + {"category": "Daily", "auto_vmt_percent": 100.0}, + ] + + +def test_personal_auto_vmt_all_time_period_filter_prefers_daily_totals() -> None: + chart_data = personal_auto_vmt_chart_data( + [ + ( + "Run", + pl.DataFrame( + { + "geography_type": ["all_geographies"] * 3, + "geography_id": ["all_geographies"] * 3, + "income_segment": ["low"] * 3, + "household_size": ["1"] * 3, + "time_period": ["AM", "PM", "Daily"], + "auto_vmt": [10.0, 5.0, 15.0], + "trip_count": [2.0, 1.0, 3.0], + } + ), + ) + ], + breakdown="Income Segment", + geography_type="all_geographies", + geography_id="all_geographies", + time_period="All", + income_segment="All", + household_size="1", + ) + + assert chart_data[0][1].to_dicts() == [ + {"category": "low", "auto_vmt": 15.0, "trip_count": 3.0}, + ] + + +def test_personal_auto_vmt_helper_breaks_down_by_mode() -> None: + chart_data = personal_auto_vmt_chart_data( + [ + ( + "Run", + pl.DataFrame( + { + "geography_type": ["all_geographies"] * 4, + "geography_id": ["all_geographies"] * 4, + "income_segment": ["low"] * 4, + "household_size": ["1"] * 4, + "time_period": ["AM", "Daily", "AM", "Daily"], + "mode": ["SOV", "SOV", "HOV2", "HOV2"], + "auto_vmt": [10.0, 10.0, 3.0, 3.0], + "trip_count": [1.0, 1.0, 1.0, 1.0], + } + ), + ) + ], + breakdown="Mode", + geography_type="all_geographies", + geography_id="all_geographies", + time_period="All", + income_segment="low", + household_size="1", + ) + + assert chart_data[0][1].to_dicts() == [ + {"category": "SOV", "auto_vmt": 10.0, "trip_count": 1.0}, + {"category": "HOV2", "auto_vmt": 3.0, "trip_count": 1.0}, + ] + + +def test_personal_auto_vmt_helper_filters_by_mode() -> None: + chart_data = personal_auto_vmt_chart_data( + [ + ( + "Run", + pl.DataFrame( + { + "geography_type": ["all_geographies"] * 4, + "geography_id": ["all_geographies"] * 4, + "income_segment": ["low", "low", "high", "high"], + "household_size": ["1"] * 4, + "time_period": ["Daily"] * 4, + "mode": ["SOV", "HOV2", "SOV", "HOV2"], + "auto_vmt": [10.0, 3.0, 20.0, 7.0], + "trip_count": [1.0, 1.0, 2.0, 2.0], + } + ), + ) + ], + breakdown="Income Segment", + geography_type="all_geographies", + geography_id="all_geographies", + time_period="Daily", + mode="HOV2", + income_segment="All", + household_size="1", + ) + + assert chart_data[0][1].to_dicts() == [ + {"category": "high", "auto_vmt": 7.0, "trip_count": 2.0}, + {"category": "low", "auto_vmt": 3.0, "trip_count": 1.0}, + ] + + +def test_vmt_page_labels_personal_auto_modes_from_display_config( + tmp_path: Path, +) -> None: + config = _write_config( + tmp_path, + extra_lines=[ + "display:", + " labels:", + " mode:", + " mapping:", + " SOV: Drive Alone", + " HOV2: Shared Ride 2", + " TAXI: Taxi", + ], + ) + personal_vmt = pl.DataFrame( + { + "geography_type": ["all_geographies"] * 3, + "geography_id": ["all_geographies"] * 3, + "income_segment": ["low"] * 3, + "household_size": ["1"] * 3, + "time_period": ["Daily"] * 3, + "mode": ["SOV", "HOV2", "TAXI"], + "auto_vmt": [10.0, 3.0, 4.0], + "trip_count": [1.0, 1.0, 1.0], + "distance_source": ["od_dist"] * 3, + "time_period_source": ["trip_period"] * 3, + } + ) + summary_run = create_summary_run( + label="Base", + run_key="base", + summaries_by_mode={ + "weighted": {PERSONAL_AUTO_VMT_SUMMARY_ID: personal_vmt}, + "unweighted": {PERSONAL_AUTO_VMT_SUMMARY_ID: personal_vmt}, + }, + ) + state = DashboardState( + summary_runs=[summary_run], + weighting_modes=config.weighting_modes, + ) + state.value_mode = "Count" + + page = VMTValidationPage(state, config) + page.refresh(force=True) + + assert list(page.personal_vmt_mode_sel.options) == [ + "All", + "Drive Alone", + "Shared Ride 2", + "Taxi", + ] + page.personal_vmt_breakdown_sel.value = "Income Segment" + page.personal_vmt_mode_sel.value = "Taxi" + page.refresh(force=True) + + assert page.selected_personal_vmt_mode_raw() == "TAXI" + + +def test_non_motorized_vmt_section_mirrors_personal_auto_controls( + tmp_path: Path, +) -> None: + config = _write_config( + tmp_path, + extra_lines=[ + "display:", + " labels:", + " mode:", + " mapping:", + " WALK: Walk", + " BIKE: Bike", + " geography:", + " mapping:", + " all_geographies: All Geographies", + " home_county: County", + ], + ) + personal_vmt = pl.DataFrame( + { + "geography_type": ["all_geographies"], + "geography_id": ["all_geographies"], + "income_segment": ["all_income_segments"], + "household_size": ["all_household_sizes"], + "time_period": ["Daily"], + "mode": ["SOV"], + "auto_vmt": [10.0], + "trip_count": [1.0], + "distance_source": ["skim_auto_distance"], + "time_period_source": ["trip_period"], + } + ) + non_motorized_vmt = pl.DataFrame( + { + "geography_type": [ + "all_geographies", + "all_geographies", + "home_county", + "home_county", + ], + "geography_id": ["all_geographies", "all_geographies", "Wake", "Wake"], + "income_segment": ["low", "high", "low", "high"], + "household_size": ["1", "2", "1", "2"], + "time_period": ["Daily", "Daily", "AM", "PM"], + "mode": ["WALK", "BIKE", "WALK", "BIKE"], + "non_motorized_vmt": [5.0, 3.0, 2.0, 3.0], + "trip_count": [3.0, 2.0, 1.0, 2.0], + "distance_source": ["prepared_non_motorized_distance"] * 4, + "time_period_source": ["trip_period"] * 4, + } + ) + external_vmt = pl.DataFrame( + { + "tod": ["Daily"], + "hbo": [20.0], + "Total": [20.0], + } + ) + commercial_vmt = pl.DataFrame( + { + "tod": ["Daily"], + "car": [7.0], + "mu": [3.0], + "su": [0.0], + } + ) + summary_run = create_summary_run( + label="Base", + run_key="base", + summaries_by_mode={ + "weighted": { + PERSONAL_AUTO_VMT_SUMMARY_ID: personal_vmt, + NON_MOTORIZED_VMT_SUMMARY_ID: non_motorized_vmt, + "external_vmt_validation_summary": external_vmt, + "commercial_vehicle_vmt_validation_summary": commercial_vmt, + }, + "unweighted": { + PERSONAL_AUTO_VMT_SUMMARY_ID: personal_vmt, + NON_MOTORIZED_VMT_SUMMARY_ID: non_motorized_vmt, + "external_vmt_validation_summary": external_vmt, + "commercial_vehicle_vmt_validation_summary": commercial_vmt, + }, + }, + ) + state = DashboardState( + summary_runs=[summary_run], + weighting_modes=config.weighting_modes, + ) + state.value_mode = "Count" + + page = VMTValidationPage(state, config) + page.refresh(force=True) + + markdown_titles = [ + obj.object for obj in page.view.objects if isinstance(obj, pn.pane.Markdown) + ] + assert markdown_titles[:4] == [ + "## VMT Validation", + "### VMT Overview", + "### Personal Auto VMT", + "### Non-Motorized VMT", + ] + assert markdown_titles[4] == "### External VMT and Travel" + overview_tables = _collect_tabulators(page._vmt_overview_body) + assert len(overview_tables) == 1 + assert overview_tables[0].value.to_dict("records") == [ + { + "Category": "Personal Auto", + "VMT": "10", + "% Share of Total": "20.83", + }, + { + "Category": "Non-Motorized", + "VMT": "8", + "% Share of Total": "16.67", + }, + { + "Category": "External", + "VMT": "20", + "% Share of Total": "41.67", + }, + { + "Category": "Commercial", + "VMT": "10", + "% Share of Total": "20.83", + }, + ] + assert list(page.non_motorized_vmt_mode_sel.options) == [ + "All", + "Walk", + "Bike", + ] + assert page.non_motorized_vmt_geography_type_sel.disabled is False + assert page.non_motorized_vmt_geography_sel.disabled is False + + page.non_motorized_vmt_breakdown_sel.value = "Home Geography" + page.refresh(force=True) + page.non_motorized_vmt_geography_type_sel.value = "County" + page.refresh(force=True) + + assert page.non_motorized_vmt_geography_type_sel.disabled is False + assert page.non_motorized_vmt_geography_sel.name == "County Name" + assert page.selected_non_motorized_vmt_geography_type_raw() == "home_county" + + page.non_motorized_vmt_breakdown_sel.value = "Mode" + page.refresh(force=True) + assert page.non_motorized_vmt_geography_type_sel.disabled is False + assert page.non_motorized_vmt_geography_sel.disabled is False + assert page.non_motorized_vmt_mode_sel.disabled is True + assert page.non_motorized_vmt_mode_sel.value == "All" + + chart = page.render_non_motorized_vmt_section()[0] + assert chart.object.layout.title.text == "Non-Motorized VMT by Mode" + assert chart.object.layout.yaxis.title.text == "Non-Motorized Miles Traveled" + assert list(chart.object.layout.xaxis.categoryarray) == ["Walk", "Bike"] + + +def test_personal_auto_vmt_helper_ignores_active_breakdown_selector() -> None: + data = [ + ( + "Run", + pl.DataFrame( + { + "geography_type": ["all_geographies"] * 3, + "geography_id": ["all_geographies"] * 3, + "income_segment": ["low", "high", "low"], + "household_size": ["1", "1", "2"], + "time_period": ["AM", "AM", "PM"], + "auto_vmt": [10.0, 20.0, 30.0], + "trip_count": [1.0, 2.0, 3.0], + } + ), + ) + ] + + chart_data = personal_auto_vmt_chart_data( + data, + breakdown="Income Segment", + geography_type="all_geographies", + geography_id="all_geographies", + time_period="AM", + income_segment="low", + household_size="All", + ) + + assert chart_data[0][1].to_dicts() == [ + {"category": "high", "auto_vmt": 20.0, "trip_count": 2.0}, + {"category": "low", "auto_vmt": 10.0, "trip_count": 1.0}, + ] + + +def test_personal_auto_vmt_helper_caps_home_geography_breakdown() -> None: + data = [ + ( + "Run", + pl.DataFrame( + { + "geography_type": ["home_taz"] * 30, + "geography_id": [str(value) for value in range(30)], + "income_segment": ["all_income_segments"] * 30, + "household_size": ["all_household_sizes"] * 30, + "time_period": ["Daily"] * 30, + "auto_vmt": [float(value) for value in range(30)], + "trip_count": [1.0] * 30, + } + ), + ) + ] + + chart_data = personal_auto_vmt_chart_data( + data, + breakdown="Home Geography", + geography_type="home_taz", + geography_id="All", + time_period="Daily", + income_segment="All", + household_size="All", + ) + + rows = chart_data[0][1].to_dicts() + assert len(rows) == 25 + assert rows[0] == {"category": "29", "auto_vmt": 29.0, "trip_count": 1.0} + assert rows[-1] == {"category": "5", "auto_vmt": 5.0, "trip_count": 1.0} + + +def test_vmt_page_registers_personal_auto_vmt_and_renders_missing_card( + tmp_path: Path, +) -> None: + config = _write_config(tmp_path) + summary_run = create_summary_run( + label="Base", + run_key="base", + summaries_by_mode={ + "weighted": {"population_totals": pl.DataFrame({"population": [1.0]})}, + "unweighted": {"population_totals": pl.DataFrame({"population": [1.0]})}, + }, + ) + state = DashboardState( + summary_runs=[summary_run], + weighting_modes=config.weighting_modes, + ) + + page = VMTValidationPage(state, config) + page.refresh(force=True) + + assert PERSONAL_AUTO_VMT_SUMMARY_ID in page.required_summary_ids + assert NON_MOTORIZED_VMT_SUMMARY_ID in page.required_summary_ids + assert page._personal_vmt_body.objects + assert page._non_motorized_vmt_body.objects + + +def test_vmt_page_disables_active_personal_auto_vmt_filter_selector( + tmp_path: Path, +) -> None: + config = _write_config(tmp_path) + personal_vmt = pl.DataFrame( + { + "geography_type": ["all_geographies", "all_geographies"], + "geography_id": ["all_geographies", "all_geographies"], + "income_segment": ["low", "high"], + "household_size": ["1", "2"], + "time_period": ["AM", "PM"], + "mode": ["SOV", "HOV2"], + "auto_vmt": [10.0, 20.0], + "trip_count": [1.0, 2.0], + "distance_source": ["skim_auto_distance", "skim_auto_distance"], + "time_period_source": ["trip_period", "trip_period"], + } + ) + summary_run = create_summary_run( + label="Base", + run_key="base", + summaries_by_mode={ + "weighted": {PERSONAL_AUTO_VMT_SUMMARY_ID: personal_vmt}, + "unweighted": {PERSONAL_AUTO_VMT_SUMMARY_ID: personal_vmt}, + }, + ) + state = DashboardState( + summary_runs=[summary_run], + weighting_modes=config.weighting_modes, + ) + + page = VMTValidationPage(state, config) + page.refresh(force=True) + + assert page.personal_vmt_geography_type_sel.disabled is False + assert page.personal_vmt_geography_sel.disabled is False + assert page.personal_vmt_time_period_sel.disabled is True + assert page.personal_vmt_mode_sel.disabled is False + assert page.personal_vmt_income_segment_sel.disabled is False + assert page.personal_vmt_household_size_sel.disabled is False + + page.personal_vmt_breakdown_sel.value = "Home Geography" + page.refresh(force=True) + + assert page.personal_vmt_geography_type_sel.disabled is False + assert page.personal_vmt_geography_sel.disabled is False + + page.personal_vmt_breakdown_sel.value = "Income Segment" + page.refresh(force=True) + + assert page.personal_vmt_geography_type_sel.disabled is False + assert page.personal_vmt_geography_sel.disabled is False + assert page.personal_vmt_time_period_sel.disabled is False + assert page.personal_vmt_mode_sel.disabled is False + assert page.personal_vmt_income_segment_sel.disabled is True + assert page.personal_vmt_income_segment_sel.value == "All" + assert page.personal_vmt_household_size_sel.disabled is False + + page.personal_vmt_breakdown_sel.value = "Mode" + page.refresh(force=True) + + assert page.personal_vmt_geography_type_sel.disabled is False + assert page.personal_vmt_geography_sel.disabled is False + assert page.personal_vmt_time_period_sel.disabled is False + assert page.personal_vmt_mode_sel.disabled is True + assert page.personal_vmt_mode_sel.value == "All" + + +def test_vmt_page_geography_selectors_use_display_labels_and_raw_filters( + tmp_path: Path, +) -> None: + config = _write_config( + tmp_path, + extra_lines=[ + "display:", + " labels:", + " geography:", + " mapping:", + " all_geographies: All Geographies", + " home_county: County", + ], + ) + personal_vmt = pl.DataFrame( + { + "geography_type": ["all_geographies", "home_county", "home_county"], + "geography_id": ["all_geographies", "Wake", "Durham"], + "income_segment": ["low", "low", "low"], + "household_size": ["1", "1", "1"], + "time_period": ["Daily", "Daily", "Daily"], + "mode": ["SOV", "SOV", "SOV"], + "auto_vmt": [30.0, 10.0, 20.0], + "trip_count": [3.0, 1.0, 2.0], + "distance_source": ["skim_auto_distance"] * 3, + "time_period_source": ["trip_period"] * 3, + } + ) + summary_run = create_summary_run( + label="Base", + run_key="base", + summaries_by_mode={ + "weighted": {PERSONAL_AUTO_VMT_SUMMARY_ID: personal_vmt}, + "unweighted": {PERSONAL_AUTO_VMT_SUMMARY_ID: personal_vmt}, + }, + ) + state = DashboardState( + summary_runs=[summary_run], + weighting_modes=config.weighting_modes, + ) + + page = VMTValidationPage(state, config) + page.refresh(force=True) + + assert list(page.personal_vmt_geography_type_sel.options) == [ + "All Geography Types", + "County", + ] + assert page.personal_vmt_breakdown_sel.value == "Time Period" + assert page.personal_vmt_geography_type_sel.disabled is False + assert page.personal_vmt_geography_sel.disabled is False + + page.personal_vmt_geography_type_sel.value = "County" + page.refresh(force=True) + + assert page.personal_vmt_geography_type_sel.disabled is False + assert page.personal_vmt_geography_sel.disabled is False + assert page.personal_vmt_geography_sel.name == "County Name" + assert list(page.personal_vmt_geography_sel.options) == [ + "All Counties", + "Durham", + "Wake", + ] + page.personal_vmt_geography_sel.value = "Durham" + + assert page.selected_personal_vmt_geography_type_raw() == "home_county" + assert page.selected_personal_vmt_geography_raw() == "Durham" + + +def test_vmt_export_states_collapse_ignored_personal_auto_selectors( + tmp_path: Path, +) -> None: + config = _write_config(tmp_path) + personal_vmt = pl.DataFrame( + { + "geography_type": [ + "all_geographies", + "home_county", + "home_county", + "home_county", + ], + "geography_id": [ + "all_geographies", + "Wake", + "Durham", + "Wake", + ], + "income_segment": ["low", "low", "high", "low"], + "household_size": ["1", "1", "2", "1"], + "time_period": ["Daily", "AM", "PM", "AM"], + "mode": ["SOV", "SOV", "HOV2", "HOV2"], + "auto_vmt": [30.0, 10.0, 20.0, 5.0], + "trip_count": [3.0, 1.0, 2.0, 1.0], + "distance_source": ["skim_auto_distance"] * 4, + "time_period_source": ["trip_period"] * 4, + } + ) + summary_run = create_summary_run( + label="Base", + run_key="base", + summaries_by_mode={ + "weighted": {PERSONAL_AUTO_VMT_SUMMARY_ID: personal_vmt}, + "unweighted": {PERSONAL_AUTO_VMT_SUMMARY_ID: personal_vmt}, + }, + ) + state = DashboardState( + summary_runs=[summary_run], + weighting_modes=config.weighting_modes, + ) + page = VMTValidationPage(state, config) + page.refresh(force=True) + + selector_widgets = { + selector.selector_id: selector.widget for selector in page.registered_selectors + } + assert selector_widgets["personal_auto_vmt_geography_type"].disabled is False + assert selector_widgets["personal_auto_vmt_geography"].disabled is False + + active_selector_ids = [ + "personal_auto_vmt_breakdown", + "personal_auto_vmt_geography_type", + "personal_auto_vmt_geography", + "personal_auto_vmt_time_period", + "personal_auto_vmt_mode", + "personal_auto_vmt_income_segment", + "personal_auto_vmt_household_size", + ] + request_modes = { + "personal_auto_vmt_breakdown": "explicit", + "personal_auto_vmt_geography_type": "all", + "personal_auto_vmt_geography": "all", + "personal_auto_vmt_time_period": "default", + "personal_auto_vmt_mode": "all", + "personal_auto_vmt_income_segment": "default", + "personal_auto_vmt_household_size": "default", + } + requested_values = { + "personal_auto_vmt_breakdown": ["Mode", "Home Geography"], + } + selector_metadata = { + selector_id: { + "id": selector_id, + "label": selector_id, + "available": True, + "request_mode": request_modes[selector_id], + "requested_values": requested_values.get(selector_id, []), + "resolved_values": [ + str(option) for option in selector_widgets[selector_id].options + ], + "default_value": str(selector_widgets[selector_id].value), + "options": [ + str(option) for option in selector_widgets[selector_id].options + ], + "export_enabled": True, + } + for selector_id in active_selector_ids + } + + states, aliases = resolve_export_section_states( + page, + page_def=VMTValidationPage.definition, + part_def=type("Part", (), {"part_id": "personal_auto_vmt_body"})(), + active_selector_ids=active_selector_ids, + selector_widgets=selector_widgets, + selector_metadata_by_id=selector_metadata, + ) + + assert all( + state["personal_auto_vmt_mode"] == "All" + for state in states + if state["personal_auto_vmt_breakdown"] == "Mode" + ) + mode_states = [ + state for state in states if state["personal_auto_vmt_breakdown"] == "Mode" + ] + assert ( + len({state["personal_auto_vmt_geography_type"] for state in mode_states}) == 1 + ) + assert {state["personal_auto_vmt_geography_type"] for state in mode_states} == { + "All Geography Types" + } + assert len({state["personal_auto_vmt_geography"] for state in mode_states}) == 1 + home_state_groups = { + tuple( + value + for selector_id, value in state.items() + if selector_id != "personal_auto_vmt_geography" + ) + for state in states + if state["personal_auto_vmt_breakdown"] == "Home Geography" + } + home_states = [ + state + for state in states + if state["personal_auto_vmt_breakdown"] == "Home Geography" + ] + assert len(home_states) == len(home_state_groups) + assert len({state["personal_auto_vmt_geography_type"] for state in home_states}) > 1 + assert aliases + + +def test_vmt_demo_commercial_vehicle_chart_uses_breakdown_and_percent_mode( + tmp_path: Path, +) -> None: + config = _write_config( + tmp_path, + extra_lines=[ + "display:", + " labels:", + " commercial_vehicle_type:", + " mapping:", + " car: Car", + " su: Single-Unit Truck", + " mu: Multi-Unit Truck", + ], + ) + demo_commercial = pl.DataFrame( + { + "tod": ["AM", "PM", "Daily"], + "car": [10.0, 20.0, 100.0], + "mu": [30.0, 10.0, 200.0], + "su": [5.0, 15.0, 300.0], + } + ) + summary_run = create_summary_run( + label="Base", + run_key="base", + summaries_by_mode={ + "weighted": { + "commercial_vehicle_validation_summary": demo_commercial, + }, + "unweighted": { + "commercial_vehicle_validation_summary": demo_commercial, + }, + }, + ) + state = DashboardState( + summary_runs=[summary_run], + weighting_modes=config.weighting_modes, + ) + state.value_mode = "Percent" + + page = VMTValidationPage(state, config) + page.refresh(force=True) + page.demo_commercial_metric_sel.value = "Trips" + page.demo_commercial_breakdown_sel.value = "Time Period" + page.demo_commercial_vehicle_type_sel.value = "Car" + + chart = page.render_demo_commercial_chart() + fig = chart.object + + assert fig.layout.title.text == "Commercial Vehicle Trips by Time Period" + assert fig.layout.showlegend is True + assert fig.layout.yaxis.title.text == "Percent of Trips (%)" + assert len(fig.data) == 1 + assert list(fig.data[0].x) == ["AM", "PM", "Daily"] + assert list(fig.data[0].y) == [10.0, 20.0, 100.0] + assert list(fig.layout.xaxis.categoryarray) == ["AM", "PM", "Daily"] + assert list(page.demo_commercial_time_period_sel.options) == [ + "Daily", + "AM", + "PM", + ] + assert list(page.demo_commercial_vehicle_type_sel.options) == [ + "All", + "Car", + "Single-Unit Truck", + "Multi-Unit Truck", + ] + assert page.selected_demo_commercial_vehicle_type_raw() == "car" + + page.refresh(force=True) + assert page.demo_commercial_time_period_sel.disabled is True + assert page.demo_commercial_time_period_sel.value == "Daily" + assert page.demo_commercial_vehicle_type_sel.disabled is False + + page.demo_commercial_breakdown_sel.value = "Commercial Vehicle Type" + page.demo_commercial_time_period_sel.value = "AM" + chart = page.render_demo_commercial_chart() + + assert chart.object.layout.title.text == ( + "Commercial Vehicle Trips by Commercial Vehicle Type" + ) + assert list(chart.object.layout.xaxis.categoryarray) == [ + "Car", + "Single-Unit Truck", + "Multi-Unit Truck", + ] + + +def test_vmt_external_travel_chart_uses_metric_breakdown_and_filters( + tmp_path: Path, +) -> None: + config = _write_config( + tmp_path, + extra_lines=[ + "display:", + " labels:", + " trip_purpose:", + " mapping:", + " hbw: Work", + " nhbw: Non-Home-Based Work", + ], + ) + external_travel = pl.DataFrame( + { + "tod": ["AM", "PM", "Daily"], + "hbw": [10.0, 20.0, 100.0], + "nhbw": [30.0, 10.0, 200.0], + "truck": [5.0, 15.0, 300.0], + "Total": [45.0, 45.0, 600.0], + } + ) + summary_run = create_summary_run( + label="Base", + run_key="base", + summaries_by_mode={ + "weighted": { + "external_trip_validation_summary": external_travel, + }, + "unweighted": { + "external_trip_validation_summary": external_travel, + }, + }, + ) + state = DashboardState( + summary_runs=[summary_run], + weighting_modes=config.weighting_modes, + ) + state.value_mode = "Percent" + + page = VMTValidationPage(state, config) + page.refresh(force=True) + page.external_travel_metric_sel.value = "Trips" + page.external_travel_breakdown_sel.value = "Time Period" + page.external_travel_trip_purpose_sel.value = "Work" + + chart = page.render_external_travel_chart() + fig = chart.object + + assert fig.layout.title.text == "External Trips by Time Period" + assert fig.layout.yaxis.title.text == "Percent of Trips (%)" + assert list(fig.data[0].x) == ["AM", "PM", "Daily"] + assert list(fig.data[0].y) == [10.0, 20.0, 100.0] + assert list(fig.layout.xaxis.categoryarray) == ["AM", "PM", "Daily"] + assert list(page.external_travel_time_period_sel.options) == [ + "Daily", + "AM", + "PM", + ] + assert page.external_travel_time_period_sel.disabled is True + assert page.external_travel_time_period_sel.value == "Daily" + assert page.external_travel_trip_purpose_sel.disabled is False + assert page.selected_external_travel_trip_purpose_raw() == "hbw" + + page.external_travel_breakdown_sel.value = "Trip Purpose" + page.external_travel_time_period_sel.value = "AM" + chart = page.render_external_travel_chart() + + assert chart.object.layout.title.text == "External Trips by Trip Purpose" + assert list(chart.object.layout.xaxis.categoryarray) == [ + "Work", + "Non-Home-Based Work", + "truck", + ] + assert list(chart.object.data[0].x) == [ + "Work", + "Non-Home-Based Work", + "truck", + ] -def _raw_trip_run() -> RunData: - return RunData( +def _export_selector_widgets(page) -> dict[str, pn.widgets.Widget]: + return { + selector.selector_id: selector.widget for selector in page.registered_selectors + } + + +def _export_selector_metadata( + selector_widgets: dict[str, pn.widgets.Widget], + selector_ids: list[str], + *, + request_modes: dict[str, str] | None = None, + requested_values: dict[str, list[str]] | None = None, +) -> dict[str, dict]: + request_modes = request_modes or {} + requested_values = requested_values or {} + return { + selector_id: { + "id": selector_id, + "label": selector_id, + "available": True, + "request_mode": request_modes.get(selector_id, "all"), + "requested_values": requested_values.get(selector_id, []), + "resolved_values": [ + str(option) for option in selector_widgets[selector_id].options + ], + "default_value": str(selector_widgets[selector_id].value), + "options": [ + str(option) for option in selector_widgets[selector_id].options + ], + "export_enabled": True, + } + for selector_id in selector_ids + } + + +def test_mandatory_location_choice_geography_labels_filter_raw_and_export( + tmp_path: Path, +) -> None: + config = _write_config( + tmp_path, + extra_lines=[ + "display:", + " labels:", + " geography:", + " mapping:", + " all_geographies: All Geographies", + " home_county: County", + ], + ) + internal_external = pl.DataFrame( + { + "geography_type": [ + "all_geographies", + "home_county", + "home_county", + ], + "geography_id": ["all_geographies", "Wake", "Durham"], + "internal_worker_count": [30.0, 10.0, 20.0], + "external_worker_count": [3.0, 1.0, 2.0], + } + ) + summary_run = create_summary_run( label="Base", - run_dir="C:/runs/base", - skim_file=None, - hh=pl.DataFrame({"household_id": [1], "finalweight": [2.0]}), - per=pl.DataFrame({"person_id": [1], "household_id": [1], "finalweight": [3.0]}), - tours=pl.DataFrame({"tour_id": [10], "finalweight": [4.0]}), - trips=pl.DataFrame( - { - "trip_id": [100, 101, 102], - "tour_id": [10, 10, 10], - "trip_mode": ["DRIVEALONE", "WALK", "WALK"], - "finalweight": [5.0, 2.0, 1.0], - } - ), - joint_participants=pl.DataFrame(), - land_use=pl.DataFrame(), - skim_matrix=None, - skim_zone_map=None, + run_key="base", + summaries_by_mode={ + "weighted": {"internal_external_worker_by_geography": internal_external}, + "unweighted": {"internal_external_worker_by_geography": internal_external}, + }, + ) + state = DashboardState( + summary_runs=[summary_run], + weighting_modes=config.weighting_modes, + export_mode=True, ) + page = MandatoryLocationChoicePage(state, config) + page.refresh(force=True) -def test_page_registry_exposes_expected_default_definitions() -> None: - definitions = default_page_definitions() + assert list(page.geo_level_sel.options) == [ + "All Geography Types", + "County", + ] + assert "Durham" in page.geography_sel.options + assert "Wake" in page.geography_sel.options - assert [definition.page_id for definition in definitions] == EXPECTED_DEFAULT_LEAF_PAGE_IDS - assert [definition.title for definition in definitions] == EXPECTED_DEFAULT_LEAF_PAGE_TITLES - assert page_definition_by_id("daily_activity_pattern") is not None - assert page_definition_by_id("daily_activity_pattern").title == "Daily Activity Pattern" - assert page_definition_by_id("daily_activity_pattern").group_id == "daily_travel" - assert not hasattr(page_definition_by_id("daily_activity_pattern"), "child_id") - assert page_definition_by_id("trip_mode").page_cls is not None - assert page_definition_by_id("raw_trip_demo") is not None - assert page_definition_by_id("raw_trip_demo").default_enabled is False - assert page_definition_by_id("raw_trip_demo").title == "Prepared Trip Demo" - assert page_definition_by_id("raw_trip_demo").prepared_data_mode == "required" - assert page_definition_by_id("raw_trip_demo").required_prepared_tables == ("trips",) + page.geo_level_sel.value = "County" + page.geography_sel.value = "Durham" + assert page._selected_geography() == ("home_county", "Durham") -def test_discovered_page_modules_export_page_definitions_without_legacy_build_api() -> None: - discovered_modules = [] - for module_info in pkgutil.iter_modules(dashboard_pages_package.__path__): - if module_info.name.startswith("_"): - continue - module = importlib.import_module(f"{dashboard_pages_package.__name__}.{module_info.name}") - discovered_modules.append(module) - if module_info.ispkg: - discovered_modules.extend( - importlib.import_module(f"{module.__name__}.{child_info.name}") - for child_info in pkgutil.iter_modules(module.__path__) - if not child_info.name.startswith("_") - ) - assert discovered_modules - assert any(hasattr(module, "GROUP") for module in discovered_modules) - assert all( - isinstance(getattr(module, "PAGE", None), DashboardPageDefinition) - or hasattr(module, "GROUP") - for module in discovered_modules +def test_mandatory_location_choice_export_aliases_invalid_geography_pairs( + tmp_path: Path, +) -> None: + config = _write_config( + tmp_path, + extra_lines=[ + "display:", + " labels:", + " geography:", + " mapping:", + " all_geographies: All Geography Types", + " home_county: County", + " district: District", + ], ) - assert all(not hasattr(module, "build") for module in discovered_modules) + internal_external = pl.DataFrame( + { + "geography_type": [ + "all_geographies", + "home_county", + "home_county", + "district", + ], + "geography_id": ["all_geographies", "Wake", "Durham", "North"], + "internal_worker_count": [30.0, 10.0, 20.0, 15.0], + "external_worker_count": [3.0, 1.0, 2.0, 1.5], + } + ) + summary_run = create_summary_run( + label="Base", + run_key="base", + summaries_by_mode={ + "weighted": {"internal_external_worker_by_geography": internal_external}, + "unweighted": {"internal_external_worker_by_geography": internal_external}, + }, + ) + state = DashboardState( + summary_runs=[summary_run], + weighting_modes=config.weighting_modes, + export_mode=True, + ) + page = MandatoryLocationChoicePage(state, config) + page.refresh(force=True) + selector_ids = ["geography_level", "geography"] + selector_widgets = _export_selector_widgets(page) + states, aliases = resolve_export_section_states( + page, + page_def=MandatoryLocationChoicePage.definition, + part_def=type("Part", (), {"part_id": "remote_work"})(), + active_selector_ids=selector_ids, + selector_widgets=selector_widgets, + selector_metadata_by_id=_export_selector_metadata( + selector_widgets, + selector_ids, + ), + ) -def test_page_registry_smoke_checks_ids_titles_and_selector_uniqueness() -> None: - definitions = all_page_definitions() + state_pairs = {(state["geography_level"], state["geography"]) for state in states} + assert ("County", "North") not in state_pairs + assert ("District", "Durham") not in state_pairs + assert ("District", "Wake") not in state_pairs + assert ("County", "Durham") in state_pairs + assert ("County", "Wake") in state_pairs + assert ("District", "North") in state_pairs + assert aliases['["County","North"]'] == '["County","All"]' + assert aliases['["District","Durham"]'] == '["District","All"]' + assert len(states) < ( + len(page.geo_level_sel.options) * len(page.geography_sel.options) + ) - assert all(definition.page_id for definition in definitions) - assert all(definition.title for definition in definitions) - assert len({definition.page_id for definition in definitions}) == len(definitions) - assert all(definition.page_cls is not None for definition in definitions) - for definition in definitions: - selector_ids = [selector.selector_id for selector in definition.selectors] - assert len(selector_ids) == len(set(selector_ids)) - assert definition.prepared_data_mode in {"none", "optional", "required"} - assert len(set(definition.required_summary_ids)) == len( - definition.required_summary_ids +def test_tour_distance_export_geography_pairs_follow_selected_level( + tmp_path: Path, +) -> None: + config = _write_config(tmp_path) + average_mandatory = pl.DataFrame( + { + "mandatory_tour_purpose": ["work"], + "geography_level": ["Region"], + "average_tour_distance": [8.0], + } + ) + average_nonmandatory = pl.DataFrame( + { + "nonmandatory_tour_purpose": ["shopping", "shopping", "shopping"], + "geography_type": ["district", "district", "county"], + "geography_id": ["North", "South", "Wake"], + "average_tour_distance": [4.0, 8.0, 12.0], + "tour_count": [2.0, 3.0, 4.0], + } + ) + distance = pl.DataFrame( + { + "tour_purpose": ["all_tour_purposes"], + "distance_bin": [0], + "tour_count": [5.0], + } + ) + summary_run = create_summary_run( + label="Base", + run_key="base", + summaries_by_mode={ + "weighted": { + "tour_distance_by_tour_purpose": distance, + "average_mandatory_tour_distance_by_purpose_and_geography": average_mandatory, + "average_nonmandatory_tour_distance_by_purpose_and_geography": average_nonmandatory, + }, + "unweighted": { + "tour_distance_by_tour_purpose": distance, + "average_mandatory_tour_distance_by_purpose_and_geography": average_mandatory, + "average_nonmandatory_tour_distance_by_purpose_and_geography": average_nonmandatory, + }, + }, + ) + state = DashboardState( + summary_runs=[summary_run], + weighting_modes=config.weighting_modes, + ) + page = TourDistancePage(state, config) + page.refresh(force=True) + + selector_ids = [ + "geography_level", + "geography", + ] + selector_widgets = _export_selector_widgets(page) + states, aliases = resolve_export_section_states( + page, + page_def=TourDistancePage.definition, + part_def=type("Part", (), {"part_id": "tour_distance_averages"})(), + active_selector_ids=selector_ids, + selector_widgets=selector_widgets, + selector_metadata_by_id=_export_selector_metadata( + selector_widgets, + selector_ids, + ), + ) + + state_pairs = {(state["geography_level"], state["geography"]) for state in states} + assert ("District", "North") in state_pairs + assert ("District", "South") in state_pairs + assert ("County", "Wake") in state_pairs + assert ("District", "Wake") not in state_pairs + assert ("County", "North") not in state_pairs + assert aliases == {} + + +def test_regional_helpers_rename_blank_origin() -> None: + matrix = pl.DataFrame( + { + "": ["A", "Total"], + "A": [1.0, 2.0], + "Total": [3.0, 4.0], + } + ) + + normalized = normalize_flow_matrix(matrix, include_totals=False) + + assert normalized.to_dicts() == [{"Origin": "A", "A": 1.0}] + + +def test_regional_flow_heatmap_labels_cells_with_matrix_values() -> None: + matrix = pl.DataFrame( + { + "": ["A", "B"], + "A": [1200.0, 30.0], + "B": [45.0, 6789.0], + } + ) + + heatmap = flow_heatmap( + [("Run", matrix)], + include_totals=False, + title="District flows", + ) + plot = heatmap.objects[0][0] + trace = plot.object.data[0] + + assert trace.text == (["1,200", "45"], ["30", "6,789"]) + assert trace.texttemplate == "%{text}" + + +def test_regional_flow_comparison_aligns_observed_and_modeled_flows() -> None: + observed = [ + ( + "Base", + pl.DataFrame( + { + "": ["A", "B", "Total"], + "A": [10.0, 3.0, 13.0], + "B": [5.0, 20.0, 25.0], + "Total": [15.0, 23.0, 38.0], + } + ), ) - assert all( - summary_id in SUMMARY_SPEC_BY_ID - for summary_id in definition.required_summary_ids + ] + modeled = [ + ( + "Base", + pl.DataFrame( + { + "origin_geography_type": ["home_county"] * 4, + "origin_geography_id": ["A", "A", "B", "B"], + "destination_geography_type": ["home_county"] * 4, + "destination_geography_id": ["A", "B", "A", "B"], + "commuter_count": [12.0, 4.0, 3.0, 18.0], + } + ), ) + ] + + comparison = flow_comparison_data( + observed, + modeled, + geography_type="home_county", + include_totals=False, + ) + + assert comparison[0][1].select( + "Origin", + "Destination", + "observed", + "modeled", + "difference", + "percent_difference", + ).to_dicts() == [ + { + "Origin": "A", + "Destination": "A", + "observed": 10.0, + "modeled": 12.0, + "difference": 2.0, + "percent_difference": 20.0, + }, + { + "Origin": "A", + "Destination": "B", + "observed": 5.0, + "modeled": 4.0, + "difference": -1.0, + "percent_difference": -20.0, + }, + { + "Origin": "B", + "Destination": "A", + "observed": 3.0, + "modeled": 3.0, + "difference": 0.0, + "percent_difference": 0.0, + }, + { + "Origin": "B", + "Destination": "B", + "observed": 20.0, + "modeled": 18.0, + "difference": -2.0, + "percent_difference": -10.0, + }, + ] + + +def test_regional_validation_page_compares_county_flows_to_commuting_flows( + tmp_path: Path, +) -> None: + config = _write_config(tmp_path) + observed = pl.DataFrame( + { + "": ["A", "B", "Total"], + "A": [10.0, 3.0, 13.0], + "B": [5.0, 20.0, 25.0], + "Total": [15.0, 23.0, 38.0], + } + ) + modeled = pl.DataFrame( + { + "origin_geography_type": ["home_county"] * 4, + "origin_geography_id": ["A", "A", "B", "B"], + "destination_geography_type": ["home_county"] * 4, + "destination_geography_id": ["A", "B", "A", "B"], + "commuter_count": [12.0, 4.0, 3.0, 18.0], + } + ) + summary_run = create_summary_run( + label="Base", + run_key="base", + summaries_by_mode={ + "weighted": { + "county_flows_joja_validation_summary": observed, + "commuting_flows": modeled, + }, + "unweighted": { + "county_flows_joja_validation_summary": observed, + "commuting_flows": modeled, + }, + }, + ) + state = DashboardState( + summary_runs=[summary_run], + weighting_modes=config.weighting_modes, + ) + + page = RegionalValidationPage(state, config) + page.refresh(force=True) + + assert list(page.flow_matrix_sel.options) == ["County flows"] + assert list(page.comparison_metric_sel.options) == [ + "Observed", + "Difference", + "Percent Difference", + "Absolute Percent Difference", + "Modeled", + ] + chart = page.render_flow_section() + tabs = chart.objects[0] + plot = tabs.objects[0][0] + + assert plot.object.layout.title.text == "Observed County flows" + assert plot.object.data[0].z == ([10.0, 5.0], [3.0, 20.0]) + + page.comparison_metric_sel.value = "Difference" + chart = page.render_flow_section() + tabs = chart.objects[0] + plot = tabs.objects[0][0] + + assert plot.object.layout.title.text == "Difference County flows" + assert plot.object.data[0].z == ([2.0, -1.0], [0.0, -2.0]) def test_resolve_page_definitions_defaults_to_default_pages_when_unconfigured( @@ -132,7 +2054,7 @@ def test_resolve_page_definitions_defaults_to_default_pages_when_unconfigured( ) -> None: config = _write_config(tmp_path, dashboard_pages=None) - resolved_pages = resolve_page_definitions(config) + resolved_pages = resolve_live_page_definitions(config) assert [page.title for page in resolved_pages] == EXPECTED_DEFAULT_LEAF_PAGE_TITLES @@ -150,7 +2072,7 @@ def test_page_selectors_render_with_widget_label_instead_of_duplicate_markdown( selector_row = page.view.objects[1] assert isinstance(selector_row, pn.Row) - assert selector_row.objects == [page.tour_purpose_sel] + assert selector_row.objects == [page.tour_purpose_sel, page.hide_drive_alone] assert page.tour_purpose_sel.name == "Tour Purpose" assert "page-selector-widget" in page.tour_purpose_sel.css_classes assert PAGE_SELECTOR_STYLESHEET in page.tour_purpose_sel.stylesheets @@ -164,7 +2086,7 @@ def test_resolve_page_definitions_respects_configured_page_order_and_subset( dashboard_pages=["trip_mode", "overview", "joint_travel"], ) - resolved_pages = resolve_page_definitions(config) + resolved_pages = resolve_live_page_definitions(config) assert [page.page_id for page in resolved_pages] == [ "trip_mode", @@ -185,7 +2107,7 @@ def test_resolve_page_definitions_supports_nested_group_child_selection( ], ) - resolved_pages = resolve_page_definitions(config) + resolved_pages = resolve_live_page_definitions(config) assert [page.page_id for page in resolved_pages] == [ "overview", @@ -208,7 +2130,9 @@ def test_enabled_prepared_data_mode_tracks_optional_and_required_page_sets( assert enabled_prepared_data_mode(raw_demo_config) == "required" -def test_data_requirements_for_pages_aggregates_summary_and_prepared_dependencies() -> None: +def test_data_requirements_for_pages_aggregates_summary_and_prepared_dependencies() -> ( + None +): overview = page_definition_by_id("overview") raw_trip_demo = page_definition_by_id("raw_trip_demo") @@ -219,15 +2143,31 @@ def test_data_requirements_for_pages_aggregates_summary_and_prepared_dependencie assert requirements.required_summary_ids == overview.required_summary_ids +def test_data_requirements_for_pages_tracks_optional_summary_dependencies() -> None: + vmt = page_definition_by_id("vmt") + regional = page_definition_by_id("regional_validation") + + requirements = data_requirements_for_pages([vmt, regional]) + + assert PERSONAL_AUTO_VMT_SUMMARY_ID in requirements.required_summary_ids + assert NON_MOTORIZED_VMT_SUMMARY_ID in requirements.required_summary_ids + assert "commercial_vmt_totals" not in requirements.required_summary_ids + assert "commercial_vmt_totals" not in requirements.optional_summary_ids + assert "auto_vmt_validation_summary" not in requirements.optional_summary_ids + assert "county_flows_validation_summary" in requirements.optional_summary_ids + assert "commuting_flows" in requirements.optional_summary_ids + assert "auto_vmt_validation_summary" not in requirements.summary_ids_for_pruning + + def test_resolve_page_definitions_rejects_unknown_configured_page_ids( tmp_path: Path, ) -> None: config = _write_config(tmp_path, dashboard_pages=["overview", "unknown_page"]) with pytest.raises( - ValueError, match="Unsupported visualizer.dashboard_pages entries" + ValueError, match="Unsupported dashboard.live.pages entries" ): - resolve_page_definitions(config) + resolve_live_page_definitions(config) def test_resolve_page_definitions_rejects_duplicate_configured_page_ids( @@ -235,7 +2175,7 @@ def test_resolve_page_definitions_rejects_duplicate_configured_page_ids( ) -> None: with pytest.raises( ValueError, - match="visualizer.dashboard_pages contains duplicate page id 'overview'", + match="dashboard.live.pages contains duplicate page id 'overview'", ): _write_config(tmp_path, dashboard_pages=["overview", "overview"]) @@ -244,9 +2184,15 @@ def test_build_dashboard_uses_expected_default_page_order(tmp_path: Path) -> Non config = _write_config(tmp_path) template = build_dashboard([], config, summary_runs=[_full_summary_run()]) - assert [page.name for page in template._dashboard_pages] == EXPECTED_DEFAULT_PAGE_TITLES - assert [page.page_id() for page in template._dashboard_pages] == EXPECTED_DEFAULT_PAGE_IDS - assert [page.page_id() for page in template._dashboard_leaf_pages] == EXPECTED_DEFAULT_LEAF_PAGE_IDS + assert [ + page.name for page in template._dashboard_pages + ] == EXPECTED_DEFAULT_PAGE_TITLES + assert [ + page.page_id() for page in template._dashboard_pages + ] == EXPECTED_DEFAULT_PAGE_IDS + assert [ + page.page_id() for page in template._dashboard_leaf_pages + ] == EXPECTED_DEFAULT_LEAF_PAGE_IDS def test_build_dashboard_sidebar_uses_shared_run_legend_markup(tmp_path: Path) -> None: @@ -272,7 +2218,9 @@ def test_build_dashboard_can_refresh_every_default_page_from_precomputed_summari for index, page in enumerate(pages): tabs.active = index assert page.view is not None - assert state.page_state["Overview"]["last_rendered_state"] == state.global_state_key() + assert ( + state.page_state["overview"]["last_rendered_state"] == state.global_state_key() + ) leaf_pages = {page.page_id(): page for page in template._dashboard_leaf_pages} assert [ @@ -280,25 +2228,37 @@ def test_build_dashboard_can_refresh_every_default_page_from_precomputed_summari for selector in leaf_pages["trip_stop_distance"].registered_selectors ] == [ "tour_purpose", + "trip_stop_distance_min", + "trip_stop_distance_max", ] assert leaf_pages["daily_activity_pattern"].person_type_sel.options == [ - "Total", + "All Person Types", "worker", ] - assert leaf_pages["joint_travel"].hhsize_sel.options == ["All", "2", "3"] - assert leaf_pages["tour_time"].purpose_sel.options == ["Total", "work"] - assert leaf_pages["tour_mode"].purpose_sel.options == ["Total", "work"] - assert leaf_pages["tour_stop_frequency"].purpose_sel.options == [ + assert leaf_pages["joint_travel"].hhsize_sel.options == [ "All", + "2", + "3", + "4", + "5+", + ] + assert leaf_pages["tour_time"].purpose_sel.options == ["All Tour Purposes", "work"] + assert leaf_pages["tour_mode"].purpose_sel.options == ["All Tour Purposes", "work"] + assert leaf_pages["tour_stop_frequency"].purpose_sel.options == [ + "All Tour Purposes", "eatout", "social", ] assert leaf_pages["trip_stop_time"].tour_purpose_sel.options == [ - "Total", + "All Tour Purposes", + "eatout", + "social", + ] + assert leaf_pages["trip_mode"].tour_purpose_sel.options == [ + "All Tour Purposes", "eatout", "social", ] - assert leaf_pages["trip_mode"].tour_purpose_sel.options == ["All", "eatout", "social"] def test_build_dashboard_loads_prepared_runs_for_optional_default_pages_when_available( @@ -312,13 +2272,17 @@ def test_build_dashboard_loads_prepared_runs_for_optional_default_pages_when_ava ) assert template._dashboard_state.prepared_run_availability == "loaded" - weighted_runs = template._dashboard_state.get_prepared_runs_if_loaded(weighted=True) + weighted_runs = template._dashboard_state.get_prepared_runs_if_loaded( + weighting_mode="weighted" + ) assert weighted_runs is not None assert weighted_runs[0][0] == "Base" -def test_build_dashboard_loads_prepared_runs_when_demo_page_is_enabled(tmp_path: Path) -> None: +def test_build_dashboard_loads_prepared_runs_when_demo_page_is_enabled( + tmp_path: Path, +) -> None: config = _write_config(tmp_path, dashboard_pages=["raw_trip_demo"]) template = build_dashboard([("Base", _raw_trip_run())], config) page = template._dashboard_pages[0] @@ -336,7 +2300,9 @@ def test_build_dashboard_shows_unavailable_card_when_demo_page_has_no_prepared_r page = template._dashboard_pages[0] assert template._dashboard_state.prepared_run_availability == "unavailable" - assert any(getattr(obj, "title", "") == "Data Not Available" for obj in page.view.objects) + assert any( + getattr(obj, "title", "") == "Data Not Available" for obj in page.view.objects + ) def test_dashboard_state_exposes_summary_first_accessors(tmp_path: Path) -> None: @@ -353,7 +2319,7 @@ def test_dashboard_state_exposes_summary_first_accessors(tmp_path: Path) -> None assert state.has_summary_table_set("missing_summary", "weighted") is False assert totals[0][0] == "Base" assert totals[0][1]["person_count"][0] == 100.0 - assert state.get_prepared_runs_if_loaded(weighted=True) is None + assert state.get_prepared_runs_if_loaded(weighting_mode="weighted") is None assert state.prepared_run_availability == "not_requested" @@ -383,8 +2349,12 @@ def test_dashboard_state_prepared_run_provider_supports_loaded_and_unavailable_m prepared_run_provider=DashboardPreparedRunProvider.unavailable(), ) - weighted_runs = loaded_state.get_prepared_runs_if_loaded(weighted=True) - unweighted_runs = loaded_state.get_prepared_runs_if_loaded(weighted=False) + weighted_runs = loaded_state.get_prepared_runs_if_loaded( + weighting_mode="weighted" + ) + unweighted_runs = loaded_state.get_prepared_runs_if_loaded( + weighting_mode="unweighted" + ) assert loaded_state.prepared_run_availability == "loaded" assert weighted_runs is not None @@ -393,7 +2363,10 @@ def test_dashboard_state_prepared_run_provider_supports_loaded_and_unavailable_m assert unweighted_runs is not None assert unweighted_runs[0][1].hh["finalweight"][0] == 1.0 assert unavailable_state.prepared_run_availability == "unavailable" - assert unavailable_state.get_prepared_runs_if_loaded(weighted=True) is None + assert ( + unavailable_state.get_prepared_runs_if_loaded(weighting_mode="weighted") + is None + ) def test_build_dashboard_switches_tabs_and_refreshes_only_the_active_page( @@ -402,41 +2375,46 @@ def test_build_dashboard_switches_tabs_and_refreshes_only_the_active_page( config = _write_config(tmp_path) template = build_dashboard([], config, summary_runs=[_full_summary_run()]) state = template._dashboard_state - pages = template._dashboard_pages tabs = template.main[0] assert state.active_tab == 0 - assert state.page_state["Overview"]["last_rendered_state"] == state.global_state_key() - assert state.page_state["Daily Activity Pattern"].get("last_rendered_state") is None + assert ( + state.page_state["overview"]["last_rendered_state"] == state.global_state_key() + ) + assert state.page_state["daily_activity_pattern"].get("last_rendered_state") is None tabs.active = 1 assert state.active_tab == 1 - assert state.page_state["Overview"]["last_rendered_state"] == state.global_state_key() assert ( - state.page_state["Daily Activity Pattern"]["last_rendered_state"] + state.page_state["overview"]["last_rendered_state"] == state.global_state_key() + ) + assert ( + state.page_state["daily_activity_pattern"]["last_rendered_state"] == state.global_state_key() ) - assert state.page_state["Tour Purpose"].get("last_rendered_state") is None + assert state.page_state["tour_purpose"].get("last_rendered_state") is None state.weight_mode = "Unweighted" - assert state.page_state["Overview"]["last_rendered_state"] is None + assert state.page_state["overview"]["last_rendered_state"] is None assert ( - state.page_state["Daily Activity Pattern"]["last_rendered_state"] + state.page_state["daily_activity_pattern"]["last_rendered_state"] == state.global_state_key() ) - assert state.page_state["Tour Purpose"].get("last_rendered_state") is None + assert state.page_state["tour_purpose"].get("last_rendered_state") is None state.value_mode = "Count" assert ( - state.page_state["Daily Activity Pattern"]["last_rendered_state"] + state.page_state["daily_activity_pattern"]["last_rendered_state"] == state.global_state_key() ) - assert state.page_state["Overview"].get("last_rendered_state") is None - assert state.page_state["Mandatory Location Choice"].get("last_rendered_state") is None - assert state.page_state["Tour Purpose"].get("last_rendered_state") is None + assert state.page_state["overview"].get("last_rendered_state") is None + assert ( + state.page_state["mandatory_location_choice"].get("last_rendered_state") is None + ) + assert state.page_state["tour_purpose"].get("last_rendered_state") is None def test_build_dashboard_preserves_widget_state_across_tab_switches( @@ -452,7 +2430,10 @@ def test_build_dashboard_preserves_widget_state_across_tab_switches( ) tabs.active = 1 - assert daily_activity_pattern_page.person_type_sel.options == ["Total", "worker"] + assert daily_activity_pattern_page.person_type_sel.options == [ + "All Person Types", + "worker", + ] daily_activity_pattern_page.person_type_sel.value = "worker" tabs.active = 0 @@ -461,7 +2442,89 @@ def test_build_dashboard_preserves_widget_state_across_tab_switches( assert daily_activity_pattern_page.person_type_sel.value == "worker" -def test_dashboard_page_cache_helpers_reuse_summary_and_filtered_view_results( +def test_build_dashboard_preserves_individual_choices_person_type_across_tab_switches( + tmp_path: Path, +) -> None: + config = _write_config(tmp_path) + summary_run = _full_summary_run() + worker_summary_run = type(summary_run)( + label=summary_run.label, + run_key=summary_run.run_key, + summaries_by_mode={ + mode: { + **summary_run.summaries_by_mode[mode], + "license_holding_status_distribution": pl.DataFrame( + { + "person_type": ["all_person_types", "worker"], + "license_holding_status": ["has_license", "has_license"], + "person_count": [80.0, 40.0], + "pct": [0.8, 1.0], + } + ), + "bicycle_comfort_level_distribution": pl.DataFrame( + { + "person_type": ["all_person_types", "worker"], + "bicycle_comfort_level": [ + "InterestedButConcerned", + "StrongAndFearless", + ], + "person_count": [50.0, 20.0], + "pct": [0.5, 0.5], + } + ), + "transit_pass_ownership_by_person_type": pl.DataFrame( + { + "person_type": ["all_person_types", "worker"], + "transit_pass_ownership_status": ["no_pass", "has_pass"], + "person_count": [70.0, 15.0], + "pct": [0.7, 0.375], + } + ), + "transit_subsidy_by_person_type": pl.DataFrame( + { + "person_type": ["all_person_types", "worker"], + "transit_subsidy_status": ["none", "full"], + "person_count": [65.0, 10.0], + "pct": [0.65, 0.25], + } + ), + } + for mode in summary_run.summaries_by_mode + }, + source_run_dir=summary_run.source_run_dir, + manifest=summary_run.manifest, + ) + template = build_dashboard([], config, summary_runs=[worker_summary_run]) + top_tabs = template.main[0] + long_term_choices_index = next( + index + for index, page in enumerate(template._dashboard_pages) + if page.page_id() == "long_term_choices" + ) + long_term_choices_page = template._dashboard_pages[long_term_choices_index] + individual_choices_index = next( + index + for index, page in enumerate(long_term_choices_page.pages) + if page.page_id() == "individual_choices" + ) + individual_choices_page = long_term_choices_page.pages[individual_choices_index] + + top_tabs.active = long_term_choices_index + long_term_choices_page.view.active = individual_choices_index + assert individual_choices_page.person_type_sel.options == [ + "All Person Types", + "worker", + ] + + individual_choices_page.person_type_sel.value = "worker" + top_tabs.active = 0 + top_tabs.active = long_term_choices_index + long_term_choices_page.view.active = individual_choices_index + + assert individual_choices_page.person_type_sel.value == "worker" + + +def test_dashboard_page_cache_helpers_reuse_summary_and_query_results( tmp_path: Path, ) -> None: config = _write_config(tmp_path) @@ -470,8 +2533,14 @@ def test_dashboard_page_cache_helpers_reuse_summary_and_filtered_view_results( label=summary_run.label, run_key=summary_run.run_key, summaries_by_mode={ - "weighted": {**summary_run.summaries_by_mode["weighted"], "probe_summary": pl.DataFrame({"value": ["summary"]})}, - "unweighted": {**summary_run.summaries_by_mode["unweighted"], "probe_summary": pl.DataFrame({"value": ["summary"]})}, + "weighted": { + **summary_run.summaries_by_mode["weighted"], + "probe_summary": pl.DataFrame({"value": ["summary"]}), + }, + "unweighted": { + **summary_run.summaries_by_mode["unweighted"], + "probe_summary": pl.DataFrame({"value": ["summary"]}), + }, }, source_run_dir=summary_run.source_run_dir, manifest=summary_run.manifest, @@ -480,24 +2549,20 @@ def test_dashboard_page_cache_helpers_reuse_summary_and_filtered_view_results( summary_runs=[probe_summary_run], weighting_modes=config.weighting_modes, ) - call_counts = {"filtered_view": 0} + call_counts = {"query": 0} class CacheProbePage(DashboardPage): def __init__(self) -> None: - super().__init__("Cache Probe", state, config) + super().__init__(state, config) self.view = pn.Column() - def _filtered_view_factory(self) -> dict[str, str]: - call_counts["filtered_view"] += 1 - return {"kind": "filtered_view"} + def _query_factory(self) -> dict[str, str]: + call_counts["query"] += 1 + return {"kind": "query"} def _refresh(self) -> None: - self.summary_value = self.require_summary("probe_summary") - self.filtered_view_value = self.get_filtered_view( - "probe_view", - "default", - factory=self._filtered_view_factory, - ) + self.summary_value = self.data.summary("probe_summary") + self.query_value = self.query(self._query_factory) page = CacheProbePage() @@ -506,10 +2571,10 @@ def _refresh(self) -> None: page.mark_stale() page.refresh_if_needed() - assert call_counts == {"filtered_view": 1} + assert call_counts == {"query": 1} assert page.summary_value[0][1]["value"][0] == "summary" - assert page.filtered_view_value == {"kind": "filtered_view"} - assert state.cache_stats["filtered_view"] == {"hits": 2, "misses": 1} + assert page.query_value == {"kind": "query"} + assert state.cache_stats["page_query"] == {"hits": 2, "misses": 1} def test_skim_pages_render_selector_controls_and_independent_sections( @@ -877,7 +2942,11 @@ def test_skim_pages_render_disaggregated_distribution_plots_when_prepared_runs_a "skimjoin_trip_component_ecdf": pl.DataFrame( { "trip_mode": ["SOV", "SOV", "SOV"], - "component": ["skim_auto_time", "skim_auto_time", "skim_auto_time"], + "component": [ + "skim_auto_time", + "skim_auto_time", + "skim_auto_time", + ], "percentile": [0.0, 0.99, 1.0], "value": [10.0, 14.0, 200.0], "n_valid": [100.0, 100.0, 100.0], @@ -968,9 +3037,9 @@ def test_skim_pages_render_disaggregated_distribution_plots_when_prepared_runs_a trip_page.refresh(force=True) assert isinstance(trip_page._distribution_section.objects[-1], pn.pane.Plotly) - assert tuple(trip_page._distribution_section.objects[-1].object.layout.xaxis.range) == pytest.approx( - (10.0, 200.0) - ) + assert tuple( + trip_page._distribution_section.objects[-1].object.layout.xaxis.range + ) == pytest.approx((10.0, 200.0)) assert trip_page._distribution_section.objects[-1].object.layout.title.text == ( "Trip Distribution - skim_auto_time / All Modes" ) @@ -978,30 +3047,32 @@ def test_skim_pages_render_disaggregated_distribution_plots_when_prepared_runs_a trip_page.trip_min_sel.value = 11.0 trip_page.trip_max_sel.value = 13.0 - assert tuple(trip_page._distribution_section.objects[-1].object.layout.xaxis.range) == pytest.approx( - (11.0, 13.0) - ) + assert tuple( + trip_page._distribution_section.objects[-1].object.layout.xaxis.range + ) == pytest.approx((11.0, 13.0)) trip_page.trip_reset_btn.clicks = trip_page.trip_reset_btn.clicks + 1 - assert tuple(trip_page._distribution_section.objects[-1].object.layout.xaxis.range) == pytest.approx( - (10.0, 200.0) - ) + assert tuple( + trip_page._distribution_section.objects[-1].object.layout.xaxis.range + ) == pytest.approx((10.0, 200.0)) tour_page = TourSkimsPage(state, config) tour_page.refresh(force=True) - outbound_plot = tour_page._distribution_section.objects[1] - inbound_plot = tour_page._distribution_section.objects[3] + outbound_note_view = tour_page._distribution_section.objects[1] + inbound_note_view = tour_page._distribution_section.objects[3] + assert "calculation-note-view" in outbound_note_view.css_classes + assert "calculation-note-view" in inbound_note_view.css_classes + outbound_plot = outbound_note_view.objects[0] + inbound_plot = inbound_note_view.objects[0] assert isinstance(outbound_plot, pn.pane.Plotly) assert isinstance(inbound_plot, pn.pane.Plotly) assert tuple(outbound_plot.object.layout.xaxis.range) == pytest.approx( (10.0, 200.0) ) - assert tuple(inbound_plot.object.layout.xaxis.range) == pytest.approx( - (10.0, 200.0) - ) + assert tuple(inbound_plot.object.layout.xaxis.range) == pytest.approx((10.0, 200.0)) assert outbound_plot.object.layout.title.text == ( "Outbound Tour Distribution - skim_auto_time_outbound / All Modes" ) @@ -1014,23 +3085,17 @@ def test_skim_pages_render_disaggregated_distribution_plots_when_prepared_runs_a tour_page.inbound_min_sel.value = 11.0 tour_page.inbound_max_sel.value = 13.0 - outbound_plot = tour_page._distribution_section.objects[1] - inbound_plot = tour_page._distribution_section.objects[3] - assert tuple(outbound_plot.object.layout.xaxis.range) == pytest.approx( - (11.0, 13.0) - ) - assert tuple(inbound_plot.object.layout.xaxis.range) == pytest.approx( - (11.0, 13.0) - ) + outbound_plot = tour_page._distribution_section.objects[1].objects[0] + inbound_plot = tour_page._distribution_section.objects[3].objects[0] + assert tuple(outbound_plot.object.layout.xaxis.range) == pytest.approx((11.0, 13.0)) + assert tuple(inbound_plot.object.layout.xaxis.range) == pytest.approx((11.0, 13.0)) tour_page.outbound_reset_btn.clicks = tour_page.outbound_reset_btn.clicks + 1 tour_page.inbound_reset_btn.clicks = tour_page.inbound_reset_btn.clicks + 1 - outbound_plot = tour_page._distribution_section.objects[1] - inbound_plot = tour_page._distribution_section.objects[3] + outbound_plot = tour_page._distribution_section.objects[1].objects[0] + inbound_plot = tour_page._distribution_section.objects[3].objects[0] assert tuple(outbound_plot.object.layout.xaxis.range) == pytest.approx( (10.0, 200.0) ) - assert tuple(inbound_plot.object.layout.xaxis.range) == pytest.approx( - (10.0, 200.0) - ) + assert tuple(inbound_plot.object.layout.xaxis.range) == pytest.approx((10.0, 200.0)) diff --git a/tests/test_export_html.py b/tests/test_export_html.py index b7ab2fd..6dee1b6 100644 --- a/tests/test_export_html.py +++ b/tests/test_export_html.py @@ -14,12 +14,13 @@ from _dashboard_expectations import EXPECTED_DEFAULT_LEAF_PAGES, EXPECTED_DEFAULT_PAGES from dashboard.export.html import ( ExportBuildError, + _iter_script_safe_payload_json, build_export_html_document, write_export_html_document, ) from dashboard.export.types import EXPORT_CLIENT_RUNTIME, EXPORT_SCHEMA_VERSION from processor.models import RunData -from processor.summarize.cache import create_summary_run +from processor.summarize.cache_types import create_summary_run from runtime.config import Config @@ -31,7 +32,7 @@ def _write_config( modes_lines: list[str] | None = None, geography_lines: list[str] | None = None, export_html_lines: list[str] | None = None, - visualizer_lines: list[str] | None = None, + display_lines: list[str] | None = None, extra_lines: list[str] | None = None, ) -> Config: weighting_modes = weighting_modes or ["weighted", "unweighted"] @@ -39,45 +40,59 @@ def _write_config( lines = [ 'name: "Test Config"', "runs: []", - "summaries:", - " root: summary_cache", + "root: summary_cache", + "summarize:", " weighting_modes:", ] lines.extend(f" - {mode}" for mode in weighting_modes) + if geography_lines: + lines.append(" geography:") + lines.extend(f" {line}" for line in geography_lines) lines.extend( [ - "visualizer:", - ' dashboard_title: "Test Dashboard"', + "dashboard:", + ' title: "Test Dashboard"', ] ) - if visualizer_lines: - lines.extend(f" {line}" for line in visualizer_lines) if dashboard_pages is ...: dashboard_pages = [page_id for page_id, _ in EXPECTED_DEFAULT_PAGES] if dashboard_pages is not None: - lines.append(" dashboard_pages:") + lines.extend([" live:", " pages:"]) for entry in dashboard_pages: if isinstance(entry, str): - lines.append(f" - {entry}") + lines.append(f" - {entry}") continue if isinstance(entry, dict) and len(entry) == 1: page_id, children = next(iter(entry.items())) - lines.append(f" - {page_id}:") + lines.append(f" - {page_id}:") for child_id in children: - lines.append(f" - {child_id}") + lines.append(f" - {child_id}") continue raise ValueError("dashboard_pages test helper only supports strings or single-key child mappings.") - if export_html_lines: - lines.append(" export_html:") + if export_html_lines is not None: + lines.append(" export:") lines.extend(f" {line}" for line in export_html_lines) + if display_lines: + lines.append("display:") + lines.extend(f" {line}" for line in display_lines) if modes_lines: lines.append("modes:") lines.extend(f" {line}" for line in modes_lines) else: lines.append("modes: {}") - if geography_lines: - lines.append("geography:") - lines.extend(f" {line}" for line in geography_lines) + if export_html_lines is not None: + pipeline_steps = ( + "segment, summarize, dashboard" + if extra_lines and "segment:" in extra_lines + else "summarize, dashboard" + ) + lines.extend( + [ + "pipeline:", + f" steps: [{pipeline_steps}]", + " dashboard_mode: export", + ] + ) if extra_lines: lines.extend(extra_lines) @@ -138,7 +153,8 @@ def _full_summary_run(): "auto_vmt_totals": pl.DataFrame({"auto_vmt": [180.0]}), "auto_ownership_distribution": pl.DataFrame( { - "household_vehicle_count": [0, 1], + "household_size": ["1", "5+"], + "household_vehicle_count": [0, 4], "household_count": [12.0, 18.0], } ), @@ -184,13 +200,6 @@ def _full_summary_run(): "person_count": [5.0, 1.0, 3.0, 0.5, 2.0, 0.5], } ), - "geo_flows": pl.DataFrame( - { - "Home Geography": ["Urban", "Suburban"], - "Work Geography": ["Urban", "Suburban"], - "Workers": [7.0, 4.0], - } - ), "internal_external_worker_by_geography": pl.DataFrame( { "geography_level": ["Urban", "Suburban"], @@ -337,19 +346,6 @@ def _full_summary_run(): "household_percent": [40.0, 60.0, 37.5, 62.5], } ), - "destination_distance": pl.DataFrame( - { - "purpose": ["All NM", "All NM", "eatout", "eatout", "social", "social"], - "distbin": [0, 1, 0, 1, 0, 1], - "freq": [5.0, 7.5, 2.0, 4.0, 3.0, 2.0], - } - ), - "destination_average_distance": pl.DataFrame( - { - "purpose": ["eatout", "social"], - "avg_distance": [3.25, 4.5], - } - ), "tour_time_of_day_by_tour_purpose": pl.DataFrame( { "tour_purpose": [ @@ -404,6 +400,12 @@ def _full_summary_run(): "tour_count": [18.0, 5.0, 10.0, 5.0, 8.0], } ), + "atwork_subtour_frequency_distribution": pl.DataFrame( + { + "atwork_subtour_frequency_category": ["0", "1+"], + "atwork_subtour_count": [6.0, 4.0], + } + ), "stop_destination_purpose_by_tour_purpose": pl.DataFrame( { "tour_purpose": [ @@ -670,8 +672,7 @@ def test_export_html_config_segmentation_defaults_to_live_dashboard_settings( config = _write_config( tmp_path, extra_lines=[ - "segmentation:", - " enabled: true", + "segment:", " dashboard:", " segmentation_type: signup_platform", " visibility: segments_only", @@ -695,9 +696,7 @@ def test_export_html_config_segmentation_defaults_to_live_dashboard_settings( " label: Male", " values: [1]", ], - export_html_lines=[ - "enabled: true", - ], + export_html_lines=[], ) assert config.export_html.dashboard.segmentation_type == "signup_platform" @@ -710,8 +709,7 @@ def test_export_html_config_supports_segmentation_overrides( config = _write_config( tmp_path, extra_lines=[ - "segmentation:", - " enabled: true", + "segment:", " dashboard:", " segmentation_type: signup_platform", " visibility: segments_only", @@ -765,7 +763,7 @@ def test_export_html_config_supports_new_summaries_and_visualizer_sections( assert config.summary_root.endswith("summary_cache") assert _configured_page_ids(config) == ["overview", "trip_mode"] assert config.export_html.enabled is True - assert list(config.export_html.pages) == ["trip_mode"] + assert list(config.export_html.pages) == ["trip_mode", "overview"] assert config.export_html.pages_configured is True @@ -775,7 +773,6 @@ def test_export_html_config_resolves_output_path_relative_to_root( config = _write_config( tmp_path, export_html_lines=[ - "enabled: true", "output_path: exports/dashboard.html", ], ) @@ -791,9 +788,7 @@ def test_export_html_enabled_without_pages_uses_all_dashboard_states_and_all_sel config = _write_config( tmp_path, dashboard_pages=["overview", "trip_mode"], - export_html_lines=[ - "enabled: true", - ], + export_html_lines=[], ) assert config.export_html.enabled is True @@ -914,26 +909,14 @@ def test_config_allows_missing_dashboard_pages(tmp_path: Path) -> None: assert config.dashboard_pages is None -def test_config_defaults_when_summaries_and_visualizer_sections_are_absent( +def test_config_defaults_when_optional_sections_are_absent( tmp_path: Path, ) -> None: config_path = tmp_path / "config.yaml" config_path.write_text( "\n".join( [ - 'name: "Legacy Layout"', - 'dashboard_title: "Ignored Legacy Title"', - "run_colors:", - ' - "#111111"', - "outputs:", - " summary_root: ignored_summary_cache", - " weighting_modes:", - " - weighted", - " export_html:", - " dashboard:", - " weighting: all", - "dashboard_pages:", - " - raw_trip_demo", + 'name: "Minimal Layout"', "runs: []", ] ), @@ -946,7 +929,7 @@ def test_config_defaults_when_summaries_and_visualizer_sections_are_absent( (tmp_path / "artifacts" / "summary_cache").resolve() ) assert config.weighting_modes == ["weighted", "unweighted"] - assert config.dashboard_title == "Ignored Legacy Title" + assert config.dashboard_title == "ActivitySim Visualizer" assert config.dashboard_pages is None assert config.run_colors == [ "#1f77b4", @@ -961,55 +944,9 @@ def test_config_defaults_when_summaries_and_visualizer_sections_are_absent( assert config.export_html.dashboard.weighting == ["weighted", "unweighted"] assert config.export_html.dashboard.values == ["percent", "count"] assert config.export_html.pages == {} - - -def test_config_prefers_visualizer_dashboard_title_over_legacy_top_level_title( - tmp_path: Path, -) -> None: - config_path = tmp_path / "config.yaml" - config_path.write_text( - "\n".join( - [ - 'name: "Dashboard Title Precedence"', - 'dashboard_title: "Legacy Dashboard Title"', - "runs: []", - "visualizer:", - ' dashboard_title: "Visualizer Dashboard Title"', - ] - ), - encoding="utf-8", - ) - - config = Config.from_yaml(config_path) - - assert config.dashboard_title == "Visualizer Dashboard Title" - - -def test_config_ignores_flat_export_html_dashboard_aliases(tmp_path: Path) -> None: - config_path = tmp_path / "config.yaml" - config_path.write_text( - "\n".join( - [ - 'name: "Flat Export Legacy"', - "runs: []", - "visualizer:", - " export_html:", - " weighting: all", - " values: all", - ] - ), - encoding="utf-8", - ) - - config = Config.from_yaml(config_path) - - assert config.export_html.dashboard.weighting == ["weighted", "unweighted"] - assert config.export_html.dashboard.values == ["percent", "count"] - - def test_export_html_config_rejects_invalid_or_empty_values(tmp_path: Path) -> None: with pytest.raises( - ValueError, match="Unsupported visualizer.export_html.dashboard.weighting" + ValueError, match="Unsupported dashboard.export.dashboard.weighting" ): _write_config( tmp_path / "invalid", @@ -1023,14 +960,13 @@ def test_export_html_config_rejects_invalid_or_empty_values(tmp_path: Path) -> N with pytest.raises( ValueError, - match="visualizer.export_html.dashboard.segmentation_type must name one configured segmentation definition", + match="dashboard.export.dashboard.segmentation_type must name one configured segment definition", ): _write_config( tmp_path / "invalid_export_segmentation_type", extra_lines=[ - "segmentation:", - " enabled: true", - " definitions:", + "segment:", + " definitions:", " signup_platform:", " source:", " type: prepared_column", @@ -1049,14 +985,13 @@ def test_export_html_config_rejects_invalid_or_empty_values(tmp_path: Path) -> N with pytest.raises( ValueError, - match="visualizer.export_html.dashboard.segmentation_visibility must be one of full_only, segments_only, or full_and_segments", + match="dashboard.export.dashboard.segmentation_visibility must be one of full_only, segments_only, or full_and_segments", ): _write_config( tmp_path / "invalid_export_segmentation_visibility", extra_lines=[ - "segmentation:", - " enabled: true", - " definitions:", + "segment:", + " definitions:", " signup_platform:", " source:", " type: prepared_column", @@ -1077,7 +1012,7 @@ def test_export_html_config_rejects_invalid_or_empty_values(tmp_path: Path) -> N def test_config_rejects_duplicate_dashboard_pages(tmp_path: Path) -> None: with pytest.raises( ValueError, - match="visualizer.dashboard_pages contains duplicate page id", + match="dashboard.live.pages contains duplicate page id", ): _write_config( tmp_path, @@ -1086,7 +1021,7 @@ def test_config_rejects_duplicate_dashboard_pages(tmp_path: Path) -> None: with pytest.raises( ValueError, - match="visualizer.export_html.dashboard.values resolved to no values", + match="dashboard.export.dashboard.values resolved to no values", ): _write_config( tmp_path / "empty", @@ -1098,7 +1033,7 @@ def test_config_rejects_duplicate_dashboard_pages(tmp_path: Path) -> None: with pytest.raises( ValueError, - match="visualizer\\.export_html\\.pages\\.trip_summaries(\\.children)?\\.trip_mode\\.tour_purpose resolved to no values", + match="dashboard\\.export\\.pages\\.trip_summaries(\\.children)?\\.trip_mode\\.tour_purpose resolved to no values", ): _write_config( tmp_path / "empty_page_values", @@ -1148,6 +1083,10 @@ def _flatten_page_descriptors(pages: list[dict]) -> dict[str, dict]: return by_id +def _selector_by_id(page: dict, selector_id: str) -> dict: + return next(selector for selector in page["selectors"] if selector["id"] == selector_id) + + def _walk_nodes(node: dict) -> list[dict]: if node.get("kind") == "page": return _walk_nodes(node["content"]) @@ -1182,15 +1121,11 @@ def _plot_node_by_title(node: dict, title: str) -> dict: ) +@pytest.mark.full_export def test_build_export_html_document_serializes_dashboard_states_and_pages( - tmp_path: Path, + representative_full_export_html: str, ) -> None: - config = _write_config(tmp_path) - html = build_export_html_document( - [], - config, - summary_runs=[_full_summary_run()], - ) + html = representative_full_export_html payload = _extract_payload(html) assert payload["schema_version"] == EXPORT_SCHEMA_VERSION @@ -1213,9 +1148,13 @@ def test_build_export_html_document_serializes_dashboard_states_and_pages( assert payload["client_runtime"] == EXPORT_CLIENT_RUNTIME enabled_selectors = payload["page_export_support"]["enabled_page_selectors"] assert {"page_id": "daily_activity_pattern", "selector_id": "person_type"} in enabled_selectors + assert {"page_id": "vehicle_ownership_type", "selector_id": "household_size"} in enabled_selectors assert {"page_id": "joint_travel", "selector_id": "household_size"} in enabled_selectors + assert {"page_id": "joint_travel", "selector_id": "hide_no_joint_tours"} in enabled_selectors assert {"page_id": "tour_stop_frequency", "selector_id": "tour_purpose"} in enabled_selectors assert {"page_id": "trip_mode", "selector_id": "tour_purpose"} in enabled_selectors + assert {"page_id": "tour_mode", "selector_id": "hide_drive_alone"} in enabled_selectors + assert {"page_id": "trip_mode", "selector_id": "hide_drive_alone"} in enabled_selectors assert sorted(payload["states"]) == [ "Unweighted||Count", "Unweighted||Percent", @@ -1239,6 +1178,25 @@ def test_build_export_html_document_serializes_dashboard_states_and_pages( "worker", ] assert page_defs["daily_activity_pattern"]["selectors"][0]["export_enabled"] is True + vehicle_household_size = _selector_by_id( + page_defs["vehicle_ownership_type"], "household_size" + ) + assert vehicle_household_size["request_mode"] == "all" + assert vehicle_household_size["resolved_values"] == [ + "All", + "1", + "2", + "3", + "4", + "5+", + ] + assert vehicle_household_size["export_enabled"] is True + joint_travel_hide_no_joint_tours = _selector_by_id( + page_defs["joint_travel"], "hide_no_joint_tours" + ) + assert joint_travel_hide_no_joint_tours["request_mode"] == "all" + assert joint_travel_hide_no_joint_tours["resolved_values"] == ["False", "True"] + assert joint_travel_hide_no_joint_tours["export_enabled"] is True assert page_defs["tour_time"]["selectors"][0]["id"] == "tour_purpose" assert page_defs["tour_time"]["selectors"][0]["request_mode"] == "all" assert page_defs["tour_time"]["selectors"][0]["resolved_values"] == [ @@ -1246,13 +1204,17 @@ def test_build_export_html_document_serializes_dashboard_states_and_pages( "work", ] assert page_defs["tour_time"]["selectors"][0]["export_enabled"] is True - assert page_defs["tour_mode"]["selectors"][0]["id"] == "tour_purpose" - assert page_defs["tour_mode"]["selectors"][0]["request_mode"] == "all" - assert page_defs["tour_mode"]["selectors"][0]["resolved_values"] == [ + tour_mode_purpose = _selector_by_id(page_defs["tour_mode"], "tour_purpose") + tour_mode_hide_drive_alone = _selector_by_id(page_defs["tour_mode"], "hide_drive_alone") + assert tour_mode_purpose["request_mode"] == "all" + assert tour_mode_purpose["resolved_values"] == [ "All Tour Purposes", "work", ] - assert page_defs["tour_mode"]["selectors"][0]["export_enabled"] is True + assert tour_mode_purpose["export_enabled"] is True + assert tour_mode_hide_drive_alone["request_mode"] == "all" + assert tour_mode_hide_drive_alone["resolved_values"] == ["False", "True"] + assert tour_mode_hide_drive_alone["export_enabled"] is True assert page_defs["tour_stop_frequency"]["selectors"][0]["id"] == "tour_purpose" assert page_defs["tour_stop_frequency"]["selectors"][0]["request_mode"] == "all" assert page_defs["tour_stop_frequency"]["selectors"][0]["resolved_values"] == [ @@ -1269,14 +1231,18 @@ def test_build_export_html_document_serializes_dashboard_states_and_pages( "social", ] assert page_defs["trip_stop_time"]["selectors"][0]["export_enabled"] is True - assert page_defs["trip_mode"]["selectors"][0]["id"] == "tour_purpose" - assert page_defs["trip_mode"]["selectors"][0]["request_mode"] == "all" - assert page_defs["trip_mode"]["selectors"][0]["resolved_values"] == [ + trip_mode_purpose = _selector_by_id(page_defs["trip_mode"], "tour_purpose") + trip_mode_hide_drive_alone = _selector_by_id(page_defs["trip_mode"], "hide_drive_alone") + assert trip_mode_purpose["request_mode"] == "all" + assert trip_mode_purpose["resolved_values"] == [ "All Tour Purposes", "eatout", "social", ] - assert page_defs["trip_mode"]["selectors"][0]["export_enabled"] is True + assert trip_mode_purpose["export_enabled"] is True + assert trip_mode_hide_drive_alone["request_mode"] == "all" + assert trip_mode_hide_drive_alone["resolved_values"] == ["False", "True"] + assert trip_mode_hide_drive_alone["export_enabled"] is True weighted_percent = payload["states"]["Weighted||Percent"] overview = weighted_percent["overview"] assert overview["kind"] == "page" @@ -1287,6 +1253,30 @@ def test_build_export_html_document_serializes_dashboard_states_and_pages( '["All Person Types"]', '["worker"]', ] + vehicle_ownership_type = weighted_percent["vehicle_ownership_type"] + assert vehicle_ownership_type["kind"] == "page" + assert _region_nodes(vehicle_ownership_type)["vehicle_ownership_summary"][ + "selector_ids" + ] == ["household_size"] + assert sorted( + _region_nodes(vehicle_ownership_type)["vehicle_ownership_summary"]["variants"] + ) == [ + '["1"]', + '["2"]', + '["3"]', + '["4"]', + '["5+"]', + '["All"]', + ] + joint_travel = weighted_percent["joint_travel"] + assert joint_travel["kind"] == "page" + assert _region_nodes(joint_travel)["joint_travel_frequency"]["selector_ids"] == [ + "hide_no_joint_tours", + ] + assert sorted(_region_nodes(joint_travel)["joint_travel_frequency"]["variants"]) == [ + '["False"]', + '["True"]', + ] tour_time = weighted_percent["tour_time"] assert tour_time["kind"] == "page" assert _region_nodes(tour_time)["tour_time_body"]["selector_ids"] == ["tour_purpose"] @@ -1298,10 +1288,13 @@ def test_build_export_html_document_serializes_dashboard_states_and_pages( assert tour_mode["kind"] == "page" assert _region_nodes(tour_mode)["tour_mode_modes"]["selector_ids"] == [ "tour_purpose", + "hide_drive_alone", ] assert sorted(_region_nodes(tour_mode)["tour_mode_modes"]["variants"]) == [ - '["All Tour Purposes"]', - '["work"]', + '["All Tour Purposes","False"]', + '["All Tour Purposes","True"]', + '["work","False"]', + '["work","True"]', ] tour_stop_frequency = weighted_percent["tour_stop_frequency"] assert tour_stop_frequency["kind"] == "page" @@ -1325,11 +1318,17 @@ def test_build_export_html_document_serializes_dashboard_states_and_pages( ] trip_mode = weighted_percent["trip_mode"] assert trip_mode["kind"] == "page" - assert _region_nodes(trip_mode)["trip_summary_mode_body"]["selector_ids"] == ["tour_purpose"] + assert _region_nodes(trip_mode)["trip_summary_mode_body"]["selector_ids"] == [ + "tour_purpose", + "hide_drive_alone", + ] assert sorted(_region_nodes(trip_mode)["trip_summary_mode_body"]["variants"]) == [ - '["All Tour Purposes"]', - '["eatout"]', - '["social"]', + '["All Tour Purposes","False"]', + '["All Tour Purposes","True"]', + '["eatout","False"]', + '["eatout","True"]', + '["social","False"]', + '["social","True"]', ] widget_nodes = [ node for node in _walk_nodes(daily_activity_pattern) if node.get("kind") == "widget" @@ -1343,7 +1342,7 @@ def test_build_export_html_document_serializes_dashboard_states_and_pages( ) -def test_build_export_html_document_respects_configured_dashboard_page_subset_and_order( +def test_build_export_html_document_applies_overrides_without_narrowing_live_pages( tmp_path: Path, ) -> None: config = _write_config( @@ -1454,10 +1453,10 @@ def test_build_export_html_document_keeps_summary_safe_skims_content_and_hides_p ) -> None: config = _write_config( tmp_path, - dashboard_pages=["skims"], + dashboard_pages=["skim_summaries"], export_html_lines=[ "pages:", - " skims: {}", + " skim_summaries: {}", ], ) @@ -1469,7 +1468,7 @@ def test_build_export_html_document_keeps_summary_safe_skims_content_and_hides_p ) assert [(page["id"], page["title"]) for page in payload["pages"]] == [ - ("skims", "Skim Summaries") + ("skim_summaries", "Skim Summaries") ] assert not any( node.get("selector_id") in {"trip_min", "trip_max", "tour_min", "tour_max"} @@ -1489,16 +1488,21 @@ def test_build_export_html_document_validates_page_selector_requests_against_reg ) -> None: config = _write_config( tmp_path, + dashboard_pages=[ + {"daily_travel": ["daily_activity_pattern"]}, + {"tour_summaries": ["tour_time", "tour_stop_frequency", "tour_mode"]}, + {"trip_summaries": ["trip_mode", "trip_stop_time"]}, + ], export_html_lines=[ "dashboard:", - " weighting: all", - " values: all", + " weighting: weighted", + " values: percent", "pages:", " daily_travel:", " children:", " daily_activity_pattern:", " person_type:", - " - total", + " - all", " - worker", " tour_summaries:", " children:", @@ -1603,17 +1607,23 @@ def test_build_export_html_document_validates_page_selector_requests_against_reg assert tour_mode_weighted_percent["kind"] == "page" assert _region_nodes(tour_mode_weighted_percent)["tour_mode_modes"]["selector_ids"] == [ "tour_purpose", + "hide_drive_alone", ] assert sorted(_region_nodes(tour_mode_weighted_percent)["tour_mode_modes"]["variants"]) == [ - '["All Tour Purposes"]', - '["work"]', + '["All Tour Purposes","False"]', + '["All Tour Purposes","True"]', + '["work","False"]', + '["work","True"]', ] trip_mode_weighted_percent = payload["states"]["Weighted||Percent"]["trip_mode"] assert trip_mode_weighted_percent["kind"] == "page" assert sorted(_region_nodes(trip_mode_weighted_percent)["trip_summary_mode_body"]["variants"]) == [ - '["All Tour Purposes"]', - '["eatout"]', - '["social"]', + '["All Tour Purposes","False"]', + '["All Tour Purposes","True"]', + '["eatout","False"]', + '["eatout","True"]', + '["social","False"]', + '["social","True"]', ] @@ -1622,6 +1632,7 @@ def test_build_export_html_document_keeps_grouped_tour_mode_chart_when_mode_grou ) -> None: config = _write_config( tmp_path, + dashboard_pages=[{"tour_summaries": ["tour_mode"]}], modes_lines=[ "groups:", " Auto:", @@ -1644,14 +1655,17 @@ def test_build_export_html_document_keeps_grouped_tour_mode_chart_when_mode_grou assert tour_mode["kind"] == "page" assert _region_nodes(tour_mode)["tour_mode_modes"]["selector_ids"] == [ "tour_purpose", + "hide_drive_alone", ] assert sorted(_region_nodes(tour_mode)["tour_mode_modes"]["variants"]) == [ - '["All Tour Purposes"]', - '["work"]', + '["All Tour Purposes","False"]', + '["All Tour Purposes","True"]', + '["work","False"]', + '["work","True"]', ] region_nodes = _region_nodes(tour_mode) variant_nodes = _walk_nodes( - region_nodes["tour_mode_modes"]["variants"]['["All Tour Purposes"]'] + region_nodes["tour_mode_modes"]["variants"]['["All Tour Purposes","False"]'] ) assert any( node.get("kind") == "plotly" @@ -1674,7 +1688,7 @@ def test_build_export_html_document_serializes_vehicle_occupancy_variants_for_to ) -> None: config = _write_config( tmp_path, - dashboard_pages=["tour_summaries"], + dashboard_pages=[{"tour_summaries": ["tour_mode"]}], export_html_lines=[ "pages:", " tour_mode:", @@ -1718,6 +1732,7 @@ def test_build_export_html_document_serializes_long_term_geography_variants( ) -> None: config = _write_config( tmp_path, + dashboard_pages=[{"long_term_choices": ["mandatory_location_choice"]}], geography_lines=[ "enabled: true", "landuse_col: COUNTY", @@ -1737,24 +1752,146 @@ def test_build_export_html_document_serializes_long_term_geography_variants( assert page_defs["mandatory_location_choice"]["selectors"][0]["id"] == "geography_level" assert page_defs["mandatory_location_choice"]["selectors"][0]["request_mode"] == "all" assert set(page_defs["mandatory_location_choice"]["selectors"][0]["resolved_values"]) == { - "All Geographies", + "All Geography Types", "Suburban", "Urban", } assert page_defs["mandatory_location_choice"]["selectors"][0]["export_enabled"] is True assert page_defs["mandatory_location_choice"]["selectors"][1]["id"] == "geography" - assert page_defs["mandatory_location_choice"]["selectors"][1]["export_enabled"] is True - assert "All" in page_defs["mandatory_location_choice"]["selectors"][1]["resolved_values"] - assert len(page_defs["mandatory_location_choice"]["selectors"][1]["resolved_values"]) > 1 + geography_selector = page_defs["mandatory_location_choice"]["selectors"][1] + assert geography_selector["export_enabled"] is True + assert "All" in geography_selector["resolved_values"] + assert len(geography_selector["resolved_values"]) > 1 + assert geography_selector["parent_selector_id"] == "geography_level" + assert set(geography_selector["options_by_parent_value"]) == { + "All Geography Types", + "Suburban", + "Urban", + } + assert geography_selector["options_by_parent_value"]["All Geography Types"] == [ + "All" + ] + assert geography_selector["disabled_parent_values"] == ["All Geography Types"] mandatory_location_choice = payload["states"]["Weighted||Percent"]["mandatory_location_choice"] assert mandatory_location_choice["kind"] == "page" - commuting_variants = sorted( - _region_nodes(mandatory_location_choice)["commuting_flows"]["variants"] + assert "commuting_flows" not in _region_nodes(mandatory_location_choice) + + +def test_mandatory_location_export_filters_dependent_names_to_exported_types( + tmp_path: Path, +) -> None: + config = _write_config( + tmp_path, + dashboard_pages=[{"long_term_choices": ["mandatory_location_choice"]}], + export_html_lines=[ + "pages:", + " long_term_choices:", + " mandatory_location_choice:", + " geography_level:", + " - All Geography Types", + " - County", + " - MPO", + ], ) - assert '["All","All"]' in commuting_variants - assert any("Urban" in key for key in commuting_variants) - assert any("Suburban" in key for key in commuting_variants) + base_run = _full_summary_run() + summaries_by_mode: dict[str, dict[str, pl.DataFrame]] = {} + for weighting_mode, summaries in base_run.summaries_by_mode.items(): + updated = dict(summaries) + factor = 1.0 if weighting_mode == "weighted" else 0.5 + updated["internal_external_worker_by_geography"] = pl.DataFrame( + { + "geography_level": ["county", "mpo", "taz", "taz"], + "geography": ["Alpha County", "Regional MPO", "101", "102"], + "internal_worker_count": [10.0, 20.0, 3.0, 4.0], + "external_worker_count": [1.0, 2.0, 0.0, 1.0], + } + ).with_columns( + pl.col("internal_worker_count") * factor, + pl.col("external_worker_count") * factor, + ) + distance = pl.DataFrame( + { + "geography_level": ["county", "mpo", "taz", "taz"], + "geography": ["Alpha County", "Regional MPO", "101", "102"], + "distance_bin": [1.0, 1.0, 1.0, 1.0], + "person_count": [10.0, 20.0, 3.0, 4.0], + } + ).with_columns(pl.col("person_count") * factor) + for summary_id in ( + "work_location_distance_distribution_by_geography", + "school_location_distance_distribution_by_geography", + "university_location_distance_distribution_by_geography", + ): + updated[summary_id] = distance + updated["average_mandatory_tour_distance_by_purpose_and_geography"] = ( + pl.DataFrame( + { + "geography_level": ["county", "mpo", "taz", "taz"], + "geography": ["Alpha County", "Regional MPO", "101", "102"], + "mandatory_tour_purpose": ["work", "work", "work", "work"], + "average_tour_distance": [8.0, 9.0, 4.0, 5.0], + } + ) + ) + updated["work_from_home_rate_by_geography"] = pl.DataFrame( + { + "geography_level": ["county", "mpo", "taz", "taz"], + "geography": ["Alpha County", "Regional MPO", "101", "102"], + "worker_count": [10.0, 20.0, 3.0, 4.0], + "work_from_home_worker_count": [1.0, 2.0, 0.0, 1.0], + } + ).with_columns( + pl.col("worker_count") * factor, + pl.col("work_from_home_worker_count") * factor, + ) + summaries_by_mode[weighting_mode] = updated + + summary_run = create_summary_run( + label="Base", + run_key="base", + summaries_by_mode=summaries_by_mode, + source_run_dir="C:/runs/base", + ) + html = build_export_html_document([], config, summary_runs=[summary_run]) + payload = _extract_payload(html) + page = _flatten_page_descriptors(payload["pages"])["mandatory_location_choice"] + geography_type = _selector_by_id(page, "geography_level") + geography_name = _selector_by_id(page, "geography") + + assert geography_type["resolved_values"] == [ + "All Geography Types", + "County", + "MPO", + ] + assert geography_name["options_by_parent_value"] == { + "All Geography Types": ["All"], + "County": ["All", "Alpha County"], + "MPO": ["All", "Regional MPO"], + } + assert geography_name["resolved_values"] == [ + "All", + "Alpha County", + "Regional MPO", + ] + assert "101" not in geography_name["options"] + assert "102" not in geography_name["options"] + + page_node = payload["states"]["Weighted||Percent"]["mandatory_location_choice"] + geography_widget = next( + node + for node in _walk_nodes(page_node) + if node.get("kind") == "widget" and node.get("selector_id") == "geography" + ) + assert geography_widget["parent_selector_id"] == "geography_level" + assert geography_widget["disabled_parent_values"] == ["All Geography Types"] + assert geography_widget["options_by_parent_value"] == ( + geography_name["options_by_parent_value"] + ) + + regions = _region_nodes(page_node) + assert regions + assert all(not region["variant_aliases"] for region in regions.values()) def test_build_export_html_document_warns_and_falls_back_when_long_term_geography_is_unavailable( @@ -1762,6 +1899,7 @@ def test_build_export_html_document_warns_and_falls_back_when_long_term_geograph ) -> None: config = _write_config( tmp_path, + dashboard_pages=[{"long_term_choices": ["shadow_pricing"]}], export_html_lines=[ "pages:", " long_term_choices:", @@ -1789,6 +1927,7 @@ def test_build_export_html_document_serializes_stop_frequency_four_chart_variant ) -> None: config = _write_config( tmp_path, + dashboard_pages=[{"tour_summaries": ["tour_stop_frequency"]}], export_html_lines=[ "pages:", " tour_stop_frequency:", @@ -1817,10 +1956,30 @@ def test_build_export_html_document_serializes_stop_frequency_four_chart_variant if node.get("kind") == "plotly" } assert { - "Tour Stop Frequency - eatout, Both", - "Tour Stop Frequency - eatout, Outbound", - "Tour Stop Frequency - eatout, Inbound", + "Tour Stop Frequency - Purpose: eatout, Direction: Both", + "Tour Stop Frequency - Purpose: eatout, Direction: Outbound", + "Tour Stop Frequency - Purpose: eatout, Direction: Inbound", }.issubset(plotly_titles) + direction_rows = [ + node + for node in variant_nodes + if node.get("kind") == "container" + and node.get("layout") == "row" + and { + descendant.get("figure", {}) + .get("layout", {}) + .get("title", {}) + .get("text") + for child in node.get("children", []) + for descendant in _walk_nodes(child) + if descendant.get("kind") == "plotly" + } + == { + "Tour Stop Frequency - Purpose: eatout, Direction: Outbound", + "Tour Stop Frequency - Purpose: eatout, Direction: Inbound", + } + ] + assert len(direction_rows) == 1 def test_build_export_html_document_serializes_stop_timing_two_chart_variant( @@ -1828,6 +1987,7 @@ def test_build_export_html_document_serializes_stop_timing_two_chart_variant( ) -> None: config = _write_config( tmp_path, + dashboard_pages=["trip_stop_time"], export_html_lines=[ "pages:", " trip_stop_time:", @@ -1853,11 +2013,112 @@ def test_build_export_html_document_serializes_stop_timing_two_chart_variant( assert sum(1 for node in variant_nodes if node.get("kind") == "plotly") == 2 +def test_build_export_html_document_serializes_trip_stop_distance_two_chart_variant( + tmp_path: Path, +) -> None: + config = _write_config( + tmp_path, + dashboard_pages=["trip_stop_distance"], + export_html_lines=[ + "pages:", + " trip_stop_distance:", + " tour_purpose: all", + ], + ) + + summary_run = _full_summary_run() + distance_summary_run = type(summary_run)( + label=summary_run.label, + run_key=summary_run.run_key, + summaries_by_mode={ + mode: { + **summary_run.summaries_by_mode[mode], + "trip_distance_by_purpose": pl.DataFrame( + { + "tour_purpose": [ + "all_tour_purposes", + "all_tour_purposes", + "eatout", + "eatout", + "social", + "social", + ], + "distance_bin": ["0-1", "1-2", "0-1", "1-2", "0-1", "1-2"], + "trip_count": [10.0, 8.0, 4.0, 2.0, 3.0, 1.0], + "pct": [0.56, 0.44, 0.67, 0.33, 0.75, 0.25], + } + ), + "stop_out_of_direction_distance_by_tour_purpose": pl.DataFrame( + { + "tour_purpose": [ + "all_tour_purposes", + "all_tour_purposes", + "eatout", + "eatout", + "social", + "social", + ], + "distance_bin": ["0-1", "1-2", "0-1", "1-2", "0-1", "1-2"], + "stop_count": [7.0, 5.0, 3.0, 1.0, 2.0, 1.0], + "pct": [0.58, 0.42, 0.75, 0.25, 0.67, 0.33], + } + ), + } + for mode in summary_run.summaries_by_mode + }, + source_run_dir=summary_run.source_run_dir, + manifest=summary_run.manifest, + ) + + html = build_export_html_document([], config, summary_runs=[distance_summary_run]) + payload = _extract_payload(html) + trip_stop_distance = payload["states"]["Weighted||Percent"]["trip_stop_distance"] + + assert trip_stop_distance["kind"] == "page" + region = _region_nodes(trip_stop_distance)["trip_stop_distance_body"] + assert region["selector_ids"] == ["tour_purpose"] + assert sorted( + region["variants"] + ) == [ + '["All Tour Purposes"]', + '["eatout"]', + '["social"]', + ] + variant_nodes = _walk_nodes( + region["variants"]['["All Tour Purposes"]'] + ) + assert sum(1 for node in variant_nodes if node.get("kind") == "plotly") == 2 + range_widgets = [ + node + for node in _walk_nodes(trip_stop_distance) + if node.get("kind") == "widget" + and node.get("name") + in {"Distance Min", "Distance Max", "Reset distance range"} + ] + assert {node["widget_type"] for node in range_widgets} == { + "button", + "float_input", + "select", + } + assert sum(node["widget_type"] == "button" for node in range_widgets) >= 1 + assert sum(node["widget_type"] == "float_input" for node in range_widgets) >= 1 + assert sum(node["widget_type"] == "select" for node in range_widgets) >= 1 + assert any( + node["widget_type"] == "select" and node.get("value") == "40+" + for node in range_widgets + ) + assert all(node.get("disabled") for node in range_widgets) + assert all(not node.get("export_enabled") for node in range_widgets) + all_variant_plots = [node for node in variant_nodes if node.get("kind") == "plotly"] + assert all_variant_plots[0]["figure"]["layout"]["xaxis"]["range"] == [0.0, 40.0] + + def test_build_export_html_document_serializes_joint_tours_hh_size_variants( tmp_path: Path, ) -> None: config = _write_config( tmp_path, + dashboard_pages=[{"daily_travel": ["daily_activity_pattern"]}], export_html_lines=[ "pages:", " daily_travel:", @@ -1883,16 +2144,95 @@ def test_build_export_html_document_serializes_joint_tours_hh_size_variants( ) +def test_build_export_html_document_serializes_joint_frequency_hide_no_joint_tours_variant( + tmp_path: Path, +) -> None: + config = _write_config( + tmp_path, + dashboard_pages=["joint_travel"], + export_html_lines=[ + "pages:", + " joint_travel:", + " hide_no_joint_tours: all", + ], + ) + summary_run = _full_summary_run() + summary_run = create_summary_run( + label=summary_run.label, + run_key=summary_run.run_key, + summaries_by_mode={ + mode: { + **summary_run.summaries_by_mode[mode], + "joint_tours_by_household_size": pl.DataFrame( + { + "household_size": [2, 3], + "household_count": [6.0, 4.0], + "joint_tour_hh_count": [3.0, 2.0], + } + ), + "joint_tour_composition_by_party_size": pl.DataFrame( + { + "tour_composition": ["adults", "mixed"], + "party_size": [2, 3], + "joint_tour_count": [3.0, 2.0], + } + ), + "person_jtp_by_household_size": pl.DataFrame( + { + "household_size": [2, 3], + "joint_tour_person_count": [2.0, 3.0], + "total_person_count": [4.0, 3.0], + } + ), + } + for mode in summary_run.summaries_by_mode + }, + summary_metadata_by_mode=summary_run.summary_metadata_by_mode, + ) + + html = build_export_html_document([], config, summary_runs=[summary_run]) + payload = _extract_payload(html) + joint_travel = payload["states"]["Weighted||Percent"]["joint_travel"] + frequency_region = _region_nodes(joint_travel)["joint_travel_frequency"] + + assert frequency_region["selector_ids"] == ["hide_no_joint_tours"] + assert sorted(frequency_region["variants"]) == ['["False"]', '["True"]'] + + checked_nodes = _walk_nodes(frequency_region["variants"]['["True"]']) + checked_plot = next( + node + for node in checked_nodes + if node.get("kind") == "plotly" + and node.get("figure", {}).get("layout", {}).get("title", {}).get("text") + == "Joint Tour Frequency by Joint Tour Pattern" + ) + checked_x_values = [ + value + for trace in checked_plot.get("figure", {}).get("data", []) + for value in trace.get("x", []) + ] + checked_y_values = [ + value + for trace in checked_plot.get("figure", {}).get("data", []) + for value in trace.get("y", []) + ] + + assert "No Joint Tours" not in checked_x_values + assert checked_x_values == ["1 Shopping", "1 Maintenance"] + assert checked_y_values == pytest.approx([25.0, 15.0]) + + def test_build_export_html_document_serializes_trip_mode_tour_purpose_variants( tmp_path: Path, ) -> None: config = _write_config( tmp_path, + dashboard_pages=["trip_mode"], export_html_lines=[ "pages:", " trip_summaries:", " trip_mode:", - " tour_purpose: all", + " tour_purpose: all", ], ) @@ -1902,9 +2242,12 @@ def test_build_export_html_document_serializes_trip_mode_tour_purpose_variants( assert trip_mode["kind"] == "page" assert sorted(_region_nodes(trip_mode)["trip_summary_mode_body"]["variants"]) == [ - '["All Tour Purposes"]', - '["eatout"]', - '["social"]', + '["All Tour Purposes","False"]', + '["All Tour Purposes","True"]', + '["eatout","False"]', + '["eatout","True"]', + '["social","False"]', + '["social","True"]', ] page_nodes = _walk_nodes(trip_mode) widget_nodes = [node for node in page_nodes if node.get("kind") == "widget"] @@ -1914,9 +2257,15 @@ def test_build_export_html_document_serializes_trip_mode_tour_purpose_variants( and not node.get("disabled") for node in widget_nodes ) + assert any( + node.get("selector_id") == "hide_drive_alone" + and node.get("export_enabled") + and not node.get("disabled") + for node in widget_nodes + ) assert not any(node.get("selector_id") == "tour_mode" for node in widget_nodes) variant_nodes = _walk_nodes( - _region_nodes(trip_mode)["trip_summary_mode_body"]["variants"]['["eatout"]'] + _region_nodes(trip_mode)["trip_summary_mode_body"]["variants"]['["eatout","False"]'] ) assert sum(1 for node in variant_nodes if node.get("kind") == "plotly") == 3 @@ -1933,7 +2282,7 @@ def test_build_export_html_document_rejects_unknown_page_and_selector_ids( ], ) with pytest.raises( - ValueError, match="Unsupported visualizer.export_html.pages entries" + ValueError, match="Unsupported dashboard.export.pages entries" ): build_export_html_document( [], bad_page_config, summary_runs=[_full_summary_run()] @@ -1950,7 +2299,7 @@ def test_build_export_html_document_rejects_unknown_page_and_selector_ids( ) with pytest.raises( ValueError, - match="Unsupported visualizer.export_html.pages.trip_summaries.trip_mode entries", + match="Unsupported dashboard.export.pages.trip_summaries.trip_mode entries", ): build_export_html_document( [], @@ -1962,10 +2311,13 @@ def test_build_export_html_document_rejects_unknown_page_and_selector_ids( def test_export_html_save_writes_single_client_side_html_file(tmp_path: Path) -> None: config = _write_config( tmp_path, + dashboard_pages=[{"trip_summaries": ["trip_mode"]}], export_html_lines=[ "dashboard:", " weighting: all", " values: all", + "pages:", + " trip_mode: {}", ], ) out_dir = tmp_path / "html_export" @@ -2001,10 +2353,57 @@ def test_export_html_save_writes_single_client_side_html_file(tmp_path: Path) -> assert "Weighted||Percent" in diagnostics["states"] +def test_export_html_save_streams_without_building_full_payload_or_document( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config = _write_config( + tmp_path, + dashboard_pages=["overview"], + export_html_lines=["pages:", " overview: {}"], + ) + out_path = tmp_path / "dashboard.html" + + def fail_in_memory_build(*args, **kwargs): + raise AssertionError("production writer used an in-memory export representation") + + monkeypatch.setattr( + "dashboard.export.html._serialize_export_payload_json", + fail_in_memory_build, + ) + monkeypatch.setattr( + "dashboard.export.html._build_export_html_shell_document", + fail_in_memory_build, + ) + + write_export_html_document(out_path, [], config, summary_runs=[_full_summary_run()]) + + payload = _extract_payload(out_path.read_text(encoding="utf-8")) + assert payload["title"] == config.dashboard_title + assert payload["states"]["Weighted||Percent"]["overview"] + + +def test_streamed_payload_escapes_script_end_token_across_encoder_chunks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def chunked_iterencode(self, payload): + del self, payload + return iter(['{"html": "<', '/script>"}']) + + monkeypatch.setattr(json.JSONEncoder, "iterencode", chunked_iterencode) + + payload_json = "".join( + _iter_script_safe_payload_json({"html": ""}) + ) + + assert payload_json == '{"html": "<\\/script>"}' + assert json.loads(payload_json) == {"html": ""} + + def test_export_html_config_supports_missing_data_display(tmp_path: Path) -> None: config = _write_config( tmp_path, - visualizer_lines=["missing_data_display: blank"], + display_lines=["missing_data_display: blank"], ) assert config.missing_data_display == "blank" @@ -2015,11 +2414,11 @@ def test_export_html_config_rejects_invalid_missing_data_display( ) -> None: with pytest.raises( ValueError, - match="visualizer.missing_data_display must be either 'card' or 'blank'", + match="display.missing_data_display must be either 'card' or 'blank'", ): _write_config( tmp_path, - visualizer_lines=["missing_data_display: loud"], + display_lines=["missing_data_display: loud"], ) @@ -2048,33 +2447,44 @@ def test_export_html_save_fails_fast_without_final_files_when_html_write_fails( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - config = _write_config(tmp_path) + config = _write_config( + tmp_path, + dashboard_pages=["overview"], + export_html_lines=["pages:", " overview: {}"], + ) out_path = tmp_path / "dashboard.html" - def fail_html_temp_write(path: Path, contents: str) -> None: - if "dashboard.html" in path.name and path.suffix == ".tmp": - raise OSError("disk full") - path.write_text(contents, encoding="utf-8") + def fail_html_temp_write(path: Path, **kwargs) -> None: + path.write_text("partial", encoding="utf-8") + raise OSError("disk full") - monkeypatch.setattr("dashboard.export.html._write_text_file", fail_html_temp_write) + monkeypatch.setattr( + "dashboard.export.html._write_streamed_export_html", + fail_html_temp_write, + ) with pytest.raises(ExportBuildError, match="write HTML atomically"): write_export_html_document(out_path, [], config, summary_runs=[_full_summary_run()]) assert not out_path.exists() assert not (tmp_path / "dashboard.diagnostics.json").exists() + assert not list(tmp_path.glob(".*.tmp")) def test_export_html_save_rejects_malformed_assembled_html_before_finalizing( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - config = _write_config(tmp_path) + config = _write_config( + tmp_path, + dashboard_pages=["overview"], + export_html_lines=["pages:", " overview: {}"], + ) out_path = tmp_path / "dashboard.html" monkeypatch.setattr( - "dashboard.export.html.build_export_html_shell", - lambda **kwargs: "broken export", + "dashboard.export.html.build_export_html_shell_parts", + lambda **kwargs: ("broken export", ""), ) with pytest.raises(ExportBuildError, match="validate assembled HTML"): diff --git a/tests/test_export_html_smoke.py b/tests/test_export_html_smoke.py index 9776065..33f7790 100644 --- a/tests/test_export_html_smoke.py +++ b/tests/test_export_html_smoke.py @@ -3,7 +3,6 @@ import json from pathlib import Path import sys -from uuid import uuid4 sys.path.insert(0, str(Path(__file__).resolve().parents[1])) sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -18,7 +17,7 @@ def _workspace_tmp_dir(label: str) -> Path: - path = Path("tmp_export_test_artifacts") / f"{label}_{uuid4().hex}" + path = Path(".pytest_tmp") / "export_helpers" / label path.mkdir(parents=True, exist_ok=True) return path @@ -27,10 +26,13 @@ def test_export_html_smoke_writes_single_self_contained_file() -> None: tmp_path = _workspace_tmp_dir("html_smoke_file") config = _write_config( tmp_path, + dashboard_pages=[{"trip_summaries": ["trip_mode"]}], export_html_lines=[ "dashboard:", " weighting: all", " values: all", + "pages:", + " trip_mode: {}", ], ) out_path = tmp_path / "smoke" / "dashboard.html" @@ -52,6 +54,10 @@ def test_export_html_smoke_writes_single_self_contained_file() -> None: assert "activitysim-export-data" in html assert "Export payload JSON could not be parsed." in html assert "Plotly.react" in html + assert 'displayModeBar: "hover"' in html + assert "scale: 2" in html + assert 'name: "Download CSV"' in html + assert "modeBarButtonsToAdd: [makePlotCsvDownloadButton(figure)]" in html assert "Offline export failed to load" in html assert "This HTML export encountered a runtime rendering error." in html assert "Unknown export node kind encountered:" in html @@ -66,7 +72,7 @@ def test_export_html_smoke_embeds_versioned_payload_and_runtime() -> None: tmp_path = _workspace_tmp_dir("html_smoke_payload") config = _write_config( tmp_path, - dashboard_pages=["trip_summaries"], + dashboard_pages=[{"trip_summaries": ["trip_mode"]}], export_html_lines=[ "pages:", " trip_summaries:", @@ -94,27 +100,24 @@ def test_export_html_smoke_embeds_versioned_payload_and_runtime() -> None: ("trip_summaries", "Trip Summaries") ] trip_summaries = payload["pages"][0] - assert trip_summaries["default_page_id"] == "trip_stop_purpose" + assert trip_summaries["default_page_id"] == "trip_mode" assert [(child["id"], child["title"]) for child in trip_summaries["children"]] == [ - ("trip_stop_purpose", "Trip and Stop Purpose"), ("trip_mode", "Trip Mode"), - ("trip_stop_time", "Trip and Stop Time"), - ("trip_stop_distance", "Trip and Stop Distance"), ] trip_mode = next(child for child in trip_summaries["children"] if child["id"] == "trip_mode") - assert trip_mode["selectors"] == [ - { - "id": "tour_purpose", - "label": "Tour Purpose", - "available": True, - "request_mode": "all", - "requested_values": [], - "resolved_values": ["All", "eatout", "social"], - "default_value": "All", - "options": ["All", "eatout", "social"], - "export_enabled": True, - } - ] + selectors = {selector["id"]: selector for selector in trip_mode["selectors"]} + assert selectors["tour_purpose"] == { + "id": "tour_purpose", + "label": "Tour Purpose", + "available": True, + "request_mode": "all", + "requested_values": [], + "resolved_values": ["All Tour Purposes", "eatout", "social"], + "default_value": "All Tour Purposes", + "options": ["All Tour Purposes", "eatout", "social"], + "export_enabled": True, + } + assert selectors["hide_drive_alone"]["resolved_values"] == ["False", "True"] assert payload["states"]["Weighted||Percent"]["trip_mode"]["kind"] == "page" assert "Unsupported export schema version." in html assert "__EXPORT_SCHEMA_VERSION__" not in html @@ -126,8 +129,10 @@ def test_export_runtime_assets_are_loaded_from_source_files() -> None: assert ".export-shell" in css assert ".export-error-panel" in css + assert ".export-table-sort" in css assert "function validatePayloadSchema(candidate)" in runtime_js assert "function renderPlot(node, context)" in runtime_js + assert "function renderTable(node)" in runtime_js assert "function renderNode(node, context, actions, leafPageId)" in runtime_js assert "function renderRegion(node, context, actions, leafPageId)" in runtime_js assert "function getLeafPageId(currentPayload, currentState)" in runtime_js @@ -141,6 +146,11 @@ def test_export_html_smoke_serializes_grouped_default_page_as_leaf_page_id() -> tmp_path = _workspace_tmp_dir("html_smoke_grouped_defaults") config = _write_config( tmp_path, + dashboard_pages=[ + {"daily_travel": ["daily_activity_pattern"]}, + {"tour_summaries": ["tour_purpose"]}, + {"trip_summaries": ["trip_stop_purpose"]}, + ], export_html_lines=[ "dashboard:", " weighting: all", diff --git a/tests/test_export_payload.py b/tests/test_export_payload.py index d73e7eb..6628c65 100644 --- a/tests/test_export_payload.py +++ b/tests/test_export_payload.py @@ -2,7 +2,8 @@ from pathlib import Path import sys -from uuid import uuid4 + +import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[1])) sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -11,12 +12,19 @@ from dashboard import DashboardState from dashboard.data_access import DashboardPreparedRunProvider from dashboard.export.context import ExportBuildContext -from dashboard.export.payload import build_export_payload +from dashboard.export.page_serializer import ( + VMT_EXPORT_DROPDOWN_NOTE, + _with_export_page_notes, +) +from dashboard.export.payload import ( + build_export_payload, +) from dashboard.export.types import ( EXPORT_CLIENT_RUNTIME, EXPORT_PAGE_SELECTOR_RUNTIME, EXPORT_SCHEMA_VERSION, ) +from dashboard.page_definitions import DashboardPageDefinition from test_export_html import ( _full_summary_run, _segmented_summary_runs, @@ -28,7 +36,7 @@ def _workspace_tmp_dir(label: str) -> Path: - path = Path("tmp_export_test_artifacts") / f"{label}_{uuid4().hex}" + path = Path(".pytest_tmp") / "export_helpers" / label path.mkdir(parents=True, exist_ok=True) return path @@ -42,6 +50,36 @@ def _plot_nodes(page_payload: dict) -> list[dict]: return [node for node in _walk_nodes(page_payload) if node.get("kind") == "plotly"] +def test_vmt_export_content_includes_dropdown_availability_note() -> None: + title_node = {"kind": "html", "html": "