diff --git a/lopper/__init__.py b/lopper/__init__.py index 79696e9a..9e4b81c2 100644 --- a/lopper/__init__.py +++ b/lopper/__init__.py @@ -289,36 +289,33 @@ def _unwrap_overlay_tree(ov_tree, base_tree): clean_node.__dict__['abs_path'] = target_abs_path clean_node.label = label - # Copy properties, resolving 0xffffffff placeholders + # Resolve 0xffffffff phandle placeholders in-place across + # overlay_child and all its children (ov_tree is temporary) + ov_node_prefix = frag_path + '/__overlay__' + for fix_label, fix_refs in label_to_fixups.items(): + ph = label_to_phandle.get(fix_label) + if not ph: continue + for ref in fix_refs: + try: + rn, rp, bo = ref.rsplit(':', 2) + if not rn.startswith(ov_node_prefix): continue + prop = ov_tree[rn].__props__.get(rp) + if not prop: continue + val = list(prop.__dict__.get('value', [])) + idx = int(bo) // 4 + if idx < len(val) and val[idx] == 4294967295: + val[idx] = ph + prop.__dict__['value'] = val + except Exception: + pass + + # Copy resolved properties to clean_node for prop in overlay_child.__props__.values(): new_prop = copy.deepcopy(prop) new_prop.node = clean_node - - val = new_prop.__dict__.get('value') - if isinstance(val, list) and 4294967295 in val: - val = list(val) - ov_node_prefix = frag_path + '/__overlay__' - for fix_label, fix_refs in label_to_fixups.items(): - ph = label_to_phandle.get(fix_label) - if ph is None: - continue - for ref in fix_refs: - try: - ref_node, ref_prop, byte_off = ref.rsplit(':', 2) - if ref_prop != prop.name: - continue - if not ref_node.startswith(ov_node_prefix): - continue - idx = int(byte_off) // 4 - if idx < len(val) and val[idx] == 4294967295: - val[idx] = ph - except Exception: - pass - new_prop.__dict__['value'] = val - clean_node.__props__[prop.name] = new_prop - # Recursively copy child nodes from __overlay__ + # Recursively copy child nodes (fixups already resolved in-place) def _copy_children(src_node, dst_node): for child in src_node.child_nodes.values(): child_copy = copy.deepcopy(child) @@ -571,6 +568,7 @@ def _deserialize_overlay_node(data, parent=None, tree=None): from lopper.tree import LopperNode, LopperProp node = LopperNode(-1, data["abs_path"]) node.tree = tree + node.parent = parent for prop_name, val, pclass in data.get("props", []): lp = LopperProp(prop_name, -1, node, val) @@ -584,7 +582,7 @@ def _deserialize_overlay_node(data, parent=None, tree=None): for child_data in data.get("children", []): child = _deserialize_overlay_node(child_data, parent=node, tree=tree) - node.child_nodes.append(child) + node.child_nodes[child.abs_path] = child return node diff --git a/lopper/__main__.py b/lopper/__main__.py index 38857c3f..25a8e866 100644 --- a/lopper/__main__.py +++ b/lopper/__main__.py @@ -91,6 +91,8 @@ def usage(): print(' schema_all (enable all schema checks)' ) print(' all (enable all warnings)' ) print(' , --memmap output file for memory map visualization (use - for stdout)' ) + print(' , --cpumap output file for CPU access map visualization (use - for stdout)' ) + print(' , --cpumap-expand expand bus nodes to show child devices in cpumap' ) print(' , --symbols generate (and maintain) the __symbols__ node during processing' ) print(' -o, --output output file') print(' , --overlay Allow input files (dts or yaml) to overlay system device tree nodes' ) @@ -569,6 +571,17 @@ def main(): f.write(memmap_output) _info(f"memory map written to {memmap_file}") + # Generate CPU access map visualization if requested + if cpumap_file: + from lopper.assists.lopper_lib import render_all_cpu_access_maps + cpumap_output = render_all_cpu_access_maps(device_tree.tree, expand=cpumap_expand) + if cpumap_file == "-": + print(cpumap_output) + else: + with open(cpumap_file, 'w') as f: + f.write(cpumap_output) + _info(f"CPU access map written to {cpumap_file}") + if not dryrun: # write any changes to the FDT, before we do our write if device_tree.dts: diff --git a/lopper/assists/lopper_lib.py b/lopper/assists/lopper_lib.py index d843f392..0ded1772 100644 --- a/lopper/assists/lopper_lib.py +++ b/lopper/assists/lopper_lib.py @@ -361,6 +361,269 @@ def find_address_in_map(entries, address): return None +def _is_cpu_cluster(node): + """Check if a node is a CPU cluster. + + A CPU cluster is identified by: + - Having an address-map property (restricted access cluster), or + - Being named 'cpus' or 'cpus-cluster*', or + - Having compatible containing 'cpus,cluster' + """ + if 'address-map' in node.__props__: + return True + + # Check node name + if node.name == 'cpus' or node.name.startswith('cpus-cluster'): + return True + + # Check compatible property + try: + compat = node['compatible'].value + if isinstance(compat, list): + for c in compat: + if 'cpus' in str(c).lower(): + return True + elif 'cpus' in str(compat).lower(): + return True + except (KeyError, TypeError): + pass + + return False + + +def _get_child_devices(tree, bus_node, parent_addr, map_size, max_children=10): + """Get child devices of a bus node with their addresses. + + Args: + tree: LopperTree + bus_node: The bus/interconnect node + parent_addr: Base address of the bus in parent's view + map_size: Size of the mapped region + max_children: Maximum number of children to return + + Returns: + List of (address, size, label) tuples for child devices + """ + children = [] + + # Get address/size cells for this bus + try: + child_ac = bus_node['#address-cells'].value[0] + child_sc = bus_node['#size-cells'].value[0] + except (KeyError, IndexError, TypeError): + child_ac = 2 + child_sc = 2 + + # Parse ranges to understand address translation + # Empty ranges (or ['']) means 1:1 mapping (identity) + # ranges format: child_addr(child_ac) parent_addr(parent_ac) size(child_sc) + ranges_offset = 0 + try: + ranges = bus_node['ranges'].value + if ranges and ranges != [''] and len(ranges) >= child_ac: + # Get the child base address from ranges + ranges_offset = cell_value_get(ranges, child_ac, 0)[0] + except (KeyError, IndexError, TypeError): + pass + + # Iterate children + for child in bus_node.child_nodes.values(): + if 'reg' not in child.__props__: + continue + + try: + reg = child['reg'].value + if not reg or reg == ['']: + continue + + # Get child's address from reg property + child_addr = cell_value_get(reg, child_ac, 0)[0] + + # Get size if available + if len(reg) >= child_ac + child_sc: + child_size = cell_value_get(reg, child_sc, child_ac)[0] + else: + child_size = 0 + + # For 1:1 mapping (empty ranges), child_addr IS the system address + # Check if this falls within the mapped region + if child_addr >= parent_addr and child_addr < parent_addr + map_size: + label = child.label if child.label else child.name + children.append((child_addr, child_size, label)) + + except (IndexError, TypeError, ValueError): + continue + + if len(children) >= max_children: + break + + # Sort by address + children.sort(key=lambda x: x[0]) + return children + + +def render_cpu_access_map(tree, cpu_cluster=None, width=60, expand=False): + """Render an ASCII visualization of CPU cluster device accessibility. + + Shows which devices each CPU cluster can access based on its address-map + property. Clusters without address-map are shown as having unrestricted + system access. + + Args: + tree: LopperTree to visualize + cpu_cluster: Specific CPU cluster node or path (None for all clusters) + width: Width of the visualization in characters + expand: If True, show child devices of bus nodes + + Returns: + String containing the ASCII visualization + """ + lines = [] + + # Find CPU clusters to visualize + clusters = [] + if cpu_cluster is not None: + if isinstance(cpu_cluster, str): + node = tree.deref(cpu_cluster) + if node: + clusters = [node] + else: + clusters = [cpu_cluster] + else: + # Find all CPU cluster nodes + for node in tree.__nodes__.values(): + if _is_cpu_cluster(node): + clusters.append(node) + + if not clusters: + return "No CPU clusters found.\n" + + # Sort clusters by path for consistent output + clusters = sorted(clusters, key=lambda n: n.abs_path) + + # Render each cluster + for cluster in clusters: + lines.append("=" * width) + cluster_label = cluster.label if cluster.label else cluster.name + title = f"CPU Cluster: {cluster_label}" + lines.append(title.center(width)) + lines.append(f"Path: {cluster.abs_path}".center(width)) + lines.append("=" * width) + + # Check if cluster has address-map + if 'address-map' not in cluster.__props__: + lines.append("") + lines.append(" (no address-map - unrestricted system access)") + lines.append("") + continue + + # Parse address-map + try: + address_map = cluster['address-map'].value + na = cluster['#ranges-address-cells'].value[0] + ns = cluster['#ranges-size-cells'].value[0] + except (KeyError, IndexError, TypeError): + lines.append(" (unable to parse address-map)") + lines.append("") + continue + + entries = parse_address_map(address_map, na, ns) + + if not entries: + lines.append(" (empty address-map)") + lines.append("") + continue + + # Header + lines.append("") + lines.append(f" {'Address Range':<36} {'Size':<12} Device") + lines.append(f" {'-' * 36} {'-' * 12} {'-' * (width - 54)}") + + # Sort entries by address + sorted_entries = sorted(entries, key=lambda e: e.child_addr) + + # Track unique devices (same phandle may appear multiple times) + seen_devices = {} + + for entry in sorted_entries: + # Skip zero-size entries (typically non-memory mapped devices) + if entry.size == 0: + continue + + # Format address range + addr_start = f"0x{entry.child_addr:08x}" + addr_end = f"0x{entry.child_addr + entry.size - 1:08x}" + addr_range = f"{addr_start} - {addr_end}" + + # Format size + if entry.size >= 0x100000: + size_str = f"{entry.size // 0x100000} MB" + elif entry.size >= 0x400: + size_str = f"{entry.size // 0x400} KB" + else: + size_str = f"{entry.size} B" + + # Resolve device node + device_node = tree.pnode(entry.phandle) + if device_node: + device_label = device_node.label if device_node.label else device_node.name + device_path = device_node.abs_path + + # Track for summary + if entry.phandle not in seen_devices: + seen_devices[entry.phandle] = device_node + else: + device_label = f"" + device_path = "" + + lines.append(f" {addr_range:<36} {size_str:<12} {device_label}") + + # Expand bus nodes to show children + if expand and device_node and device_node.child_nodes: + children = _get_child_devices(tree, device_node, entry.child_addr, entry.size) + for i, (child_addr, child_size, child_label) in enumerate(children): + # Use tree drawing characters + if i == len(children) - 1: + prefix = " └─" + else: + prefix = " ├─" + + # Format child address and size + child_addr_str = f"0x{child_addr:08x}" + if child_size >= 0x100000: + child_size_str = f"{child_size // 0x100000} MB" + elif child_size >= 0x400: + child_size_str = f"{child_size // 0x400} KB" + elif child_size > 0: + child_size_str = f"{child_size} B" + else: + child_size_str = "" + + lines.append(f"{prefix} {child_addr_str:<31} {child_size_str:<12} {child_label}") + + # Summary + lines.append("") + lines.append(f" Total mappings: {len(entries)}") + lines.append(f" Unique devices: {len(seen_devices)}") + lines.append("") + + return "\n".join(lines) + + +def render_all_cpu_access_maps(tree, width=60, expand=False): + """Render ASCII visualization for all CPU clusters in a tree. + + Args: + tree: LopperTree to visualize + width: Width of the visualization in characters + expand: If True, show child devices of bus nodes + + Returns: + String containing the ASCII visualization + """ + return render_cpu_access_map(tree, cpu_cluster=None, width=width, expand=expand) + + def _normalize_start_size_value(raw_value, default_value): """Convert YAML-provided start/size representations into integers.""" if raw_value is None: diff --git a/lopper/assists/sdt_devices.py b/lopper/assists/sdt_devices.py new file mode 100644 index 00000000..085631e0 --- /dev/null +++ b/lopper/assists/sdt_devices.py @@ -0,0 +1,1267 @@ +#/* +# * Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All Rights Reserved. +# * +# * Author: +# * Bruce Ashfield +# * +# * SPDX-License-Identifier: BSD-3-Clause +# */ + +""" +Generate YAML domain with all devices from System Device Tree. + +This assist scans the SDT for devices across multiple categories (bus devices, +CPUs, memory, firmware, etc.) and generates a YAML file containing a domain +with all devices. This generated YAML can then be used as a parent domain +for glob-based device matching. + +Usage: + lopper system.dts output.yaml -- sdt_devices [options] + +Options: + -v, --verbose Enable verbose output + -b, --bus-types Comma-separated bus compatible strings (default: simple-bus) + -n, --domain-name Name for generated domain (default: sdt_all_devices) + -o Output file path (overrides positional output argument) + -c, --categories Comma-separated device categories to include + Categories: bus,cpu,memory,firmware,toplevel (default: all) + --exclude-categories Comma-separated categories to exclude + --include-pattern Regex pattern for node names to include + --exclude-pattern Regex pattern for node names to exclude + --include-clocks Include clock nodes in device list (default: excluded) + --include-infrastructure Comma-separated infrastructure category names to include + Use --list-infrastructure to see available categories + Use 'all' to include all infrastructure devices + --list-infrastructure List available infrastructure categories and exit + +Example: + # Generate SDT devices YAML (use '-' to skip main output, -o for assist output) + lopper system-top.dts - -- sdt_devices -o /tmp/sdt-devices.yaml + + # Only CPUs and memory + lopper system.dts - -- sdt_devices -c cpu,memory -o /tmp/cpu-mem.yaml + + # Everything except firmware + lopper system.dts - -- sdt_devices --exclude-categories firmware -o output.yaml + + # Only serial devices + lopper system.dts - -- sdt_devices --include-pattern "serial@.*" -o serial.yaml + + # All-in-one command (generate and use in single pipeline) + lopper system-top.dts - -- sdt_devices -o /tmp/sdt-devices.yaml && \\ + lopper -f --permissive --enhanced \\ + -x '*.yaml' \\ + -i /tmp/sdt-devices.yaml \\ + -i my-domain.yaml \\ + system-top.dts output.dts +""" + +import sys +import os +import getopt +import re +import logging +from enum import Enum +from pathlib import Path +from lopper.tree import LopperTree +from lopper.tree import LopperNode +from lopper.tree import LopperProp +from lopper.yaml import LopperYAML +from ruamel.yaml.scalarint import HexInt +import lopper +import lopper.log + +lopper.log._init(__name__) + + +class DeviceCategory(Enum): + """Categories of devices that can be discovered from the SDT.""" + BUS = "bus" + CPU = "cpu" + MEMORY = "memory" + FIRMWARE = "firmware" + TOPLEVEL = "toplevel" + + @classmethod + def from_string(cls, s): + """Convert string to DeviceCategory.""" + try: + return cls(s.lower().strip()) + except ValueError: + return None + + @classmethod + def all_categories(cls): + """Return list of all category values.""" + return list(cls) + + @classmethod + def parse_list(cls, s): + """Parse comma-separated string into list of categories.""" + categories = [] + for item in s.split(','): + cat = cls.from_string(item) + if cat: + categories.append(cat) + return categories + + +def is_compat(node, compat_string_to_test): + """Identify whether this assist handles the provided compatibility string. + + Args: + node (LopperNode): Device tree node being evaluated + compat_string_to_test (str): Compatibility string to test + + Returns: + Callable | str: Reference to entry point function on match, empty string otherwise + """ + if re.search("sdt-devices,sdt-devices-v1", compat_string_to_test): + return sdt_devices + if re.search("module,sdt_devices", compat_string_to_test): + return sdt_devices + return "" + + +def usage(): + print(""" + Usage: sdt_devices [OPTION] + + -v, --verbose Enable verbose output + -b, --bus-types Comma-separated bus compatible strings (default: simple-bus) + -n, --domain-name Name for generated domain (default: sdt_all_devices) + -o Output file path + -c, --categories Comma-separated device categories to include + Categories: bus,cpu,memory,firmware,toplevel + (default: all categories) + --exclude-categories Comma-separated categories to exclude + --include-pattern Regex pattern for node names to include + --exclude-pattern Regex pattern for node names to exclude + --include-clocks Include clock nodes in device list (default: excluded) + --include-infrastructure Comma-separated infrastructure category names to include + (devices normally excluded as non-assignable) + Use --list-infrastructure to see available categories + Use 'all' to include all infrastructure devices + --list-infrastructure List available infrastructure categories and exit + + Generate YAML domain containing devices from the System Device Tree. + The generated YAML can be used as a parent domain for glob-based device matching. + + Device Categories: + bus - Devices under simple-bus or other bus nodes + cpu - CPU clusters and individual CPU nodes + memory - Memory nodes, reserved-memory, SRAM/TCM + firmware - Firmware nodes, IPI, power management + toplevel - Non-bus devices directly under root + + Example: + lopper system.dts - -- sdt_devices -o output.yaml + lopper system.dts - -- sdt_devices -o output.yaml -c bus,cpu + lopper system.dts - -- sdt_devices -o output.yaml --exclude-categories firmware + lopper system.dts - -- sdt_devices -o output.yaml --include-pattern "serial@.*" + lopper system.dts - -- sdt_devices -o output.yaml --include-infrastructure protection + """) + + +def list_infrastructure(): + """Print available infrastructure categories and their descriptions.""" + print(""" + Infrastructure Categories (excluded by default): + + These devices cannot be independently assigned to domains or protected + by XPPU/XMPU. Use --include-infrastructure to include them. + + Category Description Example patterns + -------- ----------- ---------------- + interrupt Interrupt controllers (shared) arm,gic, interrupt-controller + bus Bus nodes (structural, not devices) simple-bus + ipi IPI mailbox infrastructure zynqmp-ipi-mailbox + smmu SMMU/IOMMU address translation arm,smmu, iommu + power Power management and CPU states arm,psci, arm,idle-state + syscon System controller registers syscon + phy PHY providers (not standalone) phy-provider + reset Reset controllers (shared) reset-controller + pinctrl Pin control/muxing (shared) pinctrl + misc Miscellaneous structural nodes gpio-keys, chosen + slcr SLCR and clock/reset control *slcr*, *_crf_*, *_crl_* + interconnect Interconnect fabric (shared) *_gpv@*, *_cci_*, *_afi_* + protection Protection units (can't protect self) *xmpu*, *xppu* + cpu-ctrl CPU cluster control registers *_apu_*, *_rpu_* + platform Platform/IO configuration *_siou@*, *iouslcr* + + Use 'all' to include all infrastructure devices: + lopper system.dts - -- sdt_devices --include-infrastructure all -o output.yaml + + Include multiple categories: + lopper system.dts - -- sdt_devices --include-infrastructure protection,slcr -o output.yaml + """) + + +class SDTDevices: + """Generates a YAML domain containing all devices from the SDT.""" + + # Default bus compatible strings to search for devices + DEFAULT_BUS_TYPES = ['simple-bus'] + + # Memory type mappings based on node name patterns + MEMORY_TYPE_PATTERNS = { + 'sram': [r'tcm', r'ocm', r'sram', r'bram'], + 'memory': [r'memory@', r'ddr'], + } + + # Infrastructure device categories - devices that typically can't be + # independently assigned to domains or protected by XPPU/XMPU. + # Organized into categories so specific categories can be included via + # --include-infrastructure option. + INFRASTRUCTURE_CATEGORIES = { + # Core interrupt infrastructure - cannot be split between domains + 'interrupt': [ + r'interrupt-controller', + r'arm,gic', + ], + # Bus nodes - structural, not actual devices + 'bus': [ + r'simple-bus', + ], + # IPI mailbox infrastructure - shared communication + 'ipi': [ + r'xlnx,zynqmp-ipi-mailbox', + r'xlnx,versal-ipi-dest-mailbox', + ], + # SMMU/IOMMU - system-level address translation + 'smmu': [ + r'arm,smmu', + r'iommu', + ], + # Power management and CPU states + 'power': [ + r'arm,idle-state', + r'arm,psci', + r'xlnx,zynqmp-power', + ], + # System controller registers + 'syscon': [ + r'syscon', + ], + # PHY providers - not standalone devices + 'phy': [ + r'phy-provider', + ], + # Reset controllers - shared infrastructure + 'reset': [ + r'reset-controller', + ], + # Pin control - shared multiplexing + 'pinctrl': [ + r'pinctrl', + ], + # Miscellaneous structural/virtual nodes + 'misc': [ + r'gpio-keys', + r'chosen$', + r'memory$', + ], + # SLCR and clock/reset control registers - not PMU controlled + 'slcr': [ + r'slcr', + r'_crf_', + r'_crf@', + r'_crl_', + r'_crl@', + ], + # Interconnect fabric - shared infrastructure + 'interconnect': [ + r'_gpv@', + r'_gpv_', + r'_cci_', + r'_cci@', + r'_afi_', + r'_afi@', + ], + # Protection units - can't protect themselves + 'protection': [ + r'xmpu', + r'xppu', + ], + # CPU cluster control registers (psu_apu@, ps_wizard_0_ps11_0_apu_0@, etc.) + 'cpu-ctrl': [ + r'_apu@', + r'_apu_\d+@', + r'_rpu@', + r'_rpu_[a-z]@', + ], + # Platform/IO configuration + 'platform': [ + r'_siou@', + r'iou.*scntr', + r'iousecure', + r'iouslcr', + ], + } + + # All infrastructure category names for validation + INFRASTRUCTURE_CATEGORY_NAMES = list(INFRASTRUCTURE_CATEGORIES.keys()) + + # Clock-related compatible patterns - separated so they can be optionally included + # NOTE: Clock domain assignment is not currently implemented. When included, + # clocks appear in the device list but no special handling is performed. + # Future work could add exclusive clock assignment to prevent multiple domains + # from controlling the same clock frequency. + CLOCK_COMPAT_PATTERNS = [ + r'fixed-clock', # Clock providers + r'fixed-factor-clock', # Clock dividers + r'-clk$', # Clock nodes + r'clock-controller', # Clock controllers + ] + + def __init__(self, sdt, include_clocks=False, include_infrastructure=None): + """Initialize the SDT devices generator. + + Args: + sdt (LopperSDT): The system device tree instance + include_clocks (bool): If True, include clock nodes in device list. + Default is False (clocks excluded). + include_infrastructure (list): List of infrastructure category names + to include (not exclude). Use ['all'] to + include all infrastructure devices. + Default is None (all infra excluded). + """ + self.sdt = sdt + self.include_clocks = include_clocks + self.include_infrastructure = include_infrastructure or [] + self.tree = LopperTree() + self.tree.phandle_resolution = False + + # Build the active exclusion patterns based on which categories are NOT included + self._build_active_patterns() + + def _build_active_patterns(self): + """Build the active infrastructure exclusion patterns. + + Combines patterns from all infrastructure categories that are NOT + in the include_infrastructure list. If 'all' is in include_infrastructure, + no patterns are excluded. + """ + self.active_infra_patterns = [] + + # If 'all' specified, don't exclude any infrastructure + if 'all' in self.include_infrastructure: + lopper.log._info("Including all infrastructure devices") + return + + # Build exclusion patterns from categories NOT in include list + for category, patterns in self.INFRASTRUCTURE_CATEGORIES.items(): + if category not in self.include_infrastructure: + self.active_infra_patterns.extend(patterns) + else: + lopper.log._info(f"Including infrastructure category: {category}") + + def _classify_memory_type(self, node_name): + """Classify memory node into memory or sram category. + + Args: + node_name (str): Name of the memory node + + Returns: + str: 'sram' or 'memory' + """ + name_lower = node_name.lower() + for mem_type, patterns in self.MEMORY_TYPE_PATTERNS.items(): + for pattern in patterns: + if re.search(pattern, name_lower): + return mem_type + return 'memory' + + def _parse_reg_property(self, node): + """Parse reg property to extract start address and size. + + The reg property format depends on parent's #address-cells and #size-cells. + Common formats: + - 4 cells: + - 2 cells: + + Args: + node: LopperNode with reg property + + Returns: + tuple: (start_hex, size_hex) or (None, None) if not parseable + """ + reg = node.propval("reg") + if not reg or len(reg) < 2: + return None, None + + # Try to get #address-cells and #size-cells from parent + parent = node.parent + addr_cells = 2 # default + size_cells = 2 # default + + if parent: + ac = parent.propval("#address-cells") + if ac and len(ac) > 0 and isinstance(ac[0], int): + addr_cells = ac[0] + sc = parent.propval("#size-cells") + if sc and len(sc) > 0 and isinstance(sc[0], int): + size_cells = sc[0] + + try: + # Parse based on cells + if addr_cells == 2 and size_cells == 2 and len(reg) >= 4: + # 64-bit address and size + start = (reg[0] << 32) | reg[1] if isinstance(reg[0], int) else 0 + size = (reg[2] << 32) | reg[3] if isinstance(reg[2], int) else 0 + elif addr_cells == 1 and size_cells == 1 and len(reg) >= 2: + # 32-bit address and size + start = reg[0] if isinstance(reg[0], int) else 0 + size = reg[1] if isinstance(reg[1], int) else 0 + elif addr_cells == 2 and size_cells == 1 and len(reg) >= 3: + # 64-bit address, 32-bit size + start = (reg[0] << 32) | reg[1] if isinstance(reg[0], int) else 0 + size = reg[2] if isinstance(reg[2], int) else 0 + else: + # Fallback: assume first value is address, second is size + start = reg[0] if isinstance(reg[0], int) else 0 + size = reg[1] if isinstance(reg[1], int) else 0 + + # Return HexInt for proper YAML hex integer output + start_hex = HexInt(start) if start else None + size_hex = HexInt(size) if size else None + return start_hex, size_hex + except: + return None, None + + def _is_actual_device(self, node): + """Check if node represents an actual device (vs structural/infrastructure node). + + Actual devices that can be assigned to domains have: + 1. A 'compatible' property identifying the device type + 2. A compatible string that's NOT infrastructure (clocks, interrupt controllers, etc.) + 3. A node name that's NOT infrastructure (slcr, xmpu, etc.) + + Structural nodes (port@*, endpoint@*) don't have compatible properties. + Infrastructure nodes (clocks, GIC, SMMU) have compatible but can't be split. + + Args: + node: LopperNode to check + + Returns: + bool: True if node appears to be an assignable device + """ + compat = node.propval("compatible") + # Check for valid compatible - propval may return [''] for missing properties + if not compat or len(compat) == 0: + return False + + # Filter out empty strings + valid_compat = [c for c in compat if c and str(c).strip()] + if not valid_compat: + return False + + # Check if node name matches infrastructure patterns + # (for patterns like _apu@, _gpv@, xmpu, etc.) + node_name = node.name + for pattern in self.active_infra_patterns: + if re.search(pattern, node_name, re.IGNORECASE): + lopper.log._debug(f" Skipping infrastructure device (name): {node_name}") + return False + + # Check if any compatible string matches infrastructure patterns + for compat_str in valid_compat: + compat_str = str(compat_str) + for pattern in self.active_infra_patterns: + if re.search(pattern, compat_str, re.IGNORECASE): + lopper.log._debug(f" Skipping infrastructure device: {node.name} ({compat_str})") + return False + + # Check clock patterns (excluded by default, can be included with --include-clocks) + if not self.include_clocks: + for pattern in self.CLOCK_COMPAT_PATTERNS: + if re.search(pattern, compat_str, re.IGNORECASE): + lopper.log._debug(f" Skipping clock device: {node.name} ({compat_str})") + return False + + return True + + def _apply_pattern_filter(self, devices, include_pattern=None, exclude_pattern=None): + """Apply include/exclude pattern filters to device list. + + Args: + devices (list): List of device dictionaries + include_pattern (str): Regex pattern - only include matching devices + exclude_pattern (str): Regex pattern - exclude matching devices + + Returns: + list: Filtered device list + """ + if not include_pattern and not exclude_pattern: + return devices + + filtered = [] + for dev in devices: + dev_name = dev.get('dev', '') + + # Check include pattern + if include_pattern: + if not re.search(include_pattern, dev_name): + continue + + # Check exclude pattern + if exclude_pattern: + if re.search(exclude_pattern, dev_name): + continue + + filtered.append(dev) + + return filtered + + def discover_bus_devices(self, bus_types=None): + """Find all devices under bus nodes in SDT. + + Searches for nodes with compatible strings matching the specified + bus types, then collects all addressable device children. + + Args: + bus_types (list): List of compatible strings to search for bus nodes. + Defaults to ['simple-bus']. + + Returns: + list: List of device dictionaries with 'dev' and optionally 'label' keys + """ + if bus_types is None: + bus_types = self.DEFAULT_BUS_TYPES + + devices = [] + seen_devices = set() + + # Find all bus nodes matching the compatible strings + bus_nodes = [] + for bus_type in bus_types: + found = self.sdt.tree.cnodes(bus_type) + lopper.log._debug(f"Found {len(found)} nodes with compatible '{bus_type}'") + bus_nodes.extend(found) + + lopper.log._info(f"Found {len(bus_nodes)} bus nodes to scan for devices") + + # Collect devices from each bus + for bus in bus_nodes: + lopper.log._debug(f"Scanning bus node: {bus.abs_path}") + + # Use child_nodes.values() to get only direct children + # (subnodes with children_only=True still returns all descendants) + for node in bus.child_nodes.values(): + # Only include addressable devices (have @ in name) + if '@' in node.name: + # Only include actual devices (must have compatible property) + # This filters out structural nodes like port@*, endpoint@*, etc. + if not self._is_actual_device(node): + lopper.log._debug(f" Skipping node without compatible: {node.name}") + continue + + if node.abs_path in seen_devices: + continue + seen_devices.add(node.abs_path) + + entry = {'dev': node.name} + if node.label: + entry['label'] = node.label + + # Add status if not "okay" (disabled devices) + status = node.propval("status") + if status and len(status) > 0 and status[0]: + status_str = str(status[0]).strip() + if status_str and status_str != "okay": + entry['status'] = status_str + + devices.append(entry) + lopper.log._debug(f" Found bus device: {node.name}") + + lopper.log._info(f"Discovered {len(devices)} bus devices") + return devices + + def _parse_cpu_map(self, cluster_node): + """Parse cpu-map node to get linear CPU enumeration. + + The cpu-map node describes the physical topology with clusters and cores. + We use it to enumerate CPUs linearly (0, 1, 2, ...) rather than using + MPIDR-based reg values which create sparse bitmasks. + + Args: + cluster_node: The parent CPU cluster node (e.g., cpus-a78@0) + + Returns: + dict: Mapping of CPU node path to linear index, or None if no cpu-map + """ + cpu_map = None + for child in cluster_node.child_nodes.values(): + if child.name == 'cpu-map': + cpu_map = child + break + + if not cpu_map: + return None + + # Parse cpu-map: cluster0/core0/cpu, cluster0/core1/cpu, etc. + cpu_index_map = {} + linear_idx = 0 + + # Get clusters in sorted order for consistent enumeration + clusters = sorted([n for n in cpu_map.child_nodes.values() + if n.name.startswith('cluster')], + key=lambda n: n.name) + + for cluster in clusters: + # Get cores in sorted order + cores = sorted([n for n in cluster.child_nodes.values() + if n.name.startswith('core')], + key=lambda n: n.name) + + for core in cores: + # The 'cpu' property is a phandle to the actual CPU node + cpu_phandle = core.propval('cpu') + if cpu_phandle and len(cpu_phandle) > 0: + # Resolve phandle to node path + phandle_val = cpu_phandle[0] + if isinstance(phandle_val, int): + cpu_node = self.sdt.tree.pnode(phandle_val) + if cpu_node: + cpu_index_map[cpu_node.abs_path] = linear_idx + linear_idx += 1 + + return cpu_index_map if cpu_index_map else None + + def discover_cpus(self): + """Find all CPU clusters and CPUs in SDT. + + Discovers CPU nodes by looking for nodes with device_type="cpu" + and their parent clusters. Uses cpu-map for proper CPU enumeration + when available, falling back to sequential enumeration. + + Returns: + list: List of CPU entry dictionaries with cluster info including: + - dev: cluster name/label + - compatible: CPU compatible string + - cpumask: hex bitmap of available CPUs + """ + cpus = [] + cluster_info = {} # cluster_path -> {cluster, compat, cpu_nodes} + + # First pass: collect all CPU nodes grouped by cluster + for node in self.sdt.tree: + device_type = node.propval("device_type") + if device_type and "cpu" in device_type: + cluster = node.parent + if cluster and cluster.abs_path != "/": + if cluster.abs_path not in cluster_info: + cluster_info[cluster.abs_path] = { + 'cluster': cluster, + 'compat': None, + 'cpu_nodes': [], + 'cpu_map': None + } + cluster_info[cluster.abs_path]['cpu_nodes'].append(node) + # Get compatible from first CPU + if not cluster_info[cluster.abs_path]['compat']: + compat = node.propval("compatible") + if compat and len(compat) > 0: + cluster_info[cluster.abs_path]['compat'] = compat[0] + + # Parse cpu-map for each cluster if available + for cluster_path, info in cluster_info.items(): + info['cpu_map'] = self._parse_cpu_map(info['cluster']) + + # Second pass: build one entry per cluster with combined cpumask + for cluster_path, info in cluster_info.items(): + cluster = info['cluster'] + cluster_name = cluster.label if cluster.label else cluster.name + cpu_map = info['cpu_map'] + + # Build cpumask - use cpu-map indices if available, else enumerate + cpumask = 0 + for idx, cpu_node in enumerate(info['cpu_nodes']): + if cpu_map and cpu_node.abs_path in cpu_map: + # Use linear index from cpu-map + cpu_idx = cpu_map[cpu_node.abs_path] + else: + # Fallback: sequential enumeration within cluster + cpu_idx = idx + + cpumask |= (1 << cpu_idx) + + entry = {'dev': cluster_name} + entry['cpumask'] = HexInt(cpumask) if cpumask else HexInt(0) + + if info['compat']: + entry['compatible'] = info['compat'] + + cpus.append(entry) + map_source = "cpu-map" if cpu_map else "sequential" + lopper.log._debug(f" Found CPU cluster: {cluster_name} (cpumask={hex(cpumask)}, {len(info['cpu_nodes'])} CPUs, {map_source})") + + # TODO: Future enhancements for smarter CPU grouping: + # 1. Parse power-domains property to group R52 CPUs by RPU cluster + # (e.g., RPU_A: cortexr52_0+1, RPU_B: cortexr52_2+3, etc.) + # 2. Use xlnx,lockstep-select to determine lockstep vs split mode + # 3. Consider cpu-map cluster hierarchy for A78 physical topology + # 4. Add execution-level information from cpu node properties + # Currently we keep it simple: one entry per DTS cluster node with + # combined cpumask for all CPUs in that node. + + lopper.log._info(f"Discovered {len(cpus)} CPU clusters") + return cpus + + def discover_memory(self): + """Find all memory nodes in SDT. + + Discovers: + - Main memory nodes (memory@*) + - Reserved memory regions + - SRAM/TCM regions + + Returns: + dict: Dictionary with 'memory' and 'sram' lists + """ + memory_devices = {'memory': [], 'sram': []} + seen = set() + + # Find memory@* nodes + for node in self.sdt.tree: + if node.name.startswith('memory@'): + if node.abs_path in seen: + continue + seen.add(node.abs_path) + + entry = {'dev': node.name} + if node.label: + entry['label'] = node.label + + # Extract start address and size from reg property + start, size = self._parse_reg_property(node) + if start: + entry['start'] = start + if size: + entry['size'] = size + + memory_devices['memory'].append(entry) + lopper.log._debug(f" Found memory: {node.name}") + + # Find reserved-memory children + try: + reserved_mem = self.sdt.tree["/reserved-memory"] + for node in reserved_mem.child_nodes.values(): + if node.abs_path in seen: + continue + seen.add(node.abs_path) + + entry = {'dev': node.name} + if node.label: + entry['label'] = node.label + + # Extract start address and size from reg property + start, size = self._parse_reg_property(node) + if start: + entry['start'] = start + if size: + entry['size'] = size + + # Add reserved-memory flags if present + if node.propval("no-map") is not None: + # no-map is a boolean property (presence = true) + no_map = node.propval("no-map") + if no_map != ['']: + entry['no-map'] = True + if node.propval("reusable") is not None: + reusable = node.propval("reusable") + if reusable != ['']: + entry['reusable'] = True + + mem_type = self._classify_memory_type(node.name) + memory_devices[mem_type].append(entry) + lopper.log._debug(f" Found reserved memory: {node.name} ({mem_type})") + except: + pass + + # Find SRAM/TCM nodes (often under bus nodes but may be elsewhere) + for node in self.sdt.tree: + name_lower = node.name.lower() + if any(p in name_lower for p in ['tcm', 'ocm', 'sram', 'bram']): + if '@' in node.name and node.abs_path not in seen: + # Check if this is actually a memory node, not infrastructure + # (e.g., ocm_xmpu is protection unit, not memory) + is_infrastructure = False + for pattern in self.active_infra_patterns: + if re.search(pattern, node.name, re.IGNORECASE): + lopper.log._debug(f" Skipping infrastructure SRAM: {node.name}") + is_infrastructure = True + break + if is_infrastructure: + continue + + seen.add(node.abs_path) + + entry = {'dev': node.name} + if node.label: + entry['label'] = node.label + + # Extract start address and size from reg property + start, size = self._parse_reg_property(node) + if start: + entry['start'] = start + if size: + entry['size'] = size + + memory_devices['sram'].append(entry) + lopper.log._debug(f" Found SRAM: {node.name}") + + lopper.log._info(f"Discovered {len(memory_devices['memory'])} memory, " + f"{len(memory_devices['sram'])} sram nodes") + return memory_devices + + def discover_firmware(self): + """Find firmware and system nodes in SDT. + + Discovers: + - /firmware/* nodes + - IPI mailbox nodes + - Power management nodes + + Returns: + list: List of firmware device dictionaries + """ + firmware_devices = [] + seen = set() + + # Find /firmware children (direct children only, not all descendants) + try: + firmware_node = self.sdt.tree["/firmware"] + for node in firmware_node.child_nodes.values(): + if node.abs_path in seen: + continue + + # Skip infrastructure nodes + if not self._is_actual_device(node): + lopper.log._debug(f" Skipping infrastructure firmware: {node.name}") + continue + + seen.add(node.abs_path) + + # Use label or name + dev_name = node.label if node.label else node.name + entry = {'dev': dev_name} + if node.label and node.label != dev_name: + entry['label'] = node.label + + firmware_devices.append(entry) + lopper.log._debug(f" Found firmware node: {dev_name}") + except: + pass + + # Find IPI controller nodes (but not mailbox children) + # We only want addressable IPI controllers, not destination mailboxes + for node in self.sdt.tree: + # Skip non-addressable nodes + if '@' not in node.name: + continue + + # Skip if not an actual device (filters out dest-mailbox children) + if not self._is_actual_device(node): + continue + + compat = node.propval("compatible") + if compat: + compat_str = ' '.join(str(c) for c in compat) + # Only include IPI controllers, not mailbox destinations + if 'ipi' in compat_str.lower() and 'dest' not in compat_str.lower(): + if node.abs_path not in seen: + seen.add(node.abs_path) + + dev_name = node.label if node.label else node.name + entry = {'dev': dev_name} + if node.label: + entry['label'] = node.label + + firmware_devices.append(entry) + lopper.log._debug(f" Found IPI controller: {dev_name}") + + lopper.log._info(f"Discovered {len(firmware_devices)} firmware nodes") + return firmware_devices + + def discover_toplevel(self): + """Find non-bus devices directly under root. + + Discovers devices that are direct children of / but are not + bus nodes, CPU clusters, or special nodes. + + Returns: + list: List of toplevel device dictionaries + """ + toplevel_devices = [] + seen = set() + + # Nodes to skip (buses, special nodes, etc.) + skip_patterns = [ + r'^cpus', + r'^memory@', + r'^reserved-memory$', + r'^firmware$', + r'^chosen$', + r'^aliases$', + r'^__symbols__$', + r'^__fixups__$', + r'^__local_fixups__$', + ] + + try: + root = self.sdt.tree["/"] + for node in root.child_nodes.values(): + # Skip special nodes + skip = False + for pattern in skip_patterns: + if re.search(pattern, node.name): + skip = True + break + + if skip: + continue + + # Skip bus nodes (they're handled by discover_bus_devices) + compat = node.propval("compatible") + if compat: + compat_str = ' '.join(str(c) for c in compat) + if 'simple-bus' in compat_str: + continue + + # Skip infrastructure nodes (clocks, etc.) + if not self._is_actual_device(node): + lopper.log._debug(f" Skipping infrastructure toplevel: {node.name}") + continue + + if node.abs_path in seen: + continue + seen.add(node.abs_path) + + dev_name = node.label if node.label else node.name + entry = {'dev': dev_name} + if node.label: + entry['label'] = node.label + + toplevel_devices.append(entry) + lopper.log._debug(f" Found toplevel: {dev_name}") + + except: + pass + + lopper.log._info(f"Discovered {len(toplevel_devices)} toplevel nodes") + return toplevel_devices + + def discover_all(self, categories=None, bus_types=None, + include_pattern=None, exclude_pattern=None): + """Orchestrate discovery based on selected categories. + + Args: + categories (list): List of DeviceCategory to include (default: all) + bus_types (list): Bus compatible strings for bus device discovery + include_pattern (str): Regex pattern - only include matching devices + exclude_pattern (str): Regex pattern - exclude matching devices + + Returns: + dict: Dictionary with 'access', 'cpus', 'memory', 'sram' lists + """ + if categories is None: + categories = DeviceCategory.all_categories() + + if bus_types is None: + bus_types = self.DEFAULT_BUS_TYPES + + devices = { + 'access': [], + 'cpus': [], + 'memory': [], + 'sram': [] + } + + # Discover by category + if DeviceCategory.BUS in categories: + bus_devices = self.discover_bus_devices(bus_types) + bus_devices = self._apply_pattern_filter(bus_devices, include_pattern, exclude_pattern) + devices['access'].extend(bus_devices) + + if DeviceCategory.CPU in categories: + cpu_devices = self.discover_cpus() + cpu_devices = self._apply_pattern_filter(cpu_devices, include_pattern, exclude_pattern) + devices['cpus'].extend(cpu_devices) + + if DeviceCategory.MEMORY in categories: + mem_devices = self.discover_memory() + mem_devices['memory'] = self._apply_pattern_filter( + mem_devices['memory'], include_pattern, exclude_pattern) + mem_devices['sram'] = self._apply_pattern_filter( + mem_devices['sram'], include_pattern, exclude_pattern) + devices['memory'].extend(mem_devices['memory']) + devices['sram'].extend(mem_devices['sram']) + + if DeviceCategory.FIRMWARE in categories: + fw_devices = self.discover_firmware() + fw_devices = self._apply_pattern_filter(fw_devices, include_pattern, exclude_pattern) + devices['access'].extend(fw_devices) + + if DeviceCategory.TOPLEVEL in categories: + top_devices = self.discover_toplevel() + top_devices = self._apply_pattern_filter(top_devices, include_pattern, exclude_pattern) + devices['access'].extend(top_devices) + + return devices + + def generate_domain(self, domain_name='sdt_all_devices', categories=None, + bus_types=None, include_pattern=None, exclude_pattern=None): + """Generate a domain node containing discovered devices. + + Creates a tree structure with separate properties for different + device types (matching isospec format): + /domains + / + compatible = "openamp,domain-v1,devices" + id = 0 + cpus: [...] + memory: [...] + sram: [...] + access: [...] + + Args: + domain_name (str): Name for the generated domain node + categories (list): List of DeviceCategory to include + bus_types (list): List of bus compatible strings to search + include_pattern (str): Regex pattern for including devices + exclude_pattern (str): Regex pattern for excluding devices + + Returns: + LopperTree: Tree containing the generated domain + """ + # Create fresh tree + self.tree = LopperTree() + self.tree.phandle_resolution = False + + # Create /domains container + domains = LopperNode(abspath="/domains", name="domains") + domains.phandle_resolution = False + self.tree = self.tree + domains + + # Create the device domain + domain = LopperNode(name=domain_name) + domain.phandle_resolution = False + domain["compatible"] = "openamp,domain-v1,devices" + domain["id"] = 0 + domains + domain + + # Discover all devices based on categories + devices = self.discover_all( categories=categories, + bus_types=bus_types, + include_pattern=include_pattern, + exclude_pattern=exclude_pattern ) + + # Add cpus property if we have CPUs + if devices['cpus']: + cpus_prop = LopperProp("cpus", -1, domain, []) + cpus_prop.phandle_resolution = False + domain + cpus_prop + for cpu in devices['cpus']: + cpus_prop.value.append(cpu) + lopper.log._info(f"Added {len(devices['cpus'])} CPU entries") + + # Add memory property if we have memory + if devices['memory']: + memory_prop = LopperProp("memory", -1, domain, []) + memory_prop.phandle_resolution = False + domain + memory_prop + for mem in devices['memory']: + memory_prop.value.append(mem) + lopper.log._info(f"Added {len(devices['memory'])} memory entries") + + # Add sram property if we have SRAM + if devices['sram']: + sram_prop = LopperProp("sram", -1, domain, []) + sram_prop.phandle_resolution = False + domain + sram_prop + for sram in devices['sram']: + sram_prop.value.append(sram) + lopper.log._info(f"Added {len(devices['sram'])} SRAM entries") + + # Add access property for bus/firmware/toplevel devices + if devices['access']: + access_prop = LopperProp("access", -1, domain, []) + access_prop.phandle_resolution = False + domain + access_prop + for dev in devices['access']: + access_prop.value.append(dev) + lopper.log._info(f"Added {len(devices['access'])} access entries") + + total = (len(devices['cpus']) + len(devices['memory']) + + len(devices['sram']) + len(devices['access'])) + lopper.log._info(f"Generated domain '{domain_name}' with {total} total entries") + + return self.tree + + +def sdt_devices(tgt_node, sdt, options): + """Generate YAML domain with all SDT devices. + + This is the main entry point called by the lopper assist framework. + + Args: + tgt_node (LopperNode): Target node (typically root /) + sdt (LopperSDT): System device tree instance + options (dict): Options dictionary with 'verbose' and 'args' keys + + Returns: + bool: True on success, False on failure + """ + try: + verbose = options['verbose'] + except: + verbose = 0 + + try: + args = options['args'] + except: + args = [] + + # Parse command-line options + try: + opts, args2 = getopt.getopt( + args, + "hvb:n:o:c:", + ["help", "verbose", "bus-types=", "domain-name=", + "categories=", "exclude-categories=", + "include-pattern=", "exclude-pattern=", + "include-clocks", "include-infrastructure=", "list-infrastructure"] + ) + except getopt.GetoptError as e: + lopper.log._error(f"Invalid option: {e}") + usage() + return False + + # Default values + bus_types = SDTDevices.DEFAULT_BUS_TYPES + domain_name = 'sdt_all_devices' + output_file = None + categories = None # None means all categories + exclude_categories = [] + include_pattern = None + exclude_pattern = None + include_clocks = False + include_infrastructure = [] + + for o, a in opts: + if o in ('-h', '--help'): + usage() + return True + elif o in ('--list-infrastructure',): + list_infrastructure() + return True + elif o in ('-v', '--verbose'): + verbose = verbose + 1 + elif o in ('-b', '--bus-types'): + bus_types = [t.strip() for t in a.split(',')] + elif o in ('-n', '--domain-name'): + domain_name = a + elif o in ('-o'): + output_file = a + elif o in ('-c', '--categories'): + categories = DeviceCategory.parse_list(a) + elif o in ('--exclude-categories',): + exclude_categories = DeviceCategory.parse_list(a) + elif o in ('--include-pattern',): + include_pattern = a + elif o in ('--exclude-pattern',): + exclude_pattern = a + elif o in ('--include-clocks',): + include_clocks = True + elif o in ('--include-infrastructure',): + # Parse comma-separated infrastructure categories + infra_cats = [c.strip().lower() for c in a.split(',')] + for cat in infra_cats: + if cat == 'all': + include_infrastructure = ['all'] + break + elif cat in SDTDevices.INFRASTRUCTURE_CATEGORY_NAMES: + include_infrastructure.append(cat) + else: + lopper.log._warning(f"Unknown infrastructure category: {cat}") + lopper.log._warning(f"Valid categories: {', '.join(SDTDevices.INFRASTRUCTURE_CATEGORY_NAMES)}") + + # Handle category exclusions + if categories is None: + categories = DeviceCategory.all_categories() + if exclude_categories: + categories = [c for c in categories if c not in exclude_categories] + + # Set logging level based on verbosity + if verbose > 3: + desired_level = lopper.log.TRACE2 + elif verbose > 2: + desired_level = lopper.log.TRACE + elif verbose > 1: + desired_level = logging.DEBUG + elif verbose > 0: + desired_level = logging.INFO + else: + desired_level = None + + if desired_level is not None: + lopper.log._level(desired_level, __name__) + + cat_names = [c.value for c in categories] + lopper.log._info(f"sdt_devices: generating device list for domain '{domain_name}'") + lopper.log._info(f"sdt_devices: categories: {cat_names}") + lopper.log._info(f"sdt_devices: bus types: {bus_types}") + if include_pattern: + lopper.log._info(f"sdt_devices: include pattern: {include_pattern}") + if exclude_pattern: + lopper.log._info(f"sdt_devices: exclude pattern: {exclude_pattern}") + if include_clocks: + lopper.log._info(f"sdt_devices: including clock nodes") + if include_infrastructure: + lopper.log._info(f"sdt_devices: including infrastructure: {include_infrastructure}") + + # Create the generator and build the domain tree + generator = SDTDevices(sdt, include_clocks=include_clocks, + include_infrastructure=include_infrastructure) + tree = generator.generate_domain( domain_name=domain_name, + categories=categories, + bus_types=bus_types, + include_pattern=include_pattern, + exclude_pattern=exclude_pattern ) + + # Determine output file + if not output_file: + # Try to get output from sdt + if hasattr(sdt, 'output_file') and sdt.output_file: + output_file = sdt.output_file + else: + lopper.log._error("No output file specified") + usage() + return False + + # Ensure output file has .yaml extension for proper formatting + if not output_file.endswith('.yaml'): + base, _ = os.path.splitext(output_file) + output_file = base + '.yaml' + lopper.log._info(f"Output file changed to: {output_file}") + + # Ensure parent directory exists + output_dir = os.path.dirname(output_file) + if output_dir and not os.path.exists(output_dir): + os.makedirs(output_dir, exist_ok=True) + + # Write the output using LopperYAML directly + # This is more robust than sdt.write() as it doesn't require sdt.config + lopper.log._info(f"Writing SDT devices to: {output_file}") + try: + # Get config from sdt if available, otherwise use empty dict + config = getattr(sdt, 'config', {}) + yaml_writer = LopperYAML(None, tree, config=config) + yaml_writer.to_yaml(output_file) + except Exception as e: + lopper.log._error(f"Failed to write output: {e}") + return False + + lopper.log._info(f"Successfully generated {output_file}") + return True diff --git a/lopper/assists/yaml_to_dts_expansion.py b/lopper/assists/yaml_to_dts_expansion.py index 208002cd..2970e7ba 100755 --- a/lopper/assists/yaml_to_dts_expansion.py +++ b/lopper/assists/yaml_to_dts_expansion.py @@ -1064,7 +1064,11 @@ def cpu_expand( tree, subnode, verbose = 0): print( f" mode: {c['mode']}" ) if type(c) == dict: - cluster = c['cluster'] + # Support both 'cluster' (traditional) and 'dev' (sdt_devices) keys + cluster = c.get('cluster') or c.get('dev') + if not cluster: + # Skip CPU entries without cluster/dev info + continue if 'cluster_cpu' in c.keys(): cluster_cpu = c['cluster_cpu'] else: @@ -1100,35 +1104,38 @@ def cpu_expand( tree, subnode, verbose = 0): # */ if type(c) == dict: mode_mask = 0 - mode = c['mode'] - if mode: - try: - secure = mode['secure'] - if secure: - mode_mask = set_bit( mode_mask, 31 ) - except: - pass + try: + mode = c['mode'] + if mode: + try: + secure = mode['secure'] + if secure: + mode_mask = set_bit( mode_mask, 31 ) + except: + pass - try: - lockstep = mode['lockstep'] - if lockstep: - mode_mask = set_bit( mode_mask, 30 ) - except: - pass + try: + lockstep = mode['lockstep'] + if lockstep: + mode_mask = set_bit( mode_mask, 30 ) + except: + pass - try: - el = mode['el'] - if el: - mode_mask = set_bit( mode_mask, 0 ) - mode_mask = set_bit( mode_mask, 1 ) - except: - pass + try: + el = mode['el'] + if el: + mode_mask = set_bit( mode_mask, 0 ) + mode_mask = set_bit( mode_mask, 1 ) + except: + pass + except: + pass - mask = c['cpumask'] try: + mask = c['cpumask'] mask = int(mask,16) except: - pass + mask = 0 else: mode_mask = 0x0 mask = 0x0 diff --git a/lopper/audit/__init__.py b/lopper/audit/__init__.py index 347a4a88..f04e8214 100644 --- a/lopper/audit/__init__.py +++ b/lopper/audit/__init__.py @@ -73,24 +73,29 @@ # Re-export schema validation functions and classes from .schema import ( - # Enums + # Data classes (note: ConstraintType is re-exported from lopper.schema.core) ConstraintType, - # Data classes - PropertyConstraint, + PropertyConstraint, # Alias for lopper.schema.core.Constraint NodeConstraints, # Constraint definitions NODE_PROPERTY_CONSTRAINTS, - # Validation functions + # Validation functions (dt-schema based) check_forbidden_properties, check_required_properties, check_property_values, check_mutex_properties, + # Validation functions (learned schema based) + check_learned_type_violations, + check_type_frequency_anomalies, # Validator class SchemaValidator, # Convenience function validate_schema, ) +# Also export Constraint directly for code using the new unified name +from lopper.schema.core import Constraint + # Define what is exported when using `from lopper.audit import *` __all__ = [ # Base framework classes @@ -131,15 +136,19 @@ # Schema enums 'ConstraintType', # Schema data classes - 'PropertyConstraint', + 'Constraint', # New unified name + 'PropertyConstraint', # Alias for Constraint (backwards compatibility) 'NodeConstraints', # Schema constraints 'NODE_PROPERTY_CONSTRAINTS', - # Schema validation functions + # Schema validation functions (dt-schema based) 'check_forbidden_properties', 'check_required_properties', 'check_property_values', 'check_mutex_properties', + # Schema validation functions (learned schema based) + 'check_learned_type_violations', + 'check_type_frequency_anomalies', # Schema validator 'SchemaValidator', 'validate_schema', diff --git a/lopper/audit/schema.py b/lopper/audit/schema.py index 1262c626..7716cb9a 100644 --- a/lopper/audit/schema.py +++ b/lopper/audit/schema.py @@ -14,8 +14,9 @@ loading constraints from vendored dt-schema YAML files. Key components: -- ConstraintType: Enum of constraint types (required, forbidden, etc.) -- PropertyConstraint: Dataclass for individual property constraints +- ConstraintType: Enum of constraint types (from lopper.schema.core) +- Constraint: Dataclass for individual constraints (from lopper.schema.core) +- PropertyConstraint: Alias for Constraint (backwards compatibility) - NodeConstraints: Dataclass for node-level constraint groupings - load_constraints_from_schema(): Loads constraints from dt-schema YAML - Standalone check functions: check_forbidden_properties, etc. @@ -23,10 +24,12 @@ The constraints are loaded dynamically from YAML schema files in lopper/dt-schema/schemas/, making validation fully data-driven. + +Note: This module uses unified types from lopper.schema.core. The +PropertyConstraint name is preserved for backwards compatibility. """ from dataclasses import dataclass, field -from enum import Enum from fnmatch import fnmatch import glob import os @@ -48,35 +51,15 @@ ValidatorRegistry, ) +# Import unified types from lopper.schema.core +from lopper.schema.core import ConstraintType, Constraint -class ConstraintType(Enum): - """ - Property constraint types using dt-schema/JSON Schema terminology. - - These are lopper's own constraint implementations, using vocabulary - consistent with upstream dt-schema for familiarity. - """ - REQUIRED = "required" # Property must be present - FORBIDDEN = "forbidden" # Property must NOT be present - CONST = "const" # Property must have specific value - ENUM = "enum" # Property must be one of allowed values - MUTEX = "mutex" # Properties are mutually exclusive - +# Import unified property types for learned schema integration +from lopper.schema.types import PropertyType -@dataclass -class PropertyConstraint: - """A single property constraint. - - Attributes: - constraint_type: Type of constraint (required, forbidden, etc.) - properties: List of property names this constraint applies to - expected_value: Expected value for CONST/ENUM constraints - message: Optional human-readable message for violations - """ - constraint_type: ConstraintType - properties: list - expected_value: object = None - message: str = None +# Backwards compatibility alias: PropertyConstraint -> Constraint +# Existing code using PropertyConstraint will continue to work +PropertyConstraint = Constraint @dataclass @@ -85,9 +68,12 @@ class NodeConstraints: Attributes: node_pattern: Glob pattern matching node paths (e.g., "/memory@*") - constraints: List of PropertyConstraint objects + constraints: List of Constraint objects description: Optional description of this constraint set schema_file: Source schema file path + + Note: This is similar to lopper.schema.core.NodeSpec but includes + schema_file for audit diagnostics. Keeping locally for now. """ node_pattern: str constraints: list @@ -101,7 +87,7 @@ class NodeConstraints: def _get_schema_dir(): """Get the path to vendored dt-schema files.""" - return os.path.join(os.path.dirname(__file__), '..', 'dt-schema', 'schemas') + return os.path.join(os.path.dirname(__file__), '..', 'schema', 'dt-schema', 'schemas') def _parse_schema_file(schema_path): @@ -635,6 +621,369 @@ def check_mutex_properties(tree, constraints=None): return results +# ============================================================================= +# Learned Type Validation +# ============================================================================= + +def _infer_type_from_value(value): + """Infer PropertyType from a property value. + + Args: + value: Property value (may be wrapped in property object) + + Returns: + PropertyType enum value + """ + # Unwrap property object if needed + if hasattr(value, 'value'): + value = value.value + + if value is None or value == []: + return PropertyType.EMPTY + + if isinstance(value, bool): + return PropertyType.FLAG + + if isinstance(value, str): + # Empty string is also EMPTY/FLAG + if value == '': + return PropertyType.EMPTY + return PropertyType.STRING + + if isinstance(value, int): + # Could be uint8, uint16, uint32, uint64 - default to uint32 + if value < 0: + return PropertyType.INT32 + elif value <= 0xFF: + return PropertyType.UINT8 # Could be uint8 + elif value <= 0xFFFF: + return PropertyType.UINT16 # Could be uint16 + elif value <= 0xFFFFFFFF: + return PropertyType.UINT32 + else: + return PropertyType.UINT64 + + if isinstance(value, list): + if len(value) == 0: + return PropertyType.EMPTY + + first = value[0] + + # List containing only empty string is EMPTY/FLAG (e.g., ranges;) + if len(value) == 1 and first == '': + return PropertyType.EMPTY + + # List of strings + if isinstance(first, str): + if len(value) == 1: + return PropertyType.STRING + return PropertyType.STRING_ARRAY + + # List of integers + if isinstance(first, int): + # For arrays, report as array type + if len(value) == 1: + return PropertyType.UINT32 + return PropertyType.UINT32_ARRAY + + # Nested list (matrix) + if isinstance(first, list): + return PropertyType.UINT32_ARRAY + + return PropertyType.UNKNOWN + + +def _get_resolver(): + """Get the property type resolver from learned schema. + + Returns: + DTSPropertyTypeResolver instance, or None if unavailable + """ + try: + # Use internal _get_schema_manager() to avoid deprecation warnings + # in internal code paths. Public API users get the warning. + from lopper.schema.learned import _get_schema_manager + manager = _get_schema_manager() + if manager: + # Try both the method and the attribute + resolver = getattr(manager, 'resolver', None) + if resolver is None: + resolver = getattr(manager, '_resolver', None) + return resolver + except Exception: + pass + return None + + +def _types_compatible(actual, expected): + """Check if actual type is compatible with expected type. + + Some type mismatches are acceptable: + - UINT8/UINT16/UINT32/UINT64 are all "integers" + - STRING and STRING_ARRAY with single element + - Array variants of base types + - FLAG/EMPTY can also have values (ranges, dma-ranges) + + Args: + actual: PropertyType inferred from value + expected: PropertyType from learned schema + + Returns: + True if types are compatible + """ + if actual == expected: + return True + + # Integer types are compatible with each other (including arrays) + int_types = { + PropertyType.UINT8, PropertyType.UINT16, PropertyType.UINT32, PropertyType.UINT64, + PropertyType.INT8, PropertyType.INT16, PropertyType.INT32, PropertyType.INT64, + } + int_array_types = { + PropertyType.UINT8_ARRAY, PropertyType.UINT16_ARRAY, + PropertyType.UINT32_ARRAY, PropertyType.UINT64_ARRAY, + } + all_int_types = int_types | int_array_types + + # All integer types (scalar and array) are compatible + if actual in all_int_types and expected in all_int_types: + return True + + # Array types are compatible with scalar types + array_pairs = { + (PropertyType.UINT8, PropertyType.UINT8_ARRAY), + (PropertyType.UINT16, PropertyType.UINT16_ARRAY), + (PropertyType.UINT32, PropertyType.UINT32_ARRAY), + (PropertyType.UINT64, PropertyType.UINT64_ARRAY), + (PropertyType.STRING, PropertyType.STRING_ARRAY), + } + if (actual, expected) in array_pairs or (expected, actual) in array_pairs: + return True + + # PHANDLE is stored as UINT32 + if expected == PropertyType.PHANDLE and actual in (PropertyType.UINT32, PropertyType.UINT8, PropertyType.UINT16): + return True + if expected == PropertyType.PHANDLE_ARRAY and actual == PropertyType.UINT32_ARRAY: + return True + + # EMPTY and FLAG are compatible + if {actual, expected} == {PropertyType.EMPTY, PropertyType.FLAG}: + return True + + # FLAG/EMPTY can also have integer values (ranges, dma-ranges can be empty or have values) + if expected == PropertyType.FLAG and actual in all_int_types: + return True + + return False + + +def check_learned_type_violations(tree, min_confidence=0.8): + """Check properties match their learned types. + + This function validates that property values match the types learned + from observing device trees. High-confidence learned types are used + to detect anomalies. + + Args: + tree: LopperTree to validate + min_confidence: Minimum confidence threshold (0.0-1.0) + + Returns: + List of ValidationResult objects + """ + results = [] + resolver = _get_resolver() + + if not resolver: + results.append(ValidationResult( + check_name='schema_learned_types', + phase=ValidationPhase.POST_YAML, + passed=True, + message="No learned schema available - skipping type checks", + )) + return results + + violations = 0 + checked = 0 + + for node in tree: + # Get compatible string for context + compatible = None + try: + compat_prop = node['compatible'] + if compat_prop and hasattr(compat_prop, 'value'): + compatible = compat_prop.value + if isinstance(compatible, list) and len(compatible) > 0: + compatible = compatible[0] + except (KeyError, TypeError): + pass + + # Check each property (iterate over node yields property objects) + for prop in node: + try: + prop_name = prop.name + if prop is None: + continue + + # Get learned type specification + spec = resolver.resolve_property_spec(prop_name, node.abs_path, compatible) + + # Skip low-confidence or unknown types + if spec.confidence < min_confidence: + continue + if spec.type_def.property_type == PropertyType.UNKNOWN: + continue + + checked += 1 + + # Infer actual type from value + actual_type = _infer_type_from_value(prop) + + # Compare types + expected_type = spec.type_def.property_type + if not _types_compatible(actual_type, expected_type): + violations += 1 + results.append(ValidationResult( + check_name='schema_learned_types', + phase=ValidationPhase.POST_YAML, + passed=False, + message=f"{node.abs_path}: {prop_name} type mismatch - " + f"expected {expected_type.value}, got {actual_type.value}", + source_path=node.abs_path, + details={ + 'property': prop_name, + 'expected_type': expected_type.value, + 'actual_type': actual_type.value, + 'confidence': spec.confidence, + 'source': spec.source, + } + )) + + except Exception as e: + lopper.log._debug(f"schema: error checking {node.abs_path}/{prop_name}: {e}") + + if violations == 0: + results.append(ValidationResult( + check_name='schema_learned_types', + phase=ValidationPhase.POST_YAML, + passed=True, + message=f"Checked {checked} properties against learned types - no violations", + )) + + return results + + +def check_type_frequency_anomalies(tree, minority_threshold=0.1): + """Check for property type usage that differs from majority. + + Some properties have ambiguous types in the learned schema (seen as + both string and integer across different device trees). This check + identifies when a property uses a minority type. + + Args: + tree: LopperTree to validate + minority_threshold: Report if type frequency is below this (0.0-1.0) + + Returns: + List of ValidationResult objects + """ + results = [] + resolver = _get_resolver() + + if not resolver: + results.append(ValidationResult( + check_name='schema_type_frequency', + phase=ValidationPhase.POST_YAML, + passed=True, + message="No learned schema available - skipping frequency checks", + )) + return results + + anomalies = 0 + + for node in tree: + for prop in node: + try: + prop_name = prop.name + if prop is None: + continue + + # Get learned type specification + spec = resolver.resolve_property_spec(prop_name, node.abs_path) + + # Check if there are type frequencies + if not spec.type_frequencies: + continue + + # Calculate total observations + total = sum(spec.type_frequencies.values()) + if total < 5: # Not enough data + continue + + # Infer actual type + actual_type = _infer_type_from_value(prop) + + # Find matching frequency entry + # Type names in frequencies may be like 'uint32', 'string', etc. + actual_name = actual_type.value + freq = spec.type_frequencies.get(actual_name, 0) + + # Also check array variants + if freq == 0 and actual_name.endswith('-array'): + base_name = actual_name.replace('-array', '') + freq = spec.type_frequencies.get(base_name, 0) + + if freq == 0: + # Type not seen before at all + anomalies += 1 + results.append(ValidationResult( + check_name='schema_type_frequency', + phase=ValidationPhase.POST_YAML, + passed=False, + message=f"{node.abs_path}: {prop_name} uses unseen type {actual_name} " + f"(known types: {list(spec.type_frequencies.keys())})", + source_path=node.abs_path, + details={ + 'property': prop_name, + 'actual_type': actual_name, + 'type_frequencies': spec.type_frequencies, + } + )) + elif freq / total < minority_threshold: + # Type is rare + anomalies += 1 + pct = (freq / total) * 100 + results.append(ValidationResult( + check_name='schema_type_frequency', + phase=ValidationPhase.POST_YAML, + passed=False, + message=f"{node.abs_path}: {prop_name} uses minority type {actual_name} " + f"({pct:.1f}% of observations)", + source_path=node.abs_path, + details={ + 'property': prop_name, + 'actual_type': actual_name, + 'frequency': freq, + 'total': total, + 'percentage': pct, + 'type_frequencies': spec.type_frequencies, + } + )) + + except Exception as e: + lopper.log._debug(f"schema: error checking frequencies for {prop_name}: {e}") + + if anomalies == 0: + results.append(ValidationResult( + check_name='schema_type_frequency', + phase=ValidationPhase.POST_YAML, + passed=True, + message="No type frequency anomalies detected", + )) + + return results + + # ============================================================================= # Schema Validator Class # ============================================================================= @@ -659,6 +1008,8 @@ class SchemaValidator(BaseValidator): 'schema_required_props', 'schema_prop_values', 'schema_mutex_props', + 'schema_learned_types', + 'schema_type_frequency', ] # Meta-flags that enable multiple checks @@ -668,11 +1019,17 @@ class SchemaValidator(BaseValidator): 'schema_required_props', 'schema_prop_values', 'schema_mutex_props', + 'schema_learned_types', + 'schema_type_frequency', ], 'schema_reserved_memory': [ 'schema_forbidden_props', 'schema_mutex_props', ], + 'schema_learned': [ + 'schema_learned_types', + 'schema_type_frequency', + ], } # Map of warning flags to check functions and their phases @@ -681,6 +1038,8 @@ class SchemaValidator(BaseValidator): 'schema_required_props': (ValidationPhase.POST_YAML, check_required_properties), 'schema_prop_values': (ValidationPhase.POST_YAML, check_property_values), 'schema_mutex_props': (ValidationPhase.POST_YAML, check_mutex_properties), + 'schema_learned_types': (ValidationPhase.POST_YAML, check_learned_type_violations), + 'schema_type_frequency': (ValidationPhase.POST_YAML, check_type_frequency_anomalies), } def run_phase(self, phase, tree, **kwargs): diff --git a/lopper/lops/lop-sdt-devices.dts b/lopper/lops/lop-sdt-devices.dts new file mode 100644 index 00000000..31606d90 --- /dev/null +++ b/lopper/lops/lop-sdt-devices.dts @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Author: + * Bruce Ashfield + * + * SPDX-License-Identifier: BSD-3-Clause + * + * This lop file invokes the sdt_devices assist to generate a YAML domain + * containing all devices from the System Device Tree. The generated YAML + * can be used as a parent domain for glob-based device matching. + * + * Usage: + * lopper -i lop-sdt-devices.dts system-top.dts output.yaml + * + * The assist will scan all simple-bus compatible nodes and generate + * a domain with all addressable devices in its access list. + */ + +/dts-v1/; + +/ { + compatible = "system-device-tree-v1,lop"; + lops { + lop_sdt_devices { + compatible = "system-device-tree-v1,lop,assist-v1"; + node = "/"; + id = "module,sdt_devices"; + }; + }; +}; diff --git a/lopper/schema/__init__.py b/lopper/schema/__init__.py new file mode 100644 index 00000000..e28e3387 --- /dev/null +++ b/lopper/schema/__init__.py @@ -0,0 +1,196 @@ +#/* +# * Copyright (c) 2026 AMD Inc. All rights reserved. +# * +# * Author: +# * Bruce Ashfield +# * +# * SPDX-License-Identifier: BSD-3-Clause +# */ + +""" +Lopper Schema Package + +Unified schema system for device tree property types and constraints. + +This package provides: +- Type definitions from dt-schema (uint32, phandle, string, etc.) +- Constraint definitions (required, forbidden, const, enum, mutex) +- Schema registry for type resolution +- Schema loader with search path for user/vendor extensions + +For backwards compatibility, all exports from the old lopper/schema.py +are re-exported here. New code should use the unified types. + +Schema Search Path (lowest to highest priority): +1. Built-in dt-schema (lopper/schema/dt-schema/schemas/) +2. User schemas (~/.config/lopper/schemas/) +3. LOPPER_SCHEMA_PATH environment variable +4. --schema-dir command-line option + +Deprecation Notes: +- get_schema_manager() is deprecated, use get_registry() for new code +- SchemaManager is deprecated, use SchemaRegistry for new code +- DTSPropertyTypeResolver.get_property_type() is deprecated, + use resolve_property_spec() which returns a PropertySpec +""" + +import warnings + +# New unified types +from .types import ( + PropertyType, + TypeDefinition, + DT_SCHEMA_TYPES, +) + +# Core data structures +from .core import ( + ConstraintType, + Constraint, + PropertySpec, + NodeSpec, + SchemaRegistry, + get_registry, + reset_registry, +) + +# Loader functions +from .loader import ( + get_schema_search_path, + load_all_schemas, +) + +# Backwards compatibility: re-export everything from learned.py +# This ensures `from lopper.schema import ...` continues to work +from .learned import ( + # Constants + PROPERTY_DEBUG_LIST, + PROPERTY_DEBUG_SET, + PROPERTY_NAME_HEURISTICS, + PROPERTY_TYPE_HINTS, + # Schema manager (legacy API - see deprecation wrappers below) + SchemaManager as _SchemaManager, + get_schema_manager as _get_schema_manager, + _schema_manager, # Private singleton accessed by lopper/__init__.py + # Schema generator + DTSSchemaGenerator, + # Type resolver + DTSPropertyTypeResolver, + # Type checker + DTSTypeChecker, + # Validator + DTSValidator, + # Helper functions + add_property_type_hint, + add_property_heuristic, + create_all_from_schema, + update_schema, + update_schema_with_property, + update_schema_with_node_pattern, + initialize_lopper_properties, + schema_has_definition, + property_exists_in_schema, + get_property_info, + create_property_resolver, + generate_schema_from_dts, + schema_add_runtime_property, + schema_get_resolver, +) + + +# ============================================================================= +# Deprecation Wrappers +# ============================================================================= + +# Control deprecation warnings - can be disabled for testing +_DEPRECATION_WARNINGS_ENABLED = True + + +def _deprecation_warning(message): + """Issue a deprecation warning if enabled.""" + if _DEPRECATION_WARNINGS_ENABLED: + warnings.warn(message, DeprecationWarning, stacklevel=3) + + +def get_schema_manager(): + """Get the global schema manager instance. + + .. deprecated:: + Use :func:`get_registry` for new code. The SchemaRegistry provides + a unified interface for type resolution and constraint checking. + + Returns: + SchemaManager instance (legacy API) + """ + _deprecation_warning( + "get_schema_manager() is deprecated. " + "Use lopper.schema.get_registry() for new code. " + "The SchemaRegistry provides unified type resolution." + ) + return _get_schema_manager() + + +class SchemaManager(_SchemaManager): + """Legacy schema manager for learned property types. + + .. deprecated:: + Use :class:`SchemaRegistry` for new code. The SchemaRegistry provides + a unified interface combining dt-schema types, learned types, and + constraint validation. + + This class is maintained for backwards compatibility. New code should use: + - SchemaRegistry for type resolution + - PropertySpec for complete property specifications + - get_registry() to access the global registry + """ + + def __new__(cls): + # Issue deprecation warning before singleton creation + _deprecation_warning( + "SchemaManager is deprecated. " + "Use lopper.schema.SchemaRegistry for new code." + ) + return super().__new__(cls) + +__all__ = [ + # New unified types + 'PropertyType', + 'TypeDefinition', + 'DT_SCHEMA_TYPES', + # Core data structures + 'ConstraintType', + 'Constraint', + 'PropertySpec', + 'NodeSpec', + 'SchemaRegistry', + 'get_registry', + 'reset_registry', + # Loader + 'get_schema_search_path', + 'load_all_schemas', + # Legacy (from learned.py) + 'PROPERTY_DEBUG_LIST', + 'PROPERTY_DEBUG_SET', + 'PROPERTY_NAME_HEURISTICS', + 'PROPERTY_TYPE_HINTS', + 'SchemaManager', + 'get_schema_manager', + 'DTSSchemaGenerator', + 'DTSPropertyTypeResolver', + 'DTSTypeChecker', + 'DTSValidator', + 'add_property_type_hint', + 'add_property_heuristic', + 'create_all_from_schema', + 'update_schema', + 'update_schema_with_property', + 'update_schema_with_node_pattern', + 'initialize_lopper_properties', + 'schema_has_definition', + 'property_exists_in_schema', + 'get_property_info', + 'create_property_resolver', + 'generate_schema_from_dts', + 'schema_add_runtime_property', + 'schema_get_resolver', +] diff --git a/lopper/schema/core.py b/lopper/schema/core.py new file mode 100644 index 00000000..7a99e414 --- /dev/null +++ b/lopper/schema/core.py @@ -0,0 +1,382 @@ +#/* +# * Copyright (c) 2026 AMD Inc. All rights reserved. +# * +# * Author: +# * Bruce Ashfield +# * +# * SPDX-License-Identifier: BSD-3-Clause +# */ + +""" +Core schema data structures and registry. + +This module provides: +- ConstraintType: Enum for constraint kinds (required, forbidden, etc.) +- Constraint: Individual constraint definition +- PropertySpec: Complete property specification (type + constraints) +- NodeSpec: Node-level constraints by path pattern +- SchemaRegistry: Unified registry for all schema information +""" + +from dataclasses import dataclass, field +from enum import Enum +from fnmatch import fnmatch +from typing import List, Dict, Optional, Any + +from .types import PropertyType, TypeDefinition, DT_SCHEMA_TYPES + + +class ConstraintType(Enum): + """Constraint types for property validation. + + Uses JSON Schema / dt-schema vocabulary for familiarity. + """ + REQUIRED = "required" # Property must exist + FORBIDDEN = "forbidden" # Property must NOT exist + CONST = "const" # Property must equal specific value + ENUM = "enum" # Property must be one of allowed values + MUTEX = "mutex" # Properties are mutually exclusive + RANGE = "range" # Value must be in min/max range + PATTERN = "pattern" # String must match regex + + +@dataclass +class Constraint: + """A single constraint on a property or set of properties. + + Attributes: + constraint_type: Kind of constraint + properties: Property names this constraint applies to + expected_value: Value for CONST/ENUM constraints + message: Human-readable error message + """ + constraint_type: ConstraintType + properties: List[str] + expected_value: Optional[Any] = None + message: Optional[str] = None + + +@dataclass +class PropertySpec: + """Complete specification for a property. + + Unifies type information (from dt-schema or learned) with + constraints (required, forbidden, etc.). + + Attributes: + name: Property name + type_def: Type definition with validation bounds + constraints: List of constraints on this property + confidence: 1.0 = authoritative (dt-schema), <1.0 = inferred + source: Origin ("dt-schema", "learned", "heuristic") + context: Compatible string or node pattern for context-specific specs + observation_count: Number of times observed (for learned) + type_frequencies: Type occurrence counts for ambiguous properties + phandle_pattern: Detected phandle pattern (e.g., "phandle + 2 cells") + context_lookups: Properties to look up for context (e.g., "#clock-cells") + """ + name: str + type_def: TypeDefinition + constraints: List[Constraint] = field(default_factory=list) + confidence: float = 1.0 + source: str = "unknown" + context: Optional[str] = None + observation_count: int = 0 + type_frequencies: Dict[str, int] = field(default_factory=dict) + phandle_pattern: Optional[str] = None + context_lookups: List[str] = field(default_factory=list) + + +@dataclass +class NodeSpec: + """Specification for a class of nodes. + + Matches nodes by path pattern and defines property constraints. + + Attributes: + node_pattern: Glob pattern (e.g., "/memory@*", "/reserved-memory/*") + properties: Property specifications for this node type + constraints: Node-level constraints (e.g., required properties) + description: Human-readable description + schema_file: Source schema file path + """ + node_pattern: str + properties: Dict[str, PropertySpec] = field(default_factory=dict) + constraints: List[Constraint] = field(default_factory=list) + description: Optional[str] = None + schema_file: Optional[str] = None + + +class SchemaRegistry: + """Unified registry for all schema information. + + Combines: + - dt-schema authoritative specs (highest priority) + - Learned property types from observation + - Name-based heuristics (lowest priority) + + Resolution priority: dt-schema > learned > heuristics + """ + + def __init__(self): + self._node_specs: Dict[str, NodeSpec] = {} + self._property_specs: Dict[str, PropertySpec] = {} + self._compatible_specs: Dict[str, Dict[str, PropertySpec]] = {} + self._type_defs: Dict[str, TypeDefinition] = {} + self._initialized = False + + def register_type(self, name: str, type_def: TypeDefinition): + """Register a type definition. + + Args: + name: Type name (e.g., "uint32", "phandle") + type_def: Type definition with constraints + """ + self._type_defs[name] = type_def + + def get_type(self, name: str) -> Optional[TypeDefinition]: + """Get a registered type definition. + + Args: + name: Type name + + Returns: + TypeDefinition or None if not found + """ + return self._type_defs.get(name) + + def register_node_spec(self, name: str, spec: NodeSpec): + """Register a node specification. + + Args: + name: Identifier for this spec (usually schema filename) + spec: Node specification with pattern and constraints + """ + self._node_specs[name] = spec + + def get_node_spec(self, name: str) -> Optional[NodeSpec]: + """Get a registered node specification by name.""" + return self._node_specs.get(name) + + def register_property_spec( + self, + spec: PropertySpec, + context: Optional[str] = None + ): + """Register a property specification. + + Args: + spec: Property specification + context: Optional compatible string for context-specific registration + """ + if context: + if context not in self._compatible_specs: + self._compatible_specs[context] = {} + self._compatible_specs[context][spec.name] = spec + else: + self._property_specs[spec.name] = spec + + def resolve_property_type( + self, + prop_name: str, + node_path: Optional[str] = None, + compatible: Optional[str] = None + ) -> PropertySpec: + """Resolve the type specification for a property. + + Resolution order (highest to lowest priority): + 1. Node-specific (by path pattern match) + 2. Compatible-specific + 3. Global property spec + 4. Name heuristics + 5. Unknown + + Args: + prop_name: Property name to resolve + node_path: Optional node path for context + compatible: Optional compatible string for context + + Returns: + PropertySpec with type and constraints + """ + # 1. Check node-specific specs (highest priority for path matches) + if node_path: + for spec in self._node_specs.values(): + if self._matches_pattern(node_path, spec.node_pattern): + if prop_name in spec.properties: + return spec.properties[prop_name] + + # 2. Check compatible-specific specs + if compatible and compatible in self._compatible_specs: + if prop_name in self._compatible_specs[compatible]: + return self._compatible_specs[compatible][prop_name] + + # 3. Check global property specs + if prop_name in self._property_specs: + return self._property_specs[prop_name] + + # 4. Apply name heuristics + heuristic_spec = self._apply_heuristics(prop_name) + if heuristic_spec: + return heuristic_spec + + # 5. Unknown + return PropertySpec( + name=prop_name, + type_def=TypeDefinition(PropertyType.UNKNOWN, source="unknown"), + confidence=0.0, + source="unknown" + ) + + def get_node_constraints(self, node_path: str) -> List[Constraint]: + """Get all constraints that apply to a node path. + + Args: + node_path: Absolute node path (e.g., "/reserved-memory/region1") + + Returns: + List of Constraint objects from all matching node specs + """ + constraints = [] + for spec in self._node_specs.values(): + if self._matches_pattern(node_path, spec.node_pattern): + constraints.extend(spec.constraints) + return constraints + + def get_matching_node_specs(self, node_path: str) -> List[NodeSpec]: + """Get all node specs that match a path. + + Args: + node_path: Absolute node path + + Returns: + List of matching NodeSpec objects + """ + return [ + spec for spec in self._node_specs.values() + if self._matches_pattern(node_path, spec.node_pattern) + ] + + def _matches_pattern(self, path: str, pattern: str) -> bool: + """Check if a node path matches a constraint pattern. + + Supports: + - Exact match: "/reserved-memory" matches "/reserved-memory" + - Child wildcard: "/reserved-memory/*" matches "/reserved-memory/foo" + - Unit address wildcard: "/memory@*" matches "/memory@0" + + Args: + path: Node path to test + pattern: Pattern to match against + + Returns: + True if path matches pattern + """ + if pattern == path: + return True + + # Child wildcard: /parent/* matches /parent/child + if pattern.endswith('/*'): + parent_pattern = pattern[:-2] + if '/' not in path.lstrip('/'): + return False + parent_path = path.rsplit('/', 1)[0] + if not parent_path: + parent_path = '/' + return parent_path == parent_pattern + + # Unit address wildcard: /node@* matches /node@0, /node@80000000 + if '@*' in pattern: + base = pattern.replace('@*', '@') + return path.startswith(base) + + # General glob + return fnmatch(path, pattern) + + def _apply_heuristics(self, prop_name: str) -> Optional[PropertySpec]: + """Apply name-based heuristics to infer property type. + + Uses PROPERTY_NAME_HEURISTICS from learned.py for suffix/prefix rules. + + Args: + prop_name: Property name + + Returns: + PropertySpec with heuristic type, or None + """ + try: + from .learned import PROPERTY_NAME_HEURISTICS + except ImportError: + return None + + # Check exact matches first + exact = PROPERTY_NAME_HEURISTICS.get('exact', {}) + if prop_name in exact: + lopper_fmt = exact[prop_name] + return PropertySpec( + name=prop_name, + type_def=TypeDefinition( + property_type=PropertyType.from_lopper_fmt(lopper_fmt), + source="heuristic" + ), + confidence=0.7, + source="heuristic" + ) + + # Check suffix patterns + suffixes = PROPERTY_NAME_HEURISTICS.get('suffixes', {}) + for suffix, lopper_fmt in suffixes.items(): + if prop_name.endswith(suffix): + return PropertySpec( + name=prop_name, + type_def=TypeDefinition( + property_type=PropertyType.from_lopper_fmt(lopper_fmt), + source="heuristic" + ), + confidence=0.5, + source="heuristic" + ) + + return None + + def list_node_specs(self) -> Dict[str, NodeSpec]: + """Get all registered node specs.""" + return self._node_specs.copy() + + def list_property_specs(self) -> Dict[str, PropertySpec]: + """Get all registered global property specs.""" + return self._property_specs.copy() + + def list_types(self) -> Dict[str, TypeDefinition]: + """Get all registered type definitions.""" + return self._type_defs.copy() + + +# Global registry instance +_registry: Optional[SchemaRegistry] = None + + +def get_registry() -> SchemaRegistry: + """Get the global schema registry, initializing if needed. + + The registry is lazily initialized on first access and loads: + 1. Built-in dt-schema type definitions + 2. Schemas from the search path + + Returns: + The global SchemaRegistry instance + """ + global _registry + if _registry is None: + _registry = SchemaRegistry() + # Register built-in types + for name, type_def in DT_SCHEMA_TYPES.items(): + _registry.register_type(name, type_def) + _registry._initialized = True + return _registry + + +def reset_registry(): + """Reset the global registry (mainly for testing).""" + global _registry + _registry = None diff --git a/lopper/dt-schema/schemas/memory.yaml b/lopper/schema/dt-schema/schemas/memory.yaml similarity index 100% rename from lopper/dt-schema/schemas/memory.yaml rename to lopper/schema/dt-schema/schemas/memory.yaml diff --git a/lopper/dt-schema/schemas/reserved-memory/reserved-memory.yaml b/lopper/schema/dt-schema/schemas/reserved-memory/reserved-memory.yaml similarity index 100% rename from lopper/dt-schema/schemas/reserved-memory/reserved-memory.yaml rename to lopper/schema/dt-schema/schemas/reserved-memory/reserved-memory.yaml diff --git a/lopper/dt-schema/schemas/types.yaml b/lopper/schema/dt-schema/schemas/types.yaml similarity index 100% rename from lopper/dt-schema/schemas/types.yaml rename to lopper/schema/dt-schema/schemas/types.yaml diff --git a/lopper/schema.py b/lopper/schema/learned.py similarity index 95% rename from lopper/schema.py rename to lopper/schema/learned.py index fa5122bc..eb4714a4 100644 --- a/lopper/schema.py +++ b/lopper/schema/learned.py @@ -18,6 +18,10 @@ from lopper.log import _warning, _info, _error, _debug, _init, _level import logging +# Import unified types for Phase 2 integration +from .types import PropertyType, TypeDefinition +from .core import PropertySpec + _init( __name__ ) _init( "schema.py" ) @@ -170,7 +174,6 @@ 'device_type', 'label', 'bootargs', - 'stdout-path', 'phy-mode', 'dr_mode', 'maximum-speed', @@ -178,6 +181,14 @@ 'entry-method', ], + # Properties whose string value is an alias name (key in /aliases), optionally + # followed by :options (e.g. "serial0:115200n8"). These are checked at runtime + # against the live /aliases node; the list here seeds the known-good set. + 'alias_ref_properties': [ + 'stdout-path', + 'linux,stdout-path', + ], + # Properties that are always boolean (empty) 'boolean_properties': [ 'no-map', @@ -706,6 +717,9 @@ def _determine_property_type(self, name, value): value = value.strip() # Check explicit type hints first + if name in self.type_hints.get('alias_ref_properties', []): + return 'alias-ref' if '"' in value else 'unknown' + if name in self.type_hints.get('string_properties', []): return 'string' if '"' in value else 'unknown' @@ -782,8 +796,11 @@ def _determine_property_type(self, name, value): # Has quotes, so it's a string type if '", "' in value or '","' in value: return 'string-array' - else: - return 'string' + # Path-ref: quoted string whose content starts with '/' (absolute node path) + inner = value.strip('"') + if inner.startswith('/'): + return 'path-ref' + return 'string' else: return 'unknown' @@ -1509,6 +1526,11 @@ def get_property_type(self, prop_name, node_path=None, compatible=None): """ Get LopperFmt type for a property. + .. deprecated:: + For new code, use :meth:`resolve_property_spec` which returns + a :class:`PropertySpec` with richer type information including + confidence scores, type frequencies, and phandle patterns. + Args: prop_name: Property name node_path: Full path to node (e.g., "/soc/uart@ff000000") @@ -1632,6 +1654,95 @@ def get_common_properties(self): """Get a dictionary of all common property types for quick reference""" return self._property_types.copy() + def resolve_property_spec(self, prop_name, node_path=None, compatible=None): + """ + Get unified PropertySpec for a property. + + This is the Phase 2 unified API that returns PropertySpec objects + compatible with the new schema system. It wraps get_property_type() + to maintain backwards compatibility while providing richer type info. + + Args: + prop_name: Property name + node_path: Full path to node (e.g., "/soc/uart@ff000000") + compatible: Compatible string(s) for the node + + Returns: + PropertySpec with type information and metadata + """ + # Get the LopperFmt type using existing logic + lopper_fmt = self.get_property_type(prop_name, node_path, compatible) + + # Convert to PropertyType + prop_type = PropertyType.from_lopper_fmt(lopper_fmt) + + # Determine source and confidence based on how it was resolved + source = "unknown" + confidence = 0.0 + + # Check resolution path to determine source + if node_path and node_path in self._path_properties: + path_props = self._path_properties[node_path].get('properties', {}) + if prop_name in path_props: + source = "learned-path" + confidence = 0.9 + + elif compatible: + compat_list = [compatible] if isinstance(compatible, str) else compatible + for compat in compat_list: + if compat in self._compatible_properties: + if prop_name in self._compatible_properties[compat]: + source = "learned-compatible" + confidence = 0.85 + break + + elif prop_name in self._property_types: + source = "learned" + confidence = 0.8 + + elif prop_name in PROPERTY_NAME_HEURISTICS.get('exact', {}): + source = "heuristic" + confidence = 0.7 + + elif any(prop_name.endswith(s) for s in PROPERTY_NAME_HEURISTICS.get('suffixes', {})): + source = "heuristic" + confidence = 0.5 + + else: + source = "unknown" + confidence = 0.0 + + # Get type frequencies if available + type_frequencies = {} + prop_def = self.schema.get('property_definitions', {}).get(prop_name, {}) + if '_type_frequencies' in prop_def: + type_frequencies = prop_def['_type_frequencies'] + + # Get phandle pattern if available + phandle_pattern = prop_def.get('phandle-pattern') + + # Get context lookups if available + context_lookups = prop_def.get('context-lookups', []) + + # Build TypeDefinition + type_def = TypeDefinition( + property_type=prop_type, + source=source, + description=prop_def.get('description') + ) + + # Build and return PropertySpec + return PropertySpec( + name=prop_name, + type_def=type_def, + confidence=confidence, + source=source, + context=compatible if compatible else node_path, + type_frequencies=type_frequencies, + phandle_pattern=phandle_pattern, + context_lookups=context_lookups + ) + def add_property_heuristic(heuristic_type, pattern, fmt_type): """ diff --git a/lopper/schema/loader.py b/lopper/schema/loader.py new file mode 100644 index 00000000..fbf1e8dc --- /dev/null +++ b/lopper/schema/loader.py @@ -0,0 +1,487 @@ +#/* +# * Copyright (c) 2026 AMD Inc. All rights reserved. +# * +# * Author: +# * Bruce Ashfield +# * +# * SPDX-License-Identifier: BSD-3-Clause +# */ + +""" +Schema loader - loads schema definitions from multiple sources. + +Search path (lowest to highest priority): +1. Built-in dt-schema (lopper/schema/dt-schema/schemas/) +2. User schemas (~/.config/lopper/schemas/) +3. Environment variable (LOPPER_SCHEMA_PATH, colon-separated) +4. Command-line override (--schema-dir) + +Later sources override earlier ones, allowing vendor/user customization. +""" + +import os +import glob as glob_module +from typing import Dict, List, Optional + +import lopper.log + +try: + from ruamel.yaml import YAML + _yaml = YAML() + _yaml.preserve_quotes = True + _use_ruamel = True +except ImportError: + import yaml as pyyaml + _yaml = None + _use_ruamel = False + +from .types import PropertyType, TypeDefinition, DT_SCHEMA_TYPES +from .core import ( + SchemaRegistry, + NodeSpec, + PropertySpec, + Constraint, + ConstraintType, +) + + +def _get_builtin_schema_dir() -> str: + """Get path to built-in dt-schema files.""" + return os.path.join(os.path.dirname(__file__), 'dt-schema', 'schemas') + + +def get_schema_search_path(extra_dirs: List[str] = None) -> List[str]: + """Get ordered list of schema directories to search. + + Returns directories in priority order (lowest to highest). + Later directories can override schemas from earlier ones. + + Args: + extra_dirs: Additional directories (e.g., from --schema-dir) + + Returns: + List of directory paths + """ + paths = [] + + # 1. Built-in dt-schema (lowest priority) + builtin = _get_builtin_schema_dir() + if os.path.isdir(builtin): + paths.append(builtin) + + # 2. XDG user config directory + xdg_config = os.environ.get('XDG_CONFIG_HOME', os.path.expanduser('~/.config')) + user_schemas = os.path.join(xdg_config, 'lopper', 'schemas') + if os.path.isdir(user_schemas): + paths.append(user_schemas) + + # 3. Environment variable (colon-separated) + env_path = os.environ.get('LOPPER_SCHEMA_PATH', '') + if env_path: + for p in env_path.split(':'): + p = p.strip() + if p and os.path.isdir(p): + paths.append(p) + + # 4. Extra directories from caller (e.g., --schema-dir) + if extra_dirs: + for p in extra_dirs: + if os.path.isdir(p): + paths.append(p) + + return paths + + +def load_all_schemas( + registry: SchemaRegistry, + extra_dirs: List[str] = None +) -> int: + """Load schemas from all directories in search path. + + Later directories override earlier ones, allowing user/vendor + customization of built-in schemas. + + Args: + registry: SchemaRegistry to populate + extra_dirs: Additional directories (e.g., from --schema-dir) + + Returns: + Number of schemas loaded + """ + count = 0 + + # Always register built-in types first + for name, type_def in DT_SCHEMA_TYPES.items(): + registry.register_type(name, type_def) + + # Load from each directory in order + for schema_dir in get_schema_search_path(extra_dirs): + count += _load_schemas_from_dir(registry, schema_dir) + + return count + + +def _load_schemas_from_dir(registry: SchemaRegistry, schema_dir: str) -> int: + """Load all schema YAML files from a directory. + + Args: + registry: SchemaRegistry to populate + schema_dir: Directory to scan + + Returns: + Number of schemas loaded + """ + count = 0 + pattern = os.path.join(schema_dir, '**', '*.yaml') + + for schema_path in glob_module.glob(pattern, recursive=True): + basename = os.path.basename(schema_path) + + if basename == 'types.yaml': + # Types file - load type definitions + if _load_types_yaml(registry, schema_path): + count += 1 + else: + # Constraint schema - load node constraints + if _load_constraint_schema(registry, schema_path): + count += 1 + + return count + + +def _load_yaml_file(path: str) -> Optional[dict]: + """Load a YAML file. + + Args: + path: Path to YAML file + + Returns: + Parsed YAML as dict, or None on error + """ + try: + with open(path, 'r') as f: + if _use_ruamel and _yaml: + return _yaml.load(f) + else: + return pyyaml.safe_load(f) + except Exception as e: + lopper.log._debug(f"schema: failed to load {path}: {e}") + return None + + +def _load_types_yaml(registry: SchemaRegistry, path: str) -> bool: + """Parse types.yaml and register type definitions. + + Args: + registry: SchemaRegistry to populate + path: Path to types.yaml + + Returns: + True if successfully loaded + """ + data = _load_yaml_file(path) + if not data: + return False + + definitions = data.get('definitions', {}) + for name, spec in definitions.items(): + type_def = _parse_type_definition(name, spec) + if type_def: + registry.register_type(name, type_def) + lopper.log._debug(f"schema: loaded type {name} from {path}") + + return True + + +def _load_constraint_schema(registry: SchemaRegistry, path: str) -> bool: + """Parse a dt-schema YAML and register node constraints. + + Args: + registry: SchemaRegistry to populate + path: Path to schema YAML file + + Returns: + True if successfully loaded + """ + data = _load_yaml_file(path) + if not data: + return False + + # Determine node pattern from file + node_pattern = _schema_to_node_pattern(data, path) + if not node_pattern: + return False + + # Extract constraints + constraints = [] + properties = {} + + # Required properties + required = data.get('required', []) + if required: + constraints.append(Constraint( + constraint_type=ConstraintType.REQUIRED, + properties=required, + message=f"required: {', '.join(required)}" + )) + + # Forbidden properties (from 'not: required') + not_block = data.get('not', {}) + if isinstance(not_block, dict): + forbidden = not_block.get('required', []) + if forbidden: + constraints.append(Constraint( + constraint_type=ConstraintType.FORBIDDEN, + properties=forbidden, + message=f"forbidden: {', '.join(forbidden)}" + )) + + # Property type definitions and const/enum constraints + for prop_name, prop_schema in data.get('properties', {}).items(): + prop_spec = _parse_property_schema(prop_name, prop_schema) + if prop_spec: + properties[prop_name] = prop_spec + + # Extract const/enum constraints + if isinstance(prop_schema, dict): + if 'const' in prop_schema: + constraints.append(Constraint( + constraint_type=ConstraintType.CONST, + properties=[prop_name], + expected_value=prop_schema['const'], + message=f"{prop_name} must be '{prop_schema['const']}'" + )) + elif 'enum' in prop_schema: + constraints.append(Constraint( + constraint_type=ConstraintType.ENUM, + properties=[prop_name], + expected_value=prop_schema['enum'], + message=f"{prop_name} must be one of {prop_schema['enum']}" + )) + + # Mutex constraints from dependentSchemas + mutex_pairs = _extract_mutex_constraints(data.get('dependentSchemas', {})) + for mutex_props in mutex_pairs: + constraints.append(Constraint( + constraint_type=ConstraintType.MUTEX, + properties=mutex_props, + message=f"mutually exclusive: {', '.join(mutex_props)}" + )) + + if not constraints and not properties: + return False + + # Register node spec + node_spec = NodeSpec( + node_pattern=node_pattern, + properties=properties, + constraints=constraints, + description=data.get('description', data.get('title', '')), + schema_file=path + ) + + # Use basename (without parent dirs that might conflict) + name = os.path.splitext(os.path.basename(path))[0] + + # Handle name conflicts by adding parent directory + existing = registry.get_node_spec(name) + if existing and existing.schema_file != path: + parent = os.path.basename(os.path.dirname(path)) + name = f"{parent}-{name}" + + registry.register_node_spec(name, node_spec) + lopper.log._debug(f"schema: loaded {name} -> {node_pattern} from {path}") + + return True + + +def _schema_to_node_pattern(data: dict, path: str) -> Optional[str]: + """Convert schema to node pattern. + + Uses filename and/or $id to determine which nodes this schema applies to. + + Args: + data: Parsed schema data + path: Path to schema file + + Returns: + Node pattern string, or None if cannot determine + """ + # Map known filenames to patterns + patterns = { + 'reserved-memory.yaml': '/reserved-memory/*', + 'memory.yaml': '/memory@*', + } + + basename = os.path.basename(path) + if basename in patterns: + return patterns[basename] + + # Check $id field + schema_id = data.get('$id', '') + for filename, pattern in patterns.items(): + if filename in schema_id: + return pattern + + # Check for explicit node_pattern in schema (extension) + if 'node_pattern' in data: + return data['node_pattern'] + + return None + + +def _parse_type_definition(name: str, spec: dict) -> Optional[TypeDefinition]: + """Parse a type definition from dt-schema types.yaml. + + Args: + name: Type name + spec: Type specification dict + + Returns: + TypeDefinition or None + """ + if not isinstance(spec, dict): + return None + + prop_type = _json_schema_to_property_type(spec, name) + + return TypeDefinition( + property_type=prop_type, + min_value=spec.get('minimum'), + max_value=spec.get('maximum'), + min_items=spec.get('minItems'), + max_items=spec.get('maxItems'), + source="dt-schema", + description=spec.get('description') + ) + + +def _parse_property_schema(name: str, spec: dict) -> Optional[PropertySpec]: + """Parse a property schema from dt-schema. + + Args: + name: Property name + spec: Property schema dict + + Returns: + PropertySpec or None + """ + if not isinstance(spec, dict): + return None + + prop_type = _json_schema_to_property_type(spec, name) + type_def = TypeDefinition( + property_type=prop_type, + min_value=spec.get('minimum'), + max_value=spec.get('maximum'), + source="dt-schema", + description=spec.get('description') + ) + + return PropertySpec( + name=name, + type_def=type_def, + confidence=1.0, + source="dt-schema" + ) + + +def _json_schema_to_property_type(spec: dict, name: str) -> PropertyType: + """Convert JSON schema type to PropertyType. + + Args: + spec: JSON schema dict + name: Property/type name for context + + Returns: + PropertyType enum value + """ + schema_type = spec.get('type', 'unknown') + + if schema_type == 'integer': + max_val = spec.get('maximum', 0xffffffff) + min_val = spec.get('minimum', 0) + + # Check for signed + if min_val < 0: + if max_val <= 127: + return PropertyType.INT8 + elif max_val <= 32767: + return PropertyType.INT16 + elif max_val <= 2147483647: + return PropertyType.INT32 + else: + return PropertyType.INT64 + + # Unsigned + if max_val <= 255: + return PropertyType.UINT8 + elif max_val <= 65535: + return PropertyType.UINT16 + elif max_val <= 0xffffffff: + return PropertyType.UINT32 + else: + return PropertyType.UINT64 + + elif schema_type == 'string': + return PropertyType.STRING + + elif schema_type == 'boolean': + return PropertyType.FLAG + + elif schema_type == 'array': + items = spec.get('items', {}) + items_type = items.get('type', 'unknown') if isinstance(items, dict) else 'unknown' + + if items_type == 'string': + return PropertyType.STRING_ARRAY + elif items_type == 'integer': + # Determine array element size + max_val = items.get('maximum', 0xffffffff) if isinstance(items, dict) else 0xffffffff + if max_val <= 255: + return PropertyType.UINT8_ARRAY + elif max_val <= 65535: + return PropertyType.UINT16_ARRAY + elif max_val <= 0xffffffff: + return PropertyType.UINT32_ARRAY + else: + return PropertyType.UINT64_ARRAY + else: + return PropertyType.UINT32_ARRAY # Default for unknown arrays + + return PropertyType.UNKNOWN + + +def _extract_mutex_constraints(dependent: dict) -> List[List[str]]: + """Extract mutex property pairs from dependentSchemas. + + In dt-schema, mutual exclusivity is expressed as: + dependentSchemas: + prop_a: + not: + required: [prop_b] + + Args: + dependent: The dependentSchemas dict + + Returns: + List of mutex property lists + """ + mutex_pairs = [] + seen = set() + + for prop_a, dep_schema in dependent.items(): + if not isinstance(dep_schema, dict): + continue + not_block = dep_schema.get('not', {}) + if not isinstance(not_block, dict): + continue + forbidden = not_block.get('required', []) + for prop_b in forbidden: + # Avoid duplicates (a,b) and (b,a) + pair = tuple(sorted([prop_a, prop_b])) + if pair not in seen: + seen.add(pair) + mutex_pairs.append(list(pair)) + + return mutex_pairs diff --git a/lopper/schema/types.py b/lopper/schema/types.py new file mode 100644 index 00000000..47cf36ed --- /dev/null +++ b/lopper/schema/types.py @@ -0,0 +1,238 @@ +#/* +# * Copyright (c) 2026 AMD Inc. All rights reserved. +# * +# * Author: +# * Bruce Ashfield +# * +# * SPDX-License-Identifier: BSD-3-Clause +# */ + +""" +Unified type definitions for device tree schema validation. + +This module defines the core type system used across: +- dt-schema based validation (authoritative specs) +- Learned schema inference (observed usage) +- Audit constraint checking + +Types are based on dt-schema/JSON Schema vocabulary for familiarity. +""" + +from dataclasses import dataclass, field +from enum import Enum +from typing import Optional, List, Any, Dict + + +class PropertyType(Enum): + """Unified property types from dt-schema + lopper extensions. + + These map to both dt-schema types.yaml definitions and LopperFmt + for backwards compatibility. + """ + # Scalar integers (from dt-schema types.yaml) + UINT8 = "uint8" + INT8 = "int8" + UINT16 = "uint16" + INT16 = "int16" + UINT32 = "uint32" + INT32 = "int32" + UINT64 = "uint64" + INT64 = "int64" + + # References + PHANDLE = "phandle" + PHANDLE_ARRAY = "phandle-array" + PATH_REF = "path-ref" # string value is an absolute node path ("/axi/foo@0") + ALIAS_REF = "alias-ref" # string value is an alias name, optionally ":options" + + # Strings + STRING = "string" + STRING_ARRAY = "string-array" + + # Boolean/flag + FLAG = "flag" + EMPTY = "empty" # Lopper's EMPTY maps to FLAG + + # Arrays (element type determined by context) + UINT8_ARRAY = "uint8-array" + UINT16_ARRAY = "uint16-array" + UINT32_ARRAY = "uint32-array" + UINT64_ARRAY = "uint64-array" + + # Special + UNKNOWN = "unknown" + + def to_lopper_fmt(self): + """Convert to LopperFmt for backwards compatibility.""" + from lopper import LopperFmt + mapping = { + PropertyType.UINT8: LopperFmt.UINT8, + PropertyType.UINT16: LopperFmt.UINT16, + PropertyType.UINT32: LopperFmt.UINT32, + PropertyType.UINT64: LopperFmt.UINT64, + PropertyType.INT8: LopperFmt.UINT8, # No signed in LopperFmt + PropertyType.INT16: LopperFmt.UINT16, + PropertyType.INT32: LopperFmt.UINT32, + PropertyType.INT64: LopperFmt.UINT64, + PropertyType.STRING: LopperFmt.STRING, + PropertyType.STRING_ARRAY: LopperFmt.MULTI_STRING, + PropertyType.FLAG: LopperFmt.EMPTY, + PropertyType.EMPTY: LopperFmt.EMPTY, + PropertyType.PHANDLE: LopperFmt.UINT32, + PropertyType.PHANDLE_ARRAY: LopperFmt.UINT32, + PropertyType.PATH_REF: LopperFmt.STRING, + PropertyType.ALIAS_REF: LopperFmt.STRING, + PropertyType.UINT8_ARRAY: LopperFmt.UINT8, + PropertyType.UINT16_ARRAY: LopperFmt.UINT16, + PropertyType.UINT32_ARRAY: LopperFmt.UINT32, + PropertyType.UINT64_ARRAY: LopperFmt.UINT64, + PropertyType.UNKNOWN: LopperFmt.UNKNOWN, + } + return mapping.get(self, LopperFmt.UNKNOWN) + + @classmethod + def from_lopper_fmt(cls, lopper_fmt): + """Convert from LopperFmt to PropertyType.""" + from lopper import LopperFmt + mapping = { + LopperFmt.UINT8: cls.UINT8, + LopperFmt.UINT16: cls.UINT16, + LopperFmt.UINT32: cls.UINT32, + LopperFmt.UINT64: cls.UINT64, + LopperFmt.STRING: cls.STRING, + LopperFmt.MULTI_STRING: cls.STRING_ARRAY, + LopperFmt.EMPTY: cls.FLAG, + LopperFmt.UNKNOWN: cls.UNKNOWN, + } + return mapping.get(lopper_fmt, cls.UNKNOWN) + + +@dataclass +class TypeDefinition: + """A property type with validation constraints. + + Loaded from dt-schema types.yaml or inferred from observation. + + Attributes: + property_type: The PropertyType enum value + min_value: Minimum allowed value (for integers) + max_value: Maximum allowed value (for integers) + min_items: Minimum array length + max_items: Maximum array length + pattern: Regex pattern for string validation + enum_values: List of allowed values + source: Origin of this definition ("dt-schema", "learned", "heuristic") + description: Human-readable description + """ + property_type: PropertyType + min_value: Optional[int] = None + max_value: Optional[int] = None + min_items: Optional[int] = None + max_items: Optional[int] = None + pattern: Optional[str] = None + enum_values: Optional[List[Any]] = None + source: str = "unknown" + description: Optional[str] = None + + +# Pre-built type definitions from dt-schema types.yaml +DT_SCHEMA_TYPES: Dict[str, TypeDefinition] = { + 'uint8': TypeDefinition( + PropertyType.UINT8, + min_value=0, max_value=255, + source="dt-schema", + description="8-bit unsigned integer" + ), + 'int8': TypeDefinition( + PropertyType.INT8, + min_value=-128, max_value=127, + source="dt-schema", + description="8-bit signed integer" + ), + 'uint16': TypeDefinition( + PropertyType.UINT16, + min_value=0, max_value=65535, + source="dt-schema", + description="16-bit unsigned integer" + ), + 'int16': TypeDefinition( + PropertyType.INT16, + min_value=-32768, max_value=32767, + source="dt-schema", + description="16-bit signed integer" + ), + 'uint32': TypeDefinition( + PropertyType.UINT32, + min_value=0, max_value=0xffffffff, + source="dt-schema", + description="32-bit unsigned integer" + ), + 'int32': TypeDefinition( + PropertyType.INT32, + min_value=-2147483648, max_value=2147483647, + source="dt-schema", + description="32-bit signed integer" + ), + 'uint64': TypeDefinition( + PropertyType.UINT64, + min_value=0, max_value=0xffffffffffffffff, + source="dt-schema", + description="64-bit unsigned integer" + ), + 'int64': TypeDefinition( + PropertyType.INT64, + min_value=-9223372036854775808, max_value=9223372036854775807, + source="dt-schema", + description="64-bit signed integer" + ), + 'phandle': TypeDefinition( + PropertyType.PHANDLE, + min_value=1, max_value=0xffffffff, # phandle 0 is invalid + source="dt-schema", + description="Reference to another node" + ), + 'string': TypeDefinition( + PropertyType.STRING, + source="dt-schema", + description="Single string value" + ), + 'string-array': TypeDefinition( + PropertyType.STRING_ARRAY, + min_items=1, + source="dt-schema", + description="Array of strings" + ), + 'flag': TypeDefinition( + PropertyType.FLAG, + source="dt-schema", + description="Boolean flag (presence = true)" + ), + 'uint32-array': TypeDefinition( + PropertyType.UINT32_ARRAY, + min_items=1, + source="dt-schema", + description="Array of 32-bit unsigned integers" + ), + 'uint64-array': TypeDefinition( + PropertyType.UINT64_ARRAY, + min_items=1, + source="dt-schema", + description="Array of 64-bit unsigned integers" + ), + 'phandle-array': TypeDefinition( + PropertyType.PHANDLE_ARRAY, + min_items=1, + source="dt-schema", + description="Array of phandles with optional specifier cells" + ), + 'path-ref': TypeDefinition( + PropertyType.PATH_REF, + source="dt-schema", + description="Absolute device-tree node path string" + ), + 'alias-ref': TypeDefinition( + PropertyType.ALIAS_REF, + source="dt-schema", + description="Alias name string, optionally suffixed with :options" + ), +} diff --git a/lopper/selftest/domain-to-domain-axi-sdt.dts b/lopper/selftest/domain-to-domain-axi-sdt.dts new file mode 100644 index 00000000..663713e8 --- /dev/null +++ b/lopper/selftest/domain-to-domain-axi-sdt.dts @@ -0,0 +1,171 @@ +/dts-v1/; + +/* + * Test SDT for domain-to-domain phandle refcounting regression. + * + * Exercises step 2c in domain_access: subnodes of a domain node that + * reference /axi children via carveouts/elfload/mbox/timer phandles must + * survive filtering even when those /axi nodes are NOT in any domain's + * 'access' list. + * + * If the subnodes() scan is too shallow (children_only=True) or the parent + * chain is not marked, the /axi bus node gets ref=0 and the simple-bus + * filter at step 5 deletes it and every child regardless of refcount. + */ + +/ { + compatible = "xlnx,versal-net"; + #address-cells = <2>; + #size-cells = <2>; + model = "Test SDT for domain-to-domain /axi node survival"; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpus_a78: cpus-a78@0 { + compatible = "cpus,cluster"; + #address-cells = <1>; + #size-cells = <0>; + + cpu@0 { + compatible = "arm,cortex-a78"; + device_type = "cpu"; + reg = <0>; + }; + }; + + cpus_r52: cpus-r52@0 { + compatible = "cpus,cluster"; + #address-cells = <1>; + #size-cells = <0>; + + cpu@0 { + compatible = "arm,cortex-r52"; + device_type = "cpu"; + reg = <0>; + }; + }; + }; + + memory@0 { + device_type = "memory"; + reg = <0x0 0x0 0x0 0x80000000>; + }; + + /* + * The /axi simple-bus holds UIO nodes that domain-to-domain relations + * reference via phandle. These nodes are NOT in any domain's 'access' + * list — they survive only because domain subnode phandle scan marks them. + */ + axi: axi { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + + /* UIO node referenced by APU_Linux domain-to-domain carveouts */ + libmetal_uio_desc0: libmetal_uio_desc0@99c8000 { + reg = <0x0 0x99c8000 0x0 0x4000>; + compatible = "uio"; + }; + + libmetal_uio_data: libmetal_uio_data@99d0000 { + reg = <0x0 0x99d0000 0x0 0x40000>; + compatible = "uio"; + }; + + /* Timer node referenced by APU_Linux domain-to-domain timer */ + test_timer: timer@f1e90000 { + reg = <0x0 0xf1e90000 0x0 0x1000>; + compatible = "uio"; + }; + + /* Mailbox node referenced by APU_Linux domain-to-domain mbox */ + test_mbox: mailbox@eb360000 { + reg = <0x0 0xeb360000 0x0 0x1000>; + compatible = "uio"; + }; + + /* Regular /axi device that IS in the access list — survives via step 1a */ + uart: serial@f1920000 { + reg = <0x0 0xf1920000 0x0 0x1000>; + compatible = "arm,pl011", "arm,primecell"; + }; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + /* Carveout referenced by elfload from RPU domain */ + libmetal_elf_rsvd: libmetal_elf@9968000 { + reg = <0x0 0x9968000 0x0 0x60000>; + no-map; + }; + }; + + /* + * Pre-expanded domain nodes — in the real pipeline these come from + * %.yaml.lop YAML expansion, but for unit testing we pre-populate them + * with integer phandles so domain_access can run without YAML expansion. + */ + domains { + compatible = "openamp,domain-v1"; + + APU_Linux: APU_Linux { + compatible = "openamp,domain-v1"; + + cpus = <&cpus_a78 0xff 0x3>; + + /* Regular access: uart survives via step 1a */ + access = <&uart 0x0>; + + memory = <0x0 0x0 0x0 0x80000000>; + + /* + * domain-to-domain subnode tree — 3 levels deep. + * relation0 holds carveouts/mbox/timer phandles. + * Step 2c must scan ALL subnodes (not just direct children) + * and must mark the /axi parent chain so filter #1 keeps it. + */ + domain-to-domain { + compatible = "openamp,domain-to-domain-v1"; + + libmetal-relation { + compatible = "libmetal,ipc-v1"; + + relation0 { + remote = <&RPU_BM>; + carveouts = <&libmetal_uio_desc0 &libmetal_uio_data>; + mbox = <&test_mbox>; + timer = <&test_timer>; + }; + }; + }; + }; + + RPU_BM: RPU_BM { + compatible = "openamp,domain-v1"; + + cpus = <&cpus_r52 0x1 0x3>; + + memory = <0x0 0x0 0x0 0x10000000>; + + domain-to-domain { + compatible = "openamp,domain-to-domain-v1"; + + libmetal-relation { + compatible = "libmetal,ipc-v1"; + + relation0 { + host = <&APU_Linux>; + carveouts = <&libmetal_uio_desc0 &libmetal_uio_data>; + elfload = <&libmetal_elf_rsvd>; + }; + }; + }; + }; + }; +}; diff --git a/lopper/tree.py b/lopper/tree.py index 6e2af44d..70f86a51 100644 --- a/lopper/tree.py +++ b/lopper/tree.py @@ -1615,7 +1615,12 @@ def resolve( self, strict = None, sync_companions = True ): if prop_type == "comment": outstring = "" for s in prop_val: - outstring += s + if isinstance(s, int): + outstring += chr(s) + elif isinstance(s, (bytes, bytearray)): + outstring += s.decode("utf-8", errors="replace") + else: + outstring += s elif prop_type == "label": outstring = "" @@ -5194,6 +5199,68 @@ def print(self, output = None): # Without this, cached string_val entries for properties like address-map may # still reference deleted nodes, causing stale phandle records in the output. if self.strict: + # Pass A: drop dangling path-ref properties. + # Any string property whose value is an absolute node path that no longer + # exists in the tree is removed (e.g. stale /aliases entries after + # domain_access filters out a node). + try: + aliases_node = self["/aliases"] + except Exception: + aliases_node = None + + for n in self: + props_to_delete = [] + for p in n: + val = p.value + # String properties are stored as a list; extract single string value. + if not isinstance(val, list) or len(val) != 1 or not isinstance(val[0], str): + continue + raw = val[0].strip().strip('"') + if raw.startswith('/'): + try: + self[raw] + except Exception: + props_to_delete.append(p) + # Suppress noise for comment nodes — they disappear + # with their parent by design, not a real dangling ref. + if not p.name.startswith('lopper-comment-'): + lopper.log._warning( + f"strict: dropping dangling path-ref " + f"'{n.abs_path}/{p.name}' -> '{raw}' (node gone)") + for p in props_to_delete: + n - p + + # Pass B: drop dangling alias-ref properties. + # Properties like stdout-path reference an alias name; if that alias was + # removed in pass A (or never existed), the property is also stale. + # Only check properties explicitly registered as alias-ref to avoid false + # positives from generic string properties whose values look like identifiers. + if aliases_node is not None: + try: + alias_ref_props = set( + lopper.schema.PROPERTY_TYPE_HINTS.get('alias_ref_properties', []) + ) + except Exception: + alias_ref_props = {'stdout-path', 'linux,stdout-path'} + for n in self: + props_to_delete = [] + for p in n: + if p.name not in alias_ref_props: + continue + val = p.value + if not isinstance(val, list) or len(val) != 1 or not isinstance(val[0], str): + continue + raw = val[0].strip().strip('"') + alias_name = raw.split(':')[0] + if alias_name and aliases_node.propval(alias_name) == ['']: + props_to_delete.append(p) + lopper.log._warning( + f"strict: dropping dangling alias-ref " + f"'{n.abs_path}/{p.name}' -> alias '{alias_name}' " + f"(no longer in /aliases)") + for p in props_to_delete: + n - p + for n in self: for p in n: p.resolve() diff --git a/lopper/yaml.py b/lopper/yaml.py index 1d216708..526f2d1d 100644 --- a/lopper/yaml.py +++ b/lopper/yaml.py @@ -9,6 +9,7 @@ import ruamel from ruamel.yaml import YAML +from ruamel.yaml.scalarint import HexInt import json import sys @@ -1339,6 +1340,30 @@ def load_tree( self, tree = None ): importer.boolean_as_int = self.boolean_as_int self.anytree = importer.import_(in_tree["/"]) + +def _convert_ordered_dict(obj): + """Recursively convert OrderedDicts to regular dicts while preserving HexInt values. + + This is an alternative to json.loads(json.dumps(obj)) that preserves + ruamel.yaml scalar types like HexInt for proper hex formatting in YAML output. + + Args: + obj: The object to convert (dict, list, or scalar) + + Returns: + The converted object with OrderedDicts replaced by regular dicts + """ + if isinstance(obj, OrderedDict): + return {k: _convert_ordered_dict(v) for k, v in obj.items()} + elif isinstance(obj, dict): + return {k: _convert_ordered_dict(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [_convert_ordered_dict(item) for item in obj] + else: + # Preserve HexInt and other scalar types + return obj + + class LopperYAML(LopperJSON): """YAML read/writer for Lopper @@ -1429,11 +1454,10 @@ def update_custom_parent(node): update_custom_parent(start_node) # Update the custom attributes first dct = LopperDictExporter(dictcls=dcttype,attriter=sorted).export(start_node) - # This converts the ordered dicts to regular dicts at the last moment - # As a result, the order is preserved AND we don't get YAML that is all - # list based, which is what you get from OrderedDicts when they are dumped - # to yaml. - dct = json.loads(json.dumps(dct)) + # Convert OrderedDicts to regular dicts while preserving HexInt values + # for proper hex formatting in YAML output. This replaces the previous + # json.loads(json.dumps(dct)) approach which lost HexInt type info. + dct = _convert_ordered_dict(dct) lopper.log._debug("to_yaml: dumping export dictionary", level=lopper.log.TRACE) lopper.log._debug(pprint.pformat(dct), level=lopper.log.TRACE) @@ -1446,6 +1470,11 @@ def update_custom_parent(node): yaml_obj.default_flow_style = False yaml_obj.canonical = False yaml_obj.default_style = None + # Add representer for HexInt to output hex format (0xff instead of 255) + yaml_obj.representer.add_representer( + HexInt, + lambda dumper, data: dumper.represent_scalar('tag:yaml.org,2002:int', hex(data)) + ) # This stops tags from being output. # We could make this a configuration option in the future diff --git a/tests/test_address_map.py b/tests/test_address_map.py index 3b8c8207..f2a78720 100644 --- a/tests/test_address_map.py +++ b/tests/test_address_map.py @@ -13,9 +13,70 @@ parse_address_map, get_accessible_phandles, find_address_in_map, + render_cpu_access_map, + render_all_cpu_access_maps, ) +@pytest.fixture +def tree_with_address_map(): + """Build a synthetic LopperTree with address-map data. + + Layout: + /uart - device (phandle=10) + /spi - device (phandle=20) + /cpus-a72 - cluster with address-map referencing uart (10) and spi (20) + /cpus-r5 - cluster with address-map referencing uart (10) only + (uart is a shared device accessible by both clusters) + + address-map format with na=1, ns=1 per entry: + child_addr, phandle, parent_addr, size + """ + tree = LopperTree() + + uart = LopperNode(-1, "/uart") + uart + LopperProp("compatible", -1, uart, ["arm,pl011"]) + tree.add(uart) + + spi = LopperNode(-1, "/spi") + spi + LopperProp("compatible", -1, spi, ["arm,pl022"]) + tree.add(spi) + + # CPU cluster A72 - maps uart (10) and spi (20) + a72 = LopperNode(-1, "/cpus-a72") + a72 + LopperProp("#ranges-address-cells", -1, a72, [1]) + a72 + LopperProp("#ranges-size-cells", -1, a72, [1]) + tree.add(a72) + + # CPU cluster R5 - maps only uart (10) + r5 = LopperNode(-1, "/cpus-r5") + r5 + LopperProp("#ranges-address-cells", -1, r5, [1]) + r5 + LopperProp("#ranges-size-cells", -1, r5, [1]) + tree.add(r5) + + tree.sync() + tree.resolve() + + # Set phandles after nodes are in the tree so __pnodes__ index is updated + tree['/uart'].phandle = 10 + tree['/spi'].phandle = 20 + + # Add address-map props to cluster nodes after phandles are registered + a72_node = tree['/cpus-a72'] + a72_node + LopperProp("address-map", -1, a72_node, + [0xff000000, 10, 0xff000000, 0x1000, + 0xff010000, 20, 0xff010000, 0x1000]) + + r5_node = tree['/cpus-r5'] + r5_node + LopperProp("address-map", -1, r5_node, + [0xff000000, 10, 0xff000000, 0x1000]) + + tree.sync() + tree.resolve() + + return tree + + class TestLopperAddressMapEntry: """Tests for the LopperAddressMapEntry dataclass.""" @@ -282,43 +343,20 @@ def test_accessible_by_returns_list(self, lopper_sdt): result = tree.accessible_by(root) assert isinstance(result, list) - def test_accessible_by_node_target(self, lopper_sdt): + def test_accessible_by_node_target(self, tree_with_address_map): """Test accessible_by with a node target.""" - tree = lopper_sdt.tree + tree = tree_with_address_map - # Find a node that might be in an address-map - try: - # Look for any node with address-map - cluster = None - for node in tree: - if 'address-map' in node.__props__: - cluster = node - break - - if cluster is None: - pytest.skip("No CPU cluster with address-map found") - - # Get the address-map and find a phandle - address_map = cluster['address-map'].value - na = cluster['#ranges-address-cells'].value[0] - ns = cluster['#ranges-size-cells'].value[0] - - entries = parse_address_map(address_map, na, ns) - if not entries: - pytest.skip("Empty address-map") - - # Find the node for first phandle - target = tree.pnode(entries[0].phandle) - if target is None: - pytest.skip("Could not resolve phandle to node") - - # This node should be accessible by at least one cluster - result = tree.accessible_by(target) - assert len(result) >= 1 - assert cluster in result - - except (KeyError, IndexError, TypeError): - pytest.skip("Could not set up test") + # uart (phandle=10) is in cpus-a72's address-map + uart = tree.pnode(10) + assert uart is not None + + result = tree.accessible_by(uart) + assert len(result) >= 1 + + # cpus-a72 should be in the result + a72 = tree['/cpus-a72'] + assert a72 in result def test_accessible_by_path_string(self, lopper_sdt): """Test accessible_by with path string target.""" @@ -346,77 +384,137 @@ def test_accessible_by_address_integer(self, lopper_sdt): assert isinstance(result, list) # May or may not have matches depending on tree - def test_accessible_by_returns_cluster_nodes(self, lopper_sdt): + def test_accessible_by_returns_cluster_nodes(self, tree_with_address_map): """Test that accessible_by returns nodes with address-map property.""" - tree = lopper_sdt.tree + tree = tree_with_address_map - # Find a target that's definitely in some address-map - for node in tree: - if 'address-map' in node.__props__: - try: - address_map = node['address-map'].value - na = node['#ranges-address-cells'].value[0] - ns = node['#ranges-size-cells'].value[0] - entries = parse_address_map(address_map, na, ns) - if entries: - target = tree.pnode(entries[0].phandle) - if target: - result = tree.accessible_by(target) - # All returned nodes should have address-map - for cluster in result: - assert 'address-map' in cluster.__props__ - return - except (KeyError, IndexError, TypeError): - continue - - pytest.skip("No suitable test data found") + uart = tree.pnode(10) + assert uart is not None + + result = tree.accessible_by(uart) + assert len(result) >= 1 + # All returned nodes should have address-map + for cluster in result: + assert 'address-map' in cluster.__props__ class TestAccessibleByMultipleClusters: """Test accessible_by when multiple clusters can access a device.""" - def test_multiple_clusters_same_device(self, lopper_sdt): + def test_multiple_clusters_same_device(self, tree_with_address_map): """Test that accessible_by returns all clusters that can access a device.""" + tree = tree_with_address_map + + # uart (phandle=10) is mapped by both cpus-a72 and cpus-r5 + uart = tree.pnode(10) + assert uart is not None + + result = tree.accessible_by(uart) + assert len(result) == 2 + + a72 = tree['/cpus-a72'] + r5 = tree['/cpus-r5'] + assert a72 in result + assert r5 in result + + +class TestRenderCpuAccessMap: + """Tests for the CPU access map visualization.""" + + def test_render_returns_string(self, lopper_sdt): + """Test that render_cpu_access_map returns a string.""" + tree = lopper_sdt.tree + result = render_cpu_access_map(tree) + assert isinstance(result, str) + + def test_render_unrestricted_access_message(self, lopper_tree): + """Test message for clusters without address-map.""" + # lopper_tree has a /cpus node but no address-map + result = render_cpu_access_map(lopper_tree) + # Should show unrestricted access for the cpus node + assert "unrestricted" in result or "No CPU clusters" in result + + def test_render_all_returns_string(self, lopper_sdt): + """Test that render_all_cpu_access_maps returns a string.""" tree = lopper_sdt.tree + result = render_all_cpu_access_maps(tree) + assert isinstance(result, str) + + def test_render_contains_header(self, tree_with_address_map): + """Test that output contains expected header elements.""" + tree = tree_with_address_map + cluster = tree['/cpus-a72'] - # Find a phandle that appears in multiple address-maps - phandle_to_clusters = {} + result = render_cpu_access_map(tree, cluster) + assert "CPU Cluster:" in result + assert "Address Range" in result + assert "Device" in result + def test_render_by_path(self, tree_with_address_map): + """Test render with path string.""" + tree = tree_with_address_map + + result = render_cpu_access_map(tree, '/cpus-a72') + assert "CPU Cluster:" in result + + +class TestRenderCpuAccessMap: + """Tests for the CPU access map visualization.""" + + def test_render_returns_string(self, lopper_sdt): + """Test that render_cpu_access_map returns a string.""" + tree = lopper_sdt.tree + result = render_cpu_access_map(tree) + assert isinstance(result, str) + + def test_render_unrestricted_access_message(self, lopper_tree): + """Test message for clusters without address-map.""" + # lopper_tree has a /cpus node but no address-map + result = render_cpu_access_map(lopper_tree) + # Should show unrestricted access for the cpus node + assert "unrestricted" in result or "No CPU clusters" in result + + def test_render_all_returns_string(self, lopper_sdt): + """Test that render_all_cpu_access_maps returns a string.""" + tree = lopper_sdt.tree + result = render_all_cpu_access_maps(tree) + assert isinstance(result, str) + + def test_render_contains_header(self, lopper_sdt): + """Test that output contains expected header elements.""" + tree = lopper_sdt.tree + + # Find a cluster to test with + cluster = None for node in tree: - if 'address-map' not in node.__props__: - continue - try: - address_map = node['address-map'].value - na = node['#ranges-address-cells'].value[0] - ns = node['#ranges-size-cells'].value[0] - entries = parse_address_map(address_map, na, ns) - for entry in entries: - if entry.phandle not in phandle_to_clusters: - phandle_to_clusters[entry.phandle] = [] - phandle_to_clusters[entry.phandle].append(node) - except (KeyError, IndexError, TypeError): - continue - - # Find a phandle that's in multiple clusters - shared_phandle = None - expected_clusters = None - for phandle, clusters in phandle_to_clusters.items(): - if len(clusters) > 1: - shared_phandle = phandle - expected_clusters = clusters + if 'address-map' in node.__props__: + cluster = node break - if shared_phandle is None: - pytest.skip("No device shared by multiple clusters") + if cluster is None: + pytest.skip("No CPU cluster found") + + result = render_cpu_access_map(tree, cluster) + assert "CPU Cluster:" in result + assert "Address Range" in result + assert "Device" in result + + def test_render_by_path(self, lopper_sdt): + """Test render with path string.""" + tree = lopper_sdt.tree + + # Find a cluster path + cluster_path = None + for node in tree: + if 'address-map' in node.__props__: + cluster_path = node.abs_path + break - target = tree.pnode(shared_phandle) - if target is None: - pytest.skip("Could not resolve shared phandle") + if cluster_path is None: + pytest.skip("No CPU cluster found") - result = tree.accessible_by(target) - assert len(result) == len(expected_clusters) - for cluster in expected_clusters: - assert cluster in result + result = render_cpu_access_map(tree, cluster_path) + assert "CPU Cluster:" in result class TestAddressMapParsingMatchesLegacy: diff --git a/tests/test_audit_schema.py b/tests/test_audit_schema.py index 74b90e31..d079640a 100644 --- a/tests/test_audit_schema.py +++ b/tests/test_audit_schema.py @@ -6,6 +6,7 @@ - Node pattern matching: _node_matches_pattern, _get_matching_constraints - Validation checks: check_forbidden_properties, check_required_properties, etc. - SchemaValidator: Orchestration of phased validation +- Type unification: ConstraintType and Constraint come from lopper.schema.core Tests validation rules for reserved-memory nodes where device_type="memory" incorrectly applied can cause Xen boot failures. @@ -26,15 +27,26 @@ check_required_properties, check_property_values, check_mutex_properties, + check_learned_type_violations, + check_type_frequency_anomalies, + _infer_type_from_value, + _types_compatible, SchemaValidator, validate_schema, ) +from lopper.schema.types import PropertyType from lopper.audit.base import ( ValidationPhase, ValidationResult, ValidatorRegistry, ) +# Import unified types for comparison +from lopper.schema.core import ( + ConstraintType as CoreConstraintType, + Constraint as CoreConstraint, +) + class TestDataStructures: """Tests for constraint data structures.""" @@ -551,3 +563,282 @@ def test_mixed_violations(self): assert 'schema_required_props' in check_names assert 'schema_forbidden_props' in check_names assert 'schema_mutex_props' in check_names + + +class TestTypeUnification: + """Tests verifying unified types from lopper.schema.core are used.""" + + def test_constraint_type_is_from_core(self): + """ConstraintType should be the same as lopper.schema.core.ConstraintType.""" + assert ConstraintType is CoreConstraintType + + def test_property_constraint_is_constraint_alias(self): + """PropertyConstraint should be an alias for lopper.schema.core.Constraint.""" + assert PropertyConstraint is CoreConstraint + + def test_constraint_type_values_match(self): + """ConstraintType values should match between audit and core.""" + # These are the same enum now, but verify the values are correct + assert ConstraintType.REQUIRED == CoreConstraintType.REQUIRED + assert ConstraintType.FORBIDDEN == CoreConstraintType.FORBIDDEN + assert ConstraintType.CONST == CoreConstraintType.CONST + assert ConstraintType.ENUM == CoreConstraintType.ENUM + assert ConstraintType.MUTEX == CoreConstraintType.MUTEX + + def test_core_has_additional_constraint_types(self): + """Core may have additional constraint types like RANGE and PATTERN.""" + # These are available in the unified type but not used in audit yet + assert hasattr(CoreConstraintType, 'RANGE') + assert hasattr(CoreConstraintType, 'PATTERN') + + def test_property_constraint_creation_uses_core_fields(self): + """PropertyConstraint should have all fields from Constraint.""" + constraint = PropertyConstraint( + constraint_type=ConstraintType.FORBIDDEN, + properties=['device_type'], + expected_value=None, + message="device_type is forbidden" + ) + # These are core.Constraint fields + assert hasattr(constraint, 'constraint_type') + assert hasattr(constraint, 'properties') + assert hasattr(constraint, 'expected_value') + assert hasattr(constraint, 'message') + + def test_loaded_constraints_use_unified_types(self): + """Loaded constraints should use the unified ConstraintType.""" + for name, nc in NODE_PROPERTY_CONSTRAINTS.items(): + for constraint in nc.constraints: + assert isinstance(constraint.constraint_type, ConstraintType) + assert isinstance(constraint.constraint_type, CoreConstraintType) + + def test_audit_package_exports_constraint(self): + """lopper.audit should export Constraint in addition to PropertyConstraint.""" + from lopper.audit import Constraint, PropertyConstraint + assert Constraint is PropertyConstraint + assert Constraint is CoreConstraint + + def test_audit_package_constraint_type_is_unified(self): + """lopper.audit.ConstraintType should be the unified type.""" + from lopper.audit import ConstraintType as AuditConstraintType + assert AuditConstraintType is CoreConstraintType + + +class TestInferTypeFromValue: + """Tests for _infer_type_from_value function.""" + + def test_infer_none_is_empty(self): + """None should infer to EMPTY.""" + assert _infer_type_from_value(None) == PropertyType.EMPTY + + def test_infer_empty_list_is_empty(self): + """Empty list should infer to EMPTY.""" + assert _infer_type_from_value([]) == PropertyType.EMPTY + + def test_infer_string(self): + """String should infer to STRING.""" + assert _infer_type_from_value("hello") == PropertyType.STRING + + def test_infer_string_list_single(self): + """Single-element string list should infer to STRING.""" + assert _infer_type_from_value(["hello"]) == PropertyType.STRING + + def test_infer_string_list_multiple(self): + """Multi-element string list should infer to STRING_ARRAY.""" + assert _infer_type_from_value(["a", "b", "c"]) == PropertyType.STRING_ARRAY + + def test_infer_small_int(self): + """Small int should infer to UINT8.""" + assert _infer_type_from_value(42) == PropertyType.UINT8 + + def test_infer_medium_int(self): + """Medium int should infer to UINT16.""" + assert _infer_type_from_value(1000) == PropertyType.UINT16 + + def test_infer_large_int(self): + """Large int should infer to UINT32.""" + assert _infer_type_from_value(0x80000000) == PropertyType.UINT32 + + def test_infer_very_large_int(self): + """Very large int should infer to UINT64.""" + assert _infer_type_from_value(0x100000000) == PropertyType.UINT64 + + def test_infer_negative_int(self): + """Negative int should infer to INT32.""" + assert _infer_type_from_value(-1) == PropertyType.INT32 + + def test_infer_int_list(self): + """Int list should infer to UINT32_ARRAY.""" + assert _infer_type_from_value([1, 2, 3]) == PropertyType.UINT32_ARRAY + + def test_infer_bool(self): + """Bool should infer to FLAG.""" + assert _infer_type_from_value(True) == PropertyType.FLAG + assert _infer_type_from_value(False) == PropertyType.FLAG + + def test_infer_wrapped_value(self): + """Wrapped property value should be unwrapped.""" + mock_prop = MockProp("test") + assert _infer_type_from_value(mock_prop) == PropertyType.STRING + + def test_infer_empty_string(self): + """Empty string should infer to EMPTY.""" + assert _infer_type_from_value('') == PropertyType.EMPTY + + def test_infer_single_empty_string_list(self): + """[''] (empty property like ranges;) should infer to EMPTY.""" + assert _infer_type_from_value(['']) == PropertyType.EMPTY + + +class TestTypesCompatible: + """Tests for _types_compatible function.""" + + def test_same_types_compatible(self): + """Same types should be compatible.""" + assert _types_compatible(PropertyType.UINT32, PropertyType.UINT32) + assert _types_compatible(PropertyType.STRING, PropertyType.STRING) + + def test_integer_types_compatible(self): + """All integer types should be compatible.""" + assert _types_compatible(PropertyType.UINT8, PropertyType.UINT32) + assert _types_compatible(PropertyType.UINT32, PropertyType.UINT64) + assert _types_compatible(PropertyType.UINT16, PropertyType.UINT8) + + def test_array_and_scalar_compatible(self): + """Array types compatible with scalar base.""" + assert _types_compatible(PropertyType.UINT32, PropertyType.UINT32_ARRAY) + assert _types_compatible(PropertyType.UINT32_ARRAY, PropertyType.UINT32) + assert _types_compatible(PropertyType.STRING, PropertyType.STRING_ARRAY) + + def test_phandle_compatible_with_uint32(self): + """PHANDLE should be compatible with UINT32.""" + assert _types_compatible(PropertyType.UINT32, PropertyType.PHANDLE) + assert _types_compatible(PropertyType.UINT32_ARRAY, PropertyType.PHANDLE_ARRAY) + + def test_empty_and_flag_compatible(self): + """EMPTY and FLAG should be compatible.""" + assert _types_compatible(PropertyType.EMPTY, PropertyType.FLAG) + assert _types_compatible(PropertyType.FLAG, PropertyType.EMPTY) + + def test_string_and_uint32_not_compatible(self): + """STRING and UINT32 should not be compatible.""" + assert not _types_compatible(PropertyType.STRING, PropertyType.UINT32) + assert not _types_compatible(PropertyType.UINT32, PropertyType.STRING) + + +class TestCheckLearnedTypeViolations: + """Tests for check_learned_type_violations function.""" + + def test_no_resolver_returns_pass(self): + """Without resolver, should return passing result.""" + tree = MockTree([ + MockNode('/test', {'compatible': ['test-device']}), + ]) + + results = check_learned_type_violations(tree) + + assert len(results) == 1 + assert results[0].passed + assert 'No learned schema' in results[0].message or 'no violations' in results[0].message + + def test_function_handles_empty_tree(self): + """Should handle empty tree gracefully.""" + tree = MockTree([]) + + results = check_learned_type_violations(tree) + + assert len(results) >= 1 + assert all(r.passed for r in results) + + def test_check_name_is_correct(self): + """Results should have correct check_name.""" + tree = MockTree([]) + + results = check_learned_type_violations(tree) + + for r in results: + assert r.check_name == 'schema_learned_types' + + +class TestCheckTypeFrequencyAnomalies: + """Tests for check_type_frequency_anomalies function.""" + + def test_no_resolver_returns_pass(self): + """Without resolver, should return passing result.""" + tree = MockTree([ + MockNode('/test', {}), + ]) + + results = check_type_frequency_anomalies(tree) + + assert len(results) == 1 + assert results[0].passed + assert 'No learned schema' in results[0].message or 'No type frequency anomalies' in results[0].message + + def test_function_handles_empty_tree(self): + """Should handle empty tree gracefully.""" + tree = MockTree([]) + + results = check_type_frequency_anomalies(tree) + + assert len(results) >= 1 + assert all(r.passed for r in results) + + def test_check_name_is_correct(self): + """Results should have correct check_name.""" + tree = MockTree([]) + + results = check_type_frequency_anomalies(tree) + + for r in results: + assert r.check_name == 'schema_type_frequency' + + +class TestSchemaValidatorLearnedChecks: + """Tests for learned schema checks in SchemaValidator.""" + + def test_learned_types_flag_defined(self): + """schema_learned_types should be in WARNING_FLAGS.""" + assert 'schema_learned_types' in SchemaValidator.WARNING_FLAGS + + def test_type_frequency_flag_defined(self): + """schema_type_frequency should be in WARNING_FLAGS.""" + assert 'schema_type_frequency' in SchemaValidator.WARNING_FLAGS + + def test_schema_learned_meta_flag(self): + """schema_learned meta flag should exist and include both checks.""" + assert 'schema_learned' in SchemaValidator.META_FLAGS + learned_flags = SchemaValidator.META_FLAGS['schema_learned'] + assert 'schema_learned_types' in learned_flags + assert 'schema_type_frequency' in learned_flags + + def test_schema_all_includes_learned(self): + """schema_all meta flag should include learned checks.""" + all_flags = SchemaValidator.META_FLAGS['schema_all'] + assert 'schema_learned_types' in all_flags + assert 'schema_type_frequency' in all_flags + + def test_check_registry_has_learned_checks(self): + """CHECK_REGISTRY should have learned check functions.""" + assert 'schema_learned_types' in SchemaValidator.CHECK_REGISTRY + assert 'schema_type_frequency' in SchemaValidator.CHECK_REGISTRY + + # Verify they're callable + _, func = SchemaValidator.CHECK_REGISTRY['schema_learned_types'] + assert callable(func) + _, func = SchemaValidator.CHECK_REGISTRY['schema_type_frequency'] + assert callable(func) + + def test_run_phase_with_learned_checks(self): + """Running with schema_learned should execute learned checks.""" + tree = MockTree([ + MockNode('/test', {'reg': [0, 0x1000]}), + ]) + + validator = SchemaValidator(warnings=['schema_learned']) + results = validator.run_phase(ValidationPhase.POST_YAML, tree) + + # Should have run both learned checks + check_names = {r.check_name for r in results} + assert 'schema_learned_types' in check_names + assert 'schema_type_frequency' in check_names diff --git a/tests/test_domain_access_phandle.py b/tests/test_domain_access_phandle.py new file mode 100644 index 00000000..d96d7a3a --- /dev/null +++ b/tests/test_domain_access_phandle.py @@ -0,0 +1,151 @@ +""" +Regression tests for domain_access step 2c: domain subnode phandle refcounting. + +Tests the fix for a bug where nodes under /axi referenced by domain-to-domain +subnode properties (carveouts, mbox, timer, elfload) were deleted by the +simple-bus filter because: + + 1. The subnodes() scan was too shallow (children_only=True) — relation0 is + 3 levels below the domain node and was never reached. + 2. The /axi parent chain was not marked after setting ref_node.ref = 1, + so the simple-bus filter saw ref=0 on /axi and deleted it along with + every child regardless of individual refcounts. + +Fixture: lopper/selftest/domain-to-domain-axi-sdt.dts + +Copyright (C) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. + +SPDX-License-Identifier: BSD-3-Clause + +Author: + Bruce Ashfield +""" + +import os +import pytest +from lopper import LopperSDT, Lopper + + +class TestDomainSubnodePhandleRefcounting: + """Regression tests for domain_access step 2c subnode phandle scan. + + The SDT fixture contains: + /axi/libmetal_uio_desc0@99c8000 -- referenced by APU_Linux relation0.carveouts + /axi/libmetal_uio_data@99d0000 -- referenced by APU_Linux relation0.carveouts + /axi/timer@f1e90000 -- referenced by APU_Linux relation0.timer + /axi/mailbox@eb360000 -- referenced by APU_Linux relation0.mbox + /axi/serial@f1920000 -- referenced by APU_Linux access= (step 1a) + + All five must survive after domain_access runs on /domains/APU_Linux. + """ + + SDT = "lopper/selftest/domain-to-domain-axi-sdt.dts" + + def _run_pipeline(self, test_outdir, target="/domains/APU_Linux", + output_name="da-phandle-output.dts"): + if not os.path.exists(self.SDT): + pytest.skip(f"Test fixture not found: {self.SDT}") + + device_tree = LopperSDT(self.SDT) + device_tree.dryrun = False + device_tree.verbose = 0 + device_tree.werror = False + device_tree.output_file = os.path.join(test_outdir, output_name) + device_tree.cleanup_flag = True + device_tree.save_temps = False + device_tree.enhanced = True + device_tree.outdir = test_outdir + + # Register OpenAMP phandle property descriptors so resolve_phandles() + # recognises carveouts/mbox/timer/elfload slots. In production these + # come from the phandle-desc-v1 block in %.yaml.lop; here we inject + # them directly so the test is self-contained and order-independent. + base = Lopper.phandle_possible_properties() + base.pop("DEFAULT", None) + for prop, spec in { + "carveouts": "phandle", + "elfload": "phandle", + "mbox": "phandle", + "timer": "phandle", + "host": "phandle", + "remote": "phandle", + }.items(): + base[prop] = [spec] + Lopper.phandle_possible_prop_dict = base + + device_tree.setup(device_tree.dts, [], "", True, libfdt=True) + device_tree.target = target + device_tree.assists_setup(["lopper/assists/domain_access.py"]) + device_tree.assist_autorun_setup("lopper/assists/domain_access", ["-t", target]) + device_tree.perform_lops() + + return device_tree + + def _axi_children(self, tree): + """Return a set of node names directly under /axi.""" + try: + axi = tree["/axi"] + except Exception: + return set() + return {child.name for child in axi.subnodes(children_only=True)} + + # ------------------------------------------------------------------ + # Nodes referenced by domain-to-domain phandles must survive + # ------------------------------------------------------------------ + + def test_carveout_nodes_survive(self, test_outdir): + """Nodes referenced by relation0.carveouts survive after domain_access.""" + dt = self._run_pipeline(test_outdir, output_name="da-carveout.dts") + children = self._axi_children(dt.tree) + assert "libmetal_uio_desc0@99c8000" in children, \ + "libmetal_uio_desc0 (carveouts phandle) was deleted — step 2c scan too shallow" + assert "libmetal_uio_data@99d0000" in children, \ + "libmetal_uio_data (carveouts phandle) was deleted — step 2c scan too shallow" + dt.cleanup() + + def test_timer_node_survives(self, test_outdir): + """Node referenced by relation0.timer survives after domain_access.""" + dt = self._run_pipeline(test_outdir, output_name="da-timer.dts") + children = self._axi_children(dt.tree) + assert "timer@f1e90000" in children, \ + "timer@f1e90000 (timer phandle) was deleted — step 2c scan too shallow" + dt.cleanup() + + def test_mbox_node_survives(self, test_outdir): + """Node referenced by relation0.mbox survives after domain_access.""" + dt = self._run_pipeline(test_outdir, output_name="da-mbox.dts") + children = self._axi_children(dt.tree) + assert "mailbox@eb360000" in children, \ + "mailbox@eb360000 (mbox phandle) was deleted — step 2c scan too shallow" + dt.cleanup() + + def test_axi_bus_parent_survives(self, test_outdir): + """The /axi simple-bus parent node survives when children are refcounted.""" + dt = self._run_pipeline(test_outdir, output_name="da-axi-parent.dts") + try: + axi = dt.tree["/axi"] + except Exception: + axi = None + assert axi is not None, \ + "/axi was deleted — parent chain not marked after step 2c ref_node.ref=1" + dt.cleanup() + + def test_access_listed_node_survives(self, test_outdir): + """Node in the domain access= list (step 1a) still survives.""" + dt = self._run_pipeline(test_outdir, output_name="da-access.dts") + children = self._axi_children(dt.tree) + assert "serial@f1920000" in children, \ + "serial@f1920000 (access list) was deleted — step 1a regression" + dt.cleanup() + + def test_elfload_node_survives_in_reserved_memory(self, test_outdir): + """Node referenced by RPU relation0.elfload survives in /reserved-memory.""" + dt = self._run_pipeline(test_outdir, output_name="da-elfload.dts") + try: + resmem = dt.tree["/reserved-memory"] + names = {child.name for child in resmem.subnodes(children_only=True)} + except Exception: + names = set() + assert "libmetal_elf@9968000" in names, \ + "libmetal_elf@9968000 (elfload phandle) was deleted from /reserved-memory" + dt.cleanup() diff --git a/tests/test_glob_access.py b/tests/test_glob_access.py index b7608353..aabb60a5 100644 --- a/tests/test_glob_access.py +++ b/tests/test_glob_access.py @@ -571,3 +571,160 @@ def test_access_expand_processes_json_string_list(self): node.tree = tree # Must not raise AttributeError — should process without crashing access_expand(tree, node) + + +class TestPathRefPruning: + """Regression tests for strict-mode path-ref / alias-ref pruning in LopperTree.print(). + + These tests exercise the two pre-output passes added to LopperTree.print(): + Pass A: drop /aliases entries (and any other string properties) whose value is an + absolute node path that no longer exists in the tree. + Pass B: drop known alias-ref properties (e.g. stdout-path) that reference an alias + name that was removed by pass A. + """ + + def _build_tree_with_aliases(self, alias_entries, live_nodes): + """Build a minimal LopperTree with /aliases and some live nodes. + + alias_entries: dict of alias_name -> node_path string + live_nodes: list of absolute node paths that should exist in the tree + """ + from lopper.tree import LopperTree, LopperNode, LopperProp + + tree = LopperTree() + + # root node + root = LopperNode(-1, "/") + root.abs_path = "/" + tree + root + + # live nodes + for path in live_nodes: + parts = path.strip("/").split("/") + current = "/" + for part in parts: + child_path = current.rstrip("/") + "/" + part + if child_path not in [n.abs_path for n in tree]: + node = LopperNode(-1, part) + node.abs_path = child_path + tree + node + current = child_path + + # /aliases node + aliases = LopperNode(-1, "aliases") + aliases.abs_path = "/aliases" + tree + aliases + + for aname, apath in alias_entries.items(): + prop = LopperProp(aname, -1, aliases, [apath]) + prop.pclass = "string" + aliases + prop + + return tree, aliases + + def _run_pruning_passes(self, tree): + """Run the strict-mode pruning passes directly without needing a full print() call. + + tree.print() tries to open output.name which fails for StringIO. Instead, + replicate only the pruning logic so the tests remain self-contained. + """ + import re + import lopper.schema + + try: + aliases_node = tree["/aliases"] + except Exception: + aliases_node = None + + # Pass A: drop dangling path-ref properties + for n in tree: + props_to_delete = [] + for p in n: + val = p.value + if not isinstance(val, list) or len(val) != 1 or not isinstance(val[0], str): + continue + raw = val[0].strip().strip('"') + if raw.startswith('/'): + try: + tree[raw] + except Exception: + props_to_delete.append(p) + for p in props_to_delete: + n - p + + # Pass B: drop dangling alias-ref properties + if aliases_node is not None: + try: + alias_ref_props = set( + lopper.schema.PROPERTY_TYPE_HINTS.get('alias_ref_properties', []) + ) + except Exception: + alias_ref_props = {'stdout-path', 'linux,stdout-path'} + for n in tree: + props_to_delete = [] + for p in n: + if p.name not in alias_ref_props: + continue + val = p.value + if not isinstance(val, list) or len(val) != 1 or not isinstance(val[0], str): + continue + raw = val[0].strip().strip('"') + alias_name = raw.split(':')[0] + if alias_name and aliases_node.propval(alias_name) == ['']: + props_to_delete.append(p) + for p in props_to_delete: + n - p + + def test_path_ref_pruning_removes_dangling_alias(self): + """Pass A removes a /aliases entry pointing to a deleted node.""" + from lopper.tree import LopperNode, LopperProp + + tree, aliases = self._build_tree_with_aliases( + alias_entries={"serial0": "/axi/serial@f1920000", + "serial1": "/axi/serial@f1930000"}, + live_nodes=["/axi/serial@f1930000"], + ) + tree.strict = True + self._run_pruning_passes(tree) + + remaining = [p.name for p in aliases] + assert "serial0" not in remaining, "dangling serial0 alias should have been pruned" + assert "serial1" in remaining, "live serial1 alias must be preserved" + + def test_path_ref_pruning_preserves_valid_alias(self): + """Pass A leaves /aliases entries intact when the target node exists.""" + from lopper.tree import LopperNode, LopperProp + + tree, aliases = self._build_tree_with_aliases( + alias_entries={"serial1": "/axi/serial@f1930000"}, + live_nodes=["/axi/serial@f1930000"], + ) + tree.strict = True + self._run_pruning_passes(tree) + + remaining = [p.name for p in aliases] + assert "serial1" in remaining, "valid alias must not be pruned" + + def test_alias_ref_pruning_removes_dangling_stdout_path(self): + """Pass B removes stdout-path when the referenced alias was pruned by pass A.""" + from lopper.tree import LopperNode, LopperProp + + tree, aliases = self._build_tree_with_aliases( + alias_entries={"serial0": "/axi/serial@f1920000"}, + live_nodes=[], # node gone — serial0 alias will be pruned in pass A + ) + tree.strict = True + + # Add a /chosen node with stdout-path referencing serial0 + chosen = LopperNode(-1, "chosen") + chosen.abs_path = "/chosen" + tree + chosen + stdout_prop = LopperProp("stdout-path", -1, chosen, ["serial0:115200n8"]) + stdout_prop.pclass = "string" + chosen + stdout_prop + + self._run_pruning_passes(tree) + + remaining_chosen = [p.name for p in chosen] + assert "stdout-path" not in remaining_chosen, \ + "stdout-path referencing pruned alias must itself be pruned" diff --git a/tests/test_overlay_e2e.py b/tests/test_overlay_e2e.py index 35f2d926..a0f8e465 100644 --- a/tests/test_overlay_e2e.py +++ b/tests/test_overlay_e2e.py @@ -826,3 +826,107 @@ def test_base_domain_has_cdns_binding(self, tmp_path): f"cdns,ttc missing from RPU1_BM.dts — base tree was unexpectedly modified" assert "uio" not in content, \ f"uio leaked into RPU1_BM.dts — linux overlay contaminated base domain" + + +# --------------------------------------------------------------------------- +# Section 7: child-node deserialization regression +# +# Regression coverage for the bug where _deserialize_overlay_node() called +# node.child_nodes.append(child) but child_nodes is an OrderedDict, not a +# list — crashing with AttributeError on any overlay that contains nested +# child nodes (e.g. zyxclmm_drm under &amba_pl). +# --------------------------------------------------------------------------- + +class TestDeserializeOverlayChildNodes: + """_deserialize_overlay_node must correctly reconstruct nested child nodes. + + Previously crashed: AttributeError: 'collections.OrderedDict' has no + attribute 'append'. Fix: use child_nodes[child.abs_path] = child with + explicit parent assignment. + """ + + def _make_nested_overlay_node(self): + """Return a LopperNode with one level of child_nodes populated.""" + from lopper.tree import LopperNode, LopperProp + from collections import OrderedDict + + parent_node = LopperNode(-1, "/axi/amba_pl") + child_node = LopperNode(-1, "/axi/amba_pl/zyxclmm_drm") + lp = LopperProp("compatible", -1, child_node, ["xlnx,zocl"]) + child_node.__props__["compatible"] = lp + child_node.parent = parent_node + parent_node.child_nodes[child_node.abs_path] = child_node + return parent_node + + def test_serialize_nested_node_roundtrip(self): + """serialize → deserialize must not raise AttributeError for nested nodes.""" + from lopper import _serialize_overlay_node, _deserialize_overlay_node + + root = self._make_nested_overlay_node() + data = _serialize_overlay_node(root) + + # This previously raised: AttributeError: 'OrderedDict' has no attribute 'append' + restored = _deserialize_overlay_node(data) + assert restored is not None, "Deserialized node is None" + + def test_child_nodes_type_after_deserialize(self): + """child_nodes must remain an OrderedDict after deserialization.""" + from collections import OrderedDict + from lopper import _serialize_overlay_node, _deserialize_overlay_node + + root = self._make_nested_overlay_node() + data = _serialize_overlay_node(root) + restored = _deserialize_overlay_node(data) + + assert isinstance(restored.child_nodes, OrderedDict), \ + f"child_nodes is {type(restored.child_nodes)}, expected OrderedDict" + + def test_child_count_preserved_after_deserialize(self): + """All child nodes must survive the serialize → deserialize round-trip.""" + from lopper import _serialize_overlay_node, _deserialize_overlay_node + + root = self._make_nested_overlay_node() + original_count = len(root.child_nodes) + data = _serialize_overlay_node(root) + restored = _deserialize_overlay_node(data) + + assert len(restored.child_nodes) == original_count, \ + f"Expected {original_count} child nodes, got {len(restored.child_nodes)}" + + def test_child_node_props_preserved_after_deserialize(self): + """Child node properties must be intact after deserialization.""" + from lopper import _serialize_overlay_node, _deserialize_overlay_node + + root = self._make_nested_overlay_node() + data = _serialize_overlay_node(root) + restored = _deserialize_overlay_node(data) + + child = list(restored.child_nodes.values())[0] + assert "compatible" in child.__props__, \ + "compatible prop missing from deserialized child node" + assert child.__props__["compatible"].value == ["xlnx,zocl"], \ + f"compatible value wrong: {child.__props__['compatible'].value}" + + def test_child_parent_set_after_deserialize(self): + """Deserialized child nodes must have their parent correctly assigned.""" + from lopper import _serialize_overlay_node, _deserialize_overlay_node + + root = self._make_nested_overlay_node() + data = _serialize_overlay_node(root) + restored = _deserialize_overlay_node(data) + + child = list(restored.child_nodes.values())[0] + assert child.parent is restored, \ + f"child.parent is {child.parent}, expected the restored parent node" + + def test_child_abs_path_is_dict_key(self): + """Each child must be keyed by its abs_path in child_nodes.""" + from lopper import _serialize_overlay_node, _deserialize_overlay_node + + root = self._make_nested_overlay_node() + data = _serialize_overlay_node(root) + restored = _deserialize_overlay_node(data) + + child = list(restored.child_nodes.values())[0] + assert child.abs_path in restored.child_nodes, \ + f"child abs_path '{child.abs_path}' not found as key in child_nodes" diff --git a/tests/test_schema_learned.py b/tests/test_schema_learned.py new file mode 100644 index 00000000..1fdb9201 --- /dev/null +++ b/tests/test_schema_learned.py @@ -0,0 +1,757 @@ +""" +Tests for learned schema type resolution. + +These tests capture the current behavior of DTSPropertyTypeResolver and +DTSSchemaGenerator to ensure Phase 2 migration doesn't change output types. + +Copyright (c) 2026 AMD Inc. All rights reserved. + +SPDX-License-Identifier: BSD-3-Clause + +Author: + Bruce Ashfield +""" + +import pytest +from lopper.fmt import LopperFmt +import lopper.schema +from lopper.schema import ( + PROPERTY_NAME_HEURISTICS, + PROPERTY_TYPE_HINTS, + DTSSchemaGenerator, + DTSPropertyTypeResolver, + SchemaManager, + get_schema_manager, + PropertyType, + PropertySpec, + TypeDefinition, +) + + +class TestPropertyNameHeuristics: + """Test that PROPERTY_NAME_HEURISTICS produces correct types.""" + + def test_exact_compatible_is_string(self): + """compatible property should be STRING.""" + assert PROPERTY_NAME_HEURISTICS['exact']['compatible'] == LopperFmt.STRING + + def test_exact_status_is_string(self): + """status property should be STRING.""" + assert PROPERTY_NAME_HEURISTICS['exact']['status'] == LopperFmt.STRING + + def test_exact_device_type_is_string(self): + """device_type property should be STRING.""" + assert PROPERTY_NAME_HEURISTICS['exact']['device_type'] == LopperFmt.STRING + + def test_exact_phandle_is_uint32(self): + """phandle property should be UINT32.""" + assert PROPERTY_NAME_HEURISTICS['exact']['phandle'] == LopperFmt.UINT32 + + def test_exact_reg_is_uint32(self): + """reg property should be UINT32.""" + assert PROPERTY_NAME_HEURISTICS['exact']['reg'] == LopperFmt.UINT32 + + def test_exact_interrupts_is_uint32(self): + """interrupts property should be UINT32.""" + assert PROPERTY_NAME_HEURISTICS['exact']['interrupts'] == LopperFmt.UINT32 + + def test_exact_clocks_is_uint32(self): + """clocks property should be UINT32.""" + assert PROPERTY_NAME_HEURISTICS['exact']['clocks'] == LopperFmt.UINT32 + + def test_exact_no_map_is_empty(self): + """no-map property should be EMPTY (boolean flag).""" + assert PROPERTY_NAME_HEURISTICS['exact']['no-map'] == LopperFmt.EMPTY + + def test_exact_reusable_is_empty(self): + """reusable property should be EMPTY (boolean flag).""" + assert PROPERTY_NAME_HEURISTICS['exact']['reusable'] == LopperFmt.EMPTY + + def test_suffix_names_is_multi_string(self): + """-names suffix should map to MULTI_STRING.""" + assert PROPERTY_NAME_HEURISTICS['suffixes']['-names'] == LopperFmt.MULTI_STRING + + def test_suffix_cells_is_uint32(self): + """-cells suffix should map to UINT32.""" + assert PROPERTY_NAME_HEURISTICS['suffixes']['-cells'] == LopperFmt.UINT32 + + def test_suffix_gpio_is_uint32(self): + """-gpio suffix should map to UINT32.""" + assert PROPERTY_NAME_HEURISTICS['suffixes']['-gpio'] == LopperFmt.UINT32 + + def test_suffix_gpios_is_uint32(self): + """-gpios suffix should map to UINT32.""" + assert PROPERTY_NAME_HEURISTICS['suffixes']['-gpios'] == LopperFmt.UINT32 + + +class TestPropertyTypeHints: + """Test PROPERTY_TYPE_HINTS structure.""" + + def test_phandle_array_properties_contains_clocks(self): + """clocks should be in phandle_array_properties.""" + assert 'clocks' in PROPERTY_TYPE_HINTS['phandle_array_properties'] + + def test_phandle_array_properties_contains_resets(self): + """resets should be in phandle_array_properties.""" + assert 'resets' in PROPERTY_TYPE_HINTS['phandle_array_properties'] + + def test_phandle_array_properties_contains_interrupt_map(self): + """interrupt-map should be in phandle_array_properties.""" + assert 'interrupt-map' in PROPERTY_TYPE_HINTS['phandle_array_properties'] + + def test_potential_64bit_contains_reg(self): + """reg should be in potential_64bit_properties.""" + assert 'reg' in PROPERTY_TYPE_HINTS['potential_64bit_properties'] + + def test_potential_64bit_contains_ranges(self): + """ranges should be in potential_64bit_properties.""" + assert 'ranges' in PROPERTY_TYPE_HINTS['potential_64bit_properties'] + + def test_cell_groupings_reg_is_2(self): + """reg cell grouping should be 2.""" + assert PROPERTY_TYPE_HINTS['cell_groupings']['reg'] == 2 + + def test_cell_groupings_ranges_is_3(self): + """ranges cell grouping should be 3.""" + assert PROPERTY_TYPE_HINTS['cell_groupings']['ranges'] == 3 + + def test_string_properties_contains_compatible(self): + """compatible should be in string_properties.""" + assert 'compatible' in PROPERTY_TYPE_HINTS['string_properties'] + + def test_string_properties_contains_status(self): + """status should be in string_properties.""" + assert 'status' in PROPERTY_TYPE_HINTS['string_properties'] + + def test_boolean_properties_contains_no_map(self): + """no-map should be in boolean_properties.""" + assert 'no-map' in PROPERTY_TYPE_HINTS['boolean_properties'] + + def test_boolean_properties_contains_reusable(self): + """reusable should be in boolean_properties.""" + assert 'reusable' in PROPERTY_TYPE_HINTS['boolean_properties'] + + +class TestDTSSchemaGeneratorTypeDetection: + """Test DTSSchemaGenerator._determine_property_type method.""" + + @pytest.fixture + def generator(self): + """Create a fresh DTSSchemaGenerator.""" + return DTSSchemaGenerator() + + def test_empty_value_is_boolean(self, generator): + """Empty value should return boolean type.""" + result = generator._determine_property_type('test-prop', '') + assert result == 'boolean' + + def test_single_cell_is_uint32(self, generator): + """Single cell value should return uint32.""" + result = generator._determine_property_type('test-prop', '<0x1>') + assert result == 'uint32' + + def test_phandle_reference_is_phandle_array(self, generator): + """Value with & should return phandle-array.""" + result = generator._determine_property_type('test-prop', '<&clk>') + assert result == 'phandle-array' + + def test_empty_cells_is_empty(self, generator): + """Empty angle brackets should return empty.""" + result = generator._determine_property_type('test-prop', '<>') + assert result == 'empty' + + def test_single_string_is_string(self, generator): + """Quoted string should return string.""" + result = generator._determine_property_type('test-prop', '"hello"') + assert result == 'string' + + def test_multi_string_is_string_array(self, generator): + """Multiple quoted strings should return string-array.""" + result = generator._determine_property_type('test-prop', '"hello", "world"') + assert result == 'string-array' + + def test_byte_array_is_uint8_array(self, generator): + """Byte array syntax should return uint8-array.""" + result = generator._determine_property_type('test-prop', '[00 01 02 03]') + assert result == 'uint8-array' + + def test_known_string_property_is_string(self, generator): + """Known string property should return string.""" + result = generator._determine_property_type('compatible', '"foo,bar"') + assert result == 'string' + + def test_known_boolean_property_is_boolean(self, generator): + """Known boolean property should return boolean.""" + result = generator._determine_property_type('no-map', '') + assert result == 'boolean' + + def test_two_cells_phandle_array_is_uint32_matrix(self, generator): + """Two cells for phandle array property should be uint32-matrix-2.""" + result = generator._determine_property_type('clocks', '<0x1 0x2>') + assert result == 'uint32-matrix-2' + + def test_multiple_cells_is_uint32_array(self, generator): + """Multiple cells should return uint32-array.""" + result = generator._determine_property_type('test-prop', '<0x1 0x2 0x3>') + assert result == 'uint32-array' + + +class TestDTSPropertyTypeResolverHeuristics: + """Test DTSPropertyTypeResolver._apply_heuristics method.""" + + @pytest.fixture + def resolver(self): + """Create a resolver with minimal schema.""" + schema = {'property_definitions': {}} + return DTSPropertyTypeResolver(schema) + + def test_exact_match_compatible(self, resolver): + """compatible should resolve via exact heuristic.""" + result = resolver._apply_heuristics('compatible') + assert result == LopperFmt.STRING + + def test_exact_match_reg(self, resolver): + """reg should resolve via exact heuristic.""" + result = resolver._apply_heuristics('reg') + assert result == LopperFmt.UINT32 + + def test_exact_match_no_map(self, resolver): + """no-map should resolve via exact heuristic.""" + result = resolver._apply_heuristics('no-map') + assert result == LopperFmt.EMPTY + + def test_suffix_clock_names(self, resolver): + """clock-names should resolve via -names suffix.""" + result = resolver._apply_heuristics('clock-names') + assert result == LopperFmt.MULTI_STRING + + def test_suffix_interrupt_names(self, resolver): + """interrupt-names should resolve via -names suffix.""" + result = resolver._apply_heuristics('interrupt-names') + assert result == LopperFmt.MULTI_STRING + + def test_suffix_gpio_cells(self, resolver): + """#gpio-cells should resolve via -cells suffix.""" + result = resolver._apply_heuristics('#gpio-cells') + assert result == LopperFmt.UINT32 + + def test_suffix_clock_cells(self, resolver): + """#clock-cells should resolve via -cells suffix.""" + result = resolver._apply_heuristics('#clock-cells') + assert result == LopperFmt.UINT32 + + def test_suffix_reset_gpios(self, resolver): + """reset-gpios should resolve via -gpios suffix.""" + result = resolver._apply_heuristics('reset-gpios') + assert result == LopperFmt.UINT32 + + def test_unknown_property(self, resolver): + """Unknown property should return UNKNOWN.""" + result = resolver._apply_heuristics('xlnx-totally-unknown-property') + assert result == LopperFmt.UNKNOWN + + +class TestDTSPropertyTypeResolverLookup: + """Test DTSPropertyTypeResolver.get_property_type with schema data.""" + + @pytest.fixture + def schema_with_definitions(self): + """Create schema with property definitions.""" + return { + 'property_definitions': { + 'my-custom-prop': { + 'type': 'uint32', + }, + 'my-string-prop': { + 'type': 'string', + }, + 'my-64bit-prop': { + 'type': 'string', + 'format': 'uint64-bits', + }, + 'my-16bit-prop': { + 'type': 'string', + 'format': 'uint16-array', + }, + 'my-8bit-prop': { + 'type': 'string', + 'format': 'uint8-array', + }, + }, + 'compatible_mappings': { + 'vendor,device': { + 'properties': { + 'vendor-specific': { + 'type': 'uint32', + } + } + } + }, + 'path_overrides': { + '/special/node': { + 'properties': { + 'special-prop': { + 'type': 'uint64', + } + } + } + } + } + + @pytest.fixture + def resolver(self, schema_with_definitions): + """Create resolver with test schema.""" + return DTSPropertyTypeResolver(schema_with_definitions) + + def test_global_uint32_property(self, resolver): + """Global uint32 property should resolve correctly.""" + result = resolver.get_property_type('my-custom-prop') + assert result == LopperFmt.UINT32 + + def test_global_string_property(self, resolver): + """Global string property should resolve correctly.""" + result = resolver.get_property_type('my-string-prop') + assert result == LopperFmt.STRING + + def test_64bit_format_property(self, resolver): + """Property with uint64-bits format should resolve to UINT64.""" + result = resolver.get_property_type('my-64bit-prop') + assert result == LopperFmt.UINT64 + + def test_16bit_format_property(self, resolver): + """Property with uint16-array format should resolve to UINT16.""" + result = resolver.get_property_type('my-16bit-prop') + assert result == LopperFmt.UINT16 + + def test_8bit_format_property(self, resolver): + """Property with uint8-array format should resolve to UINT8.""" + result = resolver.get_property_type('my-8bit-prop') + assert result == LopperFmt.UINT8 + + def test_compatible_specific_property(self, resolver): + """Compatible-specific property should resolve correctly.""" + result = resolver.get_property_type('vendor-specific', compatible='vendor,device') + assert result == LopperFmt.UINT32 + + def test_path_override_property(self, resolver): + """Path-specific property should resolve correctly.""" + result = resolver.get_property_type('special-prop', node_path='/special/node') + assert result == LopperFmt.UINT64 + + def test_fallback_to_heuristics(self, resolver): + """Unknown property should fall back to heuristics.""" + result = resolver.get_property_type('clock-names') + assert result == LopperFmt.MULTI_STRING + + +class TestSchemaManagerSingleton: + """Test SchemaManager singleton behavior.""" + + def test_singleton_returns_same_instance(self): + """get_schema_manager should return same instance.""" + mgr1 = get_schema_manager() + mgr2 = get_schema_manager() + assert mgr1 is mgr2 + + def test_update_schema_creates_resolver(self): + """update_schema should create resolver.""" + mgr = get_schema_manager() + test_schema = { + 'property_definitions': { + 'test-prop': {'type': 'uint32'} + } + } + mgr.update_schema(test_schema) + resolver = mgr.get_resolver() + assert resolver is not None + + def test_resolver_uses_updated_schema(self): + """Resolver should use updated schema data.""" + mgr = get_schema_manager() + test_schema = { + 'property_definitions': { + 'unique-test-prop-12345': {'type': 'string'} + } + } + mgr.update_schema(test_schema) + resolver = mgr.get_resolver() + result = resolver.get_property_type('unique-test-prop-12345') + assert result == LopperFmt.STRING + + +class TestSchemaToLopperFmtConversion: + """Test _schema_to_lopper_fmt conversion for all type formats.""" + + @pytest.fixture + def resolver(self): + """Create resolver with empty schema.""" + return DTSPropertyTypeResolver({'property_definitions': {}}) + + def test_uint32_type(self, resolver): + """uint32 type should convert to UINT32.""" + result = resolver._schema_to_lopper_fmt('test', {'type': 'uint32'}) + assert result == LopperFmt.UINT32 + + def test_uint64_type(self, resolver): + """uint64 type should convert to UINT64.""" + result = resolver._schema_to_lopper_fmt('test', {'type': 'uint64'}) + assert result == LopperFmt.UINT64 + + def test_uint8_type(self, resolver): + """uint8 type should convert to UINT8.""" + result = resolver._schema_to_lopper_fmt('test', {'type': 'uint8'}) + assert result == LopperFmt.UINT8 + + def test_string_type(self, resolver): + """string type should convert to STRING.""" + result = resolver._schema_to_lopper_fmt('test', {'type': 'string'}) + assert result == LopperFmt.STRING + + def test_string_array_type(self, resolver): + """string-array type should convert to MULTI_STRING.""" + result = resolver._schema_to_lopper_fmt('test', {'type': 'string-array'}) + assert result == LopperFmt.MULTI_STRING + + def test_boolean_type(self, resolver): + """boolean type should convert to EMPTY.""" + result = resolver._schema_to_lopper_fmt('test', {'type': 'boolean'}) + assert result == LopperFmt.EMPTY + + def test_integer_type(self, resolver): + """integer type should convert to UINT32.""" + result = resolver._schema_to_lopper_fmt('test', {'type': 'integer'}) + assert result == LopperFmt.UINT32 + + def test_phandle_array_type(self, resolver): + """phandle-array type should convert to UINT32.""" + result = resolver._schema_to_lopper_fmt('test', {'type': 'phandle-array'}) + assert result == LopperFmt.UINT32 + + def test_array_of_integers(self, resolver): + """Array of integers should convert to UINT32.""" + result = resolver._schema_to_lopper_fmt('test', { + 'type': 'array', + 'items': {'type': 'integer'} + }) + assert result == LopperFmt.UINT32 + + def test_array_of_strings(self, resolver): + """Array of strings should convert to MULTI_STRING.""" + result = resolver._schema_to_lopper_fmt('test', { + 'type': 'array', + 'items': {'type': 'string'} + }) + assert result == LopperFmt.MULTI_STRING + + def test_uint8_format_override(self, resolver): + """Format uint8 should override type string.""" + result = resolver._schema_to_lopper_fmt('test', { + 'type': 'string', + 'format': 'uint8' + }) + assert result == LopperFmt.UINT8 + + def test_uint8_array_format_override(self, resolver): + """Format uint8-array should override type string.""" + result = resolver._schema_to_lopper_fmt('test', { + 'type': 'string', + 'format': 'uint8-array' + }) + assert result == LopperFmt.UINT8 + + def test_uint16_array_format_override(self, resolver): + """Format uint16-array should override type string.""" + result = resolver._schema_to_lopper_fmt('test', { + 'type': 'string', + 'format': 'uint16-array' + }) + assert result == LopperFmt.UINT16 + + def test_uint64_bits_format(self, resolver): + """Format uint64-bits should return UINT64.""" + result = resolver._schema_to_lopper_fmt('test', { + 'type': 'string', + 'format': 'uint64-bits' + }) + assert result == LopperFmt.UINT64 + + def test_uint64_bits_array_format(self, resolver): + """Format uint64-bits-array should return UINT64.""" + result = resolver._schema_to_lopper_fmt('test', { + 'type': 'string', + 'format': 'uint64-bits-array' + }) + assert result == LopperFmt.UINT64 + + def test_oneof_with_type_frequencies(self, resolver): + """oneOf with _type_frequencies should use most common type.""" + result = resolver._schema_to_lopper_fmt('test', { + 'oneOf': [ + {'type': 'integer'}, + {'type': 'string'} + ], + '_type_frequencies': { + 'uint32': 10, + 'string': 2 + } + }) + assert result == LopperFmt.UINT32 + + def test_oneof_string_majority(self, resolver): + """oneOf with string majority should return STRING.""" + result = resolver._schema_to_lopper_fmt('test', { + 'oneOf': [ + {'type': 'integer'}, + {'type': 'string'} + ], + '_type_frequencies': { + 'uint32': 2, + 'string': 10 + } + }) + assert result == LopperFmt.STRING + + def test_empty_definition(self, resolver): + """Empty definition should return UNKNOWN.""" + result = resolver._schema_to_lopper_fmt('test', {}) + assert result == LopperFmt.UNKNOWN + + def test_none_definition(self, resolver): + """None definition should return UNKNOWN.""" + result = resolver._schema_to_lopper_fmt('test', None) + assert result == LopperFmt.UNKNOWN + + +class TestDTSSchemaGeneratorCellGrouping: + """Test cell grouping determination.""" + + @pytest.fixture + def generator(self): + """Create a fresh generator.""" + return DTSSchemaGenerator() + + def test_reg_grouping_even_cells(self, generator): + """reg with even cells should group by 2.""" + cells = ['0x0', '0x1000', '0x80000000', '0x2000'] + result = generator._determine_cell_grouping('reg', cells) + assert result == 2 + + def test_ranges_grouping_divisible_by_3(self, generator): + """ranges divisible by 3 should group by 3.""" + cells = ['0x0', '0x0', '0x1000', '0x1000', '0x1000', '0x2000'] + result = generator._determine_cell_grouping('ranges', cells) + assert result == 3 + + def test_interrupts_grouping(self, generator): + """interrupts divisible by 3 should group by 3.""" + cells = ['0x0', '0x1', '0x4', '0x0', '0x2', '0x4'] + result = generator._determine_cell_grouping('interrupts', cells) + assert result == 3 + + def test_clocks_grouping(self, generator): + """clocks with even cells should group by 2.""" + cells = ['0x1', '0x0', '0x2', '0x1'] + result = generator._determine_cell_grouping('clocks', cells) + assert result == 2 + + def test_unknown_property_no_grouping(self, generator): + """Unknown property should have no grouping.""" + cells = ['0x1', '0x2', '0x3', '0x4', '0x5'] + result = generator._determine_cell_grouping('unknown-prop', cells) + assert result == 1 + + +class TestPropertyTypeConsistency: + """Test that property types are consistent across resolution paths.""" + + @pytest.fixture + def full_schema(self): + """Create schema with various property definitions.""" + return { + 'property_definitions': { + 'compatible': {'type': 'string'}, + 'reg': {'type': 'uint32'}, + 'interrupts': {'type': 'uint32'}, + 'clocks': {'type': 'uint32'}, + 'status': {'type': 'string'}, + 'no-map': {'type': 'boolean'}, + } + } + + @pytest.fixture + def resolver(self, full_schema): + """Create resolver with full schema.""" + return DTSPropertyTypeResolver(full_schema) + + def test_compatible_consistent(self, resolver): + """compatible should be STRING via all paths.""" + # Via schema + schema_result = resolver.get_property_type('compatible') + # Via heuristics + heuristic_result = resolver._apply_heuristics('compatible') + assert schema_result == LopperFmt.STRING + assert heuristic_result == LopperFmt.STRING + + def test_reg_consistent(self, resolver): + """reg should be UINT32 via all paths.""" + schema_result = resolver.get_property_type('reg') + heuristic_result = resolver._apply_heuristics('reg') + assert schema_result == LopperFmt.UINT32 + assert heuristic_result == LopperFmt.UINT32 + + def test_no_map_consistent(self, resolver): + """no-map should be EMPTY via all paths.""" + schema_result = resolver.get_property_type('no-map') + heuristic_result = resolver._apply_heuristics('no-map') + assert schema_result == LopperFmt.EMPTY + assert heuristic_result == LopperFmt.EMPTY + + +class TestResolvePropertySpec: + """Test DTSPropertyTypeResolver.resolve_property_spec method (Phase 2).""" + + @pytest.fixture + def schema_with_definitions(self): + """Create schema with various property definitions.""" + return { + 'property_definitions': { + 'my-uint32-prop': { + 'type': 'uint32', + 'description': 'A test uint32 property', + }, + 'my-string-prop': { + 'type': 'string', + }, + 'mixed-type-prop': { + 'oneOf': [ + {'type': 'integer'}, + {'type': 'string'} + ], + '_type_frequencies': { + 'uint32': 10, + 'string': 2 + } + }, + 'phandle-prop': { + 'type': 'phandle-array', + 'phandle-pattern': 'phandle + 2 cells', + 'context-lookups': ['#clock-cells'], + }, + }, + 'compatible_mappings': { + 'vendor,device': { + 'properties': { + 'vendor-prop': {'type': 'uint32'} + } + } + }, + 'path_overrides': { + '/special/node': { + 'properties': { + 'special-prop': {'type': 'uint64'} + } + } + } + } + + @pytest.fixture + def resolver(self, schema_with_definitions): + """Create resolver with test schema.""" + return DTSPropertyTypeResolver(schema_with_definitions) + + def test_returns_property_spec(self, resolver): + """resolve_property_spec should return PropertySpec.""" + result = resolver.resolve_property_spec('my-uint32-prop') + assert isinstance(result, PropertySpec) + + def test_property_spec_has_name(self, resolver): + """PropertySpec should have correct name.""" + result = resolver.resolve_property_spec('my-uint32-prop') + assert result.name == 'my-uint32-prop' + + def test_property_spec_has_type_def(self, resolver): + """PropertySpec should have TypeDefinition.""" + result = resolver.resolve_property_spec('my-uint32-prop') + assert isinstance(result.type_def, TypeDefinition) + + def test_uint32_resolves_to_correct_type(self, resolver): + """uint32 property should resolve to UINT32 PropertyType.""" + result = resolver.resolve_property_spec('my-uint32-prop') + assert result.type_def.property_type == PropertyType.UINT32 + + def test_string_resolves_to_correct_type(self, resolver): + """string property should resolve to STRING PropertyType.""" + result = resolver.resolve_property_spec('my-string-prop') + assert result.type_def.property_type == PropertyType.STRING + + def test_type_converts_to_lopper_fmt(self, resolver): + """PropertyType should convert to correct LopperFmt.""" + result = resolver.resolve_property_spec('my-uint32-prop') + lopper_fmt = result.type_def.property_type.to_lopper_fmt() + assert lopper_fmt == LopperFmt.UINT32 + + def test_source_is_learned(self, resolver): + """Source should be 'learned' for schema-defined properties.""" + result = resolver.resolve_property_spec('my-uint32-prop') + assert result.source == 'learned' + + def test_confidence_for_learned(self, resolver): + """Confidence should be 0.8 for learned properties.""" + result = resolver.resolve_property_spec('my-uint32-prop') + assert result.confidence == 0.8 + + def test_type_frequencies_preserved(self, resolver): + """Type frequencies should be preserved in PropertySpec.""" + result = resolver.resolve_property_spec('mixed-type-prop') + assert result.type_frequencies == {'uint32': 10, 'string': 2} + + def test_phandle_pattern_preserved(self, resolver): + """Phandle pattern should be preserved in PropertySpec.""" + result = resolver.resolve_property_spec('phandle-prop') + assert result.phandle_pattern == 'phandle + 2 cells' + + def test_context_lookups_preserved(self, resolver): + """Context lookups should be preserved in PropertySpec.""" + result = resolver.resolve_property_spec('phandle-prop') + assert result.context_lookups == ['#clock-cells'] + + def test_compatible_specific_source(self, resolver): + """Compatible-specific property should have learned-compatible source.""" + result = resolver.resolve_property_spec('vendor-prop', compatible='vendor,device') + assert result.source == 'learned-compatible' + assert result.confidence == 0.85 + + def test_path_specific_source(self, resolver): + """Path-specific property should have learned-path source.""" + result = resolver.resolve_property_spec('special-prop', node_path='/special/node') + assert result.source == 'learned-path' + assert result.confidence == 0.9 + + def test_heuristic_exact_match(self, resolver): + """Heuristic exact match should have source 'heuristic'.""" + result = resolver.resolve_property_spec('compatible') + assert result.source == 'heuristic' + assert result.confidence == 0.7 + + def test_heuristic_suffix_match(self, resolver): + """Heuristic suffix match should have lower confidence.""" + result = resolver.resolve_property_spec('clock-names') + assert result.source == 'heuristic' + assert result.confidence == 0.5 + + def test_unknown_property(self, resolver): + """Unknown property should have source 'unknown'.""" + result = resolver.resolve_property_spec('totally-unknown-xyz') + assert result.source == 'unknown' + assert result.confidence == 0.0 + + def test_consistency_with_get_property_type(self, resolver): + """resolve_property_spec should be consistent with get_property_type.""" + # Test several properties + for prop in ['my-uint32-prop', 'my-string-prop', 'compatible', 'clock-names']: + spec = resolver.resolve_property_spec(prop) + lopper_fmt = resolver.get_property_type(prop) + # Convert PropertySpec type to LopperFmt + spec_lopper_fmt = spec.type_def.property_type.to_lopper_fmt() + assert spec_lopper_fmt == lopper_fmt, \ + f"{prop}: PropertySpec gives {spec_lopper_fmt}, get_property_type gives {lopper_fmt}" diff --git a/tests/test_schema_types.py b/tests/test_schema_types.py new file mode 100644 index 00000000..a9afa6b7 --- /dev/null +++ b/tests/test_schema_types.py @@ -0,0 +1,454 @@ +""" +Tests for unified schema types (PropertyType, TypeDefinition). + +These tests ensure the new unified type system converts correctly to/from +LopperFmt and maintains consistency with learned schema output. + +Copyright (c) 2026 AMD Inc. All rights reserved. + +SPDX-License-Identifier: BSD-3-Clause + +Author: + Bruce Ashfield +""" + +import pytest +from lopper.fmt import LopperFmt +from lopper.schema import ( + PropertyType, + TypeDefinition, + DT_SCHEMA_TYPES, +) + + +class TestPropertyTypeToLopperFmt: + """Test PropertyType.to_lopper_fmt() conversion.""" + + def test_uint8_to_lopper(self): + """UINT8 should convert to LopperFmt.UINT8.""" + assert PropertyType.UINT8.to_lopper_fmt() == LopperFmt.UINT8 + + def test_uint16_to_lopper(self): + """UINT16 should convert to LopperFmt.UINT16.""" + assert PropertyType.UINT16.to_lopper_fmt() == LopperFmt.UINT16 + + def test_uint32_to_lopper(self): + """UINT32 should convert to LopperFmt.UINT32.""" + assert PropertyType.UINT32.to_lopper_fmt() == LopperFmt.UINT32 + + def test_uint64_to_lopper(self): + """UINT64 should convert to LopperFmt.UINT64.""" + assert PropertyType.UINT64.to_lopper_fmt() == LopperFmt.UINT64 + + def test_string_to_lopper(self): + """STRING should convert to LopperFmt.STRING.""" + assert PropertyType.STRING.to_lopper_fmt() == LopperFmt.STRING + + def test_string_array_to_lopper(self): + """STRING_ARRAY should convert to LopperFmt.MULTI_STRING.""" + assert PropertyType.STRING_ARRAY.to_lopper_fmt() == LopperFmt.MULTI_STRING + + def test_flag_to_lopper(self): + """FLAG should convert to LopperFmt.EMPTY.""" + assert PropertyType.FLAG.to_lopper_fmt() == LopperFmt.EMPTY + + def test_empty_to_lopper(self): + """EMPTY should convert to LopperFmt.EMPTY.""" + assert PropertyType.EMPTY.to_lopper_fmt() == LopperFmt.EMPTY + + def test_phandle_to_lopper(self): + """PHANDLE should convert to LopperFmt.UINT32.""" + assert PropertyType.PHANDLE.to_lopper_fmt() == LopperFmt.UINT32 + + def test_phandle_array_to_lopper(self): + """PHANDLE_ARRAY should convert to LopperFmt.UINT32.""" + assert PropertyType.PHANDLE_ARRAY.to_lopper_fmt() == LopperFmt.UINT32 + + def test_uint8_array_to_lopper(self): + """UINT8_ARRAY should convert to LopperFmt.UINT8.""" + assert PropertyType.UINT8_ARRAY.to_lopper_fmt() == LopperFmt.UINT8 + + def test_uint16_array_to_lopper(self): + """UINT16_ARRAY should convert to LopperFmt.UINT16.""" + assert PropertyType.UINT16_ARRAY.to_lopper_fmt() == LopperFmt.UINT16 + + def test_uint32_array_to_lopper(self): + """UINT32_ARRAY should convert to LopperFmt.UINT32.""" + assert PropertyType.UINT32_ARRAY.to_lopper_fmt() == LopperFmt.UINT32 + + def test_uint64_array_to_lopper(self): + """UINT64_ARRAY should convert to LopperFmt.UINT64.""" + assert PropertyType.UINT64_ARRAY.to_lopper_fmt() == LopperFmt.UINT64 + + def test_unknown_to_lopper(self): + """UNKNOWN should convert to LopperFmt.UNKNOWN.""" + assert PropertyType.UNKNOWN.to_lopper_fmt() == LopperFmt.UNKNOWN + + def test_signed_int8_to_lopper(self): + """INT8 should convert to LopperFmt.UINT8 (no signed in LopperFmt).""" + assert PropertyType.INT8.to_lopper_fmt() == LopperFmt.UINT8 + + def test_signed_int16_to_lopper(self): + """INT16 should convert to LopperFmt.UINT16 (no signed in LopperFmt).""" + assert PropertyType.INT16.to_lopper_fmt() == LopperFmt.UINT16 + + def test_signed_int32_to_lopper(self): + """INT32 should convert to LopperFmt.UINT32 (no signed in LopperFmt).""" + assert PropertyType.INT32.to_lopper_fmt() == LopperFmt.UINT32 + + def test_signed_int64_to_lopper(self): + """INT64 should convert to LopperFmt.UINT64 (no signed in LopperFmt).""" + assert PropertyType.INT64.to_lopper_fmt() == LopperFmt.UINT64 + + +class TestPropertyTypeFromLopperFmt: + """Test PropertyType.from_lopper_fmt() conversion.""" + + def test_from_uint8(self): + """LopperFmt.UINT8 should convert to UINT8.""" + assert PropertyType.from_lopper_fmt(LopperFmt.UINT8) == PropertyType.UINT8 + + def test_from_uint16(self): + """LopperFmt.UINT16 should convert to UINT16.""" + assert PropertyType.from_lopper_fmt(LopperFmt.UINT16) == PropertyType.UINT16 + + def test_from_uint32(self): + """LopperFmt.UINT32 should convert to UINT32.""" + assert PropertyType.from_lopper_fmt(LopperFmt.UINT32) == PropertyType.UINT32 + + def test_from_uint64(self): + """LopperFmt.UINT64 should convert to UINT64.""" + assert PropertyType.from_lopper_fmt(LopperFmt.UINT64) == PropertyType.UINT64 + + def test_from_string(self): + """LopperFmt.STRING should convert to STRING.""" + assert PropertyType.from_lopper_fmt(LopperFmt.STRING) == PropertyType.STRING + + def test_from_multi_string(self): + """LopperFmt.MULTI_STRING should convert to STRING_ARRAY.""" + assert PropertyType.from_lopper_fmt(LopperFmt.MULTI_STRING) == PropertyType.STRING_ARRAY + + def test_from_empty(self): + """LopperFmt.EMPTY should convert to FLAG.""" + assert PropertyType.from_lopper_fmt(LopperFmt.EMPTY) == PropertyType.FLAG + + def test_from_unknown(self): + """LopperFmt.UNKNOWN should convert to UNKNOWN.""" + assert PropertyType.from_lopper_fmt(LopperFmt.UNKNOWN) == PropertyType.UNKNOWN + + +class TestPropertyTypeRoundTrip: + """Test round-trip conversion PropertyType -> LopperFmt -> PropertyType.""" + + def test_uint8_roundtrip(self): + """UINT8 should round-trip correctly.""" + original = PropertyType.UINT8 + lopper_fmt = original.to_lopper_fmt() + result = PropertyType.from_lopper_fmt(lopper_fmt) + assert result == original + + def test_uint16_roundtrip(self): + """UINT16 should round-trip correctly.""" + original = PropertyType.UINT16 + lopper_fmt = original.to_lopper_fmt() + result = PropertyType.from_lopper_fmt(lopper_fmt) + assert result == original + + def test_uint32_roundtrip(self): + """UINT32 should round-trip correctly.""" + original = PropertyType.UINT32 + lopper_fmt = original.to_lopper_fmt() + result = PropertyType.from_lopper_fmt(lopper_fmt) + assert result == original + + def test_uint64_roundtrip(self): + """UINT64 should round-trip correctly.""" + original = PropertyType.UINT64 + lopper_fmt = original.to_lopper_fmt() + result = PropertyType.from_lopper_fmt(lopper_fmt) + assert result == original + + def test_string_roundtrip(self): + """STRING should round-trip correctly.""" + original = PropertyType.STRING + lopper_fmt = original.to_lopper_fmt() + result = PropertyType.from_lopper_fmt(lopper_fmt) + assert result == original + + def test_string_array_roundtrip(self): + """STRING_ARRAY should round-trip correctly.""" + original = PropertyType.STRING_ARRAY + lopper_fmt = original.to_lopper_fmt() + result = PropertyType.from_lopper_fmt(lopper_fmt) + assert result == original + + def test_flag_to_empty_roundtrip(self): + """FLAG -> EMPTY -> FLAG (via from_lopper_fmt).""" + original = PropertyType.FLAG + lopper_fmt = original.to_lopper_fmt() + assert lopper_fmt == LopperFmt.EMPTY + result = PropertyType.from_lopper_fmt(lopper_fmt) + assert result == PropertyType.FLAG + + +class TestTypeDefinition: + """Test TypeDefinition dataclass.""" + + def test_basic_creation(self): + """TypeDefinition should be creatable with just property_type.""" + td = TypeDefinition(PropertyType.UINT32) + assert td.property_type == PropertyType.UINT32 + assert td.min_value is None + assert td.max_value is None + assert td.source == "unknown" + + def test_with_range(self): + """TypeDefinition should accept min/max values.""" + td = TypeDefinition( + PropertyType.UINT8, + min_value=0, + max_value=255 + ) + assert td.min_value == 0 + assert td.max_value == 255 + + def test_with_source(self): + """TypeDefinition should accept source.""" + td = TypeDefinition( + PropertyType.STRING, + source="dt-schema" + ) + assert td.source == "dt-schema" + + def test_with_description(self): + """TypeDefinition should accept description.""" + td = TypeDefinition( + PropertyType.PHANDLE, + description="Reference to another node" + ) + assert td.description == "Reference to another node" + + def test_with_enum_values(self): + """TypeDefinition should accept enum_values.""" + td = TypeDefinition( + PropertyType.STRING, + enum_values=["okay", "disabled", "fail"] + ) + assert td.enum_values == ["okay", "disabled", "fail"] + + def test_with_array_bounds(self): + """TypeDefinition should accept min_items/max_items.""" + td = TypeDefinition( + PropertyType.UINT32_ARRAY, + min_items=1, + max_items=10 + ) + assert td.min_items == 1 + assert td.max_items == 10 + + +class TestDTSchemaTypes: + """Test pre-built DT_SCHEMA_TYPES dictionary.""" + + def test_uint8_exists(self): + """uint8 type should exist in DT_SCHEMA_TYPES.""" + assert 'uint8' in DT_SCHEMA_TYPES + td = DT_SCHEMA_TYPES['uint8'] + assert td.property_type == PropertyType.UINT8 + assert td.min_value == 0 + assert td.max_value == 255 + + def test_uint16_exists(self): + """uint16 type should exist in DT_SCHEMA_TYPES.""" + assert 'uint16' in DT_SCHEMA_TYPES + td = DT_SCHEMA_TYPES['uint16'] + assert td.property_type == PropertyType.UINT16 + assert td.min_value == 0 + assert td.max_value == 65535 + + def test_uint32_exists(self): + """uint32 type should exist in DT_SCHEMA_TYPES.""" + assert 'uint32' in DT_SCHEMA_TYPES + td = DT_SCHEMA_TYPES['uint32'] + assert td.property_type == PropertyType.UINT32 + assert td.min_value == 0 + assert td.max_value == 0xffffffff + + def test_uint64_exists(self): + """uint64 type should exist in DT_SCHEMA_TYPES.""" + assert 'uint64' in DT_SCHEMA_TYPES + td = DT_SCHEMA_TYPES['uint64'] + assert td.property_type == PropertyType.UINT64 + + def test_phandle_exists(self): + """phandle type should exist in DT_SCHEMA_TYPES.""" + assert 'phandle' in DT_SCHEMA_TYPES + td = DT_SCHEMA_TYPES['phandle'] + assert td.property_type == PropertyType.PHANDLE + assert td.min_value == 1 # phandle 0 is invalid + + def test_string_exists(self): + """string type should exist in DT_SCHEMA_TYPES.""" + assert 'string' in DT_SCHEMA_TYPES + td = DT_SCHEMA_TYPES['string'] + assert td.property_type == PropertyType.STRING + + def test_flag_exists(self): + """flag type should exist in DT_SCHEMA_TYPES.""" + assert 'flag' in DT_SCHEMA_TYPES + td = DT_SCHEMA_TYPES['flag'] + assert td.property_type == PropertyType.FLAG + + def test_all_have_dt_schema_source(self): + """All DT_SCHEMA_TYPES should have source='dt-schema'.""" + for name, td in DT_SCHEMA_TYPES.items(): + assert td.source == "dt-schema", f"{name} has wrong source: {td.source}" + + +class TestPropertyTypeValues: + """Test PropertyType enum values match expected strings.""" + + def test_uint8_value(self): + assert PropertyType.UINT8.value == "uint8" + + def test_uint16_value(self): + assert PropertyType.UINT16.value == "uint16" + + def test_uint32_value(self): + assert PropertyType.UINT32.value == "uint32" + + def test_uint64_value(self): + assert PropertyType.UINT64.value == "uint64" + + def test_string_value(self): + assert PropertyType.STRING.value == "string" + + def test_string_array_value(self): + assert PropertyType.STRING_ARRAY.value == "string-array" + + def test_phandle_value(self): + assert PropertyType.PHANDLE.value == "phandle" + + def test_phandle_array_value(self): + assert PropertyType.PHANDLE_ARRAY.value == "phandle-array" + + def test_flag_value(self): + assert PropertyType.FLAG.value == "flag" + + def test_empty_value(self): + assert PropertyType.EMPTY.value == "empty" + + def test_unknown_value(self): + assert PropertyType.UNKNOWN.value == "unknown" + + +class TestTypeDefinitionConversionToLopperFmt: + """Test that TypeDefinition can convert its property_type to LopperFmt.""" + + def test_uint32_typedef_to_lopper(self): + """TypeDefinition with UINT32 should convert to LopperFmt.UINT32.""" + td = TypeDefinition(PropertyType.UINT32) + assert td.property_type.to_lopper_fmt() == LopperFmt.UINT32 + + def test_string_typedef_to_lopper(self): + """TypeDefinition with STRING should convert to LopperFmt.STRING.""" + td = TypeDefinition(PropertyType.STRING) + assert td.property_type.to_lopper_fmt() == LopperFmt.STRING + + def test_flag_typedef_to_lopper(self): + """TypeDefinition with FLAG should convert to LopperFmt.EMPTY.""" + td = TypeDefinition(PropertyType.FLAG) + assert td.property_type.to_lopper_fmt() == LopperFmt.EMPTY + + def test_dt_schema_uint32_to_lopper(self): + """DT_SCHEMA_TYPES['uint32'] should convert to LopperFmt.UINT32.""" + td = DT_SCHEMA_TYPES['uint32'] + assert td.property_type.to_lopper_fmt() == LopperFmt.UINT32 + + def test_dt_schema_string_to_lopper(self): + """DT_SCHEMA_TYPES['string'] should convert to LopperFmt.STRING.""" + td = DT_SCHEMA_TYPES['string'] + assert td.property_type.to_lopper_fmt() == LopperFmt.STRING + + def test_dt_schema_phandle_to_lopper(self): + """DT_SCHEMA_TYPES['phandle'] should convert to LopperFmt.UINT32.""" + td = DT_SCHEMA_TYPES['phandle'] + assert td.property_type.to_lopper_fmt() == LopperFmt.UINT32 + + +class TestDeprecationWarnings: + """Test deprecation warnings for legacy APIs.""" + + def test_get_schema_manager_warns(self): + """get_schema_manager() should emit deprecation warning.""" + import warnings + import lopper.schema + + # Enable deprecation warnings for this test + lopper.schema._DEPRECATION_WARNINGS_ENABLED = True + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + manager = lopper.schema.get_schema_manager() + + # Check a deprecation warning was issued + assert len(w) >= 1 + assert any(issubclass(warning.category, DeprecationWarning) for warning in w) + assert any("get_schema_manager" in str(warning.message) for warning in w) + + def test_schema_manager_class_warns(self): + """SchemaManager() should emit deprecation warning.""" + import warnings + import lopper.schema + + # Enable deprecation warnings for this test + lopper.schema._DEPRECATION_WARNINGS_ENABLED = True + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + # Creating SchemaManager directly should warn + manager = lopper.schema.SchemaManager() + + assert len(w) >= 1 + assert any(issubclass(warning.category, DeprecationWarning) for warning in w) + assert any("SchemaManager" in str(warning.message) for warning in w) + + def test_deprecation_warnings_can_be_disabled(self): + """Deprecation warnings should be suppressible.""" + import warnings + import lopper.schema + + # Disable deprecation warnings + lopper.schema._DEPRECATION_WARNINGS_ENABLED = False + + try: + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + manager = lopper.schema.get_schema_manager() + + # No deprecation warnings should be issued + deprecation_warnings = [ + warning for warning in w + if issubclass(warning.category, DeprecationWarning) + and "get_schema_manager" in str(warning.message) + ] + assert len(deprecation_warnings) == 0 + finally: + # Re-enable for other tests + lopper.schema._DEPRECATION_WARNINGS_ENABLED = True + + def test_new_api_does_not_warn(self): + """get_registry() should not emit deprecation warning.""" + import warnings + import lopper.schema + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + registry = lopper.schema.get_registry() + + # No deprecation warnings for new API + deprecation_warnings = [ + warning for warning in w + if issubclass(warning.category, DeprecationWarning) + ] + assert len(deprecation_warnings) == 0 diff --git a/tests/test_sdt_devices.py b/tests/test_sdt_devices.py new file mode 100644 index 00000000..ccf6e10b --- /dev/null +++ b/tests/test_sdt_devices.py @@ -0,0 +1,802 @@ +""" +Pytest tests for SDT devices YAML generation assist. + +This module tests the sdt_devices assist that scans the System Device Tree +for devices across multiple categories and generates a YAML domain containing +all devices. + +Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. + +SPDX-License-Identifier: BSD-3-Clause + +Author: + Bruce Ashfield +""" + +import os +import re +import tempfile +import pytest +import yaml + +from lopper import LopperSDT +from lopper.tree import LopperTree, LopperNode, LopperProp +from lopper.assists.sdt_devices import ( + SDTDevices, sdt_devices, is_compat, DeviceCategory +) + + +class TestDeviceCategory: + """Test the DeviceCategory enum.""" + + def test_all_categories_returns_list(self): + """Test all_categories returns all enum values.""" + cats = DeviceCategory.all_categories() + assert len(cats) == 5 + assert DeviceCategory.BUS in cats + assert DeviceCategory.CPU in cats + assert DeviceCategory.MEMORY in cats + assert DeviceCategory.FIRMWARE in cats + assert DeviceCategory.TOPLEVEL in cats + + def test_from_string_valid(self): + """Test from_string with valid category names.""" + assert DeviceCategory.from_string("bus") == DeviceCategory.BUS + assert DeviceCategory.from_string("CPU") == DeviceCategory.CPU + assert DeviceCategory.from_string("Memory") == DeviceCategory.MEMORY + assert DeviceCategory.from_string(" firmware ") == DeviceCategory.FIRMWARE + + def test_from_string_invalid(self): + """Test from_string returns None for invalid names.""" + assert DeviceCategory.from_string("invalid") is None + assert DeviceCategory.from_string("") is None + + def test_parse_list(self): + """Test parse_list parses comma-separated categories.""" + cats = DeviceCategory.parse_list("bus,cpu,memory") + assert len(cats) == 3 + assert DeviceCategory.BUS in cats + assert DeviceCategory.CPU in cats + assert DeviceCategory.MEMORY in cats + + def test_parse_list_with_invalid(self): + """Test parse_list ignores invalid category names.""" + cats = DeviceCategory.parse_list("bus,invalid,cpu") + assert len(cats) == 2 + assert DeviceCategory.BUS in cats + assert DeviceCategory.CPU in cats + + +class TestIsCompat: + """Test the is_compat function for assist matching.""" + + def test_matches_module_pattern(self): + """Test is_compat matches 'module,sdt_devices' pattern.""" + result = is_compat(None, "module,sdt_devices") + assert result == sdt_devices + + def test_matches_sdt_devices_pattern(self): + """Test is_compat matches 'sdt-devices,sdt-devices-v1' pattern.""" + result = is_compat(None, "sdt-devices,sdt-devices-v1") + assert result == sdt_devices + + def test_no_match_returns_empty_string(self): + """Test is_compat returns empty string for non-matching patterns.""" + result = is_compat(None, "some,other-compat") + assert result == "" + + def test_no_match_partial(self): + """Test is_compat doesn't match partial patterns.""" + result = is_compat(None, "module,sdt") + assert result == "" + + +class TestSDTDevicesDiscovery: + """Unit tests for device discovery functionality.""" + + def test_discover_bus_devices_finds_simple_bus_children(self, lopper_sdt): + """Test that discover_bus_devices finds devices under simple-bus nodes.""" + generator = SDTDevices(lopper_sdt) + devices = generator.discover_bus_devices() + + assert len(devices) > 0, "Should discover at least one device" + for dev in devices: + assert 'dev' in dev, "Each device entry must have 'dev' key" + + def test_discover_bus_devices_only_addressable(self, lopper_sdt): + """Test that only addressable devices (with @) are discovered.""" + generator = SDTDevices(lopper_sdt) + devices = generator.discover_bus_devices() + + for dev in devices: + assert '@' in dev['dev'], \ + f"Device '{dev['dev']}' should be addressable (contain @)" + + def test_discover_bus_devices_includes_labels(self, lopper_sdt): + """Test that device labels are included when present.""" + generator = SDTDevices(lopper_sdt) + devices = generator.discover_bus_devices() + + devices_with_labels = [d for d in devices if 'label' in d] + for dev in devices_with_labels: + assert dev['label'], "Label should not be empty" + + def test_discover_bus_devices_no_duplicates(self, lopper_sdt): + """Test that discovered devices have no duplicates.""" + generator = SDTDevices(lopper_sdt) + devices = generator.discover_bus_devices() + + dev_names = [d['dev'] for d in devices] + assert len(devices) == len(set(dev_names)), \ + "Device list should not have duplicates" + + def test_discover_bus_devices_custom_bus_type(self, lopper_sdt): + """Test discovering devices with custom bus types.""" + generator = SDTDevices(lopper_sdt) + + devices = generator.discover_bus_devices(bus_types=['nonexistent-bus']) + assert devices == [] + + def test_discover_bus_devices_multiple_bus_types(self, lopper_sdt): + """Test discovering devices with multiple bus types.""" + generator = SDTDevices(lopper_sdt) + + devices = generator.discover_bus_devices( + bus_types=['simple-bus', 'xlnx,versal-axi'] + ) + assert isinstance(devices, list) + + +class TestCPUDiscovery: + """Test CPU cluster discovery.""" + + def test_discover_cpus_returns_list(self, lopper_sdt): + """Test that discover_cpus returns a list.""" + generator = SDTDevices(lopper_sdt) + cpus = generator.discover_cpus() + + assert isinstance(cpus, list) + + def test_discover_cpus_has_dev_key(self, lopper_sdt): + """Test that CPU entries have 'dev' key.""" + generator = SDTDevices(lopper_sdt) + cpus = generator.discover_cpus() + + for cpu in cpus: + assert 'dev' in cpu, "CPU entry must have 'dev' key" + + def test_discover_cpus_no_duplicates(self, lopper_sdt): + """Test that CPU cluster entries are not duplicated.""" + generator = SDTDevices(lopper_sdt) + cpus = generator.discover_cpus() + + # Each cluster should appear only once + dev_names = [c['dev'] for c in cpus] + assert len(cpus) == len(set(dev_names)), \ + "CPU cluster entries should not be duplicated" + + +class TestMemoryDiscovery: + """Test memory node discovery.""" + + def test_discover_memory_returns_dict(self, lopper_sdt): + """Test that discover_memory returns dict with memory and sram keys.""" + generator = SDTDevices(lopper_sdt) + memory = generator.discover_memory() + + assert isinstance(memory, dict) + assert 'memory' in memory + assert 'sram' in memory + assert isinstance(memory['memory'], list) + assert isinstance(memory['sram'], list) + + def test_discover_memory_entries_have_dev(self, lopper_sdt): + """Test that memory entries have 'dev' key.""" + generator = SDTDevices(lopper_sdt) + memory = generator.discover_memory() + + for mem in memory['memory']: + assert 'dev' in mem + for sram in memory['sram']: + assert 'dev' in sram + + def test_classify_memory_type_sram(self, lopper_sdt): + """Test SRAM classification.""" + generator = SDTDevices(lopper_sdt) + + assert generator._classify_memory_type("tcm@ffe00000") == "sram" + assert generator._classify_memory_type("ocm@fffc0000") == "sram" + assert generator._classify_memory_type("sram@10000") == "sram" + + def test_classify_memory_type_memory(self, lopper_sdt): + """Test memory classification.""" + generator = SDTDevices(lopper_sdt) + + assert generator._classify_memory_type("memory@0") == "memory" + assert generator._classify_memory_type("ddr@80000000") == "memory" + + +class TestFirmwareDiscovery: + """Test firmware node discovery.""" + + def test_discover_firmware_returns_list(self, lopper_sdt): + """Test that discover_firmware returns a list.""" + generator = SDTDevices(lopper_sdt) + firmware = generator.discover_firmware() + + assert isinstance(firmware, list) + + def test_discover_firmware_entries_have_dev(self, lopper_sdt): + """Test that firmware entries have 'dev' key.""" + generator = SDTDevices(lopper_sdt) + firmware = generator.discover_firmware() + + for fw in firmware: + assert 'dev' in fw + + +class TestToplevelDiscovery: + """Test toplevel node discovery.""" + + def test_discover_toplevel_returns_list(self, lopper_sdt): + """Test that discover_toplevel returns a list.""" + generator = SDTDevices(lopper_sdt) + toplevel = generator.discover_toplevel() + + assert isinstance(toplevel, list) + + def test_discover_toplevel_skips_special_nodes(self, lopper_sdt): + """Test that special nodes are skipped.""" + generator = SDTDevices(lopper_sdt) + toplevel = generator.discover_toplevel() + + dev_names = [d['dev'] for d in toplevel] + assert 'chosen' not in dev_names + assert 'aliases' not in dev_names + assert '__symbols__' not in dev_names + + +class TestPatternFiltering: + """Test include/exclude pattern filtering.""" + + def test_include_pattern_filters_devices(self, lopper_sdt): + """Test that include pattern filters devices.""" + generator = SDTDevices(lopper_sdt) + devices = [ + {'dev': 'serial@ff000000'}, + {'dev': 'can@ff060000'}, + {'dev': 'serial@ff010000'}, + ] + + filtered = generator._apply_pattern_filter( + devices, include_pattern="serial@.*" + ) + + assert len(filtered) == 2 + assert all('serial' in d['dev'] for d in filtered) + + def test_exclude_pattern_removes_devices(self, lopper_sdt): + """Test that exclude pattern removes devices.""" + generator = SDTDevices(lopper_sdt) + devices = [ + {'dev': 'serial@ff000000'}, + {'dev': 'can@ff060000'}, + {'dev': 'serial@ff010000'}, + ] + + filtered = generator._apply_pattern_filter( + devices, exclude_pattern="serial@.*" + ) + + assert len(filtered) == 1 + assert filtered[0]['dev'] == 'can@ff060000' + + def test_both_patterns_applied(self, lopper_sdt): + """Test that both patterns are applied.""" + generator = SDTDevices(lopper_sdt) + devices = [ + {'dev': 'serial@ff000000'}, + {'dev': 'serial@ff010000'}, + {'dev': 'can@ff060000'}, + ] + + filtered = generator._apply_pattern_filter( + devices, + include_pattern="@ff0.*", + exclude_pattern="can@.*" + ) + + assert len(filtered) == 2 + assert all('serial' in d['dev'] for d in filtered) + + def test_no_patterns_returns_original(self, lopper_sdt): + """Test that no patterns returns original list.""" + generator = SDTDevices(lopper_sdt) + devices = [{'dev': 'test@123'}] + + filtered = generator._apply_pattern_filter(devices) + assert filtered == devices + + +class TestDiscoverAll: + """Test the orchestrated discovery.""" + + def test_discover_all_default_returns_all_categories(self, lopper_sdt): + """Test discover_all with default categories.""" + generator = SDTDevices(lopper_sdt) + devices = generator.discover_all() + + assert 'access' in devices + assert 'cpus' in devices + assert 'memory' in devices + assert 'sram' in devices + + def test_discover_all_single_category(self, lopper_sdt): + """Test discover_all with single category.""" + generator = SDTDevices(lopper_sdt) + + # Only bus devices + devices = generator.discover_all(categories=[DeviceCategory.BUS]) + + assert len(devices['access']) > 0 + # CPUs should be empty since we only asked for BUS + assert len(devices['cpus']) == 0 + + def test_discover_all_multiple_categories(self, lopper_sdt): + """Test discover_all with multiple categories.""" + generator = SDTDevices(lopper_sdt) + + devices = generator.discover_all( + categories=[DeviceCategory.BUS, DeviceCategory.CPU] + ) + + # Should have bus devices in access + # May or may not have CPUs depending on test tree + assert isinstance(devices['access'], list) + assert isinstance(devices['cpus'], list) + + def test_discover_all_with_pattern_filter(self, lopper_sdt): + """Test discover_all with pattern filtering.""" + generator = SDTDevices(lopper_sdt) + + # Get all devices first + all_devices = generator.discover_all() + + # Then filter + filtered = generator.discover_all(include_pattern="serial@.*") + + # Filtered should have fewer or equal devices + total_all = sum(len(v) for v in all_devices.values()) + total_filtered = sum(len(v) for v in filtered.values()) + + assert total_filtered <= total_all + + +class TestSDTDevicesGeneration: + """Test YAML domain generation.""" + + def test_generate_domain_creates_tree(self, lopper_sdt): + """Test that generate_domain creates a valid LopperTree.""" + generator = SDTDevices(lopper_sdt) + tree = generator.generate_domain() + + assert isinstance(tree, LopperTree) + + def test_generate_domain_has_domains_container(self, lopper_sdt): + """Test that generated tree has /domains container.""" + generator = SDTDevices(lopper_sdt) + tree = generator.generate_domain() + + domains_node = tree["/domains"] + assert domains_node is not None + + def test_generate_domain_has_named_domain(self, lopper_sdt): + """Test that generated tree has named domain node.""" + generator = SDTDevices(lopper_sdt) + tree = generator.generate_domain(domain_name='test_domain') + + domains_node = tree["/domains"] + domain_found = False + for child in domains_node.subnodes(children_only=True): + if child.name == 'test_domain': + domain_found = True + break + + assert domain_found, "Domain 'test_domain' should exist" + + def test_generate_domain_has_compatible(self, lopper_sdt): + """Test that domain has correct compatible string.""" + generator = SDTDevices(lopper_sdt) + tree = generator.generate_domain(domain_name='test_domain') + + domains_node = tree["/domains"] + for child in domains_node.subnodes(children_only=True): + if child.name == 'test_domain': + compat = child["compatible"].value + assert "openamp,domain-v1,devices" in compat + break + + def test_generate_domain_has_id(self, lopper_sdt): + """Test that domain has id property.""" + generator = SDTDevices(lopper_sdt) + tree = generator.generate_domain(domain_name='test_domain') + + domains_node = tree["/domains"] + for child in domains_node.subnodes(children_only=True): + if child.name == 'test_domain': + id_prop = child["id"].value + assert id_prop is not None + break + + def test_generate_domain_default_name(self, lopper_sdt): + """Test that default domain name is 'sdt_all_devices'.""" + generator = SDTDevices(lopper_sdt) + tree = generator.generate_domain() + + domains_node = tree["/domains"] + default_found = False + for child in domains_node.subnodes(children_only=True): + if child.name == 'sdt_all_devices': + default_found = True + break + + assert default_found, "Default domain name should be 'sdt_all_devices'" + + def test_generate_domain_with_categories(self, lopper_sdt): + """Test generate_domain respects category selection.""" + generator = SDTDevices(lopper_sdt) + + # Generate with only bus category + tree = generator.generate_domain(categories=[DeviceCategory.BUS]) + + domains_node = tree["/domains"] + for child in domains_node.subnodes(children_only=True): + # Should have access property (bus devices) + access = child.propval("access") + # access may be empty list or actual devices + assert access is not None or True + + def test_generate_domain_with_pattern(self, lopper_sdt): + """Test generate_domain with include pattern.""" + generator = SDTDevices(lopper_sdt) + + tree = generator.generate_domain(include_pattern="serial@.*") + + # Verify only serial devices in access + domains_node = tree["/domains"] + for child in domains_node.subnodes(children_only=True): + access = child.propval("access") + if access: + for entry in access: + if isinstance(entry, dict) and 'dev' in entry: + assert 'serial' in entry['dev'] + + +class TestSDTDevicesIntegration: + """Integration tests for the sdt_devices entry point.""" + + @pytest.fixture + def temp_output_file(self): + """Create a temporary file for output.""" + with tempfile.NamedTemporaryFile( + mode='w', suffix='.yaml', delete=False + ) as f: + yield f.name + if os.path.exists(f.name): + os.unlink(f.name) + + def test_sdt_devices_entry_point_returns_true( + self, lopper_sdt, temp_output_file + ): + """Test that sdt_devices entry point returns True on success.""" + lopper_sdt.output_file = temp_output_file + options = { + 'verbose': 0, + 'args': [] + } + + result = sdt_devices(None, lopper_sdt, options) + assert result is True + + def test_sdt_devices_creates_output_file( + self, lopper_sdt, temp_output_file + ): + """Test that sdt_devices creates output file.""" + lopper_sdt.output_file = temp_output_file + options = { + 'verbose': 0, + 'args': [] + } + + sdt_devices(None, lopper_sdt, options) + assert os.path.exists(temp_output_file) + + def test_sdt_devices_output_is_valid_yaml( + self, lopper_sdt, temp_output_file + ): + """Test that output is valid YAML.""" + lopper_sdt.output_file = temp_output_file + options = { + 'verbose': 0, + 'args': [] + } + + sdt_devices(None, lopper_sdt, options) + + with open(temp_output_file) as f: + data = yaml.safe_load(f) + + assert data is not None + + def test_sdt_devices_output_has_domains_structure( + self, lopper_sdt, temp_output_file + ): + """Test that output YAML has correct domains structure.""" + lopper_sdt.output_file = temp_output_file + options = { + 'verbose': 0, + 'args': [] + } + + sdt_devices(None, lopper_sdt, options) + + with open(temp_output_file) as f: + data = yaml.safe_load(f) + + assert 'domains' in data, "Output should have 'domains' key" + assert 'sdt_all_devices' in data['domains'], \ + "Output should have 'sdt_all_devices' domain" + + def test_sdt_devices_custom_domain_name( + self, lopper_sdt, temp_output_file + ): + """Test sdt_devices with custom domain name option.""" + lopper_sdt.output_file = temp_output_file + options = { + 'verbose': 0, + 'args': ['-n', 'my_custom_domain'] + } + + sdt_devices(None, lopper_sdt, options) + + with open(temp_output_file) as f: + data = yaml.safe_load(f) + + assert 'my_custom_domain' in data['domains'], \ + "Output should have custom domain name" + + def test_sdt_devices_with_output_option( + self, lopper_sdt, test_outdir + ): + """Test sdt_devices with -o output option.""" + output_path = os.path.join(test_outdir, "custom-output.yaml") + options = { + 'verbose': 0, + 'args': ['-o', output_path] + } + + result = sdt_devices(None, lopper_sdt, options) + assert result is True + assert os.path.exists(output_path) + + def test_sdt_devices_with_categories_option( + self, lopper_sdt, temp_output_file + ): + """Test sdt_devices with -c categories option.""" + lopper_sdt.output_file = temp_output_file + options = { + 'verbose': 0, + 'args': ['-c', 'bus'] + } + + result = sdt_devices(None, lopper_sdt, options) + assert result is True + + with open(temp_output_file) as f: + data = yaml.safe_load(f) + + # Should have access (bus devices) but not cpus + domain = data['domains']['sdt_all_devices'] + # cpus should not be present or empty when only bus category + assert 'cpus' not in domain or not domain.get('cpus') + + def test_sdt_devices_with_exclude_categories( + self, lopper_sdt, temp_output_file + ): + """Test sdt_devices with --exclude-categories option.""" + lopper_sdt.output_file = temp_output_file + options = { + 'verbose': 0, + 'args': ['--exclude-categories', 'firmware,toplevel'] + } + + result = sdt_devices(None, lopper_sdt, options) + assert result is True + + def test_sdt_devices_with_include_pattern( + self, lopper_sdt, temp_output_file + ): + """Test sdt_devices with --include-pattern option.""" + lopper_sdt.output_file = temp_output_file + options = { + 'verbose': 0, + 'args': ['--include-pattern', 'serial@.*'] + } + + result = sdt_devices(None, lopper_sdt, options) + assert result is True + + with open(temp_output_file) as f: + data = yaml.safe_load(f) + + domain = data['domains']['sdt_all_devices'] + access = domain.get('access', []) + + # All access entries should match serial pattern + for entry in access: + if entry and isinstance(entry, dict) and 'dev' in entry: + assert 'serial' in entry['dev'].lower() or '@' not in entry['dev'] + + def test_sdt_devices_with_exclude_pattern( + self, lopper_sdt, temp_output_file + ): + """Test sdt_devices with --exclude-pattern option.""" + lopper_sdt.output_file = temp_output_file + options = { + 'verbose': 0, + 'args': ['--exclude-pattern', 'serial@.*', '-c', 'bus'] + } + + result = sdt_devices(None, lopper_sdt, options) + assert result is True + + with open(temp_output_file) as f: + data = yaml.safe_load(f) + + domain = data['domains']['sdt_all_devices'] + access = domain.get('access', []) + + # No access entries should be serial + for entry in access: + if isinstance(entry, dict) and 'dev' in entry: + assert 'serial' not in entry['dev'].lower() + + +class TestBackwardsCompatibility: + """Test backwards compatibility with original implementation.""" + + @pytest.fixture + def temp_output_file(self): + """Create a temporary file for output.""" + with tempfile.NamedTemporaryFile( + mode='w', suffix='.yaml', delete=False + ) as f: + yield f.name + if os.path.exists(f.name): + os.unlink(f.name) + + def test_no_options_includes_bus_devices( + self, lopper_sdt, temp_output_file + ): + """Test that default behavior includes bus devices.""" + lopper_sdt.output_file = temp_output_file + options = { + 'verbose': 0, + 'args': [] + } + + sdt_devices(None, lopper_sdt, options) + + with open(temp_output_file) as f: + data = yaml.safe_load(f) + + domain = data['domains']['sdt_all_devices'] + access = domain.get('access', []) + + # Should have bus devices (addressable with @) + addressable = [e for e in access if e and '@' in e.get('dev', '')] + assert len(addressable) > 0 + + def test_bus_types_option_still_works( + self, lopper_sdt, temp_output_file + ): + """Test that -b/--bus-types option still works.""" + lopper_sdt.output_file = temp_output_file + options = { + 'verbose': 0, + 'args': ['-b', 'simple-bus', '-c', 'bus'] + } + + result = sdt_devices(None, lopper_sdt, options) + assert result is True + + def test_domain_has_compatible_string( + self, lopper_sdt, temp_output_file + ): + """Test domain has openamp,domain-v1,devices compatible.""" + lopper_sdt.output_file = temp_output_file + options = { + 'verbose': 0, + 'args': [] + } + + sdt_devices(None, lopper_sdt, options) + + with open(temp_output_file) as f: + data = yaml.safe_load(f) + + domain = data['domains']['sdt_all_devices'] + assert domain.get('compatible') == 'openamp,domain-v1,devices' + + +class TestSDTDevicesGlobUsage: + """Test that generated YAML can be used for glob matching.""" + + def test_generated_yaml_has_access_devices( + self, lopper_sdt, test_outdir + ): + """Test generated YAML has devices in access list.""" + output_path = os.path.join(test_outdir, "sdt-devices-glob-test.yaml") + lopper_sdt.output_file = output_path + options = { + 'verbose': 0, + 'args': [] + } + + sdt_devices(None, lopper_sdt, options) + + with open(output_path) as f: + data = yaml.safe_load(f) + + domain = data['domains']['sdt_all_devices'] + access = domain.get('access', []) + + non_empty_entries = [e for e in access if e] + assert len(non_empty_entries) > 0, \ + "Generated domain should have devices for glob matching" + + def test_generated_yaml_device_names_are_addressable( + self, lopper_sdt, test_outdir + ): + """Test bus devices in access list are addressable (have @).""" + output_path = os.path.join(test_outdir, "sdt-devices-addr-test.yaml") + lopper_sdt.output_file = output_path + options = { + 'verbose': 0, + 'args': ['-c', 'bus'] # Only bus to ensure addressable + } + + sdt_devices(None, lopper_sdt, options) + + with open(output_path) as f: + data = yaml.safe_load(f) + + domain = data['domains']['sdt_all_devices'] + access = domain.get('access', []) + + for entry in access: + if entry and 'dev' in entry: + assert '@' in entry['dev'], \ + f"Bus device '{entry['dev']}' should be addressable" + + def test_compatible_string_for_parent_domain( + self, lopper_sdt, test_outdir + ): + """Test domain has compatible string identifiable as SDT devices.""" + output_path = os.path.join(test_outdir, "sdt-devices-compat-test.yaml") + lopper_sdt.output_file = output_path + options = { + 'verbose': 0, + 'args': [] + } + + sdt_devices(None, lopper_sdt, options) + + with open(output_path) as f: + data = yaml.safe_load(f) + + domain = data['domains']['sdt_all_devices'] + compatible = domain.get('compatible') + + assert compatible == 'openamp,domain-v1,devices', \ + "Domain should have identifiable compatible string"