diff --git a/src/loadpath/architecture/snapshot.py b/src/loadpath/architecture/snapshot.py index fbedac1..8cb50dd 100644 --- a/src/loadpath/architecture/snapshot.py +++ b/src/loadpath/architecture/snapshot.py @@ -8,7 +8,7 @@ from loadpath.architecture.depth import deepening_candidates from loadpath.architecture.rules import evaluate from loadpath.config import LoadpathConfig, load_config -from loadpath.graph.store import GraphStore +from loadpath.graph.store import GraphStore, linked_edges from loadpath.index import default_db_path, index_drift from loadpath.types import NodeType @@ -18,6 +18,7 @@ NodeType.ROUTE.value, NodeType.VIEW.value, NodeType.SERIALIZER.value, + NodeType.FORM.value, NodeType.MODEL.value, NodeType.TASK.value, NodeType.MANAGEMENT_COMMAND.value, @@ -77,9 +78,7 @@ def summarize_index(store: GraphStore, config: LoadpathConfig) -> dict[str, Any] def architecture_graph(store: GraphStore) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: nodes = [n for n in store.nodes() if n["type"] in ARCHITECTURE_NODE_TYPES] - ids = {n["id"] for n in nodes} - edges = [e for e in store.edges() if e["src"] in ids and e["dst"] in ids] - return nodes, edges + return nodes, linked_edges(nodes, store.edges()) def architecture_report(repo_root: Path, db_path: Path | None = None) -> dict[str, Any]: diff --git a/src/loadpath/detect.py b/src/loadpath/detect.py index 4d1447c..92b95a2 100644 --- a/src/loadpath/detect.py +++ b/src/loadpath/detect.py @@ -23,6 +23,8 @@ "site-packages", } +TESTISH_PARTS = {"test", "tests", "testing"} + def _skip(path: Path) -> bool: return any(part in SKIP_DIRS or part.startswith(".") for part in path.parts) @@ -115,15 +117,36 @@ def _first(repo_root: Path, name: str) -> str | None: return None +def _is_testish(rel: Path) -> bool: + return any(part in TESTISH_PARTS for part in rel.parts) + + def _detect_django_root(repo_root: Path) -> str: - manage = None - for path in repo_root.rglob("manage.py"): - if _skip(path): + """Prefer the package that holds real apps, not a nested test project's manage.py.""" + parents: list[tuple[str, ...]] = [] + for marker in repo_root.rglob("apps.py"): + if _skip(marker): + continue + app_dir = marker.parent + if app_dir.name in {"migrations", "tests", "management"}: + continue + rel = app_dir.relative_to(repo_root) + if _is_testish(rel): continue - manage = path - break - if manage is not None: - rel = manage.parent.relative_to(repo_root) + parents.append(rel.parent.parts) + if parents: + common: list[str] = [] + for items in zip(*parents): + if len(set(items)) == 1: + common.append(items[0]) + else: + break + return "/".join(common) if common else "." + + manages = [p for p in repo_root.rglob("manage.py") if not _skip(p)] + manages.sort(key=lambda p: (_is_testish(p.relative_to(repo_root)), len(p.relative_to(repo_root).parts))) + if manages: + rel = manages[0].parent.relative_to(repo_root) return rel.as_posix() if rel.parts else "." for candidate in ("backend", "server", "api", "app"): if (repo_root / candidate).is_dir(): @@ -152,14 +175,15 @@ def _django_apps(repo_root: Path, django_root: str) -> list[str]: for marker in root.rglob("apps.py"): if _skip(marker): continue - if marker.parent.name in {"migrations", "tests", "management"}: + rel = marker.relative_to(repo_root) + if _is_testish(rel) or marker.parent.name in {"migrations", "tests", "management"}: continue name = marker.parent.name if name not in apps and name not in {"config", "project", "settings"}: apps.append(name) if not apps: for marker in root.rglob("models.py"): - if _skip(marker): + if _skip(marker) or _is_testish(marker.relative_to(repo_root)): continue name = marker.parent.name if name not in apps and name not in {"migrations", "config"}: diff --git a/src/loadpath/extractors/django.py b/src/loadpath/extractors/django.py index ab81687..a200bf1 100644 --- a/src/loadpath/extractors/django.py +++ b/src/loadpath/extractors/django.py @@ -30,6 +30,7 @@ } SERIALIZER_BASES = {"Serializer", "ModelSerializer", "HyperlinkedModelSerializer", "ListSerializer"} +FORM_BASES = {"Form", "ModelForm", "BaseForm", "BaseModelForm"} MODEL_BASES = {"Model"} ADMIN_BASES = {"ModelAdmin", "StackedInline", "TabularInline"} CELERY_DECORATORS = {"shared_task", "task", "periodic_task"} @@ -141,7 +142,18 @@ def _app_from_path(rel: str) -> str | None: if idx > 0: return parts[idx - 1] for i, part in enumerate(parts): - if part in {"models.py", "views.py", "serializers.py", "urls.py", "signals.py", "tasks.py", "admin.py", "apps.py"}: + if part in { + "models.py", + "views.py", + "serializers.py", + "urls.py", + "signals.py", + "signal_handlers.py", + "forms.py", + "tasks.py", + "admin.py", + "apps.py", + }: return parts[i - 1] if i > 0 else None if part == "management" and i > 0: return parts[i - 1] @@ -216,6 +228,12 @@ def visit_ClassDef(self, node: ast.ClassDef) -> None: self._model(node) elif _has_base(node, SERIALIZER_BASES): self._serializer(node) + elif _has_base(node, FORM_BASES) or ( + node.name.endswith("Form") + and not node.name.startswith("Test") + and not _has_base(node, {"TestCase", "SimpleTestCase", "TransactionTestCase", "LiveServerTestCase", "APITestCase"}) + ): + self._serializer(node, ntype=NodeType.FORM) elif _has_base(node, DJANGO_VIEW_BASES) or node.name.endswith(("View", "ViewSet")): self._view(node) elif any(b.split(".")[-1] in {"BaseCommand", "AppCommand", "LabelCommand"} for b in _bases(node)): @@ -240,6 +258,7 @@ def visit_ClassDef(self, node: ast.ClassDef) -> None: def visit_FunctionDef(self, node: ast.FunctionDef) -> None: self._maybe_task(node) self._maybe_receiver(node) + self._maybe_plain_signal_handler(node) self._maybe_command(node) self._maybe_test(node) self._maybe_service_fn(node) @@ -282,8 +301,14 @@ def visit_Call(self, node: ast.Call) -> None: self.graph.residuals.append(f"Raw SQL ({fname}) in {self.rel_path}:{node.lineno}") elif short in {"select_related", "prefetch_related"}: pass - elif short == "connect" and any(s in fname for s in SIGNAL_NAMES | {"signal"}): - self._signal_connect(node, fname) + elif short == "connect": + handler = node.args[0] if node.args else _kw(node, "receiver") + stem = Path(self.rel_path).stem + looks_signal = stem in {"apps", "signals", "signal_handlers", "handlers"} or any( + token in fname.lower() for token in {n.lower() for n in SIGNAL_NAMES} | {"signal"} + ) + if looks_signal and isinstance(handler, (ast.Name, ast.Attribute)): + self._signal_connect(node, fname) elif short == "reverse": self._reverse(node) self.generic_visit(node) @@ -344,9 +369,12 @@ def _model(self, node: ast.ClassDef) -> None: if extra.get("on_delete") == "CASCADE": self.add_edge(model.id, rel_id, EdgeType.RELATES_TO, extra={"cascade": True}) - def _serializer(self, node: ast.ClassDef) -> None: + def _serializer(self, node: ast.ClassDef, ntype: NodeType = NodeType.SERIALIZER) -> None: qname = f"{self.app}.{node.name}" - ser = self.add_node(NodeType.SERIALIZER, node.name, qname, node.lineno, {"app": self.app}) + extra: dict = {"app": self.app} + if ntype is NodeType.FORM: + extra["django_form"] = True + ser = self.add_node(ntype, node.name, qname, node.lineno, extra) meta_model = None meta_fields: list[str] | None = None meta_exclude: list[str] | None = None @@ -805,6 +833,27 @@ def _maybe_receiver(self, node: ast.FunctionDef) -> None: model_q = sender if "." in sender else f"{self.app}.{sender.split('.')[-1]}" self.add_edge(recv.id, node_id(NodeType.MODEL, model_q), EdgeType.EMITS_SIGNAL) + def _maybe_plain_signal_handler(self, node: ast.FunctionDef) -> None: + if self.class_stack: + return + if Path(self.rel_path).stem not in {"signals", "signal_handlers", "handlers"}: + return + for dec in node.decorator_list: + dname = _name(dec.func if isinstance(dec, ast.Call) else dec) or "" + if dname.split(".")[-1] == "receiver": + return + args = [a.arg for a in node.args.args] + if not (node.args.kwarg or "instance" in args or "sender" in args): + return + qname = f"{self.app}.{node.name}" + self.add_node( + NodeType.RECEIVER, + node.name, + qname, + node.lineno, + {"app": self.app, "plain_handler": True}, + ) + def _maybe_command(self, node: ast.FunctionDef) -> None: if "management/commands" in self.rel_path and node.name == "handle": cmd = Path(self.rel_path).stem @@ -833,7 +882,7 @@ def _maybe_test(self, node: ast.FunctionDef) -> None: # crude: referenced class names in the test become tested_by for child in ast.walk(node): if isinstance(child, ast.Name) and child.id[:1].isupper(): - for ntype in (NodeType.SERIALIZER, NodeType.VIEW, NodeType.MODEL, NodeType.SERVICE): + for ntype in (NodeType.SERIALIZER, NodeType.FORM, NodeType.VIEW, NodeType.MODEL, NodeType.SERVICE, NodeType.RECEIVER): self.add_edge( node_id(ntype, f"{self.app}.{child.id}"), test.id, @@ -915,7 +964,39 @@ def _get_model(self, node: ast.Call) -> None: self.add_node(NodeType.MODEL, label.split(".")[-1], label, node.lineno, {"string_ref": True}) def _signal_connect(self, node: ast.Call, fname: str) -> None: - self.graph.residuals.append(f"signal.connect() at {self.rel_path}:{node.lineno} ({fname})") + handler_ast = node.args[0] if node.args else _kw(node, "receiver") + handler = _name(handler_ast) + signal = fname.rsplit(".", 1)[0] if "." in fname else None + sender = _name(_kw(node, "sender")) + if not handler: + self.graph.residuals.append(f"signal.connect() at {self.rel_path}:{node.lineno} ({fname})") + return + handler_short = handler.split(".")[-1] + qname = handler if "." in handler else f"{self.app}.{handler_short}" + extra = { + "app": self.app, + "signal": signal, + "sender": sender, + "referenced": True, + "via": "connect", + } + recv = self.add_node(NodeType.RECEIVER, handler_short, qname, node.lineno, extra) + if signal: + sig_short = signal.split(".")[-1] + sig_id = node_id(NodeType.SIGNAL, sig_short) + self.graph.nodes.append( + Node( + id=sig_id, + type=NodeType.SIGNAL, + name=sig_short, + qualified_name=sig_short, + extra={"referenced": True}, + ) + ) + self.add_edge(sig_id, recv.id, EdgeType.RECEIVES) + if sender: + model_q = sender if "." in sender else f"{self.app}.{sender.split('.')[-1]}" + self.add_edge(recv.id, node_id(NodeType.MODEL, model_q), EdgeType.EMITS_SIGNAL) def _reverse(self, node: ast.Call) -> None: name = _const_str(node.args[0]) if node.args else None diff --git a/src/loadpath/graph/store.py b/src/loadpath/graph/store.py index 2252872..de2cd58 100644 --- a/src/loadpath/graph/store.py +++ b/src/loadpath/graph/store.py @@ -59,6 +59,12 @@ """ +def linked_edges(nodes: Iterable[dict[str, Any]], edges: Iterable[dict[str, Any]]) -> list[dict[str, Any]]: + """Keep only edges whose src and dst both exist in `nodes`.""" + ids = {n["id"] for n in nodes if n and n.get("id")} + return [e for e in edges if e.get("src") in ids and e.get("dst") in ids] + + class GraphStore: def __init__(self, db_path: Path) -> None: self.db_path = Path(db_path) @@ -248,7 +254,8 @@ def nodes_in_files(self, paths: Iterable[str]) -> list[dict[str, Any]]: return [self._node_from_row(r) for r in rows] def subgraph(self, seed_ids: Iterable[str], hops: int = 6) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - frontier = set(seed_ids) + known = {r["id"] for r in self.conn.execute("SELECT id FROM nodes")} + frontier = set(seed_ids) & known seen_nodes = set(frontier) seen_edges: dict[str, dict[str, Any]] = {} for _ in range(hops): @@ -257,14 +264,16 @@ def subgraph(self, seed_ids: Iterable[str], hops: int = 6) -> tuple[list[dict[st nxt: set[str] = set() for nid in frontier: for edge in self.neighbors(nid, "both"): - seen_edges[edge["id"]] = edge other = edge["dst"] if edge["src"] == nid else edge["src"] + if other not in known: + continue + seen_edges[edge["id"]] = edge if other not in seen_nodes: seen_nodes.add(other) nxt.add(other) frontier = nxt - nodes = [self.get_node(i) for i in seen_nodes] - return [n for n in nodes if n], list(seen_edges.values()) + nodes = [n for n in (self.get_node(i) for i in seen_nodes) if n] + return nodes, linked_edges(nodes, list(seen_edges.values())) def iter_nodes(self) -> Iterator[dict[str, Any]]: for row in self.conn.execute("SELECT * FROM nodes"): diff --git a/src/loadpath/index.py b/src/loadpath/index.py index 4a80eb7..f68434e 100644 --- a/src/loadpath/index.py +++ b/src/loadpath/index.py @@ -15,7 +15,7 @@ PY_SKIP = {"migrations"} # still extract migrations, just not skip INDEX_EXTENSIONS = {".py", ".ts", ".tsx", ".js", ".jsx"} # Bump when extractor/stitch node identity changes so incremental indexes rebuild. -INDEX_REVISION = "3" +INDEX_REVISION = "4" def default_db_path(repo_root: Path) -> Path: @@ -143,7 +143,8 @@ def index_repo( store.set_meta("repo_root", str(repo_root)) drift = index_drift(store, repo_root, config) - if incremental and store.file_count() > 0 and not drift["stale"]: + revision_changed = (store.get_meta("index_revision") or "") != INDEX_REVISION + if incremental and store.file_count() > 0 and not drift["stale"] and not revision_changed: store.set_meta("reindex_skipped", "1") store.set_meta("files_extracted", "0") store.set_meta("files_skipped", str(drift["indexed_count"])) @@ -161,7 +162,7 @@ def index_repo( for path in files: rel = path.relative_to(repo_root).as_posix() digest = file_hash(path) - if incremental and store.file_hash(rel) == digest: + if incremental and not revision_changed and store.file_hash(rel) == digest: skipped.add(rel) continue store.delete_file_nodes(rel, drop_incoming=False) @@ -214,6 +215,7 @@ def index_repo( store.set_meta("django_boot_detail", boot_detail) store.set_meta("config_hash", _config_digest(repo_root)) store.set_meta("sidecar_hash", _sidecar_digest(repo_root, config)) + store.set_meta("index_revision", INDEX_REVISION) store.conn.commit() return store diff --git a/src/loadpath/report/graph.html b/src/loadpath/report/graph.html index 27fc3a1..cc4b7fa 100644 --- a/src/loadpath/report/graph.html +++ b/src/loadpath/report/graph.html @@ -89,8 +89,9 @@