From 8743fe5478126edcfce0395fe1f7873e48aa21df Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 7 May 2026 12:19:52 -0400 Subject: [PATCH 1/4] Add Flutter runtime inspector support --- .gitignore | 2 + AGENTS.md | 5 + README.md | 30 +- client/src/api/types.ts | 4 + client/src/app/uiState.ts | 2 + .../accessibility/AccessibilityInspector.tsx | 16 + client/src/styles/components.css | 6 + docs/.vitepress/config.mts | 4 + docs/api/inspector-protocol.md | 2 +- docs/api/rest.md | 5 +- docs/cli/commands.md | 4 +- docs/cli/flags.md | 16 +- docs/contributing.md | 3 + docs/guide/architecture.md | 2 + docs/guide/index.md | 1 + docs/inspector/accessibility.md | 2 +- docs/inspector/flutter.md | 59 + docs/inspector/index.md | 6 +- packages/flutter-inspector/README.md | 50 + .../flutter-inspector/analysis_options.yaml | 5 + .../SimDeckFlutterInspectorPlugin.swift | 38 + .../ios/simdeck_flutter_inspector.podspec | 14 + .../lib/simdeck_flutter_inspector.dart | 1087 +++++++++++++++++ packages/flutter-inspector/pubspec.yaml | 22 + packages/simdeck-test/dist/index.d.ts | 8 +- packages/simdeck-test/src/index.ts | 8 +- server/src/api/routes.rs | 76 +- server/src/main.rs | 2 + skills/simdeck/SKILL.md | 4 +- 29 files changed, 1452 insertions(+), 31 deletions(-) create mode 100644 docs/inspector/flutter.md create mode 100644 packages/flutter-inspector/README.md create mode 100644 packages/flutter-inspector/analysis_options.yaml create mode 100644 packages/flutter-inspector/ios/Classes/SimDeckFlutterInspectorPlugin.swift create mode 100644 packages/flutter-inspector/ios/simdeck_flutter_inspector.podspec create mode 100644 packages/flutter-inspector/lib/simdeck_flutter_inspector.dart create mode 100644 packages/flutter-inspector/pubspec.yaml diff --git a/.gitignore b/.gitignore index 88c21a83..3fe5e034 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ packages/inspector-agent/.build/ packages/inspector-agent/.swiftpm/ packages/nativescript-inspector/dist/ packages/react-native-inspector/dist/ +packages/flutter-inspector/.dart_tool/ +packages/flutter-inspector/pubspec.lock docs/.vitepress/dist/ docs/.vitepress/cache/ cloud/ diff --git a/AGENTS.md b/AGENTS.md index b4a1bf05..cef353a6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,6 +51,10 @@ The native side should own anything that depends on macOS frameworks, `xcrun sim React Native in-app inspector runtime that connects to the Rust server over WebSocket, publishes React Fiber component hierarchies with Metro source locations, and performs best-effort debug JS/native prop edits. +- `packages/flutter-inspector/lib/simdeck_flutter_inspector.dart` + Flutter in-app inspector runtime that connects to the Rust server over + WebSocket, publishes widget/render/semantics hierarchies with debug creation + locations, and performs best-effort semantics, focus, text, and scroll actions. ## Working Rules @@ -59,6 +63,7 @@ The native side should own anything that depends on macOS frameworks, `xcrun sim - Keep browser-only presentation logic in `client/`. - Keep NativeScript app runtime inspection logic in `packages/nativescript-inspector/`. - Keep React Native app runtime inspection logic in `packages/react-native-inspector/`. +- Keep Flutter app runtime inspection logic in `packages/flutter-inspector/`. - Prefer adding a native API endpoint before adding client-only assumptions. - Do not add a Node or Swift dependency to solve work that already fits in Foundation/AppKit. - When touching private API usage, keep the adaptation small and explicit and document any simulator/runtime assumptions here. diff --git a/README.md b/README.md index afb7a70f..ddf7fde4 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ view inside the editor. - Full simulator control & inspection using private accessibility APIs - available using `simdeck` CLI - Real-time screen `describe` command using accessibility view tree - available in token-efficient format for agents - CoreSimulator chrome asset rendering for device bezels -- NativeScript, React Native, UIKit and SwiftUI runtime inspector plugins to view app's view hierarchy live +- NativeScript, React Native, Flutter, UIKit and SwiftUI runtime inspector plugins to view app's view hierarchy live - `simdeck/test` for fast JS/TS app tests that can query accessibility state and drive simulator controls. - SimDeck Studio for sharing Simulator streams & automatic PR deployments to on-demand simulators @@ -182,9 +182,10 @@ booting is unavailable. `stream` writes an Annex B H.264 elementary stream to stdout for diagnostics or external tools such as `ffplay`. -`describe` uses the project daemon to prefer React Native, NativeScript, or -UIKit in-app inspectors, then falls back to the built-in private CoreSimulator -accessibility bridge. Use `--format agent` or `--format compact-json` for +`describe` uses the project daemon to prefer React Native, NativeScript, +Flutter, or UIKit in-app inspectors, then falls back to the built-in private +CoreSimulator accessibility bridge. Use `--format agent` or +`--format compact-json` for lower-token hierarchy dumps. Coordinate commands accept screen coordinates from the accessibility tree by default; pass `--normalized` to send `0.0..1.0` coordinates directly. @@ -241,6 +242,27 @@ so the package can capture React Fiber commits. The auto entrypoint no-ops outside development, reads `EXPO_PUBLIC_SIMDECK_PORT` when present, and otherwise scans common SimDeck daemon ports. +## Flutter Inspector + +Flutter apps can expose their widget tree, render frames, semantics metadata, +and debug widget creation locations with the Flutter inspector package: + +```dart +import 'package:flutter/foundation.dart'; +import 'package:flutter/widgets.dart'; +import 'package:simdeck_flutter_inspector/simdeck_flutter_inspector.dart'; + +void main() { + WidgetsFlutterBinding.ensureInitialized(); + + if (kDebugMode) { + startSimDeckFlutterInspector(port: 4310); + } + + runApp(const App()); +} +``` + ## VS Code Install the `nativescript.simdeck-vscode` extension from the VS Code Marketplace, then diff --git a/client/src/api/types.ts b/client/src/api/types.ts index 713a0155..d201c83d 100644 --- a/client/src/api/types.ts +++ b/client/src/api/types.ts @@ -164,6 +164,7 @@ export interface AccessibilityNode { enabled?: boolean | null; frame?: AccessibilityFrame | null; frameInScreen?: AccessibilityFrame | null; + flutter?: Record | null; help?: string | null; imageName?: string | null; inspectorId?: string | null; @@ -178,11 +179,13 @@ export interface AccessibilityNode { role?: string | null; role_description?: string | null; scroll?: Record | null; + semantics?: Record | null; source?: | "native-ax" | "in-app-inspector" | "nativescript" | "react-native" + | "flutter" | "swiftui" | string | null; @@ -207,6 +210,7 @@ export type AccessibilitySource = | "in-app-inspector" | "nativescript" | "react-native" + | "flutter" | "swiftui"; export type AccessibilitySourcePreference = AccessibilitySource | "auto"; diff --git a/client/src/app/uiState.ts b/client/src/app/uiState.ts index 60956b70..98871d0c 100644 --- a/client/src/app/uiState.ts +++ b/client/src/app/uiState.ts @@ -34,6 +34,7 @@ export const TOUCH_OVERLAY_VISIBLE_STORAGE_KEY = "xcw-touch-overlay-visible"; const ACCESSIBILITY_SOURCE_ORDER: AccessibilitySource[] = [ "nativescript", "react-native", + "flutter", "swiftui", "in-app-inspector", "native-ax", @@ -158,6 +159,7 @@ export function isAccessibilitySource( return ( value === "nativescript" || value === "react-native" || + value === "flutter" || value === "swiftui" || value === "in-app-inspector" || value === "native-ax" diff --git a/client/src/features/accessibility/AccessibilityInspector.tsx b/client/src/features/accessibility/AccessibilityInspector.tsx index 0d2d6dc8..f24f5e63 100644 --- a/client/src/features/accessibility/AccessibilityInspector.tsx +++ b/client/src/features/accessibility/AccessibilityInspector.tsx @@ -467,6 +467,7 @@ function NodeDetails({ ["Module", node.moduleName ?? ""], ["NativeScript", nativeScriptDescription(node.nativeScript)], ["React Native", reactNativeDescription(node.reactNative)], + ["Flutter", flutterDescription(node.flutter)], ["UIKit Class", node.className ?? ""], ["Last JS", lastUIKitScriptText(node)], ["Value", node.AXValue ?? ""], @@ -727,6 +728,7 @@ function errorMessage(error: unknown): string { const HIERARCHY_SOURCE_ORDER: AccessibilitySource[] = [ "nativescript", "react-native", + "flutter", "swiftui", "in-app-inspector", "native-ax", @@ -756,6 +758,9 @@ function sourceLabel(source: AccessibilitySource): string { if (source === "react-native") { return "React Native"; } + if (source === "flutter") { + return "Flutter"; + } if (source === "swiftui") { return "SwiftUI"; } @@ -795,6 +800,17 @@ function reactNativeDescription( return [tag, testID, nativeID].filter(Boolean).join(" / "); } +function flutterDescription(value: Record | null | undefined) { + if (!value) { + return ""; + } + const widgetType = + typeof value.widgetType === "string" ? value.widgetType : ""; + const stateType = typeof value.stateType === "string" ? value.stateType : ""; + const key = typeof value.key === "string" ? value.key : ""; + return [widgetType, stateType, key].filter(Boolean).join(" / "); +} + function lastUIKitScriptText(node: AccessibilityNode): string { const direct = stringRecordValue(node.uikitScript, "script"); if (direct) { diff --git a/client/src/styles/components.css b/client/src/styles/components.css index 9e69f76b..bb0275cb 100644 --- a/client/src/styles/components.css +++ b/client/src/styles/components.css @@ -624,6 +624,12 @@ color: color-mix(in srgb, #61dafb 78%, var(--text)); } +.hierarchy-source-pill.source-flutter { + border-color: color-mix(in srgb, #54c5f8 50%, var(--border)); + background: color-mix(in srgb, #54c5f8 14%, transparent); + color: color-mix(in srgb, #54c5f8 80%, var(--text)); +} + .hierarchy-source-pill.source-swiftui { border-color: color-mix(in srgb, #ff6b9d 50%, var(--border)); background: color-mix(in srgb, #ff6b9d 14%, transparent); diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 57140c85..c21eb9c3 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -135,6 +135,10 @@ export default defineConfig({ text: "React Native Runtime", link: "/inspector/react-native", }, + { + text: "Flutter Runtime", + link: "/inspector/flutter", + }, ], }, ], diff --git a/docs/api/inspector-protocol.md b/docs/api/inspector-protocol.md index 4dc26285..60bc8fb8 100644 --- a/docs/api/inspector-protocol.md +++ b/docs/api/inspector-protocol.md @@ -122,7 +122,7 @@ Params: { "includeHidden": false, "maxDepth": 20, "source": "uikit" } ``` -By default the agent returns the published framework hierarchy (e.g. NativeScript) when one exists. Pass `"source": "uikit"` to force the raw UIKit tree. +By default the agent returns the published framework hierarchy (for example NativeScript, React Native, Flutter, or SwiftUI) when one exists. Pass `"source": "uikit"` to force the raw UIKit tree when the runtime supports UIKit inspection. Published framework nodes may include `sourceLocation`: diff --git a/docs/api/rest.md b/docs/api/rest.md index 6955ee44..2d9a01ec 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -442,6 +442,7 @@ Returns the current accessibility tree. The server merges framework inspectors, | `auto` _(default)_ / unset | Use the most accurate source available, falling back to AX. | | `nativescript` / `ns` | Force the NativeScript logical tree if a NativeScript inspector is connected for the foreground app. | | `react-native` / `rn` | Force the React Native component tree if a React Native inspector is connected for the foreground app. | +| `flutter` / `fl` | Force the Flutter widget tree if a Flutter inspector is connected for the foreground app. | | `swiftui` / `swift-ui` | Force the published SwiftUI logical tree if the Swift agent root publisher is installed in the app. | | `uikit` / `in-app-inspector` | Force the raw UIKit hierarchy from the in-app inspector agent (NativeScript or Swift). | | `native-ax` / `ax` | Always use the native accessibility snapshot. | @@ -456,8 +457,8 @@ The response always includes: ```json { "roots": [...], - "source": "nativescript|react-native|swiftui|in-app-inspector|native-ax", - "availableSources": ["nativescript", "react-native", "swiftui", "in-app-inspector", "native-ax"], + "source": "nativescript|react-native|flutter|swiftui|in-app-inspector|native-ax", + "availableSources": ["nativescript", "react-native", "flutter", "swiftui", "in-app-inspector", "native-ax"], "fallbackReason": "...", "inspector": { ... } } diff --git a/docs/cli/commands.md b/docs/cli/commands.md index 7ad378be..c67386e9 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -198,13 +198,15 @@ simdeck describe simdeck describe --format agent --max-depth 4 simdeck describe --format compact-json simdeck describe --source nativescript +simdeck describe --source react-native +simdeck describe --source flutter simdeck describe --source uikit simdeck describe --source native-ax simdeck describe --point 120,240 simdeck describe --direct ``` -By default, `describe` uses the project daemon so it can prefer connected NativeScript or UIKit in-app inspectors, then fall back to the private CoreSimulator accessibility bridge. `--direct` skips the daemon and uses the native accessibility bridge directly. +By default, `describe` uses the project daemon so it can prefer connected NativeScript, React Native, Flutter, or UIKit in-app inspectors, then fall back to the private CoreSimulator accessibility bridge. `--direct` skips the daemon and uses the native accessibility bridge directly. Use `--format agent` for compact hierarchy text intended for agent planning, and `--format compact-json` when a script needs parseable lower-token output. diff --git a/docs/cli/flags.md b/docs/cli/flags.md index ecfad846..6afeb1e7 100644 --- a/docs/cli/flags.md +++ b/docs/cli/flags.md @@ -47,14 +47,14 @@ The public commands generate an access token automatically. Use `simdeck daemon ## `describe` -| Flag | Default | Description | -| ------------------ | ------------------------------ | ------------------------------------------------------------------------- | -| `--format` | `json` | Output format: `json`, `compact-json`, or `agent`. | -| `--source` | `auto` | Hierarchy source: `auto`, `nativescript`, `uikit`, or `native-ax`. | -| `--max-depth` | unlimited native / `80` daemon | Trim descendants after the requested depth. | -| `--include-hidden` | `false` | Include hidden in-app inspector views when supported. | -| `--direct` | `false` | Skip the daemon and use the private native accessibility bridge directly. | -| `--point ,` | unset | Return the native element at a screen point. | +| Flag | Default | Description | +| ------------------ | ------------------------------ | --------------------------------------------------------------------------------------------- | +| `--format` | `json` | Output format: `json`, `compact-json`, or `agent`. | +| `--source` | `auto` | Hierarchy source: `auto`, `nativescript`, `react-native`, `flutter`, `uikit`, or `native-ax`. | +| `--max-depth` | unlimited native / `80` daemon | Trim descendants after the requested depth. | +| `--include-hidden` | `false` | Include hidden in-app inspector views when supported. | +| `--direct` | `false` | Skip the daemon and use the private native accessibility bridge directly. | +| `--point ,` | unset | Return the native element at a screen point. | ## Input Flags diff --git a/docs/contributing.md b/docs/contributing.md index 4ac2b6d6..ad3e7df0 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -51,6 +51,7 @@ To run only the production server: | `cli/` | Objective-C native bridge for private CoreSimulator and SimulatorKit APIs. | | `client/` | React UI served at `/`. | | `packages/nativescript-inspector/` | TypeScript runtime for the NativeScript inspector. | +| `packages/flutter-inspector/` | Flutter runtime plugin for publishing widget, render, and semantics hierarchy data. | | `packages/inspector-agent/` | Swift Package for the Swift in-app inspector agent. | | `packages/simdeck-test/` | JS/TS testing API for daemon-backed simulator automation. | | `packages/vscode-extension/` | VS Code extension that opens the simulator inside an editor panel. | @@ -66,6 +67,8 @@ If you contribute, keep these invariants in mind. They are also enforced by the - Rust server logic stays under `server/`. - Browser-only presentation logic stays in `client/`. - NativeScript app runtime inspection logic stays in `packages/nativescript-inspector/`. +- React Native app runtime inspection logic stays in `packages/react-native-inspector/`. +- Flutter app runtime inspection logic stays in `packages/flutter-inspector/`. - Prefer adding a server endpoint before adding client-only assumptions. - Don't add a Node or Swift dependency to solve work that already fits in Foundation/AppKit. - When touching private API usage, keep the adaptation small and explicit and document any simulator/runtime assumptions in `AGENTS.md`. diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index e2af3763..4468b188 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -72,6 +72,7 @@ The client never depends on private APIs and never assumes anything not exposed - **`packages/nativescript-inspector/`** ships `@nativescript/simdeck-inspector`, a TypeScript runtime that connects from a NativeScript app to the server's WebSocket inspector hub. See [NativeScript Runtime](/inspector/nativescript). - **`packages/react-native-inspector/`** ships `react-native-simdeck`, a React Native runtime that connects from an app to the server's WebSocket inspector hub and publishes React Fiber hierarchy data. See [React Native Runtime](/inspector/react-native). +- **`packages/flutter-inspector/`** ships `simdeck_flutter_inspector`, a Flutter runtime plugin that connects from an app to the server's WebSocket inspector hub and publishes widget, render, and semantics hierarchy data. See [Flutter Runtime](/inspector/flutter). - **`packages/inspector-agent/`** ships `SimDeckInspectorAgent`, a Swift Package you can link from a debug iOS app to expose its UIKit hierarchy. See [Swift In-App Agent](/inspector/swift). - **`packages/vscode-extension/`** is the VS Code extension that opens the browser client inside a webview panel and auto-starts the server. - **`packages/simdeck-test/`** ships `simdeck/test`, a small JS/TS wrapper around daemon startup and the REST control API. See [Testing](/guide/testing). @@ -118,5 +119,6 @@ If you contribute, keep the following invariants in mind: - Rust server logic stays under `server/`. - Browser-only presentation logic stays in `client/`. - NativeScript app runtime inspection logic stays in `packages/nativescript-inspector/`. +- Flutter app runtime inspection logic stays in `packages/flutter-inspector/`. - Add a server endpoint before adding client-only assumptions. - The supported live video paths are the WebRTC H.264 offer endpoint plus the `/api/simulators/{udid}/h264` WebSocket fallback. Do not bring back legacy `/stream.h264` handling. diff --git a/docs/guide/index.md b/docs/guide/index.md index 34d858a4..b7cafac1 100644 --- a/docs/guide/index.md +++ b/docs/guide/index.md @@ -28,6 +28,7 @@ Optional companion packages: - [`@nativescript/simdeck-inspector`](/inspector/nativescript) — a debug-only NativeScript inspector runtime. - [`react-native-simdeck`](/inspector/react-native) — a debug-only React Native inspector runtime. +- [`simdeck_flutter_inspector`](/inspector/flutter) — a debug-only Flutter inspector runtime. - [`packages/inspector-agent`](/inspector/swift) — a Swift Package you can link from your iOS app to expose its UIKit hierarchy. - [`packages/vscode-extension`](/extensions/vscode) — opens the simulator inside a VS Code panel. diff --git a/docs/inspector/accessibility.md b/docs/inspector/accessibility.md index ba04befa..861bb334 100644 --- a/docs/inspector/accessibility.md +++ b/docs/inspector/accessibility.md @@ -16,7 +16,7 @@ It does **not** see: - NativeScript logical tree nodes. - UIView properties that aren't part of the accessibility surface. -For those, you need to link the [Swift in-app agent](/inspector/swift), attach the SwiftUI root publisher, or use the [NativeScript runtime inspector](/inspector/nativescript). +For those, you need to link the [Swift in-app agent](/inspector/swift), attach the SwiftUI root publisher, or use a framework runtime inspector such as [NativeScript](/inspector/nativescript), [React Native](/inspector/react-native), or [Flutter](/inspector/flutter). ## When AX is the right call diff --git a/docs/inspector/flutter.md b/docs/inspector/flutter.md new file mode 100644 index 00000000..d80e7b47 --- /dev/null +++ b/docs/inspector/flutter.md @@ -0,0 +1,59 @@ +# Flutter Runtime Inspector + +`simdeck_flutter_inspector` is a debug-only Flutter iOS plugin that publishes the live Flutter widget hierarchy to SimDeck. It uses the same [Inspector Protocol](/api/inspector-protocol) as the Swift, NativeScript, and React Native inspectors, and connects outbound to the SimDeck server over WebSocket. + +The package source lives at `packages/flutter-inspector/` in this repo. + +## Install + +```sh +flutter pub add simdeck_flutter_inspector +``` + +## Start the inspector + +Start the inspector during app startup in debug builds: + +```dart +import 'package:flutter/foundation.dart'; +import 'package:flutter/widgets.dart'; +import 'package:simdeck_flutter_inspector/simdeck_flutter_inspector.dart'; + +void main() { + WidgetsFlutterBinding.ensureInitialized(); + + if (kDebugMode) { + startSimDeckFlutterInspector(port: 4310); + } + + runApp(const App()); +} +``` + +`port` is the SimDeck server port. Flutter apps do not need to choose a local inspector port. + +## What it exposes + +`View.getHierarchy` returns the Flutter widget tree. Each node may include: + +- `type` and `displayName` — the widget runtime type. +- `title` — derived from semantics labels, common widget diagnostics, text, tooltip, value, or key. +- `frame` and `frameInScreen` — RenderObject bounds in logical screen points. +- `flutter` — widget, element, state type, key, and depth metadata. +- `semantics` — label, value, hint, identifier, role, flags, and supported semantics actions. +- `sourceLocation` — file, line, and column from Flutter widget creation tracking. + +Source locations require debug builds with `--track-widget-creation`, which Flutter enables by default for debug runs. + +## Debug actions + +The Flutter runtime supports `View.getProperties`, `View.listActions`, and `View.perform`. + +`View.perform` is best-effort and uses Flutter's own runtime APIs: + +- `tap`, `longPress`, `increase`, and `decrease` dispatch matching semantics actions. +- `focus` and `resignFirstResponder` use Flutter focus scopes. +- `setText` updates the nearest `EditableText`. +- `scrollBy` and `scrollTo` drive the nearest `Scrollable`. + +Flutter widgets are immutable, so `View.setProperty` returns an unsupported-method error. Runtime interactions should go through `View.perform`, while persistent visual changes should still be made in app source. diff --git a/docs/inspector/index.md b/docs/inspector/index.md index e8f26e31..2dfcea58 100644 --- a/docs/inspector/index.md +++ b/docs/inspector/index.md @@ -8,6 +8,7 @@ SimDeck blends three different ways to inspect what an iOS app is rendering: | **Swift in-app agent** | Apps that link `SimDeckInspectorAgent` in DEBUG. | Best for native iOS apps you control. | | **NativeScript runtime** | NativeScript apps that import `@nativescript/simdeck-inspector`. | Best for NativeScript apps — exposes the logical view tree, not just UIKit. | | **React Native runtime** | React Native apps that import `react-native-simdeck`. | Best for React Native apps — exposes components and Metro source locations. | +| **Flutter runtime** | Flutter apps that import `simdeck_flutter_inspector`. | Best for Flutter apps — exposes widgets, render frames, semantics actions, and creation locations. | | **DevTools panel** | Safari/WebKit targets, Metro React Native targets, Chrome Inspector targets, and SimDeck app runtime inspector sessions. | Best when you want a familiar browser inspector for app runtimes or web content. | The HTTP API picks the most specific source available, falls back to the next one when something goes wrong, and tells the client which sources were available so the UI can offer a switch. @@ -21,6 +22,7 @@ The HTTP API picks the most specific source available, falls back to the next on | `auto` _(default)_ / unset | Use the most accurate source available, falling back to AX. | | `nativescript` / `ns` | Force the NativeScript logical tree if a NativeScript inspector is connected for the foreground app. | | `react-native` / `rn` | Force the React Native component tree if a React Native inspector is connected for the foreground app. | +| `flutter` / `fl` | Force the Flutter widget tree if a Flutter inspector is connected for the foreground app. | | `swiftui` / `swift-ui` | Force the published SwiftUI logical tree if the Swift agent root publisher is installed in the app. | | `uikit` / `in-app-inspector` | Force the raw UIKit hierarchy from any in-app inspector (NativeScript or Swift agent). | | `native-ax` / `ax` | Always use the native accessibility snapshot. | @@ -36,10 +38,11 @@ Every accessibility tree response includes: ```json { - "source": "nativescript|react-native|swiftui|in-app-inspector|native-ax", + "source": "nativescript|react-native|flutter|swiftui|in-app-inspector|native-ax", "availableSources": [ "nativescript", "react-native", + "flutter", "swiftui", "in-app-inspector", "native-ax" @@ -57,6 +60,7 @@ Every accessibility tree response includes: - **You ship a NativeScript app.** Use the [NativeScript runtime inspector](/inspector/nativescript). It connects outbound to the SimDeck server and publishes both the NativeScript logical tree and the underlying UIKit hierarchy. - **You ship a React Native app.** Use the [React Native runtime inspector](/inspector/react-native). It connects outbound to the SimDeck server and publishes the React component tree with dev-mode source locations. - **You want DevTools for React Native, Safari/WebKit, Chrome Inspector, or a connected app runtime.** Use the DevTools toolbar toggle. It shows one target list for WebKit Remote Inspector targets, Metro React Native targets, local Chrome Inspector ports, and connected React Native or NativeScript SimDeck inspector sessions. +- **You ship a Flutter app.** Use the [Flutter runtime inspector](/inspector/flutter). It connects outbound to the SimDeck server and publishes the Flutter widget tree with render frames, semantics metadata, and debug creation locations. - **You can't link anything into the app.** Stick with [AX snapshot](/inspector/accessibility). It only sees what the iOS accessibility stack exposes, but it works for every app. ## Editing properties diff --git a/packages/flutter-inspector/README.md b/packages/flutter-inspector/README.md new file mode 100644 index 00000000..4cda9fbc --- /dev/null +++ b/packages/flutter-inspector/README.md @@ -0,0 +1,50 @@ +# Flutter Inspector + +`simdeck_flutter_inspector` is a debug-only Flutter iOS runtime inspector for SimDeck. It connects to the SimDeck server over WebSocket and publishes the live Flutter widget hierarchy with render bounds, diagnostics properties, semantics actions, and source locations when Flutter widget creation tracking is enabled. + +## Install + +Add the package to your Flutter app: + +```sh +flutter pub add simdeck_flutter_inspector +``` + +## Start + +Start the inspector during app startup in debug builds: + +```dart +import 'package:flutter/foundation.dart'; +import 'package:flutter/widgets.dart'; +import 'package:simdeck_flutter_inspector/simdeck_flutter_inspector.dart'; + +void main() { + WidgetsFlutterBinding.ensureInitialized(); + + if (kDebugMode) { + startSimDeckFlutterInspector(port: 4310); + } + + runApp(const App()); +} +``` + +`port` is the SimDeck server port. The Flutter app connects outbound to: + +```text +ws://127.0.0.1:4310/api/inspector/connect +``` + +## What It Exposes + +- Flutter widget hierarchy rooted at `WidgetsBinding.instance.rootElement`. +- RenderObject screen frames in logical screen points. +- Widget diagnostics properties and state type metadata. +- Semantics labels, values, hints, identifiers, flags, roles, and actions. +- Source locations from Flutter's widget creation tracking in debug builds. +- `View.hitTest`, `View.describeAtPoint`, `View.getProperties`, `View.listActions`, and `View.perform`. + +`View.perform` supports best-effort `tap`, `longPress`, `focus`, `resignFirstResponder`, `setText`, `scrollBy`, `scrollTo`, `increase`, and `decrease` actions when the selected widget exposes the matching Flutter semantics, text, focus, or scroll API. + +Flutter widgets are immutable, so `View.setProperty` intentionally returns an unsupported-method error. Use `View.perform` for runtime-safe text and interaction changes. diff --git a/packages/flutter-inspector/analysis_options.yaml b/packages/flutter-inspector/analysis_options.yaml new file mode 100644 index 00000000..d8aa88b1 --- /dev/null +++ b/packages/flutter-inspector/analysis_options.yaml @@ -0,0 +1,5 @@ +include: package:flutter_lints/flutter.yaml + +linter: + rules: + avoid_print: true diff --git a/packages/flutter-inspector/ios/Classes/SimDeckFlutterInspectorPlugin.swift b/packages/flutter-inspector/ios/Classes/SimDeckFlutterInspectorPlugin.swift new file mode 100644 index 00000000..fdb6404a --- /dev/null +++ b/packages/flutter-inspector/ios/Classes/SimDeckFlutterInspectorPlugin.swift @@ -0,0 +1,38 @@ +import Flutter +import UIKit + +public class SimDeckFlutterInspectorPlugin: NSObject, FlutterPlugin { + public static func register(with registrar: FlutterPluginRegistrar) { + let channel = FlutterMethodChannel( + name: "simdeck_flutter_inspector", + binaryMessenger: registrar.messenger() + ) + registrar.addMethodCallDelegate(SimDeckFlutterInspectorPlugin(), channel: channel) + } + + public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { + switch call.method { + case "getInfo": + let bundle = Bundle.main + let screen = UIScreen.main + result([ + "processIdentifier": ProcessInfo.processInfo.processIdentifier, + "bundleIdentifier": bundle.bundleIdentifier as Any, + "bundleName": bundle.object(forInfoDictionaryKey: "CFBundleName") as? String + ?? bundle.object(forInfoDictionaryKey: "CFBundleDisplayName") as? String + ?? "", + "displayScale": screen.scale, + "screenBounds": [ + "x": screen.bounds.origin.x, + "y": screen.bounds.origin.y, + "width": screen.bounds.size.width, + "height": screen.bounds.size.height, + ], + "systemName": UIDevice.current.systemName, + "systemVersion": UIDevice.current.systemVersion, + ]) + default: + result(FlutterMethodNotImplemented) + } + } +} diff --git a/packages/flutter-inspector/ios/simdeck_flutter_inspector.podspec b/packages/flutter-inspector/ios/simdeck_flutter_inspector.podspec new file mode 100644 index 00000000..bb573cd0 --- /dev/null +++ b/packages/flutter-inspector/ios/simdeck_flutter_inspector.podspec @@ -0,0 +1,14 @@ +Pod::Spec.new do |s| + s.name = 'simdeck_flutter_inspector' + s.version = '0.1.0' + s.summary = 'Debug-only Flutter runtime inspector for SimDeck.' + s.description = 'Publishes Flutter widget hierarchy metadata to SimDeck during debug sessions.' + s.homepage = 'https://github.com/NativeScript/SimDeck' + s.license = { :type => 'Apache-2.0' } + s.author = { 'SimDeck' => 'support@nativescript.org' } + s.source = { :path => '.' } + s.source_files = 'Classes/**/*' + s.dependency = 'Flutter' + s.platform = :ios, '12.0' + s.swift_version = '5.0' +end diff --git a/packages/flutter-inspector/lib/simdeck_flutter_inspector.dart b/packages/flutter-inspector/lib/simdeck_flutter_inspector.dart new file mode 100644 index 00000000..b12d5dbb --- /dev/null +++ b/packages/flutter-inspector/lib/simdeck_flutter_inspector.dart @@ -0,0 +1,1087 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io' as io; +import 'dart:math' as math; +import 'dart:ui' show FlutterView; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/rendering.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter/widgets.dart'; + +const String _protocolVersion = '0.1'; +const MethodChannel _channel = MethodChannel('simdeck_flutter_inspector'); + +SimDeckFlutterInspector? _sharedInspector; + +SimDeckFlutterInspector startSimDeckFlutterInspector({ + String host = '127.0.0.1', + String path = '/api/inspector/connect', + int port = 4310, + bool reconnect = true, + bool secure = false, +}) { + if (_sharedInspector != null) { + return _sharedInspector!; + } + final inspector = SimDeckFlutterInspector( + SimDeckFlutterInspectorOptions( + host: host, + path: path, + port: port, + reconnect: reconnect, + secure: secure, + ), + ); + _sharedInspector = inspector; + inspector.start(); + return inspector; +} + +void stopSimDeckFlutterInspector() { + _sharedInspector?.stop(); + _sharedInspector = null; +} + +@immutable +class SimDeckFlutterInspectorOptions { + const SimDeckFlutterInspectorOptions({ + this.host = '127.0.0.1', + this.path = '/api/inspector/connect', + this.port = 4310, + this.reconnect = true, + this.secure = false, + }); + + final String host; + final String path; + final int port; + final bool reconnect; + final bool secure; +} + +class SimDeckFlutterInspector { + SimDeckFlutterInspector([SimDeckFlutterInspectorOptions? options]) + : options = options ?? const SimDeckFlutterInspectorOptions(); + + final SimDeckFlutterInspectorOptions options; + final Expando _ids = Expando('simdeckFlutterInspectorId'); + final Map _objects = {}; + final Map _frameCache = {}; + int _nextObjectId = 1; + io.WebSocket? _socket; + Timer? _pollTimer; + Timer? _reconnectTimer; + bool _started = false; + bool _polling = false; + Map? _metadata; + SemanticsHandle? _semanticsHandle; + + void start() { + stop(); + _started = true; + _semanticsHandle = SemanticsBinding.instance.ensureSemantics(); + unawaited(_connect()); + _startPolling(); + } + + void stop() { + _reconnectTimer?.cancel(); + _reconnectTimer = null; + _pollTimer?.cancel(); + _pollTimer = null; + _started = false; + _polling = false; + _semanticsHandle?.dispose(); + _semanticsHandle = null; + final socket = _socket; + _socket = null; + unawaited(socket?.close()); + } + + Future _connect() async { + final scheme = options.secure ? 'wss' : 'ws'; + final url = '$scheme://${options.host}:${options.port}${options.path}'; + io.WebSocket? connectedSocket; + try { + final socket = await io.WebSocket.connect(url); + connectedSocket = socket; + _socket = socket; + await _sendReady(socket); + await for (final message in socket) { + if (message is String) { + final response = await _executeRequest(_decodeRequest(message)); + socket.add(jsonEncode(response)); + } + } + } catch (_) { + // SimDeck may not be running yet; reconnect below keeps startup cheap. + } finally { + if (_started && + options.reconnect && + identical(_socket, connectedSocket)) { + _socket = null; + _scheduleReconnect(); + } + } + } + + Future _sendReady(io.WebSocket socket) async { + socket.add( + jsonEncode({ + 'method': 'Inspector.ready', + 'params': await _info(), + }), + ); + } + + void _scheduleReconnect() { + if (_reconnectTimer != null) { + return; + } + _reconnectTimer = Timer(const Duration(seconds: 1), () { + _reconnectTimer = null; + unawaited(_connect()); + }); + } + + void _startPolling() { + if (_polling) { + return; + } + _polling = true; + _schedulePoll(Duration.zero); + } + + void _schedulePoll(Duration delay) { + if (!_polling) { + return; + } + _pollTimer?.cancel(); + _pollTimer = Timer(delay, () { + _pollTimer = null; + unawaited(_pollCommands()); + }); + } + + Future _pollCommands() async { + if (!_polling) { + return; + } + try { + final info = await _info(); + final pid = info['processIdentifier']; + final client = io.HttpClient(); + try { + final request = await client.getUrl( + Uri.parse('${_httpBaseUrl()}/api/inspector/poll?pid=$pid'), + ); + final response = await request.close(); + if (response.statusCode == io.HttpStatus.noContent) { + _schedulePoll(Duration.zero); + return; + } + if (response.statusCode < 200 || response.statusCode >= 300) { + throw StateError( + 'Inspector poll failed with HTTP ${response.statusCode}.', + ); + } + final body = await utf8.decoder.bind(response).join(); + final result = await _executeRequest(_decodeRequest(body)); + final post = await client.postUrl( + Uri.parse('${_httpBaseUrl()}/api/inspector/response'), + ); + post.headers.contentType = io.ContentType.json; + post.add( + utf8.encode( + jsonEncode({'processIdentifier': pid, ...result}), + ), + ); + await post.close(); + } finally { + client.close(force: true); + } + _schedulePoll(Duration.zero); + } catch (_) { + _schedulePoll(const Duration(milliseconds: 500)); + } + } + + String _httpBaseUrl() { + final scheme = options.secure ? 'https' : 'http'; + return '$scheme://${options.host}:${options.port}'; + } + + Map _decodeRequest(String data) { + final decoded = jsonDecode(data); + if (decoded is Map) { + return decoded; + } + if (decoded is Map) { + return decoded.cast(); + } + throw const SimDeckFlutterInspectorFailure( + -32600, + 'Inspector request must be a JSON object.', + ); + } + + Future> _executeRequest( + Map request, + ) async { + try { + final method = request['method']; + if (method is! String || method.isEmpty) { + throw const SimDeckFlutterInspectorFailure( + -32600, + 'Inspector request requires method.', + ); + } + final params = _objectMap(request['params']) ?? {}; + final result = await _dispatch(method, params); + return {'id': request['id'], 'result': result}; + } catch (error) { + return { + 'id': request['id'], + 'error': _inspectorError(error), + }; + } + } + + Future _dispatch(String method, Map params) async { + switch (method) { + case 'Runtime.ping': + return { + 'ok': true, + 'protocolVersion': _protocolVersion, + }; + case 'Inspector.getInfo': + return _info(); + case 'View.getHierarchy': + return _hierarchy(params); + case 'View.get': + return _getView(params); + case 'View.hitTest': + return _hitTest(params); + case 'View.describeAtPoint': + return _describeAtPoint(params); + case 'View.listActions': + return _listActions(params); + case 'View.perform': + return _perform(params); + case 'View.getProperties': + return _getProperties(params); + case 'View.setProperty': + throw const SimDeckFlutterInspectorFailure( + -32012, + 'Flutter widgets are immutable; View.setProperty is not supported. Use View.perform for supported runtime actions.', + ); + default: + throw SimDeckFlutterInspectorFailure( + -32601, + 'Unknown inspector method: $method', + ); + } + } + + Future> _info() async { + final metadata = await _loadMetadata(); + final root = WidgetsBinding.instance.rootElement; + return { + 'protocolVersion': _protocolVersion, + 'transport': 'websocket', + 'processIdentifier': metadata['processIdentifier'], + 'bundleIdentifier': metadata['bundleIdentifier'], + 'bundleName': metadata['bundleName'], + 'displayScale': metadata['displayScale'], + 'screenBounds': metadata['screenBounds'], + 'coordinateSpace': 'screen-points', + 'methods': [ + 'Runtime.ping', + 'Inspector.getInfo', + 'View.getHierarchy', + 'View.get', + 'View.hitTest', + 'View.describeAtPoint', + 'View.listActions', + 'View.perform', + 'View.getProperties', + 'View.setProperty', + ], + 'appHierarchy': { + 'source': 'flutter', + 'available': root != null, + 'publishedAt': DateTime.now().toUtc().toIso8601String(), + }, + 'flutter': { + 'available': true, + 'widgetCreationTracked': WidgetInspectorService.instance + .isWidgetCreationTracked(), + }, + 'uikit': {'available': false, 'propertyEditing': false}, + }; + } + + Future> _loadMetadata() async { + if (_metadata != null) { + return _metadata!; + } + final view = _firstFlutterView(); + final logicalSize = view == null + ? Size.zero + : view.physicalSize / view.devicePixelRatio; + final fallback = { + 'processIdentifier': io.pid, + 'bundleIdentifier': io.Platform.resolvedExecutable, + 'bundleName': 'Flutter', + 'displayScale': view?.devicePixelRatio ?? 1.0, + 'screenBounds': _rectJson(Offset.zero & logicalSize), + }; + try { + final native = await _channel.invokeMapMethod('getInfo'); + _metadata = {...fallback, if (native != null) ...native}; + } catch (_) { + _metadata = fallback; + } + return _metadata!; + } + + Map _hierarchy(Map params) { + if (params['source'] == 'uikit') { + throw const SimDeckFlutterInspectorFailure( + -32601, + 'Flutter inspector does not expose a raw UIKit hierarchy.', + ); + } + final root = WidgetsBinding.instance.rootElement; + final roots = >[]; + if (root != null) { + final context = _TraversalContext(); + final node = _elementNode( + root, + includeHidden: params['includeHidden'] == true, + maxDepth: _optionalInt(params['maxDepth']), + depth: 0, + context: context, + ); + if (node != null) { + roots.add(node); + } + } + return { + ..._snapshotMetadata(), + 'source': 'flutter', + 'roots': roots, + }; + } + + Map _getView(Map params) { + final id = _requiredString(params, 'id'); + final element = _objects[id]; + if (element == null) { + throw SimDeckFlutterInspectorFailure( + -32004, + 'No view was found for id $id.', + ); + } + final node = _elementNode( + element, + includeHidden: true, + maxDepth: _optionalInt(params['maxDepth']), + depth: 0, + context: _TraversalContext(), + ); + if (node == null) { + throw SimDeckFlutterInspectorFailure( + -32004, + 'No view was found for id $id.', + ); + } + return node; + } + + Map _hitTest(Map params) { + final point = Offset( + _requiredNumber(params, 'x'), + _requiredNumber(params, 'y'), + ); + final chain = _findChainAtPoint(point); + return { + 'x': point.dx, + 'y': point.dy, + 'hit': chain.isEmpty ? null : chain.last, + }; + } + + Map _describeAtPoint(Map params) { + final point = Offset( + _requiredNumber(params, 'x'), + _requiredNumber(params, 'y'), + ); + final chain = _findChainAtPoint(point); + return { + 'x': point.dx, + 'y': point.dy, + 'hit': chain.isEmpty ? null : chain.last, + 'chain': chain, + }; + } + + Map _listActions(Map params) { + final id = _requiredString(params, 'id'); + final element = _requireElement(id); + return {'id': id, 'actions': _actionsFor(element)}; + } + + Future> _perform(Map params) async { + final id = _requiredString(params, 'id'); + final action = _requiredString(params, 'action'); + final element = _requireElement(id); + switch (action) { + case 'describe': + return { + 'ok': true, + 'id': id, + 'actions': _actionsFor(element), + }; + case 'tap': + return _performSemanticsAction(element, action, SemanticsAction.tap); + case 'longPress': + return _performSemanticsAction( + element, + action, + SemanticsAction.longPress, + ); + case 'increase': + return _performSemanticsAction( + element, + action, + SemanticsAction.increase, + ); + case 'decrease': + return _performSemanticsAction( + element, + action, + SemanticsAction.decrease, + ); + case 'focus': + FocusScope.of(element).requestFocus(Focus.of(element)); + return {'ok': true, 'action': action}; + case 'resignFirstResponder': + FocusScope.of(element).unfocus(); + return {'ok': true, 'action': action}; + case 'setText': + _setEditableText(element, _stringValue(params['value'])); + return {'ok': true, 'action': action}; + case 'scrollBy': + case 'scrollTo': + await _scroll(element, params, relative: action == 'scrollBy'); + return {'ok': true, 'action': action}; + default: + throw SimDeckFlutterInspectorFailure( + -32010, + 'Unsupported view action: $action', + ); + } + } + + Map _performSemanticsAction( + Element element, + String actionName, + SemanticsAction action, + ) { + final renderObject = element.findRenderObject(); + final semantics = renderObject?.debugSemantics; + final owner = renderObject?.owner?.semanticsOwner; + if (semantics == null || owner == null) { + return {'ok': false, 'action': actionName}; + } + final data = semantics.getSemanticsData(); + if (!data.hasAction(action)) { + return {'ok': false, 'action': actionName}; + } + owner.performAction(semantics.id, action); + return {'ok': true, 'action': actionName}; + } + + Map _getProperties(Map params) { + final id = _requiredString(params, 'id'); + final element = _requireElement(id); + final widgetProperties = _diagnosticProperties(element.widget); + final state = element is StatefulElement ? element.state : null; + return { + 'id': id, + 'className': element.widget.runtimeType.toString(), + 'editableProperties': [], + 'properties': widgetProperties, + 'flutter': { + 'widgetType': element.widget.runtimeType.toString(), + 'elementType': element.runtimeType.toString(), + 'stateType': state?.runtimeType.toString(), + 'key': element.widget.key?.toString(), + 'depth': element.depth, + }, + 'renderObject': _renderObjectProperties(element.findRenderObject()), + 'semantics': _semanticsInfo(element.findRenderObject()), + }; + } + + Map? _elementNode( + Element element, { + required bool includeHidden, + required int? maxDepth, + required int depth, + required _TraversalContext context, + }) { + if (context.expired) { + return null; + } + context.remainingNodes -= 1; + + final hidden = _isHidden(element); + if (hidden && !includeHidden) { + return null; + } + + final children = >[]; + if (maxDepth == null || depth < maxDepth) { + element.visitChildren((child) { + final node = _elementNode( + child, + includeHidden: includeHidden, + maxDepth: maxDepth, + depth: depth + 1, + context: context, + ); + if (node != null) { + children.add(node); + } + }); + } + + final id = _objectId(element); + final renderObject = element.findRenderObject(); + final frame = _frameFor(renderObject); + final semantics = _semanticsInfo(renderObject); + final title = _nodeTitle(element, semantics); + final sourceLocation = _sourceLocation(element); + return { + 'id': id, + 'inspectorId': id, + 'type': element.widget.runtimeType.toString(), + 'displayName': element.widget.runtimeType.toString(), + 'title': title, + 'source': 'flutter', + 'sourceLocation': sourceLocation, + 'sourceLocations': [if (sourceLocation != null) sourceLocation], + 'frame': frame, + 'frameInScreen': frame, + 'AXIdentifier': _accessibilityIdentifier(element, semantics), + 'AXLabel': _firstString([semantics?['label'], title]), + 'AXValue': _firstString([semantics?['value']]), + 'help': _firstString([semantics?['hint']]), + 'enabled': hidden ? false : null, + 'isHidden': hidden, + 'custom_actions': semantics?['actions'], + 'flutter': { + 'widgetType': element.widget.runtimeType.toString(), + 'elementType': element.runtimeType.toString(), + 'stateType': element is StatefulElement + ? element.state.runtimeType.toString() + : null, + 'key': element.widget.key?.toString(), + 'depth': element.depth, + }, + 'semantics': semantics, + 'children': children, + }; + } + + List> _findChainAtPoint(Offset point) { + final root = WidgetsBinding.instance.rootElement; + if (root == null) { + return >[]; + } + final chain = >[]; + void visit(Element element) { + final node = _elementNode( + element, + includeHidden: false, + maxDepth: 0, + depth: 0, + context: _TraversalContext(), + ); + final frame = + _objectMap(node?['frame']) ?? _objectMap(node?['frameInScreen']); + if (node == null || frame == null || !_frameContains(frame, point)) { + return; + } + chain.add(node); + element.visitChildren(visit); + } + + visit(root); + return chain; + } + + Future _scroll( + Element element, + Map params, { + required bool relative, + }) async { + final scrollable = _findScrollable(element); + if (scrollable == null) { + throw const SimDeckFlutterInspectorFailure( + -32010, + 'Selected Flutter node is not inside a Scrollable.', + ); + } + final position = scrollable.position; + final dx = _optionalDouble(params['x']) ?? 0.0; + final dy = _optionalDouble(params['y']) ?? 0.0; + final delta = dy.abs() >= dx.abs() ? dy : dx; + final target = relative ? position.pixels + delta : delta; + final clamped = target.clamp( + position.minScrollExtent, + position.maxScrollExtent, + ); + if (params['animated'] == true) { + await position.animateTo( + clamped, + duration: const Duration(milliseconds: 250), + curve: Curves.easeOut, + ); + } else { + position.jumpTo(clamped); + } + } + + ScrollableState? _findScrollable(Element element) { + final direct = Scrollable.maybeOf(element); + if (direct != null) { + return direct; + } + ScrollableState? result; + void visit(Element child) { + result ??= Scrollable.maybeOf(child); + if (result != null) { + return; + } + child.visitChildren(visit); + } + + element.visitChildren(visit); + return result; + } + + void _setEditableText(Element element, String value) { + final editable = _findEditableTextState(element); + if (editable == null) { + throw const SimDeckFlutterInspectorFailure( + -32010, + 'Selected Flutter node does not contain EditableText.', + ); + } + editable.userUpdateTextEditingValue( + TextEditingValue( + text: value, + selection: TextSelection.collapsed(offset: value.length), + ), + SelectionChangedCause.keyboard, + ); + } + + EditableTextState? _findEditableTextState(Element element) { + if (element is StatefulElement && element.state is EditableTextState) { + return element.state as EditableTextState; + } + EditableTextState? result; + void visit(Element child) { + result ??= _findEditableTextState(child); + } + + element.visitChildren(visit); + return result; + } + + List _actionsFor(Element element) { + final actions = {'describe', 'getProperties'}; + final semantics = element.findRenderObject()?.debugSemantics; + if (semantics != null) { + final data = semantics.getSemanticsData(); + for (final action in [ + SemanticsAction.tap, + SemanticsAction.longPress, + SemanticsAction.increase, + SemanticsAction.decrease, + ]) { + if (data.hasAction(action)) { + actions.add(action.name); + } + } + } + if (_findEditableTextState(element) != null) { + actions.add('setText'); + actions.add('focus'); + actions.add('resignFirstResponder'); + } + if (_findScrollable(element) != null) { + actions.add('scrollBy'); + actions.add('scrollTo'); + } + return actions.toList()..sort(); + } + + String _objectId(Element element) { + final existing = _ids[element]; + if (existing != null) { + return existing; + } + final id = 'flutter:${_nextObjectId++}'; + _ids[element] = id; + _objects[id] = element; + return id; + } + + Element _requireElement(String id) { + final element = _objects[id]; + if (element == null) { + throw SimDeckFlutterInspectorFailure( + -32004, + 'No view was found for id $id.', + ); + } + return element; + } + + Map _snapshotMetadata() { + final metadata = _metadata ?? {}; + return { + 'capturedAt': DateTime.now().toUtc().toIso8601String(), + 'protocolVersion': _protocolVersion, + 'processIdentifier': metadata['processIdentifier'] ?? io.pid, + 'bundleIdentifier': metadata['bundleIdentifier'], + 'displayScale': + metadata['displayScale'] ?? + _firstFlutterView()?.devicePixelRatio ?? + 1.0, + 'coordinateSpace': 'screen-points', + }; + } + + Map? _frameFor(RenderObject? renderObject) { + if (renderObject == null || !renderObject.attached) { + return null; + } + final cached = _frameCache[identityHashCode(renderObject)]; + final now = DateTime.now(); + if (cached != null && + now.difference(cached.measuredAt).inMilliseconds <= 250) { + return cached.frame; + } + try { + final rect = MatrixUtils.transformRect( + renderObject.getTransformTo(null), + renderObject.paintBounds, + ); + if (!rect.isFinite || rect.isEmpty) { + return null; + } + final frame = _rectJson(rect); + _frameCache[identityHashCode(renderObject)] = _FrameCacheEntry( + frame, + now, + ); + return frame; + } catch (_) { + return null; + } + } + + Map? _semanticsInfo(RenderObject? renderObject) { + final node = renderObject?.debugSemantics; + if (node == null) { + return null; + } + try { + final data = node.getSemanticsData(); + final actions = [ + for (final action in SemanticsAction.values) + if (data.hasAction(action)) action.name, + ]; + return { + 'id': node.id, + 'identifier': data.identifier, + 'label': data.attributedLabel.string, + 'value': data.attributedValue.string, + 'hint': data.attributedHint.string, + 'tooltip': data.tooltip, + 'role': data.role.name, + 'actions': actions, + 'flags': data.flagsCollection.toString(), + 'isFocused': data.flagsCollection.isFocused.toString(), + 'isTextField': data.flagsCollection.isTextField, + 'isButton': data.flagsCollection.isButton, + 'isEnabled': data.flagsCollection.isEnabled.toString(), + 'isChecked': data.flagsCollection.isChecked.toString(), + 'isSelected': data.flagsCollection.isSelected.toString(), + }; + } catch (_) { + return null; + } + } + + Map? _sourceLocation(Element element) { + if (!WidgetInspectorService.instance.isWidgetCreationTracked()) { + return null; + } + try { + final json = element.toDiagnosticsNode().toJsonMap( + InspectorSerializationDelegate( + service: WidgetInspectorService.instance, + subtreeDepth: 0, + ), + ); + final location = _objectMap(json['creationLocation']); + if (location == null) { + return null; + } + return { + 'file': location['file'], + 'line': location['line'], + 'column': location['column'], + 'name': location['name'], + 'kind': 'flutter', + }; + } catch (_) { + return null; + } + } + + String _nodeTitle(Element element, Map? semantics) { + final widget = element.widget; + final properties = _diagnosticProperties(widget); + return _firstString([ + semantics?['label'], + properties['label'], + properties['text'], + properties['data'], + properties['message'], + properties['tooltip'], + properties['semanticLabel'], + properties['value'], + widget.key?.toString(), + widget.runtimeType.toString(), + ]) ?? + ''; + } + + String? _accessibilityIdentifier( + Element element, + Map? semantics, + ) { + return _firstString([ + semantics?['identifier'], + element.widget.key?.toString(), + ]); + } + + bool _isHidden(Element element) { + final widget = element.widget; + if (widget is Offstage && widget.offstage) { + return true; + } + if (widget is Visibility && !widget.visible) { + return true; + } + if (widget is Opacity && widget.opacity <= 0.0) { + return true; + } + final frame = _frameFor(element.findRenderObject()); + if (frame != null) { + final width = _optionalDouble(frame['width']) ?? 0.0; + final height = _optionalDouble(frame['height']) ?? 0.0; + if (width <= 0.0 || height <= 0.0) { + return true; + } + } + return false; + } + + Map _diagnosticProperties(Object object) { + if (object is! Diagnosticable) { + return {}; + } + final properties = {}; + for (final property in object.toDiagnosticsNode().getProperties()) { + final name = property.name; + if (name == null || name.isEmpty) { + continue; + } + properties[name] = _encodeDiagnosticsValue( + property.value, + property.toDescription(), + ); + } + return properties; + } + + Map? _renderObjectProperties(RenderObject? renderObject) { + if (renderObject == null) { + return null; + } + return { + 'type': renderObject.runtimeType.toString(), + 'attached': renderObject.attached, + 'needsLayout': renderObject.debugNeedsLayout, + 'needsPaint': renderObject.debugNeedsPaint, + 'paintBounds': _rectJson(renderObject.paintBounds), + }; + } +} + +class SimDeckFlutterInspectorFailure implements Exception { + const SimDeckFlutterInspectorFailure(this.code, this.message); + + final int code; + final String message; + + @override + String toString() => message; +} + +class _TraversalContext { + _TraversalContext() + : deadline = DateTime.now().add(const Duration(seconds: 3)); + + final DateTime deadline; + int remainingNodes = 3500; + + bool get expired => remainingNodes <= 0 || DateTime.now().isAfter(deadline); +} + +class _FrameCacheEntry { + const _FrameCacheEntry(this.frame, this.measuredAt); + + final Map frame; + final DateTime measuredAt; +} + +Map _inspectorError(Object error) { + if (error is SimDeckFlutterInspectorFailure) { + return {'code': error.code, 'message': error.message}; + } + return {'code': -32011, 'message': error.toString()}; +} + +Map? _objectMap(Object? value) { + if (value is Map) { + return value; + } + if (value is Map) { + return value.cast(); + } + return null; +} + +String _requiredString(Map params, String key) { + final value = params[key]; + if (value is String && value.isNotEmpty) { + return value; + } + throw SimDeckFlutterInspectorFailure( + -32600, + 'Request params.$key must be a non-empty string.', + ); +} + +double _requiredNumber(Map params, String key) { + final value = _optionalDouble(params[key]); + if (value != null && value.isFinite) { + return value; + } + throw SimDeckFlutterInspectorFailure( + -32600, + 'Request params.$key must be a finite number.', + ); +} + +int? _optionalInt(Object? value) { + if (value is int) { + return value; + } + if (value is num && value.isFinite) { + return value.toInt(); + } + return null; +} + +double? _optionalDouble(Object? value) { + if (value is num && value.isFinite) { + return value.toDouble(); + } + return null; +} + +String _stringValue(Object? value) => value == null ? '' : value.toString(); + +String? _firstString(Iterable values) { + for (final value in values) { + final text = value?.toString().trim(); + if (text != null && text.isNotEmpty && text != 'null') { + return text; + } + } + return null; +} + +Object? _encodeDiagnosticsValue(Object? value, String description) { + if (value == null || value is num || value is bool || value is String) { + return value; + } + if (value is Color) { + final argb = value.toARGB32(); + return { + r'$type': 'Color', + 'value': argb, + 'hex': '#${argb.toRadixString(16).padLeft(8, '0').toUpperCase()}', + }; + } + if (value is EdgeInsetsGeometry || + value is AlignmentGeometry || + value is TextStyle || + value is BoxDecoration) { + return description; + } + return description.isEmpty ? value.toString() : description; +} + +Map _rectJson(Rect rect) { + return { + 'x': _finiteOrZero(rect.left), + 'y': _finiteOrZero(rect.top), + 'width': math.max(0.0, _finiteOrZero(rect.width)), + 'height': math.max(0.0, _finiteOrZero(rect.height)), + }; +} + +bool _frameContains(Map frame, Offset point) { + final x = _optionalDouble(frame['x']); + final y = _optionalDouble(frame['y']); + final width = _optionalDouble(frame['width']); + final height = _optionalDouble(frame['height']); + if (x == null || y == null || width == null || height == null) { + return false; + } + return point.dx >= x && + point.dy >= y && + point.dx <= x + width && + point.dy <= y + height; +} + +double _finiteOrZero(double value) => value.isFinite ? value : 0.0; + +FlutterView? _firstFlutterView() { + final views = WidgetsBinding.instance.platformDispatcher.views; + return views.isEmpty ? null : views.first; +} diff --git a/packages/flutter-inspector/pubspec.yaml b/packages/flutter-inspector/pubspec.yaml new file mode 100644 index 00000000..4a525c75 --- /dev/null +++ b/packages/flutter-inspector/pubspec.yaml @@ -0,0 +1,22 @@ +name: simdeck_flutter_inspector +description: Debug-only Flutter runtime inspector for SimDeck. +version: 0.1.0 +homepage: https://github.com/NativeScript/SimDeck +repository: https://github.com/NativeScript/SimDeck + +environment: + sdk: ">=3.4.0 <4.0.0" + flutter: ">=3.22.0" + +flutter: + plugin: + platforms: + ios: + pluginClass: SimDeckFlutterInspectorPlugin + +dependencies: + flutter: + sdk: flutter + +dev_dependencies: + flutter_lints: ^5.0.0 diff --git a/packages/simdeck-test/dist/index.d.ts b/packages/simdeck-test/dist/index.d.ts index b0a02fff..78875ccf 100644 --- a/packages/simdeck-test/dist/index.d.ts +++ b/packages/simdeck-test/dist/index.d.ts @@ -7,7 +7,13 @@ export type SimDeckLaunchOptions = { videoCodec?: "auto" | "hardware" | "software" | "h264-software"; }; export type QueryOptions = { - source?: "auto" | "nativescript" | "uikit" | "native-ax"; + source?: + | "auto" + | "nativescript" + | "react-native" + | "flutter" + | "uikit" + | "native-ax"; maxDepth?: number; includeHidden?: boolean; }; diff --git a/packages/simdeck-test/src/index.ts b/packages/simdeck-test/src/index.ts index 3e786b34..db2ffd9f 100644 --- a/packages/simdeck-test/src/index.ts +++ b/packages/simdeck-test/src/index.ts @@ -16,7 +16,13 @@ export type SimDeckLaunchOptions = { }; export type QueryOptions = { - source?: "auto" | "nativescript" | "uikit" | "native-ax"; + source?: + | "auto" + | "nativescript" + | "react-native" + | "flutter" + | "uikit" + | "native-ax"; maxDepth?: number; includeHidden?: boolean; }; diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index 153c9b44..4a077ade 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -527,6 +527,7 @@ const CONNECTED_INSPECTOR_HIERARCHY_TIMEOUT: Duration = Duration::from_secs(8); const SOURCE_NATIVE_AX: &str = "native-ax"; const SOURCE_NATIVE_SCRIPT: &str = "nativescript"; const SOURCE_REACT_NATIVE: &str = "react-native"; +const SOURCE_FLUTTER: &str = "flutter"; const SOURCE_SWIFTUI: &str = "swiftui"; const SOURCE_UIKIT: &str = "in-app-inspector"; @@ -2372,6 +2373,7 @@ async fn accessibility_tree_value( AccessibilityHierarchySource::Auto => InAppHierarchySource::Automatic, AccessibilityHierarchySource::NativeScript => InAppHierarchySource::Automatic, AccessibilityHierarchySource::ReactNative => InAppHierarchySource::Automatic, + AccessibilityHierarchySource::Flutter => InAppHierarchySource::Automatic, AccessibilityHierarchySource::SwiftUI => InAppHierarchySource::Automatic, AccessibilityHierarchySource::UIKit => InAppHierarchySource::UIKit, AccessibilityHierarchySource::NativeAX => unreachable!(), @@ -2399,6 +2401,10 @@ async fn accessibility_tree_value( && snapshot_source != Some(SOURCE_REACT_NATIVE) { Some("React Native hierarchy is not published by the app.".to_owned()) + } else if requested_source == AccessibilityHierarchySource::Flutter + && snapshot_source != Some(SOURCE_FLUTTER) + { + Some("Flutter hierarchy is not published by the app.".to_owned()) } else if requested_source == AccessibilityHierarchySource::SwiftUI && snapshot_source != Some(SOURCE_SWIFTUI) { @@ -3289,6 +3295,7 @@ enum AccessibilityHierarchySource { Auto, NativeScript, ReactNative, + Flutter, SwiftUI, UIKit, NativeAX, @@ -3300,6 +3307,7 @@ impl AccessibilityHierarchySource { "" | "auto" => Ok(Self::Auto), "nativescript" | "ns" => Ok(Self::NativeScript), "react-native" | "reactnative" | "rn" => Ok(Self::ReactNative), + "flutter" | "fl" => Ok(Self::Flutter), "swiftui" | "swift-ui" => Ok(Self::SwiftUI), "uikit" | "in-app-inspector" => Ok(Self::UIKit), "ax" | "native-ax" | "native-accessibility" => Ok(Self::NativeAX), @@ -3664,25 +3672,32 @@ fn inspector_session_score(session: &InspectorSession) -> u8 { if session .available_sources .iter() - .any(|source| source == SOURCE_NATIVE_SCRIPT) + .any(|source| source == SOURCE_FLUTTER) { return 1; } if session .available_sources .iter() - .any(|source| source == SOURCE_SWIFTUI) + .any(|source| source == SOURCE_NATIVE_SCRIPT) { return 2; } if session .available_sources .iter() - .any(|source| source == SOURCE_UIKIT) + .any(|source| source == SOURCE_SWIFTUI) { return 3; } - 4 + if session + .available_sources + .iter() + .any(|source| source == SOURCE_UIKIT) + { + return 4; + } + 5 } async fn frontmost_process_identifier(state: &AppState, udid: &str) -> Result, String> { @@ -3787,6 +3802,14 @@ fn inspector_available_sources(info: &Value) -> Vec { if react_native_available { sources.push(SOURCE_REACT_NATIVE.to_owned()); } + let flutter_available = info + .get("flutter") + .and_then(|value| value.get("available")) + .and_then(Value::as_bool) + .unwrap_or(false); + if flutter_available { + sources.push(SOURCE_FLUTTER.to_owned()); + } let app_hierarchy = info.get("appHierarchy"); let app_hierarchy_available = app_hierarchy .and_then(|value| value.get("available")) @@ -3800,6 +3823,7 @@ fn inspector_available_sources(info: &Value) -> Vec { match app_hierarchy_source { SOURCE_NATIVE_SCRIPT => sources.push(SOURCE_NATIVE_SCRIPT.to_owned()), SOURCE_REACT_NATIVE => push_unique_source(&mut sources, SOURCE_REACT_NATIVE), + SOURCE_FLUTTER => push_unique_source(&mut sources, SOURCE_FLUTTER), SOURCE_SWIFTUI => push_unique_source(&mut sources, SOURCE_SWIFTUI), _ => {} } @@ -3808,7 +3832,7 @@ fn inspector_available_sources(info: &Value) -> Vec { .get("uikit") .and_then(|value| value.get("available")) .and_then(Value::as_bool) - .unwrap_or(!react_native_available); + .unwrap_or(!(react_native_available || flutter_available)); if uikit_available { sources.push(SOURCE_UIKIT.to_owned()); } @@ -3837,6 +3861,13 @@ fn available_sources_for_session(session: &InspectorSession) -> Vec { { push_unique_source(&mut sources, SOURCE_REACT_NATIVE); } + if session + .available_sources + .iter() + .any(|source| source == SOURCE_FLUTTER) + { + push_unique_source(&mut sources, SOURCE_FLUTTER); + } if session .available_sources .iter() @@ -3867,7 +3898,7 @@ fn available_sources_for_snapshot(base_sources: &[String], snapshot: &Value) -> let Some(source) = snapshot.get("source").and_then(Value::as_str) else { return sources; }; - if source == SOURCE_REACT_NATIVE { + if source == SOURCE_REACT_NATIVE || source == SOURCE_FLUTTER { sources.retain(|candidate| candidate != SOURCE_UIKIT); } if framework_source(source) && !sources.iter().any(|value| value == source) { @@ -3904,7 +3935,10 @@ async fn merge_connected_sources_for_pid( } } } - if sources.iter().any(|source| source == SOURCE_REACT_NATIVE) { + if sources + .iter() + .any(|source| source == SOURCE_REACT_NATIVE || source == SOURCE_FLUTTER) + { sources.retain(|source| source != SOURCE_UIKIT); } } @@ -3919,7 +3953,10 @@ fn root_process_identifier(snapshot: &Value) -> Option { } fn framework_source(source: &str) -> bool { - source == SOURCE_NATIVE_SCRIPT || source == SOURCE_REACT_NATIVE || source == SOURCE_SWIFTUI + source == SOURCE_NATIVE_SCRIPT + || source == SOURCE_REACT_NATIVE + || source == SOURCE_FLUTTER + || source == SOURCE_SWIFTUI } fn attach_available_sources(snapshot: Value, available_sources: &[String]) -> Value { @@ -4554,8 +4591,8 @@ mod tests { parse_lsof_tcp_listener, resolved_stream_quality_limits, split_filter_values, stream_quality_profile, suppress_native_ax_translation_error, tap_point_from_snapshot, trim_tree_depth, AccessibilityHierarchySource, ElementSelectorPayload, InspectorSession, - InspectorSessionTransport, StreamQualityLimits, StreamQualityPayload, SOURCE_NATIVE_AX, - SOURCE_NATIVE_SCRIPT, SOURCE_REACT_NATIVE, SOURCE_SWIFTUI, SOURCE_UIKIT, + InspectorSessionTransport, StreamQualityLimits, StreamQualityPayload, SOURCE_FLUTTER, + SOURCE_NATIVE_AX, SOURCE_NATIVE_SCRIPT, SOURCE_REACT_NATIVE, SOURCE_SWIFTUI, SOURCE_UIKIT, }; use crate::inspector::PublishedInspector; use crate::transport::packet::FramePacket; @@ -4678,6 +4715,7 @@ mod tests { fn inspector_source_detection_prefers_framework_specific_sources() { let sources = inspector_available_sources(&json!({ "reactNative": { "available": true }, + "flutter": { "available": true }, "appHierarchy": { "available": true, "source": "nativescript" }, "uikit": { "available": true } })); @@ -4686,6 +4724,7 @@ mod tests { sources, vec![ SOURCE_REACT_NATIVE.to_owned(), + SOURCE_FLUTTER.to_owned(), SOURCE_NATIVE_SCRIPT.to_owned(), SOURCE_UIKIT.to_owned() ] @@ -4768,6 +4807,19 @@ mod tests { ); } + #[test] + fn available_sources_for_flutter_snapshot_removes_uikit_fallback() { + let sources = available_sources_for_snapshot( + &[SOURCE_UIKIT.to_owned(), SOURCE_NATIVE_AX.to_owned()], + &json!({ "source": SOURCE_FLUTTER }), + ); + + assert_eq!( + sources, + vec![SOURCE_FLUTTER.to_owned(), SOURCE_NATIVE_AX.to_owned()] + ); + } + #[test] fn native_ax_expected_translation_failures_are_suppressed() { assert_eq!( @@ -4844,6 +4896,10 @@ mod tests { AccessibilityHierarchySource::parse(Some("rn")).unwrap(), AccessibilityHierarchySource::ReactNative )); + assert!(matches!( + AccessibilityHierarchySource::parse(Some("flutter")).unwrap(), + AccessibilityHierarchySource::Flutter + )); assert!(matches!( AccessibilityHierarchySource::parse(Some("swift-ui")).unwrap(), AccessibilityHierarchySource::SwiftUI diff --git a/server/src/main.rs b/server/src/main.rs index 291c0e96..6b0f6b99 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -632,6 +632,7 @@ enum DescribeUiSource { Auto, Nativescript, ReactNative, + Flutter, Uikit, NativeAx, } @@ -3555,6 +3556,7 @@ impl DescribeUiSource { Self::Auto => "auto", Self::Nativescript => "nativescript", Self::ReactNative => "react-native", + Self::Flutter => "flutter", Self::Uikit => "uikit", Self::NativeAx => "native-ax", } diff --git a/skills/simdeck/SKILL.md b/skills/simdeck/SKILL.md index 86ffe768..c37ea448 100644 --- a/skills/simdeck/SKILL.md +++ b/skills/simdeck/SKILL.md @@ -67,12 +67,14 @@ simdeck describe --format compact-json simdeck describe --point 120,240 simdeck describe --source auto simdeck describe --source nativescript +simdeck describe --source react-native +simdeck describe --source flutter simdeck describe --source uikit simdeck describe --source native-ax simdeck describe --direct ``` -Use `--source auto` with the project daemon. Use `--direct` or `--source native-ax` for the private CoreSimulator accessibility bridge. NativeScript inspector runtime can add richer hierarchy data. +Use `--source auto` with the project daemon. Use `--direct` or `--source native-ax` for the private CoreSimulator accessibility bridge. NativeScript, React Native, and Flutter inspector runtimes can add richer hierarchy data. Prefer selectors, coordinates only when needed. Selector taps go through the daemon and wait for the element server-side. From aee1014401a17bfed406563b68008938d6116e3b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 9 May 2026 15:17:07 -0400 Subject: [PATCH 2/4] Fix Flutter inspector hierarchy refresh --- client/src/app/AppShell.tsx | 11 +- .../ios/simdeck_flutter_inspector.podspec | 2 +- .../lib/simdeck_flutter_inspector.dart | 134 ++++++++++++++++-- 3 files changed, 132 insertions(+), 15 deletions(-) diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index f475fccd..d088d9bc 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -106,8 +106,10 @@ import { const ACCESSIBILITY_REFRESH_MS = 1500; const REACT_NATIVE_ACCESSIBILITY_REFRESH_MS = 500; +const FLUTTER_ACCESSIBILITY_REFRESH_MS = 1000; const DEFAULT_ACCESSIBILITY_MAX_DEPTH = 10; const LOGICAL_INSPECTOR_MAX_DEPTH = 80; +const FLUTTER_INSPECTOR_MAX_DEPTH = 48; const AUTH_REQUIRED_MESSAGE = "SimDeck API access token is required."; const LOCAL_STREAM_DEFAULTS: StreamConfig = { encoder: "auto", @@ -913,7 +915,9 @@ export function AppShell({ maxDepth: accessibilityPreferredSource === "native-ax" ? DEFAULT_ACCESSIBILITY_MAX_DEPTH - : LOGICAL_INSPECTOR_MAX_DEPTH, + : accessibilityPreferredSource === "flutter" + ? FLUTTER_INSPECTOR_MAX_DEPTH + : LOGICAL_INSPECTOR_MAX_DEPTH, }, ); if (accessibilityRequestIdRef.current !== requestId) { @@ -993,7 +997,10 @@ export function AppShell({ accessibilityPreferredSource === "react-native" || accessibilitySource === "react-native" ? REACT_NATIVE_ACCESSIBILITY_REFRESH_MS - : ACCESSIBILITY_REFRESH_MS; + : accessibilityPreferredSource === "flutter" || + accessibilitySource === "flutter" + ? FLUTTER_ACCESSIBILITY_REFRESH_MS + : ACCESSIBILITY_REFRESH_MS; const interval = window.setInterval(() => { void loadAccessibilityTree(); }, refreshMs); diff --git a/packages/flutter-inspector/ios/simdeck_flutter_inspector.podspec b/packages/flutter-inspector/ios/simdeck_flutter_inspector.podspec index bb573cd0..5b97ec5f 100644 --- a/packages/flutter-inspector/ios/simdeck_flutter_inspector.podspec +++ b/packages/flutter-inspector/ios/simdeck_flutter_inspector.podspec @@ -8,7 +8,7 @@ Pod::Spec.new do |s| s.author = { 'SimDeck' => 'support@nativescript.org' } s.source = { :path => '.' } s.source_files = 'Classes/**/*' - s.dependency = 'Flutter' + s.dependency 'Flutter' s.platform = :ios, '12.0' s.swift_version = '5.0' end diff --git a/packages/flutter-inspector/lib/simdeck_flutter_inspector.dart b/packages/flutter-inspector/lib/simdeck_flutter_inspector.dart index b12d5dbb..3e8e1520 100644 --- a/packages/flutter-inspector/lib/simdeck_flutter_inspector.dart +++ b/packages/flutter-inspector/lib/simdeck_flutter_inspector.dart @@ -11,6 +11,8 @@ import 'package:flutter/widgets.dart'; const String _protocolVersion = '0.1'; const MethodChannel _channel = MethodChannel('simdeck_flutter_inspector'); +const int _defaultHierarchyMaxDepth = 48; +const int _maxHierarchyDepth = 64; SimDeckFlutterInspector? _sharedInspector; @@ -66,6 +68,9 @@ class SimDeckFlutterInspector { final SimDeckFlutterInspectorOptions options; final Expando _ids = Expando('simdeckFlutterInspectorId'); + final Expando _sourceLocations = Expando( + 'simdeckFlutterSourceLocation', + ); final Map _objects = {}; final Map _frameCache = {}; int _nextObjectId = 1; @@ -360,7 +365,7 @@ class SimDeckFlutterInspector { final node = _elementNode( root, includeHidden: params['includeHidden'] == true, - maxDepth: _optionalInt(params['maxDepth']), + maxDepth: _hierarchyMaxDepth(params['maxDepth']), depth: 0, context: context, ); @@ -387,7 +392,7 @@ class SimDeckFlutterInspector { final node = _elementNode( element, includeHidden: true, - maxDepth: _optionalInt(params['maxDepth']), + maxDepth: _hierarchyMaxDepth(params['maxDepth']), depth: 0, context: _TraversalContext(), ); @@ -543,14 +548,23 @@ class SimDeckFlutterInspector { return null; } + final type = element.widget.runtimeType.toString(); + final renderObject = element.findRenderObject(); + final frame = _frameFor(renderObject); + final semantics = _semanticsInfo(renderObject); + final sourceLocation = _shouldReadSourceLocation(element, semantics) + ? _sourceLocation(element) + : null; + final transparent = _isTransparentWrapper(element, semantics); + final childDepth = transparent ? depth : depth + 1; final children = >[]; - if (maxDepth == null || depth < maxDepth) { + if (maxDepth == null || transparent || depth < maxDepth) { element.visitChildren((child) { final node = _elementNode( child, includeHidden: includeHidden, maxDepth: maxDepth, - depth: depth + 1, + depth: childDepth, context: context, ); if (node != null) { @@ -559,17 +573,17 @@ class SimDeckFlutterInspector { }); } + if (transparent && children.length == 1) { + return children.single; + } + final id = _objectId(element); - final renderObject = element.findRenderObject(); - final frame = _frameFor(renderObject); - final semantics = _semanticsInfo(renderObject); final title = _nodeTitle(element, semantics); - final sourceLocation = _sourceLocation(element); return { 'id': id, 'inspectorId': id, - 'type': element.widget.runtimeType.toString(), - 'displayName': element.widget.runtimeType.toString(), + 'type': type, + 'displayName': type, 'title': title, 'source': 'flutter', 'sourceLocation': sourceLocation, @@ -584,7 +598,7 @@ class SimDeckFlutterInspector { 'isHidden': hidden, 'custom_actions': semantics?['actions'], 'flutter': { - 'widgetType': element.widget.runtimeType.toString(), + 'widgetType': type, 'elementType': element.runtimeType.toString(), 'stateType': element is StatefulElement ? element.state.runtimeType.toString() @@ -732,6 +746,14 @@ class SimDeckFlutterInspector { return actions.toList()..sort(); } + int? _hierarchyMaxDepth(Object? value) { + final requested = _optionalInt(value) ?? _defaultHierarchyMaxDepth; + if (requested < 0) { + return 0; + } + return math.min(requested, _maxHierarchyDepth); + } + String _objectId(Element element) { final existing = _ids[element]; if (existing != null) { @@ -835,6 +857,12 @@ class SimDeckFlutterInspector { if (!WidgetInspectorService.instance.isWidgetCreationTracked()) { return null; } + final cached = _sourceLocations[element]; + if (cached != null) { + return identical(cached, _noSourceLocation) + ? null + : cached as Map; + } try { final json = element.toDiagnosticsNode().toJsonMap( InspectorSerializationDelegate( @@ -844,20 +872,37 @@ class SimDeckFlutterInspector { ); final location = _objectMap(json['creationLocation']); if (location == null) { + _sourceLocations[element] = _noSourceLocation; return null; } - return { + final sourceLocation = { 'file': location['file'], 'line': location['line'], 'column': location['column'], 'name': location['name'], 'kind': 'flutter', }; + _sourceLocations[element] = sourceLocation; + return sourceLocation; } catch (_) { + _sourceLocations[element] = _noSourceLocation; return null; } } + bool _shouldReadSourceLocation( + Element element, + Map? semantics, + ) { + if (!WidgetInspectorService.instance.isWidgetCreationTracked()) { + return false; + } + final type = element.widget.runtimeType.toString(); + return element.widget.key != null || + _hasSemanticContent(semantics) || + !_isFrameworkWrapperType(type); + } + String _nodeTitle(Element element, Map? semantics) { final widget = element.widget; final properties = _diagnosticProperties(widget); @@ -908,6 +953,13 @@ class SimDeckFlutterInspector { return false; } + bool _isTransparentWrapper(Element element, Map? semantics) { + if (element.widget.key != null || _hasSemanticContent(semantics)) { + return false; + } + return _isFrameworkWrapperType(element.widget.runtimeType.toString()); + } + Map _diagnosticProperties(Object object) { if (object is! Diagnosticable) { return {}; @@ -967,6 +1019,64 @@ class _FrameCacheEntry { final DateTime measuredAt; } +final Object _noSourceLocation = Object(); + +const Set _frameworkWrapperTypes = { + 'Actions', + '_ActionsScope', + 'AnimatedTheme', + 'Builder', + 'CheckedModeBanner', + 'DefaultSelectionStyle', + 'DefaultTextEditingShortcuts', + 'Directionality', + 'Focus', + '_FocusInheritedScope', + '_FocusScopeWithExternalFocusNode', + 'FocusTraversalGroup', + 'HeroControllerScope', + 'Localizations', + '_LocalizationsScope', + 'MaterialApp', + 'MediaQuery', + '_MediaQueryFromView', + 'NotificationListener', + 'Overlay', + 'RawView', + '_RawViewInternal', + 'RootWidget', + 'ScrollConfiguration', + 'Semantics', + 'Shortcuts', + '_ShortcutsMarker', + 'Theme', + 'Title', + 'View', + '_ViewScope', + 'WidgetsApp', + '_WidgetsAppState', + '_PipelineOwnerScope', + '_InheritedTheme', +}; + +bool _isFrameworkWrapperType(String type) => + type.startsWith('_') || _frameworkWrapperTypes.contains(type); + +bool _hasSemanticContent(Map? semantics) { + if (semantics == null) { + return false; + } + final label = _firstString([ + semantics['identifier'], + semantics['label'], + semantics['value'], + semantics['hint'], + semantics['tooltip'], + ]); + final actions = semantics['actions']; + return label != null || (actions is List && actions.isNotEmpty); +} + Map _inspectorError(Object error) { if (error is SimDeckFlutterInspectorFailure) { return {'code': error.code, 'message': error.message}; From e29605a3429405fad8616e678f6ec61de28d816c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 9 May 2026 15:28:55 -0400 Subject: [PATCH 3/4] Flatten Flutter inspector wrapper nodes --- .../accessibility/AccessibilityInspector.tsx | 4 +- .../accessibility/accessibilityTree.test.ts | 78 +++++++ .../accessibility/accessibilityTree.ts | 193 +++++++++++++++++- docs/inspector/index.md | 14 +- .../lib/simdeck_flutter_inspector.dart | 190 +++++++++++++++-- 5 files changed, 445 insertions(+), 34 deletions(-) diff --git a/client/src/features/accessibility/AccessibilityInspector.tsx b/client/src/features/accessibility/AccessibilityInspector.tsx index f24f5e63..a36ecd2b 100644 --- a/client/src/features/accessibility/AccessibilityInspector.tsx +++ b/client/src/features/accessibility/AccessibilityInspector.tsx @@ -146,7 +146,7 @@ export function AccessibilityInspector({ const tree = buildAccessibilityTree(roots); const storedExpandedIds = - source === "react-native" + source === "react-native" || source === "flutter" ? [] : readStoredStringArray(expandedStorageKey(udid)); setExpandedIds( @@ -154,7 +154,7 @@ export function AccessibilityInspector({ ? new Set(storedExpandedIds) : defaultExpandedAccessibilityIds( tree, - source === "react-native" ? 2 : 10, + source === "react-native" || source === "flutter" ? 2 : 10, ), ); expandedInitializedKeyRef.current = expansionKey; diff --git a/client/src/features/accessibility/accessibilityTree.test.ts b/client/src/features/accessibility/accessibilityTree.test.ts index 025d5fdf..48fd9a74 100644 --- a/client/src/features/accessibility/accessibilityTree.test.ts +++ b/client/src/features/accessibility/accessibilityTree.test.ts @@ -114,6 +114,52 @@ describe("buildAccessibilityTree", () => { expect(tree[0].node.type).toBe("Text"); expect(tree[0].chain.map((node) => node.type)).toEqual(["Wrap", "RCTView"]); }); + + it("compacts one-child Flutter layout wrappers but keeps app components", () => { + const roots: AccessibilityNode[] = [ + { + source: "flutter", + type: "InspectorDemoHome", + title: "InspectorDemoHome", + sourceLocation: { file: "/tmp/demo/lib/main.dart" }, + children: [ + { + source: "flutter", + type: "Padding", + title: "Padding", + sourceLocation: { file: "/tmp/demo/lib/main.dart" }, + flutter: { transparent: true }, + children: [ + { + source: "flutter", + type: "Center", + title: "Center", + sourceLocation: { file: "/tmp/demo/lib/main.dart" }, + flutter: { transparent: true }, + children: [ + { + source: "flutter", + type: "Text", + title: "Continue", + AXLabel: "Continue", + }, + ], + }, + ], + }, + ], + }, + ]; + + const tree = buildAccessibilityTree(roots); + + expect(tree[0].node.type).toBe("InspectorDemoHome"); + expect(tree[0].children[0].node.type).toBe("Text"); + expect(tree[0].children[0].chain.map((node) => node.type)).toEqual([ + "Padding", + "Center", + ]); + }); }); describe("findAccessibilityItemAtPoint", () => { @@ -247,4 +293,36 @@ describe("findAccessibilityItemAtPoint", () => { expect(item?.node.type).toBe("Label"); expect(item?.id).toBe("0.0"); }); + + it("ignores transparent Flutter overlays that cover selectable content", () => { + const roots: AccessibilityNode[] = [ + { + source: "flutter", + type: "Stack", + frame: { x: 0, y: 0, width: 400, height: 800 }, + flutter: { transparent: true }, + children: [ + { + source: "flutter", + type: "FilledButton", + title: "Continue", + AXLabel: "Continue", + frame: { x: 100, y: 300, width: 200, height: 60 }, + }, + { + source: "flutter", + type: "Listener", + title: "Listener", + frame: { x: 0, y: 0, width: 400, height: 800 }, + flutter: { transparent: true }, + }, + ], + }, + ]; + + const item = findAccessibilityItemAtPoint(roots, { x: 0.5, y: 0.4125 }); + + expect(item?.node.type).toBe("FilledButton"); + expect(item?.id).toBe("0.0"); + }); }); diff --git a/client/src/features/accessibility/accessibilityTree.ts b/client/src/features/accessibility/accessibilityTree.ts index 7ed4a039..d28edc40 100644 --- a/client/src/features/accessibility/accessibilityTree.ts +++ b/client/src/features/accessibility/accessibilityTree.ts @@ -131,7 +131,7 @@ function buildItem( id: string, depth: number, ): AccessibilityTreeItem { - const compacted = compactReactNativeChain(node, id); + const compacted = compactInspectorChain(node, id); return { chain: compacted.chain, children: (compacted.node.children ?? []).map((child, index) => @@ -143,7 +143,7 @@ function buildItem( }; } -function compactReactNativeChain( +function compactInspectorChain( node: AccessibilityNode, id: string, ): { @@ -155,9 +155,9 @@ function compactReactNativeChain( let current = node; let currentId = id; - while (canCompactReactNativeNode(current)) { + while (canCompactInspectorNode(current)) { const child = current.children?.[0]; - if (!child || child.source !== "react-native") { + if (!child || child.source !== current.source) { break; } chain.push(current); @@ -168,16 +168,41 @@ function compactReactNativeChain( return { chain, id: currentId, node: current }; } -function canCompactReactNativeNode(node: AccessibilityNode): boolean { - if (node.source !== "react-native" || node.children?.length !== 1) { +function canCompactInspectorNode(node: AccessibilityNode): boolean { + if (node.children?.length !== 1) { return false; } + if (node.source === "react-native") { + return canCompactReactNativeNode(node); + } + if (node.source === "flutter") { + return canCompactFlutterNode(node); + } + return false; +} + +function canCompactReactNativeNode(node: AccessibilityNode): boolean { if (primarySourceLocationFile(node) || isRouteDisplayName(node)) { return false; } return !hasMeaningfulNodeContent(node); } +function canCompactFlutterNode(node: AccessibilityNode): boolean { + if (flutterBoolean(node, "transparent")) { + return true; + } + if (hasMeaningfulNodeContent(node)) { + return false; + } + const type = cleanText(node.type); + if (!isFlutterTransparentContainerType(type)) { + return false; + } + const sourceLocation = primarySourceLocationFile(node); + return !sourceLocation || isFlutterFrameworkContainerType(type); +} + function findContainingItem( items: AccessibilityTreeItem[], point: { x: number; y: number }, @@ -200,6 +225,16 @@ function findContainingItem( function isTransparentHitTestBlocker(item: AccessibilityTreeItem): boolean { const node = item.node; + if (node.source === "flutter") { + if (flutterBoolean(node, "transparent")) { + return true; + } + return ( + !hasMeaningfulNodeContent(node) && + isFlutterTransparentContainerType(cleanText(node.type)) + ); + } + if (node.source !== "in-app-inspector" || node.nativeScript) { return false; } @@ -235,6 +270,152 @@ function unqualifiedClassName(value: string | null): string | null { return value?.split(".").pop()?.trim() || value; } +function flutterBoolean(node: AccessibilityNode, key: string): boolean { + return node.flutter?.[key] === true; +} + +function isFlutterTransparentContainerType(value: string | null): boolean { + const type = unqualifiedClassName(value); + return Boolean( + type && + (type.startsWith("_") || + flutterTransparentContainerTypes.has(type) || + flutterFrameworkContainerTypes.has(type)), + ); +} + +function isFlutterFrameworkContainerType(value: string | null): boolean { + const type = unqualifiedClassName(value); + return Boolean( + type && + (type.startsWith("_") || + flutterFrameworkContainerTypes.has(type) || + flutterTransparentContainerTypes.has(type)), + ); +} + +const flutterTransparentContainerTypes = new Set([ + "AbsorbPointer", + "Actions", + "Align", + "AnimatedBuilder", + "AnimatedContainer", + "AnimatedDefaultTextStyle", + "AnimatedOpacity", + "AnimatedPadding", + "AnimatedPhysicalModel", + "AnimatedPositioned", + "AnimatedTheme", + "AspectRatio", + "AutomaticKeepAlive", + "BlockSemantics", + "Builder", + "Center", + "ClipPath", + "ClipRRect", + "ClipRect", + "Column", + "ConstrainedBox", + "Container", + "CustomMultiChildLayout", + "CustomPaint", + "CustomSingleChildLayout", + "DecoratedBox", + "DefaultSelectionStyle", + "DefaultTextStyle", + "ExcludeSemantics", + "Directionality", + "Expanded", + "Flexible", + "Focus", + "FocusScope", + "FocusTraversalGroup", + "FractionalTranslation", + "GestureDetector", + "IconButtonTheme", + "IgnorePointer", + "Ink", + "IndexedSemantics", + "InputDecorator", + "IntrinsicHeight", + "IntrinsicWidth", + "KeepAlive", + "KeyedSubtree", + "LayoutId", + "LayoutBuilder", + "LimitedBox", + "Listener", + "ListenableBuilder", + "ListView", + "Material", + "MatrixTransition", + "MediaQuery", + "MouseRegion", + "NotificationListener", + "Offstage", + "Opacity", + "OverflowBox", + "Padding", + "PhysicalModel", + "PhysicalShape", + "Positioned", + "PositionedDirectional", + "PrimaryScrollController", + "RawGestureDetector", + "RepaintBoundary", + "RestorationScope", + "RootRestorationScope", + "Row", + "SafeArea", + "Scaffold", + "ScrollNotificationObserver", + "Scrollable", + "Semantics", + "SharedAppData", + "SizeChangedLayoutNotifier", + "SizedBox", + "SliverList", + "SliverPadding", + "Stack", + "TapRegionSurface", + "TextFieldTapRegion", + "TextSelectionGestureDetector", + "Theme", + "TickerMode", + "Transform", + "UnmanagedRestorationScope", + "UndoHistory", + "ValueListenableBuilder", + "Viewport", +]); + +const flutterFrameworkContainerTypes = new Set([ + "CheckedModeBanner", + "CupertinoPageTransition", + "CupertinoTheme", + "DecoratedBoxTransition", + "DefaultTextEditingShortcuts", + "HeroControllerScope", + "IconTheme", + "InheritedCupertinoTheme", + "ShortcutRegistrar", + "Localizations", + "MaterialApp", + "ModalBarrier", + "Navigator", + "Overlay", + "PageStorage", + "RawView", + "RootWidget", + "ScaffoldMessenger", + "ScrollConfiguration", + "Shortcuts", + "SlideTransition", + "Title", + "View", + "WidgetsApp", +]); + function hasMeaningfulNodeContent(node: AccessibilityNode): boolean { const generatedNames = new Set( [node.type, node.className, node.role, "UIView", "UIKit View"] diff --git a/docs/inspector/index.md b/docs/inspector/index.md index 2dfcea58..9a946447 100644 --- a/docs/inspector/index.md +++ b/docs/inspector/index.md @@ -2,14 +2,14 @@ SimDeck blends three different ways to inspect what an iOS app is rendering: -| Source | Coverage | When to use it | -| ------------------------ | ------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------- | -| **Native AX** | Any simulator app via the Simulator accessibility stack. | Default fallback. | -| **Swift in-app agent** | Apps that link `SimDeckInspectorAgent` in DEBUG. | Best for native iOS apps you control. | -| **NativeScript runtime** | NativeScript apps that import `@nativescript/simdeck-inspector`. | Best for NativeScript apps — exposes the logical view tree, not just UIKit. | -| **React Native runtime** | React Native apps that import `react-native-simdeck`. | Best for React Native apps — exposes components and Metro source locations. | +| Source | Coverage | When to use it | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------- | +| **Native AX** | Any simulator app via the Simulator accessibility stack. | Default fallback. | +| **Swift in-app agent** | Apps that link `SimDeckInspectorAgent` in DEBUG. | Best for native iOS apps you control. | +| **NativeScript runtime** | NativeScript apps that import `@nativescript/simdeck-inspector`. | Best for NativeScript apps — exposes the logical view tree, not just UIKit. | +| **React Native runtime** | React Native apps that import `react-native-simdeck`. | Best for React Native apps — exposes components and Metro source locations. | | **Flutter runtime** | Flutter apps that import `simdeck_flutter_inspector`. | Best for Flutter apps — exposes widgets, render frames, semantics actions, and creation locations. | -| **DevTools panel** | Safari/WebKit targets, Metro React Native targets, Chrome Inspector targets, and SimDeck app runtime inspector sessions. | Best when you want a familiar browser inspector for app runtimes or web content. | +| **DevTools panel** | Safari/WebKit targets, Metro React Native targets, Chrome Inspector targets, and SimDeck app runtime inspector sessions. | Best when you want a familiar browser inspector for app runtimes or web content. | The HTTP API picks the most specific source available, falls back to the next one when something goes wrong, and tells the client which sources were available so the UI can offer a switch. diff --git a/packages/flutter-inspector/lib/simdeck_flutter_inspector.dart b/packages/flutter-inspector/lib/simdeck_flutter_inspector.dart index 3e8e1520..487e2601 100644 --- a/packages/flutter-inspector/lib/simdeck_flutter_inspector.dart +++ b/packages/flutter-inspector/lib/simdeck_flutter_inspector.dart @@ -64,7 +64,7 @@ class SimDeckFlutterInspectorOptions { class SimDeckFlutterInspector { SimDeckFlutterInspector([SimDeckFlutterInspectorOptions? options]) - : options = options ?? const SimDeckFlutterInspectorOptions(); + : options = options ?? const SimDeckFlutterInspectorOptions(); final SimDeckFlutterInspectorOptions options; final Expando _ids = Expando('simdeckFlutterInspectorId'); @@ -320,8 +320,8 @@ class SimDeckFlutterInspector { }, 'flutter': { 'available': true, - 'widgetCreationTracked': WidgetInspectorService.instance - .isWidgetCreationTracked(), + 'widgetCreationTracked': + WidgetInspectorService.instance.isWidgetCreationTracked(), }, 'uikit': {'available': false, 'propertyEditing': false}, }; @@ -332,9 +332,8 @@ class SimDeckFlutterInspector { return _metadata!; } final view = _firstFlutterView(); - final logicalSize = view == null - ? Size.zero - : view.physicalSize / view.devicePixelRatio; + final logicalSize = + view == null ? Size.zero : view.physicalSize / view.devicePixelRatio; final fallback = { 'processIdentifier': io.pid, 'bundleIdentifier': io.Platform.resolvedExecutable, @@ -556,6 +555,7 @@ class SimDeckFlutterInspector { ? _sourceLocation(element) : null; final transparent = _isTransparentWrapper(element, semantics); + final transparentHitTarget = _isTransparentHitTarget(element, semantics); final childDepth = transparent ? depth : depth + 1; final children = >[]; if (maxDepth == null || transparent || depth < maxDepth) { @@ -605,6 +605,8 @@ class SimDeckFlutterInspector { : null, 'key': element.widget.key?.toString(), 'depth': element.depth, + 'transparent': transparentHitTarget, + 'compacted': transparent, }, 'semantics': semantics, 'children': children, @@ -630,7 +632,10 @@ class SimDeckFlutterInspector { if (node == null || frame == null || !_frameContains(frame, point)) { return; } - chain.add(node); + final flutter = _objectMap(node['flutter']); + if (flutter?['transparent'] != true) { + chain.add(node); + } element.visitChildren(visit); } @@ -783,8 +788,7 @@ class SimDeckFlutterInspector { 'protocolVersion': _protocolVersion, 'processIdentifier': metadata['processIdentifier'] ?? io.pid, 'bundleIdentifier': metadata['bundleIdentifier'], - 'displayScale': - metadata['displayScale'] ?? + 'displayScale': metadata['displayScale'] ?? _firstFlutterView()?.devicePixelRatio ?? 1.0, 'coordinateSpace': 'screen-points', @@ -865,11 +869,11 @@ class SimDeckFlutterInspector { } try { final json = element.toDiagnosticsNode().toJsonMap( - InspectorSerializationDelegate( - service: WidgetInspectorService.instance, - subtreeDepth: 0, - ), - ); + InspectorSerializationDelegate( + service: WidgetInspectorService.instance, + subtreeDepth: 0, + ), + ); final location = _objectMap(json['creationLocation']); if (location == null) { _sourceLocations[element] = _noSourceLocation; @@ -954,10 +958,46 @@ class SimDeckFlutterInspector { } bool _isTransparentWrapper(Element element, Map? semantics) { - if (element.widget.key != null || _hasSemanticContent(semantics)) { + final type = element.widget.runtimeType.toString(); + if (_isTransparentContainerType(type)) { + return !_isSemanticContentWidgetType(type, semantics); + } + if (_hasSemanticContent(semantics)) { + return false; + } + if (element.widget.key != null) { return false; } - return _isFrameworkWrapperType(element.widget.runtimeType.toString()); + return false; + } + + bool _isTransparentHitTarget( + Element element, + Map? semantics, + ) { + final type = element.widget.runtimeType.toString(); + if (_isTransparentContainerType(type)) { + return !_isSemanticContentWidgetType(type, semantics); + } + return !_hasSemanticContent(semantics) && _isTransparentContainerType(type); + } + + bool _isTransparentContainerType(String type) { + return _isFrameworkWrapperType(type) || + _isFlutterPassThroughWidgetType(type); + } + + bool _isSemanticContentWidgetType( + String type, + Map? semantics, + ) { + final baseType = _baseWidgetType(type); + if (baseType == 'Semantics') { + return _hasSemanticContent(semantics); + } + return baseType == 'Text' || + baseType == 'RichText' || + baseType == 'EditableText'; } Map _diagnosticProperties(Object object) { @@ -1004,7 +1044,7 @@ class SimDeckFlutterInspectorFailure implements Exception { class _TraversalContext { _TraversalContext() - : deadline = DateTime.now().add(const Duration(seconds: 3)); + : deadline = DateTime.now().add(const Duration(seconds: 3)); final DateTime deadline; int remainingNodes = 3500; @@ -1027,6 +1067,9 @@ const Set _frameworkWrapperTypes = { 'AnimatedTheme', 'Builder', 'CheckedModeBanner', + 'CupertinoPageTransition', + 'CupertinoTheme', + 'DecoratedBoxTransition', 'DefaultSelectionStyle', 'DefaultTextEditingShortcuts', 'Directionality', @@ -1035,20 +1078,28 @@ const Set _frameworkWrapperTypes = { '_FocusScopeWithExternalFocusNode', 'FocusTraversalGroup', 'HeroControllerScope', + 'IconTheme', + 'InheritedCupertinoTheme', 'Localizations', '_LocalizationsScope', 'MaterialApp', 'MediaQuery', '_MediaQueryFromView', + 'ModalBarrier', + 'Navigator', 'NotificationListener', 'Overlay', + 'PageStorage', 'RawView', '_RawViewInternal', 'RootWidget', + 'ScaffoldMessenger', 'ScrollConfiguration', + 'ShortcutRegistrar', 'Semantics', 'Shortcuts', '_ShortcutsMarker', + 'SlideTransition', 'Theme', 'Title', 'View', @@ -1059,8 +1110,109 @@ const Set _frameworkWrapperTypes = { '_InheritedTheme', }; -bool _isFrameworkWrapperType(String type) => - type.startsWith('_') || _frameworkWrapperTypes.contains(type); +const Set _flutterPassThroughWidgetTypes = { + 'AbsorbPointer', + 'Align', + 'AnimatedBuilder', + 'AnimatedDefaultTextStyle', + 'AnimatedOpacity', + 'AnimatedPadding', + 'AnimatedPhysicalModel', + 'AnimatedPositioned', + 'AnimatedContainer', + 'AspectRatio', + 'AutomaticKeepAlive', + 'BlockSemantics', + 'Center', + 'ClipPath', + 'ClipRRect', + 'ClipRect', + 'Column', + 'ConstrainedBox', + 'Container', + 'CustomMultiChildLayout', + 'CustomPaint', + 'CustomSingleChildLayout', + 'DecoratedBox', + 'DefaultTextStyle', + 'ExcludeSemantics', + 'Expanded', + 'Flexible', + 'FocusScope', + 'FractionalTranslation', + 'GestureDetector', + 'IgnorePointer', + 'IconButtonTheme', + 'Ink', + 'IndexedSemantics', + 'InputDecorator', + 'IntrinsicHeight', + 'IntrinsicWidth', + 'KeepAlive', + 'KeyedSubtree', + 'LayoutId', + 'LayoutBuilder', + 'LimitedBox', + 'Listener', + 'ListenableBuilder', + 'ListView', + 'Material', + 'MatrixTransition', + 'MouseRegion', + 'NotificationListener', + 'Offstage', + 'Opacity', + 'OverflowBox', + 'Padding', + 'PhysicalModel', + 'PhysicalShape', + 'Positioned', + 'PositionedDirectional', + 'PrimaryScrollController', + 'RawGestureDetector', + 'RepaintBoundary', + 'RestorationScope', + 'RootRestorationScope', + 'Row', + 'SafeArea', + 'Scaffold', + 'ScrollNotificationObserver', + 'Scrollable', + 'SharedAppData', + 'SizeChangedLayoutNotifier', + 'SizedBox', + 'SliverList', + 'SliverPadding', + 'Stack', + 'TapRegionSurface', + 'TextFieldTapRegion', + 'TextSelectionGestureDetector', + 'TickerMode', + 'Transform', + 'UndoHistory', + 'UnmanagedRestorationScope', + 'ValueListenableBuilder', + 'Viewport', +}; + +bool _isFrameworkWrapperType(String type) { + final baseType = _baseWidgetType(type); + return type.startsWith('_') || + baseType.startsWith('_') || + _frameworkWrapperTypes.contains(type) || + _frameworkWrapperTypes.contains(baseType); +} + +bool _isFlutterPassThroughWidgetType(String type) { + final baseType = _baseWidgetType(type); + return _flutterPassThroughWidgetTypes.contains(type) || + _flutterPassThroughWidgetTypes.contains(baseType); +} + +String _baseWidgetType(String type) { + final genericStart = type.indexOf('<'); + return genericStart < 0 ? type : type.substring(0, genericStart); +} bool _hasSemanticContent(Map? semantics) { if (semantics == null) { From 7af8c11286c8fb6ccdbece097096602c9da7b12c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 9 May 2026 15:53:16 -0400 Subject: [PATCH 4/4] Preserve published inspector framework sources --- server/src/api/routes.rs | 31 +++++++++++++++++++++++++- server/src/inspector.rs | 47 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index 4a077ade..a74ad820 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -3478,12 +3478,16 @@ async fn registry_inspector_session( } fn inspector_session_from_published(inspector: PublishedInspector) -> InspectorSession { + let mut available_sources = inspector_available_sources(&inspector.info); + for source in inspector.available_sources { + push_unique_source(&mut available_sources, &source); + } InspectorSession { transport: InspectorSessionTransport::RemoteDaemon { server_url: inspector.server_url, access_token: inspector.access_token, }, - available_sources: inspector.available_sources, + available_sources, info: inspector.info, process_identifier: inspector.process_identifier, } @@ -4788,6 +4792,31 @@ mod tests { assert!(metadata["port"].is_null()); } + #[test] + fn published_inspector_session_merges_sources_from_info() { + let session = inspector_session_from_published(PublishedInspector { + access_token: "secret-token".to_owned(), + available_sources: vec![SOURCE_UIKIT.to_owned()], + daemon_id: "daemon-a".to_owned(), + info: json!({ + "bundleIdentifier": "com.example.FlutterApp", + "processIdentifier": 42, + "flutter": { "available": true }, + "appHierarchy": { "available": true, "source": SOURCE_FLUTTER }, + "uikit": { "available": false } + }), + process_identifier: 42, + server_url: "http://127.0.0.1:4310".to_owned(), + updated_at_unix_ms: 1, + }); + + assert_eq!( + session.available_sources, + vec![SOURCE_FLUTTER.to_owned(), SOURCE_UIKIT.to_owned()] + ); + assert_eq!(inspector_session_score(&session), 1); + } + #[test] fn direct_inspector_request_endpoint_requires_api_auth() { assert!(is_inspector_agent_transport_path("/api/inspector/connect")); diff --git a/server/src/inspector.rs b/server/src/inspector.rs index f3f12fd7..5ca21f71 100644 --- a/server/src/inspector.rs +++ b/server/src/inspector.rs @@ -613,6 +613,14 @@ fn inspector_available_sources(info: &Value) -> Vec { if react_native_available { sources.push("react-native".to_owned()); } + let flutter_available = info + .get("flutter") + .and_then(|value| value.get("available")) + .and_then(Value::as_bool) + .unwrap_or(false); + if flutter_available { + sources.push("flutter".to_owned()); + } let app_hierarchy = info.get("appHierarchy"); let app_hierarchy_available = app_hierarchy .and_then(|value| value.get("available")) @@ -626,6 +634,7 @@ fn inspector_available_sources(info: &Value) -> Vec { match app_hierarchy_source { "nativescript" => sources.push("nativescript".to_owned()), "react-native" => push_unique_source(&mut sources, "react-native"), + "flutter" => push_unique_source(&mut sources, "flutter"), "swiftui" => push_unique_source(&mut sources, "swiftui"), _ => {} } @@ -634,7 +643,7 @@ fn inspector_available_sources(info: &Value) -> Vec { .get("uikit") .and_then(|value| value.get("available")) .and_then(Value::as_bool) - .unwrap_or(!react_native_available); + .unwrap_or(!(react_native_available || flutter_available)); if uikit_available { sources.push("in-app-inspector".to_owned()); } @@ -842,6 +851,23 @@ mod tests { }) } + fn flutter_inspector_info() -> Value { + json!({ + "protocolVersion": "1.0", + "bundleIdentifier": "com.example.FlutterApp", + "processIdentifier": 456, + "flutter": { + "available": true, + "widgetCreationTracked": true + }, + "appHierarchy": { + "available": true, + "source": "flutter" + }, + "uikit": { "available": false } + }) + } + #[tokio::test] async fn registry_advertisement_publishes_and_removes_live_inspector() { let path = registry_test_path("publish"); @@ -869,6 +895,25 @@ mod tests { let _ = fs::remove_file(path.with_extension("json.lock")); } + #[tokio::test] + async fn registry_advertisement_publishes_flutter_source() { + let path = registry_test_path("publish-flutter"); + let registry = registry_entry(&path); + + registry + .upsert(456, flutter_inspector_info()) + .await + .unwrap(); + + let entries = registry.read_live_entries().await; + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].process_identifier, 456); + assert_eq!(entries[0].available_sources, vec!["flutter".to_owned()]); + + let _ = fs::remove_file(&path); + let _ = fs::remove_file(path.with_extension("json.lock")); + } + #[tokio::test] async fn polled_inspector_info_response_publishes_registry_entry() { let path = registry_test_path("polled");