diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index d0448365b9..a52f7d651e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -539,14 +539,16 @@ - (void)agentDeviceTextEntryDidChange:(UITextField *)textField { } // An auto-submitting one-time-code field: the last digit navigates away, to a screen whose own // input takes the code field's place or to one without an input. `replace-after-input` is the - // same navigation landing before the code is complete. + // same navigation landing before the code is complete, and `populated-successor` the same on any + // edit, clearing included, to a screen whose input already holds a value. NSArray *arguments = NSProcessInfo.processInfo.arguments; BOOL codeComplete = textField.text.length >= AgentDeviceTextEntryAutoSubmitLength; if (textField.superview == nil) { return; } if ((codeComplete && [arguments containsObject:@"--agent-device-text-entry-auto-submit"]) || - (textField.text.length > 0 && [arguments containsObject:@"--agent-device-text-entry-replace-after-input"])) { + (textField.text.length > 0 && [arguments containsObject:@"--agent-device-text-entry-replace-after-input"]) || + [arguments containsObject:@"--agent-device-text-entry-populated-successor"]) { [self replaceTextEntryFieldWithNextScreenInput:textField]; } else if (codeComplete && [arguments containsObject:@"--agent-device-text-entry-auto-submit-without-successor"]) { @@ -559,6 +561,9 @@ - (void)replaceTextEntryFieldWithNextScreenInput:(UITextField *)textField { if (![NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-unnamed-input"]) { nextScreenField.accessibilityIdentifier = @"agent-device-auto-submit-next-screen-input"; } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-populated-successor"]) { + nextScreenField.text = @"next screen value"; + } nextScreenField.borderStyle = UITextBorderStyleRoundedRect; [textField.superview addSubview:nextScreenField]; [textField removeFromSuperview]; @@ -762,6 +767,10 @@ - (void)viewDidLoad { if (![NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-unnamed-input"]) { textField.accessibilityIdentifier = @"agent-device-hardware-keyboard-input"; } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-long-prefill"]) { + // Longer than one clear burst and far wider than the field, so its end is scrolled out of view. + textField.text = [@"" stringByPaddingToLength:300 withString:@"stale long value " startingAtIndex:0]; + } textField.borderStyle = UITextBorderStyleRoundedRect; // An empty input view keeps the software keyboard down, which is the hardware-keyboard responder // these routes are addressed to. `--agent-device-text-entry-soft-keyboard` leaves the real input diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift index 4fb61c8502..16e7020d20 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift @@ -251,7 +251,8 @@ extension RunnerTests { case cleared /// The field never exposes its value, so it is typed into unverified, as the commit wait leaves it. case unverified - /// The field still held text after the last pass, or could not be read. Nothing is typed. + /// The field still held text after the last pass, or could not be read. The synthesized post + /// types nothing; the element route's clear falls back to delete bursts. case notCleared /// A clear post stopped the plan. case stopped @@ -259,7 +260,7 @@ extension RunnerTests { /// Clears until the field reads empty, at most `maxClearPassCount` passes. A read that cannot /// answer ends the clear, so the post never types over text it could not see removed. - static func clearForSynthesizedReplacement( + static func clearUntilEmpty( clearOnce: () -> SynthesizedStepDispatch, read: () -> TextValueRead ) -> SynthesizedClearOutcome { @@ -378,7 +379,7 @@ extension RunnerTests { } if step.replacesExistingText { var passes = 0 - let outcome = Self.clearForSynthesizedReplacement( + let outcome = Self.clearUntilEmpty( clearOnce: { passes += 1 return enter(XCUIKeyboardKey.delete.rawValue, replacingExistingText: true) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 22184877a5..d73de3461f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -355,9 +355,17 @@ extension RunnerTests { } } - /// Clears `input`, which must have been read with nothing dispatched since. - func clearTextInput(app: XCUIApplication, _ input: ResolvedTextEntryInput) { - // Skip the clear (delete burst + moveCaretToEnd edge-tap) ONLY when we can confirm the + /// Clears the reader's input. Every post goes through `reader.dispatch` and every read back through + /// `reader.input()`, so a bound target's clear stops at an input that took its place. On iOS a + /// readable value is selected whole and deleted, wherever the caret is. Otherwise, or when text is + /// left, delete bursts run from the caret, and a value longer than one burst is read again after + /// each burst until it reads empty, stops shortening, or has had at least one burst more than it + /// needs. + func clearTextInput(app: XCUIApplication, reader: inout TextEntryReader) { + guard let input = reader.input() else { + return + } + // Skip the clear (edge-tap, select all, delete bursts) ONLY when we can confirm the // field is empty. Why skip: the edge-tap computes a point from the element frame, which can // be stale after the field repositions on focus (e.g. the Settings search bar jumps // bottom->top and reveals a "Suggestions" list) — tapping there navigates away instead of @@ -365,17 +373,91 @@ extension RunnerTests { // editableTextValue returns nil for secure (and unknown) fields, where we CANNOT confirm // emptiness — those must still be cleared, or replace would concatenate stale + new text. // So distinguish nil (clear) from "" (skip). - if let existing = editableTextValue(for: input.attributes, treatingPlaceholderAsEmpty: true), - existing.isEmpty { + let existing = editableTextValue(for: input.attributes, treatingPlaceholderAsEmpty: true) + if existing?.isEmpty == true { return } // Read before the caret tap, which dispatches to the app. - let count = estimatedDeleteCount(for: input.attributes) + var valueLength = normalizedElementText(input.attributes.value).count + // The tap also focuses the field, which a clear can reach with focus unconfirmed. #if !os(tvOS) - moveCaretToEnd(app: app, input: input) + reader.dispatch { moveCaretToEnd(app: app, input: input) } +#endif + // An input that stops answering once focused (#3060) cannot be read back, so it gets one burst. + guard reader.input() != nil else { + let burst = Self.textClearBurst(valueLength: valueLength) + reader.dispatch { input.element.typeText(String(repeating: XCUIKeyboardKey.delete.rawValue, count: burst)) } + return + } +#if os(iOS) + // The tap aims at the end of what the field shows, which a long value has scrolled past, so + // bursts from there reach only what is before the caret (#3851). Select all reaches all of it, but + // only a read can show it took: a masked field can ignore it, so an unreadable value keeps the + // bursts alone. + if existing != nil { + var passes = 0 + let outcome = Self.clearUntilEmpty( + clearOnce: { + guard let input = reader.input() else { + return .stop + } + passes += 1 + reader.dispatch { + input.element.typeKey("a", modifierFlags: .command) + input.element.typeText(XCUIKeyboardKey.delete.rawValue) + } + return .posted + }, + read: { + textValueRead(for: reader.input()?.attributes, treatingPlaceholderAsEmpty: true) + } + ) + NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_CLEAR route=element passes=%d outcome=%@", passes, String(describing: outcome)) + // Text left after every pass goes to the bursts; an input that no longer resolves is not typed into. + guard outcome == .notCleared, + let remaining = reader.input().flatMap({ editableTextValue(for: $0.attributes, treatingPlaceholderAsEmpty: true) }) + else { + return + } + valueLength = remaining.count + } #endif - let deletes = String(repeating: XCUIKeyboardKey.delete.rawValue, count: count) - input.element.typeText(deletes) + for _ in 0...(valueLength / Self.maxTextClearBurst + 1) { + guard let input = reader.input() else { + return + } + let burst = Self.textClearBurst(valueLength: valueLength) + reader.dispatch { input.element.typeText(String(repeating: XCUIKeyboardKey.delete.rawValue, count: burst)) } + // A burst that covers the value needs no read. + guard burst < valueLength + Self.textClearSlack, + let remaining = reader.input().flatMap({ editableTextValue(for: $0.attributes, treatingPlaceholderAsEmpty: true) }), + Self.textClearContinues(clearedLength: valueLength, remainingLength: remaining.count) + else { + return + } + valueLength = remaining.count + } + } + + /// Extra deletes past a value's length, for whitespace its normalized read trimmed. + private static let textClearSlack = 8 + + /// The most deletes one clear burst sends. A longer value takes more bursts. + private static let maxTextClearBurst = 120 + + /// Deletes for one clear burst on a value of `valueLength` characters, at least 24 for a value + /// that reads empty but may not be. + static func textClearBurst(valueLength: Int) -> Int { + let base = valueLength == 0 ? 24 : valueLength + textClearSlack + return max(24, min(maxTextClearBurst, base)) + } + + /// Whether a clear sends another burst after one sized from `clearedLength` characters left + /// `remainingLength`. Each burst must shorten the value, so a field that ignores deletes or + /// rewrites its value ends the clear instead of looping; the fill's verification reports what + /// remains. + static func textClearContinues(clearedLength: Int, remainingLength: Int) -> Bool { + remainingLength > 0 && remainingLength < clearedLength } func isTextEntryElement(_ element: XCUIElement) -> Bool { @@ -529,12 +611,6 @@ extension RunnerTests { #endif } - private func estimatedDeleteCount(for element: any XCUIElementAttributes) -> Int { - let valueText = normalizedElementText(element.value) - let base = valueText.isEmpty ? 24 : (valueText.count + 8) - return max(24, min(120, base)) - } - private func normalizedElementText(_ value: Any?) -> String { String(describing: value ?? "") .trimmingCharacters(in: .whitespacesAndNewlines) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index c0ad58ca1c..142f9c4415 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -23,7 +23,7 @@ extension RunnerTests { // place while the response reported "typed". Append (`type`) and unrepaired entry stay // no-ops, which is what an empty payload means for them. if repairMode == .replacement { - guard let clearTarget = reader.input() else { + guard reader.input() != nil else { // No resolvable input means no clear happened — including the synthesized // first-responder route, whose target carries no element. Success here would claim a // clear the runner never performed. @@ -36,7 +36,7 @@ extension RunnerTests { failure: .notFocused ) } - reader.dispatch { clearTextInput(app: app, clearTarget) } + clearTextInput(app: app, reader: &reader) // nil means the value is unreadable (secure fields), not "not empty": leave `verified` // nil there rather than reporting a mismatch the runner cannot actually observe. let observed = editableTextValue(for: reader.input()?.attributes, treatingPlaceholderAsEmpty: true) @@ -119,13 +119,13 @@ extension RunnerTests { var entryBaseline: TextEntryObservation? if repairMode == .replacement { - guard let replacementTarget = initialTarget else { + guard initialTarget != nil else { logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode) return TextEntryResult(verified: nil, repaired: false, expectedText: expectedText, observedText: nil) } if currentText == nil || currentText?.isEmpty == false { let clearStartedAt = Date() - reader.dispatch { clearTextInput(app: app, replacementTarget) } + clearTextInput(app: app, reader: &reader) logTextEntryPhase( commandId: commandId, phase: "clear", @@ -477,10 +477,18 @@ extension RunnerTests { expectedText.count, observedText.count ) - reader.dispatch { - clearTextInput(app: app, repairTarget) - repairTarget.element.typeText(expectedText) + clearTextInput(app: app, reader: &reader) + // An input the clear removed or let another take the place of has nowhere for the repair to go. + guard let input = reader.input() else { + return TextEntryResult( + verified: nil, + repaired: false, + expectedText: expectedText, + observedText: nil, + failure: .commitNotObserved + ) } + reader.dispatch { input.element.typeText(expectedText) } return verifyTextEntry( reader: &reader, expectedText: expectedText, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index 5afd1da377..02e1978022 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -123,6 +123,24 @@ extension RunnerTests { } } + // A clear burst covers a short value whole and caps a long one, which the clear then + // finishes in more bursts (#3851). + func testTextClearBurstCoversShortValuesAndCapsLongOnes() { + XCTAssertEqual(Self.textClearBurst(valueLength: 0), 24) + XCTAssertEqual(Self.textClearBurst(valueLength: 10), 24) + XCTAssertEqual(Self.textClearBurst(valueLength: 100), 108) + XCTAssertEqual(Self.textClearBurst(valueLength: 501), 120) + } + + // A clear goes on only while each burst shortens the value, so a field that ignores deletes or + // rewrites its value cannot keep it looping. + func testTextClearContinuesOnlyWhileTheValueShortens() { + XCTAssertTrue(Self.textClearContinues(clearedLength: 501, remainingLength: 381)) + XCTAssertFalse(Self.textClearContinues(clearedLength: 501, remainingLength: 0)) + XCTAssertFalse(Self.textClearContinues(clearedLength: 501, remainingLength: 501)) + XCTAssertFalse(Self.textClearContinues(clearedLength: 501, remainingLength: 640)) + } + // One read owns readability for every route: a secure field is unreadable whatever it reports, // a plain field is unreadable only while its value is a mask, and anything that is not a text // input has no text to read. @@ -156,7 +174,7 @@ extension RunnerTests { // ending the clear as a field that could not be read. func testSynthesizedClearLeavesAMaskedFieldUnverified() { var passes = 0 - let outcome = Self.clearForSynthesizedReplacement( + let outcome = Self.clearUntilEmpty( clearOnce: { passes += 1 return .posted @@ -459,7 +477,7 @@ extension RunnerTests { for testCase in cases { var passes = 0 var reads = testCase.reads[...] - let outcome = Self.clearForSynthesizedReplacement( + let outcome = Self.clearUntilEmpty( clearOnce: { passes += 1 return .posted @@ -470,7 +488,7 @@ extension RunnerTests { XCTAssertEqual(passes, testCase.passes, "\(testCase.reads)") } XCTAssertEqual( - Self.clearForSynthesizedReplacement(clearOnce: { .stop }, read: { .text("") }), + Self.clearUntilEmpty(clearOnce: { .stop }, read: { .text("") }), .stopped ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift index 73ee78bcd5..2eb822e0f1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift @@ -176,6 +176,57 @@ extension RunnerTests { XCTAssertEqual(editableTextValue(for: nextScreenField, treatingPlaceholderAsEmpty: true), "") } + // A value longer than one clear burst and wider than its field (#3851): the caret edge-tap lands + // mid-text and a burst deletes at most 120 characters, so neither alone can reach all of it. + @MainActor + func testFillReplacesAValueLongerThanOneClearBurst() throws { + let textField = try launchRemovableInputFixture("--agent-device-text-entry-long-prefill") + + let response = executeTypeCommand( + activeApp: app, + command: try fillCommandFixture(commandId: "fill-long-prefill", text: "Ada Lovelace", at: textField) + ) + + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(response.data?.textEntryRoute, "xctest-element") + XCTAssertEqual(editableTextValue(for: textField), "Ada Lovelace") + } + + @MainActor + func testFillClearsAValueLongerThanOneClearBurst() throws { + let textField = try launchRemovableInputFixture("--agent-device-text-entry-long-prefill") + + let response = executeTypeCommand( + activeApp: app, + command: try fillCommandFixture(commandId: "clear-long-prefill", text: "", at: textField) + ) + + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(editableTextValue(for: textField, treatingPlaceholderAsEmpty: true), "") + } + + // The clear's first delete navigates to a screen whose input, already holding a value, takes the + // cleared one's place under the same handle. The clear keeps going only on the input it started on. + @MainActor + func testFillClearDoesNotEraseTheInputThatReplacedItsTarget() throws { + let textField = try launchRemovableInputFixture( + "--agent-device-text-entry-long-prefill", + "--agent-device-text-entry-populated-successor" + ) + + let failureCountBefore = currentXCTestFailureCount() + _ = executeTypeCommand( + activeApp: app, + command: try fillCommandFixture(commandId: "fill-clear-replaced-input", text: "", at: textField) + ) + + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertFalse(textField.exists) + let nextScreenField = app.textFields["agent-device-auto-submit-next-screen-input"] + XCTAssertTrue(nextScreenField.exists) + XCTAssertEqual(editableTextValue(for: nextScreenField), "next screen value") + } + // The reported shape: the last digit navigates to a screen without an input. @MainActor func testFillSucceedsWhenAppRemovesInputAfterLastCharacter() throws {