Handle cleanup of output regardless of final line ending

helpers_unix.go

@@ -9,6 +9,6 @@ import (
 
 var ERR_ACCESS_DENIED = errors.New("only used on windows, this should never match")
 
-func cleanPtySnapshot(b []byte, cursorPos int, _ bool) ([]byte, int) {
-	return b, cursorPos
+func cleanPtySnapshot(b []byte, cursorPos int, _ bool) ([]byte, int, int) {
+	return b, cursorPos, len(b)
 }

helpers_windows.go

@@ -12,9 +12,9 @@ const UnicodeBackspaceRune = '\u0008' // Note in the docs this is \u007f, but in
 // Ultimately we want to emulate the windows console here, just like we're doing for v10x on posix.
 // The current implementation is geared towards our needs, and won't be able to handle all escape sequences as a result.
 // For details on escape sequences see https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences
-func cleanPtySnapshot(snapshot []byte, cursorPos int, isPosix bool) ([]byte, int) {
+func cleanPtySnapshot(snapshot []byte, cursorPos int, isPosix bool) (_output []byte, _cursorPos int, _cleanUptoPos int) {
 	if isPosix {
-		return snapshot, cursorPos
+		return snapshot, cursorPos, len(snapshot)
 	}
 
 	// Most escape sequences appear to end on `A-Za-z@`
@@ -37,7 +37,10 @@ func cleanPtySnapshot(snapshot []byte, cursorPos int, isPosix bool) ([]byte, int
 	}
 
 	var result []rune
+	var unterminatedEscape []rune
 	runes := bytes.Runes(snapshot)
+	escapeStartPos := -1
+
 	for pos, r := range runes {
 		// Reset code recording outside of escape sequence, so we don't have to manually handle this throughout
 		if !inEscapeSequence {
@@ -48,14 +51,15 @@ func cleanPtySnapshot(snapshot []byte, cursorPos int, isPosix bool) ([]byte, int
 		// SEQUENCE START
 
 		// Delete alert / bell sequence
-		case !inEscapeSequence && r == '\a':
+		case !inEscapeSequence && r == UnicodeBellRune:
 			dropPos(pos)
 			continue
 
 		// Detect start of escape sequence
 		case !inEscapeSequence && r == UnicodeEscapeRune:
 			inEscapeSequence = true
 			recordingCode = true
+			escapeStartPos = pos
 			dropPos(pos)
 			continue
 
@@ -71,13 +75,15 @@ func cleanPtySnapshot(snapshot []byte, cursorPos int, isPosix bool) ([]byte, int
 		// Detect end of escape sequence
 		case inEscapeSequence && !inTitleEscapeSequence && bytes.ContainsRune(plainVirtualEscapeSeqEndValues, r):
 			inEscapeSequence = false
+			escapeStartPos = -1
 			dropPos(pos)
 			continue
 
 		// Detect end of complex escape sequence
 		case inTitleEscapeSequence && r == UnicodeBellRune:
 			inEscapeSequence = false
 			inTitleEscapeSequence = false
+			escapeStartPos = -1
 			dropPos(pos)
 			continue
 
@@ -108,5 +114,13 @@ func cleanPtySnapshot(snapshot []byte, cursorPos int, isPosix bool) ([]byte, int
 			result = append(result, r)
 		}
 	}
-	return []byte(string(result)), newCursorPos
+
+	// If we're still in an escape sequence at the end, retain the unterminated sequence
+	cleanUptoPos := len(result)
+	if inEscapeSequence && escapeStartPos >= 0 {
+		unterminatedEscape = runes[escapeStartPos:]
+		result = append(result, unterminatedEscape...)
+	}
+
+	return []byte(string(result)), newCursorPos, cleanUptoPos
 }

helpers_windows_test.go

@@ -13,69 +13,79 @@ func Test_cleanPtySequences(t *testing.T) {
 		cursorPos     int
 		want          []byte
 		wantCursorPos int
+		wantCleanUpto int
 	}{
 		{
 			"Window title, cursor after",
 			[]byte("\u001B]0;C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\2642502767\\cache\\94dd3fa4\\exec\\python3.exe\u0007Hello"),
 			86, // First two characters of Hello
 			[]byte("Hello"),
 			2,
+			5,
 		},
 		{
 			"Window title, cursor preceding",
 			[]byte("Hello\u001B]0;C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\2642502767\\cache\\94dd3fa4\\exec\\python3.exe\u0007World"),
 			1, // First two characters of Hello
 			[]byte("HelloWorld"),
 			1,
+			10,
 		},
 		{
 			"Window title, cursor on top",
 			[]byte("Hello\u001B]0;C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\2642502767\\cache\\94dd3fa4\\exec\\python3.exe\u0007World"),
 			10, // Inside title escape sequence
 			[]byte("HelloWorld"),
 			4,
+			10,
 		},
 		{
 			"Backspace character",
 			[]byte("Foo \u0008Bar"),
 			7, // End of string
 			[]byte("FooBar"),
 			5,
+			6,
 		},
 		{
 			"Backspace character, cursor on top of backspace",
 			[]byte("Foo \u0008Bar"),
 			5, // End of string
 			[]byte("FooBar"),
 			3,
+			6,
 		},
 		{
 			"Cursor position preceding cleaned sequence",
 			[]byte("Foo\u001B[1mBar"), // \u001B[1m = bold
 			2,                         // End of "Foo"
 			[]byte("FooBar"),
 			2,
+			6,
 		},
 		{
 			"Cursor position succeeding cleaned sequence",
 			[]byte("Foo\u001B[1mBar"), // \u001B[1m = bold
 			9,                         // End of "Bar"
 			[]byte("FooBar"),
 			5,
+			6,
 		},
 		{
 			"Cursor position on top of cleaned sequence",
 			[]byte("Foo\u001B[1mBar"), // \u001B[1m = bold
 			4,                         // Unicode code point
 			[]byte("FooBar"),
 			2,
+			6,
 		},
 		{
 			"Negative cursor position",
 			[]byte("Foo\u001B[1mBar"), // \u001B[1m = bold
 			-10,                       // End of "Foo"
 			[]byte("FooBar"),
 			-10,
+			6,
 		},
 		{
 			// Running on ANSI escape codes obviously is not the intent, but without being able to easily identify
@@ -85,6 +95,7 @@ func Test_cleanPtySequences(t *testing.T) {
 			165,
 			[]byte("25h    25l █ Installing Runtime (Unconfigured)25h 25l █25h 25l █ Installing Runtime Environment25h 25l Setting Up Runtime       \n  Resolving Dependencies |25h"),
 			159,
+			158,
 		},
 		{
 			"Escape at first character",
@@ -94,34 +105,47 @@ func Test_cleanPtySequences(t *testing.T) {
 			// Since the cleaner handles an absolute cursor position against relative output, we can't determine start
 			// of output and so we return a negative
 			-1,
+			3,
 		},
 		{
 			"Cursor character (NOT position)",
 			[]byte("foo\u001B[?25hbar"),
 			0,
 			[]byte("foobar"),
 			0,
+			6,
 		},
 		{
 			"Home key",
 			[]byte("\x1b[Hfoo"),
 			0,
 			[]byte("foo"),
 			-1,
+			3,
 		},
 		{
 			"Home key with Window title following",
 			[]byte("\x1b[H\x1b]0;C:\\Windows\\System32\\cmd.exe\afoo"),
 			0,
 			[]byte("foo"),
 			-1,
+			3,
 		},
 		{
 			"Alert / bell character",
 			[]byte("\aP\x1b[?25lython 3.9.5"),
 			0,
 			[]byte("Python 3.9.5"),
 			-1,
+			12,
+		},
+		{
+			"Unterminated escape sequence",
+			[]byte("foo\x1b[?25"),
+			0,
+			[]byte("foo\x1b[?25"),
+			0,
+			3,
 		},
 	}
 	for _, tt := range tests {
@@ -132,9 +156,10 @@ func Test_cleanPtySequences(t *testing.T) {
 			if tt.wantCursorPos > len(tt.want) {
 				t.Fatal("Wanted cursor position cannot be larger than wanted output")
 			}
-			cleaned, cursorPos := cleanPtySnapshot(tt.b, tt.cursorPos, false)
-			assert.Equal(t, string(tt.want), string(cleaned))
-			assert.Equal(t, tt.wantCursorPos, cursorPos)
+			cleaned, cursorPos, cleanUptoPos := cleanPtySnapshot(tt.b, tt.cursorPos, false)
+			assert.Equal(t, string(tt.want), string(cleaned), "wanted output")
+			assert.Equal(t, tt.wantCursorPos, cursorPos, "wanted cursor position")
+			assert.Equal(t, tt.wantCleanUpto, cleanUptoPos, "wanted clean upto position")
 		})
 	}
 }

outputproducer.go

@@ -2,7 +2,6 @@ package termtest
 
 import (
 	"bufio"
-	"bytes"
 	"errors"
 	"fmt"
 	"io"
@@ -66,13 +65,15 @@ func (o *outputProducer) listen(r io.Reader, w io.Writer, appendBuffer func([]by
 var PtyEOF = errors.New("pty closed")
 
 func (o *outputProducer) processNextRead(r io.Reader, w io.Writer, appendBuffer func([]byte, bool) error, size int) error {
+	isEOF := false
 	o.opts.Logger.Printf("processNextRead started with size: %d\n", size)
-	defer o.opts.Logger.Println("processNextRead stopped")
+	defer func() {
+		o.opts.Logger.Printf("processNextRead stopped, isEOF: %v\n", isEOF)
+	}()
 
 	snapshot := make([]byte, size)
 	n, errRead := r.Read(snapshot)
 
-	isEOF := false
 	if errRead != nil {
 		pathError := &fs.PathError{}
 		if errors.Is(errRead, fs.ErrClosed) || errors.Is(errRead, io.EOF) || (runtime.GOOS == "linux" && errors.As(errRead, &pathError)) {
@@ -116,13 +117,13 @@ func (o *outputProducer) appendBuffer(value []byte, isFinal bool) error {
 
 	// Clean output
 	var err error
-	o.output, o.cursorPos, o.cleanUptoPos, err = o.processDirtyOutput(output, o.cursorPos, o.cleanUptoPos, isFinal, func(output []byte, cursorPos int) ([]byte, int, error) {
+	o.output, o.cursorPos, o.cleanUptoPos, err = o.processDirtyOutput(output, o.cursorPos, o.cleanUptoPos, isFinal, func(output []byte, cursorPos int) ([]byte, int, int, error) {
 		var err error
-		output, cursorPos = cleanPtySnapshot(output, cursorPos, o.opts.Posix)
+		output, cursorPos, cleanCursorPos := cleanPtySnapshot(output, cursorPos, o.opts.Posix)
 		if o.opts.OutputSanitizer != nil {
-			output, cursorPos, err = o.opts.OutputSanitizer(output, cursorPos)
+			output, cursorPos, cleanCursorPos, err = o.opts.OutputSanitizer(output, cursorPos)
 		}
-		return output, cursorPos, err
+		return output, cursorPos, cleanCursorPos, err
 	})
 	if err != nil {
 		return fmt.Errorf("cleaning output failed: %w", err)
@@ -138,48 +139,36 @@ func (o *outputProducer) appendBuffer(value []byte, isFinal bool) error {
 	return nil
 }
 
-type cleanerFunc func([]byte, int) ([]byte, int, error)
+type cleanerFunc func(snapshot []byte, cursorPos int) (newSnapshot []byte, newCursorPos int, cleanUptoPos int, err error)
 
 // processDirtyOutput will sanitize the output received, but we have to be careful not to clean output that hasn't fully arrived
 // For example we may be inside an escape sequence and the escape sequence hasn't finished
 // So instead we only process new output up to the most recent line break
 // In order for this to work properly the invoker must ensure the output and cleanUptoPos are consistent with each other.
 func (o *outputProducer) processDirtyOutput(output []byte, cursorPos int, cleanUptoPos int, isFinal bool, cleaner cleanerFunc) (_output []byte, _cursorPos int, _cleanUptoPos int, _err error) {
 	defer func() {
-		o.opts.Logger.Printf("Cleaned output from %d to %d\n", cleanUptoPos, _cleanUptoPos)
+		o.opts.Logger.Printf("Cleaned output from %d to %d (isFinal: %v)\n", cleanUptoPos, _cleanUptoPos, isFinal)
 	}()
 	alreadyCleanedOutput := copyBytes(output[:cleanUptoPos])
 	processedOutput := []byte{}
 	unprocessedOutput := copyBytes(output[cleanUptoPos:])
-	processedCursorPos := cursorPos - len(alreadyCleanedOutput)
-
-	if isFinal {
-		// If we've reached the end there's no point looking for the most recent line break as there's no guarantee the
-		// output will be terminated by a newline.
-		processedOutput = copyBytes(unprocessedOutput)
-		unprocessedOutput = []byte{}
-	} else {
-		// Find the most recent line break, and only clean until that point.
-		// Any output after the most recent line break is considered not ready for cleaning as cleaning depends on
-		// multiple consecutive characters.
-		lineSepN := bytes.LastIndex(unprocessedOutput, []byte("\n"))
-		if lineSepN != -1 {
-			processedOutput = copyBytes(unprocessedOutput[0 : lineSepN+1])
-			unprocessedOutput = unprocessedOutput[lineSepN+1:]
-		}
-	}
+	relativeCursorPos := cursorPos - len(alreadyCleanedOutput)
 
 	// Invoke the cleaner now that we have output that can be cleaned
-	if len(processedOutput) > 0 {
+	newCleanUptoPos := cleanUptoPos
+	if len(unprocessedOutput) > 0 {
 		var err error
-		processedOutput, processedCursorPos, err = cleaner(processedOutput, processedCursorPos)
+		var processedCleanUptoPos int
+		processedOutput, relativeCursorPos, processedCleanUptoPos, err = cleaner(unprocessedOutput, relativeCursorPos)
 		if err != nil {
-			return processedOutput, processedCursorPos, cleanUptoPos, fmt.Errorf("cleaner failed: %w", err)
+			return processedOutput, relativeCursorPos, processedCleanUptoPos, fmt.Errorf("cleaner failed: %w", err)
 		}
+		// Keep a record of what point we're up to
+		newCleanUptoPos += processedCleanUptoPos
 	}
 
 	// Convert cursor position back to absolute
-	processedCursorPos += len(alreadyCleanedOutput)
+	processedCursorPos := relativeCursorPos + len(alreadyCleanedOutput)
 
 	if processedCursorPos < 0 {
 		// Because the cleaner function needs to support a negative cursor position it is impossible for the cleaner
@@ -188,11 +177,8 @@ func (o *outputProducer) processDirtyOutput(output []byte, cursorPos int, cleanU
 		processedCursorPos = 0
 	}
 
-	// Keep a record of what point we're up to
-	newCleanUptoPos := cleanUptoPos + len(processedOutput)
-
 	// Stitch everything back together
-	return append(append(alreadyCleanedOutput, processedOutput...), unprocessedOutput...), processedCursorPos, newCleanUptoPos, nil
+	return append(alreadyCleanedOutput, processedOutput...), processedCursorPos, newCleanUptoPos, nil
 }
 
 func (o *outputProducer) closeConsumers(reason error) {