Reduce Text Artifacts, Fix Layout Bug, Public Undo Manager (#23)

### Description

- Reduces text drawing artifacts by turning off font smoothing, enabling subpixel positioning and font quantization, and a hidden smoothing API.
- Adds an internal ObjC target to accomplish the previous point.
- Fixes a layout bug where layout bounds would be nearly infinite due to the view being told to lay out but not be in the view hierarchy yet, causing a hang and memory explosion as every line in a potentially large document is laid out and rendered.
- Fixes a small bug with the undo manager's grouping behavior and makes it public (for a fix in CESE for undo-redo related bugs), as well as reordering some notifications in the undo manager.

### Related Issues

* N/A

### Checklist

- [x] I read and understood the [contributing guide](https://github.com/CodeEditApp/CodeEdit/blob/main/CONTRIBUTING.md) as well as the [code of conduct](https://github.com/CodeEditApp/CodeEdit/blob/main/CODE_OF_CONDUCT.md)
- [x] The issues this PR addresses are related to each other
- [x] My changes generate no new warnings
- [x] My code builds and runs on my machine
- [x] My changes are all related to the related issue above
- [x] I documented my code

### Screenshots

Before:
<img width="314" alt="Screenshot 2024-02-12 at 2 03 35 PM" src="https://github.com/CodeEditApp/CodeEditTextView/assets/35942988/b4f3b3ae-0eb7-4e0b-bde8-df0b7c8fcc65">

After (left CE, right Xcode):
![Screenshot 2024-02-13 at 2 23 36 PM](https://github.com/CodeEditApp/CodeEditTextView/assets/35942988/39d5fdce-cc3e-4dfd-94ea-4f571c6f3c27)

Author: thecoolwinter

Package.swift

@@ -36,13 +36,20 @@ let package = Package(
             name: "CodeEditTextView",
             dependencies: [
                 "TextStory",
-                .product(name: "Collections", package: "swift-collections")
+                .product(name: "Collections", package: "swift-collections"),
+                "CodeEditTextViewObjC"
             ],
             plugins: [
                 .plugin(name: "SwiftLint", package: "SwiftLintPlugin")
             ]
         ),
 
+        // ObjC addons
+        .target(
+            name: "CodeEditTextViewObjC",
+            publicHeadersPath: "include"
+        ),
+
         // Tests for the text view
         .testTarget(
             name: "CodeEditTextViewTests",

Sources/CodeEditTextView/TextLayoutManager/TextLayoutManager.swift

@@ -230,7 +230,12 @@ public class TextLayoutManager: NSObject {
 
     /// Lays out all visible lines
     func layoutLines() { // swiftlint:disable:this function_body_length
-        guard let visibleRect = delegate?.visibleRect, !isInTransaction, let textStorage else { return }
+        guard layoutView?.superview != nil,
+              let visibleRect = delegate?.visibleRect,
+              !isInTransaction,
+              let textStorage else {
+            return
+        }
         CATransaction.begin()
         let minY = max(visibleRect.minY, 0)
         let maxY = max(visibleRect.maxY, 0)

Sources/CodeEditTextView/TextLine/LineFragmentView.swift

@@ -6,6 +6,7 @@
 //
 
 import AppKit
+import CodeEditTextViewObjC
 
 /// Displays a line fragment.
 final class LineFragmentView: NSView {
@@ -23,7 +24,6 @@ final class LineFragmentView: NSView {
     override func prepareForReuse() {
         super.prepareForReuse()
         lineFragment = nil
-
     }
 
     /// Set a new line fragment for this view, updating view size.
@@ -39,13 +39,24 @@ final class LineFragmentView: NSView {
             return
         }
         context.saveGState()
-        context.setAllowsFontSmoothing(true)
-        context.setShouldSmoothFonts(true)
+
+        context.setAllowsAntialiasing(true)
+        context.setShouldAntialias(true)
+        context.setAllowsFontSmoothing(false)
+        context.setShouldSmoothFonts(false)
+        context.setAllowsFontSubpixelPositioning(true)
+        context.setShouldSubpixelPositionFonts(true)
+        context.setAllowsFontSubpixelQuantization(true)
+        context.setShouldSubpixelQuantizeFonts(true)
+
+        ContextSetHiddenSmoothingStyle(context, 16)
+
         context.textMatrix = .init(scaleX: 1, y: -1)
         context.textPosition = CGPoint(
             x: 0,
             y: lineFragment.height - lineFragment.descent + (lineFragment.heightDifference/2)
         ).pixelAligned
+
         CTLineDraw(lineFragment.ctLine, context)
         context.restoreGState()
     }

Sources/CodeEditTextView/TextView/TextView+ReplaceCharacters.swift

@@ -20,10 +20,17 @@ extension TextView {
         NotificationCenter.default.post(name: Self.textWillChangeNotification, object: self)
         layoutManager.beginTransaction()
         textStorage.beginEditing()
-        // Can't insert an ssempty string into an empty range. One must be not empty
+
+        var shouldEndGrouping = false
+        if !(_undoManager?.isGrouping ?? false) {
+            _undoManager?.beginGrouping()
+            shouldEndGrouping = true
+        }
+
+        // Can't insert an empty string into an empty range. One must be not empty
         for range in ranges.sorted(by: { $0.location > $1.location }) where
-        (delegate?.textView(self, shouldReplaceContentsIn: range, with: string) ?? true)
-        && (!range.isEmpty || !string.isEmpty) {
+        (!range.isEmpty || !string.isEmpty) &&
+        (delegate?.textView(self, shouldReplaceContentsIn: range, with: string) ?? true) {
             delegate?.textView(self, willReplaceContentsIn: range, with: string)
 
             layoutManager.willReplaceCharactersInRange(range: range, with: string)
@@ -38,6 +45,11 @@ extension TextView {
 
             delegate?.textView(self, didReplaceContentsIn: range, with: string)
         }
+
+        if shouldEndGrouping {
+            _undoManager?.endGrouping()
+        }
+
         layoutManager.endTransaction()
         textStorage.endEditing()
         selectionManager.notifyAfterEdit()

Sources/CodeEditTextView/TextView/TextView.swift

@@ -203,7 +203,7 @@ public class TextView: NSView, NSTextContent {
         (" " as NSString).size(withAttributes: [.font: font]).width
     }
 
-    var _undoManager: CEUndoManager?
+    internal(set) public var _undoManager: CEUndoManager?
     @objc dynamic open var allowsUndo: Bool
 
     var scrollView: NSScrollView? {

Sources/CodeEditTextView/Utils/CEUndoManager.swift

@@ -96,11 +96,11 @@ public class CEUndoManager {
             return
         }
         isUndoing = true
+        NotificationCenter.default.post(name: .NSUndoManagerWillUndoChange, object: self.manager)
         for mutation in item.mutations.reversed() {
-            NotificationCenter.default.post(name: .NSUndoManagerWillUndoChange, object: self.manager)
-            textView.insertText(mutation.inverse.string, replacementRange: mutation.inverse.range)
-            NotificationCenter.default.post(name: .NSUndoManagerDidUndoChange, object: self.manager)
+            textView.replaceCharacters(in: mutation.inverse.range, with: mutation.inverse.string)
         }
+        NotificationCenter.default.post(name: .NSUndoManagerDidUndoChange, object: self.manager)
         redoStack.append(item)
         isUndoing = false
     }
@@ -111,11 +111,11 @@ public class CEUndoManager {
             return
         }
         isRedoing = true
+        NotificationCenter.default.post(name: .NSUndoManagerWillRedoChange, object: self.manager)
         for mutation in item.mutations {
-            NotificationCenter.default.post(name: .NSUndoManagerWillRedoChange, object: self.manager)
-            textView.insertText(mutation.mutation.string, replacementRange: mutation.mutation.range)
-            NotificationCenter.default.post(name: .NSUndoManagerDidRedoChange, object: self.manager)
+            textView.replaceCharacters(in: mutation.mutation.range, with: mutation.mutation.string)
         }
+        NotificationCenter.default.post(name: .NSUndoManagerDidRedoChange, object: self.manager)
         undoStack.append(item)
         isRedoing = false
     }
@@ -159,11 +159,19 @@ public class CEUndoManager {
 
     /// Groups all incoming mutations.
     public func beginGrouping() {
+        guard !isGrouping else {
+            assertionFailure("UndoManager already in a group. Call `endGrouping` before this can be called.")
+            return
+        }
         isGrouping = true
     }
 
     /// Stops grouping all incoming mutations.
     public func endGrouping() {
+        guard isGrouping else {
+            assertionFailure("UndoManager not in a group. Call `beginGrouping` before this can be called.")
+            return
+        }
         isGrouping = false
     }
 

Sources/CodeEditTextViewObjC/CGContextHidden.m

@@ -0,0 +1,15 @@
+//
+//  CGContextHidden.m
+//  CodeEditTextViewObjC
+//
+//  Created by Khan Winter on 2/12/24.
+//
+
+#import <Cocoa/Cocoa.h>
+#import "CGContextHidden.h"
+
+extern void CGContextSetFontSmoothingStyle(CGContextRef, int);
+
+void ContextSetHiddenSmoothingStyle(CGContextRef context, int style) {
+    CGContextSetFontSmoothingStyle(context, style);
+}

Sources/CodeEditTextViewObjC/include/CGContextHidden.h

@@ -0,0 +1,15 @@
+//
+//  CGContextHidden.h
+//  CodeEditTextViewObjC
+//
+//  Created by Khan Winter on 2/12/24.
+//
+
+#ifndef CGContextHidden_h
+#define CGContextHidden_h
+
+#import <Cocoa/Cocoa.h>
+
+void ContextSetHiddenSmoothingStyle(CGContextRef context, int style);
+
+#endif /* CGContextHidden_h */

Sources/CodeEditTextViewObjC/include/module.modulemap

@@ -0,0 +1,3 @@
+module CodeEditTextViewObjC {
+    header "CGContextHidden.h"
+}