Fixes #4200. ExtendedCharInfo needs be enhanced to properly deal with all codepoints

Examples/UICatalog/UICatalogTop.cs

@@ -166,13 +166,23 @@ View [] CreateThemeMenuItems ()
                 CheckedState = Application.Force16Colors ? CheckState.Checked : CheckState.UnChecked
             };
 
+            _force16ColorsMenuItemCb.CheckedStateChanging += (sender, args) =>
+                                                             {
+                                                                 if (Application.Force16Colors
+                                                                     && args.Result == CheckState.UnChecked
+                                                                     && !Application.Driver!.SupportsTrueColor)
+                                                                 {
+                                                                     args.Handled = true;
+                                                                 }
+                                                             };
+
             _force16ColorsMenuItemCb.CheckedStateChanged += (sender, args) =>
-            {
-                Application.Force16Colors = args.Value == CheckState.Checked;
+                                                            {
+                                                                Application.Force16Colors = args.Value == CheckState.Checked;
 
-                _force16ColorsShortcutCb!.CheckedState = args.Value;
-                Application.LayoutAndDraw ();
-            };
+                                                                _force16ColorsShortcutCb!.CheckedState = args.Value;
+                                                                Application.LayoutAndDraw ();
+                                                            };
 
             menuItems.Add (
                            new MenuItemv2
@@ -633,11 +643,22 @@ private StatusBar CreateStatusBar ()
         };
 
         _force16ColorsShortcutCb.CheckedStateChanging += (sender, args) =>
-        {
-            Application.Force16Colors = args.Result == CheckState.Checked;
-            _force16ColorsMenuItemCb!.CheckedState = args.Result;
-            Application.LayoutAndDraw ();
-        };
+                                                         {
+                                                             if (Application.Force16Colors
+                                                                 && args.Result == CheckState.UnChecked
+                                                                 && !Application.Driver!.SupportsTrueColor)
+                                                             {
+                                                                 // If the driver does not support TrueColor, we cannot disable 16 colors
+                                                                 args.Handled = true;
+                                                             }
+                                                         };
+
+        _force16ColorsShortcutCb.CheckedStateChanged += (sender, args) =>
+                                                         {
+                                                             Application.Force16Colors = args.Value == CheckState.Checked;
+                                                             _force16ColorsMenuItemCb!.CheckedState = args.Value;
+                                                             Application.LayoutAndDraw ();
+                                                         };
 
         statusBar.Add (
                        _shQuit,

Terminal.Gui/App/Application.Run.cs

@@ -212,8 +212,15 @@ public static RunState Begin (Toplevel toplevel)
 
         NotifyNewRunState?.Invoke (toplevel, new (rs));
 
-        // Force an Idle event so that an Iteration (and Refresh) happen.
-        Invoke (() => { });
+        if (!ConsoleDriver.RunningUnitTests)
+        {
+            // Force an Idle event so that an Iteration (and Refresh) happen.
+            Task.Run (() =>
+                      {
+                          Invoke (() => { });
+                          Task.Delay (10).Wait ();
+                      });
+        }
 
         return rs;
     }

Terminal.Gui/App/ApplicationImpl.cs

@@ -293,6 +293,10 @@ public virtual void Invoke (Action action)
                                return false;
                            }
                           );
+
+        // Ensure the action is executed in the main loop
+        // Wakeup mainloop if it's waiting for events
+        Application.MainLoop.Wakeup ();
     }
 
     /// <inheritdoc />

Terminal.Gui/Drawing/Sixel/SixelSupportDetector.cs

@@ -20,7 +20,7 @@ public class SixelSupportDetector
     public void Detect (Action<SixelSupportResult> resultCallback)
     {
         var result = new SixelSupportResult ();
-        result.SupportsTransparency = IsWindowsTerminal () || IsXtermWithTransparency ();
+        result.SupportsTransparency = IsVirtualTerminal () || IsXtermWithTransparency ();
         IsSixelSupportedByDar (result, resultCallback);
     }
 
@@ -142,7 +142,7 @@ private static void QueueRequest (AnsiEscapeSequence req, Action<string> respons
 
     private static bool ResponseIndicatesSupport (string response) { return response.Split (';').Contains ("4"); }
 
-    private static bool IsWindowsTerminal ()
+    private static bool IsVirtualTerminal ()
     {
         return !string.IsNullOrWhiteSpace (Environment.GetEnvironmentVariable ("WT_SESSION"));
 

Terminal.Gui/Drivers/ConsoleDriver.cs

@@ -269,7 +269,7 @@ public void AddRune (Rune rune)
                         if (Contents [Row, Col - 1].Rune.GetColumns () > 1)
                         {
                             // Invalidate cell to left
-                            Contents [Row, Col - 1].Rune = Rune.ReplacementChar;
+                            Contents [Row, Col - 1].Rune = (Rune)'\0';
                             Contents [Row, Col - 1].IsDirty = true;
                         }
                     }
@@ -308,7 +308,7 @@ public void AddRune (Rune rune)
                             {
                                 // Invalidate cell to right so that it doesn't get drawn
                                 // TODO: Figure out if it is better to show a replacement character or ' '
-                                Contents [Row, Col + 1].Rune = Rune.ReplacementChar;
+                                Contents [Row, Col + 1].Rune = (Rune)'\0';
                                 Contents [Row, Col + 1].IsDirty = true;
                             }
                         }

Terminal.Gui/Drivers/V2/ConsoleDriverFacade.cs

@@ -28,6 +28,15 @@ IWindowSizeMonitor windowSizeMonitor
         _outputBuffer = outputBuffer;
         _ansiRequestScheduler = ansiRequestScheduler;
 
+        if (InputProcessor is WindowsInputProcessor)
+        {
+            SupportsTrueColor = new WindowsInput ().IsVirtualTerminal ();
+        }
+        else if (InputProcessor is NetInputProcessor)
+        {
+            SupportsTrueColor = Application.Driver?.SupportsTrueColor == true;
+        }
+
         InputProcessor.KeyDown += (s, e) => KeyDown?.Invoke (s, e);
         InputProcessor.KeyUp += (s, e) => KeyUp?.Invoke (s, e);
         InputProcessor.MouseEvent += (s, e) =>
@@ -145,7 +154,7 @@ public int Top
     // TODO: Probably not everyone right?
 
     /// <summary>Gets whether the <see cref="ConsoleDriver"/> supports TrueColor output.</summary>
-    public bool SupportsTrueColor => true;
+    public bool SupportsTrueColor { get; init; } = true;
 
     // TODO: Currently ignored
     /// <summary>

Terminal.Gui/Drivers/V2/IConsoleOutput.cs

@@ -26,6 +26,13 @@ public interface IConsoleOutput : IDisposable
     /// <returns></returns>
     public Size GetWindowSize ();
 
+    /// <summary>
+    ///     Sets the current size of the console window in rows/columns
+    /// </summary>
+    /// <param name="newSize"></param>
+    /// /// <returns></returns>
+    public Size SetWindowSize (Size newSize);
+
     /// <summary>
     ///     Updates the console cursor (the blinking underscore) to be hidden,
     ///     visible etc.

Terminal.Gui/Drivers/V2/MainLoopCoordinator.cs

@@ -25,7 +25,6 @@ internal class MainLoopCoordinator<T> : IMainLoopCoordinator
     private ConsoleDriverFacade<T> _facade;
     private Task _inputTask;
     private readonly ITimedEvents _timedEvents;
-    private readonly bool _isWindowsTerminal;
 
     private readonly SemaphoreSlim _startupSemaphore = new (0, 1);
 
@@ -61,7 +60,6 @@ IMainLoop<T> loop
         _inputProcessor = inputProcessor;
         _outputFactory = outputFactory;
         _loop = loop;
-        _isWindowsTerminal = Environment.GetEnvironmentVariable ("WT_SESSION") is { } || Environment.GetEnvironmentVariable ("VSAPPIDNAME") != null;
     }
 
     /// <summary>
@@ -162,7 +160,15 @@ private void BuildFacadeIfPossible ()
                            _loop.AnsiRequestScheduler,
                            _loop.WindowSizeMonitor);
 
-            if (!_isWindowsTerminal)
+            if (_facade.SupportsTrueColor)
+            {
+                if (!ConsoleDriver.RunningUnitTests)
+                {
+                    // Enable alternative screen buffer.
+                    Console.Out.Write (EscSeqUtils.CSI_SaveCursorAndActivateAltBufferNoBackscroll);
+                }
+            }
+            else
             {
                 Application.Force16Colors = _facade.Force16Colors = true;
             }
@@ -187,6 +193,13 @@ public void Stop ()
         _stopCalled = true;
 
         _tokenSource.Cancel ();
+
+        if (!ConsoleDriver.RunningUnitTests && _facade.SupportsTrueColor)
+        {
+            // Disable alternative screen buffer.
+            Console.Out.Write (EscSeqUtils.CSI_RestoreCursorAndRestoreAltBufferWithBackscroll);
+        }
+
         _output.Dispose ();
 
         // Wait for input infinite loop to exit

Terminal.Gui/Drivers/V2/NetOutput.cs

@@ -213,6 +213,12 @@ public Size GetWindowSize ()
         return new (Console.WindowWidth, Console.WindowHeight);
     }
 
+    /// <inheritdoc />
+    public Size SetWindowSize (Size newSize)
+    {
+        return newSize;
+    }
+
     private void WriteToConsole (StringBuilder output, ref int lastCol, int row, ref int outputWidth)
     {
         SetCursorPositionImpl (lastCol, row);

Terminal.Gui/Drivers/V2/OutputBuffer.cs

@@ -256,7 +256,7 @@ public void AddRune (Rune rune)
                             {
                                 // Invalidate cell to right so that it doesn't get drawn
                                 // TODO: Figure out if it is better to show a replacement character or ' '
-                                Contents [Row, Col + 1].Rune = Rune.ReplacementChar;
+                                Contents [Row, Col + 1].Rune = (Rune)'\0';
                                 Contents [Row, Col + 1].IsDirty = true;
                             }
                         }

Terminal.Gui/Drivers/V2/WindowSizeMonitor.cs

@@ -30,9 +30,16 @@ public bool Poll ()
         if (size != _lastSize)
         {
             Logging.Logger.LogInformation ($"Console size changes from '{_lastSize}' to {size}");
-            _outputBuffer.SetWindowSize (size.Width, size.Height);
-            _lastSize = size;
-            SizeChanging?.Invoke (this, new (size));
+            Size newSize = size;
+
+            if (_consoleOut.GetType().Name == "WindowsOutput")
+            {
+                newSize = _consoleOut.SetWindowSize (size);
+            }
+
+            _outputBuffer.SetWindowSize (newSize.Width, newSize.Height);
+            _lastSize = newSize;
+            SizeChanging?.Invoke (this, new (newSize));
 
             return true;
         }

Terminal.Gui/Drivers/V2/WindowsInput.cs

@@ -56,6 +56,12 @@ public WindowsInput ()
         SetConsoleMode (_inputHandle, newConsoleMode);
     }
 
+    internal bool IsVirtualTerminal ()
+    {
+        nint outputHandle = GetStdHandle (STD_OUTPUT_HANDLE);
+        return GetConsoleMode (outputHandle, out uint mode) && (mode & (uint)ConsoleModes.EnableVirtualTerminalProcessing) != 0;
+    }
+
     protected override bool Peek ()
     {
         const int bufferSize = 1; // We only need to check if there's at least one event

Terminal.Gui/Drivers/V2/WindowsInputProcessor.cs

@@ -15,6 +15,45 @@ internal class WindowsInputProcessor : InputProcessor<InputRecord>
     /// <inheritdoc/>
     public WindowsInputProcessor (ConcurrentQueue<InputRecord> inputBuffer) : base (inputBuffer, new WindowsKeyConverter ()) { }
 
+    internal char _highSurrogate = '\0';
+
+    internal bool IsValidInput (Key key, out Key result)
+    {
+        result = key;
+
+        if (char.IsHighSurrogate ((char)key))
+        {
+            _highSurrogate = (char)key;
+
+            return false;
+        }
+
+        if (_highSurrogate > 0 && char.IsLowSurrogate ((char)key))
+        {
+            result = (KeyCode)new Rune (_highSurrogate, (char)key).Value;
+            _highSurrogate = '\0';
+
+            return true;
+        }
+
+        if (char.IsSurrogate ((char)key))
+        {
+            return false;
+        }
+
+        if (_highSurrogate > 0)
+        {
+            _highSurrogate = '\0';
+        }
+
+        if (key.KeyCode == 0)
+        {
+            return false;
+        }
+
+        return true;
+    }
+
     /// <inheritdoc/>
     protected override void Process (InputRecord inputEvent)
     {
@@ -71,7 +110,7 @@ protected override void ProcessAfterParsing (InputRecord input)
     {
         var key = KeyConverter.ToKey (input);
 
-        if (key != (Key)0)
+        if (IsValidInput (key, out key))
         {
             OnKeyDown (key!);
             OnKeyUp (key!);