On Windows, query console directly, not stdout.

On Windows, we must manually enable VT processing on the console. Prior to this
commit, we attempted to obtain a handle to the console by querying stdout. If
stdout has been redirected, however, then the handle doesn't correspond to the
console but instead to the redirected output. As a result, calls to enable VT
processing fail. By default, this results in `yansi` disabling coloring.

While this ordinarily doesn't have an impact on the expected functionality as
the console and stdout are one and the same, it disproportionately affects
running Rust unit tests that assume `yansi` coloring is in effect. This is
because Cargo captures stdout, redirecting it away from the console. This commit
resolves this issue, allowing unit tests to work as expected.

src/windows.rs

@@ -1,59 +1,95 @@
 #[cfg(windows)]
+#[allow(non_camel_case_types, non_snake_case)]
 mod windows_console {
     use core::ffi::c_void;
 
-    use crate::condition::CachedBool;
+    type c_ulong = u32;
+    type c_int = i32;
+    type wchar_t = u16;
 
-    #[allow(non_camel_case_types)] type c_ulong = u32;
-    #[allow(non_camel_case_types)] type c_int = i32;
     type DWORD = c_ulong;
     type LPDWORD = *mut DWORD;
     type HANDLE = *mut c_void;
     type BOOL = c_int;
+    type LPCWSTR = *const WCHAR;
+    type WCHAR = wchar_t;
+    type LPSECURITY_ATTRIBUTES = *mut SECURITY_ATTRIBUTES;
+    type LPVOID = *mut c_void;
+
+    #[repr(C)]
+    pub struct SECURITY_ATTRIBUTES {
+        pub nLength: DWORD,
+        pub lpSecurityDescriptor: LPVOID,
+        pub bInheritHandle: BOOL,
+    }
 
     const ENABLE_VIRTUAL_TERMINAL_PROCESSING: DWORD = 0x0004;
-    const STD_OUTPUT_HANDLE: DWORD = 0xFFFFFFF5;
-    const STD_ERROR_HANDLE: DWORD = 0xFFFFFFF4;
     const INVALID_HANDLE_VALUE: HANDLE = -1isize as HANDLE;
     const FALSE: BOOL = 0;
     const TRUE: BOOL = 1;
 
+    const GENERIC_READ: DWORD = 0x80000000;
+    const GENERIC_WRITE: DWORD = 0x40000000;
+
+    const FILE_SHARE_READ: DWORD = 0x00000001;
+    const FILE_SHARE_WRITE: DWORD = 0x00000002;
+    const OPEN_EXISTING: DWORD = 3;
+
     // This is the win32 console API, taken from the 'winapi' crate.
     extern "system" {
-        fn GetStdHandle(nStdHandle: DWORD) -> HANDLE;
+        fn CreateFileW(
+            lpFileName: LPCWSTR,
+            dwDesiredAccess: DWORD,
+            dwShareMode: DWORD,
+            lpSecurityAttributes: LPSECURITY_ATTRIBUTES,
+            dwCreationDisposition: DWORD,
+            dwFlagsAndAttributes: DWORD,
+            hTemplateFile: HANDLE
+        ) -> HANDLE;
+
+        fn GetLastError() -> DWORD;
         fn GetConsoleMode(hConsoleHandle: HANDLE, lpMode: LPDWORD) -> BOOL;
         fn SetConsoleMode(hConsoleHandle: HANDLE, dwMode: DWORD) -> BOOL;
     }
 
-    unsafe fn get_handle(handle_num: DWORD) -> Result<HANDLE, ()> {
-        match GetStdHandle(handle_num) {
-            handle if handle == INVALID_HANDLE_VALUE => Err(()),
-            handle => Ok(handle)
+    fn get_output_handle() -> Result<HANDLE, DWORD> {
+        // This is "CONOUT$\0" UTF-16 encoded.
+        const CONOUT: &[u16] = &[0x43, 0x4F, 0x4E, 0x4F, 0x55, 0x54, 0x24, 0x00];
+
+        let raw_handle = unsafe {
+            CreateFileW(
+                CONOUT.as_ptr(),
+                GENERIC_READ | GENERIC_WRITE,
+                FILE_SHARE_READ | FILE_SHARE_WRITE,
+                core::ptr::null_mut(),
+                OPEN_EXISTING,
+                0,
+                core::ptr::null_mut(),
+            )
+        };
+
+        if raw_handle == INVALID_HANDLE_VALUE {
+            return Err(6);
         }
+
+        Ok(raw_handle)
     }
 
-    unsafe fn enable_vt(handle: HANDLE) -> Result<(), ()> {
+    unsafe fn enable_vt(handle: HANDLE) -> Result<(), DWORD> {
         let mut dw_mode: DWORD = 0;
         if GetConsoleMode(handle, &mut dw_mode) == FALSE {
-            return Err(());
+            return Err(GetLastError());
         }
 
         dw_mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
         match SetConsoleMode(handle, dw_mode) {
             result if result == TRUE => Ok(()),
-            _ => Err(())
+            _ => Err(GetLastError())
         }
     }
 
-    unsafe fn enable_ansi_colors_raw() -> Result<bool, ()> {
-        let stdout_handle = get_handle(STD_OUTPUT_HANDLE)?;
-        let stderr_handle = get_handle(STD_ERROR_HANDLE)?;
-
-        enable_vt(stdout_handle)?;
-        if stdout_handle != stderr_handle {
-            enable_vt(stderr_handle)?;
-        }
-
+    unsafe fn enable_ansi_colors_raw() -> Result<bool, DWORD> {
+        enable_vt(get_output_handle()?)?;
         Ok(true)
     }
 
@@ -64,8 +100,9 @@ mod windows_console {
 
     // Try to enable colors on Windows, and try to do it at most once.
     pub fn cache_enable() -> bool {
-        static ENABLED: CachedBool = CachedBool::new();
+        use crate::condition::CachedBool;
 
+        static ENABLED: CachedBool = CachedBool::new();
         ENABLED.get_or_init(enable)
     }
 }