Make entry points/inferred Windows subsystem conform better with conventions

Previously, start.zig would always export wWinMainCRTStartup as the entry symbol, regardless of whether `main` or `wWinMain` was used as the main. This meant that the linker was unable to differentiate between main/wWinMain and so anything with wWinMainCRTStartup would get the `.console` subsystem inferred. In other words, the added test would fail for `winmain_inferred` as it would have the `.console` subsystem instead of the expected `.windows` subsystem.

This commit changes the relevant logic:

- Only export wWinMainCRTStartup when using wWinMain
- Export wmainCRTStartup for non-wWinMain main functions when libc is not linked
- When libc is linked, then the libc runtime entry point is used and `main` is exported (this part hasn't changed)
- Infer `.windows` subsystem when exporting WinMainCRTStartup/wWinMainCRTStartup/WinMain/wWinMain
- Infer `.console` subsystem when exporting main/mainCRTStartup/wmainCRTStartup

This is a breaking change in the sense that a compiler built before this commit will be unable to compile a non-wWinMain exe using the updated `start.zig`. This is because the previous code did not look for mainCRTStartup/wmainCRTStartup at all, so it'd fall back to assuming the entry point is wWinMainCRTStartup which is no longer exported when using a normal main and therefore hit `error: lld-link: <root>: undefined symbol: wWinMainCRTStartup`

lib/std/start.zig

@@ -37,8 +37,10 @@ comptime {
                     @export(&main2, .{ .name = "main" });
                 }
             } else if (builtin.os.tag == .windows) {
-                if (!@hasDecl(root, "wWinMainCRTStartup") and !@hasDecl(root, "mainCRTStartup")) {
+                if (!@hasDecl(root, "wWinMainCRTStartup") and !@hasDecl(root, "WinMainCRTStartup") and
-                    @export(&wWinMainCRTStartup2, .{ .name = "wWinMainCRTStartup" });
+                    !@hasDecl(root, "mainCRTStartup") and !@hasDecl(root, "wmainCRTStartup"))
+                {
+                    @export(&WinStartup2, .{ .name = "wmainCRTStartup" });
                 }
             } else if (builtin.os.tag == .opencl or builtin.os.tag == .vulkan) {
                 if (@hasDecl(root, "main"))
@@ -65,7 +67,7 @@ comptime {
                 if (!@hasDecl(root, "WinMain") and !@hasDecl(root, "WinMainCRTStartup") and
                     !@hasDecl(root, "wWinMain") and !@hasDecl(root, "wWinMainCRTStartup"))
                 {
-                    @export(&WinStartup, .{ .name = "wWinMainCRTStartup" });
+                    @export(&WinStartup, .{ .name = "wmainCRTStartup" });
                 } else if (@hasDecl(root, "WinMain") and !@hasDecl(root, "WinMainCRTStartup") and
                     !@hasDecl(root, "wWinMain") and !@hasDecl(root, "wWinMainCRTStartup"))
                 {
@@ -113,7 +115,7 @@ fn spirvMain2() callconv(.kernel) void {
     root.main();
 }
 
-fn wWinMainCRTStartup2() callconv(.c) noreturn {
+fn WinStartup2() callconv(.c) noreturn {
     std.posix.exit(callMain());
 }
 

src/codegen/llvm.zig

@@ -1642,6 +1642,8 @@ pub const Object = struct {
                 if (name.eqlSlice("main", ip)) flags.c_main = true;
                 if (name.eqlSlice("WinMain", ip)) flags.winmain = true;
                 if (name.eqlSlice("wWinMain", ip)) flags.wwinmain = true;
+                if (name.eqlSlice("mainCRTStartup", ip)) flags.main_crt_startup = true;
+                if (name.eqlSlice("wmainCRTStartup", ip)) flags.wmain_crt_startup = true;
                 if (name.eqlSlice("WinMainCRTStartup", ip)) flags.winmain_crt_startup = true;
                 if (name.eqlSlice("wWinMainCRTStartup", ip)) flags.wwinmain_crt_startup = true;
                 if (name.eqlSlice("DllMainCRTStartup", ip)) flags.dllmain_crt_startup = true;

src/link/Lld.zig

@@ -25,6 +25,8 @@ const Coff = struct {
         c_main: bool,
         winmain: bool,
         wwinmain: bool,
+        main_crt_startup: bool,
+        wmain_crt_startup: bool,
         winmain_crt_startup: bool,
         wwinmain_crt_startup: bool,
         dllmain_crt_startup: bool,
@@ -64,6 +66,8 @@ const Coff = struct {
                 .c_main = false,
                 .winmain = false,
                 .wwinmain = false,
+                .main_crt_startup = false,
+                .wmain_crt_startup = false,
                 .winmain_crt_startup = false,
                 .wwinmain_crt_startup = false,
                 .dllmain_crt_startup = false,
@@ -562,12 +566,14 @@ fn coffLink(lld: *Lld, arena: Allocator) !void {
                         if (coff.lld_export_flags.dllmain_crt_startup or is_dyn_lib)
                             break :blk null;
                         if (coff.lld_export_flags.c_main or comp.config.is_test or
-                            coff.lld_export_flags.winmain_crt_startup or
+                            coff.lld_export_flags.main_crt_startup or
-                            coff.lld_export_flags.wwinmain_crt_startup)
+                            coff.lld_export_flags.wmain_crt_startup)
                         {
                             break :blk .console;
                         }
-                        if (coff.lld_export_flags.winmain or coff.lld_export_flags.wwinmain)
+                        if (coff.lld_export_flags.winmain or coff.lld_export_flags.wwinmain or
+                            coff.lld_export_flags.winmain_crt_startup or
+                            coff.lld_export_flags.wwinmain_crt_startup)
                             break :blk .windows;
                     }
                 },
@@ -660,13 +666,19 @@ fn coffLink(lld: *Lld, arena: Allocator) !void {
                     try argv.append("-NODEFAULTLIB");
                     if (!is_lib and entry_name == null) {
                         if (comp.zcu != null) {
-                            if (coff.lld_export_flags.winmain_crt_startup) {
+                            if (coff.lld_export_flags.wwinmain_crt_startup) {
-                                try argv.append("-ENTRY:WinMainCRTStartup");
-                            } else {
                                 try argv.append("-ENTRY:wWinMainCRTStartup");
+                            } else if (coff.lld_export_flags.winmain_crt_startup) {
+                                try argv.append("-ENTRY:WinMainCRTStartup");
+                            } else if (coff.lld_export_flags.wmain_crt_startup) {
+                                try argv.append("-ENTRY:wmainCRTStartup");
+                            } else if (coff.lld_export_flags.main_crt_startup) {
+                                try argv.append("-ENTRY:mainCRTStartup");
+                            } else {
+                                try argv.append("-ENTRY:wmainCRTStartup");
                             }
                         } else {
-                            try argv.append("-ENTRY:wWinMainCRTStartup");
+                            try argv.append("-ENTRY:wmainCRTStartup");
                         }
                     }
                 }

test/standalone/build.zig.zon

@@ -120,6 +120,9 @@
         .windows_spawn = .{
             .path = "windows_spawn",
         },
+        .windows_subsystem = .{
+            .path = "windows_subsystem",
+        },
         .windows_argv = .{
             .path = "windows_argv",
         },

test/standalone/windows_subsystem/build.zig

@@ -0,0 +1,169 @@
+const std = @import("std");
+const builtin = @import("builtin");
+
+pub fn build(b: *std.Build) void {
+    const test_step = b.step("test", "Test it");
+    b.default_step = test_step;
+
+    const optimize: std.builtin.OptimizeMode = .Debug;
+    const target = b.graph.host;
+
+    if (builtin.os.tag != .windows) return;
+
+    const expected_console_subsystem = std.fmt.comptimePrint("{}\n", .{@intFromEnum(std.coff.Subsystem.WINDOWS_CUI)});
+    const expected_windows_subsystem = std.fmt.comptimePrint("{}\n", .{@intFromEnum(std.coff.Subsystem.WINDOWS_GUI)});
+
+    // Normal Zig main, no libc linked
+    {
+        const main_mod = b.createModule(.{
+            .root_source_file = b.path("main.zig"),
+            .optimize = optimize,
+            .target = target,
+        });
+
+        const main_inferred = b.addExecutable(.{
+            .name = "main_inferred",
+            .root_module = main_mod,
+        });
+        const run_inferred = b.addRunArtifact(main_inferred);
+        run_inferred.expectStdErrEqual(expected_console_subsystem);
+        run_inferred.expectExitCode(0);
+        test_step.dependOn(&run_inferred.step);
+
+        const main_console = b.addExecutable(.{
+            .name = "main_console",
+            .root_module = main_mod,
+        });
+        main_console.subsystem = .console;
+        const run_console = b.addRunArtifact(main_console);
+        run_console.expectStdErrEqual(expected_console_subsystem);
+        run_console.expectExitCode(0);
+        test_step.dependOn(&run_console.step);
+
+        const main_windows = b.addExecutable(.{
+            .name = "main_windows",
+            .root_module = main_mod,
+        });
+        main_windows.subsystem = .windows;
+        const run_windows = b.addRunArtifact(main_windows);
+        run_windows.expectStdErrEqual(expected_windows_subsystem);
+        run_windows.expectExitCode(0);
+        test_step.dependOn(&run_windows.step);
+    }
+
+    // Normal Zig main, libc linked
+    {
+        const main_link_libc_mod = b.createModule(.{
+            .root_source_file = b.path("main.zig"),
+            .optimize = optimize,
+            .target = target,
+            .link_libc = true,
+        });
+
+        const main_link_libc_inferred = b.addExecutable(.{
+            .name = "main_link_libc_inferred",
+            .root_module = main_link_libc_mod,
+        });
+        const run_inferred = b.addRunArtifact(main_link_libc_inferred);
+        run_inferred.expectStdErrEqual(expected_console_subsystem);
+        run_inferred.expectExitCode(0);
+        test_step.dependOn(&run_inferred.step);
+
+        const main_link_libc_console = b.addExecutable(.{
+            .name = "main_link_libc_console",
+            .root_module = main_link_libc_mod,
+        });
+        main_link_libc_console.subsystem = .console;
+        const run_console = b.addRunArtifact(main_link_libc_console);
+        run_console.expectStdErrEqual(expected_console_subsystem);
+        run_console.expectExitCode(0);
+        test_step.dependOn(&run_console.step);
+
+        const main_link_libc_windows = b.addExecutable(.{
+            .name = "main_link_libc_windows",
+            .root_module = main_link_libc_mod,
+        });
+        main_link_libc_windows.subsystem = .windows;
+        const run_windows = b.addRunArtifact(main_link_libc_windows);
+        run_windows.expectStdErrEqual(expected_windows_subsystem);
+        run_windows.expectExitCode(0);
+        test_step.dependOn(&run_windows.step);
+    }
+
+    // wWinMain
+    {
+        const winmain_mod = b.createModule(.{
+            .root_source_file = b.path("winmain.zig"),
+            .optimize = optimize,
+            .target = target,
+        });
+
+        const winmain_inferred = b.addExecutable(.{
+            .name = "winmain_inferred",
+            .root_module = winmain_mod,
+        });
+        const run_inferred = b.addRunArtifact(winmain_inferred);
+        run_inferred.expectStdErrEqual(expected_windows_subsystem);
+        run_inferred.expectExitCode(0);
+        test_step.dependOn(&run_inferred.step);
+
+        const winmain_console = b.addExecutable(.{
+            .name = "winmain_console",
+            .root_module = winmain_mod,
+        });
+        winmain_console.subsystem = .console;
+        const run_console = b.addRunArtifact(winmain_console);
+        run_console.expectStdErrEqual(expected_console_subsystem);
+        run_console.expectExitCode(0);
+        test_step.dependOn(&run_console.step);
+
+        const winmain_windows = b.addExecutable(.{
+            .name = "winmain_windows",
+            .root_module = winmain_mod,
+        });
+        winmain_windows.subsystem = .windows;
+        const run_windows = b.addRunArtifact(winmain_windows);
+        run_windows.expectStdErrEqual(expected_windows_subsystem);
+        run_windows.expectExitCode(0);
+        test_step.dependOn(&run_windows.step);
+    }
+
+    // exported callconv(.c) main, libc must be linked
+    {
+        const cmain_mod = b.createModule(.{
+            .root_source_file = b.path("cmain.zig"),
+            .optimize = optimize,
+            .target = target,
+            .link_libc = true,
+        });
+
+        const cmain_inferred = b.addExecutable(.{
+            .name = "cmain_inferred",
+            .root_module = cmain_mod,
+        });
+        const run_inferred = b.addRunArtifact(cmain_inferred);
+        run_inferred.expectStdErrEqual(expected_console_subsystem);
+        run_inferred.expectExitCode(0);
+        test_step.dependOn(&run_inferred.step);
+
+        const cmain_console = b.addExecutable(.{
+            .name = "cmain_console",
+            .root_module = cmain_mod,
+        });
+        cmain_console.subsystem = .console;
+        const run_console = b.addRunArtifact(cmain_console);
+        run_console.expectStdErrEqual(expected_console_subsystem);
+        run_console.expectExitCode(0);
+        test_step.dependOn(&run_console.step);
+
+        const cmain_windows = b.addExecutable(.{
+            .name = "cmain_windows",
+            .root_module = cmain_mod,
+        });
+        cmain_windows.subsystem = .windows;
+        const run_windows = b.addRunArtifact(cmain_windows);
+        run_windows.expectStdErrEqual(expected_windows_subsystem);
+        run_windows.expectExitCode(0);
+        test_step.dependOn(&run_windows.step);
+    }
+}

test/standalone/windows_subsystem/cmain.zig

@@ -0,0 +1,6 @@
+const std = @import("std");
+
+pub export fn main() callconv(.c) c_int {
+    std.debug.print("{}\n", .{std.os.windows.peb().ImageSubSystem});
+    return 0;
+}

test/standalone/windows_subsystem/main.zig

@@ -0,0 +1,5 @@
+const std = @import("std");
+
+pub fn main() void {
+    std.debug.print("{}\n", .{std.os.windows.peb().ImageSubSystem});
+}

test/standalone/windows_subsystem/winmain.zig

@@ -0,0 +1,15 @@
+const std = @import("std");
+
+pub fn wWinMain(
+    inst: std.os.windows.HINSTANCE,
+    prev: ?std.os.windows.HINSTANCE,
+    cmd_line: std.os.windows.LPWSTR,
+    cmd_show: c_int,
+) std.os.windows.INT {
+    _ = inst;
+    _ = prev;
+    _ = cmd_line;
+    _ = cmd_show;
+    std.debug.print("{}\n", .{std.os.windows.peb().ImageSubSystem});
+    return 0;
+}