Fix illumox-x64 build with gcc and clang

eng/native/configurecompiler.cmake

@@ -336,7 +336,12 @@ elseif(CLR_CMAKE_HOST_OPENBSD)
   # The PAL's hand-written asm lacks endbr64 landing pads, so disable branch-target CFI.
   add_linker_flag("-Wl,-z,nobtcfi")
 elseif(CLR_CMAKE_HOST_SUNOS)
-  add_compile_options($<$<COMPILE_LANGUAGE:ASM>:-Wa,--noexecstack>)
+  # llvm's assembler crashes with noexecstack with solaris triplet;
+  # bug report with minimal repro at: https://github.com/llvm/llvm-project/issues/202953
+  if (CMAKE_C_COMPILER_ID MATCHES "GNU")
+    add_compile_options($<$<COMPILE_LANGUAGE:ASM>:-Wa,--noexecstack>)
+  endif()
+
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fstack-protector")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fstack-protector")
   add_definitions(-D__EXTENSIONS__ -D_XPG4_2 -D_POSIX_PTHREAD_SEMANTICS -D_REENTRANT)
@@ -382,15 +387,17 @@ elseif(CLR_CMAKE_HOST_HAIKU)
     set(CMAKE_LINK_LIBRARY_USING_WHOLE_ARCHIVE_SUPPORTED TRUE)
     set(CMAKE_LINK_LIBRARY_WHOLE_ARCHIVE_ATTRIBUTES LIBRARY_TYPE=STATIC DEDUPLICATION=YES OVERRIDE=DEFAULT)
   endif()
-  # Haiku uses the GNU toolchain, which supports RESCAN, but only CMake 4.4.0+ recognizes this.
-  if(CMAKE_VERSION VERSION_LESS 4.4)
-    set(CMAKE_LINK_GROUP_USING_RESCAN "LINKER:--start-group" "LINKER:--end-group")
-    set(CMAKE_LINK_GROUP_USING_RESCAN_SUPPORTED TRUE)
-  endif()
+
   add_compile_options($<$<COMPILE_LANGUAGE:ASM>:-Wa,--noexecstack>)
   add_linker_flag("-Wl,--build-id=sha1")
 endif()
 
+if((CLR_CMAKE_HOST_HAIKU OR CLR_CMAKE_HOST_SUNOS) AND CMAKE_VERSION VERSION_LESS 4.4)
+  # Haiku and illumos uses the GNU toolchain, which supports RESCAN, but only CMake 4.4.0+ recognizes this.
+  set(CMAKE_LINK_GROUP_USING_RESCAN "LINKER:--start-group" "LINKER:--end-group")
+  set(CMAKE_LINK_GROUP_USING_RESCAN_SUPPORTED TRUE)
+endif()
+
 #------------------------------------
 # Definitions (for platform)
 #-----------------------------------

src/coreclr/gc/unix/gcenv.unix.cpp

@@ -568,7 +568,7 @@ bool GCToOSInterface::VirtualReset(void * address, size_t size, bool unlock)
     st = madvise(address, size, MADV_DONTDUMP);
 #endif
 
-#ifdef MADV_FREE
+#if defined(MADV_FREE) && !defined(TARGET_SUNOS)
     // Tell the kernel that the application doesn't need the pages in the range.
     // Freeing the pages can be delayed until a memory pressure occurs.
     st = madvise(address, size, MADV_FREE);

src/coreclr/jit/gentree.h

@@ -4534,7 +4534,7 @@ struct AsyncCallInfo
     // Behavior where we continue for each call depends on how it was
     // configured and whether it is a task await or custom await. This field
     // records that behavior.
-    ContinuationContextHandling ContinuationContextHandling = ContinuationContextHandling::None;
+    ::ContinuationContextHandling ContinuationContextHandling = ContinuationContextHandling::None;
 
     // Tail awaits do not generate suspension points and the JIT instead
     // directly returns the callee's continuation to the caller.
@@ -8096,8 +8096,8 @@ struct GenTreeIndir : public GenTreeOp
     unsigned Scale();
     ssize_t  Offset();
 
-    unsigned  Size() const;
-    ValueSize ValueSize() const;
+    unsigned    Size() const;
+    ::ValueSize ValueSize() const;
 
     GenTreeIndir(genTreeOps oper, var_types type, GenTree* addr, GenTree* data)
         : GenTreeOp(oper, type, addr, data)

src/native/libs/System.Native/pal_io.c

@@ -1865,8 +1865,8 @@ int32_t SystemNative_ReadThreadInfo(int32_t pid, int32_t tid, ThreadInfo* thread
 
     lwpsinfo_t pr;
     int result = Common_Read(fd, &pr, sizeof(pr));
-    close(fd);
-    if (result < sizeof (pr))
+    close(ToFileDescriptor(fd));
+    if (result < (int)sizeof(pr))
     {
         errno = EIO;
         return -1;
@@ -1913,8 +1913,8 @@ int32_t SystemNative_ReadProcessInfo(int32_t pid, ProcessInfo* processInfo, uint
 
     psinfo_t pr;
     int result = Common_Read(fd, &pr, sizeof(pr));
-    close(fd);
-    if (result < sizeof (pr))
+    close(ToFileDescriptor(fd));
+    if (result < (int)sizeof(pr))
     {
         errno = EIO;
         return -1;

src/native/libs/System.Native/pal_process.c

@@ -225,7 +225,7 @@ handler_from_sigaction (struct sigaction *sa)
     }
     else
     {
-        return sa->sa_handler;
+        return (VoidIntFn)sa->sa_handler;
     }
 }
 
@@ -838,7 +838,10 @@ static int32_t ForkAndExecProcessInternal(
         struct sigaction sa_default;
         struct sigaction sa_old;
         memset(&sa_default, 0, sizeof(sa_default)); // On some architectures, sa_mask is a struct so assigning zero to it doesn't compile
+    #pragma clang diagnostic push
+    #pragma clang diagnostic ignored "-Wstrict-prototypes"
         sa_default.sa_handler = SIG_DFL;
+    #pragma clang diagnostic pop
         for (int sig = 1; sig < NSIG; ++sig)
         {
             if (sig == SIGKILL || sig == SIGSTOP)
@@ -848,7 +851,12 @@ static int32_t ForkAndExecProcessInternal(
             if (!sigaction(sig, NULL, &sa_old))
             {
                 void (*oldhandler)(int) = handler_from_sigaction (&sa_old);
-                if (oldhandler != SIG_IGN && oldhandler != SIG_DFL)
+                bool hasCustomHandler;
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wstrict-prototypes"
+                hasCustomHandler = oldhandler != SIG_IGN && oldhandler != SIG_DFL;
+#pragma clang diagnostic pop
+                if (hasCustomHandler)
                 {
                     // It has a custom handler, put the default handler back.
                     // We check first to preserve flags on default handlers.

src/native/libs/System.Native/pal_signal.c

@@ -69,18 +69,28 @@ static bool IsSaSigInfo(struct sigaction* action)
 static bool IsSigDfl(struct sigaction* action)
 {
     assert(action);
+    bool isDefault;
     // macOS can return sigaction with SIG_DFL and SA_SIGINFO.
     // SA_SIGINFO means we should use sa_sigaction, but here we want to check sa_handler.
     // So we ignore SA_SIGINFO when sa_sigaction and sa_handler are at the same address.
-    return (&action->sa_handler == (void*)&action->sa_sigaction || !IsSaSigInfo(action)) &&
-            action->sa_handler == SIG_DFL;
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wstrict-prototypes"
+    isDefault = (&action->sa_handler == (void*)&action->sa_sigaction || !IsSaSigInfo(action)) &&
+                action->sa_handler == SIG_DFL;
+#pragma clang diagnostic pop
+    return isDefault;
 }
 
 static bool IsSigIgn(struct sigaction* action)
 {
     assert(action);
-    return (&action->sa_handler == (void*)&action->sa_sigaction || !IsSaSigInfo(action)) &&
-            action->sa_handler == SIG_IGN;
+    bool isIgnored;
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wstrict-prototypes"
+    isIgnored = (&action->sa_handler == (void*)&action->sa_sigaction || !IsSaSigInfo(action)) &&
+                action->sa_handler == SIG_IGN;
+#pragma clang diagnostic pop
+    return isIgnored;
 }
 
 bool TryConvertSignalCodeToPosixSignal(int signalCode, PosixSignal* posixSignal)
@@ -239,7 +249,7 @@ static void SignalHandler(int sig, siginfo_t* siginfo, void* context)
             else
             {
                 assert(origHandler->sa_handler);
-                origHandler->sa_handler(sig);
+                ((void (*)(int))origHandler->sa_handler)(sig);
             }
 
         }

src/native/libs/System.Native/pal_threading.c

@@ -259,10 +259,12 @@ void SystemNative_LowLevelFutex_WakeByAddressSingle(int32_t* address)
 }
 #else // defined(TARGET_LINUX)
 
-#ifdef DEBUG 
-#define DEBUGNOTRETURN __attribute__((noreturn)) 
-#else 
-#define DEBUGNOTRETURN 
+// On illumos/Solaris libc's assert is not annotated noreturn, so marking these stubs noreturn would
+// trigger -Winvalid-noreturn there. Only apply the attribute on other platforms.
+#if defined(DEBUG) && !defined(TARGET_SUNOS)
+#define DEBUGNOTRETURN __attribute__((noreturn))
+#else
+#define DEBUGNOTRETURN
 #endif
 
 DEBUGNOTRETURN
@@ -281,7 +283,7 @@ int32_t SystemNative_LowLevelFutex_WaitOnAddressTimeout(int32_t* address, int32_
     (void)comparand; // unused
     (void)timeoutMilliseconds; // unused
     assert_msg(false, "Futex is not supported on this platform", 0);
-#ifndef DEBUG
+#if !defined(DEBUG) || defined(TARGET_SUNOS)
     // trivial implementation of Wait always wakes spuriously.
     return 1;
 #endif
@@ -295,6 +297,8 @@ void SystemNative_LowLevelFutex_WakeByAddressSingle(int32_t* address)
     // trivial implementation of Wake does nothing.
 }
 
+#undef DEBUGNOTRETURN
+
 #endif  // defined(TARGET_LINUX)
 
 int32_t SystemNative_CreateThread(uintptr_t stackSize, void *(*startAddress)(void*), void *parameter)

src/native/libs/System.Native/pal_time.c

@@ -32,7 +32,7 @@ int32_t SystemNative_UTimensat(const char* path, TimeSpec* times)
 
     updatedTimes[1].tv_sec = (time_t)times[1].tv_sec;
     updatedTimes[1].tv_nsec = (long)times[1].tv_nsec;
-    while (CheckInterrupted(result = utimensat(AT_FDCWD, path, updatedTimes, AT_SYMLINK_NOFOLLOW)));
+    while (CheckInterrupted(result = utimensat((int)AT_FDCWD, path, updatedTimes, AT_SYMLINK_NOFOLLOW)));
 #else
     struct timeval updatedTimes[2];
     updatedTimes[0].tv_sec = (long)times[0].tv_sec;

src/native/libs/System.Native/pal_uid.c

@@ -108,7 +108,8 @@ static pthread_mutex_t s_groupLock = PTHREAD_MUTEX_INITIALIZER;
 #endif
 
 #if !HAVE_GETGROUPLIST
-int getgrouplist(const char *uname, gid_t agroup, gid_t *groups, int *groupCount)
+// Distinct name: avoids -Wmissing-prototypes (illumos) and clashing with grp.h's decl (Emscripten).
+static int pal_getgrouplist(const char *uname, gid_t agroup, gid_t *groups, int *groupCount)
 {
     int ngroups = 1;
     int maxgroups = *groupCount;
@@ -204,8 +205,10 @@ int32_t SystemNative_GetGroupList(const char* name, uint32_t group, uint32_t* gr
 #ifdef __APPLE__
         // On OSX groups are passed as a signed int.
         rv = getgrouplist(name, (int)group, (int*)groups, &groupsAvailable);
-#else
+#elif HAVE_GETGROUPLIST
         rv = getgrouplist(name, group, groups, &groupsAvailable);
+#else
+        rv = pal_getgrouplist(name, group, groups, &groupsAvailable);
 #endif
 
 #ifdef USE_GROUPLIST_LOCK