8346383: Cannot use DllMain in libdt_socket for static builds

Reviewed-by: dholmes, sspitsyn

Author: magicus

src/jdk.jdwp.agent/share/native/libdt_socket/socketTransport.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1345,6 +1345,10 @@ jdwpTransport_OnLoad(JavaVM *vm, jdwpTransportCallback* cbTablePtr,
     jvm = vm;
     callback = cbTablePtr;
 
+    if (dbgsysPlatformInit() != 0) {
+      return JNI_ERR;
+    }
+
     /* initialize interface table */
     interface.GetCapabilities = &socketTransport_getCapabilities;
     interface.Attach = &socketTransport_attach;

src/jdk.jdwp.agent/share/native/libdt_socket/sysSocket.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -38,6 +38,7 @@
 typedef int socklen_t;
 #endif
 
+int dbgsysPlatformInit();
 int dbgsysSocketClose(int fd);
 int dbgsysConnect(int fd, struct sockaddr *him, socklen_t len);
 int dbgsysFinishConnect(int fd, int timeout);

src/jdk.jdwp.agent/unix/native/libdt_socket/socket_md.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -39,6 +39,15 @@
 #include "socket_md.h"
 #include "sysSocket.h"
 
+/* Perform platform specific initialization.
+ * Returns 0 on success, non-0 on failure */
+int
+dbgsysPlatformInit()
+{
+    // Not needed on unix
+    return 0;
+}
+
 int
 dbgsysListen(int fd, int backlog) {
     return listen(fd, backlog);

src/jdk.jdwp.agent/windows/native/libdt_socket/socket_md.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,6 +22,7 @@
  * or visit www.oracle.com if you need additional information or have any
  * questions.
  */
+#include <stdlib.h>
 #include <windows.h>
 #include <winsock2.h>
 #include <ws2tcpip.h>
@@ -88,30 +89,21 @@ static struct {
     { WSA_OPERATION_ABORTED,    "Overlapped operation aborted" },
 };
 
+static void dbgsysAtExitCallback(void)
+{
+    WSACleanup();
+}
 
-/*
- * Initialize Windows Sockets API support
- */
-BOOL WINAPI
-DllMain(HINSTANCE hinst, DWORD reason, LPVOID reserved)
+/* Perform platform specific initialization.
+ * Returns 0 on success, non-0 on failure */
+int
+dbgsysPlatformInit()
 {
     WSADATA wsadata;
 
-    switch (reason) {
-        case DLL_PROCESS_ATTACH:
-            if (WSAStartup(MAKEWORD(2,2), &wsadata) != 0) {
-                return FALSE;
-            }
-            break;
-
-        case DLL_PROCESS_DETACH:
-            WSACleanup();
-            break;
+    atexit(dbgsysAtExitCallback);
 
-        default:
-            break;
-    }
-    return TRUE;
+    return WSAStartup(MAKEWORD(2,2), &wsadata);
 }
 
 /*