8313244: NM flags handling in configure process

Reviewed-by: clanger, jwaters, mbaesken, erikj

Author: ansteiner

make/autoconf/flags-other.m4

@@ -88,6 +88,16 @@ AC_DEFUN([FLAGS_SETUP_RCFLAGS],
   AC_SUBST(RCFLAGS)
 ])
 
+AC_DEFUN([FLAGS_SETUP_NMFLAGS],
+[
+  # On AIX, we need to set NM flag -X64 for processing 64bit object files
+  if test "x$OPENJDK_TARGET_OS" = xaix; then
+    NMFLAGS="-X64"
+  fi
+
+  AC_SUBST(NMFLAGS)
+])
+
 ################################################################################
 # platform independent
 AC_DEFUN([FLAGS_SETUP_ASFLAGS],

make/autoconf/flags.m4

@@ -429,6 +429,7 @@ AC_DEFUN([FLAGS_SETUP_FLAGS],
   FLAGS_SETUP_ARFLAGS
   FLAGS_SETUP_STRIPFLAGS
   FLAGS_SETUP_RCFLAGS
+  FLAGS_SETUP_NMFLAGS
 
   FLAGS_SETUP_ASFLAGS
   FLAGS_SETUP_ASFLAGS_CPU_DEP([TARGET])

make/autoconf/spec.gmk.in

@@ -606,6 +606,7 @@ AR := @AR@
 ARFLAGS:=@ARFLAGS@
 
 NM:=@NM@
+NMFLAGS:=@NMFLAGS@
 STRIP:=@STRIP@
 OBJDUMP:=@OBJDUMP@
 CXXFILT:=@CXXFILT@

make/common/NativeCompilation.gmk

@@ -48,12 +48,12 @@ define GetSymbols
         $(SED) -e 's/#.*//;s/global://;s/local://;s/\;//;s/^[ 	]*/_/;/^_$$$$/d' | \
         $(EGREP) -v "JNI_OnLoad|JNI_OnUnload|Agent_OnLoad|Agent_OnUnload|Agent_OnAttach" > \
         $$(@D)/$$(basename $$(@F)).symbols || true; \
-    $(NM) $$($1_TARGET) | $(GREP)  " T " | \
+    $(NM) $(NMFLAGS) $$($1_TARGET) | $(GREP)  " T " | \
         $(EGREP) "JNI_OnLoad|JNI_OnUnload|Agent_OnLoad|Agent_OnUnload|Agent_OnAttach" | \
         $(CUT) -d ' ' -f 3 >>  $$(@D)/$$(basename $$(@F)).symbols || true;\
   else \
     $(ECHO) "Getting symbols from nm"; \
-    $(NM) -m $$($1_TARGET) | $(GREP)  "__TEXT" | \
+    $(NM) $(NMFLAGS) -m $$($1_TARGET) | $(GREP)  "__TEXT" | \
         $(EGREP) -v "non-external|private extern|__TEXT,__eh_frame" | \
         $(SED) -e  's/.* //' > $$(@D)/$$(basename $$(@F)).symbols; \
   fi

make/hotspot/lib/CompileJvm.gmk

@@ -281,19 +281,13 @@ ifneq ($(GENERATE_COMPILE_COMMANDS_ONLY), true)
         #
     UNDEF_PATTERN := ' U '
 
-    # 'nm' on AIX needs -X64 option
-
-    ifeq ($(call isTargetOs, aix), true)
-      NM := $(NM) -X64
-    endif
-
     define SetupOperatorNewDeleteCheck
         $1.op_check: $1
 	  $$(call ExecuteWithLog, $1.op_check, \
-	      $$(NM) $$<  2>&1 | $$(GREP) $$(addprefix -e , $$(MANGLED_SYMS)) | $$(GREP) $$(UNDEF_PATTERN) > $1.op_check || true)
+	      $$(NM) $$(NMFLAGS) $$<  2>&1 | $$(GREP) $$(addprefix -e , $$(MANGLED_SYMS)) | $$(GREP) $$(UNDEF_PATTERN) > $1.op_check || true)
 	  if [ -s $1.op_check ]; then \
 	    $$(ECHO) "$$(notdir $$<): Error: Use of global operators new and delete is not allowed in Hotspot:"; \
-	    $$(NM) $$< | $$(CXXFILT) | $$(EGREP) '$$(DEMANGLED_REGEXP)' | $$(GREP) $$(UNDEF_PATTERN); \
+	    $$(NM) $$(NMFLAGS) $$< | $$(CXXFILT) | $$(EGREP) '$$(DEMANGLED_REGEXP)' | $$(GREP) $$(UNDEF_PATTERN); \
 	    $$(ECHO) "See: $$(TOPDIR)/make/hotspot/lib/CompileJvm.gmk"; \
 	    exit 1; \
 	  fi

make/hotspot/lib/JvmMapfile.gmk

@@ -53,7 +53,7 @@ endif
 # platform dependent.
 
 ifeq ($(call isTargetOs, linux), true)
-  DUMP_SYMBOLS_CMD := $(NM) --defined-only *$(OBJ_SUFFIX)
+  DUMP_SYMBOLS_CMD := $(NM) $(NMFLAGS) --defined-only *$(OBJ_SUFFIX)
   ifneq ($(FILTER_SYMBOLS_PATTERN), )
     FILTER_SYMBOLS_PATTERN := $(FILTER_SYMBOLS_PATTERN)|
   endif
@@ -67,7 +67,7 @@ ifeq ($(call isTargetOs, linux), true)
 else ifeq ($(call isTargetOs, macosx), true)
   # nm on macosx prints out "warning: nm: no name list" to stderr for
   # files without symbols. Hide this, even at the expense of hiding real errors.
-  DUMP_SYMBOLS_CMD := $(NM) -Uj *$(OBJ_SUFFIX) 2> /dev/null
+  DUMP_SYMBOLS_CMD := $(NM) $(NMFLAGS) -Uj *$(OBJ_SUFFIX) 2> /dev/null
   ifneq ($(FILTER_SYMBOLS_PATTERN), )
     FILTER_SYMBOLS_PATTERN := $(FILTER_SYMBOLS_PATTERN)|
   endif
@@ -89,7 +89,7 @@ else ifeq ($(call isTargetOs, aix), true)
   # which may be installed under /opt/freeware/bin. So better use an absolute path here!
   # NM=/usr/bin/nm
 
-  DUMP_SYMBOLS_CMD := $(NM) -X64 -B -C *$(OBJ_SUFFIX)
+  DUMP_SYMBOLS_CMD := $(NM) $(NMFLAGS) -B -C *$(OBJ_SUFFIX)
   FILTER_SYMBOLS_AWK_SCRIPT := \
       '{ \
         if (($$2="d" || $$2="D") && ($$3 ~ /^__vft/ || $$3 ~ /^gHotSpotVM/)) print $$3; \