Customer is seeing VM crashes with the following stack trace during their application stress testing. The crash is seen with both 1.4.1_02 and
1.4.1_05.

=>[1] _lwp_kill(0x0, 0xa, 0x0, 0xff33c004, 0xff386000, 0xff340428), at 0xff31ef30
  [2] raise(0x6, 0x0, 0x0, 0xffffffff, 0xff3403b4, 0x0), at 0xff2cb9d4
  [3] abort(0xff33c004, 0xd64fdbc0, 0x0, 0x4, 0x0, 0xd64fdbe1), at 0xff2b58f4
  [4] os::abort(0x1, 0xff14fad6, 0xd64fdc60, 0x0, 0xff1d4ebc, 0xff080e7c), at 0xff082838
  [5] os::handle_unexpected_exception(0x1ac4a0, 0xb, 0xfee1451c, 0xd64fe9c0, 0xfedebac4, 0x0), at 0xff080eec
  [6] JVM_handle_solaris_signal(0xfee1451c, 0xd64fe9c0, 0xd64fe708, 0x4000, 0x416c, 0x0), at 0xfedec334
  [7] __sighndlr(0xb, 0xd64fe9c0, 0xd64fe708, 0xfedeba48, 0x0, 0x0), at 0xff374cc8
  [8] call_user_handler(0xfead1000, 0xa, 0xff3878e0, 0xd64fe708, 0xd64fe9c0, 0xb), at 0xff36fb00
  [9] sigacthandler(0xfead1000, 0xd64fe9c0, 0xd64fe708, 0xff386000, 0xd64fe9c0, 0xb), at 0xff36fccc
  ---- called from signal handler with signal 11 (SIGSEGV) ------
  [10] adjust_check(0x4de2bc, 0x3764a4, 0x5dd458, 0xff1d8da8, 0x0, 0xd64feff8), at 0xfee1451c
  [11] IfNode::Ideal(0x0, 0x0, 0xff18e000, 0xd64feff8, 0x1, 0x4ddda8), at 0xfed1053c
  [12] PhaseIterGVN::transform_old(0xd64feff8, 0x4e3a0c, 0x80, 0xd64ff148, 0x4, 0x507620), at 0xfecd0930
  [13] PhaseIterGVN::optimize(0xd64feff8, 0x0, 0xff1d5ef8, 0x0, 0x0, 0x0), at 0xfeda6d24
  [14] Compile::Optimize(0xd64ff540, 0xd64ff314, 0xd64ff454, 0x43fa50, 0xd64ff454, 0x0), at 0xfee170b0
  [15] Compile::Compile(0x5396d4, 0x2ab698, 0x0, 0x834fe8, 0xffffffff, 0x1), at 0xfee15a6c
  [16] C2Compiler::compile_method(0x2aff8, 0xd64ffd38, 0x0, 0x834fe8, 0xffffffff, 0x0), at 0xfee124a8
  [17] CompileBroker::invoke_compiler_on_method(0x267, 0x0, 0xffffffff, 0x1ac52c, 0xff1cd080, 0x1ac4a0), at 0
xfee11ce8
  [18] CompileBroker::compiler_thread_loop(0x1ac4a0, 0x1ac4a0, 0x1a79b8, 0x1aca40, 0x30beec, 0xfee81ffc), at
0xfeec958c
  [19] JavaThread::run(0x1ac4a0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfee82024
  [20] _start(0x1ac4a0, 0xfead1000, 0x0, 0x0, 0x0, 0x0), at 0xfee7e470

VM flags : 
JVM parameter      :  -server
JVM parameter      :  -Xss256k
JVM parameter      :  -Xms100m
JVM parameter      :  -Xmx512m
JVM parameter      :  -XX:SoftRefLRUPolicyMSPerMB=15000
JVM parameter      :  -XX:+OverrideDefaultLibthread
JVM parameter      :  -XX:+UseSignalChaining
JVM parameter      :  -XX:+UseParallelGC

The crash is not seen with client VM. 

use client VM.

Possible second work around:

All the cores show the crash when compiling:
Class: com/objy/pm/util/WeakKeyHashtable 
Method: put 
so add a .hotspot_compiler file containing the following directive:
exclude com/objy/pm/util/WeakKeyHashtable put
to see if that avoids the crash.

 xxxxx@xxxxx  2003-10-07

The crashes all occur here:
ifnode.cpp:
   436    // Else, adjust existing check
   436    // Else, adjust existing check
   437    Node *new_bol = gvn->transform( new (2) BoolNode( new_cmp, bol->is_Bool()->_test._test ) );

Analysis of core cvsm_core.sun4u.1442:
[1.4.1_02]
 xxxxx@xxxxx  ( xxxxx@xxxxx ) terminated by signal ABRT (Abort)
[tena/825384/cores:DBX] where
current thread:  xxxxx@xxxxx 
=>[1] 0xff31ee64(0x6, 0x0, 0x0, 0xffffffff, 0xff3403ac, 0x0), at 0xff31ee63
  [2] addsev(0xff33c000, 0xb64fdbe0, 0x0, 0x4, 0x0, 0xb64fdc01), at 0xff2b58e4
  [3] os::abort(0x1, 0xff14ce36, 0xb64fdc80, 0x0, 0xff1d0e8c, 0xff07f17c), at 0xff080a90
  [4] os::handle_unexpected_exception(0x2582c0, 0xb, 0xfee154f0, 0xb64fe9e0, 0xfedec9c4, 0x0), at 0xff07f1ec
  [5] JVM_handle_solaris_signal(0xfee154f0, 0xb64fe9e0, 0xb64fe728, 0x4000, 0x4164, 0x0), at 0xfeded234
  [6] __sighndlr(0xb, 0xb64fe9e0, 0xb64fe728, 0xfedec948, 0x0, 0x0), at 0xff374cc8
  [7] call_user_handler(0xfe7f1600, 0xd, 0xff3878e0, 0xb64fe728, 0xb64fe9e0, 0xb), at 0xff36fb00
  [8] sigacthandler(0xfe7f1600, 0xb64fe9e0, 0xb64fe728, 0xff386000, 0xb64fe9e0, 0xb), at 0xff36fccc
  ---- called from signal handler with signal -25225728 (SIG-25225728) ------
  [9] adjust_check(0x31dd9c, 0x7d97cc, 0x7648a0, 0xff1d4d78, 0x0, 0xb64feff8), at 0xfee154f0
  [10] IfNode::Ideal(0x0, 0x0, 0xff18a000, 0xb64feff8, 0x1, 0x31d888), at 0xfed10690
  [11] PhaseIterGVN::transform_old(0xb64feff8, 0x3234ec, 0x80, 0xb64ff148, 0x4, 0x247910), at 0xfecd0844
  [12] PhaseIterGVN::optimize(0xb64feff8, 0x0, 0xff1d1ec8, 0x0, 0x0, 0x0), at 0xfeda7dfc
  [13] Compile::Optimize(0xb64ff540, 0xb64ff314, 0xb64ff454, 0x3825f8, 0xb64ff454, 0x0), at 0xfee18084
  [14] Compile::Compile(0x97e274, 0x2d86f8, 0x0, 0xa86b78, 0xffffffff, 0x1), at 0xfee16a40
  [15] C2Compiler::compile_method(0x2b0c8, 0xb64ffd38, 0x0, 0xa86b78, 0xffffffff, 0x0), at 0xfee1347c
  [16] CompileBroker::invoke_compiler_on_method(0x2ac, 0x0, 0xffffffff, 0x25834c, 0xff1c907c, 0x2582c0), at 0xfee12cbc
  [17] CompileBroker::compiler_thread_loop(0x2582c0, 0x2582c0, 0x2548c8, 0x258860, 0x30603c, 0xfee83eac), at 0xfeecad58
  [18] JavaThread::run(0x2582c0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfee83ed4
  [19] _start(0x2582c0, 0xfe7f1600, 0x0, 0x0, 0x0, 0x0), at 0xfee80320

0xfee15154: adjust_check       :        save    %sp, -0x70, %sp
0xfee15158: adjust_check+0x0004:        ld      [%i0 + 0x4], %g2
...
0xfee154d0: adjust_check+0x037c:        st      %g3, [%g4 + 0xac]
0xfee154d4: adjust_check+0x0380:        addcc   %l4, 0x8, %l7
0xfee154d8: adjust_check+0x0384:        be,a    adjust_check+0x3c8
0xfee154dc: adjust_check+0x0388:        ld      [%i5], %g2
0xfee154e0: adjust_check+0x038c:        ld      [%l2], %g2
0xfee154e4: adjust_check+0x0390:        ld      [%g2 + 0x18], %l0
0xfee154e8: adjust_check+0x0394:        jmpl    %l0, %o7
0xfee154ec: adjust_check+0x0398:        mov     %l2, %o0
0xfee154f0: adjust_check+0x039c:        ld      [%o0 + 0x20], %l0
ifnode.s:
/* 0x0344        437 */         be,a,pt %icc,.L900000720
/* 0x0348            */         ld      [%i5],%g2
/* 0x034c            */         ld      [%l2],%g2
/* 0x0350            */         ld      [%g2+24],%l0
/* 0x0354            */         jmpl    %l0,%o7
/* 0x0358            */         or      %g0,%l2,%o0
/* 0x035c            */         or      %g0,%o0,%g2
/* 0x0360            */         or      %g0,%l7,%o0
/* 0x0364            */         ld      [%g2+32],%l0

ifnode.cpp:
   436    // Else, adjust existing check
   437    Node *new_bol = gvn->transform( new (2) BoolNode( new_cmp, bol->is_Bool()->_test._test ) );


[tena/825384/cores:DBX] frame 9
0xfee154f0: adjust_check+0x039c:        ld      [%o0 + 0x20], %l0
[tena/825384/cores:DBX] regs
current thread:  xxxxx@xxxxx 
current frame:  [9]
g0-g3    0x00000000 0x00005800 0xff1baf04 0x006f5558
g4-g7    0xb64ff540 0x00000000 0x00000000 0xfe7f1600
o0-o3    0x00000000 0x006f54a8 0x007648a0 0x007d97cc
o4-o7    0x0032391c 0x00000000 0xb64fea60 0xfee154e8
l0-l3    0xfedff4a0 0x00000000 0x007d3f0c 0x0031d888
l4-l7    0x006f552c 0xff18a000 0x006f54cc 0x006f5534
i0-i3    0x0031dd9c 0x007d97cc 0x007648a0 0xff1d4d78
i4-i7    0x00000000 0xb64feff8 0xb64fead0 0xfed10690
y        0x00000000
ccr      0x00000000
pc       0xfee154f0:adjust_check+0x39c  ld      [%o0 + 0x20], %l0
npc      0xfee154f4:adjust_check+0x3a0  mov     %l7, %o0
[tena/825384/cores:DBX] frame 10
0xfed10690: Ideal+0x02c4:       call    adjust_check
[tena/825384/cores:DBX] regs
current thread:  xxxxx@xxxxx 
current frame:  [10]
g0-g3    0x00000000 0x00005800 0xff1baf04 0x006f5558
g4-g7    0xb64ff540 0x00000000 0x00000000 0xfe7f1600
o0-o3    0x0031dd9c 0x007d97cc 0x007648a0 0xff1d4d78
o4-o7    0x00000000 0xb64feff8 0xb64fead0 0xfed10690
l0-l3    0xfecd2174 0x003234ec 0xb64feff8 0x0076b924
l4-l7    0x0031dd9c 0x0031dd9c 0x0031dd9c 0x00000007
i0-i3    0x00000000 0x00000000 0xff18a000 0xb64feff8
i4-i7    0x00000001 0x0031d888 0xb64feb50 0xfecd0844
y        0x00000000
ccr      0x00000000
pc       0xfed10690:Ideal+0x2c4 call    adjust_check
npc      0xfee154f4:adjust_check+0x3a0  mov     %l7, %o0
ifnode.s:
/* 0x02b0        649 */         ld      [%fp-4],%o1
/* 0x02b4            */         or      %g0,%l4,%o0
/* 0x02b8            */         or      %g0,%i4,%o3
/* 0x02bc            */         or      %g0,%i0,%o4
/* 0x02c0            */         or      %g0,%i3,%o5
/* 0x02c4            */         call    void adjust_check(Node*,Node*,Node*,int,int,PhaseIterGVN*)      ! params =  %o0 

ifnode.cpp:
   644      if( index1 ) {
   645        // Didn't find 2 prior covering checks, so cannot remove anything.
   646        if( !prev_chk2 ) return NULL;
   647        // 'Widen' the offsets of the 1st and 2nd covering check
   648        adjust_check( prev_chk1, range1, index1, flip1, off_lo, igvn );
   649        adjust_check( prev_chk2, range1, index1, flip1, off_hi, igvn );
   650        // Test is now covered by prior checks, dominate it out
   651        prev_dom = prev_chk2;

[tena/825384/cores:DBX] Get14C2methNClass
 
0xfee1347c: compile_method+0x0064:      call    Compile #Nvariant 1
Class: com/objy/pm/util/WeakKeyHashtable 
Method: put 
 
I have attached the short versions of data from the other 2 core files.

 xxxxx@xxxxx 


discussion of a proposed fix:

Yes, the additional restriction should fix this problem.

Here is the explanation from looking at adjust_check()'s call-sites
in IfNode::Ideal()

1) The problem parameters to adjust_check() are 'prev_chk1' and 'prev_chk2'

2) These are only given the values NULL and 'prev_dom'

3) prev_dom is only given the value of 'dom' or the initial 'this' pointer

4a) I initially suspected that prev_dom might not be a projection
    that points to an IfNode.  I've convinced myself that it is, even in
    the case that fails!
4b) The trick is the following two pieces of code in IfNode::Ideal()

           // If we match the test exactly, then the top test covers
           // both our lower and upper bounds.
           if( dom->in(1) == in(1) )
             prev_chk2 = prev_chk1;

    and at the end of adjust_check()

         // Else, adjust existing check
         Node *new_bol = gvn->transform( new (2) BoolNode( new_cmp, bol->is_Bool()->_test._test ) );
         igvn->hash_delete( iff );
         iff->set_req_X( 1, new_bol, igvn );

5a) Theory: both prev_chk1 and prev_chk2 are set to the same value
    by the code in IfNode::Ideal that checks for an exact match

5b) The code at the end of adjust_check() optimizes the BoolNode
    to a constant answer using BoolNode::Value()

5c) The second call to adjust_check() in IfNode::Ideal()

     if( index1 ) {
       // Didn't find 2 prior covering checks, so cannot remove anything.
       if( !prev_chk2 ) return NULL;
       // 'Widen' the offsets of the 1st and 2nd covering check
       adjust_check( prev_chk1, range1, index1, flip1, off_lo, igvn );
       adjust_check( prev_chk2, range1, index1, flip1, off_hi, igvn );

    is expecting prev_chk2 to point to an IfNode which has a canonical
    structure.  However, the canonical structure was modified by the
    first adjust_check() call since prev_chk1 == prev_chk2.

Alternate Fix:
    Do not call adjust_check() twice when prev_chk1 == prev_chk2


    Regards,
    Mike.




Chris Phillips - Member Technical Staff wrote:

> Hmmm - No response? 
> 
> Is there anyone out there? Maybe I should use the hs-compiler alias...
> 
> Additionally:
> 
> I am now thinking of trying the following simplistic extension of the change
> added  to fix bug 4780201 -
> ifnode.cpp:
> 
>    423    if( bol->is_top() ) return;   // In case a partially dead range check 
> appears
> to
>    423    if( bol->is_top() || !(bol->is_Bool())) return;   // In case a 
> partially dead range check or non bool input appears
> 
> Comments?
> 
> Chris
> 
> http://qtool.sfbay.sun.com/bin/esc_query.cgi?esc=548662
> http://sdn.sfbay.sun.com/cgi-bin/bug2html?4780201
> http://sdn.sfbay.sun.com/cgi-bin/bug2html?4933131
> http://loon.east:8888/altair/jpse/bugtraq/4933131/ifnode.cpp
> 
> ------------- Begin Forwarded Message -------------
> 
> Let me re-phrase the question...
> Given:
> [tena/825384/cores:DBX] frame 8
> 0xff36fccc: sigacthandler+0x0064:       call    call_user_handler
> 
> i0-i3    0xfead1000 0xd64fe9c0 0xd64fe708 0xff386000
>                     siginfo ptr
> [tena/825384/cores:DBX] x 0xd64fe9c0/4X 
> 0xd64fe9c0:      0x0000000b 0x00000001 0x00000000 0x00000020
> Faulting address:                                 __________
> So we faulted on a refernce to 0x20.
>                                                   
> 1 node.hpp    356 virtual BoolNode *is_Bool ()  { return 0; }
> 2 subnode.hpp 256 virtual BoolNode *is_Bool() { return this; }
> 
> [tena/825384/cores:DBX] frame 9
> 0xfee1451c: adjust_check+0x039c:        ld      [%o0 + 0x20], %l0
> 
> 0xfee14500: adjust_check+0x0380:        addcc   %l4, 0x8, %l7
> 0xfee14504: adjust_check+0x0384:        be,a    adjust_check+0x3c8
> 0xfee14508: adjust_check+0x0388:        ld      [%i5], %g2
> 0xfee1450c: adjust_check+0x038c:        ld      [%l2], %g2
> 0xfee14510: adjust_check+0x0390:        ld      [%g2 + 0x18], %l0
> 0xfee14514: adjust_check+0x0394:        jmpl    %l0, %o7  -> is_Bool
> 0xfee14518: adjust_check+0x0398:        mov     %l2, %o0
> 0xfee1451c: adjust_check+0x039c:        ld      [%o0 + 0x20], %l0
> 0xfee14514: adjust_check+0x0394:        jmpl    %l0, %o7
> 
> 
>>l0-l3    0xfedfe558 0x00000000 0x00370be4 0x004ddda8
> 
> [tena/825384/cores:DBX] x 0xfedfe558/i
> 0xfedfe558: is_Bool       :     jmp     %o7 + 0x8
> 0xfedfe55c: is_Bool+0x0004:     clr     %o0
> 
> 
>>o4-o7    0x004e3e3c 0x00000000 0xd64fea40 0xfee14514
> 
> [tena/825384/cores:DBX] x 0xfee14514+8/i
> 0xfee1451c: adjust_check+0x039c:        ld      [%o0 + 0x20], %l0
> 
> Then:
> 
> What is the significance of the NULL returned from is_Bool ?
> 
> My attempt at interpretation: 
>   We've got the node.hpp version above and therefore we 
>   have the wrong node? 
> [If so does that mean we need an additional restriction in adjust_check or 
> does it more likely mean we have a problem higher up?]
> 
> Any help, suggestions comments (thats pure BS gladly accepted...)
> 
> Cheers!
> Chris
> 
> |Date: Tue, 7 Oct 2003 14:24:39 -0400 (EDT)
> |From: Chris Phillips - Member Technical Staff <chrisph>
> |Hi,
> |
> ||	Evaluation: 
> ||The crashes all occur here:
> ||ifnode.cpp:
> ||   436    // Else, adjust existing check
> ||   436    // Else, adjust existing check
> ||   437    Node *new_bol = gvn->transform( new (2) BoolNode( new_cmp, 
> |bol->is_Bool()->_test._test ) );
> ||
> |
> |Any idea as to what would be the significance of the 
> | bol->is_Bool()->_test._test above returning a Null?
> | 
> |Chris

 xxxxx@xxxxx  2003-10-09