Details
-
Type:
Bug
-
Status: Closed
-
Priority:
Normal
-
Resolution: Fixed
-
Affects Version/s: MCO 2.5.0, MCO 2.5.1, MCO 2.5.2, MCO 2.6.0, MCO 2.6.1, MCO 2.7.0
-
Fix Version/s: MCO 2.8.0
-
Component/s: None
-
Labels:None
-
Template:customfield_10700 65822
-
Story Points:3
-
Sprint:London 2015-02-04
Description
On some AIX systems we are seeing the following failure during service startup:
/opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/runner.rb:160: [BUG] iseq_compile_each: unknown node: NODE_SCOPE
|
ruby 1.9.3p484 (2013-11-22 revision 43786) [rs6000-aix]
|
– Control frame information -----------------------------------------------
|
c:0004 p:0064 s:0010 b:0010 l:000009 d:000009 METHOD /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/runner.rb:160
|
c:0003 p:0009 s:0006 b:0006 l:00222c d:000005 BLOCK /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/runner.rb:139
|
c:0002 p:---- s:0004 b:0004 l:000003 d:000003 FINISH
|
c:0001 p:---- s:0002 b:0002 l:000001 d:000001 TOP
|
– Ruby level backtrace information ----------------------------------------
|
/opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/runner.rb:139:in `block in start_receiver_thread'
|
/opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/runner.rb:160:in `receiver_thread'
|
– Other runtime information -----------------------------------------------
|
Loaded script: /opt/puppet/sbin/mcollectived
|
Loaded features:
|
0 enumerator.so
|
1 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/enc/encdb.so
|
2 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/enc/iso_8859_1.so
|
3 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/enc/trans/transdb.so
|
4 /opt/puppet/lib/ruby/1.9.1/rubygems/defaults.rb
|
5 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/rbconfig.rb
|
6 /opt/puppet/lib/ruby/1.9.1/rubygems/deprecate.rb
|
7 /opt/puppet/lib/ruby/1.9.1/rubygems/exceptions.rb
|
8 /opt/puppet/lib/ruby/1.9.1/rubygems/custom_require.rb
|
9 /opt/puppet/lib/ruby/1.9.1/rubygems.rb
|
10 /opt/puppet/lib/ruby/site_ruby/1.9.1/stomp/constants.rb
|
11 /opt/puppet/lib/ruby/site_ruby/1.9.1/stomp/ext/hash.rb
|
12 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/socket.so
|
13 /opt/puppet/lib/ruby/1.9.1/socket.rb
|
14 /opt/puppet/lib/ruby/1.9.1/timeout.rb
|
15 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/io/wait.so
|
16 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/digest.so
|
17 /opt/puppet/lib/ruby/1.9.1/digest.rb
|
18 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/digest/sha1.so
|
19 /opt/puppet/lib/ruby/site_ruby/1.9.1/stomp/connection.rb
|
20 /opt/puppet/lib/ruby/1.9.1/thread.rb
|
21 /opt/puppet/lib/ruby/1.9.1/forwardable.rb
|
22 /opt/puppet/lib/ruby/site_ruby/1.9.1/stomp/client.rb
|
23 /opt/puppet/lib/ruby/site_ruby/1.9.1/stomp/message.rb
|
24 /opt/puppet/lib/ruby/site_ruby/1.9.1/stomp/version.rb
|
25 /opt/puppet/lib/ruby/site_ruby/1.9.1/stomp/errors.rb
|
26 /opt/puppet/lib/ruby/site_ruby/1.9.1/stomp/codec.rb
|
27 /opt/puppet/lib/ruby/site_ruby/1.9.1/stomp/sslparams.rb
|
28 /opt/puppet/lib/ruby/site_ruby/1.9.1/stomp/null_logger.rb
|
29 /opt/puppet/lib/ruby/site_ruby/1.9.1/client/utils.rb
|
30 /opt/puppet/lib/ruby/site_ruby/1.9.1/connection/utils.rb
|
31 /opt/puppet/lib/ruby/site_ruby/1.9.1/connection/netio.rb
|
32 /opt/puppet/lib/ruby/site_ruby/1.9.1/connection/heartbeats.rb
|
33 /opt/puppet/lib/ruby/site_ruby/1.9.1/connection/utf8.rb
|
34 /opt/puppet/lib/ruby/site_ruby/1.9.1/stomp.rb
|
35 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/digest/md5.so
|
36 /opt/puppet/lib/ruby/1.9.1/optparse.rb
|
37 /opt/puppet/lib/ruby/1.9.1/singleton.rb
|
38 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/strscan.so
|
39 /opt/puppet/lib/ruby/1.9.1/erb.rb
|
40 /opt/puppet/lib/ruby/1.9.1/shellwords.rb
|
41 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/stringio.so
|
42 /opt/puppet/lib/ruby/1.9.1/delegate.rb
|
43 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/etc.so
|
44 /opt/puppet/lib/ruby/1.9.1/fileutils.rb
|
45 /opt/puppet/lib/ruby/1.9.1/tmpdir.rb
|
46 /opt/puppet/lib/ruby/1.9.1/tempfile.rb
|
47 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/openssl.so
|
48 /opt/puppet/lib/ruby/1.9.1/openssl/bn.rb
|
49 /opt/puppet/lib/ruby/1.9.1/openssl/cipher.rb
|
50 /opt/puppet/lib/ruby/1.9.1/openssl/config.rb
|
51 /opt/puppet/lib/ruby/1.9.1/openssl/digest.rb
|
52 /opt/puppet/lib/ruby/1.9.1/openssl/buffering.rb
|
53 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/fcntl.so
|
54 /opt/puppet/lib/ruby/1.9.1/openssl/ssl-internal.rb
|
55 /opt/puppet/lib/ruby/1.9.1/openssl/x509-internal.rb
|
56 /opt/puppet/lib/ruby/1.9.1/openssl.rb
|
57 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/monkey_patches.rb
|
58 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/cache.rb
|
59 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/exceptions.rb
|
60 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/vendor.rb
|
61 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/log.rb
|
62 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/config.rb
|
63 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/logger.rb
|
64 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/logger/base.rb
|
65 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/logger/console_logger.rb
|
66 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/vendor/systemu/lib/systemu.rb
|
67 /opt/puppet/lib/ruby/1.9.1/json/version.rb
|
68 /opt/puppet/lib/ruby/1.9.1/json/common.rb
|
69 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/enc/utf_16be.so
|
70 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/enc/utf_16le.so
|
71 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/enc/utf_32be.so
|
72 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/enc/utf_32le.so
|
73 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/json/ext/parser.so
|
74 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/json/ext/generator.so
|
75 /opt/puppet/lib/ruby/1.9.1/json/ext.rb
|
76 /opt/puppet/lib/ruby/1.9.1/json.rb
|
77 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/vendor/require_vendored.rb
|
78 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective.rb
|
79 /opt/puppet/lib/ruby/1.9.1/getoptlong.rb
|
80 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/util.rb
|
81 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/pluginmanager.rb
|
82 /opt/puppet/lib/ruby/1.9.1/monitor.rb
|
83 /opt/puppet/lib/ruby/1.9.1/logger.rb
|
84 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/logger/file_logger.rb
|
85 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/facts.rb
|
86 /opt/puppet/lib/ruby/1.9.1/psych/syntax_error.rb
|
87 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/psych.so
|
88 /opt/puppet/lib/ruby/1.9.1/psych/nodes/node.rb
|
89 /opt/puppet/lib/ruby/1.9.1/psych/nodes/stream.rb
|
90 /opt/puppet/lib/ruby/1.9.1/psych/nodes/document.rb
|
91 /opt/puppet/lib/ruby/1.9.1/psych/nodes/sequence.rb
|
92 /opt/puppet/lib/ruby/1.9.1/psych/nodes/scalar.rb
|
93 /opt/puppet/lib/ruby/1.9.1/psych/nodes/mapping.rb
|
94 /opt/puppet/lib/ruby/1.9.1/psych/nodes/alias.rb
|
95 /opt/puppet/lib/ruby/1.9.1/psych/nodes.rb
|
96 /opt/puppet/lib/ruby/1.9.1/psych/streaming.rb
|
97 /opt/puppet/lib/ruby/1.9.1/psych/visitors/visitor.rb
|
98 /opt/puppet/lib/ruby/1.9.1/psych/scalar_scanner.rb
|
99 /opt/puppet/lib/ruby/1.9.1/psych/visitors/to_ruby.rb
|
100 /opt/puppet/lib/ruby/1.9.1/psych/visitors/emitter.rb
|
101 /opt/puppet/lib/ruby/1.9.1/psych/visitors/yaml_tree.rb
|
102 /opt/puppet/lib/ruby/1.9.1/psych/json/ruby_events.rb
|
103 /opt/puppet/lib/ruby/1.9.1/psych/visitors/json_tree.rb
|
104 /opt/puppet/lib/ruby/1.9.1/psych/visitors/depth_first.rb
|
105 /opt/puppet/lib/ruby/1.9.1/psych/visitors.rb
|
106 /opt/puppet/lib/ruby/1.9.1/psych/handler.rb
|
107 /opt/puppet/lib/ruby/1.9.1/psych/tree_builder.rb
|
108 /opt/puppet/lib/ruby/1.9.1/psych/parser.rb
|
109 /opt/puppet/lib/ruby/1.9.1/psych/omap.rb
|
110 /opt/puppet/lib/ruby/1.9.1/psych/set.rb
|
111 /opt/puppet/lib/ruby/1.9.1/psych/coder.rb
|
112 /opt/puppet/lib/ruby/1.9.1/psych/core_ext.rb
|
113 /opt/puppet/lib/ruby/1.9.1/rs6000-aix/date_core.so
|
114 /opt/puppet/lib/ruby/1.9.1/date/format.rb
|
115 /opt/puppet/lib/ruby/1.9.1/date.rb
|
116 /opt/puppet/lib/ruby/1.9.1/psych/deprecated.rb
|
117 /opt/puppet/lib/ruby/1.9.1/psych/stream.rb
|
118 /opt/puppet/lib/ruby/1.9.1/psych/json/yaml_events.rb
|
119 /opt/puppet/lib/ruby/1.9.1/psych/json/tree_builder.rb
|
120 /opt/puppet/lib/ruby/1.9.1/psych/json/stream.rb
|
121 /opt/puppet/lib/ruby/1.9.1/psych/handlers/document_stream.rb
|
122 /opt/puppet/lib/ruby/1.9.1/psych.rb
|
123 /opt/puppet/lib/ruby/1.9.1/yaml.rb
|
124 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/facts/base.rb
|
125 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/connector.rb
|
126 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/connector/base.rb
|
127 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/ddl.rb
|
128 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/ddl/base.rb
|
129 /opt/puppet/lib/ruby/1.9.1/base64.rb
|
130 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/security.rb
|
131 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/security/base.rb
|
132 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/registration.rb
|
133 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/registration/base.rb
|
134 /opt/puppet/lib/ruby/1.9.1/prettyprint.rb
|
135 /opt/puppet/lib/ruby/1.9.1/pp.rb
|
136 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/rpc.rb
|
137 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/rpc/audit.rb
|
138 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/runnerstats.rb
|
139 /opt/puppet/lib/ruby/site_ruby/1.9.1/mcollective/runner.rb
|
[NOTE]
|
You may have encountered a bug in the Ruby interpreter or extension libraries.
|
Bug reports are welcome.
|
For details: http://www.ruby-lang.org/bugreport.html
|
Which, due to the abort trap, creates the following segfault (captured here for future searching)
# (echo where; echo quit) | dbx /opt/puppet/bin/ruby>
|
Type 'help' for help.
|
[using memory image in core]
|
reading symbolic information ...
|
IOT/Abort trap in pthread_kill at 0xd05189d4 ($t4)
|
0xd05189d4 (pthread_kill+0xb4) 80410014 lwz r2,0x14(r1)
|
pthread_kill(??, ??) at 0xd05189d4
|
_p_raise(??) at 0xd0517e64
|
raise.raise(??) at 0xd0120d20
|
abort() at 0xd017c0e4
|
rb_bug(??) at 0x100a3580
|
iseq_compile_each(??, ??, ??, ??) at 0x10126ab8
|
iseq_compile_each(??, ??, ??, ??) at 0x1012d424
|
rb_iseq_compile_node(??, ??) at 0x101341e8
|
rb_iseq_new_with_opt(??, ??, ??, ??, ??, ??, ??, ??) at 0x101236ec
|
iseq_compile_each(??, ??, ??, ??) at 0x10128468
|
iseq_compile_each(??, ??, ??, ??) at 0x1012d424
|
rb_iseq_compile_node(??, ??) at 0x101341e8
|
rb_iseq_new_with_opt(??, ??, ??, ??, ??, ??, ??, ??) at 0x101236ec
|
iseq_compile_each(??, ??, ??, ??) at 0x101286e0
|
rb_iseq_compile_node(??, ??) at 0x10134090
|
rb_iseq_new_with_bopt_and_opt(??, ??, ??, ??, ??, ??, ??, ??) at 0x10121d30
|
rb_iseq_new_top(??, ??, ??, ??, ??) at 0x10122108
|
rb_load_internal(??, ??) at 0x101ace1c
|
rb_require_safe(??, ??) at 0x101ae7a8
|
rb_autoload_load(??, ??) at 0x10099cf0
|
vm_exec_core(??, ??) at 0x1001462c
|
vm_exec(??) at 0x10016c44
|
invoke_block_from_c(??, ??, ??, ??, ??, ??, ??) at 0x1001f530
|
rb_vm_invoke_proc(??, ??, ??, ??, ??, ??) at 0x1001f9a4
|
thread_start_func_2(??, ??) at 0x10113140
|
thread_start_func_1(??) at 0x101132b4
|
The cause for this appears to be an interaction between threads and autoload. On line 160 we see:
@agents ||= Agents.new
|
Agents has previously been declared to autoload, which defers compilation of agents.rb until the constant is used here.
We can avoid the rb_bug if we load the agents file in the main thread, as see in this experimental topic patch: https://github.com/richardc/marionette-collective/commit/718e1a95ad8ccfabdd7023d917aaaaf3b7ef0529
It might be worth revisiting the use of autoload at all, as autoload in a multithreaded environment is noted as being fundamentally flawed and scheduled for removal in ruby 3 https://www.ruby-forum.com/topic/3036681#1032667
Attachments
Issue Links
- links to
