< TEST_PROGS=toeplitz.py TEST_GEN_PROGS="" run_tests make: Entering directory '/srv/vmksft/testing/wt-24/tools/testing/selftests' make[1]: Entering directory '/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw' make[1]: Nothing to be done for 'all'. make[1]: Leaving directory '/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw' make[1]: Nothing to be done for 'all'. make[1]: Entering directory '/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw' # overriding timeout to 1800 # selftests: drivers/net/hw: toeplitz.py # # Interface: enp1s0, driver: fbnic # TAP version 13 # 1..12 # # Exception| Traceback (most recent call last): # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/net/lib/py/ksft.py", line 420, in ksft_run # # Exception| func(*args) # # Exception| ~~~~^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 214, in test # # Exception| with bkg(" ".join(rx_cmd), ksft_ready=True, exit_wait=True) as rx_proc: # # Exception| ~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/net/lib/py/utils.py", line 205, in __exit__ # # Exception| return self.process(terminate=terminate, fail=self.check_fail, # # Exception| ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Exception| expect_fail=self.expect_fail) # # Exception| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/net/lib/py/utils.py", line 134, in process # # Exception| raise CmdExitFailure("Command failed", self) # # Exception| net.lib.py.utils.CmdExitFailure: Command failed # # Exception| CMD: /srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/toeplitz -4 -t -d 48036 -i enp1s0 -T 4000 -s -v # # Exception| EXIT: 40 # # Exception| STDERR: RSS indirection table size: 256 # # Exception| cpu 0: rx_hash 0xbed5022c [saddr 192.0.3.2 daddr 192.0.3.1 sport 39156 dport 48036] != expected 0x30d335df # # Exception| cpu 0: rx_hash 0x007bcea7 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39180 dport 48036] != expected 0x3c468a9d # # Exception| cpu 0: rx_hash 0x23efe041 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39198 dport 48036] != expected 0xc983adfd # # Exception| cpu 0: rx_hash 0x8990015c [saddr 192.0.3.2 daddr 192.0.3.1 sport 39240 dport 48036] != expected 0x1402b7b3 # # Exception| cpu 4: rx_hash 0xfe2f41ed [saddr 192.0.3.2 daddr 192.0.3.1 sport 39164 dport 48036] != expected 0xbc3fe672 # # Exception| cpu 4: rx_hash 0xe2e12403 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39186 dport 48036] != expected 0x5cb6d90b # # Exception| cpu 4: rx_hash 0x10455e57 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39212 dport 48036] != expected 0x5f7dbe76 # # Exception| cpu 4: rx_hash 0x33d170b1 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39230 dport 48036] != expected 0xaab89916 # # Exception| cpu 5: rx_hash 0xde52600d [saddr 192.0.3.2 daddr 192.0.3.1 sport 39148 dport 48036] != expected 0x7a498fa4 # # Exception| cpu 5: rx_hash 0x732b3370 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39222 dport 48036] != expected 0x26544abb # # Exception| cpu 5: rx_hash 0x0b8d89d9 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39246 dport 48036] != expected 0x3e685e5e # # Exception| cpu 0: rx_hash 0xa9ed20bc [saddr 192.0.3.2 daddr 192.0.3.1 sport 39256 dport 48036] != expected 0xd274de65 # # Exception| cpu 0: rx_hash 0xdabddd6b [saddr 192.0.3.2 daddr 192.0.3.1 sport 39266 dport 48036] != expected 0xc8661e43 # # Exception| cpu 0: rx_hash 0x1bb31929 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39278 dport 48036] != expected 0x5d536ab5 # # Exception| cpu 0: rx_hash 0x78dd740e [saddr 192.0.3.2 daddr 192.0.3.1 sport 39284 dport 48036] != expected 0x247a9e78 # # Exception| cpu 0: rx_hash 0x2614c3fe [saddr 192.0.3.2 daddr 192.0.3.1 sport 39322 dport 48036] != expected 0xc894c79c # # Exception| cpu 0: rx_hash 0xf4cd984a [saddr 192.0.3.2 daddr 192.0.3.1 sport 39348 dport 48036] != expected 0xd29c937 # # Exception| cpu 0: rx_hash 0x4e8ce9a7 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39362 dport 48036] != expected 0xb393e792 # # Exception| cpu 0: rx_hash 0x8c6b22e3 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39372 dport 48036] != expected 0x1515ddd2 # # Exception| cpu 0: rx_hash 0x2de28480 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39384 dport 48036] != expected 0xcaba135f # # Exception| cpu 4: rx_hash 0x3bce38c9 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39294 dport 48036] != expected 0x9b250363 # # Exception| cpu 4: rx_hash 0xac160303 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39388 dport 48036] != expected 0xd363b404 # # Exception| cpu 5: rx_hash 0x879d659d [saddr 192.0.3.2 daddr 192.0.3.1 sport 39310 dport 48036] != expected 0x173b0911 # # Exception| cpu 5: rx_hash 0x165772ee [saddr 192.0.3.2 daddr 192.0.3.1 sport 39338 dport 48036] != expected 0x6dd99aa1 # # Exception| cpu 0: rx_hash 0xdf46fed4 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39398 dport 48036] != expected 0xc9717422 # # Exception| cpu 0: rx_hash 0xbfc19cf5 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39422 dport 48036] != expected 0x83ebce59 # # Exception| cpu 0: rx_hash 0x628b7235 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39432 dport 48036] != expected 0x2fcbf815 # # Exception| cpu 0: rx_hash 0x6a943a4d [saddr 192.0.3.2 daddr 192.0.3.1 sport 39496 dport 48036] != expected 0x1e566260 # # Exception| cpu 0: rx_hash 0xeb60bdce [saddr 192.0.3.2 daddr 192.0.3.1 sport 39500 dport 48036] != expected 0x78fc53b # # Exception| cpu 4: rx_hash 0x1e483a96 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39402 dport 48036] != expected 0x5c4400d4 # # Exception| cpu 4: rx_hash 0x7ecf58b7 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39410 dport 48036] != expected 0x16debaaf # # Exception| cpu 4: rx_hash 0xb0522981 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39462 dport 48036] != expected 0xea76f6be # # Exception| cpu 4: rx_hash 0x123280e4 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39472 dport 48036] != expected 0x66a7685 # # Exception| cpu 4: rx_hash 0xa873f109 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39494 dport 48036] != expected 0xb8d05820 # # Exception| cpu 5: rx_hash 0xff3bdf34 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39414 dport 48036] != expected 0xf071df4 # # Exception| cpu 5: rx_hash 0x80119891 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39446 dport 48036] != expected 0x4f3bab83 # # Exception| cpu 5: rx_hash 0x52c8c325 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39480 dport 48036] != expected 0x8a86a528 # # Exception| cpu 5: rx_hash 0xd0d54ba0 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39486 dport 48036] != expected 0xa0ec4cc5 # # Exception| cpu 0: rx_hash 0xf8b72238 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39534 dport 48036] != expected 0x5707bf66 # # Exception| cpu 0: rx_hash 0xc6f9f7e9 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39576 dport 48036] != expected 0xf1735cf9 # # Exception| count: pass=0 nohash=0 fail=40 # # Exception| # not ok 1 toeplitz.test.tcp_ipv4 # # Receiver output: # # # # RSS indirection table size: 256 # # # cpu 4: rx_hash 0xfd61c696 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54114 dport 59619] OK # # # cpu 4: rx_hash 0x7c954115 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54118 dport 59619] OK # # # cpu 4: rx_hash 0xbd9b8557 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54122 dport 59619] OK # # # cpu 4: rx_hash 0xb07fee90 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54190 dport 59619] OK # # # cpu 5: rx_hash 0x614fba22 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54146 dport 59619] OK # # # cpu 0: rx_hash 0xcecb7880 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54096 dport 59619] OK # # # cpu 0: rx_hash 0xcd227786 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54098 dport 59619] OK # # # cpu 0: rx_hash 0x5ce860f5 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54134 dport 59619] OK # # # cpu 0: rx_hash 0xe0bb3da1 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54150 dport 59619] OK # # # cpu 0: rx_hash 0xc0c61c41 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54166 dport 59619] OK # # # cpu 0: rx_hash 0x729825d4 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54176 dport 59619] OK # # # cpu 0: rx_hash 0x121f47f5 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54200 dport 59619] OK # # # cpu 4: rx_hash 0x6950f25a [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54210 dport 59619] OK # # # cpu 4: rx_hash 0x492dd3ba [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54226 dport 59619] OK # # # cpu 5: rx_hash 0x5afa4c4c [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54256 dport 59619] OK # # # cpu 5: rx_hash 0x552cde8e [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54272 dport 59619] OK # # # cpu 5: rx_hash 0x15d69d4f [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54280 dport 59619] OK # # # cpu 5: rx_hash 0x35abbcaf [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54296 dport 59619] OK # # # cpu 5: rx_hash 0x1dc9d537 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54344 dport 59619] OK # # # cpu 0: rx_hash 0x882317f8 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54238 dport 59619] OK # # # cpu 0: rx_hash 0xf89ae529 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54246 dport 59619] OK # # # cpu 0: rx_hash 0xc70fc6fb [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54310 dport 59619] OK # # # cpu 0: rx_hash 0xe49be81d [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54324 dport 59619] OK # # # cpu 0: rx_hash 0x25952c5f [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54328 dport 59619] OK # # # cpu 0: rx_hash 0x3e5dfbd1 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54362 dport 59619] OK # # # cpu 4: rx_hash 0x8feacd42 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54382 dport 59619] OK # # # cpu 4: rx_hash 0xef6daf63 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54390 dport 59619] OK # # # cpu 4: rx_hash 0x2e636b21 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54394 dport 59619] OK # # # cpu 4: rx_hash 0x92303675 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54410 dport 59619] OK # # # cpu 4: rx_hash 0x102dbef0 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54412 dport 59619] OK # # # cpu 4: rx_hash 0xf2b75454 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54418 dport 59619] OK # # # cpu 4: rx_hash 0xb1a41893 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54424 dport 59619] OK # # # cpu 4: rx_hash 0x60944c21 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54452 dport 59619] OK # # # cpu 5: rx_hash 0x7ea7b810 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54354 dport 59619] OK # # # cpu 5: rx_hash 0x30509f10 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54428 dport 59619] OK # # # cpu 5: rx_hash 0x820ea685 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54442 dport 59619] OK # # # cpu 0: rx_hash 0xcf108e83 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54374 dport 59619] OK # # # cpu 0: rx_hash 0x238700e6 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54462 dport 59619] OK # # # cpu 4: rx_hash 0x637d4327 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54454 dport 59619] OK # # # cpu 4: rx_hash 0xf941132a [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54480 dport 59619] OK # # # cpu 4: rx_hash 0x95368415 [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54534 dport 59619] OK # # # count: pass=41 nohash=0 fail=0 # ok 2 toeplitz.test.tcp_ipv6 # # Exception| Traceback (most recent call last): # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/net/lib/py/ksft.py", line 420, in ksft_run # # Exception| func(*args) # # Exception| ~~~~^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 214, in test # # Exception| with bkg(" ".join(rx_cmd), ksft_ready=True, exit_wait=True) as rx_proc: # # Exception| ~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/net/lib/py/utils.py", line 205, in __exit__ # # Exception| return self.process(terminate=terminate, fail=self.check_fail, # # Exception| ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Exception| expect_fail=self.expect_fail) # # Exception| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/net/lib/py/utils.py", line 134, in process # # Exception| raise CmdExitFailure("Command failed", self) # # Exception| net.lib.py.utils.CmdExitFailure: Command failed # # Exception| CMD: /srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/toeplitz -4 -u -d 40411 -i enp1s0 -T 4000 -s -v # # Exception| EXIT: 44 # # Exception| STDERR: RSS indirection table size: 256 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 59131 dport 40411] != expected 0xcb9de22 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 59131 dport 40411] != expected 0xcb9de22 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 48836 dport 40411] != expected 0x916f2312 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 48836 dport 40411] != expected 0x916f2312 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 47566 dport 40411] != expected 0xa77db27d # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 47566 dport 40411] != expected 0xa77db27d # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39943 dport 40411] != expected 0x5ab92ffc # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39943 dport 40411] != expected 0x5ab92ffc # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 53752 dport 40411] != expected 0x1844cd0 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 53752 dport 40411] != expected 0x1844cd0 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 58300 dport 40411] != expected 0xff568fed # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 58300 dport 40411] != expected 0xff568fed # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 41062 dport 40411] != expected 0xfd496426 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 41062 dport 40411] != expected 0xfd496426 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39744 dport 40411] != expected 0xaf65cd7d # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 39744 dport 40411] != expected 0xaf65cd7d # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 55411 dport 40411] != expected 0xc980bd2 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 55411 dport 40411] != expected 0xc980bd2 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 48499 dport 40411] != expected 0x6bb41246 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 48499 dport 40411] != expected 0x6bb41246 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 46669 dport 40411] != expected 0xa3a74c3b # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 46669 dport 40411] != expected 0xa3a74c3b # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 46197 dport 40411] != expected 0x8c3571e5 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 46197 dport 40411] != expected 0x8c3571e5 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 34682 dport 40411] != expected 0x97244e46 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 34682 dport 40411] != expected 0x97244e46 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 49943 dport 40411] != expected 0xecf94d7e # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 49943 dport 40411] != expected 0xecf94d7e # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 46113 dport 40411] != expected 0x6207251d # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 46113 dport 40411] != expected 0x6207251d # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 36030 dport 40411] != expected 0x6c768752 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 36030 dport 40411] != expected 0x6c768752 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 51059 dport 40411] != expected 0x249f9d1c # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 51059 dport 40411] != expected 0x249f9d1c # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 56170 dport 40411] != expected 0x2b30f917 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 56170 dport 40411] != expected 0x2b30f917 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 59015 dport 40411] != expected 0xd5c6d9c # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 59015 dport 40411] != expected 0xd5c6d9c # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 33867 dport 40411] != expected 0x5f5b5bc5 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 33867 dport 40411] != expected 0x5f5b5bc5 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 60191 dport 40411] != expected 0x11fa4934 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 60191 dport 40411] != expected 0x11fa4934 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 57087 dport 40411] != expected 0x449d8f7 # # Exception| cpu 5: rx_hash 0xcb9e9f10 [saddr 192.0.3.2 daddr 192.0.3.1 sport 57087 dport 40411] != expected 0x449d8f7 # # Exception| count: pass=0 nohash=0 fail=44 # # Exception| # not ok 3 toeplitz.test.udp_ipv4 # # Exception| Traceback (most recent call last): # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/net/lib/py/ksft.py", line 420, in ksft_run # # Exception| func(*args) # # Exception| ~~~~^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 214, in test # # Exception| with bkg(" ".join(rx_cmd), ksft_ready=True, exit_wait=True) as rx_proc: # # Exception| ~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/net/lib/py/utils.py", line 205, in __exit__ # # Exception| return self.process(terminate=terminate, fail=self.check_fail, # # Exception| ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Exception| expect_fail=self.expect_fail) # # Exception| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/net/lib/py/utils.py", line 134, in process # # Exception| raise CmdExitFailure("Command failed", self) # # Exception| net.lib.py.utils.CmdExitFailure: Command failed # # Exception| CMD: /srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/toeplitz -6 -u -d 48481 -i enp1s0 -T 4000 -s -v # # Exception| EXIT: 42 # # Exception| STDERR: RSS indirection table size: 256 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 41490 dport 48481] != expected 0x3c80d60 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 41490 dport 48481] != expected 0x3c80d60 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 42811 dport 48481] != expected 0x4658e6c5 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 42811 dport 48481] != expected 0x4658e6c5 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 36249 dport 48481] != expected 0xf2d3f0d3 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 36249 dport 48481] != expected 0xf2d3f0d3 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 50681 dport 48481] != expected 0x57bacd50 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 50681 dport 48481] != expected 0x57bacd50 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 60000 dport 48481] != expected 0x85351e05 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 60000 dport 48481] != expected 0x85351e05 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 37652 dport 48481] != expected 0xbde2144a # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 37652 dport 48481] != expected 0xbde2144a # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 35272 dport 48481] != expected 0x8de273c1 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 35272 dport 48481] != expected 0x8de273c1 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 49455 dport 48481] != expected 0x294b7cf6 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 49455 dport 48481] != expected 0x294b7cf6 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 34384 dport 48481] != expected 0x72af8009 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 34384 dport 48481] != expected 0x72af8009 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 58425 dport 48481] != expected 0xb3bdcd9a # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 58425 dport 48481] != expected 0xb3bdcd9a # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 39203 dport 48481] != expected 0x12d209e # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 39203 dport 48481] != expected 0x12d209e # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 45420 dport 48481] != expected 0x4e57797c # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 45420 dport 48481] != expected 0x4e57797c # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54224 dport 48481] != expected 0xa50e60c5 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54224 dport 48481] != expected 0xa50e60c5 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 38246 dport 48481] != expected 0x77d5ffac # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 38246 dport 48481] != expected 0x77d5ffac # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 47842 dport 48481] != expected 0x43fbd756 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 47842 dport 48481] != expected 0x43fbd756 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 43245 dport 48481] != expected 0x73ed9631 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 43245 dport 48481] != expected 0x73ed9631 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54942 dport 48481] != expected 0x7d70c560 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 54942 dport 48481] != expected 0x7d70c560 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 35126 dport 48481] != expected 0xf356e5d1 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 35126 dport 48481] != expected 0xf356e5d1 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 34421 dport 48481] != expected 0xe4b78977 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 34421 dport 48481] != expected 0xe4b78977 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 45972 dport 48481] != expected 0x13fd8ee6 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 45972 dport 48481] != expected 0x13fd8ee6 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 37648 dport 48481] != expected 0x3c1693c9 # # Exception| cpu 5: rx_hash 0x7781a6cd [saddr 2001:db8:1::2 daddr 2001:db8:1::1 sport 37648 dport 48481] != expected 0x3c1693c9 # # Exception| count: pass=0 nohash=0 fail=42 # # Exception| # not ok 4 toeplitz.test.udp_ipv6 # # Exception| Traceback (most recent call last): # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/net/lib/py/ksft.py", line 420, in ksft_run # # Exception| func(*args) # # Exception| ~~~~^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 203, in test # # Exception| irq_cpus = ",".join([str(x) for x in _get_irq_cpus(cfg)]) # # Exception| ~~~~~~~~~~~~~^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 88, in _get_irq_cpus # # Exception| cpus.append(_get_cpu_for_irq(napis[queues[name]["napi-id"]]["irq"])) # # Exception| ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 49, in _get_cpu_for_irq # # Exception| raise KsftFailEx(f"IRQ{irq} is not mapped to a single core: {data}") # # Exception| net.lib.py.ksft.KsftFailEx: IRQ36 is not mapped to a single core: 0-5 # # Exception| # not ok 5 toeplitz.test.rss_tcp_ipv4 # # Exception| Traceback (most recent call last): # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/net/lib/py/ksft.py", line 420, in ksft_run # # Exception| func(*args) # # Exception| ~~~~^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 203, in test # # Exception| irq_cpus = ",".join([str(x) for x in _get_irq_cpus(cfg)]) # # Exception| ~~~~~~~~~~~~~^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 88, in _get_irq_cpus # # Exception| cpus.append(_get_cpu_for_irq(napis[queues[name]["napi-id"]]["irq"])) # # Exception| ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 49, in _get_cpu_for_irq # # Exception| raise KsftFailEx(f"IRQ{irq} is not mapped to a single core: {data}") # # Exception| net.lib.py.ksft.KsftFailEx: IRQ36 is not mapped to a single core: 0-5 # # Exception| # not ok 6 toeplitz.test.rss_tcp_ipv6 # # Exception| Traceback (most recent call last): # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/net/lib/py/ksft.py", line 420, in ksft_run # # Exception| func(*args) # # Exception| ~~~~^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 203, in test # # Exception| irq_cpus = ",".join([str(x) for x in _get_irq_cpus(cfg)]) # # Exception| ~~~~~~~~~~~~~^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 88, in _get_irq_cpus # # Exception| cpus.append(_get_cpu_for_irq(napis[queues[name]["napi-id"]]["irq"])) # # Exception| ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 49, in _get_cpu_for_irq # # Exception| raise KsftFailEx(f"IRQ{irq} is not mapped to a single core: {data}") # # Exception| net.lib.py.ksft.KsftFailEx: IRQ36 is not mapped to a single core: 0-5 # # Exception| # not ok 7 toeplitz.test.rss_udp_ipv4 # # Exception| Traceback (most recent call last): # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/net/lib/py/ksft.py", line 420, in ksft_run # # Exception| func(*args) # # Exception| ~~~~^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 203, in test # # Exception| irq_cpus = ",".join([str(x) for x in _get_irq_cpus(cfg)]) # # Exception| ~~~~~~~~~~~~~^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 88, in _get_irq_cpus # # Exception| cpus.append(_get_cpu_for_irq(napis[queues[name]["napi-id"]]["irq"])) # # Exception| ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 49, in _get_cpu_for_irq # # Exception| raise KsftFailEx(f"IRQ{irq} is not mapped to a single core: {data}") # # Exception| net.lib.py.ksft.KsftFailEx: IRQ36 is not mapped to a single core: 0-5 # # Exception| # not ok 8 toeplitz.test.rss_udp_ipv6 # # Exception| Traceback (most recent call last): # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/net/lib/py/ksft.py", line 420, in ksft_run # # Exception| func(*args) # # Exception| ~~~~^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 208, in test # # Exception| rps_cpus = _get_unused_rps_cpus(cfg, count=2) # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 101, in _get_unused_rps_cpus # # Exception| rx_cpus = set(_get_irq_cpus(cfg)) # # Exception| ~~~~~~~~~~~~~^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 88, in _get_irq_cpus # # Exception| cpus.append(_get_cpu_for_irq(napis[queues[name]["napi-id"]]["irq"])) # # Exception| ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 49, in _get_cpu_for_irq # # Exception| raise KsftFailEx(f"IRQ{irq} is not mapped to a single core: {data}") # # Exception| net.lib.py.ksft.KsftFailEx: IRQ36 is not mapped to a single core: 0-5 # # Exception| # not ok 9 toeplitz.test.rps_tcp_ipv4 # # Exception| Traceback (most recent call last): # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/net/lib/py/ksft.py", line 420, in ksft_run # # Exception| func(*args) # # Exception| ~~~~^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 208, in test # # Exception| rps_cpus = _get_unused_rps_cpus(cfg, count=2) # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 101, in _get_unused_rps_cpus # # Exception| rx_cpus = set(_get_irq_cpus(cfg)) # # Exception| ~~~~~~~~~~~~~^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 88, in _get_irq_cpus # # Exception| cpus.append(_get_cpu_for_irq(napis[queues[name]["napi-id"]]["irq"])) # # Exception| ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 49, in _get_cpu_for_irq # # Exception| raise KsftFailEx(f"IRQ{irq} is not mapped to a single core: {data}") # # Exception| net.lib.py.ksft.KsftFailEx: IRQ36 is not mapped to a single core: 0-5 # # Exception| # not ok 10 toeplitz.test.rps_tcp_ipv6 # # Exception| Traceback (most recent call last): # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/net/lib/py/ksft.py", line 420, in ksft_run # # Exception| func(*args) # # Exception| ~~~~^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 208, in test # # Exception| rps_cpus = _get_unused_rps_cpus(cfg, count=2) # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 101, in _get_unused_rps_cpus # # Exception| rx_cpus = set(_get_irq_cpus(cfg)) # # Exception| ~~~~~~~~~~~~~^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 88, in _get_irq_cpus # # Exception| cpus.append(_get_cpu_for_irq(napis[queues[name]["napi-id"]]["irq"])) # # Exception| ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 49, in _get_cpu_for_irq # # Exception| raise KsftFailEx(f"IRQ{irq} is not mapped to a single core: {data}") # # Exception| net.lib.py.ksft.KsftFailEx: IRQ36 is not mapped to a single core: 0-5 # # Exception| # not ok 11 toeplitz.test.rps_udp_ipv4 # # Exception| Traceback (most recent call last): # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/net/lib/py/ksft.py", line 420, in ksft_run # # Exception| func(*args) # # Exception| ~~~~^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 208, in test # # Exception| rps_cpus = _get_unused_rps_cpus(cfg, count=2) # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 101, in _get_unused_rps_cpus # # Exception| rx_cpus = set(_get_irq_cpus(cfg)) # # Exception| ~~~~~~~~~~~~~^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 88, in _get_irq_cpus # # Exception| cpus.append(_get_cpu_for_irq(napis[queues[name]["napi-id"]]["irq"])) # # Exception| ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Exception| File "/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw/./toeplitz.py", line 49, in _get_cpu_for_irq # # Exception| raise KsftFailEx(f"IRQ{irq} is not mapped to a single core: {data}") # # Exception| net.lib.py.ksft.KsftFailEx: IRQ36 is not mapped to a single core: 0-5 # # Exception| # not ok 12 toeplitz.test.rps_udp_ipv6 # # Totals: pass:1 fail:11 xfail:0 xpass:0 skip:0 error:0 not ok 1 selftests: drivers/net/hw: toeplitz.py # exit=1 # Totals: pass:0 fail:1 xfail:0 xpass:0 skip:0 error:0 make[1]: Leaving directory '/srv/vmksft/testing/wt-24/tools/testing/selftests/drivers/net/hw' make: Leaving directory '/srv/vmksft/testing/wt-24/tools/testing/selftests' xx__-> echo $? 0 xx__-> echo scan > /sys/kernel/debug/kmemleak xx__-> /opt/nipa/contest/scripts/env_check.py Comparing to existing state file: no differences detected. Current system state saved to /tmp/nipa-env-state.json xx__-> echo $? 0 xx__->