Troubleshooting vxdg import issues with Storage Foundation Cluster File System running the AIX OS

book

Article ID: 100002287

calendar_today

Updated On:

Resolution

When troubleshooting issues of importing shared disk groups(import hangs or gets errors), the following procedures, commands, and data gathering should be followed.

a. Please get the vxconfigd log at the time of issue.

# vxconfigd -k -x mstimestamp -x 9 >/tmp/vxconfigd.log  2>&1 > /tmp/vxconfigd.log
# time vxdisk  -o alldgs list
# vxconfigd -k  /*Stop the vxconfigd in debug mode

b.  Collect the kernel trace for the vxdisk -o alldgs list command which takes more time.

# trace -al -T 128000000 -L 256000000 -o /var/tmp/trace.raw

# time vxdisk  -o alldgs list  /* which takes more time you reported it takes 36 sec*/
#trcstop
# trcrpt-O exec=on,pid=on,tid=on,svc=on,cpuid=on  /var/tmp/trace.raw >/tmp/trace_report

c. Collect the process stack at  at the timeof issue:

#/usr/bin/procstack -F
#/usr/bin/procstack -F

d.Collect the truss of vxconfigd and vxdisk -o alldgs list command

First start this command:

# truss  -dD -o /tmp/vxconfigd_truss.log -p  &
# truss -dD  -o /tmp/vxdisk_alldg_truss.log vxdisk -o alldgs list  /* As soon as this command has completed, then kill the truss of the vxconfigd command */

send /tmp/vxconfigd_truss.log and /tmp/vxdisk_alldg_truss.log file to Veritas support.

Additional commands:

# ps -ef | grep -i vxconfig
   root 14610     1   008:34:34      -  0:03 vxconfigd -k -m disable
   root 23934 22532   008:47:05  pts/0  0:00 grep -i vxconfig

# truss -o /tmp/x -p14610 &
[1]     26600

# ps
 PID    TTY  TIME CMD
22532  pts/0  0:00 -ksh
23936  pts/0  0:00 ps
26600  pts/0  0:00 truss -o /tmp/x-p 14610

# time vxdisk -o alldgs list
DEVICE      TYPE            DISK        GROUP        STATUS
Disk_0      auto:LVM        -            -            LVM
Disk_1      auto:LVM        -            -            LVM
Disk_2      auto:cdsdisk    -            -            online
Disk_3      auto:cdsdisk    -            -            online

real    0m5.30s
user    0m0.02s
sys    0m0.02s

# truss  -o /tmp/vxdisk_alldg_truss.log vxdisk -o alldgs list
DEVICE      TYPE            DISK        GROUP        STATUS
Disk_0      auto:LVM        -            -            LVM
Disk_1      auto:LVM        -            -            LVM
Disk_2      auto:cdsdisk    -            -            online
Disk_3      auto:cdsdisk    -            -            online
Disk_4      auto:cdsdisk    -            -            online
Disk_5      auto:cdsdisk    -            -            online
Disk_6      auto:cdsdisk    -            -            online
Disk_7      auto:cdsdisk    -          (chandradg)   online
Disk_8      auto:cdsdisk    -            -            online
Disk_9      auto:cdsdisk    -            -            online
Disk_10      auto:cdsdisk    -            -            online

#ps
  PID    TTY  TIME CMD
22532  pts/0  0:00 -ksh
23948  pts/0  0:00 ps
26600  pts/0  0:07 truss -o /tmp/x -p 1461

# kill -926600

# ps
  PID    TTY  TIME CMD
22532  pts/0  0:00 -ksh
26602  pts/0  0:00 ps
[1] +Killed                   truss -o/tmp/x -p 14610 &

e. When you see this issue please force the system dump.
Also, be sure and run this command after forcing the system dump

# snap -ac /* send the ***.pax.Z File to Veritas support */

# tprof -skeuj -x "command"

Issue/Introduction

Troubleshooting vxdg import issues with Storage Foundation Cluster File System running the AIX OS