Introduction
This document covers how to set up LUN multi-pathing on RedHat Linux systems.
Note that MPIO is not supported for either the root or boot partition, Autodiscovery of software RAID on top of DM MPIO is not available.
Multipathing is only available on RHEL5, and RHEL4 UL2 and above.
RedHat Linux Multi-pathing
Multi-pathing is an I/O layer that sits above the standard I/O layer. It allows
multiple LUN access under a single device name, and provides the same functionality
as PV links under HP-UX.
Installing and Configuring Multi-path
The MINIMUM patch level for RHEL4 is UL2.
[root@testsys etc]# cat /etc/redhat-release
Red Hat Enterprise Linux ES release 4 (Nahant Update 4)
[root@testsys etc]# rpm -i /media/nfs/RHEL4/RedHat/RPMS/device-mapper-multipath-0.4.516.1.RHEL4.i386.rpm
[root@testsys etc]# vi multipath.conf
Make it look like the following:
#
#
#
#
#
This is an example configuration file for device mapper multipath.
For a complete list of the default configuration values, see
/usr/share/doc/device-mapper-multipath-0.4.5/multipath.conf.defaults
For a list of configuration options with descriptions, see
/usr/share/doc/device-mapper-multipath-0.4.5/multipath.conf.annotated
# Blacklist all devices by default. Remove this to enable multipathing
# on the default devices.
# devnode_blacklist {
# devnode "*"
#}
## Use user friendly names, instead of using WWIDs as names.
defaults {
user_friendly_names yes
}
##
## This is a template multipath-tools configuration file
## Uncomment the lines relevent to your environment
##
defaults {
multipath_tool "/sbin/multipath -v0"
udev_dir /dev
polling_interval 10
selector "round-robin 0"
path_grouping_policy failover
getuid_callout "/sbin/scsi_id -g -u -s /block/%n"
prio_callout /bin/true
path_checker readsector0
rr_min_io 100
rr_weight priorities
failback immediate
no_path_retry fail
user_friendly_name yes
}
##
## The wwid line in the following blacklist section is shown as an example
## of how to blacklist devices by wwid. The 3 devnode lines are the
## compiled in default blacklist.
##
#devnode_blacklist {
# wwid 26353900f02796769
# devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"
# devnode "^hd[a-z]"
# devnode "^cciss!c[0-9]d[0-9]*"
#}
#multipaths {
# multipath {
# wwid 3600508b4000156d700012000000b0000
# alias yellow
# path_grouping_policy multibus
# path_checker readsector0
# path_selector "round-robin 0"
# failback manual
# rr_weight priorities
# no_path_retry 5
# }
# multipath {
# wwid 1DEC_____321816758474
# alias red
# }
#}
#devices {
# device {
# vendor "COMPAQ "
# product "HSV110 (C)COMPAQ"
# path_grouping_policy multibus
# getuid_callout "/sbin/scsi_id -g -u -s /block/%n"
# path_checker readsector0
# path_selector "round-robin 0"
# hardware_handler "0"
# failback 15
# rr_weight priorities
# no_path_retry queue
# }
# device {
# vendor "COMPAQ "
# product "MSA1000 "
# path_grouping_policy multibus
# }
#}
[root@testsys etc]# modprobe dm-multipath
[root@testsys etc]# modprobe dm-round-robin
[root@testsys etc]# service multipathd start
Starting multipathd daemon: [ OK ]
[root@testsys etc]# chkconfig multipathd on
[root@testsys etc]# chkconfig --list multipathd
multipathd 0:off 1:off 2:on 3:on 4:on 5:on 6:off
If a vgscan is run now, it will still show the old sdx device names:
[root@testsys etc]# vgscan
Reading all physical volumes. This may take a while...
Found duplicate PV RD2pVw5MjO1DN1iSLKgjmIKsjzaGzsqG: using /dev/sdb
Found duplicate PV fu3SP0yIdqIpgIi1PSXrhiKPYJmGMARY: using /dev/sdc
Found duplicate PV b3LYWbNRoRiJZkqBCYWOE3H0l2Rg8C41: using /dev/sdd
Found duplicate PV QB8daoIQxXyd2WEFsmya7vaAQr4VVJZo: using /dev/sde
Found duplicate PV 8DBtlJAHjgLn2uuuHP16L1OpYbG1kkzh: using /dev/sdf
Found duplicate PV HDPPBSqQPFcxSo9aIE2m0mCuO7nc06C9: using /dev/sdg
Found duplicate PV 5FLMYkyY2TwtU1I5be2pIPkTMJ8ZJzJ2: using /dev/sdn
Found duplicate PV 6hsEh2000Qh3ip6YhBsW2w92TX56eENH: using /dev/sdo
Found duplicate PV SzNIuf867dZ0rIeTVN38HbAluNMgGs7o: using /dev/sdp
Found volume group "vg_xenonapp01a" using metadata type lvm2
Found volume group "vg_orabin01a" using metadata type lvm2
Found volume group "vg_oradb01a" using metadata type lvm2
To generate the new device mappings:
[root@testsys etc]# multipath -v2
create: mpath0 (360060e8004eb21000000eb21000006aa)
[size=4 GB][features="0"][hwhandler="0"]
\_ round-robin 0
\_ 1:0:0:11 sdaa 65:160 [ready]
\_ 0:0:0:11 sdl 8:176 [ready]
create: mpath1 (360060e8004eb21000000eb21000006ab)
[size=4 GB][features="0"][hwhandler="0"]
not
not
not
not
not
not
not
not
not
/dev/sdq
/dev/sdr
/dev/sds
/dev/sdt
/dev/sdu
/dev/sdv
/dev/sdac
/dev/sdad
/dev/sda
\_ round-robin 0
\_ 1:0:0:12 sdab 65:176 [ready]
\_ 0:0:0:12 sdm 8:192 [ready]
create: mpath2 (360060e8004eb21000000eb21000006b4)
[size=4 GB][features="0"][hwhandler="0"]
\_ round-robin 0
\_ 1:0:0:13 sdac 65:192 [ready]
\_ 0:0:0:13 sdn 8:208 [ready]
create: mpath3 (360060e8004eb21000000eb21000006b5)
[size=4 GB][features="0"][hwhandler="0"]
\_ round-robin 0
\_ 1:0:0:14 sdad 65:208 [ready]
\_ 0:0:0:14 sdo 8:224 [ready]
create: mpath4 (360060e8004eb21000000eb210000060f)
[size=4 GB][features="0"][hwhandler="0"]
\_ round-robin 0
\_ 0:0:0:0 sda 8:0 [ready]
\_ 1:0:0:0 sdp 8:240 [ready]
create: mpath5 (360060e8004eb21000000eb2100000690)
[size=4 GB][features="0"][hwhandler="0"]
\_ round-robin 0
\_ 0:0:0:1 sdb 8:16 [ready]
\_ 1:0:0:1 sdq 65:0 [ready]
create: mpath6 (360060e8004eb21000000eb2100000691)
[size=4 GB][features="0"][hwhandler="0"]
\_ round-robin 0
\_ 0:0:0:2 sdc 8:32 [ready]
\_ 1:0:0:2 sdr 65:16 [ready]
create: mpath7 (360060e8004eb21000000eb2100000692)
[size=4 GB][features="0"][hwhandler="0"]
\_ round-robin 0
\_ 0:0:0:3 sdd 8:48 [ready]
\_ 1:0:0:3 sds 65:32 [ready]
create: mpath8 (360060e8004eb21000000eb21000006a3)
[size=4 GB][features="0"][hwhandler="0"]
\_ round-robin 0
\_ 0:0:0:4 sde 8:64 [ready]
\_ 1:0:0:4 sdt 65:48 [ready]
create: mpath9 (360060e8004eb21000000eb21000006a4)
[size=4 GB][features="0"][hwhandler="0"]
\_ round-robin 0
\_ 0:0:0:5 sdf 8:80 [ready]
\_ 1:0:0:5 sdu 65:64 [ready]
create: mpath10 (360060e8004eb21000000eb21000006a5)
[size=4 GB][features="0"][hwhandler="0"]
\_ round-robin 0
\_ 0:0:0:6 sdg 8:96 [ready]
\_ 1:0:0:6 sdv 65:80 [ready]
create: mpath11 (360060e8004eb21000000eb21000006a6)
[size=4 GB][features="0"][hwhandler="0"]
\_ round-robin 0
\_ 0:0:0:7 sdh 8:112 [ready]
\_ 1:0:0:7 sdw 65:96 [ready]
create: mpath12 (360060e8004eb21000000eb21000006a7)
[size=4 GB][features="0"][hwhandler="0"]
\_ round-robin 0
\_ 0:0:0:8 sdi 8:128 [ready]
\_ 1:0:0:8 sdx 65:112 [ready]
create: mpath13 (360060e8004eb21000000eb21000006a8)
[size=4 GB][features="0"][hwhandler="0"]
\_ round-robin 0
\_ 0:0:0:9 sdj 8:144 [ready]
\_ 1:0:0:9 sdy 65:128 [ready]
create: mpath14 (360060e8004eb21000000eb21000006a9)
[size=4 GB][features="0"][hwhandler="0"]
\_ round-robin 0
\_ 0:0:0:10 sdk 8:160 [ready]
\_ 1:0:0:10 sdz 65:144 [ready]
If a vgscan is run now, it will now show the new dm device names
[root@testsys etc]# vgscan
Reading all physical volumes. This may take a while...
Found volume group "vg_orabin01a" using metadata type lvm2
Found volume group "vg_xenonapp01a" using metadata type lvm2
Found volume group "vg_oradb01a" using metadata type lvm2
[root@testsys etc]# vgdisplay -v /dev/vg_orabin01a
Using volume group(s) on command line
Finding volume group "vg_orabin01a"
--- Volume group --VG Name vg_orabin01a
System ID
Format lvm2
Metadata Areas 3
Metadata Sequence No 3
VG Access read/write
VG Status resizable
MAX LV 0
Cur LV 2
Open LV 2
Max PV 0
Cur PV 3
Act PV 3
VG Size 14.66 GB
PE Size 4.00 MB
Total PE 3753
Alloc PE / Size 2529 / 9.88 GB
Free PE / Size 1224 / 4.78 GB
VG UUID qzQX2B-BGbc-IKPh-9Xez-l22z-nFUn-RWETSG
--- Logical volume --LV Name /dev/vg_orabin01a/lvol1
VG Name vg_orabin01a
LV UUID wuCHUL-wJ1Y-5VBK-qeKz-u065-ifiz-ecoBbi
LV Write Access read/write
LV Status available
# open 1
LV Size 9.77 GB
Current LE 2502
Segments 1
Allocation inherit
Read ahead sectors 0
Block device 253:0
--- Logical volume --LV Name /dev/vg_orabin01a/lvol2
VG Name vg_orabin01a
LV UUID vZRA2H-zfBM-ClvH-3YBY-I4xo-oX6g-691ROL
LV Write Access read/write
LV Status available
# open 1
LV Size 108.00 MB
Current LE 27
Segments 1
Allocation inherit
Read ahead sectors 0
Block device 253:1
--- Physical volumes ---
PV Name /dev/dm-10
PV UUID QB8dao-IQxX-yd2W-EFsm-ya7v-aAQr-4VVJZo
PV Status allocatable
Total PE / Free PE 1251 / 408
PV Name /dev/dm-11
PV UUID 8DBtlJ-AHjg-Ln2u-uuHP-16L1-OpYb-G1kkzh
PV Status allocatable
Total PE / Free PE 1251 / 408
PV Name /dev/dm-12
PV UUID HDPPBS-qQPF-cxSo-9aIE-2m0m-CuO7-nc06C9
PV Status allocatable
Total PE / Free PE 1251 / 408
To see the mpath device names:
[root@testsys etc]# dmsetup ls --target=multipath
mpath2 (253, 4)
mpath1 (253, 3)
mpath0 (253, 2)
mpath9 (253, 11)
mpath8 (253, 10)
mpath14 (253, 16)
mpath7 (253, 9)
mpath13 (253, 15)
mpath6 (253, 8)
mpath12 (253, 14)
mpath5 (253, 7)
mpath11 (253, 13)
mpath4 (253, 6)
mpath10 (253, 12)
mpath3 (253, 5)
The mpath device filenames can all be found under /dev/mapper:
[root@testsys proc]# ll /dev/mapper/mpath*
brw-rw---- 1 root disk 253, 2 Oct 27 20:55 /dev/mapper/mpath0
brw-rw---- 1 root disk 253, 3 Oct 27 20:55 /dev/mapper/mpath1
brw-rw---- 1 root disk 253, 12 Oct 27 20:55 /dev/mapper/mpath10
brw-rw---- 1 root disk 253, 13 Oct 27 20:55 /dev/mapper/mpath11
brw-rw---- 1 root disk 253, 14 Oct 27 20:55 /dev/mapper/mpath12
brw-rw---- 1 root disk 253, 15 Oct 27 20:55 /dev/mapper/mpath13
brw-rw---- 1 root disk 253, 16 Oct 27 20:55 /dev/mapper/mpath14
brw-rw---- 1 root disk 253, 4 Oct 27 20:55 /dev/mapper/mpath2
brw-rw---- 1 root disk 253, 5 Oct 27 20:55 /dev/mapper/mpath3
brw-rw---- 1 root disk 253, 6 Oct 27 20:55 /dev/mapper/mpath4
brw-rw---- 1 root disk 253, 7 Oct 27 20:55 /dev/mapper/mpath5
brw-rw---- 1 root disk 253, 8 Oct 27 20:55 /dev/mapper/mpath6
brw-rw---- 1 root disk 253, 9 Oct 27 20:55 /dev/mapper/mpath7
brw-rw---- 1 root disk 253, 10 Oct 27 20:55 /dev/mapper/mpath8
brw-rw---- 1 root disk 253, 11 Oct 27 20:55 /dev/mapper/mpath9
We're only using LVM and ASM in this instance. Should standard Linux partitions be added
to the LUNs in question, then also look at (using mpath10):
To create all of the relevant device maps
[root@testsys etc]# kpartx -a /dev/mapper/mpath10
To list the device mapping
[root@testsys etc]# kpartx -l /dev/mapper/mpath10
Known Issues
Oracle Bugzilla 3015: ASM Instance could not detect IO path failure due to incorrect
multipath.conf setting
Affects: all applictions using multipath devices
Symptom: On simulating a controller failure, the
multipath device queues all the IOs issued to the failed IO path
and does not report the failure to its user.
In this case, the ASM instance is unable to detect the failure and continues to
issue IOs, causing the instance to hang waiting for the IOs to complete.
Solution: In configuration file /etc/multipath.conf, find and comment out the
"queue_if_no_path" feature definition.Equivalently, queue_if_no_path option can be defined as
"no_path_retry queue",
Set no_path_retry option to default(null), by not having this option specified
in /etc/multipath.conf file
in Oracle Clusterware, RAC and ASM you have to use RAW devices as you mentioned. In Linux you have to bind the SCSI block devices
to a RAW device.
In RedHat:
# vi /etc/sysconfig/rawdevices
/dev/raw/raw1 /dev/disk/by-name/1HITACHI_D60085320025
/dev/raw/raw2 /dev/disk/by-name/1HITACHI_D60085320018
In Suse:
# vi /etc/raw
raw1:disk/by-name/1HITACHI_D60085320025
raw2:disk/by-name/1HITACHI_D60085320018
After restarting the RAW subsystem you only work with
/dev/raw/raw.*.
The mapper device name is fixed and depends on the storage type,
serial number and storage internal LUN number. With the binding
also the RAW device names are fixed. In Oracle ASM you enter
/dev/raw/raw.* and the complete path is bound.
If you use the Oracle ASMlib (easy to install) you do not need to
bind the RAW devices.
Example:
# multipath -ll
1HITACHI_D60H02610121
[size=1 GB][features="0"][hwhandler="0"]
\_ round-robin 0 [prio=1][active]
\_ 1:0:1:1 sdj 8:144 [active][ready]
\_ round-robin 0 [enabled]
\_ 2:0:1:1 sdan 66:112 [active][ready]
# /etc/init.d/oracleasm createdisk DISK1
/dev/disk/by-name/1HITACHI_D60H02660121
Marking disk "/dev/disk/by-name/1HITACHI_D60H02660121" as an ASM disk:
done
# /etc/init.d/oracleasm listdisks
DISK1
# /etc/init.d/oracleasm querydisk DISK1
Disk "DISK1" is a valid ASM disk on device [253, 6]
SHUTDOWN the server and exchange the LUN number with another LUN
for an different scanning sequence.
BOOT the server
Now other major and minor numbers are used:
# multipath -l
1HITACHI_D60H02610121
[size=1 GB][features="0"][hwhandler="0"]
\_ round-robin 0 [active]
\_ 1:0:1:0 sdi 8:128 [active][ready]
\_ round-robin 0 [enabled]
\_ 2:0:1:0 sdam 66:96 [active][ready]
# /etc/init.d/oracleasm listdisks
DISK1
# /etc/init.d/oracleasm querydisk DISK1
Disk "DISK1" is a valid ASM disk on device [8, 32]
# /etc/init.d/oracleasm querydisk
/dev/disk/by-name/1HITACHI_D60H02660121
Disk "/dev/disk/by-name/1HITACHI_D60H02660121" is marked an ASM disk
with the label "DISK1"
ASM still finds the correct disk.