Oracle Database Appliance (X4-2) Configuration

Components reviewed

1. ODA X4-2 – OS Configuration
2. Storage / Multipathing
3. Dev Mapper /dev/mapper
4. File System
5. Network – Node 1
6. Network – Node 2
7. Hosts File

ODA X4-2 – OS Configuration

Storage / Multipathing
4 x 200GB SSDs and 18 x 900GB 10K RPM Drives (Single Shelf with storage capacity of 17 TB) (48 cores per node)(300+GB Memory Per Node)

On each node of the ODA multipathing is configured as follows. Notice Oracle does not implement ASMLIB on these engineered systems. The systems are directly mapped to storage LUNs via MPATH and ASM.

[root@nodedb01 etc]# cat multipath.conf

#Multipath File autogenerate by Comet StorageManagement–DO NOT REMOVE THIS LINE
blacklist {
devnode “^asm/*”
devnode “ofsctl”
devnode “xvd*”
wwid 35000cca056171ea4
wwid 35000cca0561d3668
}
defaults {
user_friendly_names  yes
path_grouping_policy multibus
failback manual
no_path_retry fail
flush_on_last_del yes
queue_without_daemon no
path_checker tur
}
multipaths {
multipath {
wwid 35000cca05719bf64
alias HDD_E0_S00_1461305188
}
multipath {
wwid 35000cca057199664
alias HDD_E0_S01_1461294692
}
multipath {
wwid 35000cca05719a2c0
alias HDD_E0_S02_1461297856
}
multipath {
wwid 35000cca05719b260
alias HDD_E0_S03_1461301856
}
multipath {
wwid 35000cca057153fe4
alias HDD_E0_S04_1461010404
}
multipath {
wwid 35000cca057193390
alias HDD_E0_S05_1461269392
}
multipath {
wwid 35000cca05719c5a4
alias HDD_E0_S06_1461306788
}
multipath {
wwid 35000cca05719c768
alias HDD_E0_S07_1461307240
}
multipath {
wwid 35000cca05719b48c
alias HDD_E0_S08_1461302412
}
multipath {
wwid 35000cca057188148
alias HDD_E0_S09_1461223752
}
multipath {
wwid 35000cca05719cc14
alias HDD_E0_S10_1461308436
}
multipath {
wwid 35000cca0571a1a48
alias HDD_E0_S11_1461328456
}
multipath {
wwid 35000cca05719b1b4
alias HDD_E0_S12_1461301684
}
multipath {
wwid 35000cca05719b2cc
alias HDD_E0_S13_1461301964
}
multipath {
wwid 35000cca05719c7bc
alias HDD_E0_S14_1461307324
}
multipath {
wwid 35000cca057187158
alias HDD_E0_S15_1461219672
}
multipath {
wwid 35000cca0571a1b54
alias HDD_E0_S16_1461328724
}
multipath {
wwid 35000cca05719b350
alias HDD_E0_S17_1461302096
}
multipath {
wwid 35000cca0571a1848
alias HDD_E0_S18_1461327944
}
multipath {
wwid 35000cca0571997e4
alias HDD_E0_S19_1461295076
}
multipath {
wwid 35000a7203009b222
alias SSD_E0_S20_805941794
}
multipath {
wwid 35000a7203009a85e
alias SSD_E0_S21_805939294
}
multipath {
wwid 35000a7203009b26c
alias SSD_E0_S22_805941868
}
multipath {
wwid 35000a7203009b241
alias SSD_E0_S23_805941825
}
}

Dev Mapper /dev/mapper

[root@nodedb01 etc]# cd /dev/mapper/[root@nodedb01 mapper]# ls -ltr
total 0
crw——- 1 root root      10, 236 Feb 17 16:14 control
brw-rw—- 1 root disk     252,   3 Feb 17 16:14 VolGroupSys-LogVolSwap
brw-rw—- 1 root disk     252,   0 Feb 17 16:14 VolGroupSys-LogVolRoot
brw-rw—- 1 root disk     252,   1 Feb 17 16:14 VolGroupSys-LogVolU01
brw-rw—- 1 root disk     252,   2 Feb 17 16:14 VolGroupSys-LogVolOpt
brw-rw—- 1 grid asmadmin 252,  23 Feb 17 16:37 HDD_E0_S00_1461305188
brw-rw—- 1 grid asmadmin 252,   8 Feb 17 16:37 HDD_E0_S01_1461294692
brw-rw—- 1 grid asmadmin 252,  11 Feb 17 16:37 HDD_E0_S02_1461297856
brw-rw—- 1 grid asmadmin 252,  10 Feb 17 16:37 HDD_E0_S03_1461301856
brw-rw—- 1 grid asmadmin 252,  15 Feb 17 16:38 HDD_E0_S04_1461010404
brw-rw—- 1 grid asmadmin 252,  12 Feb 17 16:38 HDD_E0_S05_1461269392
brw-rw—- 1 grid asmadmin 252,   9 Feb 17 16:38 HDD_E0_S06_1461306788
brw-rw—- 1 grid asmadmin 252,  13 Feb 17 16:38 HDD_E0_S07_1461307240
brw-rw—- 1 grid asmadmin 252,  14 Feb 17 16:38 HDD_E0_S08_1461302412
brw-rw—- 1 grid asmadmin 252,  16 Feb 17 16:38 HDD_E0_S09_1461223752
brw-rw—- 1 grid asmadmin 252,  18 Feb 17 16:38 HDD_E0_S10_1461308436
brw-rw—- 1 grid asmadmin 252,  20 Feb 17 16:38 HDD_E0_S11_1461328456
brw-rw—- 1 grid asmadmin 252,  17 Feb 17 16:38 HDD_E0_S12_1461301684
brw-rw—- 1 grid asmadmin 252,  19 Feb 17 16:38 HDD_E0_S13_1461301964
brw-rw—- 1 grid asmadmin 252,  25 Feb 17 16:38 HDD_E0_S14_1461307324
brw-rw—- 1 grid asmadmin 252,  24 Feb 17 16:38 HDD_E0_S15_1461219672
brw-rw—- 1 grid asmadmin 252,  22 Feb 17 16:38 HDD_E0_S16_1461328724
brw-rw—- 1 grid asmadmin 252,  27 Feb 17 16:38 HDD_E0_S17_1461302096
brw-rw—- 1 grid asmadmin 252,  21 Feb 17 16:39 HDD_E0_S18_1461327944
brw-rw—- 1 grid asmadmin 252,  26 Feb 17 16:39 HDD_E0_S19_1461295076
brw-rw—- 1 grid asmadmin 252,   4 Feb 17 16:39 SSD_E0_S20_805941794
brw-rw—- 1 grid asmadmin 252,   5 Feb 17 16:39 SSD_E0_S21_805939294
brw-rw—- 1 grid asmadmin 252,   7 Feb 17 16:39 SSD_E0_S22_805941868
brw-rw—- 1 grid asmadmin 252,   6 Feb 17 16:39 SSD_E0_S23_805941825
brw-rw—- 1 grid asmadmin 252,  67 Feb 17 17:27 HDD_E0_S19_1461295076p2
brw-rw—- 1 grid asmadmin 252,  59 Feb 17 17:28 HDD_E0_S15_1461219672p2
brw-rw—- 1 grid asmadmin 252,  63 Feb 17 17:28 HDD_E0_S17_1461302096p2
brw-rw—- 1 grid asmadmin 252,  61 Feb 17 17:28 HDD_E0_S16_1461328724p2
brw-rw—- 1 grid asmadmin 252,  41 Feb 17 17:28 HDD_E0_S06_1461306788p2
brw-rw—- 1 grid asmadmin 252,  47 Feb 17 17:28 HDD_E0_S09_1461223752p2
brw-rw—- 1 grid asmadmin 252,  39 Feb 17 17:28 HDD_E0_S05_1461269392p2
brw-rw—- 1 grid asmadmin 252,  57 Feb 17 17:28 HDD_E0_S14_1461307324p2
brw-rw—- 1 grid asmadmin 252,  51 Feb 17 17:28 HDD_E0_S11_1461328456p2
brw-rw—- 1 grid asmadmin 252,  49 Feb 17 17:28 HDD_E0_S10_1461308436p2
brw-rw—- 1 grid asmadmin 252,  53 Feb 17 17:28 HDD_E0_S12_1461301684p2
brw-rw—- 1 grid asmadmin 252,  43 Feb 17 17:28 HDD_E0_S07_1461307240p2
brw-rw—- 1 grid asmadmin 252,  55 Feb 17 17:28 HDD_E0_S13_1461301964p2
brw-rw—- 1 grid asmadmin 252,  65 Feb 17 17:28 HDD_E0_S18_1461327944p2
brw-rw—- 1 grid asmadmin 252,  45 Feb 17 17:28 HDD_E0_S08_1461302412p2
brw-rw—- 1 grid asmadmin 252,  42 Feb 18 09:30 HDD_E0_S07_1461307240p1
brw-rw—- 1 grid asmadmin 252,  56 Feb 18 09:31 HDD_E0_S14_1461307324p1
brw-rw—- 1 grid asmadmin 252,  44 Feb 18 09:31 HDD_E0_S08_1461302412p1
brw-rw—- 1 grid asmadmin 252,  62 Feb 18 09:31 HDD_E0_S17_1461302096p1
brw-rw—- 1 grid asmadmin 252,  52 Feb 18 09:31 HDD_E0_S12_1461301684p1
brw-rw—- 1 grid asmadmin 252,  50 Feb 18 09:32 HDD_E0_S11_1461328456p1
brw-rw—- 1 grid asmadmin 252,  38 Feb 18 09:32 HDD_E0_S05_1461269392p1
brw-rw—- 1 grid asmadmin 252,  40 Feb 18 09:32 HDD_E0_S06_1461306788p1
brw-rw—- 1 grid asmadmin 252,  66 Feb 18 09:32 HDD_E0_S19_1461295076p1
brw-rw—- 1 grid asmadmin 252,  60 Feb 18 09:32 HDD_E0_S16_1461328724p1
brw-rw—- 1 grid asmadmin 252,  48 Feb 18 09:32 HDD_E0_S10_1461308436p1
brw-rw—- 1 grid asmadmin 252,  46 Feb 18 09:32 HDD_E0_S09_1461223752p1
brw-rw—- 1 grid asmadmin 252,  58 Feb 18 09:32 HDD_E0_S15_1461219672p1
brw-rw—- 1 grid asmadmin 252,  64 Feb 18 09:32 HDD_E0_S18_1461327944p1
brw-rw—- 1 grid asmadmin 252,  54 Feb 18 09:32 HDD_E0_S13_1461301964p1
brw-rw—- 1 grid asmadmin 252,  71 Feb 18 09:32 SSD_E0_S23_805941825p1
brw-rw—- 1 grid asmadmin 252,  70 Feb 18 09:32 SSD_E0_S22_805941868p1
brw-rw—- 1 grid asmadmin 252,  69 Feb 18 09:32 SSD_E0_S21_805939294p1
brw-rw—- 1 grid asmadmin 252,  68 Feb 18 09:32 SSD_E0_S20_805941794p1
brw-rw—- 1 grid asmadmin 252,  37 Feb 18 09:32 HDD_E0_S04_1461010404p2
brw-rw—- 1 grid asmadmin 252,  35 Feb 18 09:32 HDD_E0_S03_1461301856p2
brw-rw—- 1 grid asmadmin 252,  34 Feb 18 09:32 HDD_E0_S03_1461301856p1
brw-rw—- 1 grid asmadmin 252,  33 Feb 18 09:32 HDD_E0_S02_1461297856p2
brw-rw—- 1 grid asmadmin 252,  31 Feb 18 09:32 HDD_E0_S01_1461294692p2
brw-rw—- 1 grid asmadmin 252,  30 Feb 18 09:32 HDD_E0_S01_1461294692p1
brw-rw—- 1 grid asmadmin 252,  29 Feb 18 09:32 HDD_E0_S00_1461305188p2
brw-rw—- 1 grid asmadmin 252,  28 Feb 18 09:32 HDD_E0_S00_1461305188p1
brw-rw—- 1 grid asmadmin 252,  36 Feb 18 09:32 HDD_E0_S04_1461010404p1
brw-rw—- 1 grid asmadmin 252,  32 Feb 18 09:32 HDD_E0_S02_1461297856p1[root@nodedb01 mapper]#

— 1 grid asmadmin 252,  32 Feb 18 09:32 HDD_E0_S02_1461297856p1[root@nodedb01 mapper]# multipath -ll
HDD_E0_S13_1461301964 (35000cca05719b2cc) dm-19 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:13:0 sdn  8:208  active ready running
`- 1:0:13:0 sdal 66:80  active ready running
HDD_E0_S10_1461308436 (35000cca05719cc14) dm-18 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:10:0 sdk  8:160  active ready running
`- 1:0:10:0 sdai 66:32  active ready running
HDD_E0_S04_1461010404 (35000cca057153fe4) dm-15 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:4:0  sde  8:64   active ready running
`- 1:0:4:0  sdac 65:192 active ready running
HDD_E0_S14_1461307324 (35000cca05719c7bc) dm-25 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:14:0 sdo  8:224  active ready running
`- 1:0:14:0 sdam 66:96  active ready running
SSD_E0_S20_805941794 (35000a7203009b222) dm-4 STEC,Z16IZF4EUSUN200G
size=186G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:20:0 sdu  65:64  active ready running
`- 1:0:20:0 sdas 66:192 active ready running
HDD_E0_S15_1461219672 (35000cca057187158) dm-24 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:15:0 sdp  8:240  active ready running
`- 1:0:15:0 sdan 66:112 active ready running
HDD_E0_S07_1461307240 (35000cca05719c768) dm-13 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:7:0  sdh  8:112  active ready running
`- 1:0:7:0  sdaf 65:240 active ready running
SSD_E0_S21_805939294 (35000a7203009a85e) dm-5 STEC,Z16IZF4EUSUN200G
size=186G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:21:0 sdv  65:80  active ready running
`- 1:0:21:0 sdat 66:208 active ready running
HDD_E0_S09_1461223752 (35000cca057188148) dm-16 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:9:0  sdj  8:144  active ready running
`- 1:0:9:0  sdah 66:16  active ready running
HDD_E0_S06_1461306788 (35000cca05719c5a4) dm-9 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:6:0  sdg  8:96   active ready running
`- 1:0:6:0  sdae 65:224 active ready running
HDD_E0_S12_1461301684 (35000cca05719b1b4) dm-17 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:12:0 sdm  8:192  active ready running
`- 1:0:12:0 sdak 66:64  active ready running
HDD_E0_S11_1461328456 (35000cca0571a1a48) dm-20 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:11:0 sdl  8:176  active ready running
`- 1:0:11:0 sdaj 66:48  active ready running
HDD_E0_S08_1461302412 (35000cca05719b48c) dm-14 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:8:0  sdi  8:128  active ready running
`- 1:0:8:0  sdag 66:0   active ready running
HDD_E0_S17_1461302096 (35000cca05719b350) dm-27 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:17:0 sdr  65:16  active ready running
`- 1:0:17:0 sdap 66:144 active ready running
HDD_E0_S02_1461297856 (35000cca05719a2c0) dm-11 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:2:0  sdc  8:32   active ready running
`- 1:0:2:0  sdaa 65:160 active ready running
HDD_E0_S18_1461327944 (35000cca0571a1848) dm-21 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:18:0 sds  65:32  active ready running
`- 1:0:18:0 sdaq 66:160 active ready running
SSD_E0_S22_805941868 (35000a7203009b26c) dm-7 STEC,Z16IZF4EUSUN200G
size=186G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:22:0 sdw  65:96  active ready running
`- 1:0:22:0 sdau 66:224 active ready running
HDD_E0_S01_1461294692 (35000cca057199664) dm-8 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:1:0  sdb  8:16   active ready running
`- 1:0:1:0  sdz  65:144 active ready running
HDD_E0_S00_1461305188 (35000cca05719bf64) dm-23 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 1:0:0:0  sdy  65:128 active ready running
`- 0:0:0:0  sda  8:0    active ready running
HDD_E0_S03_1461301856 (35000cca05719b260) dm-10 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:3:0  sdd  8:48   active ready running
`- 1:0:3:0  sdab 65:176 active ready running
HDD_E0_S16_1461328724 (35000cca0571a1b54) dm-22 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:16:0 sdq  65:0   active ready running
`- 1:0:16:0 sdao 66:128 active ready running
HDD_E0_S19_1461295076 (35000cca0571997e4) dm-26 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:19:0 sdt  65:48  active ready running
`- 1:0:19:0 sdar 66:176 active ready running
SSD_E0_S23_805941825 (35000a7203009b241) dm-6 STEC,Z16IZF4EUSUN200G
size=186G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:23:0 sdx  65:112 active ready running
`- 1:0:23:0 sdav 66:240 active ready running
HDD_E0_S05_1461269392 (35000cca057193390) dm-12 HITACHI,H109090SESUN900G
size=838G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:5:0  sdf  8:80   active ready running
`- 1:0:5:0  sdad 65:208 active ready running[root@nodedb01 mapper]#

File System

[root@nodedb01 etc]#[root@nodedb01 etc]# df -h
Filesystem            Size  Used Avail Use% Mounted on
/dev/mapper/VolGroupSys-LogVolRoot
30G  6.4G   22G  23% /
/dev/md0              487M   36M  426M   8% /boot
/dev/mapper/VolGroupSys-LogVolU01
97G   11G   82G  12% /u01
/dev/mapper/VolGroupSys-LogVolOpt
59G  4.6G   51G   9% /opt
tmpfs                 127G  222M  126G   1% /dev/shm

/dev/asm/acfsvol-39    50G  178M   50G   1% /cloudfs

Instances – ASM and Custom (the ASM instance is on both nodes as ASM1 / 2
root@nodedb01 etc]# ps -ef  | grep lgwr
oracle    3074     1  0 Feb17 ?        00:00:10 ora_lgwr_cdsidb1
grid     21625     1  0 Feb17 ?        00:00:01 asm_lgwr_+ASM1
root     70116 53868  0 09:27 pts/0    00:00:00 grep lgwr[root@nodedb01 etc]#

OS Users

grid:x:1000:1001::/home/grid:/bin/bash
oracle:x:1001:1001::/home/oracle:/bin/bash

oinstall:x:1001:grid,oracle
dba:x:1002:oracle
racoper:x:1003:grid,oracle
asmdba:x:1004:grid,oracle
asmoper:x:1005:grid
asmadmin:x:1006:grid

Environments for oracle and grid users[root@nodedb01 etc]# su – oracle[oracle@nodedb01 ~]$ env | grep ORA
ORACLE_HOME=/u01/app/oracle/product/11.2.0.4/dbhome_1

[grid@nodedb01 ~]$ env | grep ORA
ORA_OAK_HOME=/opt/oracle/oak
ORACLE_HOME=/u01/app/11.2.0.4/grid[grid@nodedb01 ~]$

Network – Node 1

[root@nodedb01 etc]# ifconfig -a
bond0     Link encap:Ethernet  HWaddr 90:E2:BA:6D:34:14
inet addr:10.1.2.141  Bcast:10.202.111.255  Mask:255.255.254.0
UP BROADCAST RUNNING MASTER MULTICAST  MTU:1500  Metric:1
RX packets:268270 errors:0 dropped:0 overruns:0 frame:0
TX packets:296111 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:84475257 (80.5 MiB)  TX bytes:55171782 (52.6 MiB)

bond0:1   Link encap:Ethernet  HWaddr 90:E2:BA:6D:34:14
inet addr:10.1.2.143  Bcast:10.202.111.255  Mask:255.255.254.0
UP BROADCAST RUNNING MASTER MULTICAST  MTU:1500  Metric:1

bond0:3   Link encap:Ethernet  HWaddr 90:E2:BA:6D:34:14
inet addr:10.1.2.146  Bcast:10.202.111.255  Mask:255.255.254.0
UP BROADCAST RUNNING MASTER MULTICAST  MTU:1500  Metric:1

bond1     Link encap:Ethernet  HWaddr 00:10:E0:57:96:86
UP BROADCAST MASTER MULTICAST  MTU:1500  Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:0 (0.0 b)  TX bytes:0 (0.0 b)

eth0      Link encap:Ethernet  HWaddr 00:10:E0:57:96:84
inet addr:192.168.16.24  Bcast:192.168.16.255  Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST  MTU:9000  Metric:1
RX packets:1493871 errors:0 dropped:0 overruns:0 frame:0
TX packets:1669919 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:1337233417 (1.2 GiB)  TX bytes:5129042245 (4.7 GiB)

eth0:1    Link encap:Ethernet  HWaddr 00:10:E0:57:96:84
inet addr:169.254.89.159  Bcast:169.254.127.255  Mask:255.255.128.0
UP BROADCAST RUNNING MULTICAST  MTU:9000  Metric:1

eth1      Link encap:Ethernet  HWaddr 00:10:E0:57:96:85
inet addr:192.168.17.24  Bcast:192.168.17.255  Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST  MTU:9000  Metric:1
RX packets:1196063 errors:0 dropped:0 overruns:0 frame:0
TX packets:1153870 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:1187473713 (1.1 GiB)  TX bytes:944944820 (901.1 MiB)

eth1:1    Link encap:Ethernet  HWaddr 00:10:E0:57:96:85
inet addr:169.254.253.31  Bcast:169.254.255.255  Mask:255.255.128.0
UP BROADCAST RUNNING MULTICAST  MTU:9000  Metric:1

eth2      Link encap:Ethernet  HWaddr 90:E2:BA:6D:34:14
UP BROADCAST RUNNING SLAVE MULTICAST  MTU:1500  Metric:1
RX packets:134790 errors:0 dropped:0 overruns:0 frame:0
TX packets:147980 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:42076093 (40.1 MiB)  TX bytes:27434601 (26.1 MiB)

eth3      Link encap:Ethernet  HWaddr 90:E2:BA:6D:34:14
UP BROADCAST RUNNING SLAVE MULTICAST  MTU:1500  Metric:1
RX packets:133501 errors:0 dropped:0 overruns:0 frame:0
TX packets:148156 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:42400550 (40.4 MiB)  TX bytes:27742511 (26.4 MiB)

eth4      Link encap:Ethernet  HWaddr 00:10:E0:57:96:86
UP BROADCAST SLAVE MULTICAST  MTU:1500  Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:0 (0.0 b)  TX bytes:0 (0.0 b)

eth5      Link encap:Ethernet  HWaddr 00:10:E0:57:96:86
UP BROADCAST SLAVE MULTICAST  MTU:1500  Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:0 (0.0 b)  TX bytes:0 (0.0 b)

lo        Link encap:Local Loopback
inet addr:127.0.0.1  Mask:255.0.0.0
UP LOOPBACK RUNNING  MTU:16436  Metric:1
RX packets:1545580 errors:0 dropped:0 overruns:0 frame:0
TX packets:1545580 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:776595520 (740.6 MiB)  TX bytes:776595520 (740.6 MiB)

Network – Node 2[root@nodedb02 ~]# ifconfig -a
bond0     Link encap:Ethernet  HWaddr 90:E2:BA:6D:28:BC
inet addr:10.1.2.142  Bcast:10.202.111.255  Mask:255.255.254.0
UP BROADCAST RUNNING MASTER MULTICAST  MTU:1500  Metric:1
RX packets:239946 errors:0 dropped:0 overruns:0 frame:0
TX packets:220730 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:36197958 (34.5 MiB)  TX bytes:77968288 (74.3 MiB)

bond0:1   Link encap:Ethernet  HWaddr 90:E2:BA:6D:28:BC
inet addr:10.1.2.145  Bcast:10.202.111.255  Mask:255.255.254.0
UP BROADCAST RUNNING MASTER MULTICAST  MTU:1500  Metric:1

bond0:2   Link encap:Ethernet  HWaddr 90:E2:BA:6D:28:BC
inet addr:10.1.2.144  Bcast:10.202.111.255  Mask:255.255.254.0
UP BROADCAST RUNNING MASTER MULTICAST  MTU:1500  Metric:1

bond1     Link encap:Ethernet  HWaddr 00:10:E0:57:AE:92
UP BROADCAST MASTER MULTICAST  MTU:1500  Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:0 (0.0 b)  TX bytes:0 (0.0 b)

eth0      Link encap:Ethernet  HWaddr 00:10:E0:57:AE:90
inet addr:192.168.16.25  Bcast:192.168.16.255  Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST  MTU:9000  Metric:1
RX packets:1492126 errors:0 dropped:0 overruns:0 frame:0
TX packets:1496480 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:5117647295 (4.7 GiB)  TX bytes:1337779645 (1.2 GiB)

eth0:1    Link encap:Ethernet  HWaddr 00:10:E0:57:AE:90
inet addr:169.254.3.82  Bcast:169.254.127.255  Mask:255.255.128.0
UP BROADCAST RUNNING MULTICAST  MTU:9000  Metric:1

eth1      Link encap:Ethernet  HWaddr 00:10:E0:57:AE:91
inet addr:192.168.17.25  Bcast:192.168.17.255  Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST  MTU:9000  Metric:1
RX packets:1153932 errors:0 dropped:0 overruns:0 frame:0
TX packets:1196299 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:945263026 (901.4 MiB)  TX bytes:1187680452 (1.1 GiB)

eth1:1    Link encap:Ethernet  HWaddr 00:10:E0:57:AE:91
inet addr:169.254.187.41  Bcast:169.254.255.255  Mask:255.255.128.0
UP BROADCAST RUNNING MULTICAST  MTU:9000  Metric:1

eth2      Link encap:Ethernet  HWaddr 90:E2:BA:6D:28:BC
UP BROADCAST RUNNING SLAVE MULTICAST  MTU:1500  Metric:1
RX packets:119810 errors:0 dropped:0 overruns:0 frame:0
TX packets:110367 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:18064087 (17.2 MiB)  TX bytes:38890821 (37.0 MiB)

eth3      Link encap:Ethernet  HWaddr 90:E2:BA:6D:28:BC
UP BROADCAST RUNNING SLAVE MULTICAST  MTU:1500  Metric:1
RX packets:120159 errors:0 dropped:0 overruns:0 frame:0
TX packets:110390 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:18135389 (17.2 MiB)  TX bytes:39083009 (37.2 MiB)

eth4      Link encap:Ethernet  HWaddr 00:10:E0:57:AE:92
UP BROADCAST SLAVE MULTICAST  MTU:1500  Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:0 (0.0 b)  TX bytes:0 (0.0 b)

eth5      Link encap:Ethernet  HWaddr 00:10:E0:57:AE:92
UP BROADCAST SLAVE MULTICAST  MTU:1500  Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:0 (0.0 b)  TX bytes:0 (0.0 b)

lo        Link encap:Local Loopback
inet addr:127.0.0.1  Mask:255.0.0.0
UP LOOPBACK RUNNING  MTU:16436  Metric:1
RX packets:1058921 errors:0 dropped:0 overruns:0 frame:0
TX packets:1058921 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0

RX bytes:400060040 (381.5 MiB)  TX bytes:400060040 (381.5 MiB)

Hosts File

[root@nodedb02 ~]# cat /etc/hosts
# Do not remove the following line, or various programs
# that require network functionality will fail.
::1 localhost6.localdomain6 localhost6

192.168.17.24    nodeoda1-priv1.cdsiusa.com nodeoda1-priv1
192.168.17.25    nodeoda2-priv1.cdsiusa.com nodeoda2-priv1

192.168.17.24    nodeoda1-priv1.cdsiusa.com nodeoda1-priv1
192.168.17.25    nodeoda2-priv1.cdsiusa.com nodeoda2-priv1

# Following added by OneCommand
127.0.0.1    localhost.localdomain localhost

# PUBLIC HOSTNAMES

# PRIVATE HOSTNAMES
192.168.16.24    nodeoda1-priv0.cdsiusa.com nodeoda1-priv0
192.168.16.25    nodeoda2-priv0.cdsiusa.com nodeoda2-priv0
192.168.17.24    nodeoda1-priv1.cdsiusa.com nodeoda1-priv1
192.168.17.25    nodeoda2-priv1.cdsiusa.com nodeoda2-priv1

# VIP HOSTNAMES
10.1.2.143    nodevip01.cdsiusa.com nodevip01
10.1.2.144    nodevip02.cdsiusa.com nodevip02

# NET(0-3) HOSTNAMES
10.1.2.141    nodedb01.cdsiusa.com nodedb01
10.1.2.142    nodedb02.cdsiusa.com nodedb02

# Below are SCAN IP addresses for reference.
# SCAN_IPS=(10.1.2.145 10.1.2.146)