CentOS 7.2安装11g Grid Infrastructure

 

preface

 

    oracle claimed that 11g rac is supported on redhat linux 7 and above version,but there're still some incompatible issues in inplementing 11g gi on it.there're some main steps showed below.

 

precedure

 

###step 1.prepare phase###

 

1. modify /etc/hosts(all nodes).

 1 vim /etc/hosts
 2 cat /etc/hosts
 3 #public ip
 4 192.168.56.11 rac1    rac1-pub
 5 192.168.56.12 rac2    rac2-pub
 6 #private ip    
 7 10.10.10.11    rac1-priv
 8 10.10.10.12    rac2-priv
 9 #virtual ip
10 192.168.56.13 rac1-vip
11 192.168.56.14 rac2-vip
12 #scan ip
13 192.168.56.15 rac-scan

 

2. disable firewall & selinux(all nodes).

1 systemctl stop firewalld
2 systemctl status firewalld
3 systemctl status firewalld
4 vim /etc/selinux/config
5 cat /etc/selinux/config
6 selinux=disabled
7 sestatus -v
8 selinux status:                 disabled

 

3. install packages(all nodes).

1 yum -y install binutils compat-db compat-libcap1 compat-libstdc++-33 control-center elfutils-libelf-devel gcc gcc-c++ glibc glibc-common gnome-libs libaio-devel libstdc++ libstdc++-devel make pdksh sysstat xscreensaver libaio openmotif21 smartmontools

 

4. configure ssh connectivities(all nodes,both grid & oracle).

 1 rac1:
 2 ssh-keygen -t rsa
 3 rac2:
 4 ssh-keygen -t rsa
 5 rac1:
 6 cat  ~/.ssh/id_rsa.pub  >> ~/.ssh/authorized_keys
 7 ssh rac2 cat ~/.ssh/id_rsa.pub  >> ~/.ssh/authorized_keys
 8 rac2:
 9 cat  ~/.ssh/id_rsa.pub  >> ~/.ssh/authorized_keys
10 ssh rac1 cat ~/.ssh/id_rsa.pub  >> ~/.ssh/authorized_keys
11 
12 //test execute command on remote host without password.
13 ssh rac1 date
14 ssh rac2 date

 

5. add user & usergroups(all nodes).

1 groupadd oinstall
2 groupadd dba
3 groupadd oper
4 groupadd asmdba    
5 groupadd asmoper
6 groupadd asmadmin 
7 useradd -g oinstall -g dba,oper,asmdba oracle
8 useradd -g oinstall -g asmdba,dba,asmadmin,asmoper grid

 

6. create directory & grant privileges(all nodes).

1 mkdir /u01
2 mkdir /u01/gridbase
3 mkdir /u01/gridhome
4 mkdir /u01/oracle
5 chown -r grid:oinstall /u01
6 chown -r oracle:oinstall /u01/oracle
7 chmod -r g+w /u01

 

7. modify environment variables((all nodes,both grid & oracle).

 1 grid(rac1):
 2 vim /home/grid/.bash_profile
 3 cat /home/grid/.bash_profile
 4 oracle_base = /u01/gridbase
 5 oracle_home = /u01/gridhome
 6 oracle_sid = +asm1
 7 path = $oracle_home/bin:$path
 8 ld_library_path = $oracle_home/lib:$ld_library_path
 9 export oracle_base oracle_home oracle_sid path ld_library_path
10 
11 grid(rac2):
12 vim /home/grid/.bash_profile
13 cat /home/grid/.bash_profile
14 oracle_base = /u01/gridbase
15 oracle_home = /u01/gridhome
16 oracle_sid = +asm2
17 path=$oracle_home/bin:$path
18 ld_library_path = $oracle_home/lib:$ld_library_path
19 export oracle_base oracle_home oracle_sid path ld_library_path
20 
21 oracle(rac1):
22 vim /home/oracle/.bash_profile
23 cat /home/oracle/.bash_profile
24 oracle_base = /u01/oracle
25 oracle_home = /u01/oracle/db
26 oracle_sid = ora11g1
27 path = $oracle_home/bin:$path
28 ld_library_path = $oracle_home/lib:$ld_library_path
29 export oracle_base oracle_home oracle_sid path ld_library_path
30 
31 oracle(rac2):
32 vim /home/oracle/.bash_profile
33 cat /home/oracle/.bash_profile
34 oracle_base = /u01/oracle
35 oracle_home = /u01/oracle/db
36 oracle_sid = ora11g2
37 path = $oracle_home/bin:$path
38 ld_library_path = $oracle_home/lib:$ld_library_path
39 export oracle_base oracle_home oracle_sid path ld_library_path
40 
41 //make it take effect(rac1 & rac2).
42 source /home/oracle/.bash_profile
43 source /home/grid/.bash_profile

 

8. modify kernel parameters(all nodes).

 1 vim /etc/sysctl.conf
 2 cat vim /etc/sysctl.conf
 3 kernel.msgmnb = 65536 
 4 kernel.msgmax = 65536 
 5 kernel.shmmax = 1073741824 //1g memory for oracle.
 6 kernel.shmall = 2097152 
 7 kernel.shmmni = 4096 
 8 fs.aio-max-nr = 1048576 
 9 fs.file-max = 6815744 
10 kernel.sem = 250 32000 100 128 
11 net.ipv4.ip_local_port_range = 9000 65500 
12 net.core.rmem_default = 262144 
13 net.core.rmem_max = 4194304 
14 net.core.wmem_default = 262144 
15 net.core.wmem_max = 1048576
16 
17 //make the kernel parameters take effect.
18 sysctl -p

 

9. modify limites(all nodes).

 1 vim /etc/security/limits.conf
 2 cat /etc/security/limits.conf
 3 #oracle
 4 oracle soft nproc 2047 
 5 oracle hard nproc 16384 
 6 oracle soft nofile 1024 
 7 oracle hard nofile 65536 
 8 oracle soft stack 10240 
 9 #grid
10 grid soft nproc 2047 
11 grid hard nproc 16384 
12 grid soft nofile 1024 
13 grid hard nofile 65536 
14 grid soft stack 10240

 

10. modify pam limits(all nodes).

1 vim /etc/pam.d/login
2 cat /etc/pam.d/login
3 session required /lib64/security/pam_limits.so 
4 session required pam_limits.so

 

11. modify profile(all nodes).

1 vim /etc/profile
2 cat /etc/profile
3 if [ $user = "oracle" ]||[$user = "grid" ]; then if [ $shell = "/bin/ksh" ]; then ulimit -p 16384 ulimit -n 65536 else
4 43/ 136
5 ulimit -u 16384 -n 65536
6 fi
7 fi

 

###step 2.create shared disks with asmlib###

 

1. create three new disks and make partitions on them with "fdisk"(rac1 only).

 1 fdisk -l
 2 fdisk /dev/sdc
 3 n
 4 enter
 5 enter
 6 enter
 7 w
 8 
 9 fdisk /dev/sdd
10 n
11 enter
12 enter
13 enter
14 w
15 
16 fdisk /dev/sde
17 n
18 enter
19 enter
20 enter
21 w
22 
23 ll /dev/sd*
24 ...
25 /dev/sdc1
26 /dev/sdd1
27 /dev/sde1

 

2. install asmlib packages(both rac1 & rac2).

1 yum -y localinstall kmod-oracleasm oracleasm-support oracleasmlib

 

3. bind the shared disks with "oracleasm"(root execute).

 1 rac1:
 2 oracleasm configure -i
 3 grid
 4 asmadmin
 5 y
 6 y
 7 
 8 oracleasm init
 9 
10 oracleasm createdisk asmdisk1 sdc1
11 oracleasm createdisk asmdisk2 sdd1
12 oracleasm createdisk asmdisk3 sde1
13 
14 oracleasm listdisks
15 asmdisk1
16 asmdisk2
17 asmdisk3
18 
19 oracleasm scandisks
20 
21 rac2:
22 oracleasm  configure -i
23 grid
24 asmadmin
25 y
26 y
27 
28 oracleasm init
29 
30 //reboot rac2 and check the shared disks.
31 oracleasm listdisks
32 asmdisk1
33 asmdisk2
34 asmdisk3

 

###step 3.software installation(rac1 only)###

 

1. unzip the grid software package.

1 cd /u01
2 unzip p13390677_112040_linux-x86-64_3of7.zip
3 chown -r grid:oinstall  /u01/grid

 

2. check the prerequisite of installation.

1 cd /u01/grid
2 ./runcluvfy.sh stage -pre crsinst -n rac1,rac2 -fixup -verbose

 

3. install x windows system of linux.

1 yum -y groupinstall "x windows system"

 

4. set xhost and display variable.

1 xhost +
2 export display=192.168.56.99:0.0

 

5. install grid software with oracle gui.

1 ./runinstaller -ignoreprereq
2 
3 //i've tried many times not add "-ignoreprereq" option,but the oui will stuck at 61% when checking the semaphore.i'm afraid it's the incompatible poit installing 11g rac on centos 7.2.the picture of stuck point shows below.

 

 

6. execute below scripts in rac1,rac2 in order(this step really took me quit a long time to solve it,so i'd like to put the details below).

  1 /u01/orainventory/orainstroot.sh
  2 /u01/gridhome/root.sh
  3 
  4 i got an error bellow when execute "/u01/gridhome/root.sh" script:
  5 
  6 adding clusterware entries to inittab
  7 ohasd failed to start
  8 failed to start the clusterware. last 20 lines of the alert log follow: 
  9 2018-08-27 03:07:20.078: 
 10 [client(2762)]crs-2101:the olr was formatted using version 3.
 11 2018-08-27 03:13:24.742: 
 12 [client(3948)]crs-2101:the olr was formatted using version 3.
 13 2018-08-27 03:17:25.615: 
 14 [client(5037)]crs-2101:the olr was formatted using version 3.
 15 
 16 //it's an oracle bug 18370031,a patch should be applied before execute the "root.sh".
 17 
 18 su - root
 19 chown -r grid:oinstall /u01/
 20 su - grid
 21 export path=$path:$oracle_home/opatch
 22 mv $oracle_home/opatch $oracle_home/opatch112034
 23 cd /u01
 24 unzip p6880880_112000_linux-x86-64.zip -d $oracle_home
 25 $oracle_home/opatch/opatch version
 26 opatch version: 11.2.0.3.19
 27 
 28 opatch succeeded.
 29 
 30 cd ~
 31 $oracle_home/opatch/ocm/bin/emocmrsp
 32 ocm installation response generator 10.3.7.0.0 - production
 33 copyright (c) 2005, 2012, oracle and/or its affiliates.  all rights reserved.
 34 
 35 provide your email address to be informed of security issues, install and
 36 initiate oracle configuration manager. easier for you if you use your my
 37 oracle support email address/user name.
 38 visit http://www.oracle.com/support/policies.html for details.
 39 email address/user name: 
 40 
 41 you have not provided an email address for notification of security issues.
 42 do you wish to remain uninformed of security issues ([y]es, [n]o) [n]:  y
 43 the ocm configuration response file (ocm.rsp) was successfully created.
 44 
 45 unzip p18370031_112040_linux-x86-64.zip
 46 opatch apply /u01/18370031/ -oh $oracle_home -ocmrf /home/grid/ocm.rsp
 47 oracle interim patch installer version 11.2.0.3.19
 48 copyright (c) 2018, oracle corporation.  all rights reserved.
 49 
 50 
 51 oracle home       : /u01/gridhome
 52 central inventory : /u01/orainventory
 53    from           : /u01/gridhome/orainst.loc
 54 opatch version    : 11.2.0.3.19
 55 oui version       : 11.2.0.4.0
 56 log file location : /u01/gridhome/cfgtoollogs/opatch/opatch2018-08-27_06-28-31am_1.log
 57 
 58 verifying environment and performing prerequisite checks...
 59 opatch continues with these patches:   18370031  
 60 
 61 do you want to proceed? [y|n]
 62 y
 63 user responded with: y
 64 all checks passed.
 65 
 66 please shutdown oracle instances running out of this oracle_home on the local system.
 67 (oracle home = '/u01/gridhome')
 68 
 69 
 70 is the local system ready for patching? [y|n]
 71 y
 72 user responded with: y
 73 backing up files...
 74 applying interim patch '18370031' to oh '/u01/gridhome'
 75 
 76 patching component oracle.crs, 11.2.0.4.0...
 77 
 78 copy failed from '/u01/18370031/files/bin/ohasd.bin' to '/u01/gridhome/bin/ohasd.bin'...  
 79 please verify all applications associated with the oracle home '/u01/gridhome' are shut down. if this is aix, please perform solution documented in note 739963.1 on https://myoraclesupport.oracle.com.
 80 
 81 do you want to retry copying the file? [y|n]
 82 y
 83 user responded with: y
 84 
 85 retry copying the file to '/u01/gridhome/bin/ohasd.bin'.
 86 
 87 the following actions have failed:
 88 copy failed from '/u01/18370031/files/bin/ohasd.bin' to '/u01/gridhome/bin/ohasd.bin'... 
 89 
 90 
 91 do you want to proceed? [y|n]
 92 y  
 93 user responded with: y
 94 patch 18370031 successfully applied.
 95 opatch session completed with warnings.
 96 log file location: /u01/gridhome/cfgtoollogs/opatch/opatch2018-08-27_06-28-31am_1.log
 97 
 98 opatch completed with warnings.
 99 [grid@rac2 u01]$ ps -ef|grep ohasd
100 root     20730     1  0 06:08 ?        00:00:00 /u01/gridhome/bin/ohasd.bin reboot
101 grid     21998 21139  0 06:32 pts/0    00:00:00 grep --color=auto ohasd
102 
103 //if you want to copy it right now,then kill the ohasd process first(i didn't do that).
104 
105 kill -9 20730
106 cp /u01/18370031/ohasd.bin /u01/gridhome/bin/ohasd.bin
107 
108 //if you are using oel 7.2,there's another patch "19404309" need to apply to guarantee the grid softerware can be normally implement.
109 
110 [root@rac1 ~]# /u01/gridhome/root.sh
111 performing root user operation for oracle 11g 
112 
113 the following environment variables are set as:
114     oracle_owner= grid
115     oracle_home=  /u01/gridhome
116 
117 enter the full pathname of the local bin directory: [/usr/local/bin]: 
118 the contents of "dbhome" have not changed. no need to overwrite.
119 the contents of "oraenv" have not changed. no need to overwrite.
120 the contents of "coraenv" have not changed. no need to overwrite.
121 
122 entries will be added to the /etc/oratab file as needed by
123 database configuration assistant when a database is created
124 finished running generic part of root script.
125 now product-specific root actions will be performed.
126 using configuration parameter file: /u01/gridhome/crs/install/crsconfig_params
127 user ignored prerequisites during installation
128 installing trace file analyzer
129 adding clusterware entries to oracle-ohasd.service
130 crs-2672: attempting to start 'ora.mdnsd' on 'rac1'
131 crs-2676: start of 'ora.mdnsd' on 'rac1' succeeded
132 crs-2672: attempting to start 'ora.gpnpd' on 'rac1'
133 crs-2676: start of 'ora.gpnpd' on 'rac1' succeeded
134 crs-2672: attempting to start 'ora.cssdmonitor' on 'rac1'
135 crs-2672: attempting to start 'ora.gipcd' on 'rac1'
136 crs-2676: start of 'ora.cssdmonitor' on 'rac1' succeeded
137 crs-2676: start of 'ora.gipcd' on 'rac1' succeeded
138 crs-2672: attempting to start 'ora.cssd' on 'rac1'
139 crs-2672: attempting to start 'ora.diskmon' on 'rac1'
140 crs-2676: start of 'ora.diskmon' on 'rac1' succeeded
141 crs-2676: start of 'ora.cssd' on 'rac1' succeeded
142 
143 asm created and started successfully.
144 
145 disk group ocrvote created successfully.
146 
147 clscfg: -install mode specified
148 successfully accumulated necessary ocr keys.
149 creating ocr keys for user 'root', privgrp 'root'..
150 operation successful.
151 crs-4256: updating the profile
152 successful addition of voting disk a11c41b9357d4f48bf1b7c5e03c1510a.
153 successfully replaced voting disk group with +ocrvote.
154 crs-4256: updating the profile
155 crs-4266: voting file(s) successfully replaced
156 ##  state    file universal id                file name disk group
157 --  -----    -----------------                --------- ---------
158  1. online   a11c41b9357d4f48bf1b7c5e03c1510a (orcl:asmdisk1) [ocrvote]
159 located 1 voting disk(s).
160 sh: /bin/netstat: no such file or directory
161 crs-2672: attempting to start 'ora.asm' on 'rac1'
162 crs-2676: start of 'ora.asm' on 'rac1' succeeded
163 crs-2672: attempting to start 'ora.ocrvote.dg' on 'rac1'
164 crs-2676: start of 'ora.ocrvote.dg' on 'rac1' succeeded
165 preparing packages...
166 ls: cannot access /usr/sbin/smartctl: no such file or directory
167 /usr/sbin/smartctl not found.
168 error: %pre(cvuqdisk-1.0.9-1.x86_64) scriptlet failed, exit status 1
169 error: cvuqdisk-1.0.9-1.x86_64: install failed
170 configure oracle grid infrastructure for a cluster ... succeeded
171 [root@rac1 ~]# crs_stat -t
172 name           type           target    state     host        
173 ------------------------------------------------------------
174 ora....n1.lsnr ora....er.type online    online    rac1        
175 ora.ocrvote.dg ora....up.type online    online    rac1        
176 ora.asm        ora.asm.type   online    online    rac1        
177 ora.cvu        ora.cvu.type   online    online    rac1        
178 ora.gsd        ora.gsd.type   offline   offline               
179 ora....network ora....rk.type online    online    rac1        
180 ora.oc4j       ora.oc4j.type  online    online    rac1        
181 ora.ons        ora.ons.type   online    online    rac1        
182 ora....sm1.asm application    online    online    rac1        
183 ora.rac1.gsd   application    offline   offline               
184 ora.rac1.ons   application    online    online    rac1        
185 ora.rac1.vip   ora....t1.type online    online    rac1        
186 ora.scan1.vip  ora....ip.type online    online    rac1
187 
188 //omitted the procedure of apply patch "18370031" on rac2.
189 
190 [root@rac2 ~]# /u01/gridhome/root.sh
191 performing root user operation for oracle 11g 
192 
193 the following environment variables are set as:
194     oracle_owner= grid
195     oracle_home=  /u01/gridhome
196 
197 enter the full pathname of the local bin directory: [/usr/local/bin]: 
198 the contents of "dbhome" have not changed. no need to overwrite.
199 the contents of "oraenv" have not changed. no need to overwrite.
200 the contents of "coraenv" have not changed. no need to overwrite.
201 
202 entries will be added to the /etc/oratab file as needed by
203 database configuration assistant when a database is created
204 finished running generic part of root script.
205 now product-specific root actions will be performed.
206 using configuration parameter file: /u01/gridhome/crs/install/crsconfig_params
207 user ignored prerequisites during installation
208 installing trace file analyzer
209 crs-2672: attempting to start 'ora.mdnsd' on 'rac2'
210 crs-2676: start of 'ora.mdnsd' on 'rac2' succeeded
211 crs-2672: attempting to start 'ora.gpnpd' on 'rac2'
212 crs-2676: start of 'ora.gpnpd' on 'rac2' succeeded
213 crs-2672: attempting to start 'ora.cssdmonitor' on 'rac2'
214 crs-2672: attempting to start 'ora.gipcd' on 'rac2'
215 crs-2676: start of 'ora.cssdmonitor' on 'rac2' succeeded
216 crs-2676: start of 'ora.gipcd' on 'rac2' succeeded
217 crs-2672: attempting to start 'ora.cssd' on 'rac2'
218 crs-2672: attempting to start 'ora.diskmon' on 'rac2'
219 crs-2676: start of 'ora.diskmon' on 'rac2' succeeded
220 crs-2676: start of 'ora.cssd' on 'rac2' succeeded
221 sh: /bin/netstat: no such file or directory
222 preparing packages...
223 ls: cannot access /usr/sbin/smartctl: no such file or directory
224 /usr/sbin/smartctl not found.
225 error: %pre(cvuqdisk-1.0.9-1.x86_64) scriptlet failed, exit status 1
226 error: cvuqdisk-1.0.9-1.x86_64: install failed
227 configure oracle grid infrastructure for a cluster ... succeeded
228 [root@rac2 ~]# crs_stat -t
229 name           type           target    state     host        
230 ------------------------------------------------------------
231 ora....n1.lsnr ora....er.type online    online    rac1        
232 ora.ocrvote.dg ora....up.type online    online    rac1        
233 ora.asm        ora.asm.type   online    online    rac1        
234 ora.cvu        ora.cvu.type   online    online    rac1        
235 ora.gsd        ora.gsd.type   offline   offline               
236 ora....network ora....rk.type online    online    rac1        
237 ora.oc4j       ora.oc4j.type  online    online    rac1        
238 ora.ons        ora.ons.type   online    online    rac1        
239 ora....sm1.asm application    online    online    rac1        
240 ora.rac1.gsd   application    offline   offline               
241 ora.rac1.ons   application    online    online    rac1        
242 ora.rac1.vip   ora....t1.type online    online    rac1        
243 ora....sm2.asm application    online    online    rac2        
244 ora.rac2.gsd   application    offline   offline               
245 ora.rac2.ons   application    online    online    rac2        
246 ora.rac2.vip   ora....t1.type online    online    rac2        
247 ora.scan1.vip  ora....ip.type online    online    rac1

 

7. skip the subsequent failure of checking netca & cvu and finish the installation of gi and check crs status.

 1 [root@rac1 u01]# crsctl check crs
 2 crs-4638: oracle high availability services is online
 3 crs-4537: cluster ready services is online
 4 crs-4529: cluster synchronization services is online
 5 crs-4533: event manager is online
 6 [root@rac2 u01]# crsctl check crs
 7 crs-4638: oracle high availability services is online
 8 crs-4537: cluster ready services is online
 9 crs-4529: cluster synchronization services is online
10 crs-4533: event manager is online