-
加我公众号(orasos)
加我微信(17813235971)
加我QQ(107644445)
标签云
asm恢复 bbed bootstrap$ dul kcbzib_kcrsds_1 kccpb_sanity_check_2 kcratr_nab_less_than_odr MySQL恢复 obet ORA-00312 ORA-00704 ORA-00742 ORA-01110 ORA-01200 ORA-01555 ORA-01578 ORA-01595 ORA-600 2662 ORA-600 2663 ORA-600 3020 ORA-600 4000 ORA-600 4137 ORA-600 4193 ORA-600 4194 ORA-600 16703 ORA-600 kcbzib_kcrsds_1 ORA-600 KCLCHKBLK_4 ORA-600 kcratr_nab_less_than_odr ORA-15042 ORA-15196 ORACLE 12C oracle dul ORACLE PATCH Oracle Recovery Tools oracle加密恢复 oracle勒索 oracle勒索恢复 oracle异常恢复 ORACLE恢复 Oracle 恢复 ORACLE数据库恢复 oracle碎片 OSD-04016 YOUR FILES ARE ENCRYPTED 比特币加密文章分类
- MongoDB (1)
- MongoDB恢复 (1)
- Others (2)
- 中间件 (2)
- WebLogic (2)
- 操作系统 (112)
- 数据库 (1,883)
- DB2 (22)
- MySQL (82)
- Oracle (1,708)
- Data Guard (53)
- EXADATA (8)
- GoldenGate (24)
- ORA-xxxxx (168)
- ORACLE 12C (72)
- ORACLE 18C (6)
- ORACLE 19C (15)
- ORACLE 21C (3)
- Oracle 23ai (8)
- Oracle ASM (72)
- Oracle Bug (8)
- Oracle RAC (56)
- Oracle 安全 (6)
- Oracle 开发 (28)
- Oracle 监听 (29)
- Oracle备份恢复 (662)
- Oracle安装升级 (108)
- Oracle性能优化 (62)
- 专题索引 (5)
- 勒索恢复 (91)
- PostgreSQL (37)
- pdu工具 (7)
- PostgreSQL恢复 (13)
- SQL Server (34)
- SQL Server恢复 (14)
- TimesTen (7)
- 达梦数据库 (5)
- 达梦恢复 (3)
- 生活娱乐 (2)
- 至理名言 (11)
- 虚拟化 (2)
- VMware (2)
- 软件开发 (51)
- Asp.Net (9)
- JavaScript (12)
- PHP (2)
- 小工具 (34)
- MongoDB (1)
-
最近发表
- aix平台Oracle数据库由于断电无法启动故障处理
- 断电引起ORA-600 3005故障处理
- 记录一次云存储资源到期之后引起的数据库恢复
- Firebird 损坏数据库数据抢救工具—firebirdext
- Oracle 19c 202610补丁（RUs+OJVM）-19.33
- 由于con$.con#值达到极限，触发数据库增加约束报ORA-600 12807错误
- MongoDB 离线提取工具—mongoext
- .bres勒索病毒加密数据库dmp文件可恢复
- ORA-600 krsm_mrp_rebuild_lf_list.lno_mismatch
- 记录一次0丢失的ORA-00354: 损坏重做日志块标头故障恢复
- 不当数据库恢复操作导致一个月数据丢失
- obet快速修复oracle 位图损坏块
- 不太常见的10.2.0.1的oracle redo损坏恢复
- obet forcecopy功能抢救硬件故障中的数据文件
- kcratr_nab_less_than_odr和system坏块故障处理
- 通过obet 恢复system坏块,打开数据库
- obet dbv功能完整说明
- 分享一例运行在aix上的sap系统数据库恢复过程
- OBET-Oracle Block Editor Tool使用说明
- 几乎动用了所有手段的Oracle故障恢复
分类目录归档：数据库
公有云安装19c rac遇到问题—169网段udp异常
应客户要求在xx公有云上面安装19c rac,通过各方的努力,最后安装情况如下
1. 两个节点root.sh执行成功,crs启动正常,asm磁盘组访问正常,但是有一个节点asm实例无法启动,一个节点的db实例无法启动
---节点1
[root@dzbl1 ~]# su - grid
Last login: Thu May 20 12:32:55 CST 2021
[grid@dzbl1 ~]$ ps -ef|grep ASM
grid 477 1 0 May19 ? 00:00:24 /u01/app/19c/grid/bin/tnslsnr ASMNET1LSNR_ASM -no_crs_notify -inherit
grid 22075 22039 0 12:42 pts/1 00:00:00 grep --color=auto ASM
[grid@dzbl1 ~]$ asmcmd
ASMCMD> lsdg
State Type Rebal Sector Logical_Sector Block AU Total_MB Free_MB Req_mir_free_MB Usable_file_MB Offline_disks Voting_files Name
MOUNTED EXTERN N 512 512 4096 4194304 1907344 1904420 0 1904420 0 N DATA/
MOUNTED EXTERN N 512 512 4096 4194304 1150344 1149032 0 1149032 0 N FRA/
MOUNTED EXTERN N 512 512 4096 4194304 14304 13988 0 13988 0 Y SYSTEMDG/
ASMCMD> exit
[grid@dzbl1 ~]$ crsctl status res -t
--------------------------------------------------------------------------------
Name Target State Server State details
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.LISTENER.lsnr
ONLINE ONLINE dzbl1 STABLE
ONLINE ONLINE dzbl2 STABLE
ora.chad
ONLINE ONLINE dzbl1 STABLE
ONLINE ONLINE dzbl2 STABLE
ora.net1.network
ONLINE ONLINE dzbl1 STABLE
ONLINE ONLINE dzbl2 STABLE
ora.ons
ONLINE ONLINE dzbl1 STABLE
ONLINE ONLINE dzbl2 STABLE
ora.proxy_advm
OFFLINE OFFLINE dzbl1 STABLE
OFFLINE OFFLINE dzbl2 STABLE
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.ASMNET1LSNR_ASM.lsnr(ora.asmgroup)
1 ONLINE ONLINE dzbl1 STABLE
2 ONLINE ONLINE dzbl2 STABLE
3 ONLINE OFFLINE STABLE
ora.DATA.dg(ora.asmgroup)
1 ONLINE OFFLINE STABLE
2 ONLINE ONLINE dzbl2 STABLE
3 OFFLINE OFFLINE STABLE
ora.FRA.dg(ora.asmgroup)
1 ONLINE OFFLINE STABLE
2 ONLINE ONLINE dzbl2 STABLE
3 OFFLINE OFFLINE STABLE
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE dzbl2 STABLE
ora.SYSTEMDG.dg(ora.asmgroup)
1 OFFLINE OFFLINE STABLE
2 ONLINE ONLINE dzbl2 STABLE
3 OFFLINE OFFLINE STABLE
ora.asm(ora.asmgroup)
1 ONLINE OFFLINE STABLE
2 ONLINE ONLINE dzbl2 Started,STABLE
3 OFFLINE OFFLINE STABLE
ora.asmnet1.asmnetwork(ora.asmgroup)
1 ONLINE ONLINE dzbl1 STABLE
2 ONLINE ONLINE dzbl2 STABLE
3 OFFLINE OFFLINE STABLE
ora.cvu
1 ONLINE ONLINE dzbl2 STABLE
ora.dzbl1.vip
1 ONLINE ONLINE dzbl1 STABLE
ora.dzbl2.vip
1 ONLINE ONLINE dzbl2 STABLE
ora.dzbldb.db
1 ONLINE OFFLINE STABLE
2 ONLINE ONLINE dzbl2 Open,HOME=/u01/app/o
racle/product/19c/db
_1,STABLE
ora.qosmserver
1 ONLINE ONLINE dzbl2 STABLE
ora.scan1.vip
1 ONLINE ONLINE dzbl2 STABLE
--------------------------------------------------------------------------------
[grid@dzbl1 ~]$
---节点2
[grid@dzbl2 ~]$ ps -ef|grep ASM
grid 2464 1 0 May18 ? 00:00:29 /u01/app/19c/grid/bin/tnslsnr ASMNET1LSNR_ASM -no_crs_notify -inherit
grid 6826 1 0 May19 ? 00:00:09 oracle+ASM2_asmb_dzbldb2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
grid 14089 1 0 12:38 ? 00:00:00 asm_m000_+ASM2
grid 15670 1 0 12:40 ? 00:00:00 oracle+ASM2_crf (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
grid 16503 1 0 May18 ? 00:00:05 asm_pmon_+ASM2
grid 16505 1 0 May18 ? 00:00:04 asm_clmn_+ASM2
grid 16507 1 0 May18 ? 00:00:11 asm_psp0_+ASM2
grid 16518 1 0 12:42 ? 00:00:00 oracle+ASM2 (LOCAL=NO)
grid 16562 1 0 May18 ? 00:18:22 asm_vktm_+ASM2
grid 16567 1 0 May18 ? 00:00:08 asm_gen0_+ASM2
grid 16569 1 0 May18 ? 00:00:02 asm_mman_+ASM2
grid 16573 1 0 May18 ? 00:00:06 asm_gen1_+ASM2
grid 16577 1 0 May18 ? 00:01:13 asm_diag_+ASM2
grid 16579 1 0 May18 ? 00:00:04 asm_ping_+ASM2
grid 16581 1 0 May18 ? 00:00:09 asm_pman_+ASM2
grid 16583 1 0 May18 ? 00:03:08 asm_dia0_+ASM2
grid 16585 1 0 May18 ? 00:01:41 asm_lmon_+ASM2
grid 16587 1 0 May18 ? 00:01:55 asm_lmd0_+ASM2
grid 16589 1 0 May18 ? 00:04:26 asm_lms0_+ASM2
grid 16591 1 0 May18 ? 00:02:13 asm_lmhb_+ASM2
grid 16596 1 0 May18 ? 00:00:02 asm_lck1_+ASM2
grid 16598 1 0 May18 ? 00:00:02 asm_dbw0_+ASM2
grid 16600 1 0 May18 ? 00:00:02 asm_lgwr_+ASM2
grid 16602 1 0 May18 ? 00:00:05 asm_ckpt_+ASM2
grid 16604 1 0 May18 ? 00:00:01 asm_smon_+ASM2
grid 16606 1 0 May18 ? 00:00:02 asm_lreg_+ASM2
grid 16608 1 0 May18 ? 00:00:01 asm_pxmn_+ASM2
grid 16610 1 0 May18 ? 00:00:11 asm_rbal_+ASM2
grid 16612 1 0 May18 ? 00:00:24 asm_gmon_+ASM2
grid 16614 1 0 May18 ? 00:00:06 asm_mmon_+ASM2
grid 16616 1 0 May18 ? 00:00:47 asm_mmnl_+ASM2
grid 16618 1 0 May18 ? 00:02:52 asm_imr0_+ASM2
grid 16627 1 0 May18 ? 00:00:30 asm_scm0_+ASM2
grid 16633 1 0 May18 ? 00:00:11 asm_lck0_+ASM2
grid 16662 1 0 May18 ? 00:07:10 asm_gcr0_+ASM2
grid 16699 1 0 May19 ? 00:00:00 oracle+ASM2 (LOCAL=NO)
grid 16746 1 0 May18 ? 00:00:06 asm_asmb_+ASM2
grid 16748 1 0 May18 ? 00:00:13 oracle+ASM2_asmb_+asm2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
grid 16756 1 0 May18 ? 00:00:00 oracle+ASM2_ocr (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
grid 17567 1 0 May18 ? 00:00:00 oracle+ASM2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
grid 17622 17536 0 12:43 pts/1 00:00:00 grep --color=auto ASM
grid 27829 1 0 May18 ? 00:00:00 oracle+ASM2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
[grid@dzbl2 ~]$ asmcmd
ASMCMD> lsdg
State Type Rebal Sector Logical_Sector Block AU Total_MB Free_MB Req_mir_free_MB Usable_file_MB Offline_disks Voting_files Name
MOUNTED EXTERN N 512 512 4096 4194304 1907344 1904420 0 1904420 0 N DATA/
MOUNTED EXTERN N 512 512 4096 4194304 1150344 1149032 0 1149032 0 N FRA/
MOUNTED EXTERN N 512 512 4096 4194304 14304 13988 0 13988 0 Y SYSTEMDG/
ASMCMD> exit
[grid@dzbl2 ~]$ crsctl stat res -t
--------------------------------------------------------------------------------
Name Target State Server State details
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.LISTENER.lsnr
ONLINE ONLINE dzbl1 STABLE
ONLINE ONLINE dzbl2 STABLE
ora.chad
ONLINE ONLINE dzbl1 STABLE
ONLINE ONLINE dzbl2 STABLE
ora.net1.network
ONLINE ONLINE dzbl1 STABLE
ONLINE ONLINE dzbl2 STABLE
ora.ons
ONLINE ONLINE dzbl1 STABLE
ONLINE ONLINE dzbl2 STABLE
ora.proxy_advm
OFFLINE OFFLINE dzbl1 STABLE
OFFLINE OFFLINE dzbl2 STABLE
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.ASMNET1LSNR_ASM.lsnr(ora.asmgroup)
1 ONLINE ONLINE dzbl1 STABLE
2 ONLINE ONLINE dzbl2 STABLE
3 ONLINE OFFLINE STABLE
ora.DATA.dg(ora.asmgroup)
1 ONLINE OFFLINE STABLE
2 ONLINE ONLINE dzbl2 STABLE
3 OFFLINE OFFLINE STABLE
ora.FRA.dg(ora.asmgroup)
1 ONLINE OFFLINE STABLE
2 ONLINE ONLINE dzbl2 STABLE
3 OFFLINE OFFLINE STABLE
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE dzbl2 STABLE
ora.SYSTEMDG.dg(ora.asmgroup)
1 OFFLINE OFFLINE STABLE
2 ONLINE ONLINE dzbl2 STABLE
3 OFFLINE OFFLINE STABLE
ora.asm(ora.asmgroup)
1 ONLINE OFFLINE STABLE
2 ONLINE ONLINE dzbl2 Started,STABLE
3 OFFLINE OFFLINE STABLE
ora.asmnet1.asmnetwork(ora.asmgroup)
1 ONLINE ONLINE dzbl1 STABLE
2 ONLINE ONLINE dzbl2 STABLE
3 OFFLINE OFFLINE STABLE
ora.cvu
1 ONLINE ONLINE dzbl2 STABLE
ora.dzbl1.vip
1 ONLINE ONLINE dzbl1 STABLE
ora.dzbl2.vip
1 ONLINE ONLINE dzbl2 STABLE
ora.dzbldb.db
1 ONLINE OFFLINE STABLE
2 ONLINE ONLINE dzbl2 Open,HOME=/u01/app/o
racle/product/19c/db
_1,STABLE
ora.qosmserver
1 ONLINE ONLINE dzbl2 STABLE
ora.scan1.vip
1 ONLINE ONLINE dzbl2 STABLE
--------------------------------------------------------------------------------
[grid@dzbl2 ~]$
2. 分析db和asm有一个实例无法启动原因分析
--实例启动报错
SQL> startup
ORA-03113: end-of-file on communication channel
--无法启动节点alert日志
2021-05-19T12:41:32.143124+08:00
NOTE: ASMB (index:0) registering with ASM instance as Flex client 0xffffffffffffffff (reg:2449521867) (startid:1072960888) (new connection)
2021-05-19T12:41:32.349766+08:00
My CSS node number is 1
My CSS hostname is dzbl1
lmon registered with NM - instance number 1 (internal mem no 0)
2021-05-19T12:41:34.054865+08:00
Using default pga_aggregate_limit of 16384 MB
2021-05-19T12:42:16.978085+08:00
No connectivity to other instances in the cluster during startup. Hence, LMON is terminating the instance. Please check the LMON trace file for details.
Also, please check the network logs of this instance along with clusterwide network health for problems and then re-start this instance.
LMON (ospid: ): terminating the instance due to ORA error
Cause - 'Instance is being terminated by LMON'
2021-05-19T12:42:17.115807+08:00
System state dump requested by (instance=1, osid=29660 (LMON)), summary=[abnormal instance termination]. error - 'Instance is terminating.
System State dumped to trace file /u01/app/oracle/diag/rdbms/dzbldb/dzbldb1/trace/dzbldb1_diag_29641.trc
2021-05-19T12:42:17.227469+08:00
Dumping diagnostic data in directory=[cdmp_20210519124217], requested by (instance=1, osid=29660 (LMON)), summary=[abnormal instance termination].
2021-05-19T12:42:18.344481+08:00
Instance terminated by LMON, pid = 29660
--正常节点lmon日志
*** 2021-05-19T12:42:29.348455+08:00
IPCLW:[0.16]{-}[CNCT]:PROTO: [1621399349248289]Warning! ACNH://0x7f3d993a7990/peer=[UNKNWN]&ospid=0&msn=993097808&seq=995707504
(169.254.14.18:32056) has outstanding sends during delete.
IPCLW:[0.17]{-}[CNCT]:UTIL: [1621399349248289] ACNH 0x7f3d993a7990 State: 2 SMSN: 993097806 PKT(993097808.995707504) # Pending: 2
IPCLW:[0.18]{-}[CNCT]:UTIL: [1621399349248289] Peer: [UNKNWN].0 AckSeq: 0
IPCLW:[0.19]{-}[CNCT]:UTIL: [1621399349248289] Flags: 0x40000000 IHint: 0x30693d920000001f THint: 0x0
IPCLW:[0.20]{-}[CNCT]:UTIL: [1621399349248289] Local Address: 169.254.17.231:19443 Remote Address: 169.254.14.18:32056
IPCLW:[0.21]{-}[CNCT]:UTIL: [1621399349248289] Remote PID: ver 0 flags 1 trans 2 tos 0 opts 0 xdata3 165f xdata2 70dbd629
IPCLW:[0.22]{-}[CNCT]:UTIL: [1621399349248289] : mmsz 32768 mmr 4096 mms 4096 xdata c2a71bf9
IPCLW:[0.23]{-}[CNCT]:UTIL: [1621399349248289] IVPort: 46944 TVPort: 7161 IMPT: 25433 RMPT: 5727 Pending Sends: Yes Unacked Sends: Yes
IPCLW:[0.24]{-}[CNCT]:UTIL: [1621399349248289] Send Engine Queued: No sshdl -1 ssts 0 rtts 0 snderrchk 0 creqcnt 19 credits 0/0
IPCLW:[0.25]{-}[CNCT]:UTIL: [1621399349248289] Unackd Messages 993097806 -> 993097807. SSEQ 995707502 Send Time:
INVALID TIME SMSN # Xmits: 0 EMSN INVALID TIME
IPCLW:[0.26]{-}[CNCT]:UTIL: [1621399349248289] Pending send queue:
IPCLW:[0.27]{-}[CNCT]:UTIL: [1621399349248289] [0] mbuf 0x7f3d99397770 MSN 993097806 Seq 995707502 -> 995707503 # XMits: 0
IPCLW:[0.28]{-}[CNCT]:UTIL: [1621399349248289] [1] mbuf 0x7f3d99397350 MSN 993097807 Seq 995707503 -> 995707504 # XMits: 0
kjxgfipccb: msg 0x7f3d9934a680, mbo 0x7f3d9934a670, type 24, ack 0, ref 0, stat 34
kjxgfipccb: msg 0x7f3d9934a878, mbo 0x7f3d9934a868, type 18, ack 0, ref 0, stat 34
从日志看异常节点的169.254.14.18:32056和169.254.17.231:19443无法使用udp进行通讯,参考:Only One Instance of a RAC Database Can Start at a Time: Second Instance Fails to Start due to “No reconfig messages from other instances” – LMON is terminating the instance (Doc ID 2528588.1),从而使得asm和db实例只能启动一个节点.到目前为止,初步看很可能是公有云的对于169.254网段的某些限制导致.
对于两个节点asm磁盘组mount,crs正常启动.这个是由于使用的是fiex asm技术实现(在asm实例启动正常情况下直接启动本地asm实例,在本地asm实例无法正常启动,通过fiex asm实现磁盘组正常mount)
记录在linux 7中安装11.2.0.4 RAC遇到问题
还有客户要求在linux 7平台中安装11.2.0.4的RAC,在这次的安装过程中,参考了Installation walk-through – Oracle Grid/RAC 11.2.0.4 on Oracle Linux 7 (Doc ID 1951613.1)文档,在安装grid的时候,执行root.sh执行,打上了patch 18370031,root.sh脚本非常顺利执行成功
打patch 18370031操作
一定要在gi安装执行root.sh之前打该patch
[grid@ptdb1 soft]$ $ORACLE_HOME/OPatch/opatch napply -oh $ORACLE_HOME -local /tmp/soft/18370031 Oracle Interim Patch Installer version 11.2.0.3.27 Copyright (c) 2021, Oracle Corporation. All rights reserved. Oracle Home : /u01/app/11.2.0/grid Central Inventory : /u01/app/oraInventory from : /u01/app/11.2.0/grid/oraInst.loc OPatch version : 11.2.0.3.27 OUI version : 11.2.0.4.0 Log file location : /u01/app/11.2.0/grid/cfgtoollogs/opatch/opatch2021-05-19_20-06-19PM_1.log Verifying environment and performing prerequisite checks... -------------------------------------------------------------------------------- Start OOP by Prereq process. Launch OOP... Oracle Interim Patch Installer version 11.2.0.3.27 Copyright (c) 2021, Oracle Corporation. All rights reserved. Oracle Home : /u01/app/11.2.0/grid Central Inventory : /u01/app/oraInventory from : /u01/app/11.2.0/grid/oraInst.loc OPatch version : 11.2.0.3.27 OUI version : 11.2.0.4.0 Log file location : /u01/app/11.2.0/grid/cfgtoollogs/opatch/opatch2021-05-19_20-06-40PM_1.log Verifying environment and performing prerequisite checks... OPatch continues with these patches: 18370031 Do you want to proceed? [y|n] y User Responded with: Y All checks passed. Please shutdown Oracle instances running out of this ORACLE_HOME on the local system. (Oracle Home = '/u01/app/11.2.0/grid') Is the local system ready for patching? [y|n] y User Responded with: Y Backing up files... Applying interim patch '18370031' to OH '/u01/app/11.2.0/grid' Patching component oracle.crs, 11.2.0.4.0... Patch 18370031 successfully applied. Log file location: /u01/app/11.2.0/grid/cfgtoollogs/opatch/opatch2021-05-19_20-06-40PM_1.log OPatch succeeded.
两个节点执行root.sh操作
[root@ptdb1 rpm]# /u01/app/11.2.0/grid/root.sh
Performing root user operation for Oracle 11g
The following environment variables are set as:
ORACLE_OWNER= grid
ORACLE_HOME= /u01/app/11.2.0/grid
Enter the full pathname of the local bin directory: [/usr/local/bin]:
Copying dbhome to /usr/local/bin ...
Copying oraenv to /usr/local/bin ...
Copying coraenv to /usr/local/bin ...
Creating /etc/oratab file...
Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Using configuration parameter file: /u01/app/11.2.0/grid/crs/install/crsconfig_params
Creating trace directory
User ignored Prerequisites during installation
Installing Trace File Analyzer
OLR initialization - successful
root wallet
root wallet cert
root cert export
peer wallet
profile reader wallet
pa wallet
peer wallet keys
pa wallet keys
peer cert request
pa cert request
peer cert
pa cert
peer root cert TP
profile reader root cert TP
pa root cert TP
peer pa cert TP
pa peer cert TP
profile reader pa cert TP
profile reader peer cert TP
peer user cert
pa user cert
Adding Clusterware entries to oracle-ohasd.service
CRS-2672: Attempting to start 'ora.mdnsd' on 'ptdb1'
CRS-2676: Start of 'ora.mdnsd' on 'ptdb1' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'ptdb1'
CRS-2676: Start of 'ora.gpnpd' on 'ptdb1' succeeded
CRS-2672: Attempting to start 'ora.cssdmonitor' on 'ptdb1'
CRS-2672: Attempting to start 'ora.gipcd' on 'ptdb1'
CRS-2676: Start of 'ora.cssdmonitor' on 'ptdb1' succeeded
CRS-2676: Start of 'ora.gipcd' on 'ptdb1' succeeded
CRS-2672: Attempting to start 'ora.cssd' on 'ptdb1'
CRS-2672: Attempting to start 'ora.diskmon' on 'ptdb1'
CRS-2676: Start of 'ora.diskmon' on 'ptdb1' succeeded
CRS-2676: Start of 'ora.cssd' on 'ptdb1' succeeded
ASM created and started successfully.
Disk Group SYSDG created successfully.
clscfg: -install mode specified
Successfully accumulated necessary OCR keys.
Creating OCR keys for user 'root', privgrp 'root'..
Operation successful.
CRS-4256: Updating the profile
Successful addition of voting disk 2ad9ee4ec49f4fe2bf80f0c7006bd395.
Successfully replaced voting disk group with +SYSDG.
CRS-4256: Updating the profile
CRS-4266: Voting file(s) successfully replaced
## STATE File Universal Id File Name Disk group
-- ----- ----------------- --------- ---------
1. ONLINE 2ad9ee4ec49f4fe2bf80f0c7006bd395 (/dev/sdb) [SYSDG]
Located 1 voting disk(s).
CRS-2672: Attempting to start 'ora.asm' on 'ptdb1'
CRS-2676: Start of 'ora.asm' on 'ptdb1' succeeded
CRS-2672: Attempting to start 'ora.SYSDG.dg' on 'ptdb1'
CRS-2676: Start of 'ora.SYSDG.dg' on 'ptdb1' succeeded
Configure Oracle Grid Infrastructure for a Cluster ... succeeded
--节点2
[root@ptdb2 ~]# /u01/app/11.2.0/grid/root.sh
Performing root user operation for Oracle 11g
The following environment variables are set as:
ORACLE_OWNER= grid
ORACLE_HOME= /u01/app/11.2.0/grid
Enter the full pathname of the local bin directory: [/usr/local/bin]:
Copying dbhome to /usr/local/bin ...
Copying oraenv to /usr/local/bin ...
Copying coraenv to /usr/local/bin ...
Creating /etc/oratab file...
Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Using configuration parameter file: /u01/app/11.2.0/grid/crs/install/crsconfig_params
Creating trace directory
User ignored Prerequisites during installation
Installing Trace File Analyzer
OLR initialization - successful
Adding Clusterware entries to oracle-ohasd.service
CRS-4402: The CSS daemon was started in exclusive mode but found an active CSS daemon on node ptdb1,
number 1, and is terminating
An active cluster was found during exclusive startup, restarting to join the cluster
Configure Oracle Grid Infrastructure for a Cluster ... succeeded
不安装该patch的,手动在systemd中添加ohasd服务
在root.sh执行过程中，待/etc/init.d/目录下生成了init.ohasd 文件后执行systemctl start ohasd.service 启动ohasd服务即可。若没有/etc/init.d/init.ohasd文件 systemctl start ohasd.service 则会启动失败,如果没有及时启动ohasd服务,导致root.sh执行失败,可以在启动ohasd服务之后,重新执行root.sh
(1).创建一个空服务文件：/usr/lib/systemd/system/ohasd.service touch /usr/lib/systemd/system/ohasd.service (2).编辑文件ohasd.service添加如下内容 vi /usr/lib/systemd/system/ohasd.service [Unit] Description=Oracle High Availability Services After=syslog.target [Service] ExecStart=/etc/init.d/init.ohasd run >/dev/null 2>&1 Type=simple Restart=always [Install] WantedBy=multi-user.target (3).添加和启动服务 systemctl daemon-reload systemctl enable ohasd.service systemctl start ohasd.service
安装db软件时无法发现节点
通过参考mos文档 Database runInstaller “Nodes Selection” Window Does Not Show Cluster Nodes (Doc ID 1327486.1),应该是在安装过程中inventory信息异常导致,采用一下方法解决
[grid@ptdb1 bin]$ pwd
/u01/app/11.2.0/grid/oui/bin
[grid@ptdb1 bin]$ ./runInstaller -silent -ignoreSysPrereqs -updateNodeList
>ORACLE_HOME=$ORACLE_HOME LOCAL_NODE="ptdb1" CLUSTER_NODES="{ptdb1,ptdb2}" CRS=true
Starting Oracle Universal Installer...
Checking swap space: must be greater than 500 MB. Actual 8062 MB Passed
The inventory pointer is located at /etc/oraInst.loc
The inventory is located at /u01/app/oraInventory
'UpdateNodeList' was successful.
INS-35423错误解决
通过查询mos发现Window: Prerequisite Error INS-35423 And Warnings When Installing RAC Database Software (Doc ID 2164220.1)文章,虽然该文章描述的是windows环境,根据经验,我这个情况类似,按照该方法处理后正常
./runInstaller -jreLoc /etc/alternatives/jre_1.8.0 ORACLE_HOSTNAME="ptdb1"
安装db遇到Error in invoking target ‘agent nmhs’ of makefile ‘
这个是一个特别常见的错误,如果不使用dbcontrol,该错误可以直接忽略,或者采用类似方法处理
对$ORACLE_HOME/sysman/lib/ins_emagent.mk文件中的$(MK_EMAGENT_NMECTL)修改为$(MK_EMAGENT_NMECTL) -lnnz11,然后重试
如果要使用db console,如果在数据库安装完成之后打上patch 19692824
参考mos文档Installation of Oracle 11.2.0.4 Database Software on OL7 fails with ‘Error in invoking target ‘agent nmhs’ of makefile ‘ & “undefined reference to symbol ‘B_DestroyKeyObject’” error (Doc ID 1965691.1)
ORA-00600: internal error code, arguments: [16703], [1403], [4] 故障处理
有一个客户数据库遭遇ORA-600 16703错误,故障原因见:警告：互联网中有oracle介质被注入恶意程序导致—ORA-600 16703,这种故障已经恢复比较多,在这次的恢复中遇到一个新错误,进行记录
接手给客户报错情况ORA-00600: internal error code, arguments: [16703], [1403], [20]
Thu May 13 22:36:11 2021 SMON: enabling cache recovery Thu May 13 22:36:11 2021 NSA2 started with pid=61, OS id=6261 Archived Log entry 90224 added for thread 1 sequence 50454 ID 0x19ae1c6c dest 1: Errors in file /oracle/diag/rdbms/xff/xff/trace/xff_ora_5931.trc (incident=741052): ORA-00600: internal error code, arguments: [16703], [1403], [20], [], [], [], [], [], [], [], [], [] Incident details in: /oracle/diag/rdbms/xff/xff/incident/incdir_741052/xff_ora_5931_i741052.trc Use ADRCI or Support Workbench to package the incident. See Note 411.1 at My Oracle Support for error and packaging details. Errors in file /oracle/diag/rdbms/xff/xff/trace/xff_ora_5931.trc: ORA-00704: bootstrap process failure ORA-00704: bootstrap process failure ORA-00600: internal error code, arguments: [16703], [1403], [20], [], [], [], [], [], [], [], [], [] Errors in file /oracle/diag/rdbms/xff/xff/trace/xff_ora_5931.trc: ORA-00704: bootstrap process failure ORA-00704: bootstrap process failure ORA-00600: internal error code, arguments: [16703], [1403], [20], [], [], [], [], [], [], [], [], [] Error 704 happened during db open, shutting down database USER (ospid: 5931): terminating the instance due to error 704 Instance terminated by USER, pid = 5931 ORA-1092 signalled during: alter database open... opiodr aborting process unknown ospid (5931) as a result of ORA-1092 Thu May 13 22:36:13 2021 ORA-1092 : opitsk aborting process
这种故障,由于是恶意脚本在数据库启动的时候清空tab$所致,使用bbed对tab$进行反向删除操作,实现数据库打开.
在这次的恢复过程中遇到ORA-600 16703 1403 4的错误
SQL> startup mount pfile='/tmp/pfile'; ORACLE instance started. Total System Global Area 7.0818E+10 bytes Fixed Size 2260448 bytes Variable Size 1.3422E+10 bytes Database Buffers 5.7177E+10 bytes Redo Buffers 217030656 bytes Database mounted. SQL> alter database open ; alter database open * ERROR at line 1: ORA-01092: ORACLE instance terminated. Disconnection forced ORA-00704: bootstrap process failure ORA-00704: bootstrap process failure ORA-00600: internal error code, arguments: [16703], [1403], [4], [], [], [], [], [], [], [], [], [] Process ID: 51886 Session ID: 3497 Serial number: 3
根据ora-600 16703 1403 4基本上可以判断是由于tab$这个表中缺少obj#=4的对象导致,通过查询正常库,确认是该对象为tab$,也就是说由于tab$对象中少了tab$记录.通过bbed分析确认
SQL> SELECT a.OBJ#
2 ,TAB#
3 ,a.DATAOBJ#
4 ,BOBJ#
5 ,NAME
6 ,DBMS_ROWID.ROWID_RELATIVE_FNO (a.ROWID) FILE_ID
7 ,DBMS_ROWID.ROWID_BLOCK_NUMBER (a.ROWID) BLOCK_ID
8 FROM TAB$ a, obj$ b
9 WHERE a.obj# = b.obj#
10 AND A.OBJ# IN (4);
OBJ# TAB# DATAOBJ# BOBJ# NAME
---------- ---------- ---------- ---------- ------------------------------
FILE_ID BLOCK_ID
---------- ----------
4 1 2 2 TAB$
1 147
BBED> set dba 1,147
DBA 0x00400093 (4194451 1,147)
BBED> x /rnnnnnnnnnnnnncnnnnnnnntnnnnnnnnnncct *kdbr[14]
rowdata[6848] @7349
-------------
flag@7349: 0x20 (KDRHFH)
lock@7350: 0x02
cols@7351: 0
nrid@7352:0x00407b09.1
BBED> set dba 0x00407b09
DBA 0x00407b09 (4225801 1,31497)
BBED> p kdbt[1]
struct kdbt[1], 4 bytes @110
sb2 kdbtoffs @110 10
sb2 kdbtnrow @112 2
BBED> x /rnnnnnnnnnnnnncnnnnnnnntnnnnnnnnnncct *kdbr[11]
rowdata[815] @4436
------------
flag@4436: 0x5c (KDRHFL, KDRHFF, KDRHFD, KDRHFC)
lock@4437: 0x02
cols@4438: 0
ckix@4439: 8
BBED> x /rn *kdbr[8]
rowdata[950] @4571
------------
flag@4571: 0xac (KDRHFL, KDRHFF, KDRHFH, KDRHFK)
lock@4572: 0x00
cols@4573: 1
kref@4574: 1
hrid@4576:0x00400093.8
nrid@4582:0x00400094.0
col 0[2] @4590: 4
确认该记录发生了行迁移导致该问题,对其对应的block进行修复,数据库正常打开.
发表在 非常规恢复
标签为 arguments: [16703], ORA-00600: internal error code, ORA-600 16703, oracle tab$恢复, tab$恢复, [1403], [4]
评论关闭
