[root@bamisdbrac02 ~]# /u01/app/19.3/grid/bin/crsctl stop has
CRS-2796: The command may not proceed when Cluster Ready Services is not running
CRS-4687: Shutdown command has completed with errors.
CRS-4000: Command Stop failed, or completed with errors.
[root@bamisdbrac02 ~]#
[root@bamisdbrac02 ~]# /u01/app/19.3/grid/bin/crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
[root@bamisdbrac02 ~]#
[root@bamisdbrac02 ~]# ps -ef|grep d.bin
root 8147 1 0 Jan05 ? 02:48:40 /u01/app/19.3/grid/bin/ohasd.bin reboot
root 9033 1 0 Jan05 ? 01:04:27 /u01/app/19.3/grid/bin/orarootagent.bin
grid 9407 1 0 Jan05 ? 00:49:11 /u01/app/19.3/grid/bin/oraagent.bin
grid 9533 1 0 Jan05 ? 01:41:13 /u01/app/19.3/grid/bin/evmd.bin
grid 9534 1 0 Jan05 ? 00:43:13 /u01/app/19.3/grid/bin/mdnsd.bin
grid 9746 1 0 Jan05 ? 00:57:33 /u01/app/19.3/grid/bin/gpnpd.bin
grid 9881 9533 0 Jan05 ? 00:43:05 /u01/app/19.3/grid/bin/evmlogger.bin -o /u01/app/19.3/grid/log/[HOSTNAME]/evmd/evmlogger.info -l /u01/app/19.3/grid/log/[HOSTNAME]/evmd/evmlogger.log
grid 9930 1 0 Jan05 ? 01:43:17 /u01/app/19.3/grid/bin/gipcd.bin
root 11470 1 0 Jan05 ? 00:51:48 /u01/app/19.3/grid/bin/cssdmonitor
root 11472 1 3 Jan05 ? 08:36:10 /u01/app/19.3/grid/bin/osysmond.bin
root 11505 1 0 Jan05 ? 00:51:42 /u01/app/19.3/grid/bin/cssdagent
grid 11559 1 0 Jan05 ? 01:38:43 /u01/app/19.3/grid/bin/ocssd.bin
root 13155 1 0 Jan05 ? 02:45:58 /u01/app/19.3/grid/bin/ologgerd -M
root 95882 90626 0 09:18 pts/0 00:00:00 grep --color=auto d.bin
[root@bamisdbrac02 ~]#
Kill all process
[root@bamisdbrac02 ~]# kill -9 8147 9033 9407 9533 9534 9746 9881 9930 11470 11472 11505 11559 13155
[root@bamisdbrac02 ~]# ps -ef|grep d.bin
root 96752 90626 0 09:19 pts/0 00:00:00 grep --color=auto d.bin
[root@bamisdbrac02 ~]#
[root@bamisdbrac02 ~]# /u01/app/19.3/grid/bin/crsctl check crs
CRS-4639: Could not contact Oracle High Availability Services
[root@bamisdbrac02 ~]#
Now Start CLUSTER
[root@bamisdbrac02 ~]# /u01/app/19.3/grid/bin/crsctl start has
CRS-4123: Oracle High Availability Services has been started.
[root@bamisdbrac02 ~]#
[root@bamisdbrac02 ~]# /u01/app/19.3/grid/bin/crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
[root@bamisdbrac02 ~]#
[root@bamisdbrac02 ~]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
[root@bamisdbrac02 ~]# oracleasm listdisks
MIS_DATA_01
MIS_DATA_02
MIS_DATA_03
MIS_DATA_04
MIS_DATA_05
MIS_DATA_06
MIS_DATA_07
MIS_DATA_08
MIS_DATA_09
MIS_DATA_10
MIS_DATA_11
MIS_DATA_12
MIS_DATA_13
MIS_DATA_14
MIS_DATA_15
MIS_DATA_16
MIS_FRA_01
MIS_FRA_02
MIS_MGMT
MIS_REDO_01
MIS_REDO_02
MIS_REDO_03
OCR_VOTE
[root@bamisdbrac02 ~]#
Check the CSS service status:
[root@bamisdbrac02 ~]# ps -ef | grep css
root 98196 1 0 09:20 ? 00:00:02 /u01/app/19.3/grid/bin/cssdmonitor
root 98230 1 0 09:20 ? 00:00:02 /u01/app/19.3/grid/bin/cssdagent
grid 98257 1 0 09:20 ? 00:00:04 /u01/app/19.3/grid/bin/ocssd.bin
root 111089 90626 0 09:32 pts/0 00:00:00 grep --color=auto css
Now check if CRS (Cluster Ready Service) is online or not:
[root@bamisdbrac02 ~]# /u01/app/19.3/grid/bin/crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
[root@bamisdbrac02 ~]#
Verify if the CRS process is running:
[root@bamisdbrac02 disks]# ps -fea |grep crsd.bin
root 121203 90626 0 09:45 pts/0 00:00:00 grep --color=auto crsd.bin
[root@bamisdbrac02 disks]#
Then, if this process is not running, you will try to start the CRS through crsd.
./crsctl start res ora.crsd -init
[root@bamisdbrac02 disks]# /u01/app/19.3/grid/bin/crsctl start res ora.crsd -init
CRS-2672: Attempting to start 'ora.ctssd' on 'bamisdbrac02'
The clock on host bamisdbrac02 differs from mean cluster time by 759845901 microseconds. The Cluster Time Synchronization Service will not perform time synchronization because the time difference is beyond the permissible offset of 600 seconds. Details in /u01/app/grid/diag/crs/bamisdbrac02/crs/trace/octssd.trc.
CRS-2674: Start of 'ora.ctssd' on 'bamisdbrac02' failed
CRS-4000: Command Start failed, or completed with errors.
[root@bamisdbrac02 disks]#
Now check that Node1 and Node2 time difference
systemctl status chronyd.service
systemctl enable chronyd.service
systemctl restart chronyd.service
systemctl status chronyd.service
chronyc -a makestep
chronyc -a 'burst 4/4'
chronyc -a makestep
[root@bamisdbrac02 disks]# date
Tue Jan 17 09:54:01 +06 2023
[root@bamisdbrac01 ~]# date
Tue Jan 17 09:54:15 +06 2023
[root@bamisdbrac01 ~]#
[root@bamisdbrac02 disks]# ps -fea |grep crsd.bin
root 139179 90626 0 09:55 pts/0 00:00:00 grep --color=auto crsd.bin
Now try to start the CRS through crsd.
[root@bamisdbrac02 disks]# /u01/app/19.3/grid/bin/crsctl start res ora.crsd -init
CRS-2672: Attempting to start 'ora.ctssd' on 'bamisdbrac02'
CRS-2676: Start of 'ora.ctssd' on 'bamisdbrac02' succeeded
CRS-2672: Attempting to start 'ora.asm' on 'bamisdbrac02'
CRS-2672: Attempting to start 'ora.crsd' on 'bamisdbrac02'
CRS-2676: Start of 'ora.crsd' on 'bamisdbrac02' succeeded
CRS-2676: Start of 'ora.asm' on 'bamisdbrac02' succeeded
[root@bamisdbrac02 disks]#
[root@bamisdbrac02 disks]# /u01/app/19.3/grid/bin/crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
[root@bamisdbrac02 disks]#
Monday 16 January 2023
CRS-4535: Cannot communicate with Cluster Ready Services
Wednesday 28 December 2022
ORA-01274: cannot add data file that was originally created as '+ADATA/CDBPR/EMOB/undotbs04.dbf
Error In Standby Database:
22-12-28T12:33:32.063138+06:00
EMOB(7):File #581 added to control file as 'UNNAMED00581' because
EMOB(7):the parameter STANDBY_FILE_MANAGEMENT is set to MANUAL
EMOB(7):The file should be manually created to continue.
PR00 (PID:42551): MRP0: Background Media Recovery terminated with error 1274
2022-12-28T12:33:32.097676+06:00
Errors in file /d01/app/oracle/diag/rdbms/cdbdg/cdbdg/trace/cdbdg_pr00_42551.trc:
ORA-01274: cannot add data file that was originally created as '+ADATA/CDBPR/EMOB/undotbs04.dbf'
2022-12-28T12:33:32.098788+06:00
.... (PID:30028): Managed Standby Recovery not using Real Time Apply
2022-12-28T12:33:32.534498+06:00
Recovery interrupted!
2022-12-28T12:33:33.204594+06:00
Solution:
Perform the following steps in Standby Database
SQL> show parameter file
NAME TYPE VALUE
------------------------------------ ----------- ------------------------------
audit_file_dest string /d01/app/oracle/admin/cdbdg/ad
ump
control_file_record_keep_time integer 7
control_files string +ADATA/CDBDG/CONTROLFILE/curre
nt.261.1120417171, +FRA/CDBDG/
CONTROLFILE/current.260.112041
7171
db_create_file_dest string
db_file_multiblock_read_count integer 128
db_file_name_convert string cdbpr, cdbdg
db_files integer 4000
NAME TYPE VALUE
------------------------------------ ----------- ------------------------------
db_flash_cache_file string
db_performance_profile string
db_recovery_file_dest string +FRA
db_recovery_file_dest_size big integer 1800G
db_securefile string PREFERRED
dg_broker_config_file1 string /d01/app/oracle/product/19c/db
_1/dbs/dr1cdbdg.dat
dg_broker_config_file2 string /d01/app/oracle/product/19c/db
_1/dbs/dr2cdbdg.dat
file_mapping boolean FALSE
fileio_network_adapters string
NAME TYPE VALUE
------------------------------------ ----------- ------------------------------
filesystemio_options string asynch
ifile file
log_file_name_convert string cdbpr, cdbdg
max_dump_file_size string unlimited
pdb_file_name_convert string
remote_login_passwordfile string EXCLUSIVE
remote_recovery_file_dest string
session_max_open_files integer 10
spfile string /d01/app/oracle/product/19c/db
_1/dbs/spfilecdbdg.ora
standby_file_management string MANUAL
NAME TYPE VALUE
------------------------------------ ----------- ------------------------------
standby_pdb_source_file_dblink string
standby_pdb_source_file_directory string
tracefile_identifier string
SQL> alter system set standby_file_management=AUTO;
System altered.
SQL>
SQL> show pdbs
CON_ID CON_NAME OPEN MODE RESTRICTED
---------- ------------------------------ ---------- ----------
2 PDB$SEED READ ONLY NO
3 PDB MOUNTED
7 EMOB READ ONLY NO
SQL> alter session set container=EMOB;
Session altered.
SQL> select name from v$datafile where file#=581;
NAME
--------------------------------------------------------------------------------
/d01/app/oracle/product/19c/db_1/dbs/UNNAMED00581
SQL>
SQL> alter database recover managed standby database cancel;
alter database recover managed standby database cancel
*
ERROR at line 1:
ORA-16136: Managed Standby Recovery not active
-bash-5.1$ sqlplus
SQL*Plus: Release 19.0.0.0.0 - Production on Wed Dec 28 12:46:33 2022
Version 19.14.0.0.0
Copyright (c) 1982, 2021, Oracle. All rights reserved.
Enter user-name: /as sysdba
Connected to:
Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production
Version 19.14.0.0.0
SQL> alter system set standby_file_management=MANUAL;
System altered.
SQL> alter session set container=EMOB;
Session altered.
SQL> alter database create datafile '/d01/app/oracle/product/19c/db_1/dbs/UNNAMED00581' as '+ADATA/cdbdg/emob/undotbs04.dbf';
Database altered.
SQL> exit
Disconnected from Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production
Version 19.14.0.0.0
-bash-5.1$ sqlplus
SQL*Plus: Release 19.0.0.0.0 - Production on Wed Dec 28 12:47:16 2022
Version 19.14.0.0.0
Copyright (c) 1982, 2021, Oracle. All rights reserved.
Enter user-name: /as sysdba
Connected to:
Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production
Version 19.14.0.0.0
SQL> alter system set standby_file_management=AUTO;
System altered.
SQL>
SQL> alter database recover managed standby database using current logfile disconnect;
Database altered.
SQL>
Sunday 6 November 2022
CRS-6706: After Grid Infrastructure upgrade from 19.3 to 19.14
Error:
root@absdg1:~# /d01/app/19c/grid/bin/crsctl start has
CRS-6706: Oracle Clusterware Release patch level ('2965637783') does not match Software patch level ('2676783881'). Oracle Clusterware cannot be started.
CRS-4000: Command Start failed, or completed with errors.
root@absdg1:~#
Solution:
root@absdg1:~# /d01/app/19c/grid/bin/clscfg -localpatch
clscfg: EXISTING configuration version 0 detected.
Creating OCR keys for user 'root', privgrp 'root'..
Operation successful.
root@absdg1:~#
root@absdg1:~# /d01/app/19c/grid/cr
crf/ crs/
root@absdg1:~# /d01/app/19c/grid/crs/install/rootcrs.sh -lock
Using configuration parameter file: /d01/app/19c/grid/crs/install/crsconfig_params
The log of current session can be found at:
/d01/app/grid/crsdata/absdg1/crsconfig/crslock_absdg1_2022-11-06_03-56-05PM.log
2022/11/06 15:56:09 CLSRSC-329: Replacing Clusterware entries in file '/etc/inittab'
root@absdg1:~#
root@absdg1:~# /d01/app/19c/grid/bin/crsctl start has
CRS-4123: Oracle High Availability Services has been started.
You have new mail in /var/mail/root
root@absdg1:~# /d01/app/19c/grid/bin/crsctl stat res -t
--------------------------------------------------------------------------------
Name Target State Server State details
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.LISTENER.lsnr
ONLINE OFFLINE absdg1 STABLE
ora.MGMT.GHCHKPT.advm
OFFLINE OFFLINE absdg1 STABLE
ora.helper
OFFLINE OFFLINE absdg1 IDLE,STABLE
ora.mgmt.ghchkpt.acfs
OFFLINE OFFLINE absdg1 STABLE
ora.net1.network
ONLINE ONLINE absdg1 STABLE
ora.ons
ONLINE OFFLINE absdg1 STABLE
ora.proxy_advm
OFFLINE OFFLINE absdg1 STABLE
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.ASMNET1LSNR_ASM.lsnr(ora.asmgroup)
1 ONLINE OFFLINE STABLE
2 OFFLINE OFFLINE STABLE
3 OFFLINE OFFLINE STABLE
ora.LISTENER_SCAN1.lsnr
1 ONLINE OFFLINE STABLE
ora.MGMT.dg(ora.asmgroup)
1 ONLINE OFFLINE STABLE
2 OFFLINE OFFLINE STABLE
3 OFFLINE OFFLINE STABLE
ora.MGMTLSNR
1 ONLINE OFFLINE 169.254.23.178 10.64
.171.50,STABLE
ora.OCRVOTE.dg(ora.asmgroup)
1 ONLINE OFFLINE STABLE
2 OFFLINE OFFLINE STABLE
3 OFFLINE OFFLINE STABLE
ora.absdg1.vip
1 ONLINE OFFLINE STABLE
ora.asm(ora.asmgroup)
1 ONLINE INTERMEDIATE absdg1 Started,STABLE
2 OFFLINE OFFLINE STABLE
3 OFFLINE OFFLINE STABLE
ora.asmnet1.asmnetwork(ora.asmgroup)
1 ONLINE ONLINE absdg1 STABLE
2 OFFLINE OFFLINE STABLE
3 OFFLINE OFFLINE STABLE
ora.cvu
1 ONLINE OFFLINE STABLE
ora.mgmtdb
1 ONLINE OFFLINE Instance Shutdown,ST
ABLE
ora.qosmserver
1 ONLINE OFFLINE STABLE
ora.rhpserver
1 OFFLINE OFFLINE STABLE
ora.scan1.vip
1 ONLINE OFFLINE STABLE
--------------------------------------------------------------------------------
root@absdg1:~#