CRS-4535: Cannot communicate with Cluster Ready Services

 [root@bamisdbrac02 ~]# /u01/app/19.3/grid/bin/crsctl stop has
CRS-2796: The command may not proceed when Cluster Ready Services is not running
CRS-4687: Shutdown command has completed with errors.
CRS-4000: Command Stop failed, or completed with errors.
[root@bamisdbrac02 ~]#
                                    

[root@bamisdbrac02 ~]# /u01/app/19.3/grid/bin/crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
[root@bamisdbrac02 ~]#

[root@bamisdbrac02 ~]# ps -ef|grep d.bin
root       8147      1  0 Jan05 ?        02:48:40 /u01/app/19.3/grid/bin/ohasd.bin reboot
root       9033      1  0 Jan05 ?        01:04:27 /u01/app/19.3/grid/bin/orarootagent.bin
grid       9407      1  0 Jan05 ?        00:49:11 /u01/app/19.3/grid/bin/oraagent.bin
grid       9533      1  0 Jan05 ?        01:41:13 /u01/app/19.3/grid/bin/evmd.bin
grid       9534      1  0 Jan05 ?        00:43:13 /u01/app/19.3/grid/bin/mdnsd.bin
grid       9746      1  0 Jan05 ?        00:57:33 /u01/app/19.3/grid/bin/gpnpd.bin
grid       9881   9533  0 Jan05 ?        00:43:05 /u01/app/19.3/grid/bin/evmlogger.bin -o /u01/app/19.3/grid/log/[HOSTNAME]/evmd/evmlogger.info -l /u01/app/19.3/grid/log/[HOSTNAME]/evmd/evmlogger.log
grid       9930      1  0 Jan05 ?        01:43:17 /u01/app/19.3/grid/bin/gipcd.bin
root      11470      1  0 Jan05 ?        00:51:48 /u01/app/19.3/grid/bin/cssdmonitor
root      11472      1  3 Jan05 ?        08:36:10 /u01/app/19.3/grid/bin/osysmond.bin
root      11505      1  0 Jan05 ?        00:51:42 /u01/app/19.3/grid/bin/cssdagent
grid      11559      1  0 Jan05 ?        01:38:43 /u01/app/19.3/grid/bin/ocssd.bin
root      13155      1  0 Jan05 ?        02:45:58 /u01/app/19.3/grid/bin/ologgerd -M
root      95882  90626  0 09:18 pts/0    00:00:00 grep --color=auto d.bin
[root@bamisdbrac02 ~]#

Kill all process

[root@bamisdbrac02 ~]# kill -9 8147 9033 9407 9533 9534 9746 9881 9930 11470 11472 11505 11559 13155
[root@bamisdbrac02 ~]# ps -ef|grep d.bin
root      96752  90626  0 09:19 pts/0    00:00:00 grep --color=auto d.bin
[root@bamisdbrac02 ~]#

[root@bamisdbrac02 ~]# /u01/app/19.3/grid/bin/crsctl check crs
CRS-4639: Could not contact Oracle High Availability Services
[root@bamisdbrac02 ~]#

Now Start CLUSTER

[root@bamisdbrac02 ~]# /u01/app/19.3/grid/bin/crsctl start has
CRS-4123: Oracle High Availability Services has been started.
[root@bamisdbrac02 ~]#

[root@bamisdbrac02 ~]# /u01/app/19.3/grid/bin/crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
[root@bamisdbrac02 ~]#  
                                    
[root@bamisdbrac02 ~]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
[root@bamisdbrac02 ~]# oracleasm listdisks
MIS_DATA_01
MIS_DATA_02
MIS_DATA_03
MIS_DATA_04
MIS_DATA_05
MIS_DATA_06
MIS_DATA_07
MIS_DATA_08
MIS_DATA_09
MIS_DATA_10
MIS_DATA_11
MIS_DATA_12
MIS_DATA_13
MIS_DATA_14
MIS_DATA_15
MIS_DATA_16
MIS_FRA_01
MIS_FRA_02
MIS_MGMT
MIS_REDO_01
MIS_REDO_02
MIS_REDO_03
OCR_VOTE
[root@bamisdbrac02 ~]#

Check the CSS service status:

[root@bamisdbrac02 ~]# ps -ef | grep css
root      98196      1  0 09:20 ?        00:00:02 /u01/app/19.3/grid/bin/cssdmonitor
root      98230      1  0 09:20 ?        00:00:02 /u01/app/19.3/grid/bin/cssdagent
grid      98257      1  0 09:20 ?        00:00:04 /u01/app/19.3/grid/bin/ocssd.bin
root     111089  90626  0 09:32 pts/0    00:00:00 grep --color=auto css

Now check if CRS (Cluster Ready Service) is online or not:

[root@bamisdbrac02 ~]# /u01/app/19.3/grid/bin/crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
[root@bamisdbrac02 ~]#

Verify if the CRS process is running:

[root@bamisdbrac02 disks]# ps -fea |grep crsd.bin
root     121203  90626  0 09:45 pts/0    00:00:00 grep --color=auto crsd.bin
[root@bamisdbrac02 disks]#

Then, if this process is not running, you will try to start the CRS through crsd.

./crsctl start res ora.crsd -init

[root@bamisdbrac02 disks]# /u01/app/19.3/grid/bin/crsctl start res ora.crsd -init
CRS-2672: Attempting to start 'ora.ctssd' on 'bamisdbrac02'
The clock on host bamisdbrac02 differs from mean cluster time by 759845901 microseconds. The Cluster Time Synchronization Service will not perform time synchronization because the time difference is beyond the permissible offset of 600 seconds. Details in /u01/app/grid/diag/crs/bamisdbrac02/crs/trace/octssd.trc.
CRS-2674: Start of 'ora.ctssd' on 'bamisdbrac02' failed
CRS-4000: Command Start failed, or completed with errors.
[root@bamisdbrac02 disks]#

Now check that Node1 and Node2 time difference

systemctl status chronyd.service
systemctl enable chronyd.service
systemctl restart chronyd.service
systemctl status chronyd.service
chronyc -a makestep
chronyc -a 'burst 4/4'
chronyc -a makestep



[root@bamisdbrac02 disks]# date
Tue Jan 17 09:54:01 +06 2023

[root@bamisdbrac01 ~]# date
Tue Jan 17 09:54:15 +06 2023
[root@bamisdbrac01 ~]#


[root@bamisdbrac02 disks]# ps -fea |grep crsd.bin
root     139179  90626  0 09:55 pts/0    00:00:00 grep --color=auto crsd.bin

Now try to start the CRS through crsd.

[root@bamisdbrac02 disks]# /u01/app/19.3/grid/bin/crsctl start res ora.crsd -init
CRS-2672: Attempting to start 'ora.ctssd' on 'bamisdbrac02'
CRS-2676: Start of 'ora.ctssd' on 'bamisdbrac02' succeeded
CRS-2672: Attempting to start 'ora.asm' on 'bamisdbrac02'
CRS-2672: Attempting to start 'ora.crsd' on 'bamisdbrac02'
CRS-2676: Start of 'ora.crsd' on 'bamisdbrac02' succeeded
CRS-2676: Start of 'ora.asm' on 'bamisdbrac02' succeeded
[root@bamisdbrac02 disks]#

[root@bamisdbrac02 disks]# /u01/app/19.3/grid/bin/crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
[root@bamisdbrac02 disks]#