Shutdown blocked in case shutdown of primary is done after backup recovery and lease thread trying to clear stale replica on same time

Discussed with Yechiel who understand the problem

"GS-LeaseManager$Reaper [CECommonSpace]" #116 daemon prio=5 os_prio=0 tid=0x00007f2ee83d0180 nid=0x549b waiting for monitor entry [0x00007f2ed3df6000]
java.lang.Thread.State: BLOCKED (on object monitor)
at com.gigaspaces.internal.cluster.node.impl.replica.ReplicationNodeReplicaHandler.clearStaleReplicas(ReplicationNodeReplicaHandler.java:279)

• waiting to lock <0x00000000f92131d0> (a com.gigaspaces.internal.cluster.node.impl.replica.ReplicationNodeReplicaHandler)
  at com.gigaspaces.internal.cluster.node.impl.ReplicationNode.clearStaleReplicas(ReplicationNode.java:882)
  at com.gigaspaces.internal.cluster.node.impl.ReplicationNodeAdmin.clearStaleReplicas(ReplicationNodeAdmin.java:231)
  at com.j_spaces.core.LeaseManager$LeaseReaper.reapStaleReplicas(LeaseManager.java:1749)
  at com.j_spaces.core.LeaseManager$LeaseReaper.run(LeaseManager.java:1014)

"GS-LRMI Connection-pool-1-thread-125" #849 daemon prio=5 os_prio=0 tid=0x00007f2f1002fe80 nid=0x3577 in Object.wait() [0x00007f2ed11ce000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at com.j_spaces.core.LeaseManager$LeaseReaper.waitForCycleCompletion(LeaseManager.java:1067)

• locked <0x00000000f9409580> (a java.lang.Object)
  at com.j_spaces.core.LeaseManager.forceLeaseReaperCycle(LeaseManager.java:313)
  at com.j_spaces.core.LeaseManager.forceLeaseReaperCycle(LeaseManager.java:318)
  at com.gigaspaces.internal.cluster.node.impl.replica.SynchronizeReplicaDataProducer.onClose(SynchronizeReplicaDataProducer.java:81)
  at com.gigaspaces.internal.cluster.node.impl.replica.AbstractMultiSpaceReplicaDataProducer.close(AbstractMultiSpaceReplicaDataProducer.java:94)
  - locked <0x00000000f4a45db8> (a com.gigaspaces.internal.cluster.node.impl.replica.SynchronizeReplicaDataProducer)
  at com.gigaspaces.internal.cluster.node.impl.replica.ReplicationNodeReplicaHandler.clearReplicas(ReplicationNodeReplicaHandler.java:305)
  - locked <0x00000000f92131d0> (a com.gigaspaces.internal.cluster.node.impl.replica.ReplicationNodeReplicaHandler)
  at com.gigaspaces.internal.cluster.node.impl.replica.ReplicationNodeReplicaHandler.close(ReplicationNodeReplicaHandler.java:337)
  at com.gigaspaces.internal.cluster.node.impl.ReplicationNode.close(ReplicationNode.java:581)
  at com.gigaspaces.internal.server.space.replication.SpaceReplicationManager.close(SpaceReplicationManager.java:42)
  at com.gigaspaces.internal.server.space.SpaceEngine.close(SpaceEngine.java:3761)
  at com.gigaspaces.internal.server.space.SpaceImpl.close(SpaceImpl.java:547)
  at com.gigaspaces.internal.server.space.SpaceImpl.cleanOnDestroy(SpaceImpl.java:730)
  - locked <0x00000000f912b830> (a java.lang.Object)
  at com.gigaspaces.internal.server.space.SpaceImpl.destroy(SpaceImpl.java:567)
  at com.gigaspaces.internal.server.space.SpaceImpl.shutdown(SpaceImpl.java:1197)
  at com.j_spaces.core.JSpaceContainerImpl.shutdownInternal(JSpaceContainerImpl.java:1144)
  - locked <0x00000000f912cbb0> (a java.lang.Object)
  at com.j_spaces.core.JSpaceContainerImpl.shutdown(JSpaceContainerImpl.java:1123)
  at com.j_spaces.core.LRMIJSpaceContainer.shutdown(LRMIJSpaceContainer.java:86)
  at com.gigaspaces.internal.client.spaceproxy.SpaceProxyImpl.shutdown(SpaceProxyImpl.java:290)
  at org.openspaces.core.space.AbstractSpaceFactoryBean.close(AbstractSpaceFactoryBean.java:236)
  - locked <0x00000000f91178e8> (a org.openspaces.core.space.EmbeddedSpaceFactoryBean)
  at org.openspaces.core.space.AbstractSpaceFactoryBean.destroy(AbstractSpaceFactoryBean.java:212)
  at org.springframework.beans.factory.support.DisposableBeanAdapter$1.run(DisposableBeanAdapter.java:252)
  at java.security.AccessController.doPrivileged(Native Method)
  at org.springframework.beans.factory.support.DisposableBeanAdapter.destroy(DisposableBeanAdapter.java:249)
  at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.destroyBean(DefaultSingletonBeanRegistry.java:565)
  at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.destroySingleton(DefaultSingletonBeanRegistry.java:541)
  at org.springframework.beans.factory.support.DefaultListableBeanFactory.destroySingleton(DefaultListableBeanFactory.java:870)
  at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.destroySingletons(DefaultSingletonBeanRegistry.java:510)
  at org.springframework.context.support.AbstractApplicationContext.destroyBeans(AbstractApplicationContext.java:908)
  at org.springframework.context.support.AbstractApplicationContext.doClose(AbstractApplicationContext.java:884)
  at org.springframework.context.support.AbstractApplicationContext.close(AbstractApplicationContext.java:836)
  - locked <0x00000000f8f1fb20> (a java.lang.Object)
  at org.springframework.context.support.AbstractApplicationContext.destroy(AbstractApplicationContext.java:823)
  at org.openspaces.pu.container.integrated.IntegratedProcessingUnitContainer.close(IntegratedProcessingUnitContainer.java:81)
  at org.openspaces.pu.container.servicegrid.PUServiceBeanImpl.stopPU(PUServiceBeanImpl.java:1110)
  at org.openspaces.pu.container.servicegrid.PUServiceBeanImpl.stop(PUServiceBeanImpl.java:303)
  at org.jini.rio.jsb.ServiceBeanAdapter.destroy(ServiceBeanAdapter.java:1056)
  at org.jini.rio.jsb.ServiceBeanAdapter.destroy(ServiceBeanAdapter.java:987)
  at org.jini.rio.resources.servicecore.ServiceAdminImpl.destroy(ServiceAdminImpl.java:139)
  at com.sun.jini.admin.DestroyAdminGigaspacesMethodinternalInvoke0.internalInvoke(Unknown Source)
  at com.gigaspaces.internal.reflection.fast.AbstractMethod.invoke(AbstractMethod.java:41)
  at com.gigaspaces.lrmi.LRMIRuntime.invoked(LRMIRuntime.java:479)
  at com.gigaspaces.lrmi.nio.Pivot.consumeAndHandleRequest(Pivot.java:573)
  at com.gigaspaces.lrmi.nio.Pivot.handleRequest(Pivot.java:667)
  at com.gigaspaces.lrmi.nio.Pivot$ChannelEntryTask.run(Pivot.java:196)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  at java.lang.Thread.run(Thread.java:748)