AMBARI-9368 - Deadlock Between Dependent Cluster/Service/Component/Host Implementations (jonathanhurley)

ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java

@@ -18,12 +18,12 @@
 
 package org.apache.ambari.server.state;
 
-import com.google.gson.Gson;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.assistedinject.Assisted;
-import com.google.inject.assistedinject.AssistedInject;
-import com.google.inject.persist.Transactional;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.ServiceComponentHostNotFoundException;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
@@ -42,11 +42,12 @@ import org.apache.ambari.server.state.cluster.ClusterImpl;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
+import com.google.gson.Gson;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.assistedinject.Assisted;
+import com.google.inject.assistedinject.AssistedInject;
+import com.google.inject.persist.Transactional;
 
 public class ServiceComponentImpl implements ServiceComponent {
 
@@ -77,15 +78,15 @@ public class ServiceComponentImpl implements ServiceComponent {
   public ServiceComponentImpl(@Assisted Service service,
                               @Assisted String componentName, Injector injector) throws AmbariException {
     injector.injectMembers(this);
-    this.clusterGlobalLock = service.getClusterGlobalLock();
+    clusterGlobalLock = service.getClusterGlobalLock();
     this.service = service;
-    this.desiredStateEntity = new ServiceComponentDesiredStateEntity();
+    desiredStateEntity = new ServiceComponentDesiredStateEntity();
     desiredStateEntity.setComponentName(componentName);
     desiredStateEntity.setDesiredState(State.INIT);
 
     setDesiredStackVersion(service.getDesiredStackVersion());
 
-    this.hostComponents = new HashMap<String, ServiceComponentHost>();
+    hostComponents = new HashMap<String, ServiceComponentHost>();
 
     StackId stackId = service.getDesiredStackVersion();
     ComponentInfo compInfo = ambariMetaInfo.getComponentCategory(
@@ -99,10 +100,8 @@ public class ServiceComponentImpl implements ServiceComponent {
           + ", componentName=" + componentName
           + ", stackInfo=" + stackId.getStackId());
     }
-    this.isClientComponent = compInfo.isClient();
-    this.isMasterComponent = compInfo.isMaster();
-
-    init();
+    isClientComponent = compInfo.isClient();
+    isMasterComponent = compInfo.isMaster();
   }
 
   @AssistedInject
@@ -110,11 +109,11 @@ public class ServiceComponentImpl implements ServiceComponent {
                               @Assisted ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntity,
                               Injector injector) throws AmbariException {
     injector.injectMembers(this);
-    this.clusterGlobalLock = service.getClusterGlobalLock();
+    clusterGlobalLock = service.getClusterGlobalLock();
     this.service = service;
-    this.desiredStateEntity = serviceComponentDesiredStateEntity;
+    desiredStateEntity = serviceComponentDesiredStateEntity;
 
-    this.hostComponents = new HashMap<String, ServiceComponentHost>();
+    hostComponents = new HashMap<String, ServiceComponentHost>();
     for (HostComponentStateEntity hostComponentStateEntity : desiredStateEntity.getHostComponentStateEntities()) {
       HostComponentDesiredStateEntityPK pk = new HostComponentDesiredStateEntityPK();
       pk.setClusterId(hostComponentStateEntity.getClusterId());
@@ -141,17 +140,12 @@ public class ServiceComponentImpl implements ServiceComponent {
           + ", componentName=" + getName()
           + ", stackInfo=" + stackId.getStackId());
     }
-    this.isClientComponent = compInfo.isClient();
-    this.isMasterComponent = compInfo.isMaster();
+    isClientComponent = compInfo.isClient();
+    isMasterComponent = compInfo.isMaster();
 
     persisted = true;
   }
 
-  private void init() {
-    // TODO load during restart
-    // initialize from DB
-  }
-
   @Override
   public ReadWriteLock getClusterGlobalLock() {
     return clusterGlobalLock;
@@ -159,47 +153,17 @@ public class ServiceComponentImpl implements ServiceComponent {
 
   @Override
   public String getName() {
-    clusterGlobalLock.readLock().lock();
-    try {
-      readWriteLock.readLock().lock();
-      try {
-        return desiredStateEntity.getComponentName();
-      } finally {
-        readWriteLock.readLock().unlock();
-      }
-    } finally {
-      clusterGlobalLock.readLock().unlock();
-    }
+    return desiredStateEntity.getComponentName();
   }
 
   @Override
   public String getServiceName() {
-    clusterGlobalLock.readLock().lock();
-    try {
-      readWriteLock.readLock().lock();
-      try {
-        return service.getName();
-      } finally {
-        readWriteLock.readLock().unlock();
-      }
-    } finally {
-      clusterGlobalLock.readLock().unlock();
-    }
+    return service.getName();
   }
 
   @Override
   public long getClusterId() {
-    clusterGlobalLock.readLock().lock();
-    try {
-      readWriteLock.readLock().lock();
-      try {
-        return this.service.getClusterId();
-      } finally {
-        readWriteLock.readLock().unlock();
-      }
-    } finally {
-      clusterGlobalLock.readLock().unlock();
-    }
+    return service.getClusterId();
   }
 
   @Override
@@ -271,7 +235,7 @@ public class ServiceComponentImpl implements ServiceComponent {
         // FIXME need a better approach of caching components by host
         ClusterImpl clusterImpl = (ClusterImpl) service.getCluster();
         clusterImpl.addServiceComponentHost(hostComponent);
-        this.hostComponents.put(hostComponent.getHostName(), hostComponent);
+        hostComponents.put(hostComponent.getHostName(), hostComponent);
       } finally {
         readWriteLock.writeLock().unlock();
       }
@@ -310,7 +274,7 @@ public class ServiceComponentImpl implements ServiceComponent {
         ClusterImpl clusterImpl = (ClusterImpl) service.getCluster();
         clusterImpl.addServiceComponentHost(hostComponent);
 
-        this.hostComponents.put(hostComponent.getHostName(), hostComponent);
+        hostComponents.put(hostComponent.getHostName(), hostComponent);
 
         return hostComponent;
       } finally {
@@ -332,7 +296,7 @@ public class ServiceComponentImpl implements ServiceComponent {
           throw new ServiceComponentHostNotFoundException(getClusterName(),
               getServiceName(), getName(), hostname);
         }
-        return this.hostComponents.get(hostname);
+        return hostComponents.get(hostname);
       } finally {
         readWriteLock.readLock().unlock();
       }
@@ -343,149 +307,105 @@ public class ServiceComponentImpl implements ServiceComponent {
 
   @Override
   public State getDesiredState() {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.readLock().lock();
     try {
-      readWriteLock.readLock().lock();
-      try {
-        return desiredStateEntity.getDesiredState();
-      } finally {
-        readWriteLock.readLock().unlock();
-      }
+      return desiredStateEntity.getDesiredState();
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.readLock().unlock();
     }
   }
 
   @Override
   public void setDesiredState(State state) {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.writeLock().lock();
     try {
-      readWriteLock.writeLock().lock();
-      try {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Setting DesiredState of Service"
-              + ", clusterName=" + service.getCluster().getClusterName()
-              + ", clusterId=" + service.getCluster().getClusterId()
-              + ", serviceName=" + service.getName()
-              + ", serviceComponentName=" + getName()
-              + ", oldDesiredState=" + getDesiredState()
-              + ", newDesiredState=" + state);
-        }
-        desiredStateEntity.setDesiredState(state);
-        saveIfPersisted();
-      } finally {
-        readWriteLock.writeLock().unlock();
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Setting DesiredState of Service" + ", clusterName="
+            + service.getCluster().getClusterName() + ", clusterId="
+            + service.getCluster().getClusterId() + ", serviceName="
+            + service.getName() + ", serviceComponentName=" + getName()
+            + ", oldDesiredState=" + getDesiredState() + ", newDesiredState="
+            + state);
       }
+      desiredStateEntity.setDesiredState(state);
+      saveIfPersisted();
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.writeLock().unlock();
     }
   }
 
   @Override
   public StackId getDesiredStackVersion() {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.readLock().lock();
     try {
-      readWriteLock.readLock().lock();
-      try {
-        return gson.fromJson(desiredStateEntity.getDesiredStackVersion(), StackId.class);
-      } finally {
-        readWriteLock.readLock().unlock();
-      }
+      return gson.fromJson(desiredStateEntity.getDesiredStackVersion(),
+          StackId.class);
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.readLock().unlock();
     }
   }
 
   @Override
   public void setDesiredStackVersion(StackId stackVersion) {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.writeLock().lock();
     try {
-      readWriteLock.writeLock().lock();
-      try {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Setting DesiredStackVersion of Service"
-              + ", clusterName=" + service.getCluster().getClusterName()
-              + ", clusterId=" + service.getCluster().getClusterId()
-              + ", serviceName=" + service.getName()
-              + ", serviceComponentName=" + getName()
-              + ", oldDesiredStackVersion=" + getDesiredStackVersion()
-              + ", newDesiredStackVersion=" + stackVersion);
-        }
-        desiredStateEntity.setDesiredStackVersion(gson.toJson(stackVersion));
-        saveIfPersisted();
-      } finally {
-        readWriteLock.writeLock().unlock();
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Setting DesiredStackVersion of Service" + ", clusterName="
+            + service.getCluster().getClusterName() + ", clusterId="
+            + service.getCluster().getClusterId() + ", serviceName="
+            + service.getName() + ", serviceComponentName=" + getName()
+            + ", oldDesiredStackVersion=" + getDesiredStackVersion()
+            + ", newDesiredStackVersion=" + stackVersion);
       }
+      desiredStateEntity.setDesiredStackVersion(gson.toJson(stackVersion));
+      saveIfPersisted();
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.writeLock().unlock();
     }
   }
 
   @Override
   public ServiceComponentResponse convertToResponse() {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.readLock().lock();
     try {
-      readWriteLock.readLock().lock();
-      try {
-        ServiceComponentResponse r = new ServiceComponentResponse(
-            getClusterId(), service.getCluster().getClusterName(),
-            service.getName(), getName(),
-            getDesiredStackVersion().getStackId(),
-            getDesiredState().toString(), getTotalCount(), getStartedCount(),
-            getInstalledCount());
-        return r;
-      } finally {
-        readWriteLock.readLock().unlock();
-      }
+      ServiceComponentResponse r = new ServiceComponentResponse(getClusterId(),
+          service.getCluster().getClusterName(), service.getName(), getName(),
+          getDesiredStackVersion().getStackId(), getDesiredState().toString(),
+          getTotalCount(), getStartedCount(), getInstalledCount());
+      return r;
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.readLock().unlock();
     }
   }
 
   @Override
   public String getClusterName() {
-    clusterGlobalLock.readLock().lock();
-    try {
-      readWriteLock.readLock().lock();
-      try {
-        return service.getCluster().getClusterName();
-      } finally {
-        readWriteLock.readLock().unlock();
-      }
-    } finally {
-      clusterGlobalLock.readLock().unlock();
-    }
+    return service.getCluster().getClusterName();
   }
 
   @Override
   public void debugDump(StringBuilder sb) {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.readLock().lock();
     try {
-      readWriteLock.readLock().lock();
-      try {
-        sb.append("ServiceComponent={ serviceComponentName=" + getName()
-            + ", clusterName=" + service.getCluster().getClusterName()
-            + ", clusterId=" + service.getCluster().getClusterId()
-            + ", serviceName=" + service.getName()
-            + ", desiredStackVersion=" + getDesiredStackVersion()
-            + ", desiredState=" + getDesiredState().toString()
-            + ", hostcomponents=[ ");
-        boolean first = true;
-        for (ServiceComponentHost sch : hostComponents.values()) {
-          if (!first) {
-            sb.append(" , ");
-            first = false;
-          }
-          sb.append("\n        ");
-          sch.debugDump(sb);
-          sb.append(" ");
+      sb.append("ServiceComponent={ serviceComponentName=" + getName()
+          + ", clusterName=" + service.getCluster().getClusterName()
+          + ", clusterId=" + service.getCluster().getClusterId()
+          + ", serviceName=" + service.getName() + ", desiredStackVersion="
+          + getDesiredStackVersion() + ", desiredState="
+          + getDesiredState().toString() + ", hostcomponents=[ ");
+      boolean first = true;
+      for (ServiceComponentHost sch : hostComponents.values()) {
+        if (!first) {
+          sb.append(" , ");
+          first = false;
         }
-        sb.append(" ] }");
-      } finally {
-        readWriteLock.readLock().unlock();
+        sb.append("\n        ");
+        sch.debugDump(sb);
+        sb.append(" ");
       }
+      sb.append(" ] }");
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.readLock().unlock();
     }
   }
 
@@ -506,12 +426,22 @@ public class ServiceComponentImpl implements ServiceComponent {
 
   @Override
   public void persist() {
-    clusterGlobalLock.readLock().lock();
+    boolean clusterWriteLockAcquired = false;
+    if (!persisted) {
+      clusterGlobalLock.writeLock().lock();
+      clusterWriteLockAcquired = true;
+    }
+
     try {
       readWriteLock.writeLock().lock();
       try {
         if (!persisted) {
+          // persist the new cluster topology and then release the cluster lock
+          // as it has no more bearing on the rest of this persist() method
           persistEntities();
+          clusterGlobalLock.writeLock().unlock();
+          clusterWriteLockAcquired = false;
+
           refresh();
           service.refresh();
           persisted = true;
@@ -522,7 +452,9 @@ public class ServiceComponentImpl implements ServiceComponent {
         readWriteLock.writeLock().unlock();
       }
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      if (clusterWriteLockAcquired) {
+        clusterGlobalLock.writeLock().unlock();
+      }
     }
   }
 
@@ -541,52 +473,42 @@ public class ServiceComponentImpl implements ServiceComponent {
   @Override
   @Transactional
   public void refresh() {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.writeLock().lock();
     try {
-      readWriteLock.writeLock().lock();
-      try {
-        if (isPersisted()) {
-          ServiceComponentDesiredStateEntityPK pk = new ServiceComponentDesiredStateEntityPK();
-          pk.setComponentName(getName());
-          pk.setClusterId(getClusterId());
-          pk.setServiceName(getServiceName());
-          // TODO: desiredStateEntity is assigned in unway, may be a bug
-          desiredStateEntity = serviceComponentDesiredStateDAO.findByPK(pk);
-          serviceComponentDesiredStateDAO.refresh(desiredStateEntity);
-        }
-      } finally {
-        readWriteLock.writeLock().unlock();
+      if (isPersisted()) {
+        ServiceComponentDesiredStateEntityPK pk = new ServiceComponentDesiredStateEntityPK();
+        pk.setComponentName(getName());
+        pk.setClusterId(getClusterId());
+        pk.setServiceName(getServiceName());
+        // TODO: desiredStateEntity is assigned in unway, may be a bug
+        desiredStateEntity = serviceComponentDesiredStateDAO.findByPK(pk);
+        serviceComponentDesiredStateDAO.refresh(desiredStateEntity);
       }
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.writeLock().unlock();
     }
   }
 
   @Transactional
   private void saveIfPersisted() {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.writeLock().lock();
     try {
-      readWriteLock.writeLock().lock();
-      try {
-        if (isPersisted()) {
-          serviceComponentDesiredStateDAO.merge(desiredStateEntity);
-        }
-      } finally {
-        readWriteLock.writeLock().unlock();
+      if (isPersisted()) {
+        serviceComponentDesiredStateDAO.merge(desiredStateEntity);
       }
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.writeLock().unlock();
     }
   }
 
   @Override
   public boolean isClientComponent() {
-    return this.isClientComponent;
+    return isClientComponent;
   }
 
   @Override
   public boolean isMasterComponent() {
-    return this.isMasterComponent;
+    return isMasterComponent;
   }
 
   @Override

ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java

@@ -18,7 +18,6 @@
 
 package org.apache.ambari.server.state;
 
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.locks.ReadWriteLock;
@@ -94,14 +93,14 @@ public class ServiceImpl implements Service {
 
     this.cluster = cluster;
 
-    this.components = new HashMap<String, ServiceComponent>();
+    components = new HashMap<String, ServiceComponent>();
 
     StackId stackId = cluster.getDesiredStackVersion();
     setDesiredStackVersion(stackId);
 
     ServiceInfo sInfo = ambariMetaInfo.getServiceInfo(stackId.getStackName(),
         stackId.getStackVersion(), serviceName);
-    this.isClientOnlyService = sInfo.isClientOnlyService();
+    isClientOnlyService = sInfo.isClientOnlyService();
 
     init();
   }
@@ -115,9 +114,9 @@ public class ServiceImpl implements Service {
     this.cluster = cluster;
 
     //TODO check for null states?
-    this.serviceDesiredStateEntity = serviceEntity.getServiceDesiredStateEntity();
+    serviceDesiredStateEntity = serviceEntity.getServiceDesiredStateEntity();
 
-    this.components = new HashMap<String, ServiceComponent>();
+    components = new HashMap<String, ServiceComponent>();
 
     if (!serviceEntity.getServiceComponentDesiredStateEntities().isEmpty()) {
       for (ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntity
@@ -139,7 +138,7 @@ public class ServiceImpl implements Service {
     StackId stackId = getDesiredStackVersion();
     ServiceInfo sInfo = ambariMetaInfo.getServiceInfo(stackId.getStackName(),
         stackId.getStackVersion(), getName());
-    this.isClientOnlyService = sInfo.isClientOnlyService();
+    isClientOnlyService = sInfo.isClientOnlyService();
 
     persisted = true;
   }
@@ -151,46 +150,21 @@ public class ServiceImpl implements Service {
 
   @Override
   public String getName() {
-    clusterGlobalLock.readLock().lock();
-    try {
-      readWriteLock.readLock().lock();
-      try {
-        return serviceEntity.getServiceName();
-      } finally {
-        readWriteLock.readLock().unlock();
-      }
-    } finally {
-      clusterGlobalLock.readLock().unlock();
-    }
+    return serviceEntity.getServiceName();
   }
 
   @Override
   public long getClusterId() {
-    clusterGlobalLock.readLock().lock();
-    try {
-      readWriteLock.readLock().lock();
-      try {
-        return cluster.getClusterId();
-      } finally {
-        readWriteLock.readLock().unlock();
-      }
-    } finally {
-      clusterGlobalLock.readLock().unlock();
-    }
+    return cluster.getClusterId();
   }
 
   @Override
   public Map<String, ServiceComponent> getServiceComponents() {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.readLock().lock();
     try {
-      readWriteLock.readLock().lock();
-      try {
-        return new HashMap<String, ServiceComponent>(components);
-      } finally {
-        readWriteLock.readLock().unlock();
-      }
+      return new HashMap<String, ServiceComponent>(components);
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.readLock().unlock();
     }
   }
 
@@ -234,7 +208,7 @@ public class ServiceImpl implements Service {
               + ", serviceName=" + getName()
               + ", serviceComponentName=" + component.getName());
         }
-        this.components.put(component.getName(), component);
+        components.put(component.getName(), component);
       } finally {
         readWriteLock.writeLock().unlock();
       }
@@ -265,7 +239,7 @@ public class ServiceImpl implements Service {
               + ", serviceComponentName=" + serviceComponentName);
         }
         ServiceComponent component = serviceComponentFactory.createNew(this, serviceComponentName);
-        this.components.put(component.getName(), component);
+        components.put(component.getName(), component);
         return component;
       } finally {
         readWriteLock.writeLock().unlock();
@@ -273,134 +247,92 @@ public class ServiceImpl implements Service {
     } finally {
       clusterGlobalLock.writeLock().unlock();
     }
-
-
   }
 
   @Override
   public ServiceComponent getServiceComponent(String componentName)
       throws AmbariException {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.readLock().lock();
     try {
-      readWriteLock.readLock().lock();
-      try {
-        if (!components.containsKey(componentName)) {
-          throw new ServiceComponentNotFoundException(cluster.getClusterName(),
-              getName(),
-              componentName);
-        }
-        return this.components.get(componentName);
-      } finally {
-        readWriteLock.readLock().unlock();
+      if (!components.containsKey(componentName)) {
+        throw new ServiceComponentNotFoundException(cluster.getClusterName(),
+            getName(), componentName);
       }
+      return components.get(componentName);
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.readLock().unlock();
     }
-
-
   }
 
   @Override
   public State getDesiredState() {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.readLock().lock();
     try {
-      readWriteLock.readLock().lock();
-      try {
-        return this.serviceDesiredStateEntity.getDesiredState();
-      } finally {
-        readWriteLock.readLock().unlock();
-      }
+      return serviceDesiredStateEntity.getDesiredState();
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.readLock().unlock();
     }
-
-
   }
 
   @Override
   public void setDesiredState(State state) {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.writeLock().lock();
     try {
-      readWriteLock.writeLock().lock();
-      try {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Setting DesiredState of Service"
-              + ", clusterName=" + cluster.getClusterName()
-              + ", clusterId=" + cluster.getClusterId()
-              + ", serviceName=" + getName()
-              + ", oldDesiredState=" + this.getDesiredState()
-              + ", newDesiredState=" + state);
-        }
-        this.serviceDesiredStateEntity.setDesiredState(state);
-        saveIfPersisted();
-      } finally {
-        readWriteLock.writeLock().unlock();
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Setting DesiredState of Service" + ", clusterName="
+            + cluster.getClusterName() + ", clusterId="
+            + cluster.getClusterId() + ", serviceName=" + getName()
+            + ", oldDesiredState=" + getDesiredState() + ", newDesiredState="
+            + state);
       }
+      serviceDesiredStateEntity.setDesiredState(state);
+      saveIfPersisted();
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.writeLock().unlock();
     }
   }
 
   @Override
   public StackId getDesiredStackVersion() {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.readLock().lock();
     try {
-      readWriteLock.readLock().lock();
-      try {
-        return gson.fromJson(serviceDesiredStateEntity.getDesiredStackVersion(), StackId.class);
-      } finally {
-        readWriteLock.readLock().unlock();
-      }
+      return gson.fromJson(serviceDesiredStateEntity.getDesiredStackVersion(),
+          StackId.class);
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.readLock().unlock();
     }
   }
 
   @Override
   public void setDesiredStackVersion(StackId stackVersion) {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.writeLock().lock();
     try {
-      readWriteLock.writeLock().lock();
-      try {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Setting DesiredStackVersion of Service"
-              + ", clusterName=" + cluster.getClusterName()
-              + ", clusterId=" + cluster.getClusterId()
-              + ", serviceName=" + getName()
-              + ", oldDesiredStackVersion=" + getDesiredStackVersion()
-              + ", newDesiredStackVersion=" + stackVersion);
-        }
-        serviceDesiredStateEntity.setDesiredStackVersion(gson.toJson(stackVersion));
-        saveIfPersisted();
-      } finally {
-        readWriteLock.writeLock().unlock();
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Setting DesiredStackVersion of Service" + ", clusterName="
+            + cluster.getClusterName() + ", clusterId="
+            + cluster.getClusterId() + ", serviceName=" + getName()
+            + ", oldDesiredStackVersion=" + getDesiredStackVersion()
+            + ", newDesiredStackVersion=" + stackVersion);
       }
+      serviceDesiredStateEntity.setDesiredStackVersion(gson.toJson(stackVersion));
+      saveIfPersisted();
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.writeLock().unlock();
     }
-
-
   }
 
   @Override
   public ServiceResponse convertToResponse() {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.readLock().lock();
     try {
-      readWriteLock.readLock().lock();
-      try {
-        ServiceResponse r = new ServiceResponse(cluster.getClusterId(),
-            cluster.getClusterName(),
-            getName(),
-            getDesiredStackVersion().getStackId(),
-            getDesiredState().toString());
-        
-        r.setMaintenanceState(getMaintenanceState().name());
-        return r;
-      } finally {
-        readWriteLock.readLock().unlock();
-      }
+      ServiceResponse r = new ServiceResponse(cluster.getClusterId(),
+          cluster.getClusterName(), getName(),
+          getDesiredStackVersion().getStackId(), getDesiredState().toString());
+
+      r.setMaintenanceState(getMaintenanceState().name());
+      return r;
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.readLock().unlock();
     }
   }
 
@@ -411,32 +343,26 @@ public class ServiceImpl implements Service {
 
   @Override
   public void debugDump(StringBuilder sb) {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.readLock().lock();
     try {
-      readWriteLock.readLock().lock();
-      try {
-        sb.append("Service={ serviceName=" + getName()
-            + ", clusterName=" + cluster.getClusterName()
-            + ", clusterId=" + cluster.getClusterId()
-            + ", desiredStackVersion=" + getDesiredStackVersion()
-            + ", desiredState=" + getDesiredState().toString()
-            + ", components=[ ");
-        boolean first = true;
-        for (ServiceComponent sc : components.values()) {
-          if (!first) {
-            sb.append(" , ");
-          }
-          first = false;
-          sb.append("\n      ");
-          sc.debugDump(sb);
-          sb.append(" ");
+      sb.append("Service={ serviceName=" + getName() + ", clusterName="
+          + cluster.getClusterName() + ", clusterId=" + cluster.getClusterId()
+          + ", desiredStackVersion=" + getDesiredStackVersion()
+          + ", desiredState=" + getDesiredState().toString()
+          + ", components=[ ");
+      boolean first = true;
+      for (ServiceComponent sc : components.values()) {
+        if (!first) {
+          sb.append(" , ");
         }
-        sb.append(" ] }");
-      } finally {
-        readWriteLock.readLock().unlock();
+        first = false;
+        sb.append("\n      ");
+        sc.debugDump(sb);
+        sb.append(" ");
       }
+      sb.append(" ] }");
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.readLock().unlock();
     }
   }
 
@@ -453,12 +379,11 @@ public class ServiceImpl implements Service {
     } finally {
       clusterGlobalLock.readLock().unlock();
     }
-
   }
 
   @Override
   public void persist() {
-    clusterGlobalLock.readLock().lock();
+    clusterGlobalLock.writeLock().lock();
     try {
       readWriteLock.writeLock().lock();
       try {
@@ -474,9 +399,8 @@ public class ServiceImpl implements Service {
         readWriteLock.writeLock().unlock();
       }
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      clusterGlobalLock.writeLock().unlock();
     }
-
   }
 
   @Transactional
@@ -504,27 +428,20 @@ public class ServiceImpl implements Service {
   @Override
   @Transactional
   public void refresh() {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.writeLock().lock();
     try {
-      readWriteLock.writeLock().lock();
-      try {
-        if (isPersisted()) {
-          ClusterServiceEntityPK pk = new ClusterServiceEntityPK();
-          pk.setClusterId(getClusterId());
-          pk.setServiceName(getName());
-          serviceEntity = clusterServiceDAO.findByPK(pk);
-          serviceDesiredStateEntity = serviceEntity.getServiceDesiredStateEntity();
-          clusterServiceDAO.refresh(serviceEntity);
-          serviceDesiredStateDAO.refresh(serviceDesiredStateEntity);
-        }
-      } finally {
-        readWriteLock.writeLock().unlock();
+      if (isPersisted()) {
+        ClusterServiceEntityPK pk = new ClusterServiceEntityPK();
+        pk.setClusterId(getClusterId());
+        pk.setServiceName(getName());
+        serviceEntity = clusterServiceDAO.findByPK(pk);
+        serviceDesiredStateEntity = serviceEntity.getServiceDesiredStateEntity();
+        clusterServiceDAO.refresh(serviceEntity);
+        serviceDesiredStateDAO.refresh(serviceDesiredStateEntity);
       }
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.writeLock().unlock();
     }
-
-
   }
 
   @Override
@@ -659,26 +576,20 @@ public class ServiceImpl implements Service {
 
     clusterServiceDAO.removeByPK(pk);
   }
-  
+
   @Override
   public void setMaintenanceState(MaintenanceState state) {
-    clusterGlobalLock.readLock().lock();
+    readWriteLock.writeLock().lock();
     try {
-      try {
-        readWriteLock.writeLock().lock();
-        serviceDesiredStateEntity.setMaintenanceState(state);
-        saveIfPersisted();
-      } finally {
-        readWriteLock.writeLock().unlock();
-      }
+      serviceDesiredStateEntity.setMaintenanceState(state);
+      saveIfPersisted();
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readWriteLock.writeLock().unlock();
     }
   }
-  
+
   @Override
   public MaintenanceState getMaintenanceState() {
     return serviceDesiredStateEntity.getMaintenanceState();
   }
-
 }

ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java

@@ -131,12 +131,12 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
          State.INSTALLED,
          ServiceComponentHostEventType.HOST_SVCCOMP_OP_SUCCEEDED,
          new ServiceComponentHostOpCompletedTransition())
-         
+
      .addTransition(State.INSTALLED,
          State.INSTALLED,
          ServiceComponentHostEventType.HOST_SVCCOMP_OP_SUCCEEDED,
          new ServiceComponentHostOpCompletedTransition())
-         
+
      .addTransition(State.INSTALLING,
          State.INSTALLING,
          ServiceComponentHostEventType.HOST_SVCCOMP_OP_IN_PROGRESS,
@@ -202,7 +202,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
          State.STARTING,
          ServiceComponentHostEventType.HOST_SVCCOMP_OP_IN_PROGRESS,
          new ServiceComponentHostOpInProgressTransition())
-         
+
      .addTransition(State.STARTING,
          State.STARTING,
          ServiceComponentHostEventType.HOST_SVCCOMP_START,
@@ -211,7 +211,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
          State.STARTED,
          ServiceComponentHostEventType.HOST_SVCCOMP_STARTED,
          new ServiceComponentHostOpCompletedTransition())
-         
+
      .addTransition(State.STARTING,
          State.INSTALLED,
          ServiceComponentHostEventType.HOST_SVCCOMP_OP_FAILED,
@@ -383,7 +383,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
          State.INSTALLING,
          ServiceComponentHostEventType.HOST_SVCCOMP_INSTALL,
          new ServiceComponentHostOpStartedTransition())
-     
+
      .addTransition(State.INSTALLING,
          State.INSTALLING,
          ServiceComponentHostEventType.HOST_SVCCOMP_OP_IN_PROGRESS,
@@ -557,56 +557,44 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
 
 
   private void resetLastOpInfo() {
-    clusterGlobalLock.readLock().lock();
+    writeLock.lock();
     try {
-      try {
-        writeLock.lock();
-        setLastOpStartTime(-1);
-        setLastOpLastUpdateTime(-1);
-        setLastOpEndTime(-1);
-      } finally {
-        writeLock.unlock();
-      }
+      setLastOpStartTime(-1);
+      setLastOpLastUpdateTime(-1);
+      setLastOpEndTime(-1);
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      writeLock.unlock();
     }
-
   }
 
   private void updateLastOpInfo(ServiceComponentHostEventType eventType,
       long time) {
-    clusterGlobalLock.readLock().lock();
+    writeLock.lock();
     try {
-      try {
-        writeLock.lock();
-        switch (eventType) {
-          case HOST_SVCCOMP_INSTALL:
-          case HOST_SVCCOMP_START:
-          case HOST_SVCCOMP_STOP:
-          case HOST_SVCCOMP_UNINSTALL:
-          case HOST_SVCCOMP_WIPEOUT:
-          case HOST_SVCCOMP_OP_RESTART:
-            resetLastOpInfo();
-            setLastOpStartTime(time);
-            break;
-          case HOST_SVCCOMP_OP_FAILED:
-          case HOST_SVCCOMP_OP_SUCCEEDED:
-          case HOST_SVCCOMP_STOPPED:
-          case HOST_SVCCOMP_STARTED:
-            setLastOpLastUpdateTime(time);
-            setLastOpEndTime(time);
-            break;
-          case HOST_SVCCOMP_OP_IN_PROGRESS:
-            setLastOpLastUpdateTime(time);
-            break;
-        }
-      } finally {
-        writeLock.unlock();
+      switch (eventType) {
+        case HOST_SVCCOMP_INSTALL:
+        case HOST_SVCCOMP_START:
+        case HOST_SVCCOMP_STOP:
+        case HOST_SVCCOMP_UNINSTALL:
+        case HOST_SVCCOMP_WIPEOUT:
+        case HOST_SVCCOMP_OP_RESTART:
+          resetLastOpInfo();
+          setLastOpStartTime(time);
+          break;
+        case HOST_SVCCOMP_OP_FAILED:
+        case HOST_SVCCOMP_OP_SUCCEEDED:
+        case HOST_SVCCOMP_STOPPED:
+        case HOST_SVCCOMP_STARTED:
+          setLastOpLastUpdateTime(time);
+          setLastOpEndTime(time);
+          break;
+        case HOST_SVCCOMP_OP_IN_PROGRESS:
+          setLastOpLastUpdateTime(time);
+          break;
       }
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      writeLock.unlock();
     }
-
   }
 
   @AssistedInject
@@ -615,13 +603,13 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
     injector.injectMembers(this);
 
     if (serviceComponent.isClientComponent()) {
-      this.stateMachine = clientStateMachineFactory.make(this);
+      stateMachine = clientStateMachineFactory.make(this);
     } else {
-      this.stateMachine = daemonStateMachineFactory.make(this);
+      stateMachine = daemonStateMachineFactory.make(this);
     }
 
     this.serviceComponent = serviceComponent;
-    this.clusterGlobalLock = serviceComponent.getClusterGlobalLock();
+    clusterGlobalLock = serviceComponent.getClusterGlobalLock();
 
     stateEntity = new HostComponentStateEntity();
     stateEntity.setClusterId(serviceComponent.getClusterId());
@@ -646,14 +634,14 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
     }
 
     try {
-      this.host = clusters.getHost(hostName);
+      host = clusters.getHost(hostName);
     } catch (AmbariException e) {
       //TODO exception?
       LOG.error("Host '{}' was not found" + hostName);
       throw new RuntimeException(e);
     }
 
-    this.resetLastOpInfo();
+    resetLastOpInfo();
   }
 
   @AssistedInject
@@ -663,21 +651,21 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
                                   Injector injector) {
     injector.injectMembers(this);
     this.serviceComponent = serviceComponent;
-    this.clusterGlobalLock = serviceComponent.getClusterGlobalLock();
+    clusterGlobalLock = serviceComponent.getClusterGlobalLock();
 
     this.desiredStateEntity = desiredStateEntity;
     this.stateEntity = stateEntity;
 
     //TODO implement State Machine init as now type choosing is hardcoded in above code
     if (serviceComponent.isClientComponent()) {
-      this.stateMachine = clientStateMachineFactory.make(this);
+      stateMachine = clientStateMachineFactory.make(this);
     } else {
-      this.stateMachine = daemonStateMachineFactory.make(this);
+      stateMachine = daemonStateMachineFactory.make(this);
     }
-    this.stateMachine.setCurrentState(stateEntity.getCurrentState());
+    stateMachine.setCurrentState(stateEntity.getCurrentState());
 
     try {
-      this.host = clusters.getHost(stateEntity.getHostName());
+      host = clusters.getHost(stateEntity.getHostName());
     } catch (AmbariException e) {
       //TODO exception? impossible due to database restrictions
       LOG.error("Host '{}' was not found " + stateEntity.getHostName());
@@ -689,18 +677,9 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
 
   @Override
   public State getState() {
-    clusterGlobalLock.readLock().lock();
-    try {
-      readLock.lock();
-      try {
-        return stateMachine.getCurrentState();
-      } finally {
-        readLock.unlock();
-      }
-    } finally {
-      clusterGlobalLock.readLock().unlock();
-    }
-
+    // there's no reason to lock around the state machine for this SCH since
+    // the state machine is synchronized
+    return stateMachine.getCurrentState();
   }
 
   @Override
@@ -743,8 +722,8 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
         } catch (InvalidStateTransitionException e) {
           LOG.debug("Can't handle ServiceComponentHostEvent event at"
               + " current state"
-              + ", serviceComponentName=" + this.getServiceComponentName()
-              + ", hostName=" + this.getHostName()
+              + ", serviceComponentName=" + getServiceComponentName()
+              + ", hostName=" + getHostName()
               + ", currentState=" + oldState
               + ", eventType=" + event.getType()
               + ", event=" + event);
@@ -760,8 +739,8 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
     if (!oldState.equals(getState())) {
       if (LOG.isDebugEnabled()) {
         LOG.debug("ServiceComponentHost transitioned to a new state"
-            + ", serviceComponentName=" + this.getServiceComponentName()
-            + ", hostName=" + this.getHostName()
+            + ", serviceComponentName=" + getServiceComponentName()
+            + ", hostName=" + getHostName()
             + ", oldState=" + oldState
             + ", currentState=" + getState()
             + ", eventType=" + event.getType().name()
@@ -772,48 +751,23 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
 
   @Override
   public String getServiceComponentName() {
-    clusterGlobalLock.readLock().lock();
-    try {
-      readLock.lock();
-      try {
-        return serviceComponent.getName();
-      } finally {
-        readLock.unlock();
-      }
-    } finally {
-      clusterGlobalLock.readLock().unlock();
-    }
+    return serviceComponent.getName();
   }
 
   @Override
   public String getHostName() {
-    clusterGlobalLock.readLock().lock();
-    try {
-      readLock.lock();
-      try {
-        return host.getHostName();
-      } finally {
-        readLock.unlock();
-      }
-    } finally {
-      clusterGlobalLock.readLock().unlock();
-    }
+    return host.getHostName();
   }
 
   /**
    * @return the lastOpStartTime
    */
   public long getLastOpStartTime() {
-    clusterGlobalLock.readLock().lock();
+    readLock.lock();
     try {
-      readLock.lock();
-      try {
-        return lastOpStartTime;
-      } finally {
-        readLock.unlock();
-      }
+      return lastOpStartTime;
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readLock.unlock();
     }
   }
 
@@ -821,16 +775,11 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
    * @param lastOpStartTime the lastOpStartTime to set
    */
   public void setLastOpStartTime(long lastOpStartTime) {
-    clusterGlobalLock.readLock().lock();
+    writeLock.lock();
     try {
-      writeLock.lock();
-      try {
-        this.lastOpStartTime = lastOpStartTime;
-      } finally {
-        writeLock.unlock();
-      }
+      this.lastOpStartTime = lastOpStartTime;
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      writeLock.unlock();
     }
   }
 
@@ -838,16 +787,11 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
    * @return the lastOpEndTime
    */
   public long getLastOpEndTime() {
-    clusterGlobalLock.readLock().lock();
+    readLock.lock();
     try {
-      readLock.lock();
-      try {
-        return lastOpEndTime;
-      } finally {
-        readLock.unlock();
-      }
+      return lastOpEndTime;
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readLock.unlock();
     }
   }
 
@@ -855,16 +799,11 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
    * @param lastOpEndTime the lastOpEndTime to set
    */
   public void setLastOpEndTime(long lastOpEndTime) {
-    clusterGlobalLock.readLock().lock();
+    writeLock.lock();
     try {
-      writeLock.lock();
-      try {
-        this.lastOpEndTime = lastOpEndTime;
-      } finally {
-        writeLock.unlock();
-      }
+      this.lastOpEndTime = lastOpEndTime;
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      writeLock.unlock();
     }
   }
 
@@ -872,16 +811,11 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
    * @return the lastOpLastUpdateTime
    */
   public long getLastOpLastUpdateTime() {
-    clusterGlobalLock.readLock().lock();
+    readLock.lock();
     try {
-      readLock.lock();
-      try {
-        return lastOpLastUpdateTime;
-      } finally {
-        readLock.unlock();
-      }
+      return lastOpLastUpdateTime;
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readLock.unlock();
     }
   }
 
@@ -889,250 +823,157 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
    * @param lastOpLastUpdateTime the lastOpLastUpdateTime to set
    */
   public void setLastOpLastUpdateTime(long lastOpLastUpdateTime) {
-    clusterGlobalLock.readLock().lock();
+    writeLock.lock();
     try {
-      writeLock.lock();
-      try {
-        this.lastOpLastUpdateTime = lastOpLastUpdateTime;
-      } finally {
-        writeLock.unlock();
-      }
+      this.lastOpLastUpdateTime = lastOpLastUpdateTime;
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      writeLock.unlock();
     }
   }
 
   @Override
   public long getClusterId() {
-    clusterGlobalLock.readLock().lock();
-    try {
-      readLock.lock();
-      try {
-        return serviceComponent.getClusterId();
-      } finally {
-        readLock.unlock();
-      }
-    } finally {
-      clusterGlobalLock.readLock().unlock();
-    }
+    return serviceComponent.getClusterId();
   }
 
   @Override
   public String getServiceName() {
-    clusterGlobalLock.readLock().lock();
-    try {
-      readLock.lock();
-      try {
-        return serviceComponent.getServiceName();
-      } finally {
-        readLock.unlock();
-      }
-    } finally {
-      clusterGlobalLock.readLock().unlock();
-    }
+    return serviceComponent.getServiceName();
   }
 
   @Override
   public StackId getStackVersion() {
-    clusterGlobalLock.readLock().lock();
+    readLock.lock();
     try {
-      readLock.lock();
-      try {
-        return gson.fromJson(stateEntity.getCurrentStackVersion(), StackId.class);
-      } finally {
-        readLock.unlock();
-      }
+      return gson.fromJson(stateEntity.getCurrentStackVersion(), StackId.class);
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readLock.unlock();
     }
   }
 
   @Override
   public void setStackVersion(StackId stackVersion) {
-    clusterGlobalLock.readLock().lock();
+    writeLock.lock();
     try {
-      writeLock.lock();
-      try {
-        stateEntity.setCurrentStackVersion(gson.toJson(stackVersion));
-        saveIfPersisted();
-      } finally {
-        writeLock.unlock();
-      }
+      stateEntity.setCurrentStackVersion(gson.toJson(stackVersion));
+      saveIfPersisted();
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      writeLock.unlock();
     }
   }
 
   @Override
   public State getDesiredState() {
-    clusterGlobalLock.readLock().lock();
+    readLock.lock();
     try {
-      readLock.lock();
-      try {
-        return desiredStateEntity.getDesiredState();
-      } finally {
-        readLock.unlock();
-      }
+      return desiredStateEntity.getDesiredState();
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readLock.unlock();
     }
   }
 
   @Override
   public void setDesiredState(State state) {
-    clusterGlobalLock.readLock().lock();
+    writeLock.lock();
     try {
-      writeLock.lock();
-      try {
-        desiredStateEntity.setDesiredState(state);
-        saveIfPersisted();
-      } finally {
-        writeLock.unlock();
-      }
+      desiredStateEntity.setDesiredState(state);
+      saveIfPersisted();
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      writeLock.unlock();
     }
   }
 
   @Override
   public StackId getDesiredStackVersion() {
-    clusterGlobalLock.readLock().lock();
+    readLock.lock();
     try {
-      readLock.lock();
-      try {
-        return gson.fromJson(desiredStateEntity.getDesiredStackVersion(), StackId.class);
-      } finally {
-        readLock.unlock();
-      }
+      return gson.fromJson(desiredStateEntity.getDesiredStackVersion(),
+          StackId.class);
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readLock.unlock();
     }
   }
 
   @Override
   public void setDesiredStackVersion(StackId stackVersion) {
-    clusterGlobalLock.readLock().lock();
+    writeLock.lock();
     try {
-      writeLock.lock();
-      try {
-        desiredStateEntity.setDesiredStackVersion(gson.toJson(stackVersion));
-        saveIfPersisted();
-      } finally {
-        writeLock.unlock();
-      }
+      desiredStateEntity.setDesiredStackVersion(gson.toJson(stackVersion));
+      saveIfPersisted();
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      writeLock.unlock();
     }
   }
 
   @Override
   public HostComponentAdminState getComponentAdminState() {
-    clusterGlobalLock.readLock().lock();
+    readLock.lock();
     try {
-      readLock.lock();
-      try {
-        HostComponentAdminState adminState = desiredStateEntity.getAdminState();
-        if (adminState == null
-            && !serviceComponent.isClientComponent() && !serviceComponent.isMasterComponent()) {
-          adminState = HostComponentAdminState.INSERVICE;
-        }
-        return adminState;
-      } finally {
-        readLock.unlock();
+      HostComponentAdminState adminState = desiredStateEntity.getAdminState();
+      if (adminState == null && !serviceComponent.isClientComponent()
+          && !serviceComponent.isMasterComponent()) {
+        adminState = HostComponentAdminState.INSERVICE;
       }
+      return adminState;
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readLock.unlock();
     }
   }
 
   @Override
   public void setComponentAdminState(HostComponentAdminState attribute) {
-    clusterGlobalLock.readLock().lock();
+    writeLock.lock();
     try {
-      writeLock.lock();
-      try {
-        desiredStateEntity.setAdminState(attribute);
-        saveIfPersisted();
-      } finally {
-        writeLock.unlock();
-      }
+      desiredStateEntity.setAdminState(attribute);
+      saveIfPersisted();
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      writeLock.unlock();
     }
   }
 
   @Override
   public ServiceComponentHostResponse convertToResponse() {
-    clusterGlobalLock.readLock().lock();
+    readLock.lock();
     try {
-      readLock.lock();
-      try {
-        ServiceComponentHostResponse r = new ServiceComponentHostResponse(
-            serviceComponent.getClusterName(),
-            serviceComponent.getServiceName(),
-            serviceComponent.getName(),
-            getHostName(),
-            getState().toString(),
-            getStackVersion().getStackId(),
-            getDesiredState().toString(),
-            getDesiredStackVersion().getStackId(),
-            getComponentAdminState());
-
-        r.setActualConfigs(actualConfigs);
+      ServiceComponentHostResponse r = new ServiceComponentHostResponse(
+          serviceComponent.getClusterName(), serviceComponent.getServiceName(),
+          serviceComponent.getName(), getHostName(), getState().toString(),
+          getStackVersion().getStackId(), getDesiredState().toString(),
+          getDesiredStackVersion().getStackId(), getComponentAdminState());
 
-        try {
-          r.setStaleConfig(helper.isStaleConfigs(this));
-        } catch (Exception e) {
-          LOG.error("Could not determine stale config", e);
-        }
+      r.setActualConfigs(actualConfigs);
 
-        return r;
-      } finally {
-        readLock.unlock();
+      try {
+        r.setStaleConfig(helper.isStaleConfigs(this));
+      } catch (Exception e) {
+        LOG.error("Could not determine stale config", e);
       }
+
+      return r;
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readLock.unlock();
     }
   }
 
   @Override
   public String getClusterName() {
-    clusterGlobalLock.readLock().lock();
-    try {
-      readLock.lock();
-      try {
-        return serviceComponent.getClusterName();
-      } finally {
-        readLock.unlock();
-      }
-    } finally {
-      clusterGlobalLock.readLock().unlock();
-    }
-
+    return serviceComponent.getClusterName();
   }
 
   @Override
   public void debugDump(StringBuilder sb) {
-    clusterGlobalLock.readLock().lock();
+    readLock.lock();
     try {
-      readLock.lock();
-      try {
-        sb.append("ServiceComponentHost={ hostname=").append(getHostName())
-          .append(", serviceComponentName=").append(serviceComponent.getName())
-          .append(", clusterName=").append(serviceComponent.getClusterName())
-          .append(", serviceName=").append(serviceComponent.getServiceName())
-          .append(", desiredStackVersion=").append(getDesiredStackVersion())
-          .append(", desiredState=").append(getDesiredState())
-          .append(", stackVersion=").append(getStackVersion())
-          .append(", state=").append(getState())
-          .append(" }");
-      } finally {
-        readLock.unlock();
-      }
+      sb.append("ServiceComponentHost={ hostname=").append(getHostName()).append(
+          ", serviceComponentName=").append(serviceComponent.getName()).append(
+          ", clusterName=").append(serviceComponent.getClusterName()).append(
+          ", serviceName=").append(serviceComponent.getServiceName()).append(
+          ", desiredStackVersion=").append(getDesiredStackVersion()).append(
+          ", desiredState=").append(getDesiredState()).append(", stackVersion=").append(
+          getStackVersion()).append(", state=").append(getState()).append(" }");
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readLock.unlock();
     }
-
   }
 
   @Override
@@ -1153,12 +994,23 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
 
   @Override
   public void persist() {
-    clusterGlobalLock.readLock().lock();
+    boolean clusterWriteLockAcquired = false;
+    if (!persisted) {
+      clusterGlobalLock.writeLock().lock();
+      clusterWriteLockAcquired = true;
+    }
+
     try {
       writeLock.lock();
       try {
         if (!persisted) {
+          // persist the new cluster topology and then release the cluster lock
+          // as it has no more bearing on the rest of this persist() method
           persistEntities();
+          clusterGlobalLock.writeLock().unlock();
+          clusterWriteLockAcquired = false;
+
+          // these shoudl still be done with the internal lock
           refresh();
           host.refresh();
           serviceComponent.refresh();
@@ -1170,9 +1022,10 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
         writeLock.unlock();
       }
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      if (clusterWriteLockAcquired) {
+        clusterGlobalLock.writeLock().unlock();
+      }
     }
-
   }
 
   @Transactional
@@ -1204,33 +1057,27 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
   @Override
   @Transactional
   public void refresh() {
-    clusterGlobalLock.readLock().lock();
+    writeLock.lock();
     try {
-      writeLock.lock();
-      try {
-        if (isPersisted()) {
-          HostComponentStateEntityPK pk = new HostComponentStateEntityPK();
-          HostComponentDesiredStateEntityPK dpk = new HostComponentDesiredStateEntityPK();
-          pk.setClusterId(getClusterId());
-          pk.setComponentName(getServiceComponentName());
-          pk.setServiceName(getServiceName());
-          pk.setHostName(getHostName());
-          dpk.setClusterId(getClusterId());
-          dpk.setComponentName(getServiceComponentName());
-          dpk.setServiceName(getServiceName());
-          dpk.setHostName(getHostName());
-          stateEntity = hostComponentStateDAO.findByPK(pk);
-          desiredStateEntity = hostComponentDesiredStateDAO.findByPK(dpk);
-          hostComponentStateDAO.refresh(stateEntity);
-          hostComponentDesiredStateDAO.refresh(desiredStateEntity);
-        }
-      } finally {
-        writeLock.unlock();
+      if (isPersisted()) {
+        HostComponentStateEntityPK pk = new HostComponentStateEntityPK();
+        HostComponentDesiredStateEntityPK dpk = new HostComponentDesiredStateEntityPK();
+        pk.setClusterId(getClusterId());
+        pk.setComponentName(getServiceComponentName());
+        pk.setServiceName(getServiceName());
+        pk.setHostName(getHostName());
+        dpk.setClusterId(getClusterId());
+        dpk.setComponentName(getServiceComponentName());
+        dpk.setServiceName(getServiceName());
+        dpk.setHostName(getHostName());
+        stateEntity = hostComponentStateDAO.findByPK(pk);
+        desiredStateEntity = hostComponentDesiredStateDAO.findByPK(dpk);
+        hostComponentStateDAO.refresh(stateEntity);
+        hostComponentDesiredStateDAO.refresh(desiredStateEntity);
       }
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      writeLock.unlock();
     }
-
   }
 
   @Transactional
@@ -1244,19 +1091,21 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
   @Override
   public boolean canBeRemoved() {
     clusterGlobalLock.readLock().lock();
+    boolean schLockAcquired = false;
     try {
-      readLock.lock();
-      try {
-
-        return (getState().isRemovableState());
+      // if unable to read, then writers are writing; cannot remove SCH
+      schLockAcquired = readLock.tryLock();
+      if (!schLockAcquired) {
+        return false;
+      }
 
-      } finally {
+      return (getState().isRemovableState());
+    } finally {
+      if (schLockAcquired) {
         readLock.unlock();
       }
-    } finally {
       clusterGlobalLock.readLock().unlock();
     }
-
   }
 
   @Override
@@ -1269,7 +1118,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
           removeEntities();
           persisted = false;
         }
-        clusters.getCluster(this.getClusterName()).removeServiceComponentHost(this);
+        clusters.getCluster(getClusterName()).removeServiceComponentHost(this);
       } catch (AmbariException ex) {
         if (LOG.isDebugEnabled()) {
           LOG.error(ex.getMessage());
@@ -1301,7 +1150,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
 
     hostComponentDesiredStateDAO.removeByPK(desiredPK);
   }
-  
+
   @Override
   public void updateActualConfigs(Map<String, Map<String, String>> configTags) {
     Map<Long, ConfigGroup> configGroupMap;
@@ -1314,165 +1163,120 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
       return;
     }
 
-    clusterGlobalLock.readLock().lock();
+    writeLock.lock();
     try {
-      writeLock.lock();
-      try {
-        LOG.debug("Updating actual config tags: " + configTags);
-        actualConfigs = new HashMap<String, HostConfig>();
-
-        for (Entry<String, Map<String, String>> entry : configTags.entrySet()) {
-          String type = entry.getKey();
-          Map<String, String> values = new HashMap<String, String>(entry.getValue());
-
-          String tag = values.get(ConfigHelper.CLUSTER_DEFAULT_TAG);
-          values.remove(ConfigHelper.CLUSTER_DEFAULT_TAG);
-
-          HostConfig hc = new HostConfig();
-          hc.setDefaultVersionTag(tag);
-          actualConfigs.put(type, hc);
-
-          if (!values.isEmpty()) {
-            for (Entry<String, String> overrideEntry : values.entrySet()) {
-              Long groupId = Long.parseLong(overrideEntry.getKey());
-              hc.getConfigGroupOverrides().put(groupId, overrideEntry.getValue());
-              if (!configGroupMap.containsKey(groupId)) {
-                LOG.debug("Config group does not exist, id = " + groupId);
-              }
+      LOG.debug("Updating actual config tags: " + configTags);
+      actualConfigs = new HashMap<String, HostConfig>();
+
+      for (Entry<String, Map<String, String>> entry : configTags.entrySet()) {
+        String type = entry.getKey();
+        Map<String, String> values = new HashMap<String, String>(
+            entry.getValue());
+
+        String tag = values.get(ConfigHelper.CLUSTER_DEFAULT_TAG);
+        values.remove(ConfigHelper.CLUSTER_DEFAULT_TAG);
+
+        HostConfig hc = new HostConfig();
+        hc.setDefaultVersionTag(tag);
+        actualConfigs.put(type, hc);
+
+        if (!values.isEmpty()) {
+          for (Entry<String, String> overrideEntry : values.entrySet()) {
+            Long groupId = Long.parseLong(overrideEntry.getKey());
+            hc.getConfigGroupOverrides().put(groupId, overrideEntry.getValue());
+            if (!configGroupMap.containsKey(groupId)) {
+              LOG.debug("Config group does not exist, id = " + groupId);
             }
           }
         }
-      } finally {
-        writeLock.unlock();
       }
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      writeLock.unlock();
     }
   }
-  
-  
-  
+
+
+
   @Override
   public Map<String, HostConfig> getActualConfigs() {
-    clusterGlobalLock.readLock().lock();
+    readLock.lock();
     try {
-      readLock.lock();
-      try {
-        return actualConfigs;
-      } finally {
-        readLock.unlock();
-      }
+      return actualConfigs;
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readLock.unlock();
     }
-
   }
 
   @Override
   public HostState getHostState() {
-    clusterGlobalLock.readLock().lock();
+    readLock.lock();
     try {
-      readLock.lock();
-      try {
-        return host.getState();
-      } finally {
-        readLock.unlock();
-      }
+      return host.getState();
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readLock.unlock();
     }
   }
-  
+
   @Override
   public void setMaintenanceState(MaintenanceState state) {
-    clusterGlobalLock.readLock().lock();
+    writeLock.lock();
     try {
-      writeLock.lock();
-      try {
-        desiredStateEntity.setMaintenanceState(state);
-        saveIfPersisted();
-      } finally {
-        writeLock.unlock();
-      }
+      desiredStateEntity.setMaintenanceState(state);
+      saveIfPersisted();
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      writeLock.unlock();
     }
   }
 
   @Override
   public MaintenanceState getMaintenanceState() {
-    clusterGlobalLock.readLock().lock();
+    readLock.lock();
     try {
-      readLock.lock();
-      try {
-        return desiredStateEntity.getMaintenanceState();
-      } finally {
-        readLock.unlock();
-      }
+      return desiredStateEntity.getMaintenanceState();
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readLock.unlock();
     }
   }
-  
+
   @Override
   public void setProcesses(List<Map<String, String>> procs) {
-    clusterGlobalLock.readLock().lock();
+    writeLock.lock();
     try {
-      writeLock.lock();
-      try {
-        processes = Collections.unmodifiableList(procs);
-      } finally {
-        writeLock.unlock();
-      }
+      processes = Collections.unmodifiableList(procs);
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      writeLock.unlock();
     }
   }
-  
-  @Override  
+
+  @Override
   public List<Map<String, String>> getProcesses() {
-    clusterGlobalLock.readLock().lock();
+    readLock.lock();
     try {
-      readLock.lock();
-      try {
-        return processes;
-      } finally {
-        readLock.unlock();
-      }
+      return processes;
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readLock.unlock();
     }
   }
 
   @Override
   public boolean isRestartRequired() {
-    clusterGlobalLock.readLock().lock();
+    readLock.lock();
     try {
-      readLock.lock();
-      try {
-        return desiredStateEntity.isRestartRequired();
-      } finally {
-        readLock.unlock();
-      }
+      return desiredStateEntity.isRestartRequired();
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      readLock.unlock();
     }
   }
 
   @Override
   public void setRestartRequired(boolean restartRequired) {
-    clusterGlobalLock.readLock().lock();
+    writeLock.lock();
     try {
-      writeLock.lock();
-      try {
-        desiredStateEntity.setRestartRequired(restartRequired);
-        saveIfPersisted();
-        helper.invalidateStaleConfigsCache(this);
-      } finally {
-        writeLock.unlock();
-      }
+      desiredStateEntity.setRestartRequired(restartRequired);
+      saveIfPersisted();
+      helper.invalidateStaleConfigsCache(this);
     } finally {
-      clusterGlobalLock.readLock().unlock();
+      writeLock.unlock();
     }
   }
 }

ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterDeadlockTest.java

@@ -0,0 +1,357 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.state.cluster;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.ServiceComponentNotFoundException;
+import org.apache.ambari.server.ServiceNotFoundException;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.orm.GuiceJpaInitializer;
+import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Host;
+import org.apache.ambari.server.state.MaintenanceState;
+import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.ServiceComponent;
+import org.apache.ambari.server.state.ServiceComponentFactory;
+import org.apache.ambari.server.state.ServiceComponentHost;
+import org.apache.ambari.server.state.ServiceComponentHostFactory;
+import org.apache.ambari.server.state.ServiceFactory;
+import org.apache.ambari.server.state.StackId;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.inject.Guice;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.persist.PersistService;
+
+/**
+ * Tests AMBARI-9368 which produced a deadlock during read and writes of some of
+ * the impl classes.
+ */
+public class ClusterDeadlockTest {
+  private final AtomicInteger hostNameCounter = new AtomicInteger(0);
+
+  @Inject
+  private Injector injector;
+
+  @Inject
+  private Clusters clusters;
+
+  @Inject
+  private ServiceFactory serviceFactory;
+
+  @Inject
+  private ServiceComponentFactory serviceComponentFactory;
+
+  @Inject
+  private ServiceComponentHostFactory serviceComponentHostFactory;
+
+  @Inject
+  private AmbariMetaInfo metaInfo;
+
+
+  @Before
+  public void setup() throws Exception {
+    injector = Guice.createInjector(new InMemoryDefaultTestModule());
+    injector.getInstance(GuiceJpaInitializer.class);
+    injector.injectMembers(this);
+    clusters.addCluster("c1");
+
+    StackId stackId = new StackId("HDP-0.1");
+    Cluster c1 = clusters.getCluster("c1");
+    c1.setDesiredStackVersion(stackId);
+    metaInfo.init();
+
+    // 100 hosts
+    for (int i = 0; i < 100; i++) {
+      String hostName = "c64-" + i;
+      clusters.addHost(hostName);
+      setOsFamily(clusters.getHost(hostName), "redhat", "6.4");
+      clusters.getHost(hostName).persist();
+      clusters.mapHostToCluster(hostName, "c1");
+    }
+
+    // force creation of the service and the components on the last host
+    createNewServiceComponentHost("HDFS", "NAMENODE", "c64-99", false);
+    createNewServiceComponentHost("HDFS", "HDFS_CLIENT", "c64-99", true);
+  }
+
+  @After
+  public void teardown() {
+    injector.getInstance(PersistService.class).stop();
+  }
+
+  /**
+   * Tests that concurrent impl serialization and impl writing doesn't cause a
+   * deadlock.
+   *
+   * @throws Exception
+   */
+  @Test(timeout = 30000)
+  public void testDeadlockBetweenImplementations() throws Exception {
+    Cluster cluster = clusters.getCluster("c1");
+    Service service = cluster.getService("HDFS");
+    ServiceComponent namenodeComponent = service.getServiceComponent("NAMENODE");
+    ServiceComponent hdfsClientComponent = service.getServiceComponent("HDFS_CLIENT");
+
+    ServiceComponentHost namenodeSCH = createNewServiceComponentHost("HDFS",
+        "NAMENODE", "c64-0", false);
+
+    ServiceComponentHost hdfsClientSCH = createNewServiceComponentHost("HDFS",
+        "HDFS_CLIENT", "c64-0", true);
+
+    List<Thread> threads = new ArrayList<Thread>();
+    for (int i = 0; i < 3; i++) {
+      DeadlockExerciserThread thread = new DeadlockExerciserThread();
+      thread.setCluster(cluster);
+      thread.setService(service);
+      thread.setHdfsClientComponent(hdfsClientComponent);
+      thread.setNamenodeComponent(namenodeComponent);
+      thread.setNamenodeSCH(namenodeSCH);
+      thread.setHdfsClientSCH(hdfsClientSCH);
+      thread.start();
+      threads.add(thread);
+    }
+
+    for (Thread thread : threads) {
+      thread.join();
+    }
+  }
+
+  /**
+   * Tests that while serializing a service component, writes to that service
+   * component do not cause a deadlock with the global cluster lock.
+   *
+   * @throws Exception
+   */
+  @Test(timeout = 30000)
+  public void testAddingHostComponentsWhileReading() throws Exception {
+    Cluster cluster = clusters.getCluster("c1");
+    Service service = cluster.getService("HDFS");
+    ServiceComponent namenodeComponent = service.getServiceComponent("NAMENODE");
+    ServiceComponent hdfsClientComponent = service.getServiceComponent("HDFS_CLIENT");
+
+    List<Thread> threads = new ArrayList<Thread>();
+    for (int i = 0; i < 5; i++) {
+      ServiceComponentDeadlockThread thread = new ServiceComponentDeadlockThread();
+      thread.setHdfsClientComponent(hdfsClientComponent);
+      thread.setNamenodeComponent(namenodeComponent);
+      thread.start();
+      threads.add(thread);
+    }
+
+    for (Thread thread : threads) {
+      thread.join();
+    }
+  }
+
+  /**
+   * Tests AMBARI-9368 which saw a deadlock when adding a service component host
+   * while reading a service component.
+   */
+  private final class ServiceComponentDeadlockThread extends Thread {
+    private ServiceComponent namenodeComponent;
+    private ServiceComponent hdfsClientComponent;
+
+    /**
+     * @param namenodeComponent
+     *          the namenodeComponent to set
+     */
+    public void setNamenodeComponent(ServiceComponent namenodeComponent) {
+      this.namenodeComponent = namenodeComponent;
+    }
+
+    /**
+     * @param hdfsClientComponent
+     *          the hdfsClientComponent to set
+     */
+    public void setHdfsClientComponent(ServiceComponent hdfsClientComponent) {
+      this.hdfsClientComponent = hdfsClientComponent;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public void run() {
+      try {
+        for (int i = 0; i < 15; i++) {
+          int hostNumeric = hostNameCounter.getAndIncrement();
+
+          namenodeComponent.convertToResponse();
+          createNewServiceComponentHost("HDFS", "NAMENODE", "c64-"
+              + hostNumeric, false);
+
+          hdfsClientComponent.convertToResponse();
+          createNewServiceComponentHost("HDFS", "HDFS_CLIENT", "c64-"
+              + hostNumeric, true);
+
+          Thread.sleep(10);
+        }
+      } catch (Exception exception) {
+        throw new RuntimeException(exception);
+      }
+    }
+  }
+
+  /**
+   * Tests AMBARI-9368 which produced a deadlock during read and writes of some
+   * of the impl classes.
+   */
+  private static final class DeadlockExerciserThread extends Thread {
+    private Cluster cluster;
+    private Service service;
+    private ServiceComponent namenodeComponent;
+    private ServiceComponent hdfsClientComponent;
+    private ServiceComponentHost namenodeSCH;
+    private ServiceComponentHost hdfsClientSCH;
+
+    /**
+     * @param cluster
+     *          the cluster to set
+     */
+    public void setCluster(Cluster cluster) {
+      this.cluster = cluster;
+    }
+
+    /**
+     * @param service
+     *          the service to set
+     */
+    public void setService(Service service) {
+      this.service = service;
+    }
+
+    /**
+     * @param namenodeComponent
+     *          the namenodeComponent to set
+     */
+    public void setNamenodeComponent(ServiceComponent namenodeComponent) {
+      this.namenodeComponent = namenodeComponent;
+    }
+
+    /**
+     * @param hdfsClientComponent
+     *          the hdfsClientComponent to set
+     */
+    public void setHdfsClientComponent(ServiceComponent hdfsClientComponent) {
+      this.hdfsClientComponent = hdfsClientComponent;
+    }
+
+    /**
+     * @param namenodeSCH
+     *          the namenodeSCH to set
+     */
+    public void setNamenodeSCH(ServiceComponentHost namenodeSCH) {
+      this.namenodeSCH = namenodeSCH;
+    }
+
+    /**
+     * @param hdfsClientSCH
+     *          the hdfsClientSCH to set
+     */
+    public void setHdfsClientSCH(ServiceComponentHost hdfsClientSCH) {
+      this.hdfsClientSCH = hdfsClientSCH;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public void run() {
+      try {
+        for (int i = 0; i < 10; i++) {
+          cluster.convertToResponse();
+          service.convertToResponse();
+          namenodeComponent.convertToResponse();
+          hdfsClientComponent.convertToResponse();
+          namenodeSCH.convertToResponse();
+          hdfsClientSCH.convertToResponse();
+
+          cluster.setProvisioningState(org.apache.ambari.server.state.State.INIT);
+          service.setMaintenanceState(MaintenanceState.OFF);
+          namenodeComponent.setDesiredState(org.apache.ambari.server.state.State.STARTED);
+          hdfsClientComponent.setDesiredState(org.apache.ambari.server.state.State.INSTALLED);
+
+          namenodeSCH.setState(org.apache.ambari.server.state.State.STARTED);
+          hdfsClientSCH.setState(org.apache.ambari.server.state.State.INSTALLED);
+
+          Thread.sleep(100);
+        }
+      } catch (Exception exception) {
+        throw new RuntimeException(exception);
+      }
+    }
+  }
+
+  private void setOsFamily(Host host, String osFamily, String osVersion) {
+    Map<String, String> hostAttributes = new HashMap<String, String>(2);
+    hostAttributes.put("os_family", osFamily);
+    hostAttributes.put("os_release_version", osVersion);
+    host.setHostAttributes(hostAttributes);
+  }
+
+  private ServiceComponentHost createNewServiceComponentHost(String svc,
+      String svcComponent, String hostName, boolean isClient)
+      throws AmbariException {
+    Cluster c = clusters.getCluster("c1");
+    Assert.assertNotNull(c.getConfigGroups());
+    return createNewServiceComponentHost(c, svc, svcComponent, hostName);
+  }
+
+  private ServiceComponentHost createNewServiceComponentHost(Cluster c,
+      String svc, String svcComponent, String hostName) throws AmbariException {
+
+    Service s = null;
+
+    try {
+      s = c.getService(svc);
+    } catch (ServiceNotFoundException e) {
+      s = serviceFactory.createNew(c, svc);
+      c.addService(s);
+      s.persist();
+    }
+
+    ServiceComponent sc = null;
+    try {
+      sc = s.getServiceComponent(svcComponent);
+    } catch (ServiceComponentNotFoundException e) {
+      sc = serviceComponentFactory.createNew(s, svcComponent);
+      s.addServiceComponent(sc);
+      sc.persist();
+    }
+
+    ServiceComponentHost impl = serviceComponentHostFactory.createNew(sc,
+        hostName);
+
+    impl.persist();
+    return impl;
+  }
+}

ambari-web/package.json

@@ -19,7 +19,7 @@
   "devDependencies": {
     "phantomjs": "^1.9.2",
     "mocha":"1.9.0",
-    "mocha-phantomjs": "^3.1.6",
+    "mocha-phantomjs": "~3.1.6",
     "chai":"~1.9.0",
     "sinon":"=1.7.3",
     "sinon-chai":"~2.5.0",