Merge branch 'trunk' into branch-feature-AMBARI-14714

ambari-admin/src/main/resources/ui/admin-web/app/scripts/app.js

@@ -28,7 +28,7 @@ angular.module('ambariAdminConsole', [
 .constant('Settings', {
   siteRoot: '{proxy_root}/'.replace(/\{.+\}/g, ''),
 	baseUrl: '{proxy_root}/api/v1'.replace(/\{.+\}/g, ''),
-  testMode: (window.location.port == 8000),
+  testMode: false,
   mockDataPrefix: 'assets/data/',
   isLDAPConfigurationSupported: false,
   isLoginActivitiesSupported: false,

ambari-agent/src/main/python/ambari_agent/AlertSchedulerHandler.py

@@ -50,7 +50,7 @@ class AlertSchedulerHandler():
   TYPE_RECOVERY = 'RECOVERY'
 
   def __init__(self, initializer_module, in_minutes=True):
-
+    self.initializer_module = initializer_module
     self.cachedir = initializer_module.config.alerts_cachedir
     self.stacks_dir = initializer_module.config.stacks_dir
     self.common_services_dir = initializer_module.config.common_services_dir
@@ -169,6 +169,8 @@ class AlertSchedulerHandler():
     definitions = self.__load_definitions()
     scheduled_jobs = self.__scheduler.get_jobs()
 
+    self.initializer_module.alert_status_reporter.reported_alerts.clear()
+
     # for every scheduled job, see if its UUID is still valid
     for scheduled_job in scheduled_jobs:
       uuid_valid = False

ambari-agent/src/main/python/ambari_agent/Hardware.py

@@ -103,11 +103,11 @@ class Hardware:
   def _check_remote_mounts(self):
     """Verify if remote mount allowed to be processed or not"""
     if self.config and self.config.has_option(AmbariConfig.AMBARI_PROPERTIES_CATEGORY, Hardware.CHECK_REMOTE_MOUNTS_KEY) and \
-      self.config.get(AmbariConfig.AMBARI_PROPERTIES_CATEGORY, Hardware.CHECK_REMOTE_MOUNTS_KEY).lower() == "false":
+      self.config.get(AmbariConfig.AMBARI_PROPERTIES_CATEGORY, Hardware.CHECK_REMOTE_MOUNTS_KEY).lower() == "true":
 
-      return False
+      return True
 
-    return True
+    return False
 
   def _is_mount_blacklisted(self, blacklist, mount_point):
     """

ambari-agent/src/main/python/ambari_agent/security.py

@@ -37,7 +37,7 @@ from socket import error as socket_error
 
 logger = logging.getLogger(__name__)
 
-GEN_AGENT_KEY = 'openssl req -new -newkey rsa:1024 -nodes -keyout "%(keysdir)s' \
+GEN_AGENT_KEY = 'openssl req -new -newkey rsa -nodes -keyout "%(keysdir)s' \
                 + os.sep + '%(hostname)s.key" -subj /OU=%(hostname)s/ ' \
                 '-out "%(keysdir)s' + os.sep + '%(hostname)s.csr"'
 KEY_FILENAME = '%(hostname)s.key'

ambari-agent/src/test/python/ambari_agent/TestHardware.py

@@ -154,11 +154,11 @@ class TestHardware(TestCase):
     get_os_version_mock.return_value = "11"
     Hardware(cache_info=False).osdisks()
     timeout = 10
-    shell_call_mock.assert_called_with(['timeout', str(timeout), "df", "-kPT"], stdout = subprocess32.PIPE, stderr = subprocess32.PIPE, timeout = timeout, quiet = True)
+    shell_call_mock.assert_called_with(['timeout', str(timeout), "df", "-kPT", "-l"], stdout = subprocess32.PIPE, stderr = subprocess32.PIPE, timeout = timeout, quiet = True)
 
     config = AmbariConfig()
     Hardware(config=config, cache_info=False).osdisks()
-    shell_call_mock.assert_called_with(['timeout', str(timeout), "df", "-kPT"], stdout = subprocess32.PIPE, stderr = subprocess32.PIPE, timeout = timeout, quiet = True)
+    shell_call_mock.assert_called_with(['timeout', str(timeout), "df", "-kPT", "-l"], stdout = subprocess32.PIPE, stderr = subprocess32.PIPE, timeout = timeout, quiet = True)
 
     config.add_section(AmbariConfig.AMBARI_PROPERTIES_CATEGORY)
     config.set(AmbariConfig.AMBARI_PROPERTIES_CATEGORY, Hardware.CHECK_REMOTE_MOUNTS_KEY, "true")

ambari-common/src/main/python/ambari_ws4py/websocket.py

@@ -480,7 +480,8 @@ class WebSocket(object):
         if not bytes and self.reading_buffer_size > 0:
             return False
 
-        self.reading_buffer_size = s.parser.send(bytes) or DEFAULT_READING_SIZE
+        with self.lock:
+          self.reading_buffer_size = s.parser.send(bytes) or DEFAULT_READING_SIZE
 
         if s.closing is not None:
             logger.info("Closing message received (%d) '%s'" % (s.closing.code, s.closing.reason))

ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py

@@ -88,7 +88,7 @@ class HdfsResourceJar:
       nameservices = main_resource.resource.nameservices
 
     # non-federated cluster
-    if not nameservices:
+    if not nameservices or len(nameservices) < 2:
       self.action_delayed_for_nameservice(None, action_name, main_resource)
     else:
       default_fs_protocol = urlparse(main_resource.resource.default_fs).scheme

ambari-logsearch/ambari-logsearch-server/build.xml

@@ -48,7 +48,7 @@
       <fileset file="target/classes/logsearch.properties"/>
       <fileset file="target/classes/info.properties"/>
       <fileset file="target/classes/user_pass.json"/>
-      <fileset file="target/classes/HadoopServiceConfig.json"/>
+      <fileset file="target/classes/roles.json"/>
     </copy>
     <copy todir="target/package/conf/solr_configsets" includeEmptyDirs="yes">
       <fileset dir="src/main/configsets"/>

ambari-logsearch/ambari-logsearch-server/src/main/configsets/hadoop_logs/conf/solrconfig.xml

@@ -1213,7 +1213,7 @@
      -->
   <searchComponent name="spellcheck" class="solr.SpellCheckComponent">
 
-    <str name="queryAnalyzerFieldType">key_lower_case</str>
+    <str name="queryAnalyzerFieldType">lowercase</str>
 
     <!-- Multiple "Spell Checkers" can be declared and used by this
          component
@@ -1688,7 +1688,7 @@
       </arr>
     </processor>
     <processor class="solr.AddSchemaFieldsUpdateProcessorFactory">
-      <str name="defaultFieldType">key_lower_case</str>
+      <str name="defaultFieldType">lowercase</str>
       <lst name="typeMapping">
         <str name="valueClass">java.lang.Boolean</str>
         <str name="fieldType">booleans</str>

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/auth/filter/AbstractJWTFilter.java

@@ -86,14 +86,13 @@ public abstract class AbstractJWTFilter extends AbstractAuthenticationProcessing
         .setSigningKey(parseRSAPublicKey(getPublicKey()))
         .parseClaimsJws(getJWTFromCookie(request))
         .getBody();
-
       String userName  = claims.getSubject();
       LOG.info("USERNAME: " + userName);
       LOG.info("URL = " + request.getRequestURL());
       if (StringUtils.isNotEmpty(claims.getAudience()) && !getAudiences().contains(claims.getAudience())) {
         throw new IllegalArgumentException(String.format("Audience validation failed. (Not found: %s)", claims.getAudience()));
       }
-      Authentication authentication = new JWTAuthenticationToken(userName, getPublicKey(), getAuthorities());
+      Authentication authentication = new JWTAuthenticationToken(userName, getPublicKey(), getAuthorities(userName));
       authentication.setAuthenticated(true);
       SecurityContextHolder.getContext().setAuthentication(authentication);
       return authentication;
@@ -248,7 +247,7 @@ public abstract class AbstractJWTFilter extends AbstractAuthenticationProcessing
 
   protected abstract List<String> getAudiences();
 
-  protected abstract Collection<? extends GrantedAuthority> getAuthorities();
+  protected abstract Collection<? extends GrantedAuthority> getAuthorities(String username);
 
   protected abstract List<String> getUserAgentList();
 

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/LabelFallbackHandler.java

@@ -46,9 +46,9 @@ public class LabelFallbackHandler {
   }
 
   public String fallbackIfRequired(String field, String label, boolean replaceUnderscore,
-                                   boolean replaceUppercaseInWord, boolean capitalizeAll, List<String> prefixesToRemove) {
+                                   boolean replaceUppercaseInWord, boolean capitalizeAll, List<String> prefixesToRemove, List<String> suffixesToRemove) {
     if (isEnabled() && StringUtils.isBlank(label)) {
-      return fallback(field,replaceUnderscore, replaceUppercaseInWord, capitalizeAll, prefixesToRemove);
+      return fallback(field,replaceUnderscore, replaceUppercaseInWord, capitalizeAll, prefixesToRemove, suffixesToRemove);
     }
     return label;
   }
@@ -66,16 +66,24 @@ public class LabelFallbackHandler {
     return result;
   }
 
-  public String fallback(String field, boolean replaceUnderscore, boolean replaceUppercaseInWord, boolean capitalizeAll, List<String> prefixesToRemove) {
-    String fieldWithoutPrefix =  null;
+  public String fallback(String field, boolean replaceUnderscore, boolean replaceUppercaseInWord, boolean capitalizeAll,
+                         List<String> prefixesToRemove,  List<String> suffixesToRemove) {
+    String fieldWithoutPrefixAndSuffix =  null;
     if (!CollectionUtils.isEmpty(prefixesToRemove)) {
       for (String prefix : prefixesToRemove) {
         if (StringUtils.isNotBlank(field) && field.startsWith(prefix)) {
-          fieldWithoutPrefix = field.substring(prefix.length());
+          fieldWithoutPrefixAndSuffix = field.substring(prefix.length());
         }
       }
     }
-    return fallback(fieldWithoutPrefix != null ? fieldWithoutPrefix : field, replaceUnderscore, replaceUppercaseInWord, capitalizeAll);
+    if (!CollectionUtils.isEmpty(suffixesToRemove)) {
+      for (String suffix : suffixesToRemove) {
+        if (StringUtils.isNotBlank(field) && field.endsWith(suffix)) {
+          fieldWithoutPrefixAndSuffix = field.substring(0, field.length() - suffix.length());
+        }
+      }
+    }
+    return fallback(fieldWithoutPrefixAndSuffix != null ? fieldWithoutPrefixAndSuffix : field, replaceUnderscore, replaceUppercaseInWord, capitalizeAll);
   }
 
   private String deUnderScore(String input) {

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java

@@ -72,7 +72,8 @@ public class LogSearchConstants {
   public static final String SERVICE_FIELD_VISIBLE_DEFAULTS = "log_message,level,logtime,type";
   public static final String SERVICE_FIELD_EXCLUDES_DEFAULTS = "id,tags,text,message,seq_num,case_id,bundle_id,rowtype,event_count";
   public static final String SERVICE_FIELD_FILTERABLE_EXLUDE_DEFAULTS = "";
-  public static final String SERVICE_FIELD_FALLBACK_PREFIX_DEFAULTS = "ws_,sdi_";
+  public static final String SERVICE_FIELD_FALLBACK_PREFIX_DEFAULTS = "ws_,sdi_,std_";
+  public static final String SERVICE_FIELD_FALLBACK_SUFFIX_DEFAULTS = "_i,_l,_s,_b";
 
   // audit  field / component label defaults
   public static final String AUDIT_COMPONENT_LABELS_DEFAULTS = "ambari:Ambari,hdfs:Hdfs,RangerAudit:Ranger";
@@ -86,6 +87,7 @@ public class LogSearchConstants {
   public static final String AUDIT_FIELD_FILTERABLE_EXCLUDES_DEFAULTS = "";
   public static final String AUDIT_FIELD_FILTERABLE_EXCLUDES_COMMON_DEFAULTS = "";
   public static final String AUDIT_FIELD_FALLBACK_PREFIX_DEFAULTS = "ws_,std_";
+  public static final String AUDIT_FIELD_FALLBACK_SUFFIX_DEFAULTS = "_i,_l,_s,_b";
 
   //Facet Constant
   public static final String FACET_FIELD = "facet.field";

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/AuthPropsConfig.java

@@ -21,7 +21,6 @@ package org.apache.ambari.logsearch.conf;
 import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Configuration;
-
 import java.util.List;
 
 import static org.apache.ambari.logsearch.common.LogSearchConstants.LOGSEARCH_PROPERTIES_FILE;
@@ -187,6 +186,75 @@ public class AuthPropsConfig {
   )
   private boolean redirectForward;
 
+  @Value("${logsearch.auth.trusted.proxy:false}")
+  @LogSearchPropertyDescription(
+    name = "logsearch.auth.trusted.proxy",
+    description = "A boolean property to enable/disable trusted-proxy 'knox' authentication",
+    examples = {"true"},
+    defaultValue = "false",
+    sources = {LOGSEARCH_PROPERTIES_FILE}
+  )
+  private boolean trustedProxy;
+
+  @Value("#{propertiesSplitter.parseList('${logsearch.auth.proxyuser.users:knox}')}")
+  @LogSearchPropertyDescription(
+    name = "logsearch.auth.proxyuser.users",
+    description = "List of users which the trusted-proxy user ‘knox’ can proxy for",
+    examples = {"knox,hdfs"},
+    defaultValue = "knox",
+    sources = {LOGSEARCH_PROPERTIES_FILE}
+  )
+  private List<String> proxyUsers;
+
+  @Value("#{propertiesSplitter.parseList('${logsearch.auth.proxyuser.groups:*}')}")
+  @LogSearchPropertyDescription(
+    name = "logsearch.auth.proxyuser.groups",
+    description = "List of user-groups which trusted-proxy user ‘knox’ can proxy for",
+    examples = {"admin,user"},
+    defaultValue = "*",
+    sources = {LOGSEARCH_PROPERTIES_FILE}
+  )
+  private List<String> proxyUserGroups;
+
+  @Value("#{propertiesSplitter.parseList('${logsearch.auth.proxyuser.hosts:*}')}")
+  @LogSearchPropertyDescription(
+    name = "logsearch.auth.proxyuser.hosts",
+    description = "List of hosts from which trusted-proxy user ‘knox’ can connect from",
+    examples = {"host1,host2"},
+    defaultValue = "*",
+    sources = {LOGSEARCH_PROPERTIES_FILE}
+  )
+  private List<String> proxyUserHosts;
+
+  @Value("#{propertiesSplitter.parseList('${logsearch.auth.proxyserver.ip:}')}")
+  @LogSearchPropertyDescription(
+    name = "logsearch.auth.proxyserver.ip",
+    description = "IP of trusted Knox Proxy server(s) that Log Search will trust on",
+    examples = {"192.168.0.1,192.168.0.2"},
+    sources = {LOGSEARCH_PROPERTIES_FILE}
+  )
+  private List<String> proxyIp;
+
+  @Value("${logsearch.authr.file.enable:false}")
+  @LogSearchPropertyDescription(
+    name = "logsearch.authr.file",
+    description = "A boolean property to enable/disable file based authorization",
+    examples = {"true"},
+    defaultValue = "false",
+    sources = {LOGSEARCH_PROPERTIES_FILE}
+  )
+  private boolean fileAuthorization;
+
+  @Value("${logsearch.authr.role.file:roles.json}")
+  @LogSearchPropertyDescription(
+    name = "logsearch.authr.role.file",
+    description = "Simple file that contains user/role mappings.",
+    examples = {"logsearch-roles.json"},
+    defaultValue = "roles.json",
+    sources = {LOGSEARCH_PROPERTIES_FILE}
+  )
+  private String roleFile;
+
   public boolean isAuthFileEnabled() {
     return authFileEnabled;
   }
@@ -314,4 +382,60 @@ public class AuthPropsConfig {
   public void setUserAgentList(List<String> userAgentList) {
     this.userAgentList = userAgentList;
   }
+
+  public boolean isTrustedProxy() {
+    return trustedProxy;
+  }
+
+  public void setTrustedProxy(boolean trustedProxy) {
+    this.trustedProxy = trustedProxy;
+  }
+
+  public List<String> getProxyUsers() {
+    return proxyUsers;
+  }
+
+  public void setProxyUsers(List<String> proxyUsers) {
+    this.proxyUsers = proxyUsers;
+  }
+
+  public List<String> getProxyUserGroups() {
+    return proxyUserGroups;
+  }
+
+  public void setProxyUserGroups(List<String> proxyUserGroups) {
+    this.proxyUserGroups = proxyUserGroups;
+  }
+
+  public List<String> getProxyUserHosts() {
+    return proxyUserHosts;
+  }
+
+  public void setProxyUserHosts(List<String> proxyUserHosts) {
+    this.proxyUserHosts = proxyUserHosts;
+  }
+
+  public List<String> getProxyIp() {
+    return proxyIp;
+  }
+
+  public void setProxyIp(List<String> proxyIp) {
+    this.proxyIp = proxyIp;
+  }
+
+  public boolean isFileAuthorization() {
+    return fileAuthorization;
+  }
+
+  public void setFileAuthorization(boolean fileAuthorization) {
+    this.fileAuthorization = fileAuthorization;
+  }
+
+  public String getRoleFile() {
+    return roleFile;
+  }
+
+  public void setRoleFile(String roleFile) {
+    this.roleFile = roleFile;
+  }
 }

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java

@@ -31,6 +31,7 @@ import org.apache.ambari.logsearch.common.StatusMessage;
 import org.apache.ambari.logsearch.conf.global.LogLevelFilterManagerState;
 import org.apache.ambari.logsearch.conf.global.LogSearchConfigState;
 import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.dao.RoleDao;
 import org.apache.ambari.logsearch.web.authenticate.LogsearchAuthFailureHandler;
 import org.apache.ambari.logsearch.web.authenticate.LogsearchAuthSuccessHandler;
 import org.apache.ambari.logsearch.web.authenticate.LogsearchLogoutSuccessHandler;
@@ -42,6 +43,7 @@ import org.apache.ambari.logsearch.web.filters.LogsearchFilter;
 import org.apache.ambari.logsearch.web.filters.LogsearchJWTFilter;
 import org.apache.ambari.logsearch.web.filters.LogsearchKRBAuthenticationFilter;
 import org.apache.ambari.logsearch.web.filters.LogsearchSecurityContextFormationFilter;
+import org.apache.ambari.logsearch.web.filters.LogsearchTrustedProxyFilter;
 import org.apache.ambari.logsearch.web.filters.LogsearchUsernamePasswordAuthenticationFilter;
 import org.apache.ambari.logsearch.web.security.LogsearchAuthenticationProvider;
 import org.springframework.context.annotation.Bean;
@@ -98,20 +100,25 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
   @Inject
   private LogSearchConfigApiConfig logSearchConfigApiConfig;
 
+  @Inject
+  private RoleDao roleDao;
+
   @Override
   protected void configure(HttpSecurity http) throws Exception {
-
     http
       .csrf().disable()
       .authorizeRequests()
-        .requestMatchers(requestMatcher()).permitAll()
-        .antMatchers("/**").authenticated()
+        .requestMatchers(requestMatcher())
+          .permitAll()
+        .antMatchers("/**")
+          .hasRole("USER")
       .and()
       .authenticationProvider(logsearchAuthenticationProvider())
       .httpBasic()
         .authenticationEntryPoint(logsearchAuthenticationEntryPoint())
       .and()
-      .addFilterBefore(logsearchKRBAuthenticationFilter(), BasicAuthenticationFilter.class)
+      .addFilterBefore(logsearchTrustedProxyFilter(), BasicAuthenticationFilter.class)
+      .addFilterAfter(logsearchKRBAuthenticationFilter(), LogsearchTrustedProxyFilter.class)
       .addFilterBefore(logsearchUsernamePasswordAuthenticationFilter(), LogsearchKRBAuthenticationFilter.class)
       .addFilterAfter(securityContextFormationFilter(), FilterSecurityInterceptor.class)
       .addFilterAfter(logsearchEventHistoryFilter(), LogsearchSecurityContextFormationFilter.class)
@@ -150,9 +157,16 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
     return new LogsearchAuthenticationProvider();
   }
 
+  @Bean
+  public LogsearchTrustedProxyFilter logsearchTrustedProxyFilter() throws Exception {
+    LogsearchTrustedProxyFilter filter = new LogsearchTrustedProxyFilter(requestMatcher(), authPropsConfig);
+    filter.setAuthenticationManager(authenticationManagerBean());
+    return filter;
+  }
+
   @Bean
   public LogsearchJWTFilter logsearchJwtFilter() throws Exception {
-    LogsearchJWTFilter filter = new LogsearchJWTFilter(requestMatcher(), authPropsConfig);
+    LogsearchJWTFilter filter = new LogsearchJWTFilter(requestMatcher(), authPropsConfig, roleDao);
     filter.setAuthenticationManager(authenticationManagerBean());
     filter.setAuthenticationSuccessHandler(new LogsearchAuthSuccessHandler());
     filter.setAuthenticationFailureHandler(new LogsearchAuthFailureHandler());

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/UIMappingConfig.java

@@ -18,7 +18,6 @@
  */
 package org.apache.ambari.logsearch.conf;
 
-import org.apache.ambari.logsearch.common.LogSearchConstants;
 import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Configuration;
@@ -35,6 +34,7 @@ import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_
 import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_EXCLUDES_COMMON_DEFAULTS;
 import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_EXCLUDES_DEFAULTS;
 import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_FALLBACK_PREFIX_DEFAULTS;
+import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_FALLBACK_SUFFIX_DEFAULTS;
 import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_FILTERABLE_EXCLUDES_COMMON_DEFAULTS;
 import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_FILTERABLE_EXCLUDES_DEFAULTS;
 import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_LABELS_DEFAULTS;
@@ -42,6 +42,7 @@ import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_
 import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_VISIBLE_DEFAULTS;
 import static org.apache.ambari.logsearch.common.LogSearchConstants.LOGSEARCH_PROPERTIES_FILE;
 import static org.apache.ambari.logsearch.common.LogSearchConstants.SERVICE_FIELD_FALLBACK_PREFIX_DEFAULTS;
+import static org.apache.ambari.logsearch.common.LogSearchConstants.SERVICE_FIELD_FALLBACK_SUFFIX_DEFAULTS;
 import static org.apache.ambari.logsearch.common.LogSearchConstants.SERVICE_FIELD_FILTERABLE_EXLUDE_DEFAULTS;
 import static org.apache.ambari.logsearch.common.LogSearchConstants.SERVICE_GROUP_LABELS_DEFAULTS;
 import static org.apache.ambari.logsearch.common.LogSearchConstants.SERVICE_COMPONENT_LABELS_DEFAULTS;
@@ -222,7 +223,7 @@ public class UIMappingConfig {
   )
   private List<String> serviceFieldFallbackPrefixes;
 
-  @Value("#{propertiesSplitter.parseList('${logsearch.web.labels.service_logs.field.fallback.prefixes:" + AUDIT_FIELD_FALLBACK_PREFIX_DEFAULTS + "}')}")
+  @Value("#{propertiesSplitter.parseList('${logsearch.web.labels.audit_logs.field.fallback.prefixes:" + AUDIT_FIELD_FALLBACK_PREFIX_DEFAULTS + "}')}")
   @LogSearchPropertyDescription(
     name = "logsearch.web.labels.service_logs.field.fallback.prefixes",
     description = "List of prefixes that should be removed during fallback of audit field labels.",
@@ -232,6 +233,26 @@ public class UIMappingConfig {
   )
   private List<String> auditFieldFallbackPrefixes;
 
+  @Value("#{propertiesSplitter.parseList('${logsearch.web.labels.service_logs.field.fallback.suffixes:" + SERVICE_FIELD_FALLBACK_PREFIX_DEFAULTS +"}')}")
+  @LogSearchPropertyDescription(
+    name = "logsearch.web.labels.service_logs.field.fallback.suffixes",
+    description = "List of suffixes that should be removed during fallback of service field labels.",
+    examples = {"_i,_l,_s,_b"},
+    defaultValue = SERVICE_FIELD_FALLBACK_SUFFIX_DEFAULTS,
+    sources = {LOGSEARCH_PROPERTIES_FILE}
+  )
+  private List<String> serviceFieldFallbackSuffixes;
+
+  @Value("#{propertiesSplitter.parseList('${logsearch.web.labels.audit_logs.field.fallback.suffixes:" + AUDIT_FIELD_FALLBACK_PREFIX_DEFAULTS + "}')}")
+  @LogSearchPropertyDescription(
+    name = "logsearch.web.labels.service_logs.field.fallback.suffixes",
+    description = "List of suffixes that should be removed during fallback of audit field labels.",
+    examples = {"_i,_l,_s,_b"},
+    defaultValue = AUDIT_FIELD_FALLBACK_SUFFIX_DEFAULTS,
+    sources = {LOGSEARCH_PROPERTIES_FILE}
+  )
+  private List<String> auditFieldFallbackSuffixes;
+
   private final Map<String, Map<String, String>> mergedAuditFieldLabelMap = new HashMap<>();
 
   private final Map<String, List<String>> mergedAuditFieldVisibleMap = new HashMap<>();
@@ -368,6 +389,22 @@ public class UIMappingConfig {
     this.serviceFieldFilterableExcludesList = serviceFieldFilterableExcludesList;
   }
 
+  public List<String> getServiceFieldFallbackSuffixes() {
+    return serviceFieldFallbackSuffixes;
+  }
+
+  public void setServiceFieldFallbackSuffixes(List<String> serviceFieldFallbackSuffixes) {
+    this.serviceFieldFallbackSuffixes = serviceFieldFallbackSuffixes;
+  }
+
+  public List<String> getAuditFieldFallbackSuffixes() {
+    return auditFieldFallbackSuffixes;
+  }
+
+  public void setAuditFieldFallbackSuffixes(List<String> auditFieldFallbackSuffixes) {
+    this.auditFieldFallbackSuffixes = auditFieldFallbackSuffixes;
+  }
+
   public Map<String, List<String>> getMergedAuditFieldVisibleMap() {
     return mergedAuditFieldVisibleMap;
   }

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/RoleDao.java

@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.dao;
+
+import com.google.common.annotations.VisibleForTesting;
+import io.jsonwebtoken.lang.Collections;
+import org.apache.ambari.logsearch.conf.AuthPropsConfig;
+import org.apache.ambari.logsearch.util.FileUtil;
+import org.apache.ambari.logsearch.util.JSONUtil;
+import org.apache.ambari.logsearch.web.model.Privilege;
+import org.apache.ambari.logsearch.web.model.Role;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.security.core.GrantedAuthority;
+
+import javax.annotation.PostConstruct;
+import javax.inject.Inject;
+import javax.inject.Named;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.singletonList;
+
+/**
+ * Helper class to assign roles for authenticated users, can be used only by JWT and file based authentication.
+ */
+@Named
+public class RoleDao {
+
+  private static final Logger LOG = LoggerFactory.getLogger(RoleDao.class);
+
+  @Inject
+  private AuthPropsConfig authPropsConfig;
+
+  private final Map<String, List<String>> simpleRolesMap = new HashMap<>();
+
+  @SuppressWarnings("unchecked")
+  @PostConstruct
+  public void init() {
+    if (authPropsConfig.isFileAuthorization()) {
+      try {
+        String userRoleFileName = authPropsConfig.getRoleFile();
+        LOG.info("USER ROLE JSON file NAME:" + userRoleFileName);
+        File jsonFile = FileUtil.getFileFromClasspath(userRoleFileName);
+        if (jsonFile == null || !jsonFile.exists()) {
+          LOG.error("Role json file not found on the classpath :" + userRoleFileName);
+          System.exit(1);
+        }
+        Map<String, Object> userRoleInfo = JSONUtil.readJsonFromFile(jsonFile);
+        Map<String, Object> roles = (Map<String, Object>) userRoleInfo.get("roles");
+        for (Map.Entry<String, Object> roleEntry : roles.entrySet()) {
+          simpleRolesMap.put(roleEntry.getKey(), (List<String>) roleEntry.getValue());
+        }
+      } catch (Exception e) {
+        LOG.error("Error while reading user role file: {}", e.getMessage());
+      }
+    } else {
+      LOG.info("File authorization is disabled");
+    }
+  }
+
+  public List<GrantedAuthority> getRolesForUser(String user) {
+    List<GrantedAuthority> authorities = new ArrayList<>();
+    if (authPropsConfig.isFileAuthorization()) {
+        List<String > roles = simpleRolesMap.get(user);
+        if (!Collections.isEmpty(roles)) {
+          for (String role : roles) {
+            String roleName = "ROLE_" + role;
+            LOG.debug("Found role '{}' for user '{}'", roleName, user);
+            authorities.add(createRoleWithReadPrivilage(roleName));
+          }
+        } else {
+          LOG.warn("Not found roles for user '{}'", user);
+        }
+      return authorities;
+    } else {
+      return createDefaultAuthorities();
+    }
+  }
+
+  public Map<String, List<String>> getSimpleRolesMap() {
+    return simpleRolesMap;
+  }
+
+  @VisibleForTesting
+  public void setAuthPropsConfig(AuthPropsConfig authPropsConfig) {
+    this.authPropsConfig = authPropsConfig;
+  }
+
+  /**
+   * Helper function to create a simple default role details
+   */
+  public static List<GrantedAuthority> createDefaultAuthorities() {
+    Role r = createRoleWithReadPrivilage("ROLE_USER");
+    return singletonList(r);
+  }
+
+  private static Role createRoleWithReadPrivilage(String roleName) {
+    Role r = new Role();
+    r.setName(roleName);
+    Privilege priv = new Privilege();
+    priv.setName("READ_PRIVILEGE");
+    r.setPrivileges(singletonList(priv));
+    return r;
+  }
+}

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java

@@ -23,6 +23,7 @@ import static java.util.Collections.singletonList;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 
 import javax.annotation.PostConstruct;
 import javax.inject.Inject;
@@ -36,6 +37,7 @@ import org.apache.ambari.logsearch.web.model.Role;
 import org.apache.ambari.logsearch.web.model.User;
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
+import org.springframework.security.core.GrantedAuthority;
 import org.springframework.stereotype.Repository;
 
 @Repository
@@ -50,6 +52,9 @@ public class UserDao {
   @Inject
   private AuthPropsConfig authPropsConfig;
 
+  @Inject
+  private RoleDao roleDao;
+
   private ArrayList<HashMap<String, String>> userList = null;
 
   @SuppressWarnings("unchecked")
@@ -98,13 +103,7 @@ public class UserDao {
     user.setLastName(StringUtils.defaultString(userInfo.get(NAME), "Unknown"));
     user.setUsername(StringUtils.defaultString(userInfo.get(USER_NAME), ""));
     user.setPassword(StringUtils.defaultString(userInfo.get(ENC_PASSWORD), ""));
-
-    Role r = new Role();
-    r.setName("ROLE_USER");
-    Privilege priv = new Privilege();
-    priv.setName("READ_PRIVILEGE");
-    r.setPrivileges(singletonList(priv));
-    user.setAuthorities(singletonList(r));
+    user.setAuthorities(roleDao.getRolesForUser(user.getUsername()));
     
     return user;
   }

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java

@@ -178,7 +178,8 @@ public class AuditLogsManager extends ManagerBase<AuditLogData, AuditLogResponse
           String fieldLabel = fieldLabelMap.get(componentName) != null ? fieldLabelMap.get(componentName).get(field): null;
           String fallbackedFieldLabel = labelFallbackHandler.fallbackIfRequired(field, fieldLabel,
             true, true, true,
-            uiMappingConfig.getAuditFieldFallbackPrefixes());
+            uiMappingConfig.getAuditFieldFallbackPrefixes(),
+            uiMappingConfig.getAuditFieldFallbackSuffixes());
 
           Boolean excludeFromFilter = fieldFilterableExcludeMap.get(componentName) != null && fieldFilterableExcludeMap.get(componentName).contains(field);
           Boolean visible = fieldVisibleeMap.get(componentName) != null && fieldVisibleeMap.get(componentName).contains(field);
@@ -196,7 +197,7 @@ public class AuditLogsManager extends ManagerBase<AuditLogData, AuditLogResponse
         Boolean excludeFromFilter = commonFieldFilterableExcludeList.contains(field);
         String fallbackedFieldLabel = labelFallbackHandler.fallbackIfRequired(field, fieldLabel,
           true, true, true,
-          uiMappingConfig.getAuditFieldFallbackPrefixes());
+          uiMappingConfig.getAuditFieldFallbackPrefixes(), uiMappingConfig.getAuditFieldFallbackSuffixes());
         defaults.add(new FieldMetadata(field, fallbackedFieldLabel, !excludeFromFilter, visible));
       }
     }

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java

@@ -432,7 +432,8 @@ public class ServiceLogsManager extends ManagerBase<ServiceLogData, ServiceLogRe
           labelFallbackHandler.fallbackIfRequired(
             e.getKey(), uiMappingConfig.getServiceFieldLabels().get(e.getKey()),
             true, false, true,
-            uiMappingConfig.getServiceFieldFallbackPrefixes()),
+            uiMappingConfig.getServiceFieldFallbackPrefixes(),
+            uiMappingConfig.getServiceFieldFallbackSuffixes()),
           !uiMappingConfig.getServiceFieldFilterableExcludesList().contains(e.getKey()),
           uiMappingConfig.getServiceFieldVisibleList().contains(e.getKey())))
       .collect(Collectors.toList());

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/response/ServiceLogData.java

@@ -65,4 +65,14 @@ public interface ServiceLogData extends CommonLogData, ComponentTypeLogData, Hos
   String getGroup();
 
   void setGroup(String group);
+
+  @JsonProperty("logger_name")
+  String getLoggerName();
+
+  void setLoggerName(String loggerName);
+
+  @JsonProperty("method")
+  String getMethod();
+
+  void setMethod(String method);
 }

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/solr/SolrConstants.java

@@ -60,6 +60,7 @@ public class SolrConstants {
     public static final String GROUP = "group";
     public static final String LEVEL = "level";
     public static final String THREAD_NAME = "thread_name";
+    public static final String METHOD = "method";
     public static final String LOGGER_NAME = "logger_name";
     public static final String LINE_NUMBER = "line_number";
     public static final String PATH = "path";
@@ -68,6 +69,10 @@ public class SolrConstants {
     public static final String KEY_DYNAMIC_FIELDS = "key_*";
     public static final String WS_DYNAMIC_FIELDS = "ws_*";
     public static final String SDI_DYNAMIC_FIELDS = "sdi_*";
+    public static final String INT_DYNAMIC_FIELDS = "*_i";
+    public static final String LONG_DYNAMIC_FIELDS = "*_l";
+    public static final String BOOLEAN_DYNAMIC_FIELDS = "*_b";
+    public static final String STRING_DYNAMIC_FIELDS = "*_s";
   }
 
   public class AuditLogConstants {

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/solr/model/SolrCommonLogData.java

@@ -26,6 +26,10 @@ import java.util.HashMap;
 import java.util.Map;
 
 import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.*;
+import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.INT_DYNAMIC_FIELDS;
+import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LONG_DYNAMIC_FIELDS;
+import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.BOOLEAN_DYNAMIC_FIELDS;
+import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.STRING_DYNAMIC_FIELDS;
 import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.KEY_DYNAMIC_FIELDS;
 import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.STORED_TOKEN_DYNAMIC_FIELDS;
 import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.WS_DYNAMIC_FIELDS;
@@ -92,6 +96,18 @@ public class SolrCommonLogData implements CommonLogData {
   @Field(WS_DYNAMIC_FIELDS)
   private Map<String, Object> wsDynamicFields;
 
+  @Field(INT_DYNAMIC_FIELDS)
+  private Map<String, Object> intDynamicFields;
+
+  @Field(LONG_DYNAMIC_FIELDS)
+  private Map<String, Object> longDynamicFields;
+
+  @Field(STRING_DYNAMIC_FIELDS)
+  private Map<String, Object> stringDynamicFields;
+
+  @Field(BOOLEAN_DYNAMIC_FIELDS)
+  private Map<String, Object> booleanDynamicFields;
+
   @Override
   public String getId() {
     return this.id;
@@ -265,6 +281,7 @@ public class SolrCommonLogData implements CommonLogData {
   @Override
   public Map<String, Object> getAllDynamicFields() {
     Map<String, Object> allDynamicFields = new HashMap<>();
+
     if (stdDynamicFields != null) {
       allDynamicFields.putAll(stdDynamicFields);
     }
@@ -274,6 +291,22 @@ public class SolrCommonLogData implements CommonLogData {
     if (wsDynamicFields != null) {
       allDynamicFields.putAll(wsDynamicFields);
     }
+
+    if (intDynamicFields != null) {
+      allDynamicFields.putAll(intDynamicFields);
+    }
+
+    if (longDynamicFields != null) {
+      allDynamicFields.putAll(longDynamicFields);
+    }
+
+    if (stringDynamicFields != null) {
+      allDynamicFields.putAll(stringDynamicFields);
+    }
+
+    if (booleanDynamicFields != null) {
+      allDynamicFields.putAll(booleanDynamicFields);
+    }
     
     return allDynamicFields;
   }
@@ -289,4 +322,20 @@ public class SolrCommonLogData implements CommonLogData {
   public void setWsDynamicFields(Map<String, Object> wsDynamicFields) {
     this.wsDynamicFields = wsDynamicFields;
   }
+
+  public void setIntDynamicFields(Map<String, Object> intDynamicFields) {
+    this.intDynamicFields = intDynamicFields;
+  }
+
+  public void setLongDynamicFields(Map<String, Object> longDynamicFields) {
+    this.longDynamicFields = longDynamicFields;
+  }
+
+  public void setStringDynamicFields(Map<String, Object> stringDynamicFields) {
+    this.stringDynamicFields = stringDynamicFields;
+  }
+
+  public void setBooleanDynamicFields(Map<String, Object> booleanDynamicFields) {
+    this.booleanDynamicFields = booleanDynamicFields;
+  }
 }

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/solr/model/SolrServiceLogData.java

@@ -52,6 +52,12 @@ public class SolrServiceLogData extends SolrCommonLogData implements ServiceLogD
   @Field(GROUP)
   private String group;
 
+  @Field(LOGGER_NAME)
+  private String loggerName;
+
+  @Field(METHOD)
+  private String method;
+
   @Field(SDI_DYNAMIC_FIELDS)
   private Map<String, Object> sdiDynamicFields;
 
@@ -135,6 +141,26 @@ public class SolrServiceLogData extends SolrCommonLogData implements ServiceLogD
     this.level = level;
   }
 
+  @Override
+  public String getLoggerName() {
+    return loggerName;
+  }
+
+  @Override
+  public void setLoggerName(String loggerName) {
+    this.loggerName = loggerName;
+  }
+
+  @Override
+  public String getMethod() {
+    return method;
+  }
+
+  @Override
+  public void setMethod(String method) {
+    this.method = method;
+  }
+
   public void setSdiDynamicFields(Map<String, Object> sdiDynamicFields) {
     this.sdiDynamicFields = sdiDynamicFields;
   }

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchJWTFilter.java

@@ -20,6 +20,7 @@ package org.apache.ambari.logsearch.web.filters;
 
 import org.apache.ambari.logsearch.auth.filter.AbstractJWTFilter;
 import org.apache.ambari.logsearch.conf.AuthPropsConfig;
+import org.apache.ambari.logsearch.dao.RoleDao;
 import org.springframework.security.core.GrantedAuthority;
 import org.springframework.security.web.util.matcher.NegatedRequestMatcher;
 import org.springframework.security.web.util.matcher.RequestMatcher;
@@ -30,10 +31,12 @@ import java.util.List;
 public class LogsearchJWTFilter extends AbstractJWTFilter {
 
   private AuthPropsConfig authPropsConfig;
+  private RoleDao roleDao;
 
-  public LogsearchJWTFilter(RequestMatcher requestMatcher, AuthPropsConfig authPropsConfig) {
+  public LogsearchJWTFilter(RequestMatcher requestMatcher, AuthPropsConfig authPropsConfig, RoleDao roleDao) {
     super(new NegatedRequestMatcher(requestMatcher));
     this.authPropsConfig = authPropsConfig;
+    this.roleDao = roleDao;
   }
 
   @Override
@@ -72,9 +75,11 @@ public class LogsearchJWTFilter extends AbstractJWTFilter {
   }
 
   @Override
-  protected Collection<? extends GrantedAuthority> getAuthorities() {
-    return null; // TODO
+  protected Collection<? extends GrantedAuthority> getAuthorities(String username) {
+    if (authPropsConfig.isFileAuthorization()) {
+      return roleDao.getRolesForUser(username);
+    }
+    return RoleDao.createDefaultAuthorities();
   }
 
-
 }

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchTrustedProxyFilter.java

@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.web.filters;
+
+import org.apache.ambari.logsearch.conf.AuthPropsConfig;
+import org.apache.ambari.logsearch.dao.RoleDao;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.security.authentication.AbstractAuthenticationToken;
+import org.springframework.security.authentication.AnonymousAuthenticationToken;
+import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.AuthenticationException;
+import org.springframework.security.core.GrantedAuthority;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.security.core.userdetails.User;
+import org.springframework.security.core.userdetails.UserDetails;
+import org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter;
+import org.springframework.security.web.authentication.WebAuthenticationDetails;
+import org.springframework.security.web.util.matcher.RequestMatcher;
+
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.util.Enumeration;
+import java.util.List;
+
+/**
+ * Filter servlet to handle trusted proxy authentication.
+ * It is disabled by default (see: {@link AuthPropsConfig#isTrustedProxy()}) <br/>
+ * There are 4 main configuration properties of this filter (allow authentication only if these are matches with the request details): <br/>
+ * - {@link AuthPropsConfig#getProxyUsers()} - Proxy users <br/>
+ * - {@link AuthPropsConfig#getProxyUserGroups()} - Proxy groups <br/>
+ * - {@link AuthPropsConfig#getProxyUserHosts()} - Proxy hosts <br/>
+ * - {@link AuthPropsConfig#getProxyIp()} - Proxy server IPs<br/>
+ */
+public class LogsearchTrustedProxyFilter extends AbstractAuthenticationProcessingFilter {
+
+  private static final Logger LOG = LoggerFactory.getLogger(LogsearchTrustedProxyFilter.class);
+
+  private static final String TRUSTED_PROXY_KNOX_HEADER = "X-Forwarded-For";
+
+  private AuthPropsConfig authPropsConfig;
+
+  public LogsearchTrustedProxyFilter(RequestMatcher requestMatcher, AuthPropsConfig authPropsConfig) {
+    super(requestMatcher);
+    this.authPropsConfig = authPropsConfig;
+  }
+
+  @Override
+  public Authentication attemptAuthentication(HttpServletRequest request, HttpServletResponse response) throws AuthenticationException, IOException, ServletException {
+    String doAsUserName = request.getParameter("doAs");
+    final List<GrantedAuthority> authorities = RoleDao.createDefaultAuthorities();
+    final UserDetails principal = new User(doAsUserName, "", authorities);
+    final Authentication finalAuthentication = new UsernamePasswordAuthenticationToken(principal, "", authorities);
+    WebAuthenticationDetails webDetails = new WebAuthenticationDetails(request);
+    ((AbstractAuthenticationToken) finalAuthentication).setDetails(webDetails);
+    SecurityContextHolder.getContext().setAuthentication(finalAuthentication);
+    LOG.info("Logged into Log Search User as doAsUser = {}", doAsUserName);
+    return finalAuthentication;
+  }
+
+  @Override
+  public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain) throws IOException, ServletException {
+    Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
+    boolean skip = true;
+    if (authPropsConfig.isTrustedProxy() && !isAuthenticated(authentication) ) {
+      String doAsUserName = req.getParameter("doAs");
+      String remoteAddr = req.getRemoteAddr();
+      if (StringUtils.isNotEmpty(doAsUserName) && isTrustedProxySever(remoteAddr)
+        && isTrustedHost(getXForwardHeader((HttpServletRequest) req))) {
+        List<GrantedAuthority> grantedAuths = RoleDao.createDefaultAuthorities();
+        if (!(isTrustedProxyUser(doAsUserName) || isTrustedProxyUserGroup(grantedAuths))) {
+          skip = false;
+        }
+      }
+    }
+    if (skip) {
+      chain.doFilter(req, res);
+      return;
+    }
+    super.doFilter(req, res, chain);
+  }
+
+  private boolean isTrustedProxySever(String requestHosts) {
+    if (authPropsConfig.getProxyIp() == null || requestHosts == null) {
+      return false;
+    }
+    final List<String> proxyServers = authPropsConfig.getProxyIp();
+    return (proxyServers.size() == 1 && proxyServers.contains("*")) || authPropsConfig.getProxyIp().contains(requestHosts);
+  }
+
+  private boolean isTrustedHost(String requestHosts) {
+    if (requestHosts == null) {
+      return false;
+    }
+    List<String> trustedProxyHosts = authPropsConfig.getProxyUserHosts();
+    return (trustedProxyHosts.size() == 1 && trustedProxyHosts.contains("*")) || trustedProxyHosts.contains(requestHosts);
+  }
+
+  private boolean isTrustedProxyUser(String doAsUser) {
+    if (doAsUser == null) {
+      return false;
+    }
+    List<String> trustedProxyUsers = authPropsConfig.getProxyUsers();
+    return (trustedProxyUsers.size() == 1 && trustedProxyUsers.contains("*")) || trustedProxyUsers.contains(doAsUser);
+
+  }
+
+  private boolean isTrustedProxyUserGroup(List<GrantedAuthority> proxyUserGroup) {
+    if (proxyUserGroup == null) {
+      return false;
+    }
+    List<String> trustedProxyGroups = authPropsConfig.getProxyUserGroups();
+    if (trustedProxyGroups.size() == 1 && trustedProxyGroups.contains("*")) {
+      return true;
+    } else {
+      for (GrantedAuthority group : proxyUserGroup) {
+        if (trustedProxyGroups.contains(group.getAuthority())) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
+  private boolean isAuthenticated(Authentication authentication) {
+    return authentication != null && !(authentication instanceof AnonymousAuthenticationToken) && authentication.isAuthenticated();
+  }
+
+  private String getXForwardHeader(HttpServletRequest httpRequest) {
+    Enumeration<String> names = httpRequest.getHeaderNames();
+    while (names.hasMoreElements()) {
+      String name = names.nextElement();
+      Enumeration<String> values = httpRequest.getHeaders(name);
+      String value = "";
+      if (values != null) {
+        while (values.hasMoreElements()) {
+          value = values.nextElement();
+          if (StringUtils.isNotBlank(value)) {
+            break;
+          }
+        }
+      }
+      if (StringUtils.trimToNull(name) != null
+        && StringUtils.trimToNull(value) != null) {
+        if (name.equalsIgnoreCase(TRUSTED_PROXY_KNOX_HEADER)) {
+          return value;
+        }
+      }
+    }
+    return "";
+  }
+}

ambari-logsearch/ambari-logsearch-server/src/main/resources/roles.json

@@ -0,0 +1,5 @@
+{
+  "roles": {
+    "admin" : ["user", "admin"]
+  }
+}

ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/common/LabelFallbackHandlerTest.java

@@ -43,7 +43,7 @@ public class LabelFallbackHandlerTest {
     // GIVEN
     String testInput = "my_field";
     // WHEN
-    String result = underTest.fallbackIfRequired(testInput, "spec label", true, false, true, null);
+    String result = underTest.fallbackIfRequired(testInput, "spec label", true, false, true, null, null);
     // THEN
     assertEquals("spec label", result);
   }
@@ -109,11 +109,27 @@ public class LabelFallbackHandlerTest {
   @Test
   public void testFallbackWithRemovingPrefixes() {
     // GIVEN
-    String testInput = "ws_request_id";
+    String testInput1 = "ws_request_id";
+    String testInput2 = "std_request_username";
     // WHEN
-    String result = underTest.fallback(testInput, true, true, true, Arrays.asList("ws_", "std_"));
+    String result1 = underTest.fallback(testInput1, true, true, true, Arrays.asList("ws_", "std_"), null);
+    String result2 = underTest.fallback(testInput2, true, true, true, Arrays.asList("ws_", "std_"), null);
     // THEN
-    assertEquals("Request Id", result);
+    assertEquals("Request Id", result1);
+    assertEquals("Request Username", result2);
+  }
+
+  @Test
+  public void testFallbackWithRemovingSuffixes() {
+    // GIVEN
+    String testInput1 = "request_id_i";
+    String testInput2 = "request_username_s";
+    // WHEN
+    String result1 = underTest.fallback(testInput1, true, true, true, null, Arrays.asList("_i", "_s"));
+    String result2 = underTest.fallback(testInput2, true, true, true, null, Arrays.asList("_i", "_s"));
+    // THEN
+    assertEquals("Request Id", result1);
+    assertEquals("Request Username", result2);
   }
 
   @Test
@@ -121,7 +137,7 @@ public class LabelFallbackHandlerTest {
     // GIVEN
     String testInput = "request_id";
     // WHEN
-    String result = underTest.fallback(testInput, true, true, true, Arrays.asList("ws_", "std_"));
+    String result = underTest.fallback(testInput, true, true, true, Arrays.asList("ws_", "std_"), null);
     // THEN
     assertEquals("Request Id", result);
   }

ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/dao/RoleDaoTest.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.dao;
+
+import org.apache.ambari.logsearch.conf.AuthPropsConfig;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.security.core.GrantedAuthority;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class RoleDaoTest {
+
+  private RoleDao underTest;
+
+  @Before
+  public void setUp() {
+    underTest = new RoleDao();
+    AuthPropsConfig authPropsConfig = new AuthPropsConfig();
+    authPropsConfig.setFileAuthorization(true);
+    underTest.setAuthPropsConfig(authPropsConfig);
+  }
+
+  @Test
+  public void testCreateDefaultAuthorities() {
+    // GIVEN
+    // WHEN
+    List<GrantedAuthority> authorityList = RoleDao.createDefaultAuthorities();
+    // THEN
+    Assert.assertEquals("ROLE_USER", authorityList.get(0).getAuthority());
+  }
+
+  @Test
+  public void testGetRolesForUser() {
+    // GIVEN
+    List<String> roles = Arrays.asList("admin", "user");
+    underTest.getSimpleRolesMap().put("user1", roles);
+    // WHEN
+    List<GrantedAuthority> result1 = underTest.getRolesForUser("user1");
+    List<GrantedAuthority> result2 = underTest.getRolesForUser("user2");
+    // THEN
+    Assert.assertEquals(result1.size(), 2);
+    Assert.assertEquals(result2.size(), 0);
+  }
+}

ambari-logsearch/ambari-logsearch-web/src/app/components/search-box/search-box.component.html

@@ -21,7 +21,7 @@
 <ng-container *ngFor="let parameter of parameters">
 <label class="parameter-label" [class.exclude]="parameter.isExclude" [class.include]="!parameter.isExclude">
   {{parameter.label | translate}}:
-  <span class="parameter-value">{{parameter.value}}</span>
+  <span class="parameter-value">{{(parameter.name === 'type' ? (parameter.value | componentLabel | async) : parameter.value)}}</span>
   <span class="fa toggle-parameter action-icon" [ngClass]="{'fa-search-minus': parameter.isExclude, 'fa-search-plus': !parameter.isExclude}"
         (click)="toggleParameter($event, parameter.id)" tooltip="{{('filter.toggleTo.' + (parameter.isExclude ? 'include' : 'exclude')) | translate}}"></span>
   <span class="fa fa-times remove-parameter action-icon" (click)="removeParameter($event, parameter.id)"></span>

ambari-logsearch/ambari-logsearch-web/src/app/components/search-box/search-box.component.spec.ts

@@ -18,10 +18,14 @@
 
 import {NO_ERRORS_SCHEMA} from '@angular/core';
 import {async, ComponentFixture, TestBed} from '@angular/core/testing';
+import {StoreModule} from '@ngrx/store';
+
 import {TranslationModules} from '@app/test-config.spec';
 import {UtilsService} from '@app/services/utils.service';
 
 import {SearchBoxComponent} from './search-box.component';
+import {ComponentsService, components} from '@app/services/storage/components.service';
+import {ComponentLabelPipe} from '@app/pipes/component-label';
 
 describe('SearchBoxComponent', () => {
   let component: SearchBoxComponent;
@@ -29,9 +33,18 @@ describe('SearchBoxComponent', () => {
 
   beforeEach(async(() => {
     TestBed.configureTestingModule({
-      declarations: [SearchBoxComponent],
-      imports: TranslationModules,
+      declarations: [
+        ComponentLabelPipe,
+        SearchBoxComponent
+      ],
+      imports: [
+        ...TranslationModules,
+        StoreModule.provideStore({
+          components
+        })
+      ],
       providers: [
+        ComponentsService,
         UtilsService
       ],
       schemas: [NO_ERRORS_SCHEMA]

ambari-logsearch/docker/Dockerfile

@@ -61,7 +61,7 @@ RUN cd /root && tar -zxvf /root/solr-$SOLR_VERSION.tgz
 # Install Knox
 WORKDIR /
 RUN adduser knox
-ENV KNOX_VERSION 1.0.0
+ENV KNOX_VERSION 1.1.0
 RUN wget -q -O /knox-${KNOX_VERSION}.zip http://download.nextag.com/apache/knox/${KNOX_VERSION}/knox-${KNOX_VERSION}.zip && unzip /knox-${KNOX_VERSION}.zip && rm knox-${KNOX_VERSION}.zip && ln -nsf knox-${KNOX_VERSION} knox && chmod +x /knox/bin/*.sh && chown -R knox /knox/
 
 ADD knox/keystores /knox-secrets

ambari-logsearch/docker/bin/start.sh

@@ -50,6 +50,7 @@ function create_logsearch_configs() {
   cp /root/test-config/logsearch/log4j.xml /root/config/logsearch/
   cp /root/test-config/logsearch/logsearch-env.sh /root/config/logsearch/
   cp $LOGSEARCH_SERVER_PATH/conf/user_pass.json /root/config/logsearch/user_pass.json
+  cp $LOGSEARCH_SERVER_PATH/conf/roles.json /root/config/logsearch/roles.json
   if [ "$LOGSEARCH_HTTPS_ENABLED" == "true" ]
   then
     cp /root/test-config/logsearch/logsearch-https.properties /root/config/logsearch/logsearch.properties

ambari-logsearch/docker/test-config/logsearch/logsearch-sso.properties

@@ -61,3 +61,6 @@ logsearch.auth.jwt.public_key=MIICOjCCAaOgAwIBAgIJAMY1lA6gY1V/MA0GCSqGSIb3DQEBBQ
 logsearch.auth.jwt.provider_url=https://localhost:8443/gateway/knoxsso/api/v1/websso
 logsearch.auth.jwt.cookie.name=hadoop-jwt
 logsearch.auth.jwt.query.param.original_url=originalUrl
+
+logsearch.auth.trusted.proxy=true
+logsearch.auth.proxyuser.users=*

ambari-logsearch/docker/test-config/logsearch/logsearch.properties

@@ -51,6 +51,8 @@ logsearch.solr.jmx.port=18886
 logsearch.auth.file.enable=true
 logsearch.login.credentials.file=user_pass.json
 
+logsearch.authr.file.enable=true
+
 logsearch.auth.ldap.enable=false
 logsearch.auth.simple.enable=false
 logsearch.auth.external_auth.enable=false

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java

@@ -101,9 +101,9 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
     }
 
     if (Boolean.valueOf(metricsConf.get(TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS, "false"))) {
-      this.timelineMetricReadHelper = new TimelineMetricReadHelper(metadataManager, true);
-    } else {
       this.timelineMetricReadHelper = new TimelineMetricReadHelper(metadataManager);
+    } else {
+      this.timelineMetricReadHelper = new TimelineMetricReadHelper(metadataManager, true);
     }
   }
 

ambari-server/src/main/java/org/apache/ambari/server/api/query/render/AlertSummaryGroupedRenderer.java

@@ -228,6 +228,18 @@ public class AlertSummaryGroupedRenderer extends AlertSummaryRenderer {
     }
   }
 
+  public static Map<String, AlertDefinitionSummary> generateEmptySummary(Long definitionId, String definitionName) {
+    Map<String, AlertDefinitionSummary> summaries = new HashMap<>();
+
+    AlertDefinitionSummary groupSummaryInfo = new AlertDefinitionSummary();
+    groupSummaryInfo.Id = definitionId;
+    groupSummaryInfo.Name = definitionName;
+
+    summaries.put(definitionName, groupSummaryInfo);
+
+    return summaries;
+  }
+
   /**
    * {@inheritDoc}
    * <p/>

ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java

@@ -1414,7 +1414,8 @@ public class AmbariMetaInfo {
       for (AlertDefinitionEntity definition : definitionsToDisable) {
         definition.setEnabled(false);
         alertDefinitionDao.merge(definition);
-        eventPublisher.publish(new AlertDefinitionDisabledEvent(clusterId, definition.getDefinitionId()));
+        eventPublisher.publish(new AlertDefinitionDisabledEvent(clusterId, definition.getDefinitionId(),
+            definition.getDefinitionName()));
       }
   }
 

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertDefinitionResourceProvider.java

@@ -328,7 +328,7 @@ public class AlertDefinitionResourceProvider extends AbstractControllerResourceP
         // a disabled event
         if (oldEnabled && !entity.getEnabled()) {
           AlertDefinitionDisabledEvent event = new AlertDefinitionDisabledEvent(
-              entity.getClusterId(), entity.getDefinitionId());
+              entity.getClusterId(), entity.getDefinitionId(), entity.getDefinitionName());
 
           eventPublisher.publish(event);
         }

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/WidgetResourceProvider.java

@@ -17,7 +17,10 @@
  */
 package org.apache.ambari.server.controller.internal;
 
+import static org.apache.ambari.server.security.authorization.RoleAuthorization.CLUSTER_MANAGE_WIDGETS;
+
 import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -40,17 +43,13 @@ import org.apache.ambari.server.controller.spi.SystemException;
 import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
 import org.apache.ambari.server.controller.utilities.PropertyHelper;
 import org.apache.ambari.server.orm.dao.WidgetDAO;
-import org.apache.ambari.server.orm.entities.PermissionEntity;
 import org.apache.ambari.server.orm.entities.WidgetEntity;
 import org.apache.ambari.server.orm.entities.WidgetLayoutUserWidgetEntity;
-import org.apache.ambari.server.security.authorization.AmbariGrantedAuthority;
 import org.apache.ambari.server.security.authorization.AuthorizationHelper;
+import org.apache.ambari.server.security.authorization.ResourceType;
 import org.apache.commons.lang.ObjectUtils;
 import org.apache.commons.lang.StringUtils;
 import org.springframework.security.access.AccessDeniedException;
-import org.springframework.security.core.GrantedAuthority;
-import org.springframework.security.core.context.SecurityContext;
-import org.springframework.security.core.context.SecurityContextHolder;
 
 import com.google.gson.Gson;
 import com.google.inject.Inject;
@@ -157,7 +156,7 @@ public class WidgetResourceProvider extends AbstractControllerResourceProvider {
           String clusterName = properties.get(WIDGET_CLUSTER_NAME_PROPERTY_ID).toString();
           String scope = properties.get(WIDGET_SCOPE_PROPERTY_ID).toString();
 
-          if (!isScopeAllowedForUser(scope)) {
+          if (!isScopeAllowedForUser(scope, clusterName)) {
             throw new AccessDeniedException("Only cluster operator can create widgets with cluster scope");
           }
 
@@ -310,7 +309,8 @@ public class WidgetResourceProvider extends AbstractControllerResourceProvider {
 
           if (StringUtils.isNotBlank(ObjectUtils.toString(propertyMap.get(WIDGET_SCOPE_PROPERTY_ID)))) {
             String scope = propertyMap.get(WIDGET_SCOPE_PROPERTY_ID).toString();
-            if (!isScopeAllowedForUser(scope)) {
+            String clusterName = propertyMap.get(WIDGET_CLUSTER_NAME_PROPERTY_ID).toString();
+            if (!isScopeAllowedForUser(scope, clusterName)) {
               throw new AmbariException("Only cluster operator can create widgets with cluster scope");
             }
             entity.setScope(scope);
@@ -383,28 +383,11 @@ public class WidgetResourceProvider extends AbstractControllerResourceProvider {
     return pkPropertyIds;
   }
 
-  private boolean isScopeAllowedForUser(String scope) {
-    if (scope.equals(WidgetEntity.USER_SCOPE)) {
-      return true;
-    }
-
-    // Only cluster operators are allowed to create widgets with cluster scope
-    SecurityContext securityContext = SecurityContextHolder.getContext();
-    securityContext.getAuthentication().getAuthorities();
-    boolean hasPermissionForClusterScope = false;
-    for (GrantedAuthority grantedAuthority : securityContext.getAuthentication().getAuthorities()) {
-      if (((AmbariGrantedAuthority) grantedAuthority).getPrivilegeEntity().getPermission().getId()
-              == PermissionEntity.AMBARI_ADMINISTRATOR_PERMISSION ||
-              ((AmbariGrantedAuthority) grantedAuthority).getPrivilegeEntity().getPermission().getId()
-                      == PermissionEntity.CLUSTER_ADMINISTRATOR_PERMISSION) {
-        hasPermissionForClusterScope = true;
-      }
-    }
-    if (hasPermissionForClusterScope) {
+  private boolean isScopeAllowedForUser(String scope, String clusterName) throws AmbariException {
+    if (WidgetEntity.USER_SCOPE.equals(scope)) {
       return true;
-    } else {
-      return false;
     }
+    return AuthorizationHelper.isAuthorized(ResourceType.CLUSTER, getClusterResourceId(clusterName), EnumSet.of(CLUSTER_MANAGE_WIDGETS));
   }
 
   private String getAuthorName(Map<String, Object> properties) {

ambari-server/src/main/java/org/apache/ambari/server/events/AlertDefinitionDisabledEvent.java

@@ -30,6 +30,11 @@ public class AlertDefinitionDisabledEvent extends ClusterEvent {
    */
   private final long m_definitionId;
 
+  /**
+   * The alert definition name.
+   */
+  private final String definitionName;
+
   /**
    * Constructor.
    *
@@ -38,9 +43,10 @@ public class AlertDefinitionDisabledEvent extends ClusterEvent {
    * @param definitionId
    *          the alert definition being registered.
    */
-  public AlertDefinitionDisabledEvent(long clusterId, long definitionId) {
+  public AlertDefinitionDisabledEvent(long clusterId, long definitionId, String definitionName) {
     super(AmbariEventType.ALERT_DEFINITION_DISABLED, clusterId);
     m_definitionId = definitionId;
+    this.definitionName = definitionName;
   }
 
   /**
@@ -51,4 +57,13 @@ public class AlertDefinitionDisabledEvent extends ClusterEvent {
   public long getDefinitionId() {
     return m_definitionId;
   }
+
+  /**
+   * Gets the definition name.
+   *
+   * @return the definitionId name
+   */
+  public String getDefinitionName() {
+    return definitionName;
+  }
 }

ambari-server/src/main/java/org/apache/ambari/server/events/listeners/alerts/AlertDefinitionDisabledListener.java

@@ -17,9 +17,15 @@
  */
 package org.apache.ambari.server.events.listeners.alerts;
 
+import java.util.HashMap;
+import java.util.Map;
+
 import org.apache.ambari.server.EagerSingleton;
+import org.apache.ambari.server.api.query.render.AlertSummaryGroupedRenderer;
 import org.apache.ambari.server.events.AlertDefinitionDisabledEvent;
+import org.apache.ambari.server.events.AlertUpdateEvent;
 import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
+import org.apache.ambari.server.events.publishers.STOMPUpdatePublisher;
 import org.apache.ambari.server.orm.dao.AlertsDAO;
 import org.apache.ambari.server.orm.entities.AlertCurrentEntity;
 import org.slf4j.Logger;
@@ -46,6 +52,9 @@ public class AlertDefinitionDisabledListener {
   @Inject
   private AlertsDAO m_alertsDao = null;
 
+  @Inject
+  private STOMPUpdatePublisher STOMPUpdatePublisher;
+
   /**
    * Constructor.
    *
@@ -67,5 +76,11 @@ public class AlertDefinitionDisabledListener {
     LOG.debug("Received event {}", event);
 
     m_alertsDao.removeCurrentDisabledAlerts();
+
+    // send API STOMP alert update
+    Map<Long, Map<String, AlertSummaryGroupedRenderer.AlertDefinitionSummary>> alertUpdates = new HashMap<>();
+    alertUpdates.put(event.getClusterId(), AlertSummaryGroupedRenderer.generateEmptySummary(event.getDefinitionId(),
+        event.getDefinitionName()));
+    STOMPUpdatePublisher.publish(new AlertUpdateEvent(alertUpdates));
   }
 }

ambari-server/src/main/java/org/apache/ambari/server/events/listeners/alerts/AlertReceivedListener.java

@@ -254,6 +254,14 @@ public class AlertReceivedListener {
 
           // create the event to fire later
           alertEvents.add(new InitialAlertEvent(clusterId, alert, current));
+
+          if (!alertUpdates.containsKey(clusterId)) {
+            alertUpdates.put(clusterId, new HashMap<>());
+          }
+          Map<String, AlertSummaryGroupedRenderer.AlertDefinitionSummary> summaries = alertUpdates.get(clusterId);
+
+          AlertSummaryGroupedRenderer.updateSummary(summaries, definition.getDefinitionId(),
+              definition.getDefinitionName(), alertState, alert.getTimestamp(), maintenanceState, alert.getText());
         } finally {
           // release the lock for this alert
           lock.unlock();

ambari-server/src/main/java/org/apache/ambari/server/security/authorization/RoleAuthorization.java

@@ -57,6 +57,7 @@ public enum RoleAuthorization {
   CLUSTER_RUN_CUSTOM_COMMAND("CLUSTER.RUN_CUSTOM_COMMAND"),
   CLUSTER_MANAGE_AUTO_START("CLUSTER.MANAGE_AUTO_START"),
   CLUSTER_MANAGE_ALERT_NOTIFICATIONS("CLUSTER.MANAGE_ALERT_NOTIFICATIONS"),
+  CLUSTER_MANAGE_WIDGETS("CLUSTER.MANAGE_WIDGETS"),
   HOST_ADD_DELETE_COMPONENTS("HOST.ADD_DELETE_COMPONENTS"),
   HOST_ADD_DELETE_HOSTS("HOST.ADD_DELETE_HOSTS"),
   HOST_TOGGLE_MAINTENANCE("HOST.TOGGLE_MAINTENANCE"),

ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql

@@ -1489,6 +1489,7 @@ INSERT INTO roleauthorization(authorization_id, authorization_name)
   SELECT 'CLUSTER.RUN_CUSTOM_COMMAND', 'Perform custom cluster-level actions' FROM SYSIBM.SYSDUMMY1 UNION ALL
   SELECT 'CLUSTER.MANAGE_AUTO_START', 'Manage service auto-start configuration' FROM SYSIBM.SYSDUMMY1 UNION ALL
   SELECT 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS', 'Manage alert notifications configuration' FROM SYSIBM.SYSDUMMY1 UNION ALL
+  SELECT 'CLUSTER.MANAGE_WIDGETS', 'Manage widgets' FROM SYSIBM.SYSDUMMY1 UNION ALL
   SELECT 'AMBARI.ADD_DELETE_CLUSTERS', 'Create new clusters' FROM SYSIBM.SYSDUMMY1 UNION ALL
   SELECT 'AMBARI.RENAME_CLUSTER', 'Rename clusters' FROM SYSIBM.SYSDUMMY1 UNION ALL
   SELECT 'AMBARI.MANAGE_SETTINGS', 'Manage settings' FROM SYSIBM.SYSDUMMY1 UNION ALL
@@ -1604,6 +1605,7 @@ INSERT INTO permission_roleauthorization(permission_id, authorization_id)
   SELECT permission_id, 'CLUSTER.VIEW_ALERTS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR'  UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_CREDENTIALS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR'  UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR'  UNION ALL
+  SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR';
 
 -- Set authorizations for Cluster Administrator role
@@ -1647,6 +1649,7 @@ INSERT INTO permission_roleauthorization(permission_id, authorization_id)
   SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
+  SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.RUN_CUSTOM_COMMAND' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR';
 
 -- Set authorizations for Administrator role
@@ -1692,6 +1695,7 @@ INSERT INTO permission_roleauthorization(permission_id, authorization_id)
   SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR'  UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR'  UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
+  SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'AMBARI.ADD_DELETE_CLUSTERS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR'  UNION ALL
   SELECT permission_id, 'AMBARI.RENAME_CLUSTER' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR'  UNION ALL
   SELECT permission_id, 'AMBARI.MANAGE_SETTINGS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR'  UNION ALL

ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql

@@ -1420,6 +1420,7 @@ INSERT INTO roleauthorization(authorization_id, authorization_name)
   SELECT 'CLUSTER.RUN_CUSTOM_COMMAND', 'Perform custom cluster-level actions' UNION ALL
   SELECT 'CLUSTER.MANAGE_AUTO_START', 'Manage service auto-start configuration' UNION ALL
   SELECT 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS', 'Manage alert notifications configuration' UNION ALL
+  SELECT 'CLUSTER.MANAGE_WIDGETS', 'Manage widgets' UNION ALL
   SELECT 'AMBARI.ADD_DELETE_CLUSTERS', 'Create new clusters' UNION ALL
   SELECT 'AMBARI.RENAME_CLUSTER', 'Rename clusters' UNION ALL
   SELECT 'AMBARI.MANAGE_SETTINGS', 'Manage administrative settings' UNION ALL
@@ -1535,6 +1536,7 @@ INSERT INTO permission_roleauthorization(permission_id, authorization_id)
   SELECT permission_id, 'CLUSTER.VIEW_ALERTS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_CREDENTIALS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
+  SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR';
 
 -- Set authorizations for Cluster Administrator role
@@ -1580,6 +1582,7 @@ INSERT INTO permission_roleauthorization(permission_id, authorization_id)
   SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
+  SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.RUN_CUSTOM_COMMAND' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR';
 
 -- Set authorizations for Administrator role
@@ -1626,6 +1629,7 @@ INSERT INTO permission_roleauthorization(permission_id, authorization_id)
   SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
+  SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.RUN_CUSTOM_COMMAND' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'AMBARI.ADD_DELETE_CLUSTERS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'AMBARI.RENAME_CLUSTER' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL

ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql

@@ -1417,6 +1417,7 @@ INSERT INTO roleauthorization(authorization_id, authorization_name)
   SELECT 'CLUSTER.RUN_CUSTOM_COMMAND', 'Perform custom cluster-level actions' FROM dual UNION ALL
   SELECT 'CLUSTER.MANAGE_AUTO_START', 'Manage service auto-start configuration' FROM dual UNION ALL
   SELECT 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS', 'Manage alert notifications configuration' FROM dual UNION ALL
+  SELECT 'CLUSTER.MANAGE_WIDGETS', 'Manage widgets' FROM dual UNION ALL
   SELECT 'AMBARI.ADD_DELETE_CLUSTERS', 'Create new clusters' FROM dual UNION ALL
   SELECT 'AMBARI.RENAME_CLUSTER', 'Rename clusters' FROM dual UNION ALL
   SELECT 'AMBARI.MANAGE_SETTINGS', 'Manage settings' FROM dual UNION ALL
@@ -1532,6 +1533,7 @@ INSERT INTO permission_roleauthorization(permission_id, authorization_id)
   SELECT permission_id, 'CLUSTER.VIEW_ALERTS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_CREDENTIALS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
+  SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR';
 
 -- Set authorizations for Cluster Administrator role
@@ -1577,6 +1579,7 @@ INSERT INTO permission_roleauthorization(permission_id, authorization_id)
   SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
+  SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.RUN_CUSTOM_COMMAND' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR';
 
 -- Set authorizations for Administrator role
@@ -1623,6 +1626,7 @@ INSERT INTO permission_roleauthorization(permission_id, authorization_id)
   SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
+  SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.RUN_CUSTOM_COMMAND' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'AMBARI.ADD_DELETE_CLUSTERS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'AMBARI.RENAME_CLUSTER' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL

ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql

@@ -1405,6 +1405,7 @@ INSERT INTO roleauthorization(authorization_id, authorization_name)
   SELECT 'CLUSTER.RUN_CUSTOM_COMMAND', 'Perform custom cluster-level actions' UNION ALL
   SELECT 'CLUSTER.MANAGE_AUTO_START', 'Manage service auto-start configuration' UNION ALL
   SELECT 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS', 'Manage alert notifications configuration' UNION ALL
+  SELECT 'CLUSTER.MANAGE_WIDGETS', 'Manage widgets' UNION ALL
   SELECT 'AMBARI.ADD_DELETE_CLUSTERS', 'Create new clusters' UNION ALL
   SELECT 'AMBARI.RENAME_CLUSTER', 'Rename clusters' UNION ALL
   SELECT 'AMBARI.MANAGE_SETTINGS', 'Manage administrative settings' UNION ALL
@@ -1520,6 +1521,7 @@ INSERT INTO permission_roleauthorization(permission_id, authorization_id)
   SELECT permission_id, 'CLUSTER.VIEW_ALERTS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_CREDENTIALS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
+  SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR';
 
 -- Set authorizations for Cluster Administrator role
@@ -1565,6 +1567,7 @@ INSERT INTO permission_roleauthorization(permission_id, authorization_id)
   SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
+  SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.RUN_CUSTOM_COMMAND' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR';
 
 -- Set authorizations for Administrator role
@@ -1611,6 +1614,7 @@ INSERT INTO permission_roleauthorization(permission_id, authorization_id)
   SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
+  SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'CLUSTER.RUN_CUSTOM_COMMAND' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'AMBARI.ADD_DELETE_CLUSTERS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
   SELECT permission_id, 'AMBARI.RENAME_CLUSTER' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL

ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql

@@ -1418,6 +1418,7 @@ insert into adminpermission(permission_id, permission_name, resource_type_id, pe
     SELECT 'CLUSTER.RUN_CUSTOM_COMMAND', 'Perform custom cluster-level actions' UNION ALL
     SELECT 'CLUSTER.MANAGE_AUTO_START', 'Manage service auto-start configuration' UNION ALL
     SELECT 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS', 'Manage alert notifications configuration' UNION ALL
+    SELECT 'CLUSTER.MANAGE_WIDGETS', 'Manage widgets' UNION ALL
     SELECT 'AMBARI.ADD_DELETE_CLUSTERS', 'Create new clusters' UNION ALL
     SELECT 'AMBARI.RENAME_CLUSTER', 'Rename clusters' UNION ALL
     SELECT 'AMBARI.MANAGE_SETTINGS', 'Manage settings' UNION ALL
@@ -1533,6 +1534,7 @@ insert into adminpermission(permission_id, permission_name, resource_type_id, pe
     SELECT permission_id, 'CLUSTER.VIEW_ALERTS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.MANAGE_CREDENTIALS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
+    SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR';
 
   -- Set authorizations for Cluster Administrator role
@@ -1578,6 +1580,7 @@ insert into adminpermission(permission_id, permission_name, resource_type_id, pe
     SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
+    SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.RUN_CUSTOM_COMMAND' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR';
 
   -- Set authorizations for Administrator role
@@ -1624,6 +1627,7 @@ insert into adminpermission(permission_id, permission_name, resource_type_id, pe
     SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
+    SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.RUN_SERVICE_CHECK' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
     SELECT permission_id, 'AMBARI.ADD_DELETE_CLUSTERS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
     SELECT permission_id, 'AMBARI.RENAME_CLUSTER' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL

ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql

@@ -1432,6 +1432,7 @@ BEGIN TRANSACTION
     SELECT 'CLUSTER.RUN_CUSTOM_COMMAND', 'Perform custom cluster-level actions' UNION ALL
     SELECT 'CLUSTER.MANAGE_AUTO_START', 'Manage service auto-start configuration' UNION ALL
     SELECT 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS', 'Manage alert notifications configuration' UNION ALL
+    SELECT 'CLUSTER.MANAGE_WIDGETS', 'Manage widgets' UNION ALL
     SELECT 'AMBARI.ADD_DELETE_CLUSTERS', 'Create new clusters' UNION ALL
     SELECT 'AMBARI.RENAME_CLUSTER', 'Rename clusters' UNION ALL
     SELECT 'AMBARI.MANAGE_SETTINGS', 'Manage settings' UNION ALL
@@ -1547,6 +1548,7 @@ BEGIN TRANSACTION
     SELECT permission_id, 'CLUSTER.VIEW_ALERTS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.MANAGE_CREDENTIALS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
+    SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='CLUSTER.OPERATOR';
 
   -- Set authorizations for Cluster Administrator role
@@ -1592,6 +1594,7 @@ BEGIN TRANSACTION
     SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
+    SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.RUN_CUSTOM_COMMAND' FROM adminpermission WHERE permission_name='CLUSTER.ADMINISTRATOR';
 
   -- Set authorizations for Administrator role
@@ -1638,6 +1641,7 @@ BEGIN TRANSACTION
     SELECT permission_id, 'CLUSTER.MANAGE_USER_PERSISTED_DATA' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.MANAGE_AUTO_START' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.MANAGE_ALERT_NOTIFICATIONS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
+    SELECT permission_id, 'CLUSTER.MANAGE_WIDGETS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
     SELECT permission_id, 'CLUSTER.RUN_CUSTOM_COMMAND' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
     SELECT permission_id, 'AMBARI.ADD_DELETE_CLUSTERS' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL
     SELECT permission_id, 'AMBARI.RENAME_CLUSTER' FROM adminpermission WHERE permission_name='AMBARI.ADMINISTRATOR' UNION ALL

ambari-server/src/main/resources/stacks/PERF/check_host.sed

@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+/    Logger.info("Check execute list: " + str(check_execute_list))/{i\
+    if CHECK_INSTALLED_PACKAGES in check_execute_list: check_execute_list.remove(CHECK_INSTALLED_PACKAGES)\
+    if CHECK_INSTALLED_PACKAGES in check_execute_list: check_execute_list.remove(CHECK_EXISTING_REPOS)\
+    Logger.info("Check execute list: " + str(check_execute_list))
+d
+}

ambari-server/src/main/resources/stacks/stack_advisor.py

@@ -2989,6 +2989,10 @@ class DefaultStackAdvisor(StackAdvisor):
       return None
 
     dir = re.sub("^file://", "", dir, count=1)
+
+    if not dir:
+      return self.getErrorItem("Value has wrong format")
+
     mountPoints = {}
     for mountPoint in hostInfo["disk_info"]:
       mountPoints[mountPoint["mountpoint"]] = self.to_number(mountPoint["available"])
@@ -2997,6 +3001,9 @@ class DefaultStackAdvisor(StackAdvisor):
     if not mountPoints:
       return self.getErrorItem("No disk info found on host %s" % hostInfo["host_name"])
 
+    if mountPoint is None:
+      return self.getErrorItem("No mount point in directory %s. Mount points: %s" % (dir, ', '.join(mountPoints.keys())))
+
     if mountPoints[mountPoint] < reqiuredDiskSpace:
       msg = "Ambari Metrics disk space requirements not met. \n" \
             "Recommended disk space for partition {0} is {1}G"

ambari-server/src/test/java/org/apache/ambari/server/controller/internal/WidgetResourceProviderTest.java

@@ -435,7 +435,7 @@ public class WidgetResourceProviderTest {
     Cluster cluster = createMock(Cluster.class);
     expect(amc.getClusters()).andReturn(clusters).atLeastOnce();
     expect(clusters.getCluster((String) anyObject())).andReturn(cluster).atLeastOnce();
-    expect(cluster.getClusterId()).andReturn(Long.valueOf(1)).atLeastOnce();
+    expect(cluster.getResourceId()).andReturn(Long.valueOf(1)).atLeastOnce();
 
     Capture<WidgetEntity> entityCapture = EasyMock.newCapture();
     dao.create(capture(entityCapture));

ambari-web/app/assets/test/tests.js

@@ -96,6 +96,7 @@ var files = [
   'test/controllers/main/admin/highAvailability/journalNode/progress_controller_test',
   'test/controllers/main/admin/highAvailability/journalNode/step1_controller_test',
   'test/controllers/main/admin/highAvailability/journalNode/step2_controller_test',
+  'test/controllers/main/admin/highAvailability/journalNode/step3_controller_test',
   'test/controllers/main/admin/highAvailability/journalNode/step4_controller_test',
   'test/controllers/main/admin/highAvailability/journalNode/step6_controller_test',
   'test/controllers/main/admin/highAvailability/journalNode/step7_controller_test',

ambari-web/app/controllers/installer.js

@@ -756,7 +756,7 @@ App.InstallerController = App.WizardController.extend(App.Persist, {
       existedOS.isSelected = true;
       existedMap[existedOS.OperatingSystems.os_type] = existedOS;
     });
-    if (response.Versions['stack-errors'] && response.Versions['stack-errors'].length) {
+    if (response.Versions && response.Versions['stack-errors'] && response.Versions['stack-errors'].length) {
       this.showStackErrorAndSkipStepIfNeeded(response);
       return;
     }

ambari-web/app/controllers/main/host/details.js

@@ -1576,7 +1576,8 @@ App.MainHostDetailsController = Em.Controller.extend(App.SupportClientConfigsDow
   },
 
   getZookeeperConnectionString: function () {
-    return this.getRangerKMSServerHosts().map(function (host) {
+    var zookeeperHosts = App.MasterComponent.find('ZOOKEEPER_SERVER').get('hostNames');
+    return zookeeperHosts.map(function (host) {
       return host + ':2181';
     }).join(',');
   },
@@ -2286,7 +2287,8 @@ App.MainHostDetailsController = Em.Controller.extend(App.SupportClientConfigsDow
     var message = Em.I18n.t('hosts.host.details.for.postfix').format(context.label);
     var popupInfo = Em.I18n.t('hosts.passiveMode.popup').format(context.active ? 'On' : 'Off', this.get('content.hostName'));
     if (state === 'OFF') {
-      var hostVersion = this.get('content.stackVersions') && this.get('content.stackVersions').findProperty('isCurrent').get('repoVersion'),
+      var currentHostVersion = this.get('content.stackVersions') && this.get('content.stackVersions').findProperty('isCurrent'),
+        hostVersion = currentHostVersion && currentHostVersion.get('repoVersion'),
         currentVersion = App.StackVersion.find().findProperty('isCurrent'),
         clusterVersion = currentVersion && currentVersion.get('repositoryVersion.repositoryVersion');
       if (hostVersion !== clusterVersion) {

ambari-web/app/controllers/main/service/item.js

@@ -590,7 +590,7 @@ App.MainServiceItemController = Em.Controller.extend(App.SupportClientConfigsDow
           sender: this,
           data: {
             context,
-            serviceName: serviceName.toUpperCase(),
+            serviceName: serviceName,
             serviceGroupName: serviceGroupName,
             state: serviceHealth,
             query: requestQuery

ambari-web/app/messages.js

@@ -29,7 +29,7 @@ Em.I18n.translations = {
   'app.redirectIssuePopup.header': 'Login Redirect Issue',
   'app.redirectIssuePopup.body': 'For single sign-on, make sure that Knox Gateway and Ambari Server are located on the same host or subdomain.' +
     '<br/>Alternatively login as an Ambari local user using the local login page.<br />' +
-    '<a href="{0}" target="_blank">{0}</a>',
+    '<a rel="noopener noreferrer" href="{0}" target="_blank">{0}</a>',
 
   'app.versionMismatchAlert.title': 'Ambari Server / Web Client Version Mismatch',
   'app.versionMismatchAlert.body': 'Ambari Server and Web Client versions do not match:<br> ' +
@@ -2483,7 +2483,7 @@ Em.I18n.translations = {
   'services.service.config.configHistory.makeCurrent.message': 'Created from service config version {0}',
   'services.service.config.configHistory.comparing': 'Comparing Changes in',
   'services.service.config.setRecommendedValue': 'Set Recommended',
-  'services.service.config.database.msg.jdbcSetup.detailed': 'To use {0} with {6}, you must <a href="{3}" target="_blank">' +
+  'services.service.config.database.msg.jdbcSetup.detailed': 'To use {0} with {6}, you must <a rel="noopener noreferrer" href="{3}" target="_blank">' +
     'download the {4} from {0}</a>. Once downloaded to the Ambari Server host, run: <br/>' +
     '<b>ambari-server setup --jdbc-db={1} --jdbc-driver=/path/to/{1}/{2}</b>',
 

ambari-web/app/mixins/common/configs/enhanced_configs.js

@@ -669,22 +669,25 @@ App.EnhancedConfigsMixin = Em.Mixin.create(App.ConfigWithOverrideRecommendationP
 
   updateAttributesFromTheme: function (serviceName) {
     this.prepareSectionsConfigProperties(serviceName);
-    const serviceConfigs = this.get('stepConfigs').findProperty('serviceName', serviceName).get('configs'),
-      configConditions = App.ThemeCondition.find().filter(condition => {
-        const dependentConfigName = condition.get('configName'),
-          dependentConfigFileName = condition.get('fileName'),
-          configsToDependOn = condition.getWithDefault('configs', []);
-        return serviceConfigs.some(serviceConfig => {
-          const serviceConfigName = Em.get(serviceConfig, 'name'),
-            serviceConfigFileName = Em.get(serviceConfig, 'filename');
-          return (serviceConfigName === dependentConfigName && serviceConfigFileName === dependentConfigFileName)
-            || configsToDependOn.some(config => {
-              const {configName, fileName} = config;
-              return serviceConfigName === configName && serviceConfigFileName === fileName;
-            });
+    const service = this.get('stepConfigs').findProperty('serviceName', serviceName);
+    if (service) {
+      const serviceConfigs = service.get('configs'),
+        configConditions = App.ThemeCondition.find().filter(condition => {
+          const dependentConfigName = condition.get('configName'),
+            dependentConfigFileName = condition.get('fileName'),
+            configsToDependOn = condition.getWithDefault('configs', []);
+          return serviceConfigs.some(serviceConfig => {
+            const serviceConfigName = Em.get(serviceConfig, 'name'),
+              serviceConfigFileName = Em.get(serviceConfig, 'filename');
+            return (serviceConfigName === dependentConfigName && serviceConfigFileName === dependentConfigFileName)
+              || configsToDependOn.some(config => {
+                const {configName, fileName} = config;
+                return serviceConfigName === configName && serviceConfigFileName === fileName;
+              });
+          });
         });
-      });
-    this.updateAttributesFromConditions(configConditions, serviceConfigs, serviceName);
+      this.updateAttributesFromConditions(configConditions, serviceConfigs, serviceName);
+    }
   },
 
   prepareSectionsConfigProperties: function (serviceName) {

ambari-web/app/mixins/common/widgets/time_range_mixin.js

@@ -79,7 +79,9 @@ App.TimeRangeMixin = Em.Mixin.create({
 
   didInsertElement: function () {
     App.tooltip(this.$(), {
-      selector: '.dropdown-toggle[rel="tooltip"]'
+      selector: '.dropdown-toggle[rel="tooltip"]',
+      html: true,
+      placement: 'left'
     });
   },
 

ambari-web/app/models/host.js

@@ -216,7 +216,7 @@ App.Host = DS.Model.extend({
       'ALERT': 'health-status-DEAD-ORANGE'
     };
     return statusMap[this.get('healthStatus')] || 'health-status-DEAD-YELLOW';
-  }.property('healthStatus'),
+  }.property('healthStatus', 'passiveState'),
 
   healthIconClass: Em.computed.getByKey('healthIconClassMap', 'healthClass', ''),
 

ambari-web/app/models/host_component.js

@@ -119,7 +119,17 @@ App.HostComponent = DS.Model.extend({
    * User friendly host component status
    * @returns {String}
    */
-  isActive: Em.computed.equal('passiveState', 'OFF'),
+  isActive: function() {
+    let passiveState = this.get('passiveState');
+    if (passiveState === 'IMPLIED_FROM_HOST') {
+      passiveState = this.get('host.passiveState');
+    } else if (passiveState === 'IMPLIED_FROM_SERVICE') {
+      passiveState = this.get('service.passiveState');
+    } else if (passiveState === 'IMPLIED_FROM_SERVICE_AND_HOST') {
+      return this.get('service.passiveState') === 'OFF' && this.get('host.passiveState') === 'OFF';
+    }
+    return passiveState === 'OFF';
+  }.property('passiveState', 'host.passiveState', 'service.passiveState'),
 
   /**
    * Determine if passiveState is implied from host or/and service

ambari-web/app/styles/common.less

@@ -239,5 +239,22 @@
 @navbar-header-padding-left: 0;
 @navbar-header-font-size: 20px;
 
+/************************************************************************
+* Table styles
+***********************************************************************/
 @table-color: @gray-text;
 @table-font-size: 13px;
+@table-hover-background-color: #E7F6FC;
+@table-hover-border-color: #A7DFF2;
+@table-margin-bottom: 20px;
+@table-footer-color: #999;
+@table-footer-border: 2px solid #EEE;
+@table-cell-padding: 8px;
+
+/************************************************************************
+* Checkbox styles
+***********************************************************************/
+@checkbox-color: #1491C1;
+@checkbox-size: 10px;
+@checkbox-border-radius: 2px;
+@checkbox-top: 4px;

ambari-web/app/styles/service_configurations.less

@@ -44,6 +44,9 @@
   .comparison-row {
     margin-left: -25px;
     padding: 10px 0;
+    .compare-config-cell {
+      white-space: pre;
+    }
   }
 
   .undefined {

ambari-web/app/styles/wizard.less

@@ -19,6 +19,7 @@
 @import 'common.less';
 
 @wizard-side-nav-width: 250px;
+@host-column-width: 210px;
 
 #enable_security {
   .step3 {
@@ -336,64 +337,106 @@
   }
   #step6 {
     .pre-scrollable {
-      max-height: 440px;
+      width: 100%;
+      overflow: auto;
       position: relative;
-      display: flex;
-      overflow-y: auto;
+      max-height: 440px;
+      border-bottom: @table-footer-border;
       #component_assign_table {
-        display: inline-block;
-        overflow-x: auto;
-        margin-left: 190px;
-        margin-bottom: 10px;
-        height: 100%;
-        tbody td, th {
-          white-space: nowrap;
-        }
-        tfoot td {
-          padding-bottom: 30px;
-        }
-        .trim_hostname {
-          .ellipsis-overflow-nowrap;
-          max-width: 180px;
-          float:left;
+        margin-bottom: 0;
+        tr {
           display: block;
-        }
-        .checkbox {
-          margin: 0;
-        }
-        .host-component-checkbox {
-          line-height: 17px;
-          font-size: 12px;
-          top: 0;
-        }
-        .freeze {
-          position: absolute;
-          left: 0;
-          width: 210px;
-          border-top-color: #eee;
-          border-bottom: 1px solid #eee;
-        }
-        th.freeze {
-          //Added this because due to its absolute positioning, 
-          //the frozen column header was overlapping and blocking
-          //the next column header from being clicked, and that one
-          //contains the "all | none" links for checkbox selections.
-          pointer-events: none;
-        }
-        td.freeze {
-          padding-bottom: 11px;
-        }
-        tr.last-row {
-          td {
-            border-bottom: 2px solid #eee;
+          padding-left: @host-column-width;
+          td, th {
+            white-space: nowrap;
+          }
+          .freeze {
+            width: @host-column-width;
+            position: absolute;
+            left: 0;
+            background-color: #fff;
+            border-left: 1px solid transparent;
+          }
+          .trim_hostname {
+            .ellipsis-overflow-nowrap;
+            max-width: 180px;
+            float: left;
+            display: block;
+          }
+          .checkbox {
+            margin: 0;
+            position: static;
+            input[type="checkbox"] {
+              + label.host-component-checkbox {
+                display: inline;
+                line-height: 17px;
+                font-size: 12px;
+                position: static;
+                padding-left: 0;
+                &:before {
+                  position: static;
+                  display: inline-block;
+                  margin-top: @checkbox-top;
+                  margin-right: @checkbox-size;
+                }
+                &:after {
+                  background-color: @checkbox-color;
+                  border-color: @checkbox-color;
+                  position: static;
+                  display: inline-block;
+                  float: left;
+                  margin-right: @checkbox-size;
+                  width: @checkbox-size;
+                  height: @checkbox-size;
+                  border-radius: @checkbox-border-radius;
+                  margin-top: -1 * (@checkbox-size + @checkbox-top);
+                  line-height: 1.2em;
+                  padding-left: 2px;
+                }
+              }
+              &:checked + label:before {
+                visibility: hidden;
+              }
+              &:not(:checked) + label:after {
+                content: '';
+                visibility: hidden;
+              }
+            }
+          }
+          &:hover {
+            &:not(:first-of-type) {
+              margin-top: -1px;
+            }
+            td.freeze {
+              background-color: @table-hover-background-color;
+              border-color: @table-hover-border-color;
+            }
+            + tr {
+              border-top-width: 0;
+            }
+          }
+          &.last-row {
+            border-bottom-color: transparent;
+            td {
+              border-bottom-color: transparent;
+            }
+            &:hover {
+              border-bottom-color: @table-hover-border-color;
+            }
           }
         }
-        .static-pagination {
-          position: absolute;
-          right: 10px;
+        .host-messages {
+          margin-left: @host-column-width;
+          padding: @table-cell-padding;
         }
       }
     }
+    .static-pagination {
+      margin-bottom: @table-margin-bottom;
+      font-size: @table-font-size;
+      color: @table-footer-color;
+      padding: @table-cell-padding;
+    }
     .spinner-overlay {
       position: absolute;
       width: 100%;

ambari-web/app/templates/common/configs/config_versions_control.hbs

@@ -34,7 +34,7 @@
   {{else}}
     {{view App.ConfigVersionsDropdownView serviceVersionsBinding="view.serviceVersions"}}
     {{#unless view.displayedServiceVersion.isCurrent}}
-      <button class="btn btn-secondary make-current" {{action makeCurrent view.displayedServiceVersion target="view"}}>
+      <button class="btn btn-secondary make-current" {{action makeCurrent target="view"}}>
         {{t dashboard.configHistory.info-bar.revert.button}}
       </button>
     {{/unless}}

ambari-web/app/templates/common/configs/configs_comparison_cell.hbs

@@ -16,9 +16,7 @@
 * limitations under the License.
 }}
 
-<span {{bindAttr class="compareConfig.isMock:undefined"}}>
-  {{compareConfig.value}}&nbsp;{{compareConfig.unit}}
-</span>
+<span {{bindAttr class="compareConfig.isMock:undefined :compare-config-cell"}}>{{compareConfig.value}}&nbsp;{{compareConfig.unit}}</span>
 {{#unless compareConfig.isMock}}
   {{#if compareConfig.supportsFinal}}
     <i {{bindAttr class=":glyphicon :glyphicon-lock compareConfig.isFinal::hidden" }}></i>

ambari-web/app/templates/common/configs/configs_comparison_row.hbs

@@ -32,7 +32,7 @@
   <div class="col-md-4 property-value-column">
     {{#if controller.selectedConfigGroup.isDefault}}
       {{! Comparing config-versions from Default config-group}}
-      <span {{bindAttr class="configData.isMock:undefined"}}>{{configData.value}}&nbsp;{{configData.unit}}</span>
+      <span {{bindAttr class="configData.isMock:undefined :compare-config-cell"}}>{{configData.value}}&nbsp;{{configData.unit}}</span>
       {{#unless configData.isMock}}
         {{#if configData.supportsFinal}}
           <i {{bindAttr class=":glyphicon :glyphicon-lock configData.isFinal::hidden" }}></i>

ambari-web/app/templates/common/configs/services_config.hbs

@@ -28,10 +28,10 @@
             <a href="#" {{action selectService tab target="controller"}} {{bindAttr data-target="tab.headingClass"}}
                data-toggle="tab">
               {{formatRole tab.serviceName}}
-              {{#if tab.errorsCount}}
+              {{#if tab.configsWithErrors}}
                 <span class="alert-badge">
                   <span class="counter label alerts-crit-count">
-                    {{tab.errorsCount}}
+                    {{tab.configsWithErrors.length}}
                   </span>
                 </span>
               {{/if}}

ambari-web/app/templates/common/host_progress_popup.hbs

@@ -343,7 +343,7 @@
                     <strong class="muted">{{hostLog.fileName}}</strong>
                     {{#view App.LogSearchUILinkView linkQueryParamsBinding="hostLog.linkTail" tagName="span"}}
                       <a {{bindAttr href="view.formatedLink" class=":pull-right view.isLodaded::disabled"}}
-                              target="_blank">
+                              target="_blank" rel="noopener noreferrer">
                         <i class="icon-external-link"></i>
                         {{t popup.logTail.openInLogSearch}}</a>
                     {{/view}}

ambari-web/app/templates/common/modal_popups/log_tail_popup.hbs

@@ -29,7 +29,7 @@
         <i class="icon-external-link"></i>
         {{t common.open}}
       </a>
-      <a class="open-in-log-search" {{bindAttr href="view.logSearchUrl"}} target="_blank">
+      <a class="open-in-log-search" {{bindAttr href="view.logSearchUrl"}} target="_blank" rel="noopener noreferrer">
         <i class="icon-external-link"></i>
         {{t popup.logTail.openInLogSearch}}
       </a>

ambari-web/app/templates/main/alerts/definition_details.hbs

@@ -193,7 +193,7 @@
           {{#if controller.content.hasHelpUrl}}
             <div class="row">
               <div class="col-md-5 property-name">{{t alerts.table.header.helpUrl}}:</div>
-              <div class="col-md-7"><label for=""><a {{bindAttr href="controller.content.helpUrl"}} target="_blank">{{t common.link}}</a></label></div>
+              <div class="col-md-7"><label for=""><a {{bindAttr href="controller.content.helpUrl"}} target="_blank" rel="noopener noreferrer">{{t common.link}}</a></label></div>
             </div>
           {{/if}}
         </div>

ambari-web/app/templates/main/dashboard/widgets/hbase_links.hbs

@@ -54,7 +54,7 @@
               <td>
                 {{#if view.activeMaster}}
                   <a {{bindAttr href="view.hbaseMasterWebUrl"}}
-                          target="_blank">{{t dashboard.services.hbase.masterWebUI}}</a>
+                          target="_blank" rel="noopener noreferrer">{{t dashboard.services.hbase.masterWebUI}}</a>
                 {{else}}
                   {{t services.service.summary.notAvailable}}
                 {{/if}}
@@ -81,7 +81,7 @@
                             <a href="javascript:void(null)">{{quickLinks.publicHostNameLabel}} &nbsp;</a>
                             <ul class="dropdown-menu">
                               {{#each quickLinks}}
-                                <li><a {{bindAttr href="url"}} target="_blank">{{label}}</a></li>
+                                <li><a {{bindAttr href="url"}} target="_blank" rel="noopener noreferrer">{{label}}</a></li>
                               {{/each}}
                             </ul>
                           </li>
@@ -89,7 +89,7 @@
                       {{/each}}
                     {{else}}
                       {{#each view.quickLinks}}
-                        <li><a {{bindAttr href="url"}} target="_blank">{{label}}</a></li>
+                        <li><a {{bindAttr href="url"}} target="_blank" rel="noopener noreferrer">{{label}}</a></li>
                       {{/each}}
                     {{/if}}
                   {{else}}

ambari-web/app/templates/main/dashboard/widgets/hdfs_links.hbs

@@ -101,7 +101,7 @@
                           <a href="javascript:void(null)">{{quickLinks.publicHostNameLabel}} &nbsp;</a>
                           <ul class="dropdown-menu">
                             {{#each quickLinks}}
-                              <li><a {{bindAttr href="url"}} target="_blank">{{label}}</a></li>
+                              <li><a {{bindAttr href="url"}} target="_blank" rel="noopener noreferrer">{{label}}</a></li>
                             {{/each}}
                           </ul>
                         </li>
@@ -109,7 +109,7 @@
                     {{/each}}
                   {{else}}
                     {{#each view.quickLinks}}
-                      <li><a {{bindAttr href="url"}} target="_blank">{{label}}</a></li>
+                      <li><a {{bindAttr href="url"}} target="_blank" rel="noopener noreferrer">{{label}}</a></li>
                     {{/each}}
                   {{/if}}
                 {{else}}

ambari-web/app/templates/main/dashboard/widgets/yarn_links.hbs

@@ -66,7 +66,7 @@
                             <a href="javascript:void(null)">{{quickLinks.publicHostNameLabel}} &nbsp;</a>
                             <ul class="dropdown-menu">
                               {{#each quickLinks}}
-                                <li><a {{bindAttr href="url"}} target="_blank">{{label}}</a></li>
+                                <li><a {{bindAttr href="url"}} target="_blank" rel="noopener noreferrer">{{label}}</a></li>
                               {{/each}}
                             </ul>
                           </li>
@@ -74,7 +74,7 @@
                       {{/each}}
                     {{else}}
                       {{#each view.quickLinks}}
-                        <li><a {{bindAttr href="url"}} target="_blank">{{label}}</a></li>
+                        <li><a {{bindAttr href="url"}} target="_blank" rel="noopener noreferrer">{{label}}</a></li>
                       {{/each}}
                     {{/if}}
                   {{else}}

ambari-web/app/templates/main/host/logs.hbs

@@ -43,7 +43,7 @@
               <div>
                 <a {{action openLogFile row file.filePath target="view.parentView"}} href="#" rel="log-file-name-tooltip" {{bindAttr data-original-title="file.filePath"}}>{{file.fileName}}</a>
                 {{#view App.LogSearchUILinkView linkQueryParamsBinding="file.linkTail" tagName="span"}}
-                  <a {{bindAttr href="view.formatedLink"}} target="_blank" rel="log-file-name-tooltip" {{translateAttr title="popup.logTail.openInLogSearch"}} class="pull-right external-link">
+                  <a {{bindAttr href="view.formatedLink"}} target="_blank" rel="log-file-name-tooltip noopener noreferrer" {{translateAttr title="popup.logTail.openInLogSearch"}} class="pull-right external-link">
                     <i class="icon-external-link"></i>
                     {{t popup.logTail.openInLogSearch}}
                   </a>

ambari-web/app/templates/main/service/info/metrics/flume/flume_agent_metrics_section.hbs

@@ -26,9 +26,7 @@
           {{view.header}}
         </h4>
       </div>
-        {{#if showTimeRangeControl}}
-          {{view view.timeRangeListView}}
-        {{/if}}
+      {{view view.timeRangeListView}}
       <div class="clearfix"></div>
     </div>
     <div class="panel-body collapse in" {{bindAttr id="view.id"}}>

ambari-web/app/templates/main/service/info/summary.hbs

@@ -100,7 +100,7 @@
                     {{#each quickLinks in group.links}}
                       <h6>{{quickLinks.publicHostNameLabel}}</h6>
                       {{#each quickLinks}}
-                        <a {{bindAttr href="url"}} target="_blank">{{label}}</a>
+                        <a {{bindAttr href="url"}} target="_blank" rel="noopener noreferrer">{{label}}</a>
                       {{/each}}
                     {{/each}}
                   </div>
@@ -108,7 +108,7 @@
               {{else}}
                 {{#if view.quickLinks}}
                   {{#each view.quickLinks}}
-                    <a {{bindAttr href="url"}} target="_blank">{{label}}</a>
+                    <a {{bindAttr href="url"}} target="_blank" rel="noopener noreferrer">{{label}}</a>
                   {{/each}}
                 {{else}}
                   <div class="alert alert-danger">

ambari-web/app/templates/wizard/step6.hbs

@@ -53,24 +53,28 @@
         </div>
       {{/if}}
 
-      <div class="pre-scrollable">
-        <table class="table table-hover" id="component_assign_table" {{QAAttr "slave-clients-table"}}>
-          <thead>
-          <tr>
-            <th class="host-column freeze">{{t common.host}}</th>
-            {{#each header in controller.headers}}
-              <th {{bindAttr class="header.name"}}>
-                <a href="#" {{QAAttr "select-all"}} {{bindAttr class="header.allChecked:selected:deselected header.isDisabled:remove-link" id="header.allId"}} {{action "selectAllNodes" header target="controller"}}>{{t all}}</a>
-                &nbsp;|&nbsp;
-                <a href="#" {{QAAttr "deselect-all"}} {{bindAttr class="header.noChecked:selected:deselected header.isDisabled:remove-link" id="header.noneId"}} {{action "deselectAllNodes" header target="controller"}}>{{t none}}</a>
-              </th>
-            {{/each}}
-          </tr>
-          </thead>
-          <tbody>
+      <div class="col-sm-12">
+        <div class="pre-scrollable">
+          <table class="table table-hover" id="component_assign_table" {{QAAttr "slave-clients-table"}}>
+            <thead>
+            <tr>
+              <th class="host-column freeze">{{t common.host}}</th>
+              {{#each header in controller.headers}}
+                <th {{bindAttr class="header.name"}}>
+                  <a
+                    href="#" {{QAAttr "select-all"}} {{bindAttr class="header.allChecked:selected:deselected header.isDisabled:remove-link" id="header.allId"}}
+                    {{action "selectAllNodes" header target="controller"}}>{{t all}}</a> &nbsp;|&nbsp; <a
+                  {{QAAttr "deselect-all"}}
+                  href="#" {{bindAttr class="header.noChecked:selected:deselected header.isDisabled:remove-link" id="header.noneId"}}
+                  {{action "deselectAllNodes" header target="controller"}}>{{t none}}</a>
+                </th>
+              {{/each}}
+            </tr>
+            </thead>
+            <tbody>
             {{#each host in view.pageContent}}
               <tr {{QAAttr "host-row"}} {{bindAttr class="host.isLast:last-row"}}>
-                {{#view App.WizardStep6HostView hostBinding="host" }}
+                {{#view App.WizardStep6HostView hostBinding="host"}}
                   <span class="trim_hostname">{{host.hostName}}</span>
                   {{#if host.hasMaster}}
                     <i {{QAAttr "has-masters"}} class="glyphicon glyphicon-asterisks">&#10037;</i>
@@ -79,35 +83,28 @@
                 {{#each checkbox in host.checkboxes}}
                   <td {{QAAttr "toggle-component"}} {{bindAttr class="checkbox.hasErrorMessage:error checkbox.hasWarnMessage:warning checkbox.component"}}>
                     <div class="checkbox" {{bindAttr data-qa="checkbox.dataQaAttr"}}>
-                        <input {{bindAttr id="checkbox.uId" checked="checkbox.checked" disabled="checkbox.isDisabled"}} {{action "checkboxClick" checkbox target="view" }}
-                                type="checkbox"/>
+                      <input {{bindAttr id="checkbox.uId" checked="checkbox.checked" disabled="checkbox.isDisabled"}} {{action "checkboxClick" checkbox target="view"}}
+                        type="checkbox"/>
                       <label class="host-component-checkbox" {{bindAttr for="checkbox.uId"}}>{{checkbox.title}}</label>
                     </div>
                   </td>
                 {{/each}}
               </tr>
               {{#if host.anyMessage}}
-                <tr>
-                  <td {{bindAttr colspan="view.columnCount"}} class="no-borders">
-                    {{#each errorMsg in host.errorMessages}}
-                        <div class="alert alert-danger">{{errorMsg}}</div>
-                    {{/each}}
-                    {{#each warnMsg in host.warnMessages}}
-                      <div class="alert alert-warning">{{warnMsg}}</div>
-                    {{/each}}
-                  </td>
-                </tr>
+                <div class="host-messages">
+                  {{#each errorMsg in host.errorMessages}}
+                    <div class="alert alert-danger">{{errorMsg}}</div>
+                  {{/each}}
+                  {{#each warnMsg in host.warnMessages}}
+                    <div class="alert alert-warning">{{warnMsg}}</div>
+                  {{/each}}
+                </div>
               {{/if}}
             {{/each}}
-          </tbody>
-          <tfoot>
-            <tr>
-              <td colspan="100">
-                {{view App.PaginationView classNames="static-pagination"}}
-              </td>
-            </tr>
-          </tfoot>
-        </table>
+            </tbody>
+          </table>
+        </div>
+        {{view App.PaginationView classNames="static-pagination"}}
       </div>
     </div>
   </div>

ambari-web/app/views/common/configs/config_versions_control_view.js

@@ -155,9 +155,9 @@ App.ConfigVersionsControlView = Em.View.extend({
   /**
    * revert config values to chosen version and apply reverted configs to server
    */
-  makeCurrent: function (event) {
+  makeCurrent: function () {
     const self = this;
-    const serviceConfigVersion = event.contexts[0];
+    const serviceConfigVersion = this.get('displayedServiceVersion');
     const versionText = serviceConfigVersion.get('versionText');
     return App.ModalPopup.show({
       header: Em.I18n.t('dashboard.configHistory.info-bar.makeCurrent.popup.title'),

ambari-web/app/views/common/configs/service_config_container_view.js

@@ -25,6 +25,8 @@ App.ServiceConfigContainerView = Em.ContainerView.extend({
   view: null,
 
   lazyLoading: null,
+  
+  pushViewTimeout: null,
 
   didInsertElement: function () {
     if (this.get('controller.isInstallWizard')) {
@@ -112,7 +114,8 @@ App.ServiceConfigContainerView = Em.ContainerView.extend({
     if (this.get('controller.isRecommendedLoaded')) {
       this.pushView();
     } else {
-      Em.run.later(this.pushViewAfterRecommendation.bind(this), 300);
+      clearTimeout(this.get('pushViewTimeout'));
+      this.set('pushViewTimeout', setTimeout(() => this.pushViewAfterRecommendation(), 300));
     }
   }
 

ambari-web/app/views/main/host/details.js

@@ -39,7 +39,6 @@ App.MainHostDetailsView = Em.View.extend({
   hasManyClientsWithConfigs: Em.computed.gt('clientsWithConfigs.length', 1),
 
   maintenance: function () {
-    var onOff = this.get('controller.content.isActive') ? "On" : "Off";
     var result = [];
     if (App.isAuthorized("SERVICE.START_STOP")) {
       result = result.concat([
@@ -74,8 +73,8 @@ App.MainHostDetailsView = Em.View.extend({
         action: 'onOffPassiveModeForHost',
         liClass: '',
         cssClass: 'icon-medkit',
-        active: this.get('isActive'),
-        label: this.t('passiveState.turn' + onOff)
+        active: this.get('controller.content.isActive'),
+        label: this.t('passiveState.turn' + (this.get('controller.content.isActive') ? "On" : "Off"))
       });
     }
     if (App.get('isKerberosEnabled') && App.get('supports.regenerateKeytabsOnSingleHost')){
@@ -97,7 +96,7 @@ App.MainHostDetailsView = Em.View.extend({
       label: this.t('host.host.details.checkHost')
     });
     return result;
-  }.property('controller.content', 'isActive', 'controller.content.isNotHeartBeating'),
+  }.property('controller.content', 'controller.content.isActive', 'controller.content.isNotHeartBeating'),
 
   didInsertElement: function () {
     var self = this;

ambari-web/app/views/main/service/info/metrics_view.js

@@ -100,6 +100,7 @@ App.MainServiceInfoMetricsView = Em.View.extend(App.Persist, App.TimeRangeMixin,
     this.makeSortable('#widget_layout');
     this.makeSortable('#ns_widget_layout', true);
     this.addWidgetTooltip();
+    this._super();
   },
 
   loadActiveWidgetLayout: function () {

ambari-web/app/views/wizard/step6_view.js

@@ -50,15 +50,36 @@ App.WizardStep6View = App.TableView.extend({
   didInsertElement: function () {
     this.setLabel();
     this.get('controller').loadStep();
+    this.$('.pre-scrollable').on('scroll', event => {
+      this.$('.pre-scrollable .freeze').css('transform', `translate(${event.target.scrollLeft}px,0)`);
+    });
     Em.run.next(this, this.adjustColumnWidth);
   },
 
   adjustColumnWidth: function() {
-    const table = $('#component_assign_table');
-    const columnsCount = this.get('controller.headers.length');
-    if (table.width() > table.find('tbody').width()) {
-      const columnWidth = Math.floor(table.width() / columnsCount);
+    const table = $('#component_assign_table'),
+      tableWrapper = $('.pre-scrollable').first(),
+      tableCells = table.find('tbody > tr:first-of-type > td');
+    let cellsWidth = 0;
+    $.each(tableCells, (i, td) => cellsWidth += $(td).width());
+    if (tableWrapper.width() > cellsWidth) {
+      const columnsCount = this.get('controller.headers.length'),
+        hostColumnWidth = 210, // from ambari-web/app/styles/wizard.less
+        columnWidth = Math.floor((table.width() - hostColumnWidth)/ columnsCount);
       table.find("th:not('.freeze'), td:not('.freeze')").width(columnWidth);
+      // a trick to keep checkbox abd label on the single line
+      table.find('.host-component-checkbox').css({
+        display: 'inline-block',
+        width: '0'
+      });
+    } else {
+      const tds = $('#component_assign_table > tbody > tr:first-of-type > td');
+      $.each(tds, (i, td) => {
+        const element = $(td),
+          className = element.attr('class'),
+          width = element.width();
+        $(`#component_assign_table th.${className}`).width(width);
+      });
     }
   },
 
@@ -93,12 +114,7 @@ App.WizardStep6View = App.TableView.extend({
     Em.set(checkbox, 'checked', !checkbox.checked);
     this.get('controller').checkCallback(checkbox.component);
     this.get('controller').hostsChanged();
-  },
-
-  columnCount: function() {
-    var hosts = this.get('controller.hosts');
-    return hosts && hosts.length > 0 ? hosts[0].get('checkboxes').length + 1 : 1;
-  }.property('controller.hosts.@each.checkboxes')
+  }
 });
 
 App.WizardStep6HostView = Em.View.extend({

ambari-web/test/controllers/main/admin/highAvailability/journalNode/step3_controller_test.js

@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var App = require('app');
+var testHelpers = require('test/helpers');
+
+describe('App.ManageJournalNodeWizardStep3Controller', function () {
+  var controller;
+
+  beforeEach(function () {
+    controller = App.ManageJournalNodeWizardStep3Controller.create({
+      content: Em.Object.create()
+    });
+  });
+
+  describe('#pullCheckPointsStatuses', function () {
+    it('should removeObserver if HDFS namespaces are loaded', function () {
+      controller.set('isHDFSNameSpacesLoaded', true);
+      controller.set('content.masterComponentHosts', [{component: 'NAMENODE', isInstalled: true, hostName: 'test'}]);
+      sinon.stub(controller, 'addObserver');
+      sinon.stub(controller, 'removeObserver');
+      controller.pullCheckPointsStatuses();
+      expect(controller.addObserver.calledOnce).to.be.false;
+      expect(controller.removeObserver.calledWith('isHDFSNameSpacesLoaded', controller, 'pullCheckPointsStatuses')).to.be.true;
+      controller.addObserver.restore();
+      controller.removeObserver.restore();
+    });
+
+    it('should send ajax request if nameSpacesCount is more than 1', function () {
+      sinon.stub(App.HDFSService, 'find').returns(Em.Object.create({
+        masterComponentGroups: [{}, {}],
+        activeNameNodes: [{hostName: 'test1'}, {hostName: 'test2'}]
+      }));
+      controller.pullCheckPointsStatuses();
+      var args = testHelpers.findAjaxRequest('admin.high_availability.getNnCheckPointsStatuses');
+      expect(args[0]).to.be.eql({
+        name: 'admin.high_availability.getNnCheckPointsStatuses',
+        sender: controller,
+        data: {
+          hostNames: ['test1', 'test2']
+        },
+        success: 'checkNnCheckPointsStatuses'
+      });
+      App.HDFSService.find.restore();
+    });
+
+    it('should send ajax request if nameSpacesCount is more than 1 and add started hosts', function () {
+      sinon.stub(App.HDFSService, 'find').returns(Em.Object.create({
+        masterComponentGroups: [{
+          hosts: ['test1']
+        }, {
+          hosts: ['test2']
+        }, {
+          hosts: ['test3']
+        }],
+        activeNameNodes: [{hostName: 'test1'}, {hostName: 'test2'}]
+      }));
+
+      controller.pullCheckPointsStatuses();
+      var args = testHelpers.findAjaxRequest('admin.high_availability.getNnCheckPointsStatuses');
+      expect(args[0].data.hostNames.length).to.be.equal(3);
+      App.HDFSService.find.restore();
+    });
+  });
+});

ambari-web/test/controllers/main/host/details_test.js

@@ -3440,7 +3440,7 @@ describe('App.MainHostDetailsController', function () {
 
     var cases = [
       {
-        'kmsHosts': ['host1'],
+        'zookeeperHosts': ['host1'],
         'kmsPort': 'port',
         'title': 'single host',
         'hostToInstall': undefined,
@@ -3473,7 +3473,7 @@ describe('App.MainHostDetailsController', function () {
         ]
       },
       {
-        'kmsHosts': ['host1', 'host2'],
+        'zookeeperHosts': ['host1', 'host2'],
         'kmsPort': 'port',
         'title': 'two hosts',
         'hostToInstall': 'host2',
@@ -3563,10 +3563,15 @@ describe('App.MainHostDetailsController', function () {
 
         beforeEach(function () {
           controller.set('rangerKMSServerHost', item.hostToInstall);
-          sinon.stub(controller, 'getRangerKMSServerHosts').returns(item.kmsHosts);
+          sinon.stub(App.MasterComponent, 'find').returns(Em.Object.create({hostNames: item.zookeeperHosts}))
+          sinon.stub(controller, 'getRangerKMSServerHosts').returns(item.zookeeperHosts);
           controller.onLoadRangerConfigs(data);
         });
 
+        afterEach(function () {
+          App.MasterComponent.find.restore();
+        });
+
         it('setConfigsChanges is called with valid arguments', function () {
           expect(controller.setConfigsChanges.calledWith(item.result)).to.be.true;
         });

ambari-web/test/models/host_component_test.js

@@ -146,21 +146,41 @@ describe('App.HostComponent', function() {
     });
   });
 
-  App.TestAliases.testAsComputedEqual(hc, 'isActive', 'passiveState', 'OFF');
-
   App.TestAliases.testAsComputedIfThenElse(hc, 'passiveTooltip', 'isActive', '', Em.I18n.t('hosts.component.passive.mode'));
 
   describe('#isActive', function() {
-    it('passiveState is ON', function() {
-      hc.set('passiveState', "ON");
-      hc.propertyDidChange('isActive');
-      expect(hc.get('isActive')).to.be.false;
-    });
     it('passiveState is OFF', function() {
       hc.set('passiveState', "OFF");
       hc.propertyDidChange('isActive');
       expect(hc.get('isActive')).to.be.true;
     });
+    it('passiveState is IMPLIED_FROM_HOST', function() {
+      hc.set('passiveState', "IMPLIED_FROM_HOST");
+      hc.set('host', {
+        passiveState: 'OFF'
+      });
+      hc.propertyDidChange('isActive');
+      expect(hc.get('isActive')).to.be.true;
+    });
+    it('passiveState is IMPLIED_FROM_SERVICE', function() {
+      hc.set('passiveState', "IMPLIED_FROM_SERVICE");
+      hc.set('service', {
+        passiveState: 'OFF'
+      });
+      hc.propertyDidChange('isActive');
+      expect(hc.get('isActive')).to.be.true;
+    });
+    it('passiveState is IMPLIED_FROM_SERVICE_AND_HOST', function() {
+      hc.set('passiveState', "IMPLIED_FROM_SERVICE_AND_HOST");
+      hc.set('host', {
+        passiveState: 'OFF'
+      });
+      hc.set('service', {
+        passiveState: 'OFF'
+      });
+      hc.propertyDidChange('isActive');
+      expect(hc.get('isActive')).to.be.true;
+    });
   });
 
   describe('#statusClass', function() {

ambari-web/test/views/common/configs/config_versions_control_view_test.js

@@ -101,12 +101,12 @@ describe('App.ConfigVersionsControlView', function () {
       view.sendRevertCall.restore();
     });
     it('context passed', function () {
-      view.makeCurrent({contexts: [
-        Em.Object.create({
-          version: 1,
-          serviceName: 'S1'
-        })
-      ]});
+      view.set('displayedServiceVersion', Em.Object.create({
+        version: 1,
+        serviceName: 'S1'
+      }));
+
+      view.makeCurrent();
 
       expect(App.ModalPopup.show.calledOnce).to.be.true;
       expect(view.sendRevertCall.calledWith(Em.Object.create({

ambari-web/test/views/wizard/step6_view_test.js

@@ -27,7 +27,12 @@ function getView() {
   var controller = App.WizardStep6Controller.create();
   controller.set('wizardController', App.InstallerController.create());
   return App.WizardStep6View.create({
-    controller: controller
+    controller: controller,
+    $: function () {
+      return {
+        on: Em.K
+      };
+    }
   });
 }
 
@@ -138,20 +143,6 @@ describe('App.WizardStep6View', function() {
     });
   });
 
-  describe("#columnCount", function() {
-    it("hosts present", function() {
-      view.set('controller.hosts', [
-        Em.Object.create({checkboxes: [{}, {}, {}]})
-      ]);
-      view.propertyDidChange('columnCount');
-      expect(view.get('columnCount')).to.equal(4);
-    });
-    it("hosts absent", function() {
-      view.set('controller.hosts', []);
-      view.propertyDidChange('columnCount');
-      expect(view.get('columnCount')).to.equal(1);
-    });
-  });
 });
 
 describe('App.WizardStep6HostView', function() {

contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py

@@ -1511,6 +1511,9 @@ class HDF20StackAdvisor(DefaultStackAdvisor):
     if not mountPoints:
       return self.getErrorItem("No disk info found on host %s" % hostInfo["host_name"])
 
+    if mountPoint is None:
+      return self.getErrorItem("No mount point in directory %s. Mount points: %s" % (dir, ', '.join(mountPoints.keys())))
+
     if mountPoints[mountPoint] < reqiuredDiskSpace:
       msg = "Ambari Metrics disk space requirements not met. \n" \
             "Recommended disk space for partition {0} is {1}G"

contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/stack_advisor.py

@@ -1313,6 +1313,9 @@ class ODPi20StackAdvisor(DefaultStackAdvisor):
     if not mountPoints:
       return self.getErrorItem("No disk info found on host %s" % hostInfo["host_name"])
 
+    if mountPoint is None:
+      return self.getErrorItem("No mount point in directory %s. Mount points: %s" % (dir, ', '.join(mountPoints.keys())))
+
     if mountPoints[mountPoint] < reqiuredDiskSpace:
       msg = "Ambari Metrics disk space requirements not met. \n" \
             "Recommended disk space for partition {0} is {1}G"

contrib/utils/perf/deploy-gce-perf-cluster.py

@@ -28,7 +28,7 @@ import re
 import socket
 
 cluster_prefix = "perf"
-ambari_repo_file_url = "http://s3.amazonaws.com/dev.hortonworks.com/ambari/centos6/2.x/updates/2.5.0.0/ambaribn.repo"
+ambari_repo_file_url = "http://s3.amazonaws.com/dev.hortonworks.com/ambari/centos7/2.x/updates/2.7.1.0/ambaribn.repo"
 
 public_hostname_script = "foo"
 hostname_script = "foo"
@@ -279,14 +279,14 @@ def create_vms(args, number_of_nodes):
   :param args: Command line args
   :param number_of_nodes: Number of VMs to request.
   """
-  print "Creating server VM {0}-server-{1} with xxlarge nodes on centos6...".format(cluster_prefix, args.cluster_suffix)
-  execute_command(args, args.controller, "/usr/sbin/gce up {0}-server-{1} 1 --centos6 --xxlarge --ex --disk-xxlarge --ssd".format(cluster_prefix, args.cluster_suffix),
+  print "Creating server VM {0}-server-{1} with xxlarge nodes on centos7...".format(cluster_prefix, args.cluster_suffix)
+  execute_command(args, args.controller, "/opt/gce-utils/gce up {0}-server-{1} 1 --centos7 --xxlarge --ex --disk-xxlarge --ssd".format(cluster_prefix, args.cluster_suffix),
                   "Failed to create server, probably not enough resources!", "-tt")
   time.sleep(10)
 
   # trying to create cluster with needed params
-  print "Creating agent VMs {0}-agent-{1} with {2} xlarge nodes on centos6...".format(cluster_prefix, args.cluster_suffix, str(number_of_nodes))
-  execute_command(args, args.controller, "/usr/sbin/gce up {0}-agent-{1} {2} --centos6 --xlarge --ex --disk-xlarge".format(cluster_prefix, args.cluster_suffix, str(number_of_nodes)),
+  print "Creating agent VMs {0}-agent-{1} with {2} xlarge nodes on centos7...".format(cluster_prefix, args.cluster_suffix, str(number_of_nodes))
+  execute_command(args, args.controller, "/opt/gce-utils/gce up {0}-agent-{1} {2} --centos7 --xlarge --ex --disk-xlarge".format(cluster_prefix, args.cluster_suffix, str(number_of_nodes)),
                   "Failed to create cluster VMs, probably not enough resources!", "-tt")
 
   # VMs are not accessible immediately
@@ -329,6 +329,7 @@ def create_server_script(server_host_name):
   # echo "arg=value" >> .../ambari.properties
 
   contents = "#!/bin/bash\n" + \
+  "yum install wget -y\n" + \
   "wget -O /etc/yum.repos.d/ambari.repo {0}\n".format(ambari_repo_file_url) + \
   "yum clean all; yum install git ambari-server -y\n" + \
   "mkdir /home ; cd /home ; git clone https://github.com/apache/ambari.git ; cd ambari ; git checkout branch-2.5\n" + \
@@ -399,11 +400,13 @@ def create_agent_script(server_host_name):
 
   # TODO, instead of cloning Ambari repo on each VM, do it on the server once and distribute to all of the agents.
   contents = "#!/bin/bash\n" + \
+  "yum install wget -y\n" + \
   "wget -O /etc/yum.repos.d/ambari.repo {0}\n".format(ambari_repo_file_url) + \
   "yum clean all; yum install krb5-workstation git ambari-agent -y\n" + \
   "mkdir /home ; cd /home; git clone https://github.com/apache/ambari.git ; cd ambari ; git checkout branch-2.5\n" + \
   "cp -r /home/ambari/ambari-server/src/main/resources/stacks/PERF /var/lib/ambari-agent/cache/stacks/PERF\n" + \
-  "sed -i -f /var/lib/ambari-agent/cache/stacks/PERF/PythonExecutor.sed /usr/lib/python2.6/site-packages/ambari_agent/PythonExecutor.py\n" + \
+  "sed -i -f /var/lib/ambari-agent/cache/stacks/PERF/PythonExecutor.sed /usr/lib/ambari-agent/lib/ambari_agent/PythonExecutor.py\n" + \
+  "sed -i -f /var/lib/ambari-agent/cache/stacks/PERF/check_host.sed /var/lib/ambari-agent/cache/custom_actions/scripts/check_host.py\n" + \
   "sed -i -e 's/hostname=localhost/hostname={0}/g' /etc/ambari-agent/conf/ambari-agent.ini\n".format(server_host_name) + \
   "sed -i -e 's/agent]/agent]\\nhostname_script={0}\\npublic_hostname_script={1}\\n/1' /etc/ambari-agent/conf/ambari-agent.ini\n".format(hostname_script, public_hostname_script) + \
   "python /home/ambari/ambari-agent/conf/unix/agent-multiplier.py start\n" + \
@@ -476,7 +479,7 @@ def __get_vms_list_from_name(args, cluster_name):
   :param args: Command line args
   :return: Mapping of VM host name to ip.
   """
-  gce_fqdb_cmd = '/usr/sbin/gce fqdn {0}'.format(cluster_name)
+  gce_fqdb_cmd = '/opt/gce-utils/gce fqdn {0}'.format(cluster_name)
   out = execute_command(args, args.controller, gce_fqdb_cmd, "Failed to get VMs list!", "-tt")
   lines = out.split('\n')
   #print "LINES=" + str(lines)
@@ -485,7 +488,7 @@ def __get_vms_list_from_name(args, cluster_name):
     for s in lines[2:]:  # Ignore non-meaningful lines
       if not s:
         continue
-      match = re.match(r'^([\d\.]*)\s+([\w\.-]*)\s+([\w\.-]*)\s$', s, re.M)
+      match = re.match(r'^([\d\.]*)\s+([\w\.-]*)\s+([\w\.-]*)\s+$', s, re.M)
       if match:
         result[match.group(2)] = match.group(1)
       else:

install-ambari-python.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# requires: pip setuptools wheel
+
+readlinkf(){
+  # get real path on mac OSX
+  perl -MCwd -e 'print Cwd::abs_path shift' "$1";
+}
+
+if [ "$(uname -s)" = 'Linux' ]; then
+  SCRIPT_DIR="`dirname "$(readlink -f "$0")"`"
+else
+  SCRIPT_DIR="`dirname "$(readlinkf "$0")"`"
+fi
+
+function print_help() {
+  cat << EOF
+   Usage: ./install-ambari-python.sh [additional options]
+
+   -c, --clean                  clean generated python distribution directories
+   -d, --deploy                 deploy ambari-python artifact to maven remote repository
+   -v, --version <version>      override ambari-python artifact versison
+   -i, --repository-id <id>     repository id in settings.xml for remote repository
+   -r, --repository-url <url>   repository url of remote repository
+   -h, --help                   print help
+EOF
+}
+
+function get_python_artifact_file() {
+  local artifact_file=$(ls $SCRIPT_DIR/dist/ | head -n 1)
+  echo $artifact_file
+}
+
+function get_version() {
+  local artifact_file=$(get_python_artifact_file)
+  local artifact_version=$(echo $artifact_file | perl -lne '/ambari-python-(.*?)\.tar\.gz/ && print $1')
+  echo $artifact_version
+}
+
+function clean() {
+  if [[ -d "$SCRIPT_DIR/dist" ]]; then
+    echo "Removing '$SCRIPT_DIR/dist' directoy ..."
+    rm -r "$SCRIPT_DIR/dist"
+    echo "Directory '$SCRIPT_DIR/dist' successfully deleted."
+  fi
+  if [[ -d "$SCRIPT_DIR/ambari_python.egg-info" ]]; then
+    echo "Removing '$SCRIPT_DIR/ambari_python.egg-info' directoy ..."
+    rm -r "$SCRIPT_DIR/ambari_python.egg-info"
+    echo "Directory '$SCRIPT_DIR/ambari_python.egg-info' successfully deleted."
+  fi
+  if [[ -d "$SCRIPT_DIR/target/ambari-python-dist" ]]; then
+    echo "Removing '$SCRIPT_DIR/target/ambari-python' directoy ..."
+    rm -r "$SCRIPT_DIR/target/ambari-python-dist"
+    echo "Directory '$SCRIPT_DIR/target/ambari-python' successfully deleted."
+  fi
+}
+
+function generate_site_packages() {
+  local version="$1"
+  pip install $SCRIPT_DIR/dist/ambari-python-$version.tar.gz -I --install-option="--prefix=$SCRIPT_DIR/target/ambari-python-dist"
+}
+
+function archive_python_dist() {
+  local artifact="$1"
+  local site_packages_dir=$(find $SCRIPT_DIR/target/ambari-python-dist -name "site-packages")
+  local base_dir="`dirname $site_packages_dir`" # use this to make it work with different python versions
+  if [[ -f "$SCRIPT_DIR/target/$artifact" ]]; then
+    echo "Removing '$SCRIPT_DIR/target/$artifact' file ..."
+    echo "File '$SCRIPT_DIR/target/$artifact' successfully deleted."
+  fi
+  tar -zcf $SCRIPT_DIR/target/$artifact -C $base_dir site-packages
+}
+
+function install() {
+  local artifact_file="$1"
+  local version="$2"
+  mvn install:install-file -Dfile=$artifact_file -DgeneratePom=true -Dversion=$version -DartifactId=ambari-python -DgroupId=org.apache.ambari -Dpackaging=tar.gz
+}
+
+function deploy() {
+  local artifact_file="$1"
+  local version="$2"
+  local repo_id="$3"
+  local repo_url="$4"
+  mvn deploy:deploy-file -Dfile=$artifact_file -Dpackaging=tar.gz -DgeneratePom=true -Dversion=$version -DartifactId=ambari-python -DgroupId=org.apache.ambari -Durl="$repo_url" -DrepositoryId="$repo_url"
+}
+
+function run_setup_py() {
+  local version="$1"
+  if [[ ! -z "$version" ]]; then
+    env AMBARI_VERSION="$version" python setup.py sdist
+  else
+    python setup.py sdist
+  fi
+}
+
+function main() {
+  while [[ $# -gt 0 ]]
+    do
+      key="$1"
+      case $key in
+        -d|--deploy)
+          local DEPLOY="true"
+          shift 1
+        ;;
+        -c|--clean)
+          local CLEAN="true"
+          shift 1
+        ;;
+        -v|--version)
+          local VERSION="$2"
+          shift 2
+        ;;
+        -i|--repository-id)
+          local REPOSITORY_ID="$2"
+          shift 2
+        ;;
+        -r|--repository-url)
+          local REPOSITORY_URL="$2"
+          shift 2
+        ;;
+        -h|--help)
+          shift 1
+          print_help
+          exit 0
+        ;;
+        *)
+          echo "Unknown option: $1"
+          exit 1
+        ;;
+      esac
+  done
+
+  if [[ -z "$DEPLOY" ]] ; then
+    DEPLOY="false"
+  fi
+
+  clean
+  if [[ "$CLEAN" == "true" ]]; then
+    return 0
+  fi
+
+  run_setup_py "$VERSION"
+  local artifact_name=$(get_python_artifact_file)
+  local artifact_version=$(get_version)
+
+  generate_site_packages "$artifact_version"
+  archive_python_dist "$artifact_name"
+
+  install "$SCRIPT_DIR/target/$artifact_name" "$artifact_version"
+
+  if [[ "$DEPLOY" == "true" ]] ; then
+    if [[ -z "$REPOSITORY_ID" ]] ; then
+      echo "Repository id option is required  for deploying ambari-python artifact (-i or --repository-id)"
+      exit 1
+    fi
+    if [[ -z "$REPOSITORY_URL" ]] ; then
+      echo "Repository url option is required for deploying ambari-python artifact (-r or --repository-url)"
+      exit 1
+    fi
+    deploy "$SCRIPT_DIR/target/$artifact_name" "$artifact_version" "$REPOSITORY_ID" "$REPOSITORY_URL"
+  else
+    echo "Skip deploying ambari-python artifact to remote repository."
+  fi
+}
+
+main ${1+"$@"}

setup.py

@@ -48,11 +48,35 @@ def create_package_dir_map():
 
   return package_dirs
 
-__version__ = "3.0.0.dev0"
+__version__ = "2.0.0.0-SNAPSHOT"
+
 def get_version():
-  ambari_version = os.environ["AMBARI_VERSION"] if "AMBARI_VERSION" in os.environ else __version__
-  print ambari_version
-  return ambari_version
+  """
+  Obtain ambari version during the build from pom.xml, which will be stored in PKG-INFO file.
+  During installation from pip, pom.xml is not included in the distribution but PKG-INFO is, so it can be used
+  instead of pom.xml file. If for some reason both are not exists use the default __version__ variable.
+  All of these can be overridden by AMBARI_VERSION environment variable.
+  """
+  base_dir = dirname(__file__)
+  if "AMBARI_VERSION" in os.environ:
+    return os.environ["AMBARI_VERSION"]
+  elif os.path.exists(os.path.join(base_dir, 'pom.xml')):
+    from xml.etree import ElementTree as et
+    ns = "http://maven.apache.org/POM/4.0.0"
+    et.register_namespace('', ns)
+    tree = et.ElementTree()
+    tree.parse(os.path.join(base_dir, 'pom.xml'))
+    parent_version_tag = tree.getroot().find("{%s}version" % ns)
+    return parent_version_tag.text if parent_version_tag is not None else __version__
+  elif os.path.exists(os.path.join(base_dir, 'PKG-INFO')):
+    import re
+    version = None
+    version_re = re.compile('^Version: (.+)$', re.M)
+    with open(os.path.join(base_dir, 'PKG-INFO')) as f:
+      version = version_re.search(f.read()).group(1)
+    return version if version is not None else __version__
+  else:
+    return __version__
 
 """
 Example usage:
@@ -64,9 +88,7 @@ Example usage:
   python setup.py sdist -d "my/dist/location" upload -r "http://localhost:8080"
 
 Installing from pip:
-- pip install --extra-index-url=http://localhost:8080 ambari-python==2.7.1  // 3.0.0.dev0 is the snapshot version
-
-Note: using 'export AMBARI_VERSION=2.7.1' before commands you can redefine the package version, but you will need this export during the pip install as well
+- pip install --extra-index-url=http://localhost:8080 ambari-python==2.7.1.0  // 2.0.0.0-SNAPSHOT is the snapshot version
 """
 setup(
   name = "ambari-python",