Merge pull request #318 from mpapirkovskyy/trunk-branch-3.0-perf

Trunk branch 3.0 perf

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,10 @@
+## What changes were proposed in this pull request?
+
+(Please fill in changes proposed in this fix)
+
+## How was this patch tested?
+
+(Please explain how this patch was tested. Ex: unit tests, manual tests)
+(If this patch involves UI changes, please attach a screen-shot; otherwise, remove this)
+
+Please review [Ambari Contributing Guide](https://cwiki.apache.org/confluence/display/AMBARI/How+to+Contribute) before opening a pull request.

ambari-admin/src/main/resources/ui/admin-web/app/index.html

@@ -54,15 +54,15 @@
        <div class="container">
          <div class="navbar-header navbar-nav">
            <ol class="breadcrumb">
-             <li ng-repeat="breadcrumb in breadcrumbs" ng-class="$last && 'active'">{{breadcrumb}}</li>
+             <li ng-repeat="breadcrumb in breadcrumbs" ng-class="$last && 'active'" ng-cloak>{{breadcrumb}}</li>
            </ol>
          </div>
          <ul class="nav navbar-nav navbar-right">
            <li>
-             <p class="navbar-text">{{cluster.Clusters.cluster_name}}</p>
+             <p class="navbar-text" ng-cloak>{{cluster.Clusters.cluster_name}}</p>
            </li>
            <li class="dropdown">
-             <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" ng-disabled="disabled">
+             <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" ng-disabled="disabled" ng-cloak>
                <i class="fa fa-user"></i>&nbsp;{{currentUser}}&nbsp;<span class="caret"></span>
              </a>
              <ul class="dropdown-menu">
@@ -150,6 +150,7 @@
 <script src="scripts/directives/PasswordVerify.js"></script>
 <script src="scripts/directives/disabledTooltip.js"></script>
 <script src="scripts/directives/editableList.js"></script>
+<script src="scripts/directives/comboSearch.js"></script>
 <script src="scripts/services/Utility.js"></script>
 <script src="scripts/services/UserConstants.js"></script>
 <script src="scripts/services/User.js"></script>
@@ -169,6 +170,8 @@
 <script src="scripts/services/AddRepositoryModal.js"></script>
 <script src="scripts/services/AddVersionModal.js"></script>
 <script src="scripts/services/RoleDetailsModal.js"></script>
+<script src="scripts/services/Pagination.js"></script>
+<script src="scripts/services/Filters.js"></script>
 <!-- endbuild -->
 </body>
 </html>

ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/SideNavCtrl.js

@@ -29,11 +29,7 @@ angular.module('ambariAdminConsole')
   }, true);
 
   function loadRepos() {
-    Stack.allRepos({version: '',
-      cluster: {
-        options: [],
-        current: null
-      }}, {}).then(function (repos) {
+    Stack.allRepos().then(function (repos) {
       $scope.totalRepos = repos.itemTotal;
     });
   }

ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/ambariViews/ViewsListCtrl.js

@@ -18,154 +18,65 @@
 'use strict';
 
 angular.module('ambariAdminConsole')
-.controller('ViewsListCtrl',['$scope', 'View','$modal', 'Alert', 'ConfirmationModal', '$translate', 'Settings', function($scope, View, $modal, Alert, ConfirmationModal, $translate, Settings) {
+.controller('ViewsListCtrl',
+['$scope', 'View','$modal', 'Alert', 'ConfirmationModal', '$translate', 'Settings', 'Pagination', 'Filters',
+function($scope, View, $modal, Alert, ConfirmationModal, $translate, Settings, Pagination, Filters) {
   var $t = $translate.instant;
   var VIEWS_VERSION_STATUS_TIMEOUT = 5000;
   $scope.isLoading = false;
   $scope.minInstanceForPagination = Settings.minRowsToShowPagination;
 
-  function checkViewVersionStatus(view, versionObj, versionNumber) {
-    var deferred = View.checkViewVersionStatus(view.view_name, versionNumber);
-
-    deferred.promise.then(function (status) {
-      if (versionNeedStatusUpdate(status)) {
-        setTimeout(function() {
-          checkViewVersionStatus(view, versionObj, versionNumber);
-        }, VIEWS_VERSION_STATUS_TIMEOUT);
-      } else {
-        versionObj.status = status;
-        angular.forEach(view.versions, function (version) {
-          if (version.status === 'DEPLOYED') {
-            view.canCreateInstance = true;
-          }
-        })
-      }
-    });
-  }
-
-  function versionNeedStatusUpdate(status) {
-    return status !== 'DEPLOYED' && status !== 'ERROR';
-  }
-
-  function loadViews() {
-    $scope.isLoading = true;
-    View.all().then(function (views) {
-      $scope.isLoading = false;
-      $scope.views = views;
-      $scope.instances = [];
-      angular.forEach(views, function (view) {
-        angular.forEach(view.versions, function (versionObj, versionNumber) {
-          if (versionNeedStatusUpdate(versionObj.status)) {
-            checkViewVersionStatus(view, versionObj, versionNumber);
-          }
-        });
-        angular.forEach(view.instances, function (instance) {
-          instance.ViewInstanceInfo.short_url_name = instance.ViewInstanceInfo.short_url_name || '';
-          instance.ViewInstanceInfo.short_url = instance.ViewInstanceInfo.short_url || '';
-          instance.ViewInstanceInfo.versionObj = view.versions[instance.ViewInstanceInfo.version] || {};
-          $scope.instances.push(instance.ViewInstanceInfo);
-        });
-      });
-      initTypeFilter();
-      $scope.filterInstances();
-    }).catch(function (data) {
-      Alert.error($t('views.alerts.cannotLoadViews'), data.data.message);
-    });
-  }
-
-  function initTypeFilter() {
-    var uniqTypes = $.unique($scope.instances.map(function(instance) {
-      return instance.view_name;
-    }));
-    $scope.typeFilterOptions = [ { label: $t('common.all'), value: '*'} ]
-      .concat(uniqTypes.map(function(type) {
-        return {
-          label: type,
-          value: type
-        };
-      }));
-    $scope.instanceTypeFilter = $scope.typeFilterOptions[0];
-  }
-
-  function showInstancesOnPage() {
-    var startIndex = ($scope.currentPage - 1) * $scope.instancesPerPage + 1;
-    var endIndex = $scope.currentPage * $scope.instancesPerPage;
-    var showedCount = 0;
-    var filteredCount = 0;
-
-    angular.forEach($scope.instances, function(instance) {
-      instance.isShowed = false;
-      if (instance.isFiltered) {
-        filteredCount++;
-        if (filteredCount >= startIndex && filteredCount <= endIndex) {
-          instance.isShowed = true;
-          showedCount++;
-        }
-      }
-    });
-    $scope.tableInfo.showed = showedCount;
-  }
-
+  $scope.filters = [
+    {
+      key: 'short_url_name',
+      label: $t('common.name'),
+      options: []
+    },
+    {
+      key: 'url',
+      label: $t('urls.url'),
+      customValueConverter: function(item) {
+        return '/main/view/' + item.view_name + '/' + item.short_url;
+      },
+      options: []
+    },
+    {
+      key: 'view_name',
+      label: $t('views.table.viewType'),
+      options: []
+    },
+    {
+      key: 'instance_name',
+      label: $t('urls.viewInstance'),
+      options: []
+    }
+  ];
   $scope.views = [];
   $scope.instances = [];
-  $scope.instancesPerPage = 10;
-  $scope.currentPage = 1;
-  $scope.instanceNameFilter = '';
-  $scope.instanceUrlFilter = '';
-  $scope.maxVisiblePages = 10;
-  $scope.isNotEmptyFilter = true;
-  $scope.instanceTypeFilter = '';
   $scope.tableInfo = {
     filtered: 0,
-    showed: 0
+    showed: 0,
+    total: 0
   };
+  $scope.pagination = Pagination.create();
 
-  loadViews();
-
-  $scope.filterInstances = function() {
-    var filteredCount = 0;
-    angular.forEach($scope.instances, function(instance) {
-      if ($scope.instanceNameFilter && instance.short_url_name.indexOf($scope.instanceNameFilter) === -1) {
-        return instance.isFiltered = false;
-      }
-      if ($scope.instanceUrlFilter && ('/main/view/'+ instance.view_name + '/' + instance.short_url).indexOf($scope.instanceUrlFilter) === -1) {
-        return instance.isFiltered = false;
-      }
-      if ($scope.instanceTypeFilter.value !== '*' && instance.view_name.indexOf($scope.instanceTypeFilter.value) === -1) {
-        return instance.isFiltered = false;
-      }
-      filteredCount++;
-      instance.isFiltered = true;
-    });
-    $scope.tableInfo.filtered = filteredCount;
-    $scope.resetPagination();
+  $scope.resetPagination = function() {
+    $scope.pagination.resetPagination($scope.instances, $scope.tableInfo);
   };
 
   $scope.pageChanged = function() {
-    showInstancesOnPage();
+    $scope.pagination.pageChanged($scope.instances, $scope.tableInfo);
   };
 
-  $scope.resetPagination = function() {
-    $scope.currentPage = 1;
-    showInstancesOnPage();
+  $scope.filterInstances = function(appliedFilters) {
+    $scope.tableInfo.filtered = Filters.filterItems(appliedFilters, $scope.instances, $scope.filters);
+    $scope.pagination.resetPagination($scope.instances, $scope.tableInfo);
   };
 
-  $scope.clearFilters = function () {
-    $scope.instanceNameFilter = '';
-    $scope.instanceUrlFilter = '';
-    $scope.instanceTypeFilter = $scope.typeFilterOptions[0];
-    $scope.resetPagination();
+  $scope.toggleSearchBox = function() {
+    $('.search-box-button .popup-arrow-up, .search-box-row').toggleClass('hide');
   };
 
-  $scope.$watch(
-    function (scope) {
-      return Boolean(scope.instanceNameFilter || scope.instanceUrlFilter || (scope.instanceTypeFilter && scope.instanceTypeFilter.value !== '*'));
-    },
-    function (newValue, oldValue, scope) {
-      scope.isNotEmptyFilter = newValue;
-    }
-  );
-
   $scope.cloneInstance = function(instanceClone) {
     $scope.createInstance(instanceClone);
   };
@@ -207,4 +118,57 @@ angular.module('ambariAdminConsole')
         });
     });
   };
+
+  loadViews();
+
+  function checkViewVersionStatus(view, versionObj, versionNumber) {
+    var deferred = View.checkViewVersionStatus(view.view_name, versionNumber);
+
+    deferred.promise.then(function (status) {
+      if (versionNeedStatusUpdate(status)) {
+        setTimeout(function() {
+          checkViewVersionStatus(view, versionObj, versionNumber);
+        }, VIEWS_VERSION_STATUS_TIMEOUT);
+      } else {
+        versionObj.status = status;
+        angular.forEach(view.versions, function (version) {
+          if (version.status === 'DEPLOYED') {
+            view.canCreateInstance = true;
+          }
+        })
+      }
+    });
+  }
+
+  function versionNeedStatusUpdate(status) {
+    return status !== 'DEPLOYED' && status !== 'ERROR';
+  }
+
+  function loadViews() {
+    $scope.isLoading = true;
+    View.all().then(function (views) {
+      $scope.isLoading = false;
+      $scope.views = views;
+      $scope.instances = [];
+      angular.forEach(views, function (view) {
+        angular.forEach(view.versions, function (versionObj, versionNumber) {
+          if (versionNeedStatusUpdate(versionObj.status)) {
+            checkViewVersionStatus(view, versionObj, versionNumber);
+          }
+        });
+        angular.forEach(view.instances, function (instance) {
+          instance.ViewInstanceInfo.short_url_name = instance.ViewInstanceInfo.short_url_name || '';
+          instance.ViewInstanceInfo.short_url = instance.ViewInstanceInfo.short_url || '';
+          instance.ViewInstanceInfo.versionObj = view.versions[instance.ViewInstanceInfo.version] || {};
+          $scope.instances.push(instance.ViewInstanceInfo);
+        });
+      });
+      $scope.tableInfo.total = $scope.instances.length;
+      Filters.initFilterOptions($scope.filters, $scope.instances);
+      $scope.filterInstances();
+    }).catch(function (data) {
+      Alert.error($t('views.alerts.cannotLoadViews'), data.data.message);
+    });
+  }
+
 }]);

ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/remoteClusters/RemoteClustersListCtrl.js

@@ -18,92 +18,74 @@
 'use strict';
 
 angular.module('ambariAdminConsole')
-.controller('RemoteClustersListCtrl', ['$scope', '$routeParams', '$translate', 'RemoteCluster', function ($scope, $routeParams, $translate, RemoteCluster) {
+.controller('RemoteClustersListCtrl',
+['$scope', '$routeParams', '$translate', 'RemoteCluster', 'Settings', 'Pagination', 'Filters',
+function ($scope, $routeParams, $translate, RemoteCluster, Settings, Pagination, Filters) {
   var $t = $translate.instant;
 
+  $scope.minInstanceForPagination = Settings.minRowsToShowPagination;
   $scope.clusterName = $routeParams.clusterName;
   $scope.isLoading = false;
-
   $scope.constants = {
     groups: $t('common.clusters').toLowerCase()
   };
-
-  $scope.groupsPerPage = 10;
-  $scope.currentPage = 1;
-  $scope.totalGroups = 1;
-  $scope.currentNameFilter = '';
-  $scope.maxVisiblePages=20;
   $scope.tableInfo = {
+    filtered: 0,
     total: 0,
     showed: 0
   };
+  $scope.pagination = Pagination.create();
+  $scope.filters = [
+    {
+      key: 'clusterName',
+      label: $t('views.clusterName'),
+      options: []
+    },
+    {
+      key: 'service',
+      label: $t('common.services'),
+      customValueConverter: function (item) {
+        return item.ClusterInfo.services;
+      },
+      isMultiple: true,
+      options: []
+    }
+  ];
 
-  $scope.isNotEmptyFilter = true;
-
-  $scope.pageChanged = function() {
-    loadRemoteClusters();
-  };
-  $scope.groupsPerPageChanges = function() {
-    loadRemoteClusters();
+  $scope.toggleSearchBox = function () {
+    $('.search-box-button .popup-arrow-up, .search-box-row').toggleClass('hide');
   };
 
-  $scope.resetPagination = function() {
-    $scope.currentPage = 1;
-    loadRemoteClusters();
+  $scope.filterClusters = function (appliedFilters) {
+    $scope.tableInfo.filtered = Filters.filterItems(appliedFilters, $scope.remoteClusters, $scope.filters);
+    $scope.pagination.resetPagination($scope.remoteClusters, $scope.tableInfo);
   };
 
-  $scope.typeFilterOptions = [
-    $t('common.any')
-  ];
-
-  $scope.currentTypeFilter = $scope.typeFilterOptions[0];
-
-  $scope.clearFilters = function () {
-    $scope.currentNameFilter = '';
-    $scope.currentTypeFilter = $scope.typeFilterOptions[0];
-    $scope.resetPagination();
+  $scope.pageChanged = function () {
+    $scope.pagination.pageChanged($scope.remoteClusters, $scope.tableInfo);
   };
 
-  function loadRemoteClusters(){
-      $scope.isLoading = true;
-      RemoteCluster.all({
-        currentPage: $scope.currentPage,
-        groupsPerPage: $scope.groupsPerPage,
-        searchString: $scope.currentNameFilter,
-        service: $scope.currentTypeFilter
-      }).then(function(remoteclusters) {
-        $scope.isLoading = false;
-
-        $scope.totalGroups = remoteclusters.itemTotal;
-        $scope.tableInfo.total = remoteclusters.itemTotal;
-        $scope.tableInfo.showed = remoteclusters.items.length;
-
-        $scope.remoteClusters = remoteclusters.items;
-
-        remoteclusters.items.map(function(clusteritem){
-          clusteritem.ClusterInfo.services.map(function(service){
-            var serviceIndex = $scope.typeFilterOptions.indexOf(service);
-            if(serviceIndex == -1){
-              $scope.typeFilterOptions.push(service);
-            }
-          })
-        })
+  $scope.resetPagination = function () {
+    $scope.pagination.resetPagination($scope.remoteClusters, $scope.tableInfo);
+  };
 
-      })
-      .catch(function(data) {
+  function loadRemoteClusters() {
+    $scope.isLoading = true;
+    RemoteCluster.all().then(function (remoteclusters) {
+      $scope.isLoading = false;
+      $scope.remoteClusters = remoteclusters.items.map(function (item) {
+        item.clusterName = item.ClusterInfo.name;
+        return item;
+      });
+      $scope.tableInfo.total = $scope.remoteClusters.length;
+      $scope.filterClusters();
+      Filters.initFilterOptions($scope.filters, $scope.remoteClusters);
+    })
+      .catch(function (data) {
         console.error($t('remoteClusters.alerts.fetchError'), data);
       });
-  };
+  }
 
   loadRemoteClusters();
 
-  $scope.$watch(
-    function (scope) {
-      return Boolean(scope.currentNameFilter || (scope.currentTypeFilter));
-    },
-    function (newValue, oldValue, scope) {
-      scope.isNotEmptyFilter = newValue;
-    }
-  );
-
 }]);

ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsCreateCtrl.js

@@ -43,7 +43,7 @@ angular.module('ambariAdminConsole')
   $scope.isGPLAccepted = false;
 
   $scope.isGPLRepo = function (repository) {
-    return repository.Repositories.tags.indexOf('GPL') >= 0;
+    return  repository.Repositories.tags && repository.Repositories.tags.indexOf('GPL') >= 0;
   };
 
   $scope.showRepo = function (repository) {

ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsEditCtrl.js

@@ -36,7 +36,7 @@ angular.module('ambariAdminConsole')
   $scope.isGPLAccepted = false;
 
   $scope.isGPLRepo = function (repository) {
-    return repository.Repositories.tags && repository.Repositories.tags.indexOf('GPL') >= 0;
+    return repository.Repositories.tags.indexOf('GPL') >= 0;
   };
 
   $scope.showRepo = function (repository) {
@@ -136,7 +136,7 @@ angular.module('ambariAdminConsole')
   };
 
   $scope.isDeletable = function() {
-    return !($scope.repoStatus == 'current' || $scope.repoStatus == 'installed');
+    return !($scope.repoStatus === 'CURRENT' || $scope.repoStatus === 'INSTALLED');
   };
 
   $scope.disableUnusedOS = function() {

ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsListCtrl.js

@@ -18,93 +18,96 @@
 'use strict';
 
 angular.module('ambariAdminConsole')
-  .controller('StackVersionsListCtrl', ['$scope', 'Cluster', 'Stack', '$routeParams', '$translate', 'Settings', function ($scope, Cluster, Stack, $routeParams, $translate, Settings) {
+  .controller('StackVersionsListCtrl',
+  ['$scope', 'Cluster', 'Stack', '$routeParams', '$translate', 'Settings', 'Pagination', '$q', 'Filters',
+  function ($scope, Cluster, Stack, $routeParams, $translate, Settings, Pagination, $q, Filters) {
     var $t = $translate.instant;
     $scope.getConstant = function (key) {
       return $t(key).toLowerCase();
     };
+    $scope.minInstanceForPagination = Settings.minRowsToShowPagination;
     $scope.isLoading = false;
     $scope.clusterName = $routeParams.clusterName;
-    $scope.filter = {
-      name: '',
-      version: '',
-      type: '',
-      cluster: {
-        options: [],
-        current: null
-      },
-      stack: {
-        options: [],
-        current: null
-      }
-    };
-    $scope.isNotEmptyFilter = true;
-
-    $scope.pagination = {
-      totalRepos: 10,
-      maxVisiblePages: 20,
-      itemsPerPage: 10,
-      currentPage: 1
-    };
-
     $scope.tableInfo = {
       total: 0,
       showed: 0,
       filtered: 0
     };
-
     $scope.repos = [];
     $scope.dropDownClusters = [];
     $scope.selectedCluster = $scope.dropDownClusters[0];
+    $scope.filters = [
+      {
+        key: 'stack',
+        label: $t('common.stack'),
+        customValueConverter: function(item) {
+          return item.stack_name + '-' + item.stack_version;
+        },
+        options: []
+      },
+      {
+        key: 'display_name',
+        label: $t('common.name'),
+        options: []
+      },
+      {
+        key: 'type',
+        label: $t('common.type'),
+        options: []
+      },
+      {
+        key: 'repository_version',
+        label: $t('common.version'),
+        options: []
+      },
+      {
+        key: 'cluster',
+        label: $t('common.cluster'),
+        options: []
+      }
+    ];
+    $scope.pagination = Pagination.create();
 
-    $scope.resetPagination = function () {
-      $scope.pagination.currentPage = 1;
-      $scope.loadAllData();
+    $scope.resetPagination = function() {
+      $scope.pagination.resetPagination($scope.repos, $scope.tableInfo);
     };
 
-    $scope.pageChanged = function () {
-      $scope.loadAllData();
+    $scope.pageChanged = function() {
+      $scope.pagination.pageChanged($scope.repos, $scope.tableInfo);
     };
 
-    $scope.goToCluster = function() {
-      window.location.replace(Settings.siteRoot + '#/main/admin/stack/versions');
+    $scope.filterRepos = function (appliedFilters) {
+      $scope.tableInfo.filtered = Filters.filterItems(appliedFilters, $scope.repos, $scope.filters);
+      $scope.pagination.resetPagination($scope.repos, $scope.tableInfo);
+    };
+
+    $scope.toggleSearchBox = function() {
+      $('.search-box-button .popup-arrow-up, .search-box-row').toggleClass('hide');
     };
 
-    $scope.clearFilters = function () {
-      $scope.filter.name = '';
-      $scope.filter.version = '';
-      $scope.filter.cluster.current = $scope.filter.cluster.options[0];
-      $scope.filter.stack.current = $scope.filter.stack.options[0];
-      $scope.resetPagination();
+    $scope.goToCluster = function() {
+      window.location.replace(Settings.siteRoot + '#/main/admin/stack/versions');
     };
 
     $scope.fetchRepoClusterStatus = function (allRepos) {
+      var calls = [];
       if (allRepos && allRepos.length) {
-        var clusterName = ($scope.clusters && $scope.clusters.length > 0) ? $scope.clusters[0].Clusters.cluster_name : null, // only support one cluster at the moment
-          repos = [],
-          processedRepos = 0;
+        // only support one cluster at the moment
+        var clusterName = $scope.cluster && $scope.cluster.Clusters.cluster_name;
         if (clusterName) {
-          angular.forEach(allRepos, function (repo) {
-            Cluster.getRepoVersionStatus(clusterName, repo.id).then(function (response) {
-              repo.cluster = (response.status == 'current' || response.status == 'installed') ? clusterName : '';
-              if (!$scope.filter.cluster.current.value || repo.cluster) {
+          $scope.repos = allRepos;
+          $scope.tableInfo.total = allRepos.length;
+          angular.forEach($scope.repos, function (repo) {
+            calls.push(Cluster.getRepoVersionStatus(clusterName, repo.id).then(function (response) {
+              repo.cluster = (response.status === 'CURRENT' || response.status === 'INSTALLED') ? clusterName : '';
+              if (repo.cluster) {
                 repo.status = response.status;
                 repo.totalHosts = response.totalHosts;
                 repo.currentHosts = response.currentHosts;
                 repo.installedHosts = response.installedHosts;
                 repo.stackVersionId = response.stackVersionId;
-                repos.push(repo);
               }
-              processedRepos++;
-              if (processedRepos === allRepos.length) {
-                var from = ($scope.pagination.currentPage - 1) * $scope.pagination.itemsPerPage;
-                var to = (repos.length - from > $scope.pagination.itemsPerPage) ? from + $scope.pagination.itemsPerPage : repos.length;
-                $scope.repos = repos.slice(from, to);
-                $scope.tableInfo.total = repos.length;
-                $scope.pagination.totalRepos = repos.length;
-                $scope.tableInfo.showed = to - from;
-              }
-            });
+            }));
           });
         }
       } else {
@@ -113,101 +116,52 @@ angular.module('ambariAdminConsole')
         $scope.pagination.totalRepos = 0;
         $scope.tableInfo.showed = 0;
       }
+      $scope.tableInfo.total = $scope.repos.length;
+      return $q.all(calls);
     };
 
     $scope.fetchRepos = function () {
-      return Stack.allRepos($scope.filter).then(function (repos) {
+      return Stack.allRepos().then(function (repos) {
         $scope.isLoading = false;
         return repos.items;
       });
     };
 
-    $scope.fillClusters = function (clusters) {
-      $scope.dropDownClusters = [].concat(clusters);
-      var options = [{label: $t('common.all'), value: ''}];
-      angular.forEach(clusters, function (cluster) {
-        options.push({
-          label: cluster.Clusters.cluster_name,
-          value: cluster.Clusters.cluster_name
-        });
-      });
-      $scope.filter.cluster.options = options;
-      if (!$scope.filter.cluster.current) {
-        $scope.filter.cluster.current = options[0];
-      }
-    };
-
     $scope.fetchClusters = function () {
       return Cluster.getAllClusters().then(function (clusters) {
         if (clusters && clusters.length > 0) {
-          $scope.clusters = clusters;
-          $scope.fillClusters(clusters);
-        }
-      });
-    };
-
-    $scope.fetchStacks = function () {
-      return Stack.allStackVersions().then(function (clusters) {
-        if (clusters && clusters.length > 0) {
-          $scope.stacks = clusters;
-          $scope.fillStacks(clusters);
-        }
-      });
-    };
-
-    $scope.fillStacks = function() {
-      var options = [{label: $t('common.all'), value: ''}];
-      angular.forEach($scope.stacks, function (stack) {
-        if (stack.active) {
-          options.push({
-            label: stack.displayName,
-            value: stack.displayName
-          });
+          $scope.dropDownClusters = clusters;
         }
       });
-      $scope.filter.stack.options = options;
-      if (!$scope.filter.stack.current) {
-        $scope.filter.stack.current = options[0];
-      }
     };
 
     $scope.loadAllData = function () {
       $scope.isLoading = true;
-      $scope.fetchStacks()
-        .then(function () {
-          return $scope.fetchClusters();
-        })
-        .then(function () {
-          return $scope.fetchRepos();
-        })
+      $scope.fetchRepos()
         .then(function (repos) {
-          $scope.fetchRepoClusterStatus(repos);
+          $scope.fetchClusters();
+          $scope.fetchRepoClusterStatus(repos).then(function() {
+            Filters.initFilterOptions($scope.filters, $scope.repos);
+          });
+          $scope.filterRepos();
         });
     };
 
     $scope.loadAllData();
 
-    $scope.$watch('filter', function (filter) {
-      $scope.isNotEmptyFilter = Boolean(filter.name
-        || filter.version
-        || filter.type
-        || (filter.cluster.current && filter.cluster.current.value)
-        || (filter.stack.current && filter.stack.current.value));
-    }, true);
-
     $scope.toggleVisibility = function (repo) {
       repo.isProccessing = true;
       var payload = {
-        RepositoryVersions:{
+        RepositoryVersions: {
           hidden: repo.hidden
         }
-      }
-      Stack.updateRepo(repo.stack_name, repo.stack_version, repo.id, payload).then( null, function () {
+      };
+      Stack.updateRepo(repo.stack_name, repo.stack_version, repo.id, payload).then(null, function () {
         repo.hidden = !repo.hidden;
-      }).finally( function () {
+      }).finally(function () {
         delete repo.isProccessing;
       });
-    }
+    };
 
     $scope.isHideCheckBoxEnabled = function ( repo ) {
       return !repo.isProccessing && ( !repo.cluster || repo.isPatch && ( repo.status === 'installed' || repo.status === 'install_failed') );

ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/userManagement/GroupsListCtrl.js

@@ -19,8 +19,8 @@
 
 angular.module('ambariAdminConsole')
 .controller('GroupsListCtrl',
-['$scope', 'Group', '$modal', 'ConfirmationModal', '$rootScope', '$translate', 'Settings', 'Cluster', 'View', 'Alert',
-function($scope, Group, $modal, ConfirmationModal, $rootScope, $translate, Settings, Cluster, View, Alert) {
+['$scope', 'Group', '$modal', 'ConfirmationModal', '$rootScope', '$translate', 'Settings', 'Cluster', 'View', 'Alert', 'Pagination', 'Filters',
+function($scope, Group, $modal, ConfirmationModal, $rootScope, $translate, Settings, Cluster, View, Alert, Pagination, Filters) {
   var $t = $translate.instant;
   $scope.constants = {
     groups: $t('common.groups').toLowerCase()
@@ -28,84 +28,64 @@ function($scope, Group, $modal, ConfirmationModal, $rootScope, $translate, Setti
   $scope.minRowsToShowPagination = Settings.minRowsToShowPagination;
   $scope.isLoading = false;
   $scope.groups = [];
-
-  $scope.groupsPerPage = 10;
-  $scope.currentPage = 1;
-  $scope.totalGroups = 0;
-  $scope.filter = {
-    name: '',
-    type: null
-  };
-  $scope.maxVisiblePages=20;
   $scope.tableInfo = {
+    filtered: 0,
     total: 0,
     showed: 0
   };
-  $scope.isNotEmptyFilter = true;
+  $scope.pagination = Pagination.create();
+  $scope.filters = [
+    {
+      key: 'group_name',
+      label: $t('groups.name'),
+      options: []
+    },
+    {
+      key: 'groupTypeName',
+      label: $t('common.type'),
+      options: []
+    }
+  ];
+
+  $scope.resetPagination = function() {
+    $scope.pagination.resetPagination($scope.groups, $scope.tableInfo);
+  };
 
   $scope.pageChanged = function() {
-    loadGroups();
+    $scope.pagination.pageChanged($scope.groups, $scope.tableInfo);
   };
-  $scope.groupsPerPageChanges = function() {
-    loadGroups();
+
+  $scope.filterGroups = function(appliedFilters) {
+    $scope.tableInfo.filtered = Filters.filterItems(appliedFilters, $scope.groups, $scope.filters);
+    $scope.pagination.resetPagination($scope.groups, $scope.tableInfo);
   };
 
-  $scope.resetPagination = function() {
-    $scope.currentPage = 1;
-    loadGroups();
+  $scope.toggleSearchBox = function() {
+    $('.search-box-button .popup-arrow-up, .search-box-row').toggleClass('hide');
   };
 
-  function loadGroups(){
+  $scope.loadGroups = function() {
     $scope.isLoading = true;
-    Group.all({
-      currentPage: $scope.currentPage, 
-      groupsPerPage: $scope.groupsPerPage, 
-      searchString: $scope.filter.name,
-      group_type: $scope.filter.type.value
-    }).then(function(groups) {
+    Group.all().then(function(groups) {
       $scope.isLoading = false;
-      $scope.totalGroups = groups.itemTotal;
       $scope.groups = groups.map(Group.makeGroup);
-      $scope.tableInfo.total = groups.itemTotal;
-      $scope.tableInfo.showed = groups.length;
+      $scope.tableInfo.total = $scope.groups.length;
+      Filters.initFilterOptions($scope.filters, $scope.groups);
+      $scope.filterGroups();
     })
     .catch(function(data) {
       Alert.error($t('groups.alerts.getGroupsListError'), data.data.message);
     });
-  }
-
-  $scope.typeFilterOptions = [{ label: $t('common.all'), value: '*'}]
-    .concat(Object.keys(Group.getTypes()).map(function(key) {
-      return {
-        label: $t(Group.getTypes()[key].LABEL_KEY),
-        value: Group.getTypes()[key].VALUE
-      };
-  }));
-  $scope.filter.type = $scope.typeFilterOptions[0];
-
-  $scope.clearFilters = function () {
-    $scope.filter.name = '';
-    $scope.filter.type = $scope.typeFilterOptions[0];
-    $scope.resetPagination();
   };
-  
-  loadGroups();
 
-  $scope.$watch(
-    function (scope) {
-      return Boolean(scope.filter.name || (scope.filter.type && scope.filter.type.value !== '*'));
-    },
-    function (newValue, oldValue, scope) {
-      scope.isNotEmptyFilter = newValue;
-    }
-  );
+  $scope.loadGroups();
 
   $rootScope.$watch(function(scope) {
     return scope.LDAPSynced;
   }, function(LDAPSynced) {
     if(LDAPSynced === true){
       $rootScope.LDAPSynced = false;
-      loadGroups();
+      $scope.loadGroups();
     }
   });
 
@@ -116,7 +96,7 @@ function($scope, Group, $modal, ConfirmationModal, $rootScope, $translate, Setti
       backdrop: 'static'
     });
 
-    modalInstance.result.catch(loadGroups);
+    modalInstance.result.finally($scope.loadGroups);
   };
 
   $scope.deleteGroup = function(group) {
@@ -158,7 +138,7 @@ function($scope, Group, $modal, ConfirmationModal, $rootScope, $translate, Setti
           angular.forEach(viewsPrivileges, function(privilege) {
             View.deletePrivilege(privilege);
           });
-          loadGroups();
+          $scope.loadGroups();
         });
       });
     });

ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/userManagement/UsersListCtrl.js

@@ -19,95 +19,105 @@
 
 angular.module('ambariAdminConsole')
 .controller('UsersListCtrl',
-['$scope', 'User', '$modal', '$rootScope', 'UserConstants', '$translate', 'Cluster', 'View', 'ConfirmationModal', 'Settings',
-function($scope, User, $modal, $rootScope, UserConstants, $translate, Cluster, View, ConfirmationModal, Settings) {
+['$scope', 'User', '$modal', '$rootScope', 'UserConstants', '$translate', 'Cluster', 'View', 'ConfirmationModal', 'Settings', 'Pagination', 'Filters',
+function($scope, User, $modal, $rootScope, UserConstants, $translate, Cluster, View, ConfirmationModal, Settings, Pagination, Filters) {
   var $t = $translate.instant;
   $scope.constants = {
     admin: $t('users.ambariAdmin'),
     users: $t('common.users').toLowerCase()
   };
+  $scope.users = [];
   $scope.minRowsToShowPagination = Settings.minRowsToShowPagination;
   $scope.isLoading = false;
-  $scope.users = [];
-  $scope.usersPerPage = 10;
-  $scope.currentPage = 1;
-  $scope.totalUsers = 0;
-  $scope.filters = {
-    name: '',
-    status: null,
-    type: null
-  };
-  $scope.maxVisiblePages = 20;
+  $scope.pagination = Pagination.create();
   $scope.tableInfo = {
+    filtered: 0,
     total: 0,
     showed: 0
   };
-  $scope.isNotEmptyFilter = true;
+  $scope.filters = [
+    {
+      key: 'user_name',
+      label: $t('users.username'),
+      customValueConverter: function(item) {
+        return item.Users.user_name;
+      },
+      options: []
+    },
+    {
+      key: 'role',
+      label: $t('clusters.role'),
+      customValueConverter: function(item) {
+        return item.Users.roles[0] ? item.Users.roles[0].permission_label : '';
+      },
+      options: []
+    },
+    {
+      key: 'status',
+      label: $t('users.status'),
+      isStatic: true,
+      customValueConverter: function(item) {
+        return item.Users.active ? $t('users.active') : $t('users.inactive');
+      },
+      options: [
+        {
+          key: $t('users.active'),
+          label: $t('users.active')
+        },
+        {
+          key: $t('users.inactive'),
+          label: $t('users.inactive')
+        }
+      ]
+    },
+    {
+      key: 'type',
+      label: $t('common.type'),
+      customValueConverter: function(item) {
+        return item.Users.userTypeName;
+      },
+      options: []
+    },
+    {
+      key: 'group',
+      label: $t('common.group'),
+      isMultiple: true,
+      customValueConverter: function(item) {
+        return item.Users.groups;
+      },
+      options: []
+    }
+  ];
 
   function loadUsers() {
     $scope.isLoading = true;
-    User.list({
-      currentPage: $scope.currentPage,
-      usersPerPage: $scope.usersPerPage,
-      searchString: $scope.filters.name,
-      user_type: $scope.filters.type.value,
-      active: $scope.filters.status.value
-    }).then(function (data) {
-      $scope.totalUsers = data.data.itemTotal;
+    User.list().then(function (data) {
       $scope.users = data.data.items.map(User.makeUser);
-      $scope.tableInfo.showed = data.data.items.length;
-      $scope.tableInfo.total = data.data.itemTotal;
+      $scope.tableInfo.total = $scope.users.length;
+      $scope.filterUsers();
+      Filters.initFilterOptions($scope.filters, $scope.users);
     }).finally(function () {
       $scope.isLoading = false;
     });
   }
 
-  $scope.pageChanged = function () {
-    loadUsers();
+  $scope.toggleSearchBox = function() {
+    $('.search-box-button .popup-arrow-up, .search-box-row').toggleClass('hide');
   };
-  $scope.usersPerPageChanges = function () {
-    $scope.resetPagination();
+
+  $scope.pageChanged = function () {
+    $scope.pagination.pageChanged($scope.users, $scope.tableInfo);
   };
 
   $scope.resetPagination = function () {
-    $scope.currentPage = 1;
-    loadUsers();
+    $scope.pagination.resetPagination($scope.users, $scope.tableInfo);
   };
 
-  $scope.activeFilterOptions = [
-    {label: $t('common.all'), value: '*'},
-    {label: $t('users.active'), value: true},
-    {label: $t('users.inactive'), value: false}
-  ];
-  $scope.filters.status = $scope.activeFilterOptions[0];
-
-  $scope.typeFilterOptions = [{label: $t('common.all'), value: '*'}]
-    .concat(Object.keys(UserConstants.TYPES).map(function (key) {
-      return {
-        label: $t(UserConstants.TYPES[key].LABEL_KEY),
-        value: UserConstants.TYPES[key].VALUE
-      };
-    }));
-
-  $scope.filters.type = $scope.typeFilterOptions[0];
-
-  $scope.clearFilters = function () {
-    $scope.filters.name = '';
-    $scope.filters.type = $scope.typeFilterOptions[0];
-    $scope.filters.status = $scope.activeFilterOptions[0];
-    $scope.resetPagination();
+  $scope.filterUsers = function(appliedFilters) {
+    $scope.tableInfo.filtered = Filters.filterItems(appliedFilters, $scope.users, $scope.filters);
+    $scope.pagination.resetPagination($scope.users, $scope.tableInfo);
   };
 
-  $scope.$watch(
-    function (scope) {
-      return Boolean(scope.filters.name || (scope.filters.status && scope.filters.status.value !== '*')
-        || (scope.filters.type && scope.filters.type.value !== '*'));
-    },
-    function (newValue, oldValue, scope) {
-      scope.isNotEmptyFilter = newValue;
-    }
-  );
-
   $rootScope.$watch(function (scope) {
     return scope.LDAPSynced;
   }, function (LDAPSynced) {

ambari-admin/src/main/resources/ui/admin-web/app/scripts/directives/comboSearch.js

@@ -0,0 +1,501 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+'use strict';
+
+
+/**
+ *  Example:
+ *  <combo-search suggestions="filters"
+ *                filter-change="filterItems"
+ *                placeholder="Search"
+ *                supportCategories="true">
+ *  </combo-search>
+ *
+ *  filters = [
+ *    {
+ *      key: 'property1',
+ *      label: $t('propertyLabel'),
+ *      category: 'category1'
+ *      options: []
+ *    }
+ *  ]
+ *  Note: "category" field is optional, should be used only when supportCategories="true"
+ *
+ */
+
+angular.module('ambariAdminConsole')
+.directive('comboSearch', function() {
+  return {
+    restrict: 'E',
+    templateUrl: 'views/directives/comboSearch.html',
+    scope: {
+      suggestions: '=',
+      filterChange: '=',
+      placeholder: '@',
+      supportCategories: '@'
+    },
+    controller: ['$scope', function($scope) {
+      return {
+        suggestions: $scope.suggestions,
+        placeholder: $scope.placeholder,
+        filterChange: $scope.filterChange,
+        supportCategories: $scope.supportCategories === "true"
+      }
+    }],
+    link: function($scope, $elem, $attr, $ctrl) {
+      var idCounter = 1;
+      var suggestions = $ctrl.suggestions;
+      var supportCategories = $ctrl.supportCategories;
+      var mainInputElement = $elem.find('.main-input.combo-search-input');
+      $scope.placeholder = $ctrl.placeholder;
+      $scope.searchFilterInput = '';
+      $scope.filterSuggestions = [];
+      $scope.showAutoComplete = false;
+      $scope.appliedFilters = [];
+
+      attachInputWidthSetter(mainInputElement);
+      initKeyHandlers();
+      initBlurHandler();
+
+      $scope.$watch(function () {
+        return $scope.appliedFilters.length;
+      }, function () {
+        attachInputWidthSetter($elem.find('.combo-search-input'));
+      });
+
+      $scope.removeFilter = function(filter) {
+        $scope.appliedFilters = $scope.appliedFilters.filter(function(item) {
+          return filter.id !== item.id;
+        });
+        $scope.observeSearchFilterInput(event);
+        mainInputElement.focus();
+        $scope.updateFilters($scope.appliedFilters);
+      };
+
+      $scope.clearFilters = function() {
+        $scope.appliedFilters = [];
+        $scope.updateFilters($scope.appliedFilters);
+      };
+
+      $scope.selectFilter = function(filter, event) {
+        var newAppliedFilter = {
+          id: 'filter_' + idCounter++,
+          currentOption: null,
+          filteredOptions: [],
+          searchOptionInput: '',
+          key: filter.key,
+          label: filter.label,
+          options: filter.options || [],
+          showAutoComplete: false
+        };
+        $scope.appliedFilters.push(newAppliedFilter);
+        if (event) {
+          event.stopPropagation();
+          event.preventDefault();
+        }
+        $scope.isEditing = false;
+        $scope.showAutoComplete = false;
+        $scope.searchFilterInput = '';
+        _.debounce(function() {
+          $('input[name=' + newAppliedFilter.id + ']').focus().width(4);
+        }, 100)();
+      };
+
+      $scope.selectOption = function(event, option, filter) {
+        $('input[name=' + filter.id + ']').val(option.label).trigger('input');
+        filter.showAutoComplete = false;
+        mainInputElement.focus();
+        $scope.observeSearchFilterInput(event);
+        filter.currentOption = option;
+        $scope.updateFilters($scope.appliedFilters);
+      };
+
+      $scope.hideAutocomplete = function(filter) {
+        _.debounce(function() {
+          if (filter) {
+            filter.showAutoComplete = false;
+          } else {
+            if (!$scope.isEditing) {
+              $scope.showAutoComplete = false;
+            }
+          }
+          $scope.$apply();
+        }, 100)();
+      };
+
+      $scope.forceFocus = function(event, filter) {
+        $(event.currentTarget).find('.combo-search-input').focus();
+        $scope.showAutoComplete = false;
+        $scope.observeSearchOptionInput(filter);
+        event.stopPropagation();
+        event.preventDefault();
+      };
+
+      $scope.makeActive = function(active, all) {
+        if (active.isCategory) {
+          return false;
+        }
+        all.forEach(function(item) {
+          item.active = active.key === item.key;
+        });
+      };
+
+      $scope.observeSearchFilterInput = function(event) {
+        if (event) {
+          mainInputElement.focus();
+          $scope.isEditing = true;
+          event.stopPropagation();
+          event.preventDefault();
+        }
+
+        var filteredSuggestions = suggestions.filter(function(item) {
+          return (!$scope.searchFilterInput || item.label.toLowerCase().indexOf($scope.searchFilterInput.toLowerCase()) !== -1);
+        });
+        if (filteredSuggestions.length > 0) {
+          $scope.makeActive(filteredSuggestions[0], filteredSuggestions);
+          $scope.showAutoComplete = true;
+        } else {
+          $scope.showAutoComplete = false;
+        }
+        $scope.filterSuggestions = supportCategories ? formatCategorySuggestions(filteredSuggestions) : filteredSuggestions;
+      };
+
+      $scope.observeSearchOptionInput = function(filter) {
+        var appliedOptions = {};
+        $scope.appliedFilters.forEach(function(item) {
+          if (item.key === filter.key && item.currentOption) {
+            appliedOptions[item.currentOption.key] = true;
+          }
+        });
+
+        if (filter.currentOption && filter.currentOption.key !== filter.searchOptionInput) {
+          filter.currentOption = null;
+        }
+        filter.filteredOptions = filter.options.filter(function(option) {
+          return !(option.key === '' || option.key === undefined || appliedOptions[option.key])
+            && (!filter.searchOptionInput || option.label.toLowerCase().indexOf(filter.searchOptionInput.toLowerCase()) !== -1);
+        });
+        resetActive(filter.filteredOptions);
+        filter.showAutoComplete = filter.filteredOptions.length > 0;
+      };
+
+      $scope.extractFilters = function(filters) {
+        var map = {};
+        var result = [];
+
+        filters.forEach(function(filter) {
+          if (filter.currentOption) {
+            if (!map[filter.key]) {
+              map[filter.key] = [];
+            }
+            map[filter.key].push(filter.currentOption.key);
+          }
+        });
+        for(var key in map) {
+          result.push({
+            key: key,
+            values: map[key]
+          });
+        }
+        return result;
+      };
+
+      $scope.updateFilters = function(appliedFilters) {
+        $ctrl.filterChange($scope.extractFilters(appliedFilters));
+      };
+
+      function formatCategorySuggestions(suggestions) {
+        var categories = {};
+        var result = [];
+        suggestions.forEach(function(item) {
+          if (!item.category) {
+            item.category = 'default';
+          }
+          if (!categories[item.category]) {
+            categories[item.category] = [];
+          }
+          categories[item.category].push(item);
+        });
+
+        for(var cat in categories) {
+          result.push({
+            key: cat,
+            label: cat,
+            isCategory: true,
+            isDefault: cat === 'default'
+          });
+          result = result.concat(categories[cat]);
+        }
+        return result;
+      }
+
+      function initBlurHandler() {
+        $(document).click(function() {
+          $scope.isEditing = false;
+          $scope.hideAutocomplete();
+        });
+      }
+
+      function findActiveByName(array, name) {
+        for (var i = 0; i < array.length; i++) {
+          if (array[i].id === name) {
+            return i;
+          }
+        }
+        return null;
+      }
+
+      function findActiveByProperty(array) {
+        for (var i = 0; i < array.length; i++) {
+          if (array[i].active) {
+            return i;
+          }
+        }
+        return -1;
+      }
+
+      function resetActive(array) {
+        array.forEach(function(item) {
+          item.active = false;
+        });
+      }
+
+      function focusInput(filter) {
+        $('input[name=' + filter.id + ']').focus();
+        $scope.showAutoComplete = false;
+        $scope.observeSearchOptionInput(filter);
+      }
+
+      function initKeyHandlers() {
+        $($elem).keydown(function(event) {
+          if (event.which === 13) { // "Enter" key
+            enterKeyHandler();
+            $scope.$apply();
+          }
+          if (event.which === 8) { // "Backspace" key
+            backspaceKeyHandler(event);
+            $scope.$apply();
+          }
+          if (event.which === 38) { // "Up" key
+            upKeyHandler();
+            $scope.$apply();
+          }
+          if (event.which === 40) { // "Down" key
+            downKeyHandler();
+            $scope.$apply();
+          }
+          if (event.which === 39) { // "Right Arrow" key
+            rightArrowKeyHandler();
+            $scope.$apply();
+          }
+          if (event.which === 37) { // "Left Arrow" key
+            leftArrowKeyHandler();
+            $scope.$apply();
+          }
+          if (event.which === 27) { // "Escape" key
+            $scope.showAutoComplete = false;
+            $scope.$apply();
+          }
+        });
+      }
+
+      function leftArrowKeyHandler() {
+        var activeElement = $(document.activeElement);
+        if (activeElement.is('input') && activeElement[0].selectionStart === 0 && $scope.appliedFilters.length > 0) {
+          if (activeElement.hasClass('main-input')) {
+            focusInput($scope.appliedFilters[$scope.appliedFilters.length - 1]);
+          } else {
+            var activeIndex = findActiveByName($scope.appliedFilters, activeElement.attr('name'));
+            if (activeIndex !== null && activeIndex > 0) {
+              focusInput($scope.appliedFilters[activeIndex - 1]);
+            }
+          }
+        }
+      }
+
+      function rightArrowKeyHandler() {
+        var activeElement = $(document.activeElement);
+        if (activeElement.is('input') && activeElement[0].selectionStart === activeElement.val().length) {
+          if (!activeElement.hasClass('main-input')) {
+            var activeIndex = findActiveByName($scope.appliedFilters, activeElement.attr('name'));
+            if (activeIndex !== null) {
+              if (activeIndex === $scope.appliedFilters.length - 1) {
+                mainInputElement.focus();
+                $scope.observeSearchFilterInput();
+              } else {
+                focusInput($scope.appliedFilters[activeIndex + 1]);
+              }
+            }
+          }
+        }
+      }
+
+      function downKeyHandler() {
+        var activeIndex = 0;
+        var nextIndex = null;
+
+        if ($scope.showAutoComplete) {
+          activeIndex = findActiveByProperty($scope.filterSuggestions);
+          if (activeIndex < $scope.filterSuggestions.length - 1) {
+            if ($scope.filterSuggestions[activeIndex + 1].isCategory && activeIndex + 2 < $scope.filterSuggestions.length) {
+              nextIndex = activeIndex + 2;
+            } else {
+              nextIndex = activeIndex + 1;
+            }
+          } else {
+            nextIndex = ($scope.filterSuggestions[0].isCategory) ? 1 : 0;
+          }
+          if (nextIndex !== null) {
+            $scope.makeActive($scope.filterSuggestions[nextIndex], $scope.filterSuggestions);
+          }
+        } else {
+          var activeAppliedFilters = $scope.appliedFilters.filter(function(item) {
+            return item.showAutoComplete;
+          });
+          if (activeAppliedFilters.length > 0) {
+            var filteredOptions = activeAppliedFilters[0].filteredOptions;
+            activeIndex = findActiveByProperty(filteredOptions);
+            if (activeIndex < filteredOptions.length - 1) {
+              nextIndex = activeIndex + 1;
+            } else {
+              //switch to input of option
+              nextIndex = null;
+              resetActive(filteredOptions);
+              focusInput(activeAppliedFilters[0]);
+            }
+          }
+          if (nextIndex !== null) {
+            $scope.makeActive(filteredOptions[nextIndex], filteredOptions);
+          }
+        }
+      }
+
+      function upKeyHandler() {
+        var activeIndex = 0;
+        var nextIndex = null;
+
+        if ($scope.showAutoComplete) {
+          activeIndex = findActiveByProperty($scope.filterSuggestions);
+          if (activeIndex > 0) {
+            if ($scope.filterSuggestions[activeIndex - 1].isCategory) {
+              nextIndex = (activeIndex - 2 > 0) ? activeIndex - 2 : $scope.filterSuggestions.length - 1;
+            } else {
+              nextIndex = activeIndex - 1;
+            }
+          } else {
+            nextIndex = $scope.filterSuggestions.length - 1;
+          }
+          if (nextIndex !== null) {
+            $scope.makeActive($scope.filterSuggestions[nextIndex], $scope.filterSuggestions);
+          }
+        } else {
+          var activeAppliedFilters = $scope.appliedFilters.filter(function(item) {
+            return item.showAutoComplete;
+          });
+          if (activeAppliedFilters.length > 0) {
+            var filteredOptions = activeAppliedFilters[0].filteredOptions;
+            activeIndex = findActiveByProperty(filteredOptions);
+            if (activeIndex > 0) {
+              nextIndex = activeIndex - 1;
+            } else if (activeIndex === 0) {
+              //switch to input of option
+              nextIndex = null;
+              resetActive(filteredOptions);
+              focusInput(activeAppliedFilters[0]);
+            } else {
+              nextIndex = filteredOptions.length - 1;
+            }
+          }
+          if (nextIndex !== null) {
+            $scope.makeActive(filteredOptions[nextIndex], filteredOptions);
+          }
+        }
+      }
+
+      function enterKeyHandler() {
+        if ($scope.showAutoComplete) {
+          var activeFilters = $scope.filterSuggestions.filter(function(item) {
+            return item.active;
+          });
+          if (activeFilters.length > 0) {
+            $scope.selectFilter(activeFilters[0]);
+          }
+        } else {
+          var activeAppliedFilters = $scope.appliedFilters.filter(function(item) {
+            return item.showAutoComplete;
+          });
+          if (activeAppliedFilters.length > 0) {
+            var activeOptions = activeAppliedFilters[0].filteredOptions.filter(function(item) {
+              return item.active;
+            });
+            if (activeOptions.length > 0) {
+              $scope.selectOption(null, activeOptions[0], activeAppliedFilters[0]);
+            }
+          }
+          if (activeAppliedFilters.length === 0 || activeOptions.length === 0) {
+            $scope.appliedFilters.filter(function(item) {
+              return !item.currentOption;
+            }).forEach(function(item) {
+              if (item.searchOptionInput !== '') {
+                $scope.selectOption(null, {
+                  key: item.searchOptionInput,
+                  label: item.searchOptionInput
+                }, item);
+              }
+            });
+          }
+        }
+      }
+
+      function backspaceKeyHandler (event) {
+        if ($(document.activeElement).is('input') && $(document.activeElement)[0].selectionStart === 0) {
+          if ($(document.activeElement).hasClass('main-input') && $scope.appliedFilters.length > 0) {
+            var lastFilter = $scope.appliedFilters[$scope.appliedFilters.length - 1];
+            focusInput(lastFilter);
+            event.stopPropagation();
+            event.preventDefault();
+          } else {
+            var name = $(document.activeElement).attr('name');
+            var activeFilter = $scope.appliedFilters.filter(function(item) {
+              return name === item.id;
+            })[0];
+            if (activeFilter) {
+              $scope.removeFilter(activeFilter);
+            }
+          }
+        }
+      }
+
+      function attachInputWidthSetter(element) {
+        var textPadding = 4;
+        element.on('input', function() {
+          var inputWidth = $(this).textWidth();
+          $(this).css({
+            width: inputWidth + textPadding
+          })
+        }).trigger('input');
+      }
+    }
+  };
+});
+
+$.fn.textWidth = function(text, font) {
+  if (!$.fn.textWidth.fakeEl) $.fn.textWidth.fakeEl = $('<span>').hide().appendTo(document.body);
+  $.fn.textWidth.fakeEl.text(text || this.val() || this.text() || this.attr('placeholder')).css('font', font || this.css('font'));
+  return $.fn.textWidth.fakeEl.width();
+};

ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Cluster.js

@@ -292,8 +292,7 @@ angular.module('ambariAdminConsole')
           angular.forEach(hostStatus, function(status) {
             totalHosts += status.length;
           });
-          response.status = currentHosts > 0? 'current' :
-                            installedHosts > 0? 'installed' : '';
+          response.status = data[0].ClusterStackVersions.state;
           response.currentHosts = currentHosts;
           response.installedHosts = installedHosts;
           response.totalHosts = totalHosts;

ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Filters.js

@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+'use strict';
+
+angular.module('ambariAdminConsole')
+.factory('Filters', function() {
+
+  function initFilterOptions(filters, items) {
+    filters.filter(function(filter) {
+      return !filter.isStatic;
+    }).forEach(function(filter) {
+      var preOptions = [];
+      if (filter.isMultiple) {
+        items.forEach(function(item) {
+          if (typeof filter.customValueConverter === 'function') {
+            preOptions = preOptions.concat(filter.customValueConverter(item));
+          } else {
+            preOptions = preOptions.concat(item[filter.key]);
+          }
+        });
+      } else {
+        preOptions = items.map(function(item) {
+          if (typeof filter.customValueConverter === 'function') {
+            return filter.customValueConverter(item);
+          }
+          return item[filter.key];
+        });
+      }
+      filter.options = $.unique(preOptions).filter(function(item) {
+        return item !== undefined && item !== null;
+      }).map(function(item) {
+        return {
+          key: item,
+          label: item
+        }
+      });
+    });
+  }
+
+  function filterItems(appliedFilters, items, filterDefinitions) {
+    var filteredCount = 0;
+    angular.forEach(items, function(item) {
+      item.isFiltered = !(appliedFilters && appliedFilters.length > 0 && appliedFilters.some(function(filter) {
+        var customValueFilter = filterDefinitions.filter(function(filterDefinition) {
+          return filterDefinition.key === filter.key && typeof filterDefinition.customValueConverter === 'function';
+        })[0];
+        if (customValueFilter) {
+          return filter.values.every(function(value) {
+            var itemValue = customValueFilter.customValueConverter(item);
+            return String(Array.isArray(itemValue) ? itemValue.join() : itemValue).indexOf(value) === -1;
+          });
+        }
+        return filter.values.every(function(value) {
+          var itemValue = item[filter.key];
+          return String(Array.isArray(itemValue) ? itemValue.join() : itemValue).indexOf(value) === -1;
+
+        });
+      }));
+
+      filteredCount += ~~item.isFiltered;
+    });
+    return filteredCount;
+  }
+
+  return {
+    initFilterOptions: initFilterOptions,
+    filterItems: filterItems
+  };
+});

ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Group.js

@@ -100,18 +100,11 @@ angular.module('ambariAdminConsole')
     return $http.post(Settings.baseUrl + '/groups/' + groupName + '/members/'+memberName);
   };
 
-  Group.all = function(params) {
+  Group.all = function() {
     var deferred = $q.defer();
 
-    $http.get(Settings.baseUrl + '/groups?'
-      + 'Groups/group_name.matches(.*'+params.searchString+'.*)'
-      + '&fields=*'
-      + '&from='+ (params.currentPage-1)*params.groupsPerPage
-      + '&page_size=' + params.groupsPerPage
-      + (params.group_type === '*' ? '' : '&Groups/group_type=' + params.group_type)
-    )
+    $http.get(Settings.baseUrl + '/groups?fields=*')
     .success(function(data) {
-      data.items.itemTotal = data.itemTotal;
       deferred.resolve(data.items);
     })
     .error(function(data) {

ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Pagination.js

@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+'use strict';
+
+angular.module('ambariAdminConsole')
+.factory('Pagination', function() {
+
+  function showItemsOnPage(items, tableInfo) {
+    var startIndex = (this.currentPage - 1) * this.itemsPerPage + 1;
+    var endIndex = this.currentPage * this.itemsPerPage;
+    var showedCount = 0;
+    var filteredCount = 0;
+
+    angular.forEach(items, function (item) {
+      item.isShowed = false;
+      if (item.isFiltered) {
+        filteredCount++;
+        if (filteredCount >= startIndex && filteredCount <= endIndex) {
+          item.isShowed = true;
+          showedCount++;
+        }
+      }
+    });
+    tableInfo.showed = showedCount;
+  }
+
+  return {
+    create: function(options) {
+      options = options || {};
+      return {
+        itemsPerPage: options.itemsPerPage || 10,
+        currentPage: options.currentPage || 1,
+        maxVisiblePages: options.maxVisiblePages || 10,
+        pageChanged: function(items, tableInfo) {
+          showItemsOnPage.call(this, items, tableInfo);
+        },
+        resetPagination: function(items, tableInfo) {
+          this.currentPage = 1;
+          showItemsOnPage.call(this, items, tableInfo);
+        }
+      }
+    }
+  };
+});

ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/RemoteCluster.js

@@ -80,16 +80,10 @@ angular.module('ambariAdminConsole')
         return deferred.promise;
     }
 
-    RemoteCluster.all = function(params) {
+    RemoteCluster.all = function() {
       var deferred = $q.defer();
 
-      $http.get(Settings.baseUrl + "/remoteclusters?"
-          + 'ClusterInfo/name.matches(.*'+params.searchString+'.*)'
-          + '&fields=*'
-          + '&from='+ (params.currentPage-1)*params.groupsPerPage
-          + '&page_size=' + params.groupsPerPage
-          + (params.service === 'Any' ? '' : '&ClusterInfo/services.matches(.*'+params.service+'.*)')
-        )
+      $http.get(Settings.baseUrl + "/remoteclusters")
         .success(function(response) {
           deferred.resolve(response);
         })

ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Stack.js

@@ -92,7 +92,7 @@ angular.module('ambariAdminConsole')
 
       $http.get(Settings.baseUrl + '/services/AMBARI/components/AMBARI_SERVER?fields=RootServiceComponents/properties/gpl.license.accepted&minimal_response=true', {mock: 'true'})
         .then(function(data) {
-          deferred.resolve(data.data.RootServiceComponents.properties['gpl.license.accepted']);
+          deferred.resolve(data.data.RootServiceComponents.properties && data.data.RootServiceComponents.properties['gpl.license.accepted']);
         })
         .catch(function(data) {
           deferred.reject(data);
@@ -189,28 +189,8 @@ angular.module('ambariAdminConsole')
       }
     },
 
-    allRepos: function (filter, pagination) {
-      var versionFilter = filter.version;
-      var nameFilter = filter.name;
-      var typeFilter = filter.type;
-      var stackFilter = filter.stack && filter.stack.current && filter.stack.current.value;
+    allRepos: function () {
       var url = '/stacks?fields=versions/repository_versions/RepositoryVersions';
-      if (versionFilter) {
-        url += '&versions/repository_versions/RepositoryVersions/repository_version.matches(.*' + versionFilter + '.*)';
-      }
-      if (nameFilter) {
-        url += '&versions/repository_versions/RepositoryVersions/display_name.matches(.*' + nameFilter + '.*)';
-      }
-      if (typeFilter){
-        url += '&versions/repository_versions/RepositoryVersions/type.matches(.*' + typeFilter.toUpperCase() + '.*)';
-      }
-      if (stackFilter) {
-        var stack = filter.stack.current.value.split('-'),
-          stackNameFilter = stack[0],
-          stackVersionFilter = stack[1];
-        url += '&versions/repository_versions/RepositoryVersions/stack_name=' + stackNameFilter;
-        url += '&versions/repository_versions/RepositoryVersions/stack_version=' + stackVersionFilter;
-      }
       var deferred = $q.defer();
       $http.get(Settings.baseUrl + url, {mock: 'version/versions.json'})
       .success(function (data) {
@@ -230,16 +210,8 @@ angular.module('ambariAdminConsole')
         });
         // prepare response data with client side pagination
         var response = {};
+        response.items = repos;
         response.itemTotal = repos.length;
-        if (pagination) {
-          var from = (pagination.currentPage - 1) * pagination.itemsPerPage;
-          var to = (repos.length - from > pagination.itemsPerPage)? from + pagination.itemsPerPage : repos.length;
-          response.items = repos.slice(from, to);
-          response.showed = to - from;
-        } else {
-          response.items = repos;
-          response.showed = repos.length;
-        }
         deferred.resolve(response);
       })
       .error(function (data) {

ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/User.js

@@ -33,17 +33,8 @@ angular.module('ambariAdminConsole')
   var $t = $translate.instant;
 
   return {
-    list: function(params) {
-      return $http.get(
-        Settings.baseUrl + '/users/?'
-        + 'Users/user_name.matches(.*'+params.searchString+'.*)'
-        + '&fields=privileges/PrivilegeInfo/*,Users'
-        + '&from=' + (params.currentPage-1)*params.usersPerPage
-        + '&page_size=' + params.usersPerPage
-        + (params.user_type === '*' ? '' : '&Users/user_type=' + params.user_type)
-        + (params.active === '*' ? '' : '&Users/active=' + params.active)
-        + (params.admin ? '&Users/admin=true' : '')
-      );
+    list: function() {
+      return $http.get(Settings.baseUrl + '/users?fields=Users/*,privileges/*');
     },
     listByName: function(name) {
       return $http.get(
@@ -90,6 +81,15 @@ angular.module('ambariAdminConsole')
         }
       });
     },
+    resetLoginFailures: function(userId) {
+      return $http({
+        method: 'PUT',
+        url: Settings.baseUrl + '/users/' + userId,
+        data: {
+          'Users/consecutive_failures': 0
+        }
+      });
+    },
     /**
      * Generate user info to display by response data from API.
      * Generally this is a single point to manage all required and useful data

ambari-admin/src/main/resources/ui/admin-web/app/styles/cluster-information.css

@@ -52,7 +52,30 @@
   padding: 30px;
 }
 
-#cluster-information .fa-cloud {
-  font-size: 150px;
-  color: #f0f0f0;
+#cluster-information .install-button {
+  height: 40px;
+}
+
+#cluster-information .install-button a {
+  height: 100%;
+  padding: 0;
+  width: 225px;
+  font-size: 16px;
+  line-height: 40px;
+}
+
+@keyframes INSTALL-BOX-ROTATE {
+  0% {transform: rotate(0deg)}
+  12.5% {transform: rotate(-30deg)}
+  37.5% {transform: rotate(30deg)}
+  62.5% {transform: rotate(-30deg)}
+  87.5% {transform: rotate(30deg)}
+  100% {transform: rotate(0deg)}
+}
+
+#cluster-information #install-box {
+  animation: INSTALL-BOX-ROTATE 2s;
+  width: 116px;
+  margin-bottom: 20px;
+  margin-top: 40px;
 }

ambari-admin/src/main/resources/ui/admin-web/app/styles/combo-search.css

@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.combo-search .combo-search-inner {
+  position: relative;
+  border: 1px solid #ccc;
+  border-radius: 4px;
+  box-shadow: 0 0 1px #fff inset;
+  min-height: 34px;
+  line-height: 1em;
+  cursor: text;
+  padding-right: 30px;
+}
+
+.combo-search .combo-search-close {
+  font-size: 13px;
+  color: #999;
+  position: absolute;
+  right: 10px;
+  top: 30%;
+  cursor: pointer;
+}
+.combo-search .combo-search-close:hover {
+  color: #333;
+}
+.combo-search .combo-search-input {
+  background: transparent;
+  display: inline-block;
+  min-width: 4px;
+  width: 4px;
+  line-height: 10px;
+  height: 100%;
+  border: none;
+  outline: none;
+  margin-left: 1px;
+}
+
+.combo-search .combo-search-input-wrapper {
+  display: inline-block;
+  position: relative;
+  height: 32px;
+  margin-left: 5px;
+}
+
+.combo-search .combo-search-content {
+  display: inline-block;
+}
+
+.combo-search .combo-search-dropdown {
+  position: absolute;
+  border: 1px solid #ccc;
+  border-radius: 4px;
+  background-color: #fff;
+  cursor: pointer;
+  z-index: 10;
+  padding: 0;
+  margin: 0;
+  width: auto;
+  min-width: 80px;
+  max-width: 220px;
+  max-height: 240px;
+  overflow-y: auto;
+  overflow-x: hidden;
+  font-size: 13px;
+  top: 30px;
+  left: 5px;
+  box-shadow: 3px 4px 5px -2px rgba(0, 0, 0, 0.5);
+}
+
+.combo-search .combo-search-dropdown ul {
+  max-height: 250px;
+  list-style: none;
+  padding: 0;
+  margin: 0;
+}
+
+.combo-search .filter a {
+  display: block;
+  width: auto;
+  text-decoration: none;
+  color: initial;
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  background: none;
+  border: none;
+  padding: 3px 10px 5px 5px;
+}
+
+.combo-search .category a {
+  display: block;
+  width: auto;
+  text-decoration: none;
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  padding: 3px 10px 5px 5px;
+  text-transform: capitalize;
+  font-size: 11px;
+  font-weight: bold;
+  color: white;
+  cursor: default;
+  border-bottom: 1px solid #A2A2A2;
+  background-color: #B7B7B7;
+  text-shadow: 0 -1px 0 #999;
+}
+
+.combo-search .filter a.active {
+  background-color: #ddd;
+}
+
+.combo-search .combo-search-applied-filter {
+  position: relative;
+  display: inline-block;
+  padding: 0 3px 0 18px;
+  background-color: #dddddd;
+  border-radius: 4px;
+  margin: 4px 0 0 4px;
+  vertical-align: top;
+  border: 1px solid #d2d2d2;
+}
+
+.combo-search .combo-search-applied-filter i {
+  position: absolute;
+  left: 5px;
+  font-size: 12px;
+  top: 5px;
+  color: #999;
+  cursor: pointer;
+}
+
+.combo-search .combo-search-applied-filter i:hover {
+  color: #333;
+}
+
+.combo-search .combo-search-applied-filter span,
+.combo-search .combo-search-applied-filter input {
+  color: #666;
+  font-size: 11px;
+}
+
+.combo-search .combo-search-applied-filter span {
+  font-weight: bold;
+}
+
+.combo-search .combo-search-applied-filter .combo-search-input-wrapper {
+  height: 22px;
+  margin-left: 0;
+}

ambari-admin/src/main/resources/ui/admin-web/app/styles/main.css

@@ -18,6 +18,15 @@
 
 
 
+/**
+ * When this css rule is loaded by the browser, all html elements (including their children)
+ * that are tagged with the ngCloak directive are hidden. When AngularJS encounters this directive
+ * during the compilation of the template it deletes the ngCloak element attribute, making the compiled element visible.
+ */
+[ng\:cloak], [ng-cloak], [data-ng-cloak], [x-ng-cloak], .ng-cloak, .x-ng-cloak {
+  display: none !important;
+}
+
 /*
   ------ START editable-list DIRECTIVE SECTION ------ -
 */
@@ -1331,3 +1340,39 @@ th.entity-actions {
 .entity-actions a:focus:hover {
   text-decoration: none;
 }
+
+.search-box-button {
+  position: relative;
+  margin-right: 5px;
+}
+
+.search-box-button .btn {
+  padding: 10px;
+}
+
+.search-box-row {
+  padding-top: 15px;
+  padding-bottom: 5px;
+}
+
+.popup-arrow-up {
+  background: inherit;
+  z-index: 1;
+  left: 6px;
+  position: absolute;
+  width: 24px;
+  height: 16px;
+  overflow: hidden;
+}
+
+.popup-arrow-up:after {
+  content: "";
+  position: absolute;
+  width: 20px;
+  height: 20px;
+  background: #fff;
+  transform: rotate(45deg);
+  top: 10px;
+  left: 2px;
+  box-shadow: -2px -2px 10px -3px rgba(0, 0, 0, 0.5);
+}

ambari-admin/src/main/resources/ui/admin-web/app/styles/user-management.css

@@ -20,6 +20,10 @@
   vertical-align: baseline;
 }
 
+#user-management .search-box-row {
+  margin-top: -1px;
+}
+
 #user-management .nav.nav-tabs {
   margin-bottom: 0;
 }

ambari-admin/src/main/resources/ui/admin-web/app/views/ambariViews/viewsList.html

@@ -24,11 +24,21 @@
                 {{'views.create' | translate}}
             </button>
         </div>
+        <div class="search-box-button pull-right">
+            <button class="btn btn-default" ng-click="toggleSearchBox()">
+                <i class="fa fa-filter" aria-hidden="true"></i>
+            </button>
+            <div class="popup-arrow-up hide"></div>
+        </div>
+    </div>
+
+    <div class="search-box-row hide">
+        <combo-search suggestions="filters" filter-change="filterInstances" placeholder="Search"></combo-search>
     </div>
 
     <table class="table table-striped table-hover">
         <thead>
-        <tr class="fix-bottom">
+        <tr>
             <th class="col-md-2">
                 <span>{{'common.name' | translate}}</span>
             </th>
@@ -45,39 +55,6 @@
                 <span>{{'common.actions' | translate}}</span>
             </th>
         </tr>
-        <tr class="fix-top">
-            <th>
-                <div class="search-container">
-                    <input type="text" class="form-control" placeholder="{{'common.any' | translate}}"
-                           ng-model="instanceNameFilter" ng-change="filterInstances()">
-                    <button type="button" class="close clearfilter" ng-show="instanceNameFilter"
-                            ng-click="instanceNameFilter=''; filterInstances()">
-                        <span aria-hidden="true">&times;</span>
-                        <span class="sr-only">{{'common.controls.close' | translate}}</span>
-                    </button>
-                </div>
-            </th>
-            <th>
-                <div class="search-container">
-                    <input type="text" class="form-control" placeholder="{{'common.any' | translate}}"
-                           ng-model="instanceUrlFilter" ng-change="filterInstances()">
-                    <button type="button" class="close clearfilter" ng-show="instanceUrlFilter"
-                            ng-click="instanceUrlFilter=''; filterInstances()">
-                        <span aria-hidden="true">&times;</span>
-                        <span class="sr-only">{{'common.controls.close' | translate}}</span>
-                    </button>
-                </div>
-            </th>
-            <th>
-                <select class="form-control typefilter v-small-input"
-                        ng-model="instanceTypeFilter"
-                        ng-options="item.label for item in typeFilterOptions"
-                        ng-change="filterInstances()">
-                </select>
-            </th>
-            <th></th>
-            <th></th>
-        </tr>
         </thead>
 
         <tbody>
@@ -138,13 +115,12 @@
     <div class="col-sm-12 table-bar" ng-show="instances.length >= minInstanceForPagination">
         <div class="pull-left filtered-info">
             <span>{{'common.filterInfo' | translate:{showed: tableInfo.showed, total: tableInfo.filtered, term: urs.urls} }}</span>
-            <span ng-show="isNotEmptyFilter">- <a href ng-click="clearFilters()">{{'common.controls.clearFilters' | translate}}</a></span>
         </div>
         <div class="pull-right left-margin">
-            <pagination class="paginator" total-items="tableInfo.filtered" max-size="maxVisiblePages" items-per-page="instancesPerPage" ng-model="currentPage" ng-change="pageChanged()"></pagination>
+            <pagination class="paginator" total-items="tableInfo.filtered" max-size="pagination.maxVisiblePages" items-per-page="pagination.itemsPerPage" ng-model="pagination.currentPage" ng-change="pageChanged()"></pagination>
         </div>
         <div class="pull-right">
-            <select class="form-control" ng-model="instancesPerPage" ng-change="resetPagination()" ng-options="currOption for currOption in [10, 25, 50, 100]"></select>
+            <select class="form-control" ng-model="pagination.itemsPerPage" ng-change="resetPagination()" ng-options="currOption for currOption in [10, 25, 50, 100]"></select>
         </div>
     </div>
 

ambari-admin/src/main/resources/ui/admin-web/app/views/clusters/clusterInformation.html

@@ -20,17 +20,17 @@
   <div ng-show="cluster.Clusters.provisioning_state !== 'INSTALLED'">
     <div class="welcome-header">
       <h1>{{'main.title' | translate}}</h1>
-      <span>{{'main.noClusterDescription' | translate}}</span>
+      <span class="help-block">{{'main.noClusterDescription' | translate}}</span>
     </div>
     <div class="create-cluster-section">
       <h2>{{'main.createCluster.title' | translate}}</h2>
       <div>
-        <span>
+        <span class="help-block">
           {{'main.createCluster.description' | translate}}
         </span>
       </div>
-      <div><i class="fa fa-cloud" aria-hidden="true"></i></div>
-      <div>
+      <div><img id="install-box" src="/img/install-box.svg"></div>
+      <div class="install-button">
         <a href="{{fromSiteRoot('/#/installer/step0')}}" class="btn btn-primary">
           {{'main.createCluster.launchInstallWizard' | translate}}
         </a>

ambari-admin/src/main/resources/ui/admin-web/app/views/directives/comboSearch.html

@@ -0,0 +1,63 @@
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+<div class="combo-search">
+  <div class="combo-search-inner" ng-click="observeSearchFilterInput($event)">
+    <div class="combo-search-applied-filter" ng-repeat="filter in appliedFilters" ng-click="forceFocus($event, filter)">
+      <i class="fa fa-times-circle" ng-click="removeFilter(filter)"></i>
+      <span>{{filter.label}}:</span>
+      <div class="combo-search-input-wrapper">
+        <input type="text"
+               autocomplete="off"
+               ng-attr-name="{{filter.id}}"
+               class="combo-search-input"
+               ng-model="filter.searchOptionInput"
+               ng-change="observeSearchOptionInput(filter)"
+               ng-blur="hideAutocomplete(filter)"/>
+        <div class="combo-search-dropdown" ng-show="filter.showAutoComplete">
+          <ul>
+            <li ng-repeat="item in filter.filteredOptions" class="filter">
+              <a ng-click="selectOption($event, item, filter)"
+                 ng-class="{active: item.active}"
+                 ng-mouseover="makeActive(item, filter.filteredOptions)">{{item.label}}</a>
+            </li>
+          </ul>
+        </div>
+      </div>
+    </div>
+    <div class="combo-search-input-wrapper">
+      <input type="text"
+             autocomplete="off"
+             placeholder="{{appliedFilters.length === 0 ? placeholder : ''}}"
+             class="combo-search-input main-input"
+             ng-model="searchFilterInput"
+             ng-change="observeSearchFilterInput()"/>
+      <div class="combo-search-dropdown" ng-show="showAutoComplete">
+        <ul>
+          <li ng-repeat="item in filterSuggestions"
+              ng-class="{category: item.isCategory, filter: !item.isCategory, hide: (item.isCategory && item.isDefault)}">
+            <a ng-click="selectFilter(item, $event)"
+               ng-class="{active: item.active}"
+               ng-mouseover="makeActive(item, filterSuggestions)">{{item.label}}</a>
+          </li>
+        </ul>
+      </div>
+    </div>
+    <i class="combo-search-close fa fa-times-circle" ng-show="appliedFilters.length" ng-click="clearFilters()"></i>
+  </div>
+</div>

ambari-admin/src/main/resources/ui/admin-web/app/views/remoteClusters/list.html

@@ -23,30 +23,34 @@
         {{'views.registerRemoteCluster' | translate}}
       </a>
     </div>
+    <div class="search-box-button pull-right">
+      <button class="btn btn-default" ng-click="toggleSearchBox()">
+        <i class="fa fa-filter" aria-hidden="true"></i>
+      </button>
+      <div class="popup-arrow-up hide"></div>
+    </div>
+  </div>
+
+  <div class="search-box-row hide">
+    <combo-search suggestions="filters" filter-change="filterClusters" placeholder="Search"></combo-search>
   </div>
+
   <table class="table table-striped table-hover">
     <thead>
       <tr>
         <th class="col-sm-3">
           <div class="search-container">
-            <label for="">{{'views.clusterName' | translate}}</label>
-            <input type="text" class="form-control namefilter" placeholder="{{'common.any' | translate}}" ng-model="currentNameFilter" ng-change="resetPagination()">
-            <button type="button" class="close" ng-show="currentNameFilter" ng-click="currentNameFilter=''; resetPagination()"><span aria-hidden="true">&times;</span><span class="sr-only">{{'common.controls.close' | translate}}</span></button>
+            <label>{{'views.clusterName' | translate}}</label>
           </div>
         </th>
         <th class="col-sm-9">
-          <label for="">{{'common.services' | translate}}</label>
-          <select class="form-control typefilter"
-                  ng-model="currentTypeFilter"
-                  ng-options="item for item in typeFilterOptions"
-                  ng-change="resetPagination();">
-          </select>
+          <label>{{'common.services' | translate}}</label>
         </th>
       </tr>
     </thead>
     <tbody>
-    <tr ng-repeat="remoteCluster in remoteClusters" >
-      <td class="col-sm-3"><a href="#/remoteClusters/{{remoteCluster.ClusterInfo.name}}/edit">{{ remoteCluster.ClusterInfo.name }}</a></td>
+    <tr ng-repeat="remoteCluster in remoteClusters  | filter : { isShowed: true }">
+      <td class="col-sm-3"><a href="#/remoteClusters/{{remoteCluster.ClusterInfo.name}}/edit">{{ remoteCluster.clusterName }}</a></td>
       <td class="col-sm-9">
         <span ng-repeat="service in remoteCluster.ClusterInfo.services" ng-if="remoteCluster.ClusterInfo.services.length > 0">{{ service }}{{$last ? '' : ','}} </span>
         <span ng-if="remoteCluster.ClusterInfo.services.length == 0">--</span>
@@ -58,19 +62,18 @@
   <div ng-if="isLoading" class="spinner-container">
     <i class="fa fa-2x fa-spinner fa-spin" aria-hidden="true"></i>
   </div>
-  <div class="alert empty-table-alert col-sm-12" ng-show="!remoteClusters.length && !isLoading">
+  <div class="alert empty-table-alert col-sm-12" ng-show="!tableInfo.showed && !isLoading">
     {{'common.alerts.noRemoteClusterDisplay' | translate}}
   </div>
-  <div class="col-sm-12 table-bar">
+  <div class="col-sm-12 table-bar" ng-show="tableInfo.total >= minInstanceForPagination">
     <div class="pull-left filtered-info">
       <span>{{'common.filterInfo' | translate:{showed: tableInfo.showed, total: tableInfo.total, term: constants.groups} }}</span>
-      <span ng-show="isNotEmptyFilter">- <a href ng-click="clearFilters()">{{'common.controls.clearFilters' | translate}}</a></span>
     </div>
     <div class="pull-right left-margin">
-      <pagination class="paginator" total-items="totalGroups" max-size="maxVisiblePages" items-per-page="groupsPerPage" ng-model="currentPage" ng-change="pageChanged()"></pagination>
+      <pagination class="paginator" total-items="tableInfo.filtered" max-size="pagination.maxVisiblePages" items-per-page="groupsPerPage" ng-model="pagination.currentPage" ng-change="pageChanged()"></pagination>
     </div>
     <div class="pull-right">
-      <select class="form-control" ng-model="groupsPerPage" ng-change="groupsPerPageChanges()" ng-options="currOption for currOption in [10, 25, 50, 100]"></select>
+      <select class="form-control" ng-model="pagination.itemsPerPage" ng-change="resetPagination()" ng-options="currOption for currOption in [10, 25, 50, 100]"></select>
     </div>
   </div>
 

ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/list.html

@@ -23,47 +23,42 @@
         {{'versions.register.title' | translate}}
       </a>
     </div>
+    <div class="search-box-button pull-right">
+      <button class="btn btn-default" ng-click="toggleSearchBox()">
+        <i class="fa fa-filter" aria-hidden="true"></i>
+      </button>
+      <div class="popup-arrow-up hide"></div>
+    </div>
+  </div>
+
+  <div class="search-box-row hide">
+    <combo-search suggestions="filters" filter-change="filterRepos" placeholder="Search"></combo-search>
   </div>
+
   <table class="table table-striped table-hover">
     <thead>
     <tr>
       <th class="col-small">
         <label>{{'common.stack' | translate}}</label>
-        <select class="form-control"
-                ng-change="resetPagination()"
-                ng-model="filter.stack.current"
-                ng-options="item.label for item in filter.stack.options track by item.value"
-          ></select>
       </th>
       <th class="col-medium text-search-container">
         <label>{{'common.name' | translate}}</label>
-        <input type="text" class="form-control" ng-change="resetPagination()" ng-model="filter.name" placeholder="{{'common.any' | translate}}">
-        <button type="button" class="close clearfilter" ng-show="filter.name" ng-click="filter.name=''; resetPagination()"><span aria-hidden="true">&times;</span><span class="sr-only">{{'common.controls.close' | translate}}</span></button>
       </th>
       <th class="col-medium text-search-container">
         <label>{{'common.type' | translate}}</label>
-        <input type="text" class="form-control" ng-change="resetPagination()" ng-model="filter.type" placeholder="{{'common.any' | translate}}">
-        <button type="button" class="close clearfilter" ng-show="filter.type" ng-click="filter.type=''; resetPagination()"><span aria-hidden="true">&times;</span><span class="sr-only">{{'common.controls.close' | translate}}</span></button>
       </th>
       <th class="col-medium text-search-container">
         <label>{{'common.version' | translate}}</label>
-        <input type="text" class="form-control" ng-change="resetPagination()" ng-model="filter.version" placeholder="{{'common.any' | translate}}">
-        <button type="button" class="close clearfilter" ng-show="filter.version" ng-click="filter.version=''; resetPagination()"><span aria-hidden="true">&times;</span><span class="sr-only">{{'common.controls.close' | translate}}</span></button>
       </th>
       <th class="col-small">
         <label>{{'common.cluster' | translate}}</label>
-        <select class="form-control"
-                ng-change="resetPagination()"
-                ng-model="filter.cluster.current"
-                ng-options="item.label for item in filter.cluster.options track by item.value"
-          ></select>
       </th>
       <th class="col-small"></th>
       <th class="col-small text-center vertical-top">{{'common.hidden' | translate}}</th>
     </tr>
     </thead>
     <tbody>
-    <tr ng-repeat="repo in repos">
+    <tr ng-repeat="repo in repos | filter : { isShowed: true }">
       <td class="col-small">
         <span>{{repo.stack_name}}-{{repo.stack_version}}</span>
       </td>
@@ -120,17 +115,16 @@
   <div ng-if="isLoading" class="spinner-container">
     <i class="fa fa-2x fa-spinner fa-spin" aria-hidden="true"></i>
   </div>
-  <div class="alert empty-table-alert col-sm-12" ng-show="!repos.length && !isLoading">
+  <div class="alert empty-table-alert col-sm-12" ng-show="!tableInfo.showed && !isLoading">
     {{'common.alerts.nothingToDisplay' | translate:{term: getConstant("common.version")} }}
   </div>
-  <div class="col-sm-12 table-bar">
+  <div class="col-sm-12 table-bar" ng-show="tableInfo.total >= minInstanceForPagination">
     <div class="pull-left filtered-info">
       <span>{{'common.filterInfo' | translate:{showed: tableInfo.showed, total: tableInfo.total, term: getConstant("common.versions")} }}</span>
-      <span ng-show="isNotEmptyFilter">- <a href ng-click="clearFilters()">{{'common.controls.clearFilters' | translate}}</a></span>
     </div>
     <div class="pull-right left-margin">
       <pagination class="paginator"
-                  total-items="pagination.totalRepos"
+                  total-items="tableInfo.total"
                   max-size="pagination.maxVisiblePages"
                   items-per-page="pagination.itemsPerPage"
                   ng-model="pagination.currentPage"

ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/groupsList.html

@@ -16,11 +16,24 @@
 * limitations under the License.
 -->
 <div class="groups-pane">
-  <div class="clearfix panel">
-    <button class="btn btn-default creategroup-btn pull-right" ng-click="createGroup()">
-      {{'groups.createLocal' | translate}}
-    </button>
+  <div class="clearfix">
+    <div class="pull-right">
+      <button class="btn btn-default creategroup-btn pull-right" ng-click="createGroup()">
+        {{'groups.createLocal' | translate}}
+      </button>
+    </div>
+    <div class="search-box-button pull-right">
+      <button class="btn btn-default" ng-click="toggleSearchBox()">
+        <i class="fa fa-filter" aria-hidden="true"></i>
+      </button>
+      <div class="popup-arrow-up hide"></div>
+    </div>
   </div>
+
+  <div class="search-box-row hide">
+    <combo-search suggestions="filters" filter-change="filterGroups" placeholder="Search"></combo-search>
+  </div>
+
   <table class="table table-striped table-hover col-sm-12">
     <thead>
       <tr>
@@ -37,26 +50,9 @@
           <span>{{'common.actions' | translate}}</span>
         </th>
       </tr>
-      <tr>
-        <th class="col-sm-6">
-          <div class="search-container">
-            <input type="text" class="form-control namefilter" placeholder="{{'common.any' | translate}}" ng-model="filter.name" ng-change="resetPagination()">
-            <button type="button" class="close" ng-show="filter.name" ng-click="filter.name=''; resetPagination()"><span aria-hidden="true">&times;</span><span class="sr-only">{{'common.controls.close' | translate}}</span></button>
-          </div>
-        </th>
-        <th class="col-sm-2">
-          <select class="form-control typefilter"
-                  ng-model="filter.type"
-                  ng-options="item.label for item in typeFilterOptions"
-                  ng-change="resetPagination();">
-          </select>
-        </th>
-        <th class="col-sm-2"></th>
-        <th class="col-sm-2"></th>
-      </tr>
     </thead>
     <tbody>
-      <tr ng-repeat="group in groups">
+      <tr ng-repeat="group in groups | filter : { isShowed: true }">
         <td class="col-sm-8">
           <span>{{group.group_name}}</span>
         </td>
@@ -76,19 +72,18 @@
   <div ng-if="isLoading" class="spinner-container">
     <i class="fa fa-2x fa-spinner fa-spin" aria-hidden="true"></i>
   </div>
-  <div class="alert empty-table-alert col-sm-12" ng-show="!groups.length && !isLoading">
+  <div class="alert empty-table-alert col-sm-12" ng-show="!tableInfo.showed && !isLoading">
     {{'common.alerts.nothingToDisplay' | translate:{term: constants.groups} }}
   </div>
-  <div class="col-sm-12 table-bar" ng-show="totalGroups > minRowsToShowPagination">
+  <div class="col-sm-12 table-bar" ng-show="tableInfo.total > minRowsToShowPagination">
     <div class="pull-left filtered-info">
-      <span>{{'common.filterInfo' | translate:{showed: tableInfo.showed, total: tableInfo.total, term: constants.groups} }}</span>
-      <span ng-show="isNotEmptyFilter">- <a href ng-click="clearFilters()">{{'common.controls.clearFilters' | translate}}</a></span>
+      <span>{{'common.filterInfo' | translate:{showed: tableInfo.showed, total: tableInfo.filtered, term: constants.groups} }}</span>
     </div>
     <div class="pull-right left-margin">
-      <pagination class="paginator" total-items="totalGroups" max-size="maxVisiblePages" items-per-page="groupsPerPage" ng-model="currentPage" ng-change="pageChanged()"></pagination>
+      <pagination class="paginator" total-items="tableInfo.filtered" max-size="pagination.maxVisiblePages" items-per-page="pagination.itemsPerPage" ng-model="pagination.currentPage" ng-change="pageChanged()"></pagination>
     </div>
     <div class="pull-right">
-      <select class="form-control" ng-model="groupsPerPage" ng-change="groupsPerPageChanges()" ng-options="currOption for currOption in [10, 25, 50, 100]"></select>
+      <select class="form-control" ng-model="pagination.itemsPerPage" ng-change="resetPagination()" ng-options="currOption for currOption in [10, 25, 50, 100]"></select>
     </div>
   </div>
 </div>

ambari-admin/src/main/resources/ui/admin-web/app/views/userManagement/usersList.html

@@ -17,14 +17,27 @@
 -->
 
 <div class="users-pane">
-  <div class="clearfix panel">
-    <button class="btn btn-default createuser-btn pull-right" ng-click="createUser();">
-      {{'users.create' | translate}}
-    </button>
+  <div class="clearfix">
+    <div class="pull-right">
+      <button class="btn btn-default createuser-btn pull-right" ng-click="createUser();">
+        {{'users.create' | translate}}
+      </button>
+    </div>
+    <div class="search-box-button pull-right">
+      <button class="btn btn-default" ng-click="toggleSearchBox()">
+        <i class="fa fa-filter" aria-hidden="true"></i>
+      </button>
+      <div class="popup-arrow-up hide"></div>
+    </div>
   </div>
+
+  <div class="search-box-row hide">
+    <combo-search suggestions="filters" filter-change="filterUsers" placeholder="Search"></combo-search>
+  </div>
+
   <table class="table table-striped table-hover">
     <thead>
-      <tr class="fix-bottom">
+      <tr>
         <th>
           <span>{{'users.username' | translate}}</span>
         </th>
@@ -44,36 +57,9 @@
           <span>{{'common.actions' | translate}}</span>
         </th>
       </tr>
-      <tr class="fix-top">
-        <th>
-          <div class="search-container">
-            <input type="text" class="form-control namefilter" placeholder="{{'common.any' | translate}}" ng-model="filters.name" ng-change="resetPagination()">
-            <button type="button" class="close clearfilter" ng-show="filters.name" ng-click="filters.name=''; resetPagination()">
-              <span aria-hidden="true">&times;</span><span class="sr-only">{{'common.controls.close' | translate}}</span>
-            </button>
-          </div>
-        </th>
-        <th></th>
-        <th>
-          <select class="form-control statusfilter"
-                  ng-model="filters.status"
-                  ng-options="item.label for item in activeFilterOptions"
-                  ng-change="resetPagination()">
-          </select>
-        </th>
-        <th>
-          <select class="form-control typefilter"
-                  ng-model="filters.type"
-                  ng-options="item.label for item in typeFilterOptions"
-                  ng-change="resetPagination()">
-          </select>
-        </th>
-        <th></th>
-        <th></th>
-      </tr>
     </thead>
     <tbody>
-      <tr ng-repeat="user in users">
+      <tr ng-repeat="user in users | filter : { isShowed: true }">
         <td>
           <span>{{user.Users.user_name}}</span>
         </td>
@@ -101,19 +87,18 @@
   <div ng-if="isLoading" class="spinner-container">
     <i class="fa fa-2x fa-spinner fa-spin" aria-hidden="true"></i>
   </div>
-  <div class="alert empty-table-alert col-sm-12" ng-show="!users.length && !isLoading">
+  <div class="alert empty-table-alert col-sm-12" ng-show="!tableInfo.showed && !isLoading">
     {{'common.alerts.nothingToDisplay' | translate:{term: constants.users} }}
   </div>
-  <div class="col-sm-12 table-bar" ng-show="totalUsers > minRowsToShowPagination">
+  <div class="col-sm-12 table-bar" ng-show="users.length > minRowsToShowPagination">
     <div class="pull-left filtered-info">
       <span>{{'common.filterInfo' | translate:{showed: tableInfo.showed, total: tableInfo.total, term: constants.users} }}</span>
-      <span ng-show="isNotEmptyFilter">- <a href ng-click="clearFilters()">{{'common.controls.clearFilters' | translate}}</a></span>
     </div>
     <div class="pull-right left-margin">
-      <pagination class="paginator" total-items="totalUsers" max-size="maxVisiblePages" items-per-page="usersPerPage" ng-model="currentPage" ng-change="pageChanged()"></pagination>
+      <pagination class="paginator" total-items="tableInfo.filtered" max-size="pagination.maxVisiblePages" items-per-page="pagination.itemsPerPage" ng-model="pagination.currentPage" ng-change="pageChanged()"></pagination>
     </div>
     <div class="pull-right">
-      <select class="form-control" ng-model="usersPerPage" ng-change="usersPerPageChanges()" ng-options="currOption for currOption in [10, 25, 50, 100]"></select>
+      <select class="form-control" ng-model="pagination.itemsPerPage" ng-change="resetPagination()" ng-options="currOption for currOption in [10, 25, 50, 100]"></select>
     </div>
   </div>
 </div>

ambari-admin/src/main/resources/ui/admin-web/package.json

@@ -24,7 +24,7 @@
     "karma-jasmine": "0.1.5",
     "karma-ng-html2js-preprocessor": "^0.1.0",
     "karma-phantomjs-launcher": "0.1",
-    "phantomjs": "1.9.20",
+    "phantomjs": "^2.1.7",
     "protractor": "1.0.0"
   },
   "scripts": {

ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/ambariViews/ViewsListCtrl_test.js

@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+describe('#Cluster', function () {
+  describe('ViewsListCtrl', function() {
+    var scope, ctrl;
+
+    beforeEach(function () {
+      module('ambariAdminConsole');
+      inject(function($rootScope, $controller) {
+        scope = $rootScope.$new();
+        scope.pagination = {
+          resetPagination: angular.noop
+        };
+        ctrl = $controller('ViewsListCtrl', {$scope: scope});
+      });
+      scope.instances = [
+        {
+          short_url_name: 'sun1',
+          url: 'url1',
+          view_name: 'vn1',
+          instance_name: 'in1',
+          short_url: 'su1'
+        },
+        {
+          short_url_name: 'sun2',
+          url: 'url2',
+          view_name: 'vn2',
+          instance_name: 'in2',
+          short_url: 'su2'
+        }
+      ];
+    });
+
+    describe('#filterInstances', function() {
+      beforeEach(function() {
+        spyOn(scope.pagination, 'resetPagination');
+      });
+
+      it('all should be filtered when filters not applied', function() {
+        scope.filterInstances();
+        expect(scope.tableInfo.filtered).toEqual(2);
+        scope.filterInstances([]);
+        expect(scope.tableInfo.filtered).toEqual(2);
+      });
+
+      it('resetPagination should be called', function() {
+        scope.filterInstances();
+        expect(scope.pagination.resetPagination).toHaveBeenCalled();
+      });
+
+      it('one view should be filtered', function() {
+        var appliedFilters = [
+          {
+            key: 'view_name',
+            values: ['vn1']
+          }
+        ];
+        scope.filterInstances(appliedFilters);
+        expect(scope.tableInfo.filtered).toEqual(1);
+        expect(scope.instances[0].isFiltered).toBeTruthy();
+        expect(scope.instances[1].isFiltered).toBeFalsy();
+      });
+
+      it('two views should be filtered', function() {
+        var appliedFilters = [
+          {
+            key: 'view_name',
+            values: ['vn1', 'vn2']
+          }
+        ];
+        scope.filterInstances(appliedFilters);
+        expect(scope.tableInfo.filtered).toEqual(2);
+        expect(scope.instances[0].isFiltered).toBeTruthy();
+        expect(scope.instances[1].isFiltered).toBeTruthy();
+      });
+
+      it('one views should be filtered with combo filter', function() {
+        var appliedFilters = [
+          {
+            key: 'view_name',
+            values: ['vn1', 'vn2']
+          },
+          {
+            key: 'instance_name',
+            values: ['in2']
+          }
+        ];
+        scope.filterInstances(appliedFilters);
+        expect(scope.tableInfo.filtered).toEqual(1);
+        expect(scope.instances[0].isFiltered).toBeFalsy();
+        expect(scope.instances[1].isFiltered).toBeTruthy();
+      });
+    });
+  });
+});

ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/userManagement/GroupsListCtrl_test.js

@@ -20,110 +20,20 @@ describe('#Cluster', function () {
 
   describe('GroupsListCtrl', function() {
 
-    var scope, ctrl, $t, $httpBackend;
+    var $scope, ctrl, $t, $httpBackend, $group;
 
     beforeEach(module('ambariAdminConsole', function () {}));
 
-    beforeEach(inject(function($rootScope, $controller, _$translate_, _$httpBackend_) {
-      scope = $rootScope.$new();
+    beforeEach(inject(function($rootScope, $controller, _$translate_, _$httpBackend_, _Group_) {
+      $scope = $rootScope.$new();
       $t = _$translate_.instant;
       $httpBackend = _$httpBackend_;
+      $group = _Group_;
       ctrl = $controller('GroupsListCtrl', {
-        $scope: scope
+        $scope: $scope
       });
     }));
 
-    describe('#clearFilters()', function () {
-
-      it('should clear filters and reset pagination', function () {
-        scope.currentPage = 2;
-        scope.filter.name = 'a';
-        scope.filter.type = {
-          label: $t('common.local'),
-          value: false
-        };
-        scope.clearFilters();
-        expect(scope.filter.name).toEqual('');
-        expect(scope.filter.type).toEqual({
-          label: $t('common.all'),
-          value: '*'
-        });
-        expect(scope.currentPage).toEqual(1);
-      });
-
-    });
-
-    describe('#isNotEmptyFilter', function () {
-
-      var cases = [
-        {
-          currentNameFilter: '',
-          currentTypeFilter: null,
-          isNotEmptyFilter: false,
-          title: 'no filters'
-        },
-        {
-          currentNameFilter: '',
-          currentTypeFilter: {
-            value: '*'
-          },
-          isNotEmptyFilter: false,
-          title: 'empty filters'
-        },
-        {
-          currentNameFilter: 'a',
-          currentTypeFilter: {
-            value: '*'
-          },
-          isNotEmptyFilter: true,
-          title: 'name filter'
-        },
-        {
-          currentNameFilter: '0',
-          currentTypeFilter: {
-            value: '*'
-          },
-          isNotEmptyFilter: true,
-          title: 'name filter with "0" as string'
-        },
-        {
-          currentNameFilter: '',
-          currentTypeFilter: {
-            value: false
-          },
-          isNotEmptyFilter: true,
-          title: 'type filter'
-        },
-        {
-          currentNameFilter: 'a',
-          currentTypeFilter: {
-            value: false
-          },
-          isNotEmptyFilter: true,
-          title: 'both filters'
-        },
-        {
-          currentNameFilter: '0',
-          currentTypeFilter: {
-            value: false
-          },
-          isNotEmptyFilter: true,
-          title: 'both filters with "0" as string'
-        }
-      ];
-
-      cases.forEach(function (item) {
-        it(item.title, function () {
-          $httpBackend.expectGET(/\/api\/v1\/groups/).respond(200);
-          scope.filter.name = item.currentNameFilter;
-          scope.filter.type = item.currentTypeFilter;
-          scope.$digest();
-          expect(scope.isNotEmptyFilter).toEqual(item.isNotEmptyFilter);
-        });
-      });
-
-    });
-
   });
 
 });

ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/userManagement/UsersListCtrl_test.js

@@ -33,312 +33,6 @@ describe('#Cluster', function () {
       });
     }));
 
-    describe('#clearFilters()', function () {
-
-      it('should clear filters and reset pagination', function () {
-        scope.currentPage = 2;
-        scope.filters.name = 'a';
-        scope.filters.status = {
-          label: $t('common.local'),
-          value: false
-        };
-        scope.filters.type = {
-          label: $t('common.local'),
-          value: 'LOCAL'
-        };
-        scope.clearFilters();
-        expect(scope.filters.name).toEqual('');
-        expect(scope.filters.status).toEqual({
-          label: $t('common.all'),
-          value: '*'
-        });
-        expect(scope.filters.type).toEqual({
-          label: $t('common.all'),
-          value: '*'
-        });
-        expect(scope.currentPage).toEqual(1);
-      });
-
-    });
-
-    describe('#isNotEmptyFilter', function () {
-
-      var cases = [
-        {
-          currentNameFilter: '',
-          currentTypeFilter: null,
-          currentActiveFilter: null,
-          isNotEmptyFilter: false,
-          title: 'no filters'
-        },
-        {
-          currentNameFilter: '',
-          currentTypeFilter: {
-            value: '*'
-          },
-          currentActiveFilter: {
-            value: '*'
-          },
-          isNotEmptyFilter: false,
-          title: 'empty filters'
-        },
-        {
-          currentNameFilter: 'a',
-          currentTypeFilter: {
-            value: '*'
-          },
-          currentActiveFilter: {
-            value: '*'
-          },
-          isNotEmptyFilter: true,
-          title: 'name filter'
-        },
-        {
-          currentNameFilter: '0',
-          currentTypeFilter: {
-            value: '*'
-          },
-          currentActiveFilter: {
-            value: '*'
-          },
-          isNotEmptyFilter: true,
-          title: 'name filter with "0" as string'
-        },
-        {
-          currentNameFilter: '',
-          currentTypeFilter: {
-            value: 'LOCAL'
-          },
-          currentActiveFilter: {
-            value: '*'
-          },
-          isNotEmptyFilter: true,
-          title: 'type filter'
-        },
-        {
-          currentNameFilter: '',
-          currentTypeFilter: {
-            value: '*'
-          },
-          currentActiveFilter: {
-            value: false
-          },
-          isNotEmptyFilter: true,
-          title: 'activity filter'
-        },
-        {
-          currentNameFilter: 'a',
-          currentTypeFilter: {
-            value: 'LOCAL'
-          },
-          currentActiveFilter: {
-            value: '*'
-          },
-          isNotEmptyFilter: true,
-          title: 'name and type filters'
-        },
-        {
-          currentNameFilter: 'a',
-          currentTypeFilter: {
-            value: '*'
-          },
-          currentActiveFilter: {
-            value: false
-          },
-          isNotEmptyFilter: true,
-          title: 'name and activity filters'
-        },
-        {
-          currentNameFilter: 'a',
-          currentTypeFilter: {
-            value: '*'
-          },
-          currentActiveFilter: {
-            value: '*'
-          },
-          isNotEmptyFilter: true,
-          title: 'name and admin filters'
-        },
-        {
-          currentNameFilter: '0',
-          currentTypeFilter: {
-            value: 'LOCAL'
-          },
-          currentActiveFilter: {
-            value: '*'
-          },
-          isNotEmptyFilter: true,
-          title: 'name and type filters with "0" as string'
-        },
-        {
-          currentNameFilter: '0',
-          currentTypeFilter: {
-            value: '*'
-          },
-          currentActiveFilter: {
-            value: false
-          },
-          isNotEmptyFilter: true,
-          title: 'name and activity filters with "0" as string'
-        },
-        {
-          currentNameFilter: '0',
-          currentTypeFilter: {
-            value: '*'
-          },
-          currentActiveFilter: {
-            value: '*'
-          },
-          isNotEmptyFilter: true,
-          title: 'name and admin filters with "0" as string'
-        },
-        {
-          currentNameFilter: '',
-          currentTypeFilter: {
-            value: 'LOCAL'
-          },
-          currentActiveFilter: {
-            value: false
-          },
-          isNotEmptyFilter: true,
-          title: 'type and activity filters'
-        },
-        {
-          currentNameFilter: '',
-          currentTypeFilter: {
-            value: 'LOCAL'
-          },
-          currentActiveFilter: {
-            value: '*'
-          },
-          isNotEmptyFilter: true,
-          title: 'type and admin filters'
-        },
-        {
-          currentNameFilter: '',
-          currentTypeFilter: {
-            value: '*'
-          },
-          currentActiveFilter: {
-            value: false
-          },
-          isNotEmptyFilter: true,
-          title: 'activity and admin filters'
-        },
-        {
-          currentNameFilter: '',
-          currentTypeFilter: {
-            value: 'LOCAL'
-          },
-          currentActiveFilter: {
-            value: false
-          },
-          isNotEmptyFilter: true,
-          title: 'all filters except name one'
-        },
-        {
-          currentNameFilter: 'a',
-          currentTypeFilter: {
-            value: '*'
-          },
-          currentActiveFilter: {
-            value: false
-          },
-          isNotEmptyFilter: true,
-          title: 'all filters except type one'
-        },
-        {
-          currentNameFilter: 'a',
-          currentTypeFilter: {
-            value: 'LOCAL'
-          },
-          currentActiveFilter: {
-            value: '*'
-          },
-          isNotEmptyFilter: true,
-          title: 'all filters except activity one'
-        },
-        {
-          currentNameFilter: 'a',
-          currentTypeFilter: {
-            value: 'LOCAL'
-          },
-          currentActiveFilter: {
-            value: false
-          },
-          isNotEmptyFilter: true,
-          title: 'all filters except admin one'
-        },
-        {
-          currentNameFilter: '0',
-          currentTypeFilter: {
-            value: '*'
-          },
-          currentActiveFilter: {
-            value: false
-          },
-          isNotEmptyFilter: true,
-          title: 'all filters with "0" as string except type one'
-        },
-        {
-          currentNameFilter: '0',
-          currentTypeFilter: {
-            value: 'LOCAL'
-          },
-          currentActiveFilter: {
-            value: '*'
-          },
-          isNotEmptyFilter: true,
-          title: 'all filters with "0" as string except activity one'
-        },
-        {
-          currentNameFilter: '0',
-          currentTypeFilter: {
-            value: 'LOCAL'
-          },
-          currentActiveFilter: {
-            value: false
-          },
-          isNotEmptyFilter: true,
-          title: 'all filters with "0" as string except admin one'
-        },
-        {
-          currentNameFilter: 'a',
-          currentTypeFilter: {
-            value: false
-          },
-          currentActiveFilter: {
-            value: 'LOCAL'
-          },
-          isNotEmptyFilter: true,
-          title: 'all filters'
-        },
-        {
-          currentNameFilter: '0',
-          currentTypeFilter: {
-            value: false
-          },
-          currentActiveFilter: {
-            value: 'LOCAL'
-          },
-          isNotEmptyFilter: true,
-          title: 'all filters with "0" as string'
-        }
-      ];
-
-      cases.forEach(function (item) {
-        it(item.title, function () {
-          $httpBackend.expectGET(/\/api\/v1\/users/).respond(200);
-          scope.filters.name = item.currentNameFilter;
-          scope.filters.status = item.currentActiveFilter;
-          scope.filters.type = item.currentTypeFilter;
-          scope.$digest();
-          expect(scope.isNotEmptyFilter).toEqual(item.isNotEmptyFilter);
-        });
-      });
-
-    });
-
   });
 
 });

ambari-admin/src/main/resources/ui/admin-web/test/unit/directives/comboSearch_test.js

@@ -0,0 +1,362 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+describe('#comboSearch', function () {
+  var scope, element;
+
+  beforeEach(module('ambariAdminConsole'));
+  beforeEach(module('views/directives/comboSearch.html'));
+
+  beforeEach(inject(function($rootScope, $compile) {
+    scope = $rootScope.$new();
+
+    var preCompiledElement = '<combo-search suggestions="filters" filter-change="filterItems" placeholder="Search"></combo-search>';
+
+    scope.filters = [
+      {
+        key: 'f1',
+        label: 'filter1',
+        options: []
+      },
+      {
+        key: 'f2',
+        label: 'filter2',
+        options: []
+      }
+    ];
+    scope.filterItems = angular.noop;
+    spyOn(scope, 'filterItems');
+
+
+    element = $compile(preCompiledElement)(scope);
+    scope.$digest();
+  }));
+
+  afterEach(function() {
+    element.remove();
+  });
+
+
+  describe('#removeFilter', function() {
+    it('should remove filter by id', function () {
+      var isoScope = element.isolateScope();
+      isoScope.appliedFilters.push({
+        id: 1
+      });
+      spyOn(isoScope, 'observeSearchFilterInput');
+      spyOn(isoScope, 'updateFilters');
+
+      isoScope.removeFilter({id: 1});
+
+      expect(isoScope.appliedFilters).toEqual([]);
+      expect(isoScope.observeSearchFilterInput).toHaveBeenCalled();
+      expect(isoScope.updateFilters).toHaveBeenCalledWith([]);
+    });
+  });
+
+  describe('#clearFilters', function() {
+    it('should empty appliedFilters', function () {
+      var isoScope = element.isolateScope();
+      isoScope.appliedFilters.push({
+        id: 1
+      });
+      spyOn(isoScope, 'updateFilters');
+
+      isoScope.clearFilters();
+
+      expect(isoScope.appliedFilters).toEqual([]);
+      expect(isoScope.updateFilters).toHaveBeenCalledWith([]);
+    });
+  });
+
+  describe('#selectFilter', function() {
+    it('should add new filter to appliedFilters', function () {
+      var isoScope = element.isolateScope();
+
+      isoScope.selectFilter({
+        key: 'f1',
+        label: 'filter1',
+        options: []
+      });
+
+      expect(isoScope.appliedFilters[0]).toEqual({
+        id: 'filter_1',
+        currentOption: null,
+        filteredOptions: [],
+        searchOptionInput: '',
+        key: 'f1',
+        label: 'filter1',
+        options: [],
+        showAutoComplete: false
+      });
+      expect(isoScope.isEditing).toBeFalsy();
+      expect(isoScope.showAutoComplete).toBeFalsy();
+      expect(isoScope.searchFilterInput).toEqual('');
+    });
+  });
+
+  describe('#selectOption', function() {
+    it('should set value to appliedFilter', function () {
+      var isoScope = element.isolateScope();
+      var filter = {};
+
+      spyOn(isoScope, 'observeSearchFilterInput');
+      spyOn(isoScope, 'updateFilters');
+
+      isoScope.selectOption(null, {
+        key: 'o1',
+        label: 'option1'
+      }, filter);
+
+      expect(filter.currentOption).toEqual({
+        key: 'o1',
+        label: 'option1'
+      });
+      expect(filter.showAutoComplete).toBeFalsy();
+      expect(isoScope.observeSearchFilterInput).toHaveBeenCalled();
+      expect(isoScope.updateFilters).toHaveBeenCalled();
+    });
+  });
+
+  describe('#makeActive', function() {
+    it('category option can not be active', function () {
+      var isoScope = element.isolateScope();
+      var active = {
+        key: 'o1',
+        isCategory: true,
+        active: false
+      };
+
+      isoScope.makeActive(active, [active]);
+
+      expect(active.active).toBeFalsy();
+    });
+
+    it('value option can be active', function () {
+      var isoScope = element.isolateScope();
+      var active = {
+        key: 'o1',
+        isCategory: false,
+        active: false
+      };
+
+      isoScope.makeActive(active, [active]);
+
+      expect(active.active).toBeTruthy();
+    });
+  });
+
+  describe('#updateFilters', function() {
+    it('filter function from parent scope should be called', function () {
+      var isoScope = element.isolateScope();
+      spyOn(isoScope, 'extractFilters').andReturn([{}]);
+
+      isoScope.updateFilters([{}]);
+
+      expect(scope.filterItems).toHaveBeenCalledWith([{}]);
+    });
+  });
+
+  describe('#extractFilters', function() {
+    it('should extract filters', function () {
+      var isoScope = element.isolateScope();
+      var filters = [
+        {
+          currentOption: { key: 'o1'},
+          key: 'f1'
+        },
+        {
+          currentOption: { key: 'o2'},
+          key: 'f1'
+        },
+        {
+          currentOption: null,
+          key: 'f2'
+        }
+      ];
+
+      expect(isoScope.extractFilters(filters)).toEqual([
+        {
+          key: 'f1',
+          values: ['o1', 'o2']
+        }
+      ]);
+    });
+  });
+
+  describe('#observeSearchFilterInput', function() {
+    it('should show all filters when search filter empty', function () {
+      var isoScope = element.isolateScope();
+      isoScope.searchFilterInput = '';
+
+      isoScope.observeSearchFilterInput();
+
+      expect(isoScope.showAutoComplete).toBeTruthy();
+      expect(isoScope.filterSuggestions).toEqual([
+        {
+          key: 'f1',
+          label: 'filter1',
+          options: [  ],
+          active: true
+        },
+        {
+          key: 'f2',
+          label: 'filter2',
+          options: [  ],
+          active: false
+        }
+      ]);
+    });
+
+    it('should show only searched filter when search filter not empty', function () {
+      var isoScope = element.isolateScope();
+      isoScope.searchFilterInput = 'filter1';
+
+      isoScope.observeSearchFilterInput();
+
+      expect(isoScope.showAutoComplete).toBeTruthy();
+      expect(isoScope.filterSuggestions).toEqual([
+        {
+          key: 'f1',
+          label: 'filter1',
+          options: [  ],
+          active: true
+        }
+      ]);
+    });
+
+    it('should show no filter when search filter not found', function () {
+      var isoScope = element.isolateScope();
+      isoScope.searchFilterInput = 'unknown-filter';
+
+      isoScope.observeSearchFilterInput();
+
+      expect(isoScope.showAutoComplete).toBeFalsy();
+      expect(isoScope.filterSuggestions).toEqual([]);
+    });
+  });
+
+  describe('#observeSearchOptionInput', function() {
+    it('should show all options when options search empty', function () {
+      var isoScope = element.isolateScope();
+      var filter = {
+        key: 'p1',
+        searchOptionInput: '',
+        currentOption: null,
+        options: [
+          {
+            key: 'op1',
+            label: 'op1'
+          },
+          {
+            key: 'op2',
+            label: 'op2'
+          }
+        ]
+      };
+      isoScope.appliedFilters = [
+        {
+          key: 'p5',
+          currentOption: {
+            key: 'op5'
+          }
+        }
+      ];
+
+      isoScope.observeSearchOptionInput(filter);
+
+      expect(filter.showAutoComplete).toBeTruthy();
+      expect(filter.filteredOptions).toEqual([
+        {
+          key: 'op1',
+          label: 'op1',
+          active: false
+        },
+        {
+          key: 'op2',
+          label: 'op2',
+          active: false
+        }
+      ]);
+    });
+
+    it('should show only filtered options when options search not empty', function () {
+      var isoScope = element.isolateScope();
+      var filter = {
+        key: 'p1',
+        currentOption: null,
+        searchOptionInput: 'op1',
+        options: [
+          {
+            key: 'op1',
+            label: 'op1'
+          },
+          {
+            key: 'op2',
+            label: 'op2'
+          }
+        ]
+      };
+      isoScope.appliedFilters = [
+        {
+          key: 'p5',
+          currentOption: {
+            key: 'op5'
+          }
+        }
+      ];
+
+      isoScope.observeSearchOptionInput(filter);
+
+      expect(filter.showAutoComplete).toBeTruthy();
+      expect(filter.filteredOptions).toEqual([
+        {
+          key: 'op1',
+          label: 'op1',
+          active: false
+        }
+      ]);
+    });
+
+    it('should show no options when options search not found', function () {
+      var isoScope = element.isolateScope();
+      var filter = {
+        key: 'p1',
+        currentOption: null,
+        searchOptionInput: 'op3',
+        options: [
+          {
+            key: 'op1',
+            label: 'op1'
+          },
+          {
+            key: 'op2',
+            label: 'op2'
+          }
+        ]
+      };
+      isoScope.appliedFilters = [];
+
+      isoScope.observeSearchOptionInput(filter);
+
+      expect(filter.showAutoComplete).toBeFalsy();
+      expect(filter.filteredOptions).toEqual([]);
+    });
+  });
+
+});

ambari-admin/src/main/resources/ui/admin-web/test/unit/services/Filters_test.js

@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+describe('Filters Service', function () {
+  var Filters;
+
+  beforeEach(function () {
+    module('ambariAdminConsole', angular.noop);
+    inject(function (_Filters_) {
+      Filters = _Filters_;
+    });
+  });
+  describe('#initFilterOptions', function() {
+    var items = [
+      {
+        k1: {
+          values: ['val1', 'val3']
+        },
+        k2: 'val2'
+      }
+    ];
+    var filters = [
+      {
+        isMultiple: true,
+        key: 'k1',
+        options: [],
+        customValueConverter: function(item) {
+          return item.k1.values;
+        }
+      },
+      {
+        key: 'k2',
+        options: []
+      },
+      {
+        isStatic: true,
+        options: [
+          {
+            key: 'static1',
+            label: 'static1'
+          }
+        ]
+      }
+    ];
+    beforeEach(function() {
+      Filters.initFilterOptions(filters, items);
+    });
+
+    it('should set static options of filters', function() {
+      expect(filters[2].options).toEqual([{
+        key: 'static1',
+        label: 'static1'
+      }]);
+    });
+
+    it('should set options of filters', function() {
+      expect(filters[1].options).toEqual([{
+        key: 'val2',
+        label: 'val2'
+      }]);
+    });
+
+    it('should set multiple options of filters', function() {
+      expect(filters[0].options).toEqual([
+        {
+          key: 'val1',
+          label: 'val1'
+        },
+        {
+          key: 'val3',
+          label: 'val3'
+        }
+      ]);
+    });
+  });
+
+  describe('#filterItems', function() {
+
+    it('all items should be filtered when no filters applied', function() {
+      var items = [{}];
+
+      expect(Filters.filterItems(null, items, [])).toEqual(1);
+      expect(items[0].isFiltered).toBeTruthy();
+    });
+
+    it('items should be filtered when simple filter applied', function() {
+      var appliedFilters = [
+        {
+          key: 'p1',
+          values: ['val1']
+        }
+      ];
+      var items = [
+        { p1: 'val1' },
+        { p1: 'val2' }
+      ];
+      var filterDefinitions = [];
+
+      expect(Filters.filterItems(appliedFilters, items, filterDefinitions)).toEqual(1);
+      expect(items[0].isFiltered).toBeTruthy();
+      expect(items[1].isFiltered).toBeFalsy();
+    });
+
+    it('items should be filtered when filter applied on array values', function() {
+      var appliedFilters = [
+        {
+          key: 'p1',
+          values: ['a']
+        }
+      ];
+      var items = [
+        { p1: ['a', 'b'] },
+        { p1: ['c', 'b'] }
+      ];
+      var filterDefinitions = [];
+
+      expect(Filters.filterItems(appliedFilters, items, filterDefinitions)).toEqual(1);
+      expect(items[0].isFiltered).toBeTruthy();
+      expect(items[1].isFiltered).toBeFalsy();
+    });
+
+    it('items should be filtered when custom filter applied', function() {
+      var appliedFilters = [
+        {
+          key: 'p1',
+          values: ['a']
+        }
+      ];
+      var items = [
+        { p1: { customValue: 'a' } },
+        { p1: { customValue: 'b' } }
+      ];
+      var filterDefinitions = [{
+        key: 'p1',
+        customValueConverter: function(item) {
+          return item.p1.customValue;
+        }
+      }];
+
+      expect(Filters.filterItems(appliedFilters, items, filterDefinitions)).toEqual(1);
+      expect(items[0].isFiltered).toBeTruthy();
+      expect(items[1].isFiltered).toBeFalsy();
+    });
+  });
+
+});

ambari-admin/src/main/resources/ui/admin-web/test/unit/services/Pagination_test.js

@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+describe('Pagination Service', function () {
+  var Pagination, $pagination;
+
+  beforeEach(function () {
+    module('ambariAdminConsole', angular.noop);
+    inject(function (_Pagination_) {
+      Pagination = _Pagination_;
+      $pagination = Pagination.create({
+        itemsPerPage: 1
+      });
+    });
+  });
+
+  describe('#pageChanged', function() {
+
+    it('should show items on second page', function() {
+      var items = [
+        { isFiltered: true },
+        { isFiltered: true },
+        { isFiltered: true }
+      ];
+      var tableInfo = {
+        showed: 0
+      };
+      $pagination.currentPage = 2;
+      $pagination.pageChanged(items, tableInfo);
+      expect(items[0].isShowed).toBeFalsy();
+      expect(items[1].isShowed).toBeTruthy();
+      expect(items[2].isShowed).toBeFalsy();
+      expect(tableInfo.showed).toEqual(1);
+    });
+  });
+
+  describe('#resetPagination', function() {
+
+    it('should show items on first page', function() {
+      var items = [
+        { isFiltered: true },
+        { isFiltered: true },
+        { isFiltered: true }
+      ];
+      var tableInfo = {
+        showed: 0
+      };
+      $pagination.currentPage = 2;
+      $pagination.resetPagination(items, tableInfo);
+      expect(items[0].isShowed).toBeTruthy();
+      expect(items[1].isShowed).toBeFalsy();
+      expect(items[2].isShowed).toBeFalsy();
+      expect(tableInfo.showed).toEqual(1);
+    });
+  });
+
+});

ambari-agent/conf/unix/ambari-agent

@@ -62,7 +62,7 @@ export PATH=/usr/sbin:/sbin:/usr/lib/ambari-server/*:$PATH
 export AMBARI_CONF_DIR=$HOME_DIR/etc/ambari-server/conf:$PATH
 
 # Because Ambari rpm unpacks modules here on all systems
-export PYTHONPATH=/usr/lib/python2.6/site-packages:${PYTHONPATH:-}
+export PYTHONPATH=/usr/lib/ambari-agent/lib:${PYTHONPATH:-}
 
 export AMBARI_PID_DIR=`get_agent_property piddir`
 export AMBARI_PID_DIR=`valid_path "${AMBARI_PID_DIR:?}"`
@@ -79,11 +79,11 @@ PYTHON_WRAP=/usr/bin/ambari-python-wrap
 PIDFILE=$AMBARI_PID_DIR/$AMBARI_AGENT.pid
 OUTFILE=$AMBARI_AGENT_LOG_DIR/ambari-agent.out
 LOGFILE=$AMBARI_AGENT_LOG_DIR/ambari-agent.log
-AGENT_SCRIPT=/usr/lib/python2.6/site-packages/ambari_agent/main.py
+AGENT_SCRIPT=/usr/lib/ambari-agent/lib/ambari_agent/main.py
 AGENT_TMP_DIR=/var/lib/ambari-agent/tmp
 AGENT_WORKING_DIR=/var/lib/ambari-agent
-AMBARI_AGENT_PY_SCRIPT=/usr/lib/python2.6/site-packages/ambari_agent/AmbariAgent.py
-COMMON_DIR=/usr/lib/python2.6/site-packages/ambari_commons
+AMBARI_AGENT_PY_SCRIPT=/usr/lib/ambari-agent/lib/ambari_agent/AmbariAgent.py
+COMMON_DIR=/usr/lib/ambari-agent/lib/ambari_commons
 COMMON_DIR_AGENT=/usr/lib/ambari-agent/lib/ambari_commons
 OK=0
 NOTOK=1

ambari-agent/conf/unix/ambari-env.sh

@@ -17,7 +17,7 @@
 # given through environment variable
 AMBARI_PASSPHRASE="DEV"
 export PATH=$PATH:/var/lib/ambari-agent
-export PYTHONPATH=$PYTHONPATH:/usr/lib/python2.6/site-packages
+export PYTHONPATH=/usr/lib/ambari-agent/lib:$PYTHONPATH
 
 # customize python binary for ambari
 # export PYTHON=/usr/bin/python2

ambari-agent/conf/unix/install-helper.sh

@@ -18,26 +18,25 @@
 #                      AGENT INSTALL HELPER                      #
 ##################################################################
 
-COMMON_DIR="/usr/lib/python2.6/site-packages/ambari_commons"
-RESOURCE_MANAGEMENT_DIR="/usr/lib/python2.6/site-packages/resource_management"
-JINJA_DIR="/usr/lib/python2.6/site-packages/ambari_jinja2"
-SIMPLEJSON_DIR="/usr/lib/python2.6/site-packages/ambari_simplejson"
-STOMP_DIR="/usr/lib/python2.6/site-packages/ambari_stomp"
-WS4PY_DIR="/usr/lib/python2.6/site-packages/ambari_ws4py"
-OLD_COMMON_DIR="/usr/lib/python2.6/site-packages/common_functions"
 INSTALL_HELPER_SERVER="/var/lib/ambari-server/install-helper.sh"
-COMMON_DIR_AGENT="/usr/lib/ambari-agent/lib/ambari_commons"
-RESOURCE_MANAGEMENT_DIR_AGENT="/usr/lib/ambari-agent/lib/resource_management"
-JINJA_AGENT_DIR="/usr/lib/ambari-agent/lib/ambari_jinja2"
-SIMPLEJSON_AGENT_DIR="/usr/lib/ambari-agent/lib/ambari_simplejson"
-STOMP_AGENT_DIR="/usr/lib/ambari-agent/lib/ambari_stomp"
-WS4PY_AGENT_DIR="/usr/lib/ambari-agent/lib/ambari_ws4py"
-AMBARI_AGENT="/usr/lib/python2.6/site-packages/ambari_agent"
+COMMON_DIR="/usr/lib/ambari-agent/lib/ambari_commons"
+RESOURCE_MANAGEMENT_DIR="/usr/lib/ambari-agent/lib/resource_management"
+JINJA_DIR="/usr/lib/ambari-agent/lib/ambari_jinja2"
+SIMPLEJSON_DIR="/usr/lib/ambari-agent/lib/ambari_simplejson"
+OLD_OLD_COMMON_DIR="/usr/lib/ambari-agent/lib/common_functions"
+AMBARI_AGENT="/usr/lib/ambari-agent/lib/ambari_agent"
 PYTHON_WRAPER_TARGET="/usr/bin/ambari-python-wrap"
 AMBARI_AGENT_VAR="/var/lib/ambari-agent"
 AMBARI_AGENT_BINARY="/etc/init.d/ambari-agent"
 AMBARI_AGENT_BINARY_SYMLINK="/usr/sbin/ambari-agent"
 
+
+OLD_COMMON_DIR="/usr/lib/python2.6/site-packages/ambari_commons"
+OLD_RESOURCE_MANAGEMENT_DIR="/usr/lib/python2.6/site-packages/resource_management"
+OLD_JINJA_DIR="/usr/lib/python2.6/site-packages/ambari_jinja2"
+OLD_SIMPLEJSON_DIR="/usr/lib/python2.6/site-packages/ambari_simplejson"
+OLD_AMBARI_AGENT_DIR="/usr/lib/python2.6/site-packages/ambari_agent"
+
 clean_pyc_files(){
   # cleaning old *.pyc files
   find ${RESOURCE_MANAGEMENT_DIR:?} -name *.pyc -exec rm {} \;
@@ -53,41 +52,17 @@ do_install(){
     mv /etc/ambari-agent/conf.save /etc/ambari-agent/conf_$(date '+%d_%m_%y_%H_%M').save
   fi
 
+  # these symlinks (or directories) where created in ambari releases prior to ambari-2.6.2. Do clean up.   
+  rm -rf "$OLD_COMMON_DIR" "$OLD_RESOURCE_MANAGEMENT_DIR" "$OLD_JINJA_DIR" "$OLD_SIMPLEJSON_DIR" "$OLD_OLD_COMMON_DIR" "$OLD_AMBARI_AGENT_DIR"
+
   # setting up /usr/sbin/ambari-agent symlink
   rm -f "$AMBARI_AGENT_BINARY_SYMLINK"
   ln -s "$AMBARI_AGENT_BINARY" "$AMBARI_AGENT_BINARY_SYMLINK"
-    
-  # setting ambari_commons shared resource
-  rm -rf "$OLD_COMMON_DIR"
-  if [ ! -d "$COMMON_DIR" ]; then
-    ln -s "$COMMON_DIR_AGENT" "$COMMON_DIR"
-  fi
-  # setting resource_management shared resource
-  if [ ! -d "$RESOURCE_MANAGEMENT_DIR" ]; then
-    ln -s "$RESOURCE_MANAGEMENT_DIR_AGENT" "$RESOURCE_MANAGEMENT_DIR"
-  fi
-  # setting jinja2 shared resource
-  if [ ! -d "$JINJA_DIR" ]; then
-    ln -s "$JINJA_AGENT_DIR" "$JINJA_DIR"
-  fi
-  # setting simplejson shared resource
-  if [ ! -d "$SIMPLEJSON_DIR" ]; then
-    ln -s "$SIMPLEJSON_AGENT_DIR" "$SIMPLEJSON_DIR"
-  fi
-  # setting stomp shared resource
-  if [ ! -d "$STOMP_DIR" ]; then
-    ln -s "$STOMP_AGENT_DIR" "$STOMP_DIR"
-  fi
 
-  # setting ws4py shared resource
-  if [ ! -d "$WS4PY_DIR" ]; then
-    ln -s "$WS4PY_AGENT_DIR" "$WS4PY_DIR"
-  fi
-  
   # on nano Ubuntu, when umask=027 those folders are created without 'x' bit for 'others'.
   # which causes failures when hadoop users try to access tmp_dir
   chmod a+x $AMBARI_AGENT_VAR
-  
+
   chmod 1777 $AMBARI_AGENT_VAR/tmp
   chmod 700 $AMBARI_AGENT_VAR/keys
   chmod 700 $AMBARI_AGENT_VAR/data
@@ -137,6 +112,14 @@ do_install(){
       mv $BAK ${BAK}_$(date '+%d_%m_%y_%H_%M').save
     fi
   fi
+
+  if [ -f "$AMBARI_ENV_RPMSAVE" ] ; then
+    PYTHON_PATH_LINE='export PYTHONPATH=/usr/lib/ambari-agent/lib:$PYTHONPATH'
+    grep "^$PYTHON_PATH_LINE\$" "$AMBARI_ENV_RPMSAVE" > /dev/null
+    if [ $? -ne 0 ] ; then
+      echo -e "\n$PYTHON_PATH_LINE" >> $AMBARI_ENV_RPMSAVE
+    fi
+  fi
 }
 
 do_remove(){
@@ -150,37 +133,25 @@ do_remove(){
     mv /etc/ambari-agent/conf.save /etc/ambari-agent/conf_$(date '+%d_%m_%y_%H_%M').save
   fi
   mv /etc/ambari-agent/conf /etc/ambari-agent/conf.save
-    
+
   if [ -f "$PYTHON_WRAPER_TARGET" ]; then
     rm -f "$PYTHON_WRAPER_TARGET"
   fi
-  
+
   if [ -d "$COMMON_DIR" ]; then
-    rm -f $COMMON_DIR
+    rm -rf $COMMON_DIR
   fi
-  
+
   if [ -d "$RESOURCE_MANAGEMENT_DIR" ]; then
     rm -rf $RESOURCE_MANAGEMENT_DIR
   fi
-  
+
   if [ -d "$JINJA_DIR" ]; then
     rm -rf $JINJA_DIR
   fi
 
   if [ -d "$SIMPLEJSON_DIR" ]; then
-    rm -f $SIMPLEJSON_DIR
-  fi
-
-  if [ -d "$STOMP_DIR" ]; then
-    rm -f $STOMP_DIR
-  fi
-
-  if [ -d "$WS4PY_DIR" ]; then
-    rm -f $WS4PY_DIR
-  fi
-
-  if [ -d "$OLD_COMMON_DIR" ]; then
-    rm -f $OLD_COMMON_DIR
+    rm -rf $SIMPLEJSON_DIR
   fi
 
   # if server package exists, restore their settings

ambari-agent/pom.xml

@@ -36,7 +36,7 @@
     <package.log.dir>/var/log/ambari-agent</package.log.dir>
     <package.pid.dir>/var/run/ambari-agent</package.pid.dir>
     <skipTests>false</skipTests>
-    <agent.install.dir>/usr/lib/python2.6/site-packages/ambari_agent</agent.install.dir>
+    <agent.install.dir>/usr/lib/ambari-agent/lib/ambari_agent</agent.install.dir>
     <ambari_commons.install.dir>/usr/lib/ambari-agent/lib/ambari_commons</ambari_commons.install.dir>
     <resource_management.install.dir>/usr/lib/ambari-agent/lib/resource_management</resource_management.install.dir>
     <jinja.install.dir>/usr/lib/ambari-agent/lib/ambari_jinja2</jinja.install.dir>
@@ -286,17 +286,6 @@
                 </source>
               </sources>
             </mapping>
-            <mapping>
-              <directory>/usr/lib/python2.6/site-packages</directory>
-              <filemode>755</filemode>
-              <username>root</username>
-              <groupname>root</groupname>
-              <sources>
-                <source>
-                  <location>${project.build.directory}${dirsep}${project.artifactId}-${project.version}/usr/lib/python2.6/site-packages</location>
-                </source>
-              </sources>
-            </mapping>
             <mapping>
               <directory>/var/lib/ambari-agent</directory>
               <filemode>755</filemode>

ambari-agent/src/main/package/rpm/posttrans_agent.sh

@@ -14,29 +14,9 @@
 # limitations under the License
 
 
-RESOURCE_MANAGEMENT_DIR="/usr/lib/python2.6/site-packages/resource_management"
-RESOURCE_MANAGEMENT_DIR_AGENT="/usr/lib/ambari-agent/lib/resource_management"
-JINJA_DIR="/usr/lib/python2.6/site-packages/ambari_jinja2"
-JINJA_AGENT_DIR="/usr/lib/ambari-agent/lib/ambari_jinja2"
 AMBARI_AGENT_BINARY="/etc/init.d/ambari-agent"
 AMBARI_AGENT_BINARY_SYMLINK="/usr/sbin/ambari-agent"
 
-# remove RESOURCE_MANAGEMENT_DIR if it's a directory
-if [ -d "$RESOURCE_MANAGEMENT_DIR" ]; then  # resource_management dir exists
-  if [ ! -L "$RESOURCE_MANAGEMENT_DIR" ]; then # resource_management dir is not link
-    rm -rf "$RESOURCE_MANAGEMENT_DIR"
-  fi
-fi
-# setting resource_management shared resource
-if [ ! -d "$RESOURCE_MANAGEMENT_DIR" ]; then
-  ln -s "$RESOURCE_MANAGEMENT_DIR_AGENT" "$RESOURCE_MANAGEMENT_DIR"
-fi
-
-# setting jinja2 shared resource
-if [ ! -d "$JINJA_DIR" ]; then
-  ln -s "$JINJA_AGENT_DIR" "$JINJA_DIR"
-fi
-
 # setting ambari-agent binary symlink
 if [ ! -f "$AMBARI_AGENT_BINARY_SYMLINK" ]; then
   ln -s "$AMBARI_AGENT_BINARY" "$AMBARI_AGENT_BINARY_SYMLINK"

ambari-agent/src/main/python/ambari_agent/ActionQueue.py

@@ -110,7 +110,8 @@ class ActionQueue(threading.Thread):
     for command in commands:
 
       logger.info("Canceling command with taskId = {tid}".format(tid = str(command['target_task_id'])))
-      logger.debug(pprint.pformat(command))
+      if logger.isEnabledFor(logging.DEBUG):
+        logger.debug(pprint.pformat(command))
 
       task_id = command['target_task_id']
       reason = command['reason']
@@ -133,8 +134,8 @@ class ActionQueue(threading.Thread):
       self.customServiceOrchestrator.cancel_command(task_id, reason)
 
   def run(self):
-    try:
-      while not self.stop_event.is_set():
+    while not self.stop_event.is_set():
+      try:
         self.processBackgroundQueueSafeEmpty()
         self.fillRecoveryCommands()
         try:
@@ -172,10 +173,8 @@ class ActionQueue(threading.Thread):
             pass
         except (Queue.Empty):
           pass
-    except:
-      logger.exception("ActionQueue thread failed with exception:")
-      raise
-
+      except:
+        logger.exception("ActionQueue thread failed with exception. Re-running it")
     logger.info("ActionQueue thread has successfully finished")
 
   def fillRecoveryCommands(self):
@@ -200,7 +199,7 @@ class ActionQueue(threading.Thread):
   def process_command(self, command):
     # make sure we log failures
     commandType = command['commandType']
-    logger.debug("Took an element of Queue (command type = %s)." % commandType)
+    logger.debug("Took an element of Queue (command type = %s).", commandType)
     try:
       if commandType in [self.EXECUTION_COMMAND, self.BACKGROUND_EXECUTION_COMMAND, self.AUTO_EXECUTION_COMMAND]:
         try:
@@ -213,7 +212,7 @@ class ActionQueue(threading.Thread):
           if self.recovery_manager.enabled():
             self.recovery_manager.on_execution_command_finish()
       else:
-        logger.error("Unrecognized command " + pprint.pformat(command))
+        logger.error("Unrecognized command %s", pprint.pformat(command))
     except Exception:
       logger.exception("Exception while processing {0} command".format(commandType))
 
@@ -447,14 +446,14 @@ class ActionQueue(threading.Thread):
     self.customServiceOrchestrator
 
   def on_background_command_complete_callback(self, process_condensed_result, handle):
-    logger.debug('Start callback: %s' % process_condensed_result)
-    logger.debug('The handle is: %s' % handle)
+    logger.debug('Start callback: %s', process_condensed_result)
+    logger.debug('The handle is: %s', handle)
     status = self.COMPLETED_STATUS if handle.exitCode == 0 else self.FAILED_STATUS
 
     aborted_postfix = self.customServiceOrchestrator.command_canceled_reason(handle.command['taskId'])
     if aborted_postfix:
       status = self.FAILED_STATUS
-      logger.debug('Set status to: %s , reason = %s' % (status, aborted_postfix))
+      logger.debug('Set status to: %s , reason = %s', status, aborted_postfix)
     else:
       aborted_postfix = ''
 

ambari-agent/src/main/python/ambari_agent/AmbariAgent.py

@@ -27,7 +27,7 @@ from Controller import AGENT_AUTO_RESTART_EXIT_CODE
 if os.environ.has_key("PYTHON_BIN"):
   AGENT_SCRIPT = os.path.join(os.environ["PYTHON_BIN"],"site-packages/ambari_agent/main.py")
 else:
-  AGENT_SCRIPT = "/usr/lib/python2.6/site-packages/ambari_agent/main.py"
+  AGENT_SCRIPT = "/usr/lib/ambari-agent/lib/ambari_agent/main.py"
 if os.environ.has_key("AMBARI_PID_DIR"):
   AGENT_PID_FILE = os.path.join(os.environ["AMBARI_PID_DIR"],"ambari-agent.pid")
 else:

ambari-agent/src/main/python/ambari_agent/CommandStatusDict.py

@@ -114,7 +114,7 @@ class CommandStatusDict():
             in_progress_report = self.generate_in_progress_report(command, report)
             resultReports[cluster_id].append(in_progress_report)
         elif command ['commandType'] in [ActionQueue.AUTO_EXECUTION_COMMAND]:
-          logger.debug("AUTO_EXECUTION_COMMAND task deleted " + str(command['commandId']))
+          logger.debug("AUTO_EXECUTION_COMMAND task deleted %s", command['commandId'])
           self.reported_reports.add(key)
           pass
       return resultReports

ambari-agent/src/main/python/ambari_agent/Constants.py

@@ -43,5 +43,4 @@ ALERTS_STATUS_REPORTS_ENDPOINT = '/reports/alerts_status'
 HEARTBEAT_ENDPOINT = '/heartbeat'
 REGISTRATION_ENDPOINT = '/register'
 
-AGENT_TMP_DIR = "/var/lib/ambari-agent/tmp"
 CORRELATION_ID_STRING = 'correlationId'

ambari-agent/src/main/python/ambari_agent/Controller.py

@@ -169,18 +169,17 @@ class Controller(threading.Thread):
     while not self.isRegistered:
       try:
         data = json.dumps(self.register.build(self.version))
-        prettyData = pprint.pformat(data)
-
         try:
           server_ip = socket.gethostbyname(self.hostname)
-          logger.info("Registering with %s (%s) (agent=%s)", self.hostname, server_ip, prettyData)
+          logger.info("Registering with %s (%s) (agent=%s)", self.hostname, server_ip, data)
         except socket.error:
           logger.warn("Unable to determine the IP address of '%s', agent registration may fail (agent=%s)",
-                      self.hostname, prettyData)
+                      self.hostname, data)
 
         ret = self.sendRequest(self.registerUrl, data)
-        prettyData = pprint.pformat(ret)
-        logger.debug("Registration response is %s", prettyData)
+
+        if logger.isEnabledFor(logging.DEBUG):
+          logger.debug("Registration response is %s", pprint.pformat(ret))
 
         # exitstatus is a code of error which was raised on server side.
         # exitstatus = 0 (OK - Default)
@@ -208,7 +207,7 @@ class Controller(threading.Thread):
         self.update_caches_from_heartbeat(ret)
         self.recovery_manager.update_configuration_from_registration(ret)
         self.config.update_configuration_from_registration(ret)
-        logger.debug("Updated config:" + str(self.config))
+        logger.debug("Updated config: %s", self.config)
 
         # Start StatusCommandExecutor child process or restart it if already running
         # in order to receive up to date agent config.
@@ -453,7 +452,7 @@ class Controller(threading.Thread):
           logger.log(logging_level, "Executing alert commands")
           self.alert_scheduler_handler.execute_alert(response['alertExecutionCommands'])
 
-        if "true" == response['restartAgent']:
+        if response['restartAgent']:
           logger.error("Received the restartAgent command")
           self.restartAgent()
         else:
@@ -591,11 +590,11 @@ class Controller(threading.Thread):
     if LiveStatus.SERVICES:
       return
 
-    logger.debug("Updating components map of cluster " + cluster_name)
+    logger.debug("Updating components map of cluster %s", cluster_name)
 
     # May throw IOError on server connection error
     response = self.sendRequest(self.componentsUrl + cluster_name, None)
-    logger.debug("Response from %s was %s", self.serverHostname, str(response))
+    logger.debug("Response from %s was %s", self.serverHostname, response)
 
     services, client_components, server_components = [], [], []
     for service, components in response['components'].items():
@@ -612,9 +611,9 @@ class Controller(threading.Thread):
     LiveStatus.COMPONENTS = server_components
 
     logger.debug("Components map updated")
-    logger.debug("LiveStatus.SERVICES" + str(LiveStatus.SERVICES))
-    logger.debug("LiveStatus.CLIENT_COMPONENTS" + str(LiveStatus.CLIENT_COMPONENTS))
-    logger.debug("LiveStatus.COMPONENTS" + str(LiveStatus.COMPONENTS))
+    logger.debug("LiveStatus.SERVICES %s", LiveStatus.SERVICES)
+    logger.debug("LiveStatus.CLIENT_COMPONENTS %s", LiveStatus.CLIENT_COMPONENTS)
+    logger.debug("LiveStatus.COMPONENTS %s", LiveStatus.COMPONENTS)
 
   def get_status_commands_executor(self):
     return self.statusCommandsExecutor

ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py

@@ -31,6 +31,8 @@ from resource_management.libraries.functions.log_process_information import log_
 from resource_management.core.utils import PasswordString
 from ambari_agent.Utils import Utils
 import subprocess
+from ambari_commons.constants import AGENT_TMP_DIR
+import hostname
 import Constants
 
 
@@ -82,7 +84,7 @@ class CustomServiceOrchestrator():
     self.tmp_dir = self.config.get('agent', 'prefix')
     self.force_https_protocol = self.config.get_force_https_protocol_name()
     self.ca_cert_file_path = self.config.get_ca_cert_file_path()
-    self.exec_tmp_dir = Constants.AGENT_TMP_DIR
+    self.exec_tmp_dir = AGENT_TMP_DIR
     self.file_cache = initializer_module.file_cache
     self.status_commands_stdout = os.path.join(self.tmp_dir,
                                                'status_command_stdout.txt')
@@ -108,7 +110,7 @@ class CustomServiceOrchestrator():
 
   def map_task_to_process(self, task_id, processId):
     with self.commands_in_progress_lock:
-      logger.debug('Maps taskId=%s to pid=%s' % (task_id, processId))
+      logger.debug('Maps taskId=%s to pid=%s', task_id, processId)
       self.commands_in_progress[task_id] = processId
 
   def cancel_command(self, task_id, reason):
@@ -440,7 +442,7 @@ class CustomServiceOrchestrator():
   def command_canceled_reason(self, task_id):
     with self.commands_in_progress_lock:
       if self.commands_in_progress.has_key(task_id):#Background command do not push in this collection (TODO)
-        logger.debug('Pop with taskId %s' % task_id)
+        logger.debug('Pop with taskId %s', task_id)
         pid = self.commands_in_progress.pop(task_id)
         if not isinstance(pid, int):
           reason = pid

ambari-agent/src/main/python/ambari_agent/DataCleaner.py

@@ -68,7 +68,7 @@ class DataCleaner(threading.Thread):
     logger.info('Data cleanup thread killed.')
 
   def cleanup(self):
-    logger.debug("Cleaning up inside directory " + self.data_dir)
+    logger.debug("Cleaning up inside directory %s", self.data_dir)
     now = time.time()
     total_size_bytes = 0
     file_path_to_timestamp = {}
@@ -82,7 +82,7 @@ class DataCleaner(threading.Thread):
             file_age = now - os.path.getmtime(file_path)
             if file_age > self.file_max_age:
               os.remove(os.path.join(file_path))
-              logger.debug('Removed file: ' + file_path)
+              logger.debug('Removed file: %s', file_path)
             else:
               # Since file wasn't deleted in first pass, consider it for the second one with oldest files first
               file_size = os.path.getsize(file_path)

ambari-agent/src/main/python/ambari_agent/Heartbeat.py

@@ -75,12 +75,12 @@ class Heartbeat:
     if int(id) == 0:
       componentsMapped = False
 
-    logger.debug("Building Heartbeat: {responseId = %s, timestamp = %s, "
-                "commandsInProgress = %s, componentsMapped = %s,"
-                "recoveryTimestamp = %s}",
-        str(id), str(timestamp), repr(commandsInProgress), repr(componentsMapped), str(recovery_timestamp))
-
-    logger.debug("Heartbeat: %s", pformat(heartbeat))
+    if logger.isEnabledFor(logging.DEBUG):
+      logger.debug("Building Heartbeat: {responseId = %s, timestamp = %s, "
+                   "commandsInProgress = %s, componentsMapped = %s, "
+                   "recoveryTimestamp = %s}",
+                   id, timestamp, commandsInProgress, componentsMapped, recovery_timestamp)
+      logger.debug("Heartbeat: %s", pformat(heartbeat))
 
     hostInfo = HostInfo(self.config)
     if add_state:
@@ -93,8 +93,9 @@ class Heartbeat:
       mounts = Hardware(config=self.config, cache_info=False).osdisks()
       heartbeat['mounts'] = mounts
 
-      logger.debug("agentEnv: %s", str(nodeInfo))
-      logger.debug("mounts: %s", str(mounts))
+      if logger.isEnabledFor(logging.DEBUG):
+        logger.debug("agentEnv: %s", nodeInfo)
+        logger.debug("mounts: %s", mounts)
 
     if self.collector is not None:
       heartbeat['alerts'] = self.collector.alerts()

ambari-agent/src/main/python/ambari_agent/HostCleanup.py

@@ -20,7 +20,7 @@ limitations under the License.
 # For compatibility with different OSes
 # Edit PYTHONPATH to be able to import common_functions
 import sys
-sys.path.append("/usr/lib/python2.6/site-packages/")
+sys.path.append("/usr/lib/ambari-agent/lib/")
 ########################################################
 
 import os
@@ -39,7 +39,7 @@ import glob
 import pwd
 import re
 from AmbariConfig import AmbariConfig
-from ambari_agent.Constants import AGENT_TMP_DIR
+from ambari_commons.constants import AGENT_TMP_DIR
 from ambari_commons import OSCheck, OSConst
 from ambari_commons.constants import AMBARI_SUDO_BINARY
 from ambari_commons.os_family_impl import OsFamilyImpl, OsFamilyFuncImpl

ambari-agent/src/main/python/ambari_agent/LiveStatus.py

@@ -62,6 +62,5 @@ class LiveStatus:
     if active_config is not None:
       livestatus['configurationTags'] = active_config
 
-    logger.debug("The live status for component " + str(self.component) +
-                 " of service " + str(self.service) + " is " + str(livestatus))
+    logger.debug("The live status for component %s of service %s is %s", self.component, self.service, livestatus)
     return livestatus

ambari-agent/src/main/python/ambari_agent/PythonExecutor.py

@@ -93,7 +93,8 @@ class PythonExecutor(object):
     The structured out file, however, is preserved during multiple invocations that use the same file.
     """
     pythonCommand = self.python_command(script, script_params)
-    logger.debug("Running command " + pprint.pformat(pythonCommand))
+    if logger.isEnabledFor(logging.DEBUG):
+      logger.debug("Running command %s", pprint.pformat(pythonCommand))
 
     if handle is None:
       tmpout, tmperr = self.open_subprocess_files(tmpoutfile, tmperrfile, override_output_files, backup_log_files)
@@ -128,7 +129,7 @@ class PythonExecutor(object):
     Log some useful information after task failure.
     """
     pass
-    #logger.info("Command " + pprint.pformat(pythonCommand) + " failed with exitcode=" + str(result['exitcode']))
+    #logger.info("Command %s failed with exitcode=%s", pprint.pformat(pythonCommand), result['exitcode'])
     #log_process_information(logger)
 
   def prepare_process_result(self, returncode, tmpoutfile, tmperrfile, tmpstructedoutfile, timeout=None):
@@ -139,7 +140,7 @@ class PythonExecutor(object):
               (" after waiting %s secs" % str(timeout) if timeout else "")
       returncode = 999
     result = self.condenseOutput(out, error, returncode, structured_out)
-    logger.debug("Result: %s" % result)
+    logger.debug("Result: %s", result)
     return result
 
   def read_result_from_files(self, out_path, err_path, structured_out_path):
@@ -224,10 +225,10 @@ class BackgroundThread(threading.Thread):
   def run(self):
     process_out, process_err = self.pythonExecutor.open_subprocess_files(self.holder.out_file, self.holder.err_file, True)
 
-    logger.debug("Starting process command %s" % self.holder.command)
+    logger.debug("Starting process command %s", self.holder.command)
     process = self.pythonExecutor.launch_python_subprocess(self.holder.command, process_out, process_err)
 
-    logger.debug("Process has been started. Pid = %s" % process.pid)
+    logger.debug("Process has been started. Pid = %s", process.pid)
 
     self.holder.handle.pid = process.pid
     self.holder.handle.status = BackgroundCommandExecutionHandle.RUNNING_STATUS
@@ -237,6 +238,6 @@ class BackgroundThread(threading.Thread):
 
     self.holder.handle.exitCode = process.returncode
     process_condensed_result = self.pythonExecutor.prepare_process_result(process.returncode, self.holder.out_file, self.holder.err_file, self.holder.structured_out_file)
-    logger.debug("Calling callback with args %s" % process_condensed_result)
+    logger.debug("Calling callback with args %s", process_condensed_result)
     self.holder.handle.on_background_command_complete_callback(process_condensed_result, self.holder.handle)
-    logger.debug("Exiting from thread for holder pid %s" % self.holder.handle.pid)
+    logger.debug("Exiting from thread for holder pid %s", self.holder.handle.pid)

ambari-agent/src/main/python/ambari_agent/PythonReflectiveExecutor.py

@@ -46,7 +46,8 @@ class PythonReflectiveExecutor(PythonExecutor):
                override_output_files = True, backup_log_files = True,
                handle = None, log_info_on_failure=True):
     pythonCommand = self.python_command(script, script_params)
-    logger.debug("Running command reflectively " + pprint.pformat(pythonCommand))
+    if logger.isEnabledFor(logging.DEBUG):
+      logger.debug("Running command reflectively %s", pprint.pformat(pythonCommand))
     
     script_dir = os.path.dirname(script)
     self.open_subprocess_files(tmpoutfile, tmperrfile, override_output_files, backup_log_files)

ambari-agent/src/main/python/ambari_agent/RecoveryManager.py

@@ -568,7 +568,8 @@ class RecoveryManager:
 
 
     if dictionary and "recoveryConfig" in dictionary:
-      logger.info("RecoverConfig = " + pprint.pformat(dictionary["recoveryConfig"]))
+      if logger.isEnabledFor(logging.INFO):
+        logger.info("RecoverConfig = %s", pprint.pformat(dictionary["recoveryConfig"]))
       config = dictionary["recoveryConfig"]
       if "type" in config:
         if config["type"] in ["AUTO_INSTALL_START", "AUTO_START", "FULL"]:
@@ -677,10 +678,10 @@ class RecoveryManager:
     """
     if not self.enabled():
       return
-      
+
     if not command.has_key(self.ROLE_COMMAND) or not self.configured_for_recovery(command['role']):
       return
-      
+
     if status == ActionQueue.COMPLETED_STATUS:
       if command[self.ROLE_COMMAND] == self.ROLE_COMMAND_START:
         self.update_current_status(command[self.ROLE], LiveStatus.LIVE_STATUS)
@@ -712,10 +713,10 @@ class RecoveryManager:
     """
     if not self.enabled():
       return
-      
+
     if not self.COMMAND_TYPE in command or not command[self.COMMAND_TYPE] == ActionQueue.EXECUTION_COMMAND:
       return
-      
+
     if not self.ROLE in command:
       return
 

ambari-agent/src/main/python/ambari_agent/StatusCommandsExecutor.py

@@ -64,7 +64,8 @@ class SingleProcessStatusCommandsExecutor(StatusCommandsExecutor):
                   command['serviceName'] + " of cluster " + \
                   command['clusterName'] + " to the queue.")
       self.statusCommandQueue.put(command)
-      logger.debug(pprint.pformat(command))
+      if logger.isEnabledFor(logging.DEBUG):
+        logger.debug(pprint.pformat(command))
 
   def process_results(self):
     """

ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py

@@ -22,7 +22,6 @@ import httplib
 import imp
 import time
 import urllib
-from alerts.base_alert import BaseAlert
 from alerts.metric_alert import MetricAlert
 import ambari_simplejson as json
 import logging
@@ -30,6 +29,7 @@ import re
 import uuid
 
 from resource_management.libraries.functions.get_port_from_url import get_port_from_url
+from ambari_commons import inet_utils
 
 logger = logging.getLogger()
 
@@ -64,10 +64,11 @@ class AmsAlert(MetricAlert):
     # use the URI lookup keys to get a final URI value to query
     alert_uri = self._get_uri_from_structure(self.uri_property_keys)
 
-    logger.debug("[Alert][{0}] Calculated metric URI to be {1} (ssl={2})".format(
-      self.get_name(), alert_uri.uri, str(alert_uri.is_ssl_enabled)))
+    if logger.isEnabledFor(logging.DEBUG):
+      logger.debug("[Alert][{0}] Calculated metric URI to be {1} (ssl={2})".format(
+        self.get_name(), alert_uri.uri, str(alert_uri.is_ssl_enabled)))
 
-    host = BaseAlert.get_host_from_url(alert_uri.uri)
+    host = inet_utils.get_host_from_url(alert_uri.uri)
     if host is None:
       host = self.host_name
 
@@ -94,7 +95,8 @@ class AmsAlert(MetricAlert):
 
         collect_result = self._get_result(value_list[0] if compute_result is None else compute_result)
 
-        logger.debug("[Alert][{0}] Computed result = {1}".format(self.get_name(), str(value_list)))
+        if logger.isEnabledFor(logging.DEBUG):
+          logger.debug("[Alert][{0}] Computed result = {1}".format(self.get_name(), str(value_list)))
 
     return (collect_result, value_list)
 

ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py

@@ -532,49 +532,4 @@ class BaseAlert(object):
     :param state: the state of the alert in uppercase (such as OK, WARNING, etc)
     :return:  the parameterized text
     '''
-    return '{0}'
-
-  """
-  See RFC3986, Appendix B
-  Tested on the following cases:
-    "192.168.54.1"
-    "192.168.54.2:7661
-    "hdfs://192.168.54.3/foo/bar"
-    "ftp://192.168.54.4:7842/foo/bar"
-
-    Returns None if only a port is passed in
-  """
-  @staticmethod
-  def get_host_from_url(uri):
-    if uri is None:
-      return None
-
-    # if not a string, return None
-    if not isinstance(uri, basestring):
-      return None
-
-    # RFC3986, Appendix B
-    parts = re.findall('^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?', uri)
-
-    # index of parts
-    # scheme    = 1
-    # authority = 3
-    # path      = 4
-    # query     = 6
-    # fragment  = 8
-
-    host_and_port = uri
-    if 0 == len(parts[0][1]):
-      host_and_port = parts[0][4]
-    elif 0 == len(parts[0][2]):
-      host_and_port = parts[0][1]
-    elif parts[0][2].startswith("//"):
-      host_and_port = parts[0][3]
-
-    if -1 == host_and_port.find(':'):
-      if host_and_port.isdigit():
-        return None
-
-      return host_and_port
-    else:
-      return host_and_port.split(':')[0]
+    return '{0}'

ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py

@@ -30,7 +30,8 @@ from alerts.base_alert import BaseAlert
 from ambari_commons.urllib_handlers import RefreshHeaderProcessor
 from resource_management.libraries.functions.get_port_from_url import get_port_from_url
 from resource_management.libraries.functions.curl_krb_request import curl_krb_request
-from ambari_agent import Constants
+from ambari_commons import inet_utils
+from ambari_commons.constants import AGENT_TMP_DIR
 
 logger = logging.getLogger(__name__)
 
@@ -80,7 +81,7 @@ class MetricAlert(BaseAlert):
     logger.debug("[Alert][{0}] Calculated metric URI to be {1} (ssl={2})".format(
         self.get_name(), alert_uri.uri, str(alert_uri.is_ssl_enabled)))
 
-    host = BaseAlert.get_host_from_url(alert_uri.uri)
+    host = inet_utils.get_host_from_url(alert_uri.uri)
     if host is None:
       host = self.host_name
 
@@ -107,8 +108,8 @@ class MetricAlert(BaseAlert):
 
         collect_result = self._get_result(value_list[0] if check_value is None else check_value)
 
-        logger.debug("[Alert][{0}] Resolved values = {1}".format(self.get_name(), str(value_list)))
-
+        if logger.isEnabledFor(logging.DEBUG):
+          logger.debug("[Alert][{0}] Resolved values = {1}".format(self.get_name(), str(value_list)))
     return (collect_result, value_list)
 
 
@@ -202,7 +203,7 @@ class MetricAlert(BaseAlert):
       content = ''
       try:
         if kerberos_principal is not None and kerberos_keytab is not None and security_enabled:
-          tmp_dir = Constants.AGENT_TMP_DIR
+          tmp_dir = AGENT_TMP_DIR
           if tmp_dir is None:
             tmp_dir = gettempdir()
 

ambari-agent/src/main/python/ambari_agent/alerts/port_alert.py

@@ -24,7 +24,7 @@ import time
 from alerts.base_alert import BaseAlert
 from resource_management.libraries.functions.get_port_from_url import get_port_from_url
 from ambari_commons import OSCheck
-from ambari_commons.inet_utils import resolve_address
+from ambari_commons.inet_utils import resolve_address, get_host_from_url
 logger = logging.getLogger(__name__)
 
 # default timeouts
@@ -112,7 +112,7 @@ class PortAlert(BaseAlert):
           break
 
 
-    host = BaseAlert.get_host_from_url(uri_value)
+    host = get_host_from_url(uri_value)
     if host is None or host == "localhost" or host == "0.0.0.0":
       host = self.host_name
       host_not_specified = True

ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py

@@ -26,7 +26,7 @@ from alerts.base_alert import BaseAlert
 from resource_management.core.environment import Environment
 from resource_management.libraries.script.script import Script
 from resource_management.libraries.functions.curl_krb_request import KERBEROS_KINIT_TIMER_PARAMETER
-from ambari_agent import Constants
+from ambari_commons.constants import AGENT_TMP_DIR
 
 logger = logging.getLogger(__name__)
 
@@ -115,7 +115,7 @@ class ScriptAlert(BaseAlert):
       matchObj = re.match( r'((.*)services(.*)package)', self.path_to_script)
       if matchObj:
         basedir = matchObj.group(1)
-        with Environment(basedir, tmp_dir=Constants.AGENT_TMP_DIR, logger=logging.getLogger('ambari_alerts')) as env:
+        with Environment(basedir, tmp_dir=AGENT_TMP_DIR, logger=logging.getLogger('ambari_alerts')) as env:
           result = cmd_module.execute(configurations, self.parameters, self.host_name)
       else:
         result = cmd_module.execute(configurations, self.parameters, self.host_name)

ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py

@@ -21,9 +21,7 @@ limitations under the License.
 import logging
 import time
 import urllib2
-import ssl
 
-from functools import wraps
 from urllib2 import HTTPError
 
 from tempfile import gettempdir
@@ -33,8 +31,8 @@ from resource_management.libraries.functions.get_port_from_url import get_port_f
 from resource_management.libraries.functions.get_path_from_url import get_path_from_url
 from resource_management.libraries.functions.curl_krb_request import curl_krb_request
 from ambari_commons import OSCheck
-from ambari_commons.inet_utils import resolve_address, ensure_ssl_using_protocol
-from ambari_agent import Constants
+from ambari_commons.inet_utils import resolve_address, ensure_ssl_using_protocol, get_host_from_url
+from ambari_commons.constants import AGENT_TMP_DIR
 from ambari_agent.AmbariConfig import AmbariConfig
 
 # hashlib is supplied as of Python 2.5 as the replacement interface for md5
@@ -135,7 +133,7 @@ class WebAlert(BaseAlert):
       uri_path = get_path_from_url(string_uri)
 
     # start building the URL manually
-    host = BaseAlert.get_host_from_url(alert_uri.uri)
+    host = get_host_from_url(alert_uri.uri)
     if host is None:
       host = self.host_name
 
@@ -194,7 +192,7 @@ class WebAlert(BaseAlert):
         # Create the kerberos credentials cache (ccache) file and set it in the environment to use
         # when executing curl. Use the md5 hash of the combination of the principal and keytab file
         # to generate a (relatively) unique cache filename so that we can use it as needed.
-        tmp_dir = Constants.AGENT_TMP_DIR
+        tmp_dir = AGENT_TMP_DIR
         if tmp_dir is None:
           tmp_dir = gettempdir()
 

ambari-agent/src/main/python/ambari_agent/security.py

@@ -52,8 +52,7 @@ class VerifiedHTTPSConnection(httplib.HTTPSConnection):
 
   def connect(self):
     self.two_way_ssl_required = self.config.isTwoWaySSLConnection(self.host)
-    logger.debug("Server two-way SSL authentication required: %s" % str(
-      self.two_way_ssl_required))
+    logger.debug("Server two-way SSL authentication required: %s", self.two_way_ssl_required)
     if self.two_way_ssl_required is True:
       logger.info(
         'Server require two-way SSL authentication. Use it instead of one-way...')
@@ -277,9 +276,10 @@ class CertificateManager():
     f.close()
     try:
       data = json.loads(response)
-      logger.debug("Sign response from Server: \n" + pprint.pformat(data))
+      if logger.isEnabledFor(logging.DEBUG):
+        logger.debug("Sign response from Server: \n" + pprint.pformat(data))
     except Exception:
-      logger.warn("Malformed response! data: " + str(data))
+      logger.warn("Malformed response! data: %s", data)
       data = {'result': 'ERROR'}
     result = data['result']
     if result == 'OK':

ambari-agent/src/test/python/ambari_agent/TestController.py

@@ -447,7 +447,7 @@ class TestController(unittest.TestCase):
     self.controller.sendRequest = sendRequest
 
     self.controller.responseId = 1
-    response = {"responseId":"2", "restartAgent":"false"}
+    response = {"responseId":"2", "restartAgent":False}
     sendRequest.return_value = response
 
     def one_heartbeat(*args, **kwargs):
@@ -521,7 +521,7 @@ class TestController(unittest.TestCase):
 
     # wrong responseId => restart
     self.controller.responseId = 2
-    response = {"responseId":"2", "restartAgent":"false"}
+    response = {"responseId":"2", "restartAgent":False}
 
     restartAgent = MagicMock(name="restartAgent")
     self.controller.restartAgent = restartAgent
@@ -553,7 +553,7 @@ class TestController(unittest.TestCase):
     # restartAgent command
     self.controller.responseId = 1
     self.controller.DEBUG_STOP_HEARTBEATING = False
-    response["restartAgent"] = "true"
+    response["restartAgent"] = True
     restartAgent = MagicMock(name="restartAgent")
     self.controller.restartAgent = restartAgent
     self.controller.heartbeatWithServer()
@@ -564,7 +564,7 @@ class TestController(unittest.TestCase):
     self.controller.responseId = 1
     self.controller.DEBUG_STOP_HEARTBEATING = False
     actionQueue.isIdle.return_value = False
-    response["restartAgent"] = "false"
+    response["restartAgent"] = False
     self.controller.heartbeatWithServer()
 
 

ambari-agent/src/test/python/ambari_agent/TestProcessUtils.py

@@ -1,224 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-from ambari_agent import main
-
-main.MEMORY_LEAK_DEBUG_FILEPATH = "/tmp/memory_leak_debug.out"
-import unittest
-import signal
-import subprocess, time
-from mock.mock import patch, MagicMock, PropertyMock, call
-from ambari_commons import process_utils
-
-process_tree = {"111": "222\n 22",
-                "222": "333\n 33",
-                "22": "44\n 444",}
-
-
-class TestProcessUtils(unittest.TestCase):
-  @patch("subprocess.Popen")
-  def test_kill(self, popen_mock):
-    process_mock = MagicMock()
-    process_mock.communicate.return_value = (None, None)
-    returncode_mock = PropertyMock()
-    returncode_mock.return_value = 0
-    type(process_mock).returncode = returncode_mock
-    popen_mock.return_value = process_mock
-    process_utils.kill_pids(["12321113230", "2312415453"], signal.SIGTERM)
-    expected = [call(['kill', '-15', '12321113230', '2312415453'], stderr=-1, stdout=-1)]
-    self.assertEquals(popen_mock.call_args_list, expected)
-
-  @patch("subprocess.Popen")
-  def test_get_children(self, popen_mock):
-
-    process_mock = MagicMock()
-    process_mock.communicate.return_value = ("123 \n \n 321\n", None)
-    popen_mock.return_value = process_mock
-    returncode_mock = PropertyMock()
-    returncode_mock.return_value = 0
-    type(process_mock).returncode = returncode_mock
-    result = process_utils.get_children("2312415453")
-
-    self.assertEquals(result, ["123", "321"])
-
-    expected = [
-      call(['ps', '-o', 'pid', '--no-headers', '--ppid', '2312415453'], stderr=subprocess.PIPE, stdout=subprocess.PIPE)]
-    self.assertEquals(popen_mock.call_args_list, expected)
-
-  @patch("subprocess.Popen")
-  def test_get_flat_process_tree(self, popen_mock):
-    def side_effect(*args, **kwargs):
-      process_mock = MagicMock()
-      returncode_mock = PropertyMock()
-      returncode_mock.return_value = 0
-      type(process_mock).returncode = returncode_mock
-      if args[0][5] in process_tree.keys():
-        process_mock.communicate.return_value = (process_tree[args[0][5]], None)
-      else:
-        process_mock.communicate.return_value = ("", None)
-      return process_mock
-
-    popen_mock.side_effect = side_effect
-    result = process_utils.get_flat_process_tree("111")
-    self.assertEquals(result, ['111', '222', '333', '33', '22', '44', '444'])
-
-    expected = [call(['ps', '-o', 'pid', '--no-headers', '--ppid', '111'], stderr=-1, stdout=-1),
-                call(['ps', '-o', 'pid', '--no-headers', '--ppid', '222'], stderr=-1, stdout=-1),
-                call(['ps', '-o', 'pid', '--no-headers', '--ppid', '333'], stderr=-1, stdout=-1),
-                call(['ps', '-o', 'pid', '--no-headers', '--ppid', '33'], stderr=-1, stdout=-1),
-                call(['ps', '-o', 'pid', '--no-headers', '--ppid', '22'], stderr=-1, stdout=-1),
-                call(['ps', '-o', 'pid', '--no-headers', '--ppid', '44'], stderr=-1, stdout=-1),
-                call(['ps', '-o', 'pid', '--no-headers', '--ppid', '444'], stderr=-1, stdout=-1)]
-    self.assertEquals(popen_mock.call_args_list, expected)
-
-  @patch("subprocess.Popen")
-  def test_get_command_by_pid(self, popen_mock):
-
-    process_mock = MagicMock()
-    process_mock.communicate.return_value = ("yum something", None)
-    returncode_mock = PropertyMock()
-    returncode_mock.return_value = 0
-    type(process_mock).returncode = returncode_mock
-    popen_mock.return_value = process_mock
-
-    result = process_utils.get_command_by_pid("2312415453")
-
-    self.assertEquals(result, "yum something")
-
-    expected = [call(['ps', '-p', '2312415453', '-o', 'command', '--no-headers'], stderr=-1, stdout=-1)]
-    self.assertEquals(popen_mock.call_args_list, expected)
-
-  @patch("subprocess.Popen")
-  def test_get_command_by_pid_not_exist(self, popen_mock):
-
-    process_mock = MagicMock()
-    process_mock.communicate.return_value = ("", None)
-    returncode_mock = PropertyMock()
-    returncode_mock.return_value = 1
-    type(process_mock).returncode = returncode_mock
-    popen_mock.return_value = process_mock
-
-    result = process_utils.get_command_by_pid("2312415453")
-
-    self.assertEquals(result, "NOT_FOUND[2312415453]")
-
-    expected = [call(['ps', '-p', '2312415453', '-o', 'command', '--no-headers'], stderr=-1, stdout=-1)]
-    self.assertEquals(popen_mock.call_args_list, expected)
-
-  @patch("subprocess.Popen")
-  def test_is_process_running(self, popen_mock):
-
-    process_mock = MagicMock()
-    process_mock.communicate.return_value = ("2312415453", None)
-    returncode_mock = PropertyMock()
-    returncode_mock.return_value = 0
-    type(process_mock).returncode = returncode_mock
-    popen_mock.return_value = process_mock
-
-    result = process_utils.is_process_running("2312415453")
-
-    self.assertEquals(result, True)
-
-    expected = [call(['ps', '-p', '2312415453', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1)]
-    self.assertEquals(popen_mock.call_args_list, expected)
-
-  @patch("subprocess.Popen")
-  def test_is_process_not_running(self, popen_mock):
-
-    process_mock = MagicMock()
-    process_mock.communicate.return_value = ("", None)
-    returncode_mock = PropertyMock()
-    returncode_mock.return_value = 1
-    type(process_mock).returncode = returncode_mock
-    popen_mock.return_value = process_mock
-
-    result = process_utils.is_process_running("2312415453")
-
-    self.assertEquals(result, False)
-
-    expected = [call(['ps', '-p', '2312415453', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1)]
-    self.assertEquals(popen_mock.call_args_list, expected)
-
-  @patch("subprocess.Popen")
-  def test_get_processes_running(self, popen_mock):
-    def side_effect(*args, **kwargs):
-      process_mock = MagicMock()
-      returncode_mock = PropertyMock()
-      if args[0][2] == "4321":
-        returncode_mock.return_value = 0
-        process_mock.communicate.return_value = ("4321", None)
-      else:
-        returncode_mock.return_value = 1
-        process_mock.communicate.return_value = (None, None)
-      type(process_mock).returncode = returncode_mock
-      return process_mock
-
-    popen_mock.side_effect = side_effect
-
-    result = process_utils.get_processes_running(["1234", "4321"])
-
-    self.assertEquals(result, ["4321"])
-
-    expected = [call(['ps', '-p', '1234', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1),
-                call(['ps', '-p', '4321', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1)]
-    self.assertEquals(popen_mock.call_args_list, expected)
-
-  @patch("time.sleep")
-  @patch("subprocess.Popen")
-  def test_wait_for_process_death(self, popen_mock, sleep_mock):
-
-    process_mock = MagicMock()
-    process_mock.communicate.side_effect = [("4321", None),("4321", None),(None, None)]
-    returncode_mock = PropertyMock()
-    returncode_mock.side_effect = [0, 0, 1]
-    type(process_mock).returncode = returncode_mock
-    popen_mock.return_value = process_mock
-
-    process_utils.wait_for_process_death("4321")
-
-    expected = [call(['ps', '-p', '4321', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1),
-                call(['ps', '-p', '4321', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1),
-                call(['ps', '-p', '4321', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1)]
-    self.assertEquals(popen_mock.call_args_list, expected)
-    expected = [call(0.1), call(0.1)]
-    self.assertEquals(sleep_mock.call_args_list, expected)
-
-  @patch("time.sleep")
-  @patch("subprocess.Popen")
-  def test_wait_for_entire_process_tree_death(self, popen_mock, sleep_mock):
-
-    process_mock = MagicMock()
-    process_mock.communicate.side_effect = [("1234", None), (None, None), ("4321", None), ("4321", None), (None, None)]
-    returncode_mock = PropertyMock()
-    returncode_mock.side_effect = [0, 1, 0, 0, 1]
-    type(process_mock).returncode = returncode_mock
-    popen_mock.return_value = process_mock
-
-    process_utils.wait_for_entire_process_tree_death(["1234", "4321"])
-
-    expected = [call(['ps', '-p', '1234', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1),
-                call(['ps', '-p', '1234', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1),
-                call(['ps', '-p', '4321', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1),
-                call(['ps', '-p', '4321', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1),
-                call(['ps', '-p', '4321', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1)]
-    self.assertEquals(popen_mock.call_args_list, expected)
-    expected = [call(0.1), call(0.1), call(0.1)]
-    self.assertEquals(sleep_mock.call_args_list, expected)

ambari-agent/src/test/python/ambari_agent/TestShell.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-'''
+"""
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
@@ -9,71 +9,138 @@ regarding copyright ownership.  The ASF licenses this file
 to you under the Apache License, Version 2.0 (the
 "License"); you may not use this file except in compliance
 with the License.  You may obtain a copy of the License at
-
     http://www.apache.org/licenses/LICENSE-2.0
-
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-'''
+"""
+from contextlib import contextmanager
 
-from ambari_agent import main
-main.MEMORY_LEAK_DEBUG_FILEPATH = "/tmp/memory_leak_debug.out"
-import os
 import unittest
-import tempfile
+import signal
 from mock.mock import patch, MagicMock, call
-from ambari_agent.AmbariConfig import AmbariConfig
 from ambari_commons import shell
-from ambari_commons.shell import shellRunner
-from sys import platform as _platform
-from only_for_platform import not_for_platform, PLATFORM_WINDOWS
-import subprocess, time
+from ambari_commons import OSCheck
+from StringIO import StringIO
+
+ROOT_PID = 10
+ROOT_PID_CHILDREN = [10, 11, 12, 13]
+shell.logger = MagicMock()  # suppress any log output
+
+__proc_fs = {
+  "/proc/10/task/10/children": "11 12",
+  "/proc/10/comm": "a",
+  "/proc/10/cmdline": "",
+
+  "/proc/11/task/11/children": "13",
+  "/proc/11/comm": "b",
+  "/proc/11/cmdline": "",
+
+  "/proc/12/task/12/children": "",
+  "/proc/12/comm": "c",
+  "/proc/12/cmdline": "",
+
+  "/proc/13/task/13/children": "",
+  "/proc/13/comm": "d",
+  "/proc/13/cmdline": ""
+}
+
+__proc_fs_yum = {
+  "/proc/10/task/10/children": "11",
+  "/proc/10/comm": "a",
+  "/proc/10/cmdline": "",
+
+  "/proc/11/task/11/children": "",
+  "/proc/11/comm": "yum",
+  "/proc/11/cmdline": "yum install something"
+}
+
+# Remove any wait delay, no need for tests
+__old_waiter = shell.wait_for_process_list_kill
+
+
+def __wait_for_process_list_kill(pids, timeout=5, check_step_time=0.1):
+  return __old_waiter(pids, 0, check_step_time)
+
+
+shell.wait_for_process_list_kill = __wait_for_process_list_kill
+
+
+class FakeSignals(object):
+  SIGTERM = signal.SIG_IGN
+  SIGKILL = signal.SIG_IGN
+
+
+@contextmanager
+def _open_mock(path, open_mode):
+  if path in __proc_fs:
+    yield StringIO(__proc_fs[path])
+  else:
+    yield StringIO("")
+
+
+@contextmanager
+def _open_mock_yum(path, open_mode):
+  if path in __proc_fs:
+    yield StringIO(__proc_fs_yum[path])
+  else:
+    yield StringIO("")
+
 
-@not_for_platform(PLATFORM_WINDOWS)
 class TestShell(unittest.TestCase):
 
+  @patch("__builtin__.open", new=MagicMock(side_effect=_open_mock))
+  def test_get_all_children(self):
+
+    pid_list = [item[0] for item in shell.get_all_children(ROOT_PID)]
+
+    self.assertEquals(len(ROOT_PID_CHILDREN), len(pid_list))
+    self.assertEquals(ROOT_PID, pid_list[0])
+
+    for i in ROOT_PID_CHILDREN:
+      self.assertEquals(True, i in pid_list)
+
+  @patch("__builtin__.open", new=MagicMock(side_effect=_open_mock))
+  @patch.object(OSCheck, "get_os_family", new=MagicMock(return_value="redhat"))
+  @patch.object(shell, "signal", new_callable=FakeSignals)
+  @patch("os.listdir")
+  @patch("os.kill")
+  def test_kill_process_with_children(self, os_kill_mock, os_list_dir_mock, fake_signals):
+    pid_list = [item[0] for item in shell.get_all_children(ROOT_PID)]
+    pid_list_str = [str(i) for i in ROOT_PID_CHILDREN]
+    reverse_pid_list = sorted(pid_list, reverse=True)
+    os_list_dir_mock.side_effect = [pid_list_str, [], [], []]
+
+    shell.kill_process_with_children(ROOT_PID)
+
+    # test pid kill by SIGTERM
+    os_kill_pids = [item[0][0] for item in os_kill_mock.call_args_list]
+    self.assertEquals(len(os_kill_pids), len(pid_list))
+    self.assertEquals(reverse_pid_list, os_kill_pids)
+
+    os_kill_mock.reset_mock()
+    os_list_dir_mock.reset_mock()
+
+    os_list_dir_mock.side_effect = [pid_list_str, pid_list_str, pid_list_str, pid_list_str, [], []]
+    shell.kill_process_with_children(ROOT_PID)
+
+    # test pid kill by SIGKILL
+    os_kill_pids = [item[0][0] for item in os_kill_mock.call_args_list]
+    self.assertEquals(len(os_kill_pids), len(pid_list)*2)
+    self.assertEquals(reverse_pid_list + reverse_pid_list, os_kill_pids)
+
+  @patch("__builtin__.open", new=MagicMock(side_effect=_open_mock_yum))
+  @patch.object(OSCheck, "get_os_family", new=MagicMock(return_value="redhat"))
+  @patch.object(shell, "signal", new_callable=FakeSignals)
+  @patch("os.listdir")
+  @patch("os.kill")
+  def test_kill_process_with_children_except_yum(self, os_kill_mock, os_list_dir_mock, fake_signals):
+    os_list_dir_mock.side_effect = [["10", "12", "20"], [], [], []]
+    shell.kill_process_with_children(ROOT_PID)
 
-  @patch("os.setuid")
-  def test_changeUid(self, os_setUIDMock):
-    shell.threadLocal.uid = 9999
-    shell.changeUid()
-    self.assertTrue(os_setUIDMock.called)
-
-
-  @patch("pwd.getpwnam")
-  def test_shellRunner_run(self, getpwnamMock):
-    sh = shellRunner()
-    result = sh.run(['echo'])
-    self.assertEquals(result['exitCode'], 0)
-    self.assertEquals(result['error'], '')
-
-    getpwnamMock.return_value = [os.getuid(), os.getuid(), os.getuid()]
-    result = sh.run(['echo'], 'non_exist_user_name')
-    self.assertEquals(result['exitCode'], 0)
-    self.assertEquals(result['error'], '')
-
-  def test_kill_process_with_children(self):
-    if _platform == "linux" or _platform == "linux2": # Test is Linux-specific
-      sleep_cmd = "sleep 314"
-      test_cmd = """ (({0}) & ({0} & {0})) """.format(sleep_cmd)
-      # Starting process tree (multiple process groups)
-      test_process = subprocess.Popen(test_cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True)
-      time.sleep(0.3) # Delay to allow subprocess to start
-      # Check if processes are running
-      ps_cmd = """ps auxww """
-      ps_process = subprocess.Popen(ps_cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True)
-      (out, err) = ps_process.communicate()
-      self.assertTrue(sleep_cmd in out)
-      # Kill test process
-      shell.kill_process_with_children(test_process.pid)
-      test_process.communicate()
-      # Now test process should not be running
-      ps_process = subprocess.Popen(ps_cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True)
-      (out, err) = ps_process.communicate()
-      self.assertFalse(sleep_cmd in out)
-    else:
-      # Do not run under other systems
-      pass
+    # test clean pid by SIGTERM
+    os_kill_pids = [item[0][0] for item in os_kill_mock.call_args_list]
+    self.assertEquals(len(os_kill_pids), 1)
+    self.assertEquals([10], os_kill_pids)

ambari-agent/src/test/python/ambari_agent/examples/ControllerTester.py

@@ -47,7 +47,7 @@ responces = [
   """
   {
     "responseId":"n",
-    "restartAgent": "False",
+    "restartAgent": false,
     "executionCommands":
       [{
         "commandId": "31-1",
@@ -89,7 +89,7 @@ responces = [
   """
   {
     "responseId":"n",
-    "restartAgent": "False",
+    "restartAgent": false,
     "executionCommands": [],
     "statusCommands":[]
   }
@@ -183,7 +183,7 @@ if __name__ == '__main__':
 #  s =   """
 #  {
 #    "responseId":"n",
-#    "restartAgent": "False",
+#    "restartAgent": false,
 #    "executionCommands":
 #      [{
 #        "commandId": "31-1",

ambari-common/src/main/python/ambari_commons/constants.py

@@ -24,6 +24,7 @@ UPGRADE_TYPE_ROLLING = "rolling"
 UPGRADE_TYPE_NON_ROLLING = "nonrolling"
 UPGRADE_TYPE_HOST_ORDERED = "host_ordered"
 
+AGENT_TMP_DIR = "/var/lib/ambari-agent/tmp"
 
 class SERVICE:
   """

ambari-common/src/main/python/ambari_commons/inet_utils.py

@@ -23,6 +23,7 @@ import time
 import sys
 import urllib2
 import socket
+import re
 from ambari_commons import OSCheck
 from functools import wraps
 
@@ -266,3 +267,48 @@ def ensure_ssl_using_protocol(protocol="PROTOCOL_TLSv1", ca_certs=None):
         return context
       _create_default_https_context_patched._ambari_patched = True
       ssl._create_default_https_context = _create_default_https_context_patched
+
+"""
+See RFC3986, Appendix B
+Tested on the following cases:
+  "192.168.54.1"
+  "192.168.54.2:7661
+  "hdfs://192.168.54.3/foo/bar"
+  "ftp://192.168.54.4:7842/foo/bar"
+
+  Returns None if only a port is passed in
+"""
+def get_host_from_url(uri):
+  if uri is None:
+    return None
+
+  # if not a string, return None
+  if not isinstance(uri, basestring):
+    return None
+
+    # RFC3986, Appendix B
+  parts = re.findall('^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?', uri)
+
+  # index of parts
+  # scheme    = 1
+  # authority = 3
+  # path      = 4
+  # query     = 6
+  # fragment  = 8
+
+  host_and_port = uri
+  if 0 == len(parts[0][1]):
+    host_and_port = parts[0][4]
+  elif 0 == len(parts[0][2]):
+    host_and_port = parts[0][1]
+  elif parts[0][2].startswith("//"):
+    host_and_port = parts[0][3]
+
+  if -1 == host_and_port.find(':'):
+    if host_and_port.isdigit():
+      return None
+
+    return host_and_port
+  else:
+    return host_and_port.split(':')[0]
+

ambari-common/src/main/python/ambari_commons/kerberos/__init__.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python2.6
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''

ambari-common/src/main/python/ambari_commons/kerberos/kerberos_common.py

@@ -0,0 +1,168 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import base64
+import getpass
+import os
+import stat
+from ambari_commons.constants import AGENT_TMP_DIR
+from collections import namedtuple
+from resource_management.core import sudo
+from resource_management.core.logger import Logger
+from resource_management.core.resources.klist import Klist
+from resource_management.core.resources.system import Directory, File
+from resource_management.core.source import InlineTemplate
+from tempfile import gettempdir
+from .utils import get_property_value
+
+KRB5_REALM_PROPERTIES = [
+  'kdc',
+  'admin_server',
+  'default_domain',
+  'master_kdc'
+]
+
+
+class MissingKeytabs(object):
+  class Identity(namedtuple('Identity', ['principal', 'keytab_file_path'])):
+    @staticmethod
+    def from_kerberos_record(item, hostname):
+      return MissingKeytabs.Identity(
+        get_property_value(item, 'principal').replace("_HOST", hostname),
+        get_property_value(item, 'keytab_file_path'))
+
+    def __str__(self):
+      return "Keytab: %s Principal: %s" % (self.keytab_file_path, self.principal)
+
+  @classmethod
+  def from_kerberos_records(self, kerberos_record, hostname):
+    with_missing_keytab = (each for each in kerberos_record \
+                           if not self.keytab_exists(each) or not self.keytab_has_principal(each, hostname))
+    return MissingKeytabs(
+      set(MissingKeytabs.Identity.from_kerberos_record(each, hostname) for each in with_missing_keytab))
+
+  @staticmethod
+  def keytab_exists(kerberos_record):
+    return sudo.path_exists(get_property_value(kerberos_record, 'keytab_file_path'))
+
+  @staticmethod
+  def keytab_has_principal(kerberos_record, hostname):
+    principal = get_property_value(kerberos_record, 'principal').replace("_HOST", hostname)
+    keytab = get_property_value(kerberos_record, 'keytab_file_path')
+    klist = Klist.find_in_search_path()
+    return principal in klist.list_principals(keytab)
+
+  def __init__(self, items):
+    self.items = items
+
+  def as_dict(self):
+    return [each._asdict() for each in self.items]
+
+  def __str__(self):
+    return "Missing keytabs:\n%s" % ("\n".join(map(str, self.items))) if self.items else 'No missing keytabs'
+
+
+def write_krb5_conf(params):
+  Directory(params.krb5_conf_dir,
+            owner='root',
+            create_parents=True,
+            group='root',
+            mode=0755
+            )
+
+  content = InlineTemplate(params.krb5_conf_template)
+
+  File(params.krb5_conf_path,
+       content=content,
+       owner='root',
+       group='root',
+       mode=0644
+       )
+
+
+def clear_tmp_cache():
+  tmp_dir = AGENT_TMP_DIR
+  if tmp_dir is None:
+    tmp_dir = gettempdir()
+  curl_krb_cache_path = os.path.join(tmp_dir, "curl_krb_cache")
+  Directory(curl_krb_cache_path, action="delete")
+
+
+def write_keytab_file(params, output_hook=lambda principal, keytab_file_path: None):
+  if params.kerberos_command_params is not None:
+    for item in params.kerberos_command_params:
+      keytab_content_base64 = get_property_value(item, 'keytab_content_base64')
+      if (keytab_content_base64 is not None) and (len(keytab_content_base64) > 0):
+        keytab_file_path = get_property_value(item, 'keytab_file_path')
+        if (keytab_file_path is not None) and (len(keytab_file_path) > 0):
+          head, tail = os.path.split(keytab_file_path)
+          if head:
+            Directory(head, create_parents=True, mode=0755, owner="root", group="root")
+
+          owner = get_property_value(item, 'keytab_file_owner_name')
+          if not owner:
+            owner = getpass.getuser()
+          owner_access = get_property_value(item, 'keytab_file_owner_access')
+          group = get_property_value(item, 'keytab_file_group_name')
+          group_access = get_property_value(item, 'keytab_file_group_access')
+          mode = 0
+
+          if owner_access == 'rw':
+            mode |= stat.S_IREAD | stat.S_IWRITE
+          else:
+            mode |= stat.S_IREAD
+
+          if group_access == 'rw':
+            mode |= stat.S_IRGRP | stat.S_IWGRP
+          elif group_access == 'r':
+            mode |= stat.S_IRGRP
+
+          keytab_content = base64.b64decode(keytab_content_base64)
+
+          # to hide content in command output
+          def make_lambda(data):
+            return lambda: data
+
+          File(keytab_file_path,
+               content=make_lambda(keytab_content),
+               mode=mode,
+               owner=owner,
+               group=group)
+
+          principal = get_property_value(item, 'principal')
+
+          output_hook(principal, keytab_file_path)
+
+
+def delete_keytab_file(params, output_hook=lambda principal, keytab_file_path: None):
+  if params.kerberos_command_params is not None:
+    for item in params.kerberos_command_params:
+      keytab_file_path = get_property_value(item, 'keytab_file_path')
+      if (keytab_file_path is not None) and (len(keytab_file_path) > 0):
+        # Delete the keytab file
+        File(keytab_file_path, action="delete")
+
+        principal = get_property_value(item, 'principal')
+        output_hook(principal, keytab_file_path)
+
+
+def find_missing_keytabs(params, output_hook=lambda missing_keytabs: None):
+  missing_keytabs = MissingKeytabs.from_kerberos_records(params.kerberos_command_params, params.hostname)
+  Logger.info(str(missing_keytabs))
+  output_hook(missing_keytabs.as_dict())

ambari-common/src/main/python/ambari_commons/kerberos/utils.py

@@ -0,0 +1,109 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+
+def get_property_value(dictionary, property_name, default_value=None, trim_string=False,
+                       empty_value=""):
+  """
+  Get a property value from a dictionary, applying applying rules as necessary.
+
+  If dictionary does not contain a value for property_name or the value for property_name is None,
+  null_value is used as the value to return.  Then, if trim_string is True and the value is None
+  or the value is an empty string, empty_value will be return else the (current) value is returned.
+
+  Note: the property value will most likely be a string or a unicode string, however in the event
+  it is not (for example a number), this method will behave properly and return the value as is.
+
+  :param dictionary: a dictionary of values
+  :param property_name: the name of a dictionary item to retrieve
+  :param default_value: the value to use if the item is not in the dictionary or the value of the item is None
+  :param trim_string: a Boolean value indicating whether to strip whitespace from the value (True) or not (False)
+  :param empty_value: the value to use if the (current) value is None or an empty string, if trim_string is True
+  :return: the requested property value with rules applied
+  """
+  # If property_name is not in the dictionary, set value to null_value
+  if property_name in dictionary:
+    value = dictionary[property_name]
+    if value is None:
+      value = default_value
+  else:
+    value = default_value
+
+  if trim_string:
+    # If the value is none, consider it empty...
+    if value is None:
+      value = empty_value
+    elif (type(value) == str) or (type(value) == unicode):
+      value = value.strip()
+
+      if len(value) == 0:
+        value = empty_value
+
+  return value
+
+
+def get_unstructured_data(dictionary, property_name):
+  prefix = property_name + '/'
+  prefix_len = len(prefix)
+  return dict((k[prefix_len:], v) for k, v in dictionary.iteritems() if k.startswith(prefix))
+
+
+def split_host_and_port(host):
+  """
+  Splits a string into its host and port components
+
+  :param host: a string matching the following pattern: <host name | ip address>[:port]
+  :return: a Dictionary containing 'host' and 'port' entries for the input value
+  """
+
+  if host is None:
+    host_and_port = None
+  else:
+    host_and_port = {}
+    parts = host.split(":")
+
+    if parts is not None:
+      length = len(parts)
+
+      if length > 0:
+        host_and_port['host'] = parts[0]
+
+        if length > 1:
+          host_and_port['port'] = int(parts[1])
+
+  return host_and_port
+
+
+def set_port(host, port):
+  """
+  Sets the port for a host specification, potentially replacing an existing port declaration
+
+  :param host: a string matching the following pattern: <host name | ip address>[:port]
+  :param port: a string or integer declaring the (new) port
+  :return: a string declaring the new host/port specification
+  """
+  if port is None:
+    return host
+  else:
+    host_and_port = split_host_and_port(host)
+
+    if (host_and_port is not None) and ('host' in host_and_port):
+      return "%s:%s" % (host_and_port['host'], port)
+    else:
+      return host

ambari-common/src/main/python/ambari_commons/os_check.py

@@ -164,6 +164,8 @@ class OS_CONST_TYPE(type):
 class OSConst:
   __metaclass__ = OS_CONST_TYPE
 
+  systemd_redhat_os_major_versions = ["7"]
+
 
 class OSCheck:
 

ambari-common/src/main/python/ambari_commons/process_utils.py

@@ -1,100 +0,0 @@
-# !/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-import subprocess
-import time
-
-check_time_delay = 0.1  # seconds between checks of process killed
-
-
-def get_children(pid):
-  PSCMD = ["ps", "-o", "pid", "--no-headers", "--ppid", str(pid)]
-  ps_process = subprocess.Popen(PSCMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-  stdout, stderr = ps_process.communicate()
-  if ps_process.returncode != 0:
-    return []
-  return stdout.split()
-
-
-def get_flat_process_tree(pid):
-  """
-  :param pid: process id of parent process
-  :return: list of child process pids. Resulting list also includes parent pid
-  """
-  res = [str(pid)]
-  children = get_children(pid)
-  for child in children:
-    res += get_flat_process_tree(child)
-  return res
-
-
-def kill_pids(pids, signal):
-  from resource_management.core.exceptions import Fail
-  CMD = ["kill", "-" + str(signal)]
-  CMD.extend(pids)
-  process = subprocess.Popen(CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-  stdout, stderr = process.communicate()
-  if process.returncode != 0:
-    raise Fail("Unable to kill PIDs {0} : {1}".format(str(pids),stderr))
-
-
-def get_command_by_pid(pid):
-  CMD = ["ps", "-p", str(pid), "-o", "command", "--no-headers"]
-  process = subprocess.Popen(CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-  stdout, stderr = process.communicate()
-  if process.returncode != 0:
-    return "NOT_FOUND[%s]" % pid
-  return stdout
-
-
-def wait_for_entire_process_tree_death(pids):
-  for child in pids:
-    wait_for_process_death(child)
-
-
-def wait_for_process_death(pid, timeout=5):
-  start = time.time()
-  current_time = start
-  while is_process_running(pid) and current_time < start + timeout:
-    time.sleep(check_time_delay)
-    current_time = time.time()
-
-
-def is_process_running(pid):
-  CMD = ["ps", "-p", str(pid), "-o", "pid", "--no-headers"]
-  process = subprocess.Popen(CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-  stdout, stderr = process.communicate()
-  if process.returncode != 0:
-    return False
-  return pid in stdout
-
-
-
-def get_processes_running(process_pids):
-  """
-  Checks what processes are still running
-  :param process_pids: list of process pids
-  :return: list of pids for processes that are still running
-  """
-  result = []
-  for pid in process_pids:
-    if is_process_running(pid):
-      result.append(pid)
-  return result

ambari-common/src/main/python/ambari_commons/shell.py

@@ -27,8 +27,7 @@ from contextlib import contextmanager
 
 from ambari_commons import OSConst
 from ambari_commons.os_family_impl import OsFamilyImpl, OsFamilyFuncImpl
-from ambari_commons.process_utils import get_flat_process_tree, kill_pids, wait_for_entire_process_tree_death, \
-  get_processes_running, get_command_by_pid
+from resource_management.core import sudo
 
 logger = logging.getLogger()
 
@@ -231,35 +230,170 @@ class shellRunnerWindows(shellRunner):
     return _dict_to_object({'exitCode': code, 'output': out, 'error': err})
 
 
-#linux specific code
-@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
-def kill_process_with_children(parent_pid):
+def get_all_children(base_pid):
   """
-  Kills process tree starting from a given pid.
-  :param parent_pid: head of tree
-  :param graceful_kill_delays: map <command name, custom delay between SIGTERM and SIGKILL>
-  :return:
+  Return all child PIDs of base_pid process
+
+  :param base_pid starting PID to scan for children
+  :return tuple of the following: pid, binary name, command line incl. binary
+
+  :type base_pid int
+  :rtype list[(int, str, str)]
   """
+  parent_pid_path_pattern = "/proc/{0}/task/{0}/children"
+  comm_path_pattern = "/proc/{0}/comm"
+  cmdline_path_pattern = "/proc/{0}/cmdline"
+
+  def read_children(pid):
+    try:
+      with open(parent_pid_path_pattern.format(pid), "r") as f:
+        return [int(item) for item in f.readline().strip().split(" ")]
+    except (IOError, ValueError):
+      return []
+
+  def read_command(pid):
+    try:
+      with open(comm_path_pattern.format(pid), "r") as f:
+        return f.readline().strip()
+    except IOError:
+      return ""
+
+  def read_cmdline(pid):
+    try:
+      with open(cmdline_path_pattern.format(pid), "r") as f:
+        return f.readline().strip()
+    except IOError:
+      return ""
+
+  pids = []
+  scan_pending = [int(base_pid)]
+
+  while scan_pending:
+    curr_pid = scan_pending.pop(0)
+    children = read_children(curr_pid)
 
-  pids = get_flat_process_tree(parent_pid)
+    pids.append((curr_pid, read_command(curr_pid), read_cmdline(curr_pid)))
+    scan_pending.extend(children)
+
+  return pids
+
+
+def is_pid_exists(pid):
+  """
+  Check if process with PID still exist (not counting it real state)
+
+  :type pid int
+  :rtype bool
+  """
+  pid_path = "/proc/{0}"
   try:
-    kill_pids(pids, signal.SIGTERM)
-  except Exception, e:
-    logger.warn("Failed to kill PID %d" % parent_pid)
-    logger.warn("Reported error: " + repr(e))
+    return os.path.exists(pid_path.format(pid))
+  except (OSError, IOError):
+    logger.debug("Failed to check PID existence")
+    return False
+
+
+def get_existing_pids(pids):
+  """
+  Check if process with pid still exists (not counting it real state).
+
+  Optimized to check PID list at once.
+
+  :param pids list of PIDs to filter
+  :return list of still existing PID
 
-  wait_for_entire_process_tree_death(pids)
+  :type pids list[int]
+  :rtype list[int]
+  """
+
+  existing_pid_list = []
 
   try:
-    running_processes = get_processes_running(pids)
-    if running_processes:
-      process_names = map(lambda x: get_command_by_pid(x),  running_processes)
-      logger.warn("These PIDs %s did not die after SIGTERM, sending SIGKILL. Exact commands to be killed:\n %s" %
-                  (", ".join(running_processes), "\n".join(process_names)))
-      kill_pids(running_processes, signal.SIGKILL)
-  except Exception, e:
-    logger.error("Failed to send SIGKILL to PID %d. Process exited?" % parent_pid)
-    logger.error("Reported error: " + repr(e))
+    all_existing_pid_list = [int(item) for item in os.listdir("/proc") if item.isdigit()]
+  except (OSError, IOError):
+    logger.debug("Failed to check PIDs existence")
+    return existing_pid_list
+
+  for pid_item in pids:
+    if pid_item in all_existing_pid_list:
+      existing_pid_list.append(pid_item)
+
+  return existing_pid_list
+
+
+def wait_for_process_list_kill(pids, timeout=5, check_step_time=0.1):
+  """
+  Process tree waiter
+
+  :type pids list[int]
+  :type timeout int|float
+  :type check_step_time int|float
+
+  :param pids list of PIDs to watch
+  :param timeout how long wait till giving up, seconds. Set 0 for nowait or None for infinite time
+  :param check_step_time how often scan for existing PIDs, seconds
+  """
+  from threading import Thread, Event
+  import time
+
+  stop_waiting = Event()
+
+  def _wait_loop():
+    while not stop_waiting.is_set() and get_existing_pids(pids):
+      time.sleep(check_step_time)
+
+  if timeout == 0:  # no need for loop if no timeout is set
+    return
+
+  th = Thread(target=_wait_loop)
+  stop_waiting.clear()
+
+  th.start()
+  th.join(timeout=timeout)
+  stop_waiting.set()
+
+  th.join()
+
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def kill_process_with_children(base_pid):
+  """
+  Process tree killer
+
+  :type base_pid int
+  """
+  exception_list = ["apt-get", "apt", "yum", "zypper", "zypp"]
+  signals_to_post = {
+    "SIGTERM": signal.SIGTERM,
+    "SIGKILL": signal.SIGKILL
+  }
+  full_child_pids = get_all_children(base_pid)
+  all_child_pids = [item[0] for item in full_child_pids if item[1].lower() not in exception_list and item[0] != os.getpid()]
+  error_log = []
+
+  for sig_name, sig in signals_to_post.items():
+    # we need to kill processes from the bottom of the tree
+    pids_to_kill = sorted(get_existing_pids(all_child_pids), reverse=True)
+    for pid in pids_to_kill:
+      try:
+        sudo.kill(pid, sig)
+      except OSError as e:
+        error_log.append((sig_name, pid, repr(e)))
+
+    if pids_to_kill:
+      wait_for_process_list_kill(pids_to_kill)
+      still_existing_pids = get_existing_pids(pids_to_kill)
+      if still_existing_pids:
+        logger.warn("These PIDs {0} did not respond to {1} signal. Detailed commands list:\n {2}".format(
+          ", ".join([str(i) for i in still_existing_pids]),
+          sig_name,
+          "\n".join([i[2] for i in full_child_pids if i[0] in still_existing_pids])
+        ))
+
+  if get_existing_pids(all_child_pids) and error_log:  # we're unable to kill all requested PIDs
+    logger.warn("Process termination error log:\n")
+    for error_item in error_log:
+      logger.warn("PID: {0}, Process: {1}, Exception message: {2}".format(*error_item))
 
 
 def _changeUid():

ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py

@@ -211,6 +211,13 @@ def convert_conf_directories_to_symlinks(package, version, dirs):
   # determine which directories would be created, if any are needed
   dry_run_directory = create(stack_name, package, version, dry_run = True)
 
+  # if the dry run reported an error, then we must assume that the package does not exist in
+  # the conf-select tool
+  if len(dry_run_directory) == 0:
+    Logger.info("The conf-select tool reported an error for the package {0}. The configuration linking will be skipped.".format(package))
+    return
+
+
   need_dirs = []
   for d in dry_run_directory:
     if not os.path.exists(d):

ambari-common/src/main/python/resource_management/libraries/functions/log_process_information.py

@@ -19,7 +19,6 @@ limitations under the License.
 Ambari Agent
 
 """
-from ambari_commons.shell import shellRunner
 from ambari_commons.os_check import OSCheck
 
 __all__ = ["log_process_information"]
@@ -28,6 +27,8 @@ def log_process_information(logger):
   """
   Check if certain configuration sent from the server has been received.
   """
+
+  from ambari_commons.shell import shellRunner
   if OSCheck.is_windows_family():
     cmd_list = ["WMIC path win32_process get Caption,Processid,Commandline", "netstat -an"]
   else:

ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py

@@ -54,6 +54,9 @@ def get_lzo_packages():
 
   return lzo_packages
 
+def is_gpl_license_accepted():
+  return default("/ambariLevelParams/gpl_license_accepted", False)
+
 def should_install_lzo():
   """
   Return true if lzo is enabled via core-site.xml and GPL license (required for lzo) is accepted.
@@ -65,8 +68,7 @@ def should_install_lzo():
   if not lzo_enabled:
     return False
 
-  is_gpl_license_accepted = default("/ambariLevelParams/gpl_license_accepted", False)
-  if not is_gpl_license_accepted:
+  if not is_gpl_license_accepted():
     Logger.warning(INSTALLING_LZO_WITHOUT_GPL)
     return False
 
@@ -79,6 +81,9 @@ def install_lzo_if_needed():
   if not should_install_lzo():
     return
 
+  # If user has just accepted GPL license. GPL repository can not yet be present.
+  Script.repository_util.create_repo_files()
+
   lzo_packages = get_lzo_packages()
 
   config = Script.get_config()

ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py

@@ -239,11 +239,13 @@ def get_nameservice(hdfs_site):
   """
   name_service = hdfs_site.get('dfs.internal.nameservices', None)
   if not name_service:
-    import re
     name_service = hdfs_site.get('dfs.nameservices', None)
-    if name_service:
-      for ns in name_service.split(","):
-        if 'dfs.namenode.shared.edits.dir' in hdfs_site and re.match(r'.*%s$' % ns, hdfs_site['dfs.namenode.shared.edits.dir']): # better would be core_site['fs.defaultFS'] but it's not available
-          return ns
-      return name_service.split(",")[0] # default to return the first nameservice
+
+  if name_service and ',' in name_service:
+    import re
+    for ns in name_service.split(","):
+      if 'dfs.namenode.shared.edits.dir' in hdfs_site and re.match(r'.*%s$' % ns, hdfs_site['dfs.namenode.shared.edits.dir']): # better would be core_site['fs.defaultFS'] but it's not available
+        return ns
+    return name_service.split(",")[0] # default to return the first nameservice
+
   return name_service

ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py

@@ -17,61 +17,83 @@ limitations under the License.
 
 """
 
+from ambari_commons.os_check import OSCheck
 from resource_management.core.exceptions import Fail
 from resource_management.core.logger import Logger
 from resource_management.libraries.resources.repository import Repository
+from resource_management.libraries.functions.is_empty import is_empty
 import ambari_simplejson as json
 
 
-__all__ = ["create_repo_files", "CommandRepository"]
+__all__ = ["RepositoryUtil", "CommandRepository"]
 
 # components_lits = repoName + postfix
 UBUNTU_REPO_COMPONENTS_POSTFIX = "main"
 
+class RepositoryUtil:
+  def __init__(self, config, tags_to_skip):
+    self.tags_to_skip = tags_to_skip
 
-def create_repo_files(template, command_repository):
-  """
-  Creates repositories in a consistent manner for all types
-  :param command_repository: a CommandRepository instance
-  :type command_repository CommandRepository
-  :return: a dictionary with repo ID => repo file name mapping
-  """
+    # repo templates
+    repo_file = config['repositoryFile']
+    repo_rhel_suse =  config['configurations']['cluster-env']['repo_suse_rhel_template']
+    repo_ubuntu =  config['configurations']['cluster-env']['repo_ubuntu_template']
 
-  if command_repository.version_id is None:
-    raise Fail("The command repository was not parsed correctly")
+    if is_empty(repo_file):
+      return
 
-  if 0 == len(command_repository.items):
-    Logger.warning(
-      "Repository for {0}/{1} has no repositories.  Ambari may not be managing this version.".format(
-        command_repository.stack_name, command_repository.version_string))
-    return {}
+    self.template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else repo_ubuntu
+    self.command_repository = CommandRepository(repo_file)
 
-  append_to_file = False  # initialize to False to create the file anew.
-  repo_files = {}
-
-  for repository in command_repository.items:
-
-    if repository.repo_id is None:
-      raise Fail("Repository with url {0} has no id".format(repository.base_url))
+  def create_repo_files(self):
+    """
+    Creates repositories in a consistent manner for all types
+    :return: a dictionary with repo ID => repo file name mapping
+    """
+    if self.command_repository.version_id is None:
+      raise Fail("The command repository was not parsed correctly")
 
-    if not repository.ambari_managed:
+    if 0 == len(self.command_repository.items):
       Logger.warning(
-        "Repository for {0}/{1}/{2} is not managed by Ambari".format(
-          command_repository.stack_name, command_repository.version_string, repository.repo_id))
-    else:
-      Repository(repository.repo_id,
-                 action="create",
-                 base_url=repository.base_url,
-                 mirror_list=repository.mirrors_list,
-                 repo_file_name=command_repository.repo_filename,
-                 repo_template=template,
-                 components=repository.ubuntu_components,
-                 append_to_file=append_to_file)
-      append_to_file = True
-      repo_files[repository.repo_id] = command_repository.repo_filename
-
-  return repo_files
+        "Repository for {0}/{1} has no repositories.  Ambari may not be managing this version.".format(
+          self.command_repository.stack_name, self.command_repository.version_string))
+      return {}
+
+    append_to_file = False  # initialize to False to create the file anew.
+    repo_files = {}
+    for repository in self.command_repository.items:
+      if repository.repo_id is None:
+        raise Fail("Repository with url {0} has no id".format(repository.base_url))
+
+      if self.tags_to_skip & repository.tags:
+        Logger.info("Repository with url {0} is not created due to its tags: {1}".format(repository.base_url, repository.tags))
+        continue
+
+      if not repository.ambari_managed:
+        Logger.warning(
+          "Repository for {0}/{1}/{2} is not managed by Ambari".format(
+            self.command_repository.stack_name, self.command_repository.version_string, repository.repo_id))
+      else:
+        Repository(repository.repo_id,
+                   action="create",
+                   base_url=repository.base_url,
+                   mirror_list=repository.mirrors_list,
+                   repo_file_name=self.command_repository.repo_filename,
+                   repo_template=self.template,
+                   components=repository.ubuntu_components,
+                   append_to_file=append_to_file)
+        append_to_file = True
+        repo_files[repository.repo_id] = self.command_repository.repo_filename
+
+    return repo_files
 
+def create_repo_files(template, command_repository):
+  """
+  DEPRECATED. Is present for usage by old mpacks.
+  Please use Script.repository_util.create_repo_files() instead.
+  """
+  from resource_management.libraries.script import Script
+  return RepositoryUtil(Script.get_config(), set()).create_repo_files()
 
 def _find_value(dictionary, key, default=None):
   """
@@ -146,6 +168,7 @@ class CommandRepositoryItem(object):
     self.components = _find_value(json_dict, 'components')
     self.base_url = _find_value(json_dict, 'baseUrl')
     self.mirrors_list = _find_value(json_dict, 'mirrorsList')
+    self.tags = set(_find_value(json_dict, 'tags', default=[]))
     self.ambari_managed = _find_value(json_dict, 'ambariManaged', default=True)
 
     self.ubuntu_components = [self.distribution if self.distribution else self.repo_name] + \

ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py

@@ -162,14 +162,9 @@ def setup_atlas_jar_symlinks(hook_name, jar_source_dir):
   import params
 
   stack_root = Script.get_stack_root()
-  atlas_home_dir = os.path.join(stack_root, "current", "atlas-server")
-
-  # if this is an upgrade/downagrade, then we must link in the correct version
-  # which may not be "current", so change the home directory location
-  upgrade_type = Script.get_upgrade_type(default("/commandParams/upgrade_type", ""))
-  if upgrade_type is not None:
-    version_dir_segment = stack_features.get_stack_feature_version(Script.get_config())
-    atlas_home_dir = os.path.join(stack_root, version_dir_segment, "atlas")
+  atlas_component_name = "atlas"
+  stack_version = stack_features.get_stack_feature_version(Script.get_config())
+  atlas_home_dir = os.path.join(stack_root, stack_version, atlas_component_name)
 
   # Will only exist if this host contains Atlas Server
   atlas_hook_dir = os.path.join(atlas_home_dir, "hook", hook_name)

ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py

@@ -32,9 +32,11 @@ from resource_management.core.logger import Logger
 __all__ = ["upload_configuration_to_zk", "create_collection", "setup_kerberos", "set_cluster_prop",
            "setup_kerberos_plugin", "create_znode", "check_znode", "secure_solr_znode", "secure_znode"]
 
-def __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, separated_znode=False):
+def __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts=None, separated_znode=False):
   sudo = AMBARI_SUDO_BINARY
-  solr_cli_prefix = format('{sudo} JAVA_HOME={java64_home} /usr/lib/ambari-infra-solr-client/solrCloudCli.sh ' \
+
+  infra_solr_cli_opts= format(' INFRA_SOLR_CLI_OPTS="{java_opts}"') if java_opts is not None else ''
+  solr_cli_prefix = format('{sudo} JAVA_HOME={java64_home}{infra_solr_cli_opts} /usr/lib/ambari-infra-solr-client/solrCloudCli.sh ' \
                            '--zookeeper-connect-string {zookeeper_quorum}')
   if separated_znode:
     solr_cli_prefix+=format(' --znode {solr_znode}')
@@ -49,7 +51,7 @@ def __append_flags_if_exists(command, flagsDict):
   return command
 
 def upload_configuration_to_zk(zookeeper_quorum, solr_znode, config_set, config_set_dir, tmp_dir,
-                         java64_home, retry = 5, interval = 10, solrconfig_content = None, jaas_file=None):
+                         java64_home, retry = 5, interval = 10, solrconfig_content = None, jaas_file=None, java_opts=None):
   """
   Upload configuration set to zookeeper with solrCloudCli.sh
   At first, it tries to download configuration set if exists into a temporary location, then upload that one to
@@ -57,7 +59,7 @@ def upload_configuration_to_zk(zookeeper_quorum, solr_znode, config_set, config_
   """
   random_num = random.random()
   tmp_config_set_dir = format('{tmp_dir}/solr_config_{config_set}_{random_num}')
-  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home)
+  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts)
   Execute(format('{solr_cli_prefix} --download-config --config-dir {tmp_config_set_dir} --config-set {config_set} --retry {retry} --interval {interval}'),
           only_if=format("{solr_cli_prefix} --check-config --config-set {config_set} --retry {retry} --interval {interval}"))
   appendableDict = {}
@@ -88,7 +90,7 @@ def create_collection(zookeeper_quorum, solr_znode, collection, config_set, java
                       shards = 1, replication_factor = 1, max_shards = 1, retry = 5, interval = 10,
                       router_name = None, router_field = None, jaas_file = None, key_store_location = None,
                       key_store_password = None, key_store_type = None, trust_store_location = None,
-                      trust_store_password = None, trust_store_type = None):
+                      trust_store_password = None, trust_store_type = None, java_opts=None):
   """
   Create Solr collection based on a configuration set in zookeeper.
   If this method called again the with higher shard number (or max_shard number), then it will indicate
@@ -98,7 +100,7 @@ def create_collection(zookeeper_quorum, solr_znode, collection, config_set, java
   If you would like to add shards later to a collection, then use implicit routing, e.g.:
   router_name = "implicit", router_field = "_router_field_"
   """
-  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home)
+  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts)
 
   if max_shards == 1: # if max shards is not specified use this strategy
     max_shards = replication_factor * shards
@@ -121,59 +123,59 @@ def create_collection(zookeeper_quorum, solr_znode, collection, config_set, java
 
   Execute(create_collection_cmd)
 
-def setup_kerberos(zookeeper_quorum, solr_znode, copy_from_znode, java64_home, secure=False, jaas_file=None):
+def setup_kerberos(zookeeper_quorum, solr_znode, copy_from_znode, java64_home, secure=False, jaas_file=None, java_opts=None):
   """
   Copy all unsecured (or secured) Znode content to a secured (or unsecured) Znode,
   and restrict the world permissions there.
   """
-  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, True)
+  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts, True)
   setup_kerberos_cmd = format('{solr_cli_prefix} --setup-kerberos --copy-from-znode {copy_from_znode}')
   if secure and jaas_file is not None:
     setup_kerberos_cmd+=format(' --secure --jaas-file {jaas_file}')
   Execute(setup_kerberos_cmd)
 
-def check_znode(zookeeper_quorum, solr_znode, java64_home, retry = 5, interval = 10):
+def check_znode(zookeeper_quorum, solr_znode, java64_home, retry = 5, interval = 10, java_opts=None):
   """
   Check znode exists or not, throws exception if does not accessible.
   """
-  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, True)
+  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts, True)
   check_znode_cmd = format('{solr_cli_prefix} --check-znode --retry {retry} --interval {interval}')
   Execute(check_znode_cmd)
 
-def create_znode(zookeeper_quorum, solr_znode, java64_home, retry = 5 , interval = 10):
+def create_znode(zookeeper_quorum, solr_znode, java64_home, retry = 5 , interval = 10, java_opts=None):
   """
   Create znode if does not exists, throws exception if zookeeper is not accessible.
   """
-  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, True)
+  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts, True)
   create_znode_cmd = format('{solr_cli_prefix} --create-znode --retry {retry} --interval {interval}')
   Execute(create_znode_cmd)
 
-def setup_kerberos_plugin(zookeeper_quorum, solr_znode, java64_home, secure=False, security_json_location = None, jaas_file = None):
+def setup_kerberos_plugin(zookeeper_quorum, solr_znode, java64_home, secure=False, security_json_location = None, jaas_file = None, java_opts=None):
   """
   Set Kerberos plugin on the Solr znode in security.json, if secure is False, then clear the security.json
   """
-  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, True)
+  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts, True)
   setup_kerberos_plugin_cmd = format('{solr_cli_prefix} --setup-kerberos-plugin')
   if secure and jaas_file is not None and security_json_location is not None:
     setup_kerberos_plugin_cmd+=format(' --jaas-file {jaas_file} --secure --security-json-location {security_json_location}')
   Execute(setup_kerberos_plugin_cmd)
 
-def set_cluster_prop(zookeeper_quorum, solr_znode, prop_name, prop_value, java64_home, jaas_file = None):
+def set_cluster_prop(zookeeper_quorum, solr_znode, prop_name, prop_value, java64_home, jaas_file = None, java_opts=None):
   """
   Set a cluster property on the Solr znode in clusterprops.json
   """
-  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home)
+  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts)
   set_cluster_prop_cmd = format('{solr_cli_prefix} --cluster-prop --property-name {prop_name} --property-value {prop_value}')
   if jaas_file is not None:
     set_cluster_prop_cmd+=format(' --jaas-file {jaas_file}')
   Execute(set_cluster_prop_cmd)
 
-def secure_znode(config, zookeeper_quorum, solr_znode, jaas_file, java64_home, sasl_users=[], retry = 5 , interval = 10):
+def secure_znode(config, zookeeper_quorum, solr_znode, jaas_file, java64_home, sasl_users=[], retry = 5 , interval = 10, java_opts=None):
   """
   Secure znode, set a list of sasl users acl to 'cdrwa', and set acl to 'r' only for the world.
   Add infra-solr user by default if its available.
   """
-  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, True)
+  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts, True)
   if "infra-solr-env" in config['configurations']:
     sasl_users.append(__get_name_from_principal(config['configurations']['infra-solr-env']['infra_solr_kerberos_principal']))
   sasl_users_str = ",".join(str(__get_name_from_principal(x)) for x in sasl_users)
@@ -181,20 +183,20 @@ def secure_znode(config, zookeeper_quorum, solr_znode, jaas_file, java64_home, s
   Execute(secure_znode_cmd)
 
 
-def secure_solr_znode(zookeeper_quorum, solr_znode, jaas_file, java64_home, sasl_users_str=''):
+def secure_solr_znode(zookeeper_quorum, solr_znode, jaas_file, java64_home, sasl_users_str='', java_opts=None):
   """
   Secure solr znode - setup acls to 'cdrwa' for solr user, set 'r' only for the world, skipping /znode/configs and znode/collections (set those to 'cr' for the world)
   sasl_users_str: comma separated sasl users
   """
-  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, True)
+  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts, True)
   secure_solr_znode_cmd = format('{solr_cli_prefix} --secure-solr-znode --jaas-file {jaas_file} --sasl-users {sasl_users_str}')
   Execute(secure_solr_znode_cmd)
 
-def remove_admin_handlers(zookeeper_quorum, solr_znode, java64_home, collection, jaas_file, retry = 5, interval = 10):
+def remove_admin_handlers(zookeeper_quorum, solr_znode, java64_home, collection, jaas_file, retry = 5, interval = 10, java_opts=None):
   """
   Remove "solr.admin.AdminHandlers" request handler from collection config. Required for migrating to Solr 6 from Solr 5.
   """
-  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home)
+  solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts)
   remove_admin_handlers_cmd = format('{solr_cli_prefix} --remove-admin-handlers --collection {collection} --retry {retry} --interval {interval}')
   if jaas_file is not None:
     remove_admin_handlers_cmd+=format(' --jaas-file {jaas_file}')

ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py

@@ -25,7 +25,7 @@ from contextlib import closing
 from resource_management.core.resources.system import Execute
 
 def archive_dir(output_filename, input_dir):
-  Execute(('tar', '-zcvf', output_filename, input_dir),
+  Execute(('tar', '-zcf', output_filename, '-C', input_dir, '.'),
     sudo = True,
     tries = 3,
     try_sleep = 1,
@@ -40,7 +40,7 @@ def archive_directory_dereference(archive, directory):
   :return:  None
   """
 
-  Execute(('tar', '-zcvhf', archive, directory),
+  Execute(('tar', '-zchf', archive, '-C', directory, '.'),
     sudo = True,
     tries = 3,
     try_sleep = 1,

ambari-common/src/main/python/resource_management/libraries/script/script.py

@@ -54,7 +54,7 @@ from resource_management.libraries.functions.version import format_stack_version
 from resource_management.libraries.functions import stack_tools
 from resource_management.libraries.functions.constants import Direction
 from resource_management.libraries.script.config_dictionary import ConfigDictionary, UnknownConfiguration
-from resource_management.libraries.functions.repository_util import CommandRepository
+from resource_management.libraries.functions.repository_util import CommandRepository, RepositoryUtil
 from resource_management.core.resources.system import Execute
 from contextlib import closing
 from resource_management.libraries.functions.stack_features import check_stack_feature
@@ -357,6 +357,14 @@ class Script(object):
       Logger.logger.exception("Can not read json file with command parameters: ")
       sys.exit(1)
 
+    from resource_management.libraries.functions import lzo_utils
+
+    repo_tags_to_skip = set()
+    if not lzo_utils.is_gpl_license_accepted():
+      repo_tags_to_skip.add("GPL")
+
+    Script.repository_util = RepositoryUtil(Script.config, repo_tags_to_skip)
+
     # Run class method depending on a command type
     try:
       method = self.choose_method_to_execute(self.command_name)

ambari-infra/ambari-infra-assembly/pom.xml

@@ -25,7 +25,7 @@
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>ambari-infra-assembly</artifactId>
-  <name>Ambari Infra Solr Assembly</name>
+  <name>Ambari Infra Assembly</name>
   <url>http://maven.apache.org</url>
 
   <properties>
@@ -41,7 +41,7 @@
     <infra-manager.package.name>ambari-infra-manager</infra-manager.package.name>
     <infra-manager.dir>${project.basedir}/../ambari-infra-manager</infra-manager.dir>
     <infra-manager.mapping.path>${mapping.base.path}/${infra-manager.package.name}</infra-manager.mapping.path>
-    <infra-manager.conf.mapping.path>/etc/${infra-manager.package.name}/conf</infra-manager.conf.mapping.path>
+    <infra-manager.conf.mapping.path>${infra-manager.mapping.path}/conf</infra-manager.conf.mapping.path>
   </properties>
 
   <profiles>
@@ -142,30 +142,16 @@
                     <scriptFile>${project.basedir}/src/main/package/rpm/manager/postinstall.sh</scriptFile>
                     <fileEncoding>utf-8</fileEncoding>
                   </postinstallScriptlet>
+                  <postremoveScriptlet>
+                    <scriptFile>${project.basedir}/src/main/package/rpm/manager/postremove.sh</scriptFile>
+                    <fileEncoding>utf-8</fileEncoding>
+                  </postremoveScriptlet>
                   <mappings>
                     <mapping>
                       <directory>${infra-manager.mapping.path}</directory>
                       <sources>
                         <source>
                           <location>${infra-manager.dir}/target/package</location>
-                          <excludes>
-                            <exclude>log4j.xml</exclude>
-                            <exclude>infra-manager.properties</exclude>
-                            <exclude>infra-manager-env.sh</exclude>
-                          </excludes>
-                        </source>
-                      </sources>
-                    </mapping>
-                    <mapping>
-                      <directory>${infra-manager.conf.mapping.path}</directory>
-                      <sources>
-                        <source>
-                          <location>${infra-manager.dir}/target/package</location>
-                          <includes>
-                            <include>log4j.xml</include>
-                            <include>infra-manager.properties</include>
-                            <include>infra-manager-env.sh</include>
-                          </includes>
                         </source>
                       </sources>
                     </mapping>
@@ -354,23 +340,6 @@
                         <group>root</group>
                         <prefix>${infra-manager.mapping.path}</prefix>
                       </mapper>
-                      <excludes>
-                        log4j.xml,infra-manager.properties,infra-manager-env.sh
-                      </excludes>
-                    </data>
-                    <data>
-                      <src>${infra-manager.dir}/target/package</src>
-                      <type>directory</type>
-                      <mapper>
-                        <prefix>${infra-manager.conf.mapping.path}</prefix>
-                        <type>perm</type>
-                        <user>root</user>
-                        <group>root</group>
-                        <filemode>644</filemode>
-                      </mapper>
-                      <includes>
-                        log4j.xml,infra-manager.properties,infra-manager-env.sh
-                      </includes>
                     </data>
                   </dataSet>
                 </configuration>

ambari-infra/ambari-infra-assembly/src/main/package/deb/manager/postinst

@@ -15,6 +15,13 @@
 # limitations under the License
 
 INFRA_MANAGER_LINK_NAME="/usr/bin/infra-manager"
-INFRA_MANAGER_SOURCE="/usr/lib/ambari-infra-manager/infraManager.sh"
+INFRA_MANAGER_SOURCE="/usr/lib/ambari-infra-manager/bin/infraManager.sh"
+INFRA_MANAGER_CONF_LINK_DIR="/etc/ambari-infra-manager"
+INFRA_MANAGER_CONF_LINK_NAME="$INFRA_MANAGER_CONF_LINK_DIR/conf"
+INFRA_MANAGER_CONF_SOURCE="/usr/lib/ambari-infra-manager/conf"
 
 rm -f $INFRA_MANAGER_LINK_NAME ; ln -s $INFRA_MANAGER_SOURCE $INFRA_MANAGER_LINK_NAME
+rm -f $INFRA_MANAGER_CONF_LINK_NAME
+rm -rf $INFRA_MANAGER_CONF_LINK_DIR
+mkdir -p $INFRA_MANAGER_CONF_LINK_DIR
+ln -s $INFRA_MANAGER_CONF_SOURCE $INFRA_MANAGER_CONF_LINK_NAME

ambari-infra/ambari-infra-assembly/src/main/package/deb/manager/postrm

@@ -13,3 +13,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License
+
+INFRA_MANAGER_CONF_LINK_DIR="/etc/ambari-infra-manager"
+INFRA_MANAGER_CONF_LINK_NAME="$INFRA_MANAGER_CONF_LINK_DIR/conf"
+INFRA_MANAGER_LINK_NAME="/usr/bin/infra-manager"
+
+rm -f $INFRA_MANAGER_LINK_NAME
+rm -f $INFRA_MANAGER_CONF_LINK_NAME
+rm -rf $INFRA_MANAGER_CONF_LINK_DIR

ambari-infra/ambari-infra-assembly/src/main/package/rpm/manager/postinstall.sh

@@ -15,6 +15,13 @@
 # limitations under the License
 
 INFRA_MANAGER_LINK_NAME="/usr/bin/infra-manager"
-INFRA_MANAGER_SOURCE="/usr/lib/ambari-infra-manager/infraManager.sh"
+INFRA_MANAGER_SOURCE="/usr/lib/ambari-infra-manager/bin/infraManager.sh"
+INFRA_MANAGER_CONF_LINK_DIR="/etc/ambari-infra-manager"
+INFRA_MANAGER_CONF_LINK_NAME="$INFRA_MANAGER_CONF_LINK_DIR/conf"
+INFRA_MANAGER_CONF_SOURCE="/usr/lib/ambari-infra-manager/conf"
 
-rm -f $INFRA_MANAGER_LINK_NAME ; ln -s $INFRA_MANAGER_SOURCE $INFRA_MANAGER_LINK_NAME
+rm -f $INFRA_MANAGER_LINK_NAME ; ln -s $INFRA_MANAGER_SOURCE $INFRA_MANAGER_LINK_NAME
+rm -f $INFRA_MANAGER_CONF_LINK_NAME
+rm -rf $INFRA_MANAGER_CONF_LINK_DIR
+mkdir -p $INFRA_MANAGER_CONF_LINK_DIR
+ln -s $INFRA_MANAGER_CONF_SOURCE $INFRA_MANAGER_CONF_LINK_NAME

ambari-infra/ambari-infra-assembly/src/main/package/rpm/manager/postremove.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+INFRA_MANAGER_CONF_LINK_DIR="/etc/ambari-infra-manager"
+INFRA_MANAGER_CONF_LINK_NAME="$INFRA_MANAGER_CONF_LINK_DIR/conf"
+INFRA_MANAGER_LINK_NAME="/usr/bin/infra-manager"
+
+rm -f $INFRA_MANAGER_LINK_NAME
+rm -f $INFRA_MANAGER_CONF_LINK_NAME
+rm -rf $INFRA_MANAGER_CONF_LINK_DIR

ambari-infra/ambari-infra-manager-it/pom.xml

@@ -0,0 +1,167 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+  <parent>
+    <artifactId>ambari-infra</artifactId>
+    <groupId>org.apache.ambari</groupId>
+    <version>2.0.0.0-SNAPSHOT</version>
+  </parent>
+
+  <name>Ambari Infra Manager Integration Tests</name>
+  <url>http://maven.apache.org</url>
+  <modelVersion>4.0.0</modelVersion>
+
+  <artifactId>ambari-infra-manager-it</artifactId>
+
+  <properties>
+    <jbehave.version>4.0.5</jbehave.version>
+    <failsafe-plugin.version>2.20</failsafe-plugin.version>
+    <docker.host>localhost</docker.host>
+    <stories.location>NONE</stories.location>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.solr</groupId>
+      <artifactId>solr-solrj</artifactId>
+      <version>${solr.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.amazonaws</groupId>
+      <artifactId>aws-java-sdk-s3</artifactId>
+      <version>1.11.5</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <version>2.5</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>1.7.20</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+      <version>1.7.20</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.jbehave</groupId>
+      <artifactId>jbehave-core</artifactId>
+      <version>${jbehave.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.easymock</groupId>
+      <artifactId>easymock</artifactId>
+      <version>3.4</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.hamcrest</groupId>
+      <artifactId>hamcrest-all</artifactId>
+      <version>1.3</version>
+      <scope>test</scope>
+    </dependency>
+    <!-- Hadoop -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>${hadoop.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs-client</artifactId>
+      <version>${hadoop.version}</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <testOutputDirectory>target/classes</testOutputDirectory>
+    <testResources>
+      <testResource>
+        <directory>src/test/java/</directory>
+        <includes>
+          <include>**/*.story</include>
+        </includes>
+      </testResource>
+      <testResource>
+        <directory>src/test/resources</directory>
+      </testResource>
+    </testResources>
+  </build>
+
+  <profiles>
+    <profile>
+      <id>it</id>
+      <activation>
+        <property>
+          <name>it</name>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-failsafe-plugin</artifactId>
+            <version>${failsafe-plugin.version}</version>
+            <executions>
+              <execution>
+                <id>run-integration-tests</id>
+                <phase>integration-test</phase>
+                <goals>
+                  <goal>integration-test</goal>
+                </goals>
+                <configuration>
+                  <includes>
+                    <include>**/*Stories.java</include>
+                  </includes>
+                  <systemPropertyVariables>
+                    <log4j.configuration>file:${project.build.testOutputDirectory}/log4j.properties</log4j.configuration>
+                    <docker.host>${docker.host}</docker.host>
+                    <backend.stories.location>${stories.location}</backend.stories.location>
+                  </systemPropertyVariables>
+                </configuration>
+              </execution>
+              <execution>
+                <id>verify-integration-tests</id>
+                <phase>verify</phase>
+                <goals>
+                  <goal>verify</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
+
+</project>

ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/HttpResponse.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra;
+
+public class HttpResponse {
+  private final int code;
+  private final String body;
+
+  public HttpResponse(int code, String body) {
+    this.code = code;
+    this.body = body;
+  }
+
+  public int getCode() {
+    return code;
+  }
+
+  public String getBody() {
+    return body;
+  }
+}

ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/InfraClient.java

@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.io.IOUtils;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.commons.lang.StringUtils.isBlank;
+
+// TODO: use swagger
+public class InfraClient implements AutoCloseable {
+  private static final Logger LOG = LoggerFactory.getLogger(InfraClient.class);
+
+  private final CloseableHttpClient httpClient;
+  private final URI baseUrl;
+
+  public InfraClient(String baseUrl) {
+    try {
+      this.baseUrl = new URI(baseUrl);
+    } catch (URISyntaxException e) {
+      throw new RuntimeException(e);
+    }
+    httpClient = HttpClientBuilder.create().setRetryHandler(new DefaultHttpRequestRetryHandler(0, false)).build();
+  }
+
+  @Override
+  public void close() throws Exception {
+    httpClient.close();
+  }
+
+  // TODO: return job data
+  public void getJobs() {
+    execute(new HttpGet(baseUrl));
+  }
+
+  private HttpResponse execute(HttpRequestBase post) {
+    try (CloseableHttpResponse response = httpClient.execute(post)) {
+      String responseBodyText = IOUtils.toString(response.getEntity().getContent(), Charset.defaultCharset());
+      int statusCode = response.getStatusLine().getStatusCode();
+      LOG.info("Response code {} body {} ", statusCode, responseBodyText);
+      if (!(200 <= statusCode && statusCode <= 299))
+        throw new RuntimeException("Error while executing http request: " + responseBodyText);
+      return new HttpResponse(statusCode, responseBodyText);
+    } catch (ClientProtocolException e) {
+      throw new RuntimeException(e);
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+  }
+
+  public JobExecutionInfo startJob(String jobName, String parameters) {
+    URIBuilder uriBuilder = new URIBuilder(baseUrl);
+    uriBuilder.setScheme("http");
+    uriBuilder.setPath(uriBuilder.getPath() + "/" + jobName);
+    if (!isBlank(parameters))
+      uriBuilder.addParameter("params", parameters);
+    try {
+      String responseText = execute(new HttpPost(uriBuilder.build())).getBody();
+      Map<String, Object> responseContent = new ObjectMapper().readValue(responseText, new TypeReference<HashMap<String,Object>>() {});
+      return new JobExecutionInfo(responseContent.get("jobId").toString(), ((Map)responseContent.get("jobExecutionData")).get("id").toString());
+    } catch (URISyntaxException | JsonParseException | JsonMappingException e) {
+      throw new RuntimeException(e);
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+  }
+
+  public void restartJob(String jobName, String jobId) {
+    URIBuilder uriBuilder = new URIBuilder(baseUrl);
+    uriBuilder.setScheme("http");
+    uriBuilder.setPath(String.format("%s/%s/%s/executions", uriBuilder.getPath(), jobName, jobId));
+    uriBuilder.addParameter("operation", "RESTART");
+    try {
+      HttpResponse httpResponse = execute(new HttpPost(uriBuilder.build()));
+      if (httpResponse.getCode() != 200)
+        throw new RuntimeException(httpResponse.getBody());
+    } catch (URISyntaxException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  public void stopJob(String jobExecutionId) {
+    URIBuilder uriBuilder = new URIBuilder(baseUrl);
+    uriBuilder.setScheme("http");
+    uriBuilder.setPath(String.format("%s/executions/%s", uriBuilder.getPath(), jobExecutionId));
+    uriBuilder.addParameter("operation", "STOP");
+    try {
+      execute(new HttpDelete(uriBuilder.build()));
+    } catch (URISyntaxException e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/InfraManagerStories.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra;
+
+import org.apache.ambari.infra.steps.ExportJobsSteps;
+import org.apache.commons.lang.StringUtils;
+import org.jbehave.core.configuration.Configuration;
+import org.jbehave.core.configuration.MostUsefulConfiguration;
+import org.jbehave.core.io.LoadFromClasspath;
+import org.jbehave.core.io.LoadFromRelativeFile;
+import org.jbehave.core.io.StoryFinder;
+import org.jbehave.core.io.StoryLoader;
+import org.jbehave.core.junit.JUnitStories;
+import org.jbehave.core.reporters.Format;
+import org.jbehave.core.reporters.StoryReporterBuilder;
+import org.jbehave.core.steps.InjectableStepsFactory;
+import org.jbehave.core.steps.InstanceStepsFactory;
+import org.jbehave.core.steps.ParameterConverters;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+import static org.jbehave.core.io.CodeLocations.codeLocationFromClass;
+
+public class InfraManagerStories extends JUnitStories {
+  private static final String BACKEND_STORIES_LOCATION_PROPERTY = "backend.stories.location";
+  private static final String STORY_SUFFIX = ".story";
+
+  @Override
+  public Configuration configuration() {
+    return new MostUsefulConfiguration()
+            .useStoryLoader(getStoryLoader(BACKEND_STORIES_LOCATION_PROPERTY, this.getClass()))
+            .useParameterConverters(new ParameterConverters().addConverters(new OffsetDateTimeConverter()))
+            .useStoryReporterBuilder(
+                    new StoryReporterBuilder().withFailureTrace(true).withDefaultFormats().withFormats(Format.CONSOLE, Format.TXT));
+  }
+
+  private static StoryLoader getStoryLoader(String property, Class clazz) {
+    boolean useExternalStoryLocation = useExternalStoryLocation(property);
+    if (useExternalStoryLocation) {
+      try {
+        return new LoadFromRelativeFile(new URL("file://" + System.getProperty(property)));
+      } catch (Exception e) {
+        throw new RuntimeException("Cannot load story files from url: file://" + System.getProperty(property));
+      }
+    } else {
+      return new LoadFromClasspath(clazz);
+    }
+  }
+
+  @Override
+  public InjectableStepsFactory stepsFactory() {
+    return new InstanceStepsFactory(configuration(), new ExportJobsSteps());
+  }
+
+  @Override
+  protected List<String> storyPaths() {
+    return findStories(BACKEND_STORIES_LOCATION_PROPERTY, STORY_SUFFIX, this.getClass());
+  }
+
+  private static List<String> findStories(String property, String suffix, Class clazz) {
+    if (useExternalStoryLocation(property)) {
+      return findStoriesInFolder(System.getProperty(property), suffix);
+    } else {
+      return new StoryFinder()
+              .findPaths(codeLocationFromClass(clazz).getFile(), singletonList(String.format("**/*%s", suffix)), null);
+    }
+  }
+
+  private static List<String> findStoriesInFolder(String folderAbsolutePath, String suffix) {
+    List<String> results = new ArrayList<>();
+    File folder = new File(folderAbsolutePath);
+    File[] listOfFiles = folder.listFiles();
+    if (listOfFiles != null) {
+      for (File file : listOfFiles) {
+        if (file.getName().endsWith(suffix)) {
+          results.add(file.getName());
+        }
+      }
+    }
+    return results;
+  }
+
+  private static boolean useExternalStoryLocation(String property) {
+    String storyLocationProp = System.getProperty(property);
+    return StringUtils.isNotEmpty(storyLocationProp) && !"NONE".equals(storyLocationProp);
+  }
+
+}

ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/JobExecutionInfo.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra;
+
+public class JobExecutionInfo {
+  private final String jobId;
+  private final String executionId;
+
+  public JobExecutionInfo(String jobId, String executionId) {
+    this.jobId = jobId;
+    this.executionId = executionId;
+  }
+
+  public String getJobId() {
+    return jobId;
+  }
+
+  public String getExecutionId() {
+    return executionId;
+  }
+
+  @Override
+  public String toString() {
+    return "JobExecutionInfo{" +
+            "jobId='" + jobId + '\'' +
+            ", executionId='" + executionId + '\'' +
+            '}';
+  }
+}

ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/OffsetDateTimeConverter.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra;
+
+import org.jbehave.core.steps.ParameterConverters;
+
+import java.lang.reflect.Type;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeFormatter;
+
+public class OffsetDateTimeConverter implements ParameterConverters.ParameterConverter {
+  public static final DateTimeFormatter SOLR_DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX");
+
+  @Override
+  public boolean accept(Type type) {
+    return type instanceof Class<?> && OffsetDateTime.class.isAssignableFrom((Class<?>) type);
+  }
+
+  @Override
+  public Object convertValue(String value, Type type) {
+    return OffsetDateTime.parse(value, SOLR_DATETIME_FORMATTER);
+  }
+}

ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/AbstractInfraSteps.java

@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.steps;
+
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import org.apache.ambari.infra.InfraClient;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
+import org.apache.solr.common.SolrInputDocument;
+import org.jbehave.core.annotations.AfterStories;
+import org.jbehave.core.annotations.BeforeStories;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.time.OffsetDateTime;
+import java.util.Date;
+import java.util.function.BooleanSupplier;
+
+import static java.lang.System.currentTimeMillis;
+
+public abstract class AbstractInfraSteps {
+  private static final Logger LOG = LoggerFactory.getLogger(AbstractInfraSteps.class);
+
+  private static final int SOLR_PORT = 8983;
+  private static final int INFRA_MANAGER_PORT = 61890;
+  private static final int FAKE_S3_PORT = 4569;
+  private static final int HDFS_PORT = 9000;
+  private static final String AUDIT_LOGS_COLLECTION = "audit_logs";
+  private static final String HADOOP_LOGS_COLLECTION = "hadoop_logs";
+  protected static final String S3_BUCKET_NAME = "testbucket";
+  private String ambariFolder;
+  private String shellScriptLocation;
+  private String dockerHost;
+  private SolrClient solrClient;
+  private AmazonS3Client s3client;
+  private int documentId = 0;
+
+  public InfraClient getInfraClient() {
+    return new InfraClient(String.format("http://%s:%d/api/v1/jobs", dockerHost, INFRA_MANAGER_PORT));
+  }
+
+  public SolrClient getSolrClient() {
+    return solrClient;
+  }
+
+  public AmazonS3Client getS3client() {
+    return s3client;
+  }
+
+  public String getLocalDataFolder() {
+    return ambariFolder + "/ambari-infra/ambari-infra-manager/docker/test-out";
+  }
+
+  @BeforeStories
+  public void initDockerContainer() throws Exception {
+    System.setProperty("HADOOP_USER_NAME", "root");
+
+    URL location = AbstractInfraSteps.class.getProtectionDomain().getCodeSource().getLocation();
+    ambariFolder = new File(location.toURI()).getParentFile().getParentFile().getParentFile().getParent();
+
+    LOG.info("Clean local data folder {}", getLocalDataFolder());
+    FileUtils.cleanDirectory(new File(getLocalDataFolder()));
+
+    shellScriptLocation = ambariFolder + "/ambari-infra/ambari-infra-manager/docker/infra-manager-docker-compose.sh";
+    LOG.info("Create new docker container for testing Ambari Infra Manager ...");
+    runCommand(new String[]{shellScriptLocation, "start"});
+
+    dockerHost = System.getProperty("docker.host") != null ? System.getProperty("docker.host") : "localhost";
+
+    waitUntilSolrIsUp();
+
+    solrClient = new LBHttpSolrClient.Builder().withBaseSolrUrls(String.format("http://%s:%d/solr/%s_shard1_replica1",
+            dockerHost,
+            SOLR_PORT,
+            AUDIT_LOGS_COLLECTION)).build();
+
+    createSolrCollection(AUDIT_LOGS_COLLECTION);
+    createSolrCollection(HADOOP_LOGS_COLLECTION);
+
+    LOG.info("Initializing s3 client");
+    s3client = new AmazonS3Client(new BasicAWSCredentials("remote-identity", "remote-credential"));
+    s3client.setEndpoint(String.format("http://%s:%d", dockerHost, FAKE_S3_PORT));
+    s3client.createBucket(S3_BUCKET_NAME);
+
+    checkInfraManagerReachable();
+  }
+
+  private void createSolrCollection(String collectionName) {
+    LOG.info("Creating collection");
+    runCommand(new String[]{"docker", "exec", "docker_solr_1", "solr", "create_collection", "-c", collectionName, "-d", "configsets/"+ collectionName +"/conf", "-n", collectionName + "_conf"});
+  }
+
+  private void runCommand(String[] command) {
+    try {
+      LOG.info("Exec command: {}", StringUtils.join(command, " "));
+      Process process = Runtime.getRuntime().exec(command);
+      String stdout = IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8);
+      LOG.info("Exec command result {}", stdout);
+    } catch (Exception e) {
+      throw new RuntimeException("Error during execute shell command: ", e);
+    }
+  }
+
+  private void waitUntilSolrIsUp() throws Exception {
+    try(CloseableHttpClient httpClient = HttpClientBuilder.create().setRetryHandler(new DefaultHttpRequestRetryHandler(0, false)).build()) {
+      doWithin(60, "Start Solr", () -> pingSolr(httpClient));
+    }
+  }
+
+  protected void doWithin(int sec, String actionName, BooleanSupplier predicate) {
+    doWithin(sec, actionName, () -> {
+      if (!predicate.getAsBoolean())
+        throw new RuntimeException("Predicate was false!");
+    });
+  }
+
+  protected void doWithin(int sec, String actionName, Runnable runnable) {
+    long start = currentTimeMillis();
+    Exception exception;
+    while (true) {
+      try {
+        runnable.run();
+        return;
+      }
+      catch (Exception e) {
+        exception = e;
+      }
+
+      if (currentTimeMillis() - start > sec * 1000) {
+        throw new AssertionError(String.format("Unable to perform action '%s' within %d seconds", actionName, sec), exception);
+      }
+      else {
+        LOG.info("Performing action '{}' failed. retrying...", actionName);
+      }
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        throw new RuntimeException(e);
+      }
+    }
+  }
+
+  private boolean pingSolr(CloseableHttpClient httpClient) {
+    try (CloseableHttpResponse response = httpClient.execute(new HttpGet(String.format("http://%s:%d/solr/admin/collections?action=LIST", dockerHost, SOLR_PORT)))) {
+      return response.getStatusLine().getStatusCode() == 200;
+    }
+    catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+  }
+
+  private void checkInfraManagerReachable() throws Exception {
+    try (InfraClient httpClient = getInfraClient()) {
+      doWithin(30, "Start Ambari Infra Manager", httpClient::getJobs);
+      LOG.info("Ambari Infra Manager is up and running");
+    }
+  }
+
+  protected void addDocument(OffsetDateTime logtime) throws SolrServerException, IOException {
+    SolrInputDocument solrInputDocument = new SolrInputDocument();
+    solrInputDocument.addField("logType", "HDFSAudit");
+    solrInputDocument.addField("cluster", "cl1");
+    solrInputDocument.addField("event_count", 1);
+    solrInputDocument.addField("repo", "hdfs");
+    solrInputDocument.addField("reqUser", "ambari-qa");
+    solrInputDocument.addField("type", "hdfs_audit");
+    solrInputDocument.addField("seq_num", 9);
+    solrInputDocument.addField("result", 1);
+    solrInputDocument.addField("path", "/root/test-logs/hdfs-audit/hdfs-audit.log");
+    solrInputDocument.addField("ugi", "ambari-qa (auth:SIMPLE)");
+    solrInputDocument.addField("host", "logfeeder.apache.org");
+    solrInputDocument.addField("action", "getfileinfo");
+    solrInputDocument.addField("log_message", "allowed=true\tugi=ambari-qa (auth:SIMPLE)\tip=/192.168.64.102\tcmd=getfileinfo\tsrc=/ats/active\tdst=null\tperm=null\tproto=rpc\tcallerContext=HIVE_QUERY_ID:ambari-qa_20160317200111_223b3079-4a2d-431c-920f-6ba37ed63e9f");
+    solrInputDocument.addField("logger_name", "FSNamesystem.audit");
+    solrInputDocument.addField("id", Integer.toString(documentId++));
+    solrInputDocument.addField("authType", "SIMPLE");
+    solrInputDocument.addField("logfile_line_number", 1);
+    solrInputDocument.addField("cliIP", "/192.168.64.102");
+    solrInputDocument.addField("level", "INFO");
+    solrInputDocument.addField("resource", "/ats/active");
+    solrInputDocument.addField("ip", "172.18.0.2");
+    solrInputDocument.addField("evtTime", "2017-12-08T10:23:16.452Z");
+    solrInputDocument.addField("req_caller_id", "HIVE_QUERY_ID:ambari-qa_20160317200111_223b3079-4a2d-431c-920f-6ba37ed63e9f");
+    solrInputDocument.addField("repoType", 1);
+    solrInputDocument.addField("enforcer", "hadoop-acl");
+    solrInputDocument.addField("cliType", "rpc");
+    solrInputDocument.addField("message_md5", "-6778765776916226588");
+    solrInputDocument.addField("event_md5", "5627261521757462732");
+    solrInputDocument.addField("logtime", new Date(logtime.toInstant().toEpochMilli()));
+    solrInputDocument.addField("_ttl_", "+7DAYS");
+    solrInputDocument.addField("_expire_at_", "2017-12-15T10:23:19.106Z");
+    solrClient.add(solrInputDocument);
+  }
+
+  @AfterStories
+  public void shutdownContainers() throws Exception {
+    Thread.sleep(2000); // sync with s3 server
+    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(S3_BUCKET_NAME);
+    ObjectListing objectListing = getS3client().listObjects(listObjectsRequest);
+    LOG.info("Found {} files on s3.", objectListing.getObjectSummaries().size());
+    objectListing.getObjectSummaries().forEach(s3ObjectSummary ->  LOG.info("Found file on s3 with key {}", s3ObjectSummary.getKey()));
+
+    LOG.info("Listing files on hdfs.");
+    try (FileSystem fileSystem = getHdfs()) {
+      int count = 0;
+      RemoteIterator<LocatedFileStatus> it = fileSystem.listFiles(new Path("/test_audit_logs"), true);
+      while (it.hasNext()) {
+        LOG.info("Found file on hdfs with name {}", it.next().getPath().getName());
+        ++count;
+      }
+      LOG.info("{} files found on hfds", count);
+    }
+
+    LOG.info("shutdown containers");
+    runCommand(new String[]{shellScriptLocation, "stop"});
+  }
+
+  protected FileSystem getHdfs() throws IOException {
+    Configuration conf = new Configuration();
+    conf.set("fs.defaultFS", String.format("hdfs://%s:%d/", dockerHost, HDFS_PORT));
+    return FileSystem.get(conf);
+  }
+}

ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/ExportJobsSteps.java

@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.steps;
+
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import org.apache.ambari.infra.InfraClient;
+import org.apache.ambari.infra.JobExecutionInfo;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.jbehave.core.annotations.Given;
+import org.jbehave.core.annotations.Then;
+import org.jbehave.core.annotations.When;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.time.Duration;
+import java.time.OffsetDateTime;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.ambari.infra.OffsetDateTimeConverter.SOLR_DATETIME_FORMATTER;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasProperty;
+import static org.hamcrest.core.IsCollectionContaining.hasItem;
+import static org.junit.Assert.assertThat;
+
+public class ExportJobsSteps extends AbstractInfraSteps {
+  private static final Logger LOG = LoggerFactory.getLogger(ExportJobsSteps.class);
+
+  private Map<String, JobExecutionInfo> launchedJobs = new HashMap<>();
+
+  @Given("$count documents in solr")
+  public void addDocuments(int count) throws Exception {
+    OffsetDateTime intervalEnd = OffsetDateTime.now();
+    for (int i = 0; i < count; ++i) {
+      addDocument(intervalEnd.minusMinutes(i % (count / 10)));
+    }
+    getSolrClient().commit();
+  }
+
+  @Given("$count documents in solr with logtime from $startLogtime to $endLogtime")
+  public void addDocuments(long count, OffsetDateTime startLogtime, OffsetDateTime endLogtime) throws Exception {
+    Duration duration = Duration.between(startLogtime, endLogtime);
+    long increment = duration.toNanos() / count;
+    for (int i = 0; i < count; ++i)
+      addDocument(startLogtime.plusNanos(increment * i));
+    getSolrClient().commit();
+  }
+
+  @Given("a file on s3 with key $key")
+  public void addFileToS3(String key) throws Exception {
+    try (ByteArrayInputStream inputStream = new ByteArrayInputStream("anything".getBytes())) {
+      getS3client().putObject(S3_BUCKET_NAME, key, inputStream, new ObjectMetadata());
+    }
+  }
+
+  @When("start $jobName job")
+  public void startJob(String jobName) throws Exception {
+    startJob(jobName, null, 0);
+  }
+
+  @When("start $jobName job with parameters $parameters after $waitSec seconds")
+  public void startJob(String jobName, String parameters, int waitSec) throws Exception {
+    Thread.sleep(waitSec * 1000);
+    try (InfraClient httpClient = getInfraClient()) {
+      JobExecutionInfo jobExecutionInfo = httpClient.startJob(jobName, parameters);
+      LOG.info("Job {} started: {}", jobName, jobExecutionInfo);
+      launchedJobs.put(jobName, jobExecutionInfo);
+    }
+  }
+
+  @When("restart $jobName job within $waitSec seconds")
+  public void restartJob(String jobName, int waitSec) {
+    doWithin(waitSec, "Restarting job " + jobName, () -> {
+      try (InfraClient httpClient = getInfraClient()) {
+        httpClient.restartJob(jobName, launchedJobs.get(jobName).getJobId());
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    });
+  }
+
+  @When("stop job $jobName after at least $count file exists in s3 with filename containing text $text within $waitSec seconds")
+  public void stopJob(String jobName, int count, String text, int waitSec) throws Exception {
+    AmazonS3Client s3Client = getS3client();
+    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(S3_BUCKET_NAME);
+    doWithin(waitSec, "check uploaded files to s3", () -> s3Client.doesBucketExist(S3_BUCKET_NAME)
+            && fileCountOnS3(text, s3Client, listObjectsRequest) > count);
+
+    try (InfraClient httpClient = getInfraClient()) {
+      httpClient.stopJob(launchedJobs.get(jobName).getExecutionId());
+    }
+  }
+
+  @When("delete file with key $key from s3")
+  public void deleteFileFromS3(String key) {
+    getS3client().deleteObject(S3_BUCKET_NAME, key);
+  }
+
+  @Then("Check filenames contains the text $text on s3 server after $waitSec seconds")
+  public void checkS3After(String text, int waitSec) {
+    AmazonS3Client s3Client = getS3client();
+    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(S3_BUCKET_NAME);
+    doWithin(waitSec, "check uploaded files to s3", () -> s3Client.doesBucketExist(S3_BUCKET_NAME)
+            && !s3Client.listObjects(listObjectsRequest).getObjectSummaries().isEmpty());
+
+    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
+    assertThat(objectListing.getObjectSummaries(), hasItem(hasProperty("key", containsString(text))));
+  }
+
+  @Then("Check $count files exists on s3 server with filenames containing the text $text after $waitSec seconds")
+  public void checkNumberOfFilesOnS3(long count, String text, int waitSec) {
+    AmazonS3Client s3Client = getS3client();
+    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(S3_BUCKET_NAME);
+    doWithin(waitSec, "check uploaded files to s3", () -> s3Client.doesBucketExist(S3_BUCKET_NAME)
+            && fileCountOnS3(text, s3Client, listObjectsRequest) == count);
+  }
+
+  private long fileCountOnS3(String text, AmazonS3Client s3Client, ListObjectsRequest listObjectsRequest) {
+    return s3Client.listObjects(listObjectsRequest).getObjectSummaries().stream()
+    .filter(s3ObjectSummary -> s3ObjectSummary.getKey().contains(text))
+    .count();
+  }
+
+  @Then("Less than $count files exists on s3 server with filenames containing the text $text after $waitSec seconds")
+  public void checkLessThanFileExistsOnS3(long count, String text, int waitSec) {
+    AmazonS3Client s3Client = getS3client();
+    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(S3_BUCKET_NAME);
+    doWithin(waitSec, "check uploaded files to s3", () -> s3Client.doesBucketExist(S3_BUCKET_NAME) && between(
+            fileCountOnS3(text, s3Client, listObjectsRequest), 1L, count - 1L));
+  }
+
+  private boolean between(long count, long from, long to) {
+    return from <= count && count <= to;
+  }
+
+  @Then("No file exists on s3 server with filenames containing the text $text")
+  public void fileNotExistOnS3(String text) {
+    AmazonS3Client s3Client = getS3client();
+    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(S3_BUCKET_NAME);
+    assertThat(s3Client.listObjects(listObjectsRequest).getObjectSummaries().stream()
+            .anyMatch(s3ObjectSummary -> s3ObjectSummary.getKey().contains(text)), is(false));
+  }
+
+  @Then("solr contains $count documents between $startLogtime and $endLogtime")
+  public void documentCount(int count, OffsetDateTime startLogTime, OffsetDateTime endLogTime) throws Exception {
+    SolrQuery query = new SolrQuery();
+    query.setRows(count * 2);
+    query.setQuery(String.format("logtime:[\"%s\" TO \"%s\"]", SOLR_DATETIME_FORMATTER.format(startLogTime), SOLR_DATETIME_FORMATTER.format(endLogTime)));
+    assertThat(getSolrClient().query(query).getResults().size(), is(count));
+  }
+
+  @Then("solr does not contain documents between $startLogtime and $endLogtime after $waitSec seconds")
+  public void isSolrEmpty(OffsetDateTime startLogTime, OffsetDateTime endLogTime, int waitSec) {
+    SolrQuery query = new SolrQuery();
+    query.setRows(1);
+    query.setQuery(String.format("logtime:[\"%s\" TO \"%s\"]", SOLR_DATETIME_FORMATTER.format(startLogTime), SOLR_DATETIME_FORMATTER.format(endLogTime)));
+    doWithin(waitSec, "check solr is empty", () -> isSolrEmpty(query));
+  }
+
+  private boolean isSolrEmpty(SolrQuery query) {
+    try {
+      return getSolrClient().query(query).getResults().isEmpty();
+    } catch (SolrServerException e) {
+      throw new RuntimeException(e);
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+  }
+
+  @Then("Check $count files exists on hdfs with filenames containing the text $text in the folder $path after $waitSec seconds")
+  public void checkNumberOfFilesOnHdfs(int count, String text, String path, int waitSec) throws Exception {
+    try (FileSystem fileSystem = getHdfs()) {
+      doWithin(waitSec, "check uploaded files to hdfs", () -> {
+        try {
+          int fileCount = 0;
+          RemoteIterator<LocatedFileStatus> it = fileSystem.listFiles(new Path(path), true);
+          while (it.hasNext()) {
+            if (it.next().getPath().getName().contains(text))
+              ++fileCount;
+          }
+          return fileCount == count;
+        }
+        catch (IOException e) {
+          throw new UncheckedIOException(e);
+        }
+      });
+    }
+  }
+
+  @Then("Check $count files exists on local filesystem with filenames containing the text $text in the folder $path for job $jobName")
+  public void checkNumberOfFilesOnLocalFilesystem(long count, String text, String path, String jobName) {
+    File destinationDirectory = new File(getLocalDataFolder(), path.replace("${jobId}", launchedJobs.get(jobName).getJobId()));
+    LOG.info("Destination directory path: {}", destinationDirectory.getAbsolutePath());
+    doWithin(5, "Destination directory exists", destinationDirectory::exists);
+
+    File[] files = requireNonNull(destinationDirectory.listFiles(),
+            String.format("Path %s is not a directory or an I/O error occurred!", destinationDirectory.getAbsolutePath()));
+    assertThat(Arrays.stream(files)
+            .filter(file -> file.getName().contains(text))
+            .count(), is(count));
+  }
+}

ambari-infra/ambari-infra-manager-it/src/test/resources/log4j.properties

@@ -0,0 +1,16 @@
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+log4j.rootLogger=INFO, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n

ambari-infra/ambari-infra-manager-it/src/test/resources/stories/infra_api_tests.story

@@ -0,0 +1,67 @@
+Scenario: Exporting documents form solr and upload them to s3 using defult configuration
+
+Given 1000 documents in solr
+When start archive_audit_logs job
+Then Check filenames contains the text audit_logs on s3 server after 20 seconds
+
+
+Scenario: Exporting 10 documents using writeBlockSize=3 produces 4 files
+
+Given 10 documents in solr with logtime from 2010-10-09T05:00:00.000Z to 2010-10-09T20:00:00.000Z
+When start archive_audit_logs job with parameters writeBlockSize=3,start=2010-10-09T00:00:00.000Z,end=2010-10-11T00:00:00.000Z after 2 seconds
+Then Check 4 files exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2010-10-09 after 20 seconds
+And solr does not contain documents between 2010-10-09T05:00:00.000Z and 2010-10-09T20:00:00.000Z after 5 seconds
+
+
+Scenario: Running archiving job with a bigger start value than end value exports and deletes 0 documents
+
+Given 10 documents in solr with logtime from 2010-01-01T05:00:00.000Z to 2010-01-04T05:00:00.000Z
+When start archive_audit_logs job with parameters writeBlockSize=3,start=2010-01-03T05:00:00.000Z,end=2010-01-02T05:00:00.000Z after 2 seconds
+Then No file exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2010-01-0
+And solr contains 10 documents between 2010-01-01T05:00:00.000Z and 2010-01-04T05:00:00.000Z
+
+
+Scenario: Archiving job fails when part of the data is exported. After resolving the issue and restarting the job exports the rest of the data.
+
+Given 200 documents in solr with logtime from 2011-10-09T05:00:00.000Z to 2011-10-09T20:00:00.000Z
+And a file on s3 with key solr_archive_audit_logs_-_2011-10-09T08-00-00.000Z.json.tar.gz
+When start archive_audit_logs job with parameters writeBlockSize=20,start=2010-11-09T00:00:00.000Z,end=2011-10-11T00:00:00.000Z after 2 seconds
+Then Check 3 files exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2011-10-09 after 20 seconds
+And solr does not contain documents between 2011-10-09T05:00:00.000Z and 2011-10-09T07:59:59.999Z after 5 seconds
+When delete file with key solr_archive_audit_logs_-_2011-10-09T08-00-00.000Z.json.tar.gz from s3
+And restart archive_audit_logs job within 2 seconds
+Then Check 10 files exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2011-10-09 after 20 seconds
+And solr does not contain documents between 2011-10-09T05:00:00.000Z and 2011-10-09T20:00:00.000Z after 5 seconds
+
+
+Scenario: After Deleting job deletes documents from solr no document found in the specified interval
+
+Given 10 documents in solr with logtime from 2012-10-09T05:00:00.000Z to 2012-10-09T20:00:00.000Z
+When start delete_audit_logs job with parameters start=2012-10-09T05:00:00.000Z,end=2012-10-09T20:00:00.000Z after 2 seconds
+Then solr does not contain documents between 2012-10-09T05:00:00.000Z and 2012-10-09T20:00:00.000Z after 5 seconds
+
+
+Scenario: Archiving documents to hdfs
+
+Given 1000 documents in solr with logtime from 2014-01-04T05:00:00.000Z to 2014-01-06T20:00:00.000Z
+When start archive_audit_logs job with parameters start=2014-01-04T05:00:00.000Z,end=2014-01-06T20:00:00.000Z,destination=HDFS after 2 seconds
+Then Check 7 files exists on hdfs with filenames containing the text audit_logs_-_2014-01-0 in the folder /test_audit_logs after 10 seconds
+And solr does not contain documents between 2014-01-04T05:00:00.000Z and 2014-01-06T20:00:00.000Z after 10 seconds
+
+
+Scenario: Archiving documents to local filesystem
+
+Given 200 documents in solr with logtime from 2014-02-04T05:00:00.000Z to 2014-02-06T20:00:00.000Z
+When start archive_audit_logs job with parameters start=2014-02-04T05:00:00.000Z,end=2014-02-06T20:00:00.000Z,destination=LOCAL,localDestinationDirectory=/root/archive after 2 seconds
+Then Check 2 files exists on local filesystem with filenames containing the text audit_logs_-_2014-02-0 in the folder audit_logs_${jobId}_2014-02-06T20-00-00.000Z for job archive_audit_logs
+And solr does not contain documents between 2014-02-04T05:00:00.000Z and 2014-02-06T20:00:00.000Z after 10 seconds
+
+
+Scenario: Launch Archiving job. Initiate stop and check that part of the data is archived. After restart all data must be extracted.
+
+Given 200 documents in solr with logtime from 2014-03-09T05:00:00.000Z to 2014-03-09T20:00:00.000Z
+When start archive_audit_logs job with parameters writeBlockSize=20,start=2014-03-09T05:00:00.000Z,end=2014-03-09T20:00:00.000Z after 2 seconds
+And stop job archive_audit_logs after at least 1 file exists in s3 with filename containing text solr_archive_audit_logs_-_2014-03-09 within 10 seconds
+Then Less than 10 files exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2014-03-09 after 20 seconds
+When restart archive_audit_logs job within 10 seconds
+Then Check 10 files exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2014-03-09 after 20 seconds

ambari-infra/ambari-infra-manager/.gitignore

@@ -1,2 +1,5 @@
 out/*
-*.pid
+*.pid
+Profile
+.env
+test-out

ambari-infra/ambari-infra-manager/build.xml

@@ -33,13 +33,16 @@
     <copy todir="target/package/libs" includeEmptyDirs="no">
       <fileset file="target/*.jar"/>
     </copy>
-    <copy todir="target/package" includeEmptyDirs="no">
-      <fileset file="src/main/resources/infraManager.sh"/>
+    <copy todir="target/package/conf" includeEmptyDirs="no">
       <fileset file="src/main/resources/infra-manager-env.sh"/>
       <fileset file="target/classes/infra-manager.properties"/>
-      <fileset file="target/classes/log4j.xml"/>
+      <fileset file="target/classes/log4j2.xml"/>
+    </copy>
+    <copy todir="target/package/bin" includeEmptyDirs="no">
+      <fileset file="src/main/resources/infraManager.sh"/>
     </copy>
-    <chmod file="target/package/*.sh" perm="755"/>
+    <chmod file="target/package/conf/infra-manager-env.sh" perm="755"/>
+    <chmod file="target/package/bin/infraManager.sh" perm="755"/>
     <tar compression="gzip" destfile="target/ambari-infra-manager.tar.gz">
       <tarfileset mode="755" dir="target/package">
         <include name="*.sh"/>