AMBARI-1607. HDFS Mirroring: Create Mapper, Model and Mock Data. (Arun Kandregula via yusaku)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/trunk@1456275 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -12,6 +12,9 @@ Trunk (unreleased changes):
 
  NEW FEATURES
 
+ AMBARI-1607. HDFS Mirroring: Create Mapper, Model and Mock Data.
+ (Arun Kandregula via yusaku)
+
  AMBARI-1602. Edit User - drop the requirement to specify the old 
  password. (swagle)
 

ambari-web/app/assets/data/mirroring/all_dataset_jobs.json

@@ -0,0 +1,32 @@
+{
+  "href" : "http://ec2...com:8080/api/v1/feeds/feed1/instances",
+  "items" : [
+    {
+      "href":"http://ec2...com:8080/api/v1/feeds/feed1/instances/99",
+      "Instances":{
+        "id":"99",
+        "cluster":"drtarget1",
+        "sourceCluster":"drsource",
+        "status":"RUNNING",
+        "start":"2013-03-06T03:17Z",
+        "end":"2013-03-06T03:19Z",
+        "details":"",
+        "log":""
+      }
+    },
+    {
+      "href":"http://ec2...com:8080/api/v1/feeds/feed1/instances/95",
+      "Instances":{
+        "id":"95",
+        "cluster":"drtarget1",
+        "sourceCluster":"drsource",
+        "status":"RUNNING",
+        "start":"2013-03-06T03:17Z",
+        "end":"2013-03-06T03:19Z",
+        "details":"",
+        "log":""
+      }
+    }
+  ]
+}
+

ambari-web/app/assets/data/mirroring/all_datasets.json

@@ -0,0 +1,309 @@
+{
+  "href":"http://ec2...com:8080/api/v1/feeds",
+  "items":[
+    {
+      "href":"http://ec2...com:8080/api/v1/feeds/feed1",
+      "Feeds":{
+        "description":"test feed 1",
+        "name":"feed 1",
+        "status":"SCHEDULED",
+        "groups":"online,bi",
+        "frequency":"minutes(1)",
+        "timezone":"UTC",
+        "late-arrival":{
+          "cut-off":"minutes(10)"
+        },
+        "clusters":{
+          "cluster":[
+            {
+              "name":"drsource",
+              "type":"source",
+              "validity":{
+                "start":"2010-01-01T00:00Z",
+                "end":"2015-01-01T02:00Z"
+              },
+              "retention":{
+                "limit":"minutes(15)",
+                "action":"delete"
+              }
+            },
+            {
+              "name":"drtarget",
+              "type":"target",
+              "validity":{
+                "start":"2010-01-01T00:00Z",
+                "end":"2015-01-01T02:00Z"
+              },
+              "retention":{
+                "limit":"minutes(17)",
+                "action":"delete"
+              }
+            }
+          ]
+        },
+        "locations":{
+          "location":{
+            "type":"data",
+            "path":"/mirrorthis"
+          }
+        },
+        "ACL":{
+          "owner":"testuser",
+          "group":"group",
+          "permission":"0x755"
+        },
+        "schema":{
+          "location":"/schema/clicks",
+          "provider":"protobuf"
+        },
+        "properties":{
+          "property":[
+            {
+              "name":"field3",
+              "value":"value3"
+            },
+            {
+              "name":"field2",
+              "value":"value2"
+            },
+            {
+              "name":"field4",
+              "value":"value2"
+            }
+          ]
+        }
+      },
+      "instances":[
+        {
+          "href":"http://ec2...com:8080/api/v1/feeds/feed1/instances/99",
+          "Instances":{
+            "id":"99",
+            "cluster":"drtarget1",
+            "sourceCluster":"drsource",
+            "status":"RUNNING",
+            "start":"2013-03-06T03:17Z",
+            "end":"2013-03-06T03:19Z",
+            "details":"",
+            "log":""
+          }
+        },
+        {
+          "href":"http://ec2...com:8080/api/v1/feeds/feed1/instances/95",
+          "Instances":{
+            "id":"95",
+            "cluster":"drtarget1",
+            "sourceCluster":"drsource",
+            "status":"RUNNING",
+            "start":"2013-03-06T03:17Z",
+            "end":"2013-03-06T03:19Z",
+            "details":"",
+            "log":""
+          }
+        }
+      ]
+    },
+    {
+      "href":"http://ec2...com:8080/api/v1/feeds/feed2",
+      "Feeds":{
+        "description":"test feed 2",
+        "name":"feed 2",
+        "status":"SCHEDULED",
+        "groups":"online,bi",
+        "frequency":"minutes(1)",
+        "timezone":"UTC",
+        "late-arrival":{
+          "cut-off":"minutes(10)"
+        },
+        "clusters":{
+          "cluster":[
+            {
+              "name":"drsource",
+              "type":"source",
+              "validity":{
+                "start":"2010-01-01T00:00Z",
+                "end":"2015-01-01T02:00Z"
+              },
+              "retention":{
+                "limit":"minutes(15)",
+                "action":"delete"
+              }
+            },
+            {
+              "name":"drtarget",
+              "type":"target",
+              "validity":{
+                "start":"2010-01-01T00:00Z",
+                "end":"2015-01-01T02:00Z"
+              },
+              "retention":{
+                "limit":"minutes(17)",
+                "action":"delete"
+              }
+            }
+          ]
+        },
+        "locations":{
+          "location":{
+            "type":"data",
+            "path":"/mirrorthis"
+          }
+        },
+        "ACL":{
+          "owner":"testuser",
+          "group":"group",
+          "permission":"0x755"
+        },
+        "schema":{
+          "location":"/schema/clicks",
+          "provider":"protobuf"
+        },
+        "properties":{
+          "property":[
+            {
+              "name":"field3",
+              "value":"value3"
+            },
+            {
+              "name":"field2",
+              "value":"value2"
+            },
+            {
+              "name":"field4",
+              "value":"value2"
+            }
+          ]
+        }
+      },
+      "instances":[
+        {
+          "href":"http://ec2...com:8080/api/v1/feeds/feed1/instances/99",
+          "Instances":{
+            "id":"99",
+            "cluster":"drtarget1",
+            "sourceCluster":"drsource",
+            "status":"RUNNING",
+            "start":"2013-03-06T03:17Z",
+            "end":"2013-03-06T03:19Z",
+            "details":"",
+            "log":""
+          }
+        },
+        {
+          "href":"http://ec2...com:8080/api/v1/feeds/feed1/instances/95",
+          "Instances":{
+            "id":"95",
+            "cluster":"drtarget1",
+            "sourceCluster":"drsource",
+            "status":"RUNNING",
+            "start":"2013-03-06T03:17Z",
+            "end":"2013-03-06T03:19Z",
+            "details":"",
+            "log":""
+          }
+        }
+      ]
+    },
+    {
+      "href":"http://ec2...com:8080/api/v1/feeds/feed1",
+      "Feeds":{
+        "description":"feed 3",
+        "name":"feed 3",
+        "status":"SCHEDULED",
+        "groups":"online,bi",
+        "frequency":"minutes(1)",
+        "timezone":"UTC",
+        "late-arrival":{
+          "cut-off":"minutes(10)"
+        },
+        "clusters":{
+          "cluster":[
+            {
+              "name":"drsource",
+              "type":"source",
+              "validity":{
+                "start":"2010-01-01T00:00Z",
+                "end":"2015-01-01T02:00Z"
+              },
+              "retention":{
+                "limit":"minutes(15)",
+                "action":"delete"
+              }
+            },
+            {
+              "name":"drtarget",
+              "type":"target",
+              "validity":{
+                "start":"2010-01-01T00:00Z",
+                "end":"2015-01-01T02:00Z"
+              },
+              "retention":{
+                "limit":"minutes(17)",
+                "action":"delete"
+              }
+            }
+          ]
+        },
+        "locations":{
+          "location":{
+            "type":"data",
+            "path":"/mirrorthis"
+          }
+        },
+        "ACL":{
+          "owner":"testuser",
+          "group":"group",
+          "permission":"0x755"
+        },
+        "schema":{
+          "location":"/schema/clicks",
+          "provider":"protobuf"
+        },
+        "properties":{
+          "property":[
+            {
+              "name":"field3",
+              "value":"value3"
+            },
+            {
+              "name":"field2",
+              "value":"value2"
+            },
+            {
+              "name":"field4",
+              "value":"value2"
+            }
+          ]
+        }
+      },
+      "instances":[
+        {
+          "href":"http://ec2...com:8080/api/v1/feeds/feed1/instances/99",
+          "Instances":{
+            "id":"99",
+            "cluster":"drtarget1",
+            "sourceCluster":"drsource",
+            "status":"RUNNING",
+            "start":"2013-03-06T03:17Z",
+            "end":"2013-03-06T03:19Z",
+            "details":"",
+            "log":""
+          }
+        },
+        {
+          "href":"http://ec2...com:8080/api/v1/feeds/feed1/instances/95",
+          "Instances":{
+            "id":"95",
+            "cluster":"drtarget1",
+            "sourceCluster":"drsource",
+            "status":"RUNNING",
+            "start":"2013-03-06T03:17Z",
+            "end":"2013-03-06T03:19Z",
+            "details":"",
+            "log":""
+          }
+        }
+      ]
+    }
+
+  ]
+}

ambari-web/app/assets/data/mirroring/each_dataset.json

@@ -0,0 +1,101 @@
+{
+  "href":"http://ec2...com:8080/api/v1/feeds/feed1",
+  "Feeds":{
+    "description":"test feed 4",
+    "name":"test4",
+    "status":"SCHEDULED",
+    "groups":"online,bi",
+    "frequency":"minutes(1)",
+    "timezone":"UTC",
+    "late-arrival":{
+      "cut-off":"minutes(10)"
+    },
+    "clusters":{
+      "cluster":[
+        {
+          "name":"drsource",
+          "type":"source",
+          "validity":{
+            "start":"2010-01-01T00:00Z",
+            "end":"2015-01-01T02:00Z"
+          },
+          "retention":{
+            "limit":"minutes(15)",
+            "action":"delete"
+          }
+        },
+        {
+          "name":"drtarget",
+          "type":"target",
+          "validity":{
+            "start":"2010-01-01T00:00Z",
+            "end":"2015-01-01T02:00Z"
+          },
+          "retention":{
+            "limit":"minutes(17)",
+            "action":"delete"
+          }
+        }
+      ]
+    },
+    "locations":{
+      "location":{
+        "type":"data",
+        "path":"/mirrorthis"
+      }
+    },
+    "ACL":{
+      "owner":"testuser",
+      "group":"group",
+      "permission":"0x755"
+    },
+    "schema":{
+      "location":"/schema/clicks",
+      "provider":"protobuf"
+    },
+    "properties":{
+      "property":[
+        {
+          "name":"field3",
+          "value":"value3"
+        },
+        {
+          "name":"field2",
+          "value":"value2"
+        },
+        {
+          "name":"field4",
+          "value":"value2"
+        }
+      ]
+    }
+  },
+  "instances":[
+    {
+      "href":"http://ec2...com:8080/api/v1/feeds/feed1/instances/99",
+      "Instances":{
+        "id":"99",
+        "cluster":"drtarget1",
+        "sourceCluster":"drsource",
+        "status":"RUNNING",
+        "start":"2013-03-06T03:17Z",
+        "end":"2013-03-06T03:19Z",
+        "details":"",
+        "log":""
+      }
+    },
+    {
+      "href":"http://ec2...com:8080/api/v1/feeds/feed1/instances/95",
+      "Instances":{
+        "id":"95",
+        "cluster":"drtarget1",
+        "sourceCluster":"drsource",
+        "status":"RUNNING",
+        "start":"2013-03-06T03:17Z",
+        "end":"2013-03-06T03:19Z",
+        "details":"",
+        "log":""
+      }
+    }
+  ]
+}

ambari-web/app/assets/data/mirroring/each_dataset_job.json

@@ -0,0 +1,13 @@
+{
+  "href":"http://ec2...com:8080/api/v1/feeds/feed1/instances/99",
+  "Instances":{
+    "id":"99",
+    "cluster":"drtarget1",
+    "sourceCluster":"drsource",
+    "status":"RUNNING",
+    "start":"2013-03-06T03:17Z",
+    "end":"2013-03-06T03:19Z",
+    "details":"",
+    "log":""
+  }
+}

ambari-web/app/assets/data/mirroring/past_jobs.json

@@ -0,0 +1,20 @@
+{
+  "datasetJobs" : [
+    {
+      "dataset_id" : "dataset1",
+      "job_id" : "1",
+      "start_date" : 0,
+      "end_date" : 0,
+      "duration" : 1,
+      "data" : 100
+    },
+    {
+      "dataset_id" : "dataset2",
+      "job_id" : "1",
+      "start_date" : 0,
+      "end_date" : 0,
+      "duration" : 1,
+      "data" : 100
+    }
+  ]
+}

ambari-web/app/assets/data/mirroring/upcoming_jobs.json

@@ -0,0 +1,21 @@
+{
+  "datasetJobs" : [
+    {
+      "dataset_id" : "dataset1",
+      "job_id" : "1",
+      "start_date" : 0,
+      "end_date" : 0,
+      "duration" : 1,
+      "data" : 100
+    },
+    {
+      "dataset_id" : "dataset2",
+      "job_id" : "1",
+      "start_date" : 0,
+      "end_date" : 0,
+      "duration" : 1,
+      "data" : 100
+    }
+  ]
+}
+w

ambari-web/app/controllers/global/cluster_controller.js

@@ -42,7 +42,7 @@ App.ClusterController = Em.Controller.extend({
       }
     }
     this.set('isLoaded', loaded);
-    this.set('clusterDataLoadedPercent', 'width:' + (Math.floor(numLoaded/6*100)).toString() + '%');
+    this.set('clusterDataLoadedPercent', 'width:' + (Math.floor(numLoaded/7*100)).toString() + '%');
   },
 
   dataLoadList:Em.Object.create({
@@ -51,7 +51,8 @@ App.ClusterController = Em.Controller.extend({
     'cluster':false,
     'racks':false,
     'alerts':false,
-    'users':false
+    'users':false,
+    'datasets':false
   }),
 
   /**
@@ -287,6 +288,17 @@ App.ClusterController = Em.Controller.extend({
     var hostsUrl = this.getUrl('/data/hosts/hosts.json', '/hosts?fields=Hosts/host_name,Hosts/public_host_name,Hosts/disk_info,Hosts/cpu_count,Hosts/total_mem,Hosts/host_status,Hosts/last_heartbeat_time,Hosts/os_arch,Hosts/os_type,Hosts/ip,host_components,metrics/disk,metrics/load/load_one');
     var usersUrl = App.testMode ? '/data/users/users.json' : App.apiPrefix + '/users/?fields=*';
     var racksUrl = "/data/racks/racks.json";
+    var dataSetUrl = "/data/mirroring/all_datasets.json";
+
+    debugger;
+
+    App.HttpClient.get(dataSetUrl, App.dataSetMapper, {
+      complete:function (jqXHR, textStatus) {
+        self.updateLoadStatus('datasets');
+      }
+    }, function (jqXHR, textStatus) {
+      self.updateLoadStatus('datasets');
+    });
 
     App.HttpClient.get(racksUrl, App.racksMapper, {
       complete:function (jqXHR, textStatus) {

ambari-web/app/initialize.js

@@ -44,7 +44,7 @@ require('mappers/racks_mapper');
 require('mappers/alerts_mapper');
 require('mappers/users_mapper');
 require('mappers/service_mapper');
-
+require('mappers/dataset_mapper');
 require('utils/http_client');
 require('utils/host_progress_popup');
 

ambari-web/app/mappers/dataset_mapper.js

@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+App.dataSetMapper = App.QuickDataMapper.create({
+  model : App.DataSet,
+  Jobs_model : App.DataSetJob,
+  config : {
+    name: 'Feeds.name', // from json
+    source_cluster_name: 'Feeds.clusters.cluster[0].name', // approach1 : from json
+    target_cluster_name: 'target_cluster_name', // approach 2 : to be calculated (TBC1)
+    source_dir: 'Feeds.locations.location.path',
+    schedule: 'Feeds.frequency',
+    dataset_jobs: 'dataset_jobs', // TBC2 ( set of ids will be added )
+
+    // all below are unknown at present and may be blank
+    last_failed_date: 'last_failed_date',
+    avg_data: 'avg_data',
+    data_creation_date: 'data_creation_date',
+    target_dir: 'target_dir'
+  },
+  jobs_config : {
+    // $dataset_id: 'none', // will be loaded outside parser
+    id: 'Instances.id',
+    start_date: 'start_date_str',
+    end_date: 'end_date_str',
+    duration: 'duration'
+    //data: 'Instances.details'
+  },
+
+  map:function(json){
+    debugger;
+    if(!this.get('model')) {return;}
+    if(json && json.items && json.items.length > 0){
+      var dataset_results = [];
+      json.items.forEach(function(item){
+
+        try{
+          // TBC1
+          item.target_cluster_name = (item.Feeds.clusters.cluster.findProperty("type","target")).name;
+
+          // TBC2
+          item.dataset_jobs = [];
+
+          item.instances.forEach(function(job){
+            item.dataset_jobs.push(job.Instances.id);
+          });
+
+          item.last_failed_date ='';
+          item.avg_data ='';
+          item.data_creation_date ='';
+          item.target_dir ='';
+
+          debugger;
+          var newitem = this.parseIt(item, this.config);
+          dataset_results.push(newitem);
+        }catch(ex){
+          console.debug('Exception occured : ' + ex);
+        }
+      },this);
+      console.debug('Before load: App.DataSet.find().content : ' + App.DataSet.find().content );
+      App.store.loadMany(this.get('model'), dataset_results);
+      console.debug('After load: App.DataSet.find().content : ' + App.DataSet.find().content );
+
+      try{
+        // Child records
+        var dataset_job_results = [];
+        json.items.forEach(function(item){
+          item.instances.forEach(function(instance){
+            instance.Instances.start = new Date(instance.Instances.start); // neeed to be calulated end -start
+            instance.Instances.end = new Date(instance.Instances.end); // neeed to be calulated end -start
+
+            var milliseconds =  instance.Instances.end - instance.Instances.start;
+            var date = new Date(milliseconds);
+            var h = Math.floor(milliseconds / 3600000);
+            var m = Math.floor((milliseconds % 3600000) / 60000);
+            var s = Math.floor(((milliseconds % 360000) % 60000) / 1000);
+            instance.duration = (h==0?'':h+'hr ') + (m==0?'':m+'mins ') + (s==0?'':s+'secs ');
+            instance.start_date_str = instance.Instances.start.toString();
+            instance.end_date_str = instance.Instances.end.toString();
+
+            var result = this.parseIt(instance, this.jobs_config);
+            dataset_job_results.push(result);
+          }, this)
+        }, this);
+
+        console.debug('Before load: App.DataSetJob.find().content : ' + App.DataSetJob.find().content );
+        App.store.loadMany(this.get('Jobs_model'), dataset_job_results);
+        console.debug('After load: App.DataSetJob.find().content : ' + App.DataSetJob.find().content );
+      }catch(ex){
+        console.debug('Exception occured : ' + ex);
+      }
+    }
+  }
+
+});

ambari-web/app/models.js

@@ -41,7 +41,8 @@ require('models/run');
 require('models/app');
 require('models/background_operation');
 require('models/host_component');
-
+require('models/dataset');
+require('models/dataset_job');
 require('classes/run_class');
 require('classes/job_class');
 require('classes/job_class');

ambari-web/app/models/dataset.js

@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+var App = require('app');
+
+App.DataSet = DS.Model.extend({
+  name: DS.attr('string'),
+  sourceClusterName: DS.attr('string'),
+  targetClusterName: DS.attr('string'),
+  sourceDir : DS.attr('string'),
+  targetDir : DS.attr('string'),
+  schedule: DS.attr('string'),
+  lastFailedDate : DS.attr('date'),
+  avgData : DS.attr('number'),
+  dataCreationDate : DS.attr('string'),
+  datasetJobs: DS.hasMany('App.DataSetJob')
+
+});
+
+
+
+App.DataSet.FIXTURES = [/*
+  {
+    id: 1,
+    cluster_name: 'cluster1',
+    stack_name: 'HDP',
+    hosts: [1, 2, 3, 4],
+    racks: [1, 2, 3, 4, 5, 6],
+    max_hosts_per_rack: 10
+  }*/
+];

ambari-web/app/models/dataset_job.js

@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+var App = require('app');
+
+App.DataSetJob = DS.Model.extend({
+ // dataSet: DS.belongsTo('App.DataSet'),
+  startDate: DS.attr('string'),
+  endDate: DS.attr('string'),
+  duration : DS.attr('string')
+  //data : DS.attr('string')
+});
+
+
+
+App.DataSetJob.FIXTURES = [/*
+  {
+    id: 1,
+    cluster_name: 'cluster1',
+    stack_name: 'HDP',
+    hosts: [1, 2, 3, 4],
+    racks: [1, 2, 3, 4, 5, 6],
+    max_hosts_per_rack: 10
+  }*/
+];