YARN-7781. Update YARN service documentation.

           Contributed by Gour Saha

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md

@@ -15,7 +15,7 @@
 ## Examples
 
 ### Create a simple single-component service with most attribute values as defaults
-POST URL - http://localhost:9191/ws/v1/services
+POST URL - http://localhost:8088/app/v1/services
 
 ##### POST Request JSON
 ```json
@@ -27,7 +27,7 @@ POST URL - http://localhost:9191/ws/v1/services
     [
       {
         "name": "hello",
-        "number_of_containers": 1,
+        "number_of_containers": 2,
         "artifact": {
           "id": "nginx:latest",
           "type": "DOCKER"
@@ -36,14 +36,14 @@ POST URL - http://localhost:9191/ws/v1/services
         "resource": {
           "cpus": 1,
           "memory": "256"
-       }
+        }
       }
     ]
 }
 ```
 
 ##### GET Response JSON
-GET URL - http://localhost:9191/ws/v1/services/hello-world
+GET URL - http://localhost:8088/app/v1/services/hello-world
 
 Note, lifetime value of -1 means unlimited lifetime.
 
@@ -54,10 +54,11 @@ Note, lifetime value of -1 means unlimited lifetime.
     "description": "hello world example",
     "id": "application_1503963985568_0002",
     "lifetime": -1,
+    "state": "STABLE",
     "components": [
         {
             "name": "hello",
-            "dependencies": [],
+            "state": "STABLE",
             "resource": {
                 "cpus": 1,
                 "memory": "256"
@@ -70,21 +71,21 @@ Note, lifetime value of -1 means unlimited lifetime.
             "quicklinks": [],
             "containers": [
                 {
-                    "id": "container_e03_1503963985568_0002_01_000001",
+                    "id": "container_e03_1503963985568_0002_01_000002",
                     "ip": "10.22.8.143",
-                    "hostname": "myhost.local",
+                    "hostname": "ctr-e03-1503963985568-0002-01-000002.example.site",
                     "state": "READY",
                     "launch_time": 1504051512412,
-                    "bare_host": "10.22.8.143",
+                    "bare_host": "host100.cloud.com",
                     "component_instance_name": "hello-0"
                 },
                 {
-                    "id": "container_e03_1503963985568_0002_01_000002",
-                    "ip": "10.22.8.143",
-                    "hostname": "myhost.local",
+                    "id": "container_e03_1503963985568_0002_01_000003",
+                    "ip": "10.22.8.144",
+                    "hostname": "ctr-e03-1503963985568-0002-01-000003.example.site",
                     "state": "READY",
                     "launch_time": 1504051536450,
-                    "bare_host": "10.22.8.143",
+                    "bare_host": "host100.cloud.com",
                     "component_instance_name": "hello-1"
                 }
             ],
@@ -103,7 +104,7 @@ Note, lifetime value of -1 means unlimited lifetime.
 
 ```
 ### Update to modify the lifetime of a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world
+PUT URL - http://localhost:8088/app/v1/services/hello-world
 
 ##### PUT Request JSON
 
@@ -115,43 +116,59 @@ Note, irrespective of what the current lifetime value is, this update request wi
 }
 ```
 ### Stop a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world
+PUT URL - http://localhost:8088/app/v1/services/hello-world
 
 ##### PUT Request JSON
 ```json
 {
-    "state": "STOPPED"
+  "state": "STOPPED"
 }
 ```
 
 ### Start a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world
+PUT URL - http://localhost:8088/app/v1/services/hello-world
 
 ##### PUT Request JSON
 ```json
 {
-    "state": "STARTED"
+  "state": "STARTED"
 }
 ```
 
-### Update to flex up/down the no of containers (instances) of a component of a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello
+### Update to flex up/down the number of containers (instances) of a component of a service
+PUT URL - http://localhost:8088/app/v1/services/hello-world/components/hello
 
 ##### PUT Request JSON
 ```json
 {
-    "name": "hello",
-    "number_of_containers": 3
+  "number_of_containers": 3
+}
+```
+
+Alternatively, you can specify the entire "components" section instead.
+
+PUT URL - http://localhost:8088/app/v1/services/hello-world
+##### PUT Request JSON
+```json
+{
+  "state": "FLEX",
+  "components" :
+    [
+      {
+        "name": "hello",
+        "number_of_containers": 3
+      }
+    ]
 }
 ```
 
 ### Destroy a service
-DELETE URL - http://localhost:9191/ws/v1/services/hello-world
+DELETE URL - http://localhost:8088/app/v1/services/hello-world
 
 ***
 
 ### Create a complicated service  - HBase
-POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
+POST URL - http://localhost:8088:/app/v1/services/hbase-app-1
 
 ##### POST Request JSON
 
@@ -249,3 +266,176 @@ POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
   }
 }
 ```
+
+### Create a service requesting GPUs in addition to CPUs and RAM
+POST URL - http://localhost:8088/app/v1/services
+
+##### POST Request JSON
+```json
+{
+  "name": "hello-world",
+  "version": "1.0.0",
+  "description": "hello world example with GPUs",
+  "components" :
+    [
+      {
+        "name": "hello",
+        "number_of_containers": 2,
+        "artifact": {
+          "id": "nginx:latest",
+          "type": "DOCKER"
+        },
+        "launch_command": "./start_nginx.sh",
+        "resource": {
+          "cpus": 1,
+          "memory": "256",
+          "additional" : {
+            "yarn.io/gpu" : {
+              "value" : 4,
+              "unit" : ""
+            }
+          }
+        }
+      }
+    ]
+}
+```
+
+### Create a service with a component requesting anti-affinity placement policy
+POST URL - http://localhost:8088/app/v1/services
+
+##### POST Request JSON
+```json
+{
+  "name": "hello-world",
+  "version": "1.0.0",
+  "description": "hello world example with anti-affinity",
+  "components" :
+    [
+      {
+        "name": "hello",
+        "number_of_containers": 3,
+        "artifact": {
+          "id": "nginx:latest",
+          "type": "DOCKER"
+        },
+        "launch_command": "./start_nginx.sh",
+        "resource": {
+          "cpus": 1,
+          "memory": "256"
+        },
+        "placement_policy": {
+          "constraints": [
+            {
+              "type": "ANTI_AFFINITY",
+              "scope": "NODE",
+              "node_attributes": {
+                "os": ["linux", "windows"],
+                "fault_domain": ["fd1", "fd2"]
+              },
+              "node_partitions": [
+                "gpu",
+                "fast-disk"
+              ],
+              "target_tags": [
+                "hello"
+              ]
+            }
+          ]
+        }
+      }
+    ]
+}
+```
+
+##### GET Response JSON
+GET URL - http://localhost:8088/app/v1/services/hello-world
+
+Note, that the 3 containers will come up on 3 different nodes. If there are less
+than 3 NMs running in the cluster, then all 3 container requests will not be
+fulfilled and the service will be in non-STABLE state.
+
+```json
+{
+    "name": "hello-world",
+    "version": "1.0.0",
+    "description": "hello world example with anti-affinity",
+    "id": "application_1503963985568_0003",
+    "lifetime": -1,
+    "state": "STABLE",
+    "components": [
+        {
+            "name": "hello",
+            "state": "STABLE",
+            "resource": {
+                "cpus": 1,
+                "memory": "256"
+            },
+            "placement_policy": {
+              "constraints": [
+                {
+                  "type": "ANTI_AFFINITY",
+                  "scope": "NODE",
+                  "node_attributes": {
+                    "os": ["linux", "windows"],
+                    "fault_domain": ["fd1", "fd2"]
+                  },
+                  "node_partitions": [
+                    "gpu",
+                    "fast-disk"
+                  ],
+                  "target_tags": [
+                    "hello"
+                  ]
+                }
+              ]
+            },
+            "configuration": {
+                "properties": {},
+                "env": {},
+                "files": []
+            },
+            "quicklinks": [],
+            "containers": [
+                {
+                    "id": "container_e03_1503963985568_0003_01_000002",
+                    "ip": "10.22.8.143",
+                    "hostname": "ctr-e03-1503963985568-0003-01-000002.example.site",
+                    "state": "READY",
+                    "launch_time": 1504051512412,
+                    "bare_host": "host100.cloud.com",
+                    "component_instance_name": "hello-0"
+                },
+                {
+                    "id": "container_e03_1503963985568_0003_01_000003",
+                    "ip": "10.22.8.144",
+                    "hostname": "ctr-e03-1503963985568-0003-01-000003.example.site",
+                    "state": "READY",
+                    "launch_time": 1504051536450,
+                    "bare_host": "host101.cloud.com",
+                    "component_instance_name": "hello-1"
+                },
+                {
+                    "id": "container_e03_1503963985568_0003_01_000004",
+                    "ip": "10.22.8.145",
+                    "hostname": "ctr-e03-1503963985568-0003-01-000004.example.site",
+                    "state": "READY",
+                    "launch_time": 1504051536450,
+                    "bare_host": "host102.cloud.com",
+                    "component_instance_name": "hello-2"
+                }
+            ],
+            "launch_command": "./start_nginx.sh",
+            "number_of_containers": 1,
+            "run_privileged_container": false
+        }
+    ],
+    "configuration": {
+        "properties": {},
+        "env": {},
+        "files": []
+    },
+    "quicklinks": {}
+}
+```
+

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml

@@ -36,8 +36,8 @@ info:
     name: Apache 2.0
     url: http://www.apache.org/licenses/LICENSE-2.0.html
 # the domain of the service
-host: host.mycompany.com
-port: 9191(default)
+host: localhost
+port: 8088(default)
 # array of all schemes that your API supports
 schemes:
   - http
@@ -249,7 +249,7 @@ definitions:
         type: string
         description: The YARN queue that this service should be submitted to.
       kerberos_principal:
-        description: The Kerberos Principal of the service
+        description: The principal info of the user who launches the service.
         $ref: '#/definitions/KerberosPrincipal'
       docker_client_config:
         type: string
@@ -283,7 +283,7 @@ definitions:
         type: object
         additionalProperties:
           $ref: '#/definitions/ResourceInformation'
-        description: Map of resource name to ResourceInformation
+        description: A map of resource type name to resource type information. Including value (integer), and unit (string). This will be used to specify resource other than cpu and memory. Please refer to example below.
   PlacementPolicy:
     description: Advanced placement policy of the components of a service.
     required:
@@ -535,6 +535,7 @@ definitions:
           - STOPPED
           - FAILED
           - FLEX
+          - UPGRADING
   ContainerState:
     description: The current state of the container of a service.
     properties:
@@ -552,8 +553,10 @@ definitions:
         type: string
         description: enum of the state of the component
         enum:
+          - INIT
           - FLEXING
           - STABLE
+          - UPGRADING
   ServiceStatus:
     description: The current status of a submitted service, returned as a response to the GET API.
     properties:
@@ -572,10 +575,10 @@ definitions:
     properties:
       principal_name:
         type: string
-        description: The principal name of the user who launches the service.
+        description: The principal name of the user who launches the service. Note that `_HOST` is required in the `principal_name` field such as `testuser/_HOST@EXAMPLE.COM` because Hadoop client validates that the server's (in this case, the AM's) principal has hostname present when communicating to the server.
       keytab:
         type: string
-        description: |
-          The URI of the kerberos keytab. It supports two modes:
-          URI starts with "hdfs://": A path on hdfs where the keytab is stored. The keytab will be localized by YARN to each host.
-          URI starts with "file://": A path on the local host where the keytab is stored. It is assumed that the keytabs are pre-installed by admins before AM launches.
+        description: The URI of the kerberos keytab. Currently supports only files present on the bare host. URI starts with "file\://" - A path on the local host where the keytab is stored. It is assumed that admin pre-installs the keytabs on the local host before AM launches.
+
+
+

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java

@@ -30,7 +30,7 @@ public class BaseResource implements Serializable {
 
   /**
    * Resource location for a service, e.g.
-   * /ws/v1/services/helloworld
+   * /app/v1/services/helloworld
    *
    **/
   public String getUri() {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md

@@ -207,7 +207,7 @@ If you are building from source code, make sure you use `-Pyarn-ui` in the `mvn`
 ```
 
 # Run with security
-YARN service framework supports running in a secure(kerberized) environment. User needs to specify the kerberos principal name and keytab when they launch the service.
+YARN service framework supports running in a secure (kerberized) environment. User needs to specify the kerberos principal name and keytab when they launch the service.
 E.g. A typical configuration looks like below:
 ```
 {
@@ -215,15 +215,16 @@ E.g. A typical configuration looks like below:
   ...
   ...
   "kerberos_principal" : {
-    "principal_name" : "hdfs-demo@EXAMPLE.COM",
-    "keytab" : "hdfs:///etc/security/keytabs/hdfs.headless.keytab"
+    "principal_name" : "hdfs-demo/_HOST@EXAMPLE.COM",
+    "keytab" : "file:///etc/security/keytabs/hdfs.headless.keytab"
   }
 }
 ```
+Note that `_HOST` is required in the `principal_name` field because Hadoop client validates that the server's (in this case, the AM's) principal has hostname present when communicating to the server.
 * principal_name : the principal name of the user who launches the service
-* keytab : URI of the keytab. It supports two modes:
-    * URI starts with `hdfs://`: The URI where the keytab is stored on hdfs. The keytab will be localized to each node by YARN.
-    * URI starts with `file://`: The URI where the keytab is stored on local host. It is assumed that admin pre-installs the keytabs on the local host before AM launches.
+* keytab : URI of the keytab. Currently supports only files present on the bare host.
+    * URI starts with `file://` - A path on the local host where the keytab is stored. It is assumed that admin pre-installs the keytabs on the local host before AM launches.
+
 # Run with Docker
 The above example is only for a non-docker container based service. YARN Service Framework also provides first-class support for managing docker based services.
 Most of the steps for managing docker based services are the same except that in docker the `Artifact` type for a component is `DOCKER` and the Artifact `id` is the name of the docker image.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md

@@ -241,7 +241,7 @@ The state of the component
 
 |Name|Description|Required|Schema|Default|
 |----|----|----|----|----|
-|state|enum of the state of the component|false|enum (FLEXING, STABLE)||
+|state|enum of the state of the component|false|enum (INIT, FLEXING, STABLE, UPGRADING)||
 
 
 ### ConfigFile
@@ -262,7 +262,7 @@ Set of configuration properties that can be injected into the service components
 
 |Name|Description|Required|Schema|Default|
 |----|----|----|----|----|
-|properties|A blob of key-value pairs for configuring YARN service AM.|false|object||
+|properties|A blob of key-value pairs for configuring the YARN service AM.|false|object||
 |env|A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.|false|object||
 |files|Array of list of files that needs to be created and made available as volumes in the service component containers.|false|ConfigFile array||
 
@@ -300,8 +300,8 @@ The kerberos principal info of the user who launches the service.
 
 |Name|Description|Required|Schema|Default|
 |----|----|----|----|----|
-|principal_name|The principal name of the user who launches the service.|false|string||
-|keytab|The URI of the kerberos keytab. It supports two modes, URI starts with "hdfs://": A path on hdfs where the keytab is stored. The keytab will be localized by YARN to each host; URI starts with "file://": A path on the local host where the keytab is stored. It is assumed that the keytabs are pre-installed by admins before AM launches.|false|string||
+|principal_name|The principal name of the user who launches the service. Note that `_HOST` is required in the `principal_name` field such as `testuser/_HOST@EXAMPLE.COM` because Hadoop client validates that the server's (in this case, the AM's) principal has hostname present when communicating to the server.|false|string||
+|keytab|The URI of the kerberos keytab. Currently supports only files present on the bare host. URI starts with "file://" - A path on the local host where the keytab is stored. It is assumed that admin pre-installs the keytabs on the local host before AM launches.|false|string||
 
 
 ### PlacementConstraint
@@ -369,7 +369,7 @@ Resource determines the amount of resources (vcores, memory, network, etc.) usab
 |profile|Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.|false|string||
 |cpus|Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).|false|integer (int32)||
 |memory|Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.|false|string||
-|additional|A map of resource type name to resource type information. Including value (integer), and unit (string). This will be used to specify resource other than cpu and memory. Please refer to example below. |  false | object ||
+|additional|A map of resource type name to resource type information. Including value (integer), and unit (string). This will be used to specify resource other than cpu and memory. Please refer to example below.|false|object||
 
 
 ### ResourceInformation
@@ -402,16 +402,17 @@ a service resource has the following attributes.
 |state|State of the service. Specifying a value for this attribute for the PUT payload means update the service to this desired state.|false|ServiceState||
 |quicklinks|A blob of key-value pairs of quicklinks to be exported for a service.|false|object||
 |queue|The YARN queue that this service should be submitted to.|false|string||
-|kerberos_principal | The principal info of the user who launches the service|false|KerberosPrincipal||
+|kerberos_principal|The principal info of the user who launches the service|false|KerberosPrincipal||
 |docker_client_config|URI of the file containing the docker client configuration (e.g. hdfs:///tmp/config.json)|false|string||
 
+
 ### ServiceState
 
 The current state of a service.
 
 |Name|Description|Required|Schema|Default|
 |----|----|----|----|----|
-|state|enum of the state of the service|false|enum (ACCEPTED, STARTED, STABLE, STOPPED, FAILED, FLEX)||
+|state|enum of the state of the service|false|enum (ACCEPTED, STARTED, STABLE, STOPPED, FAILED, FLEX, UPGRADING)||
 
 
 ### ServiceStatus
@@ -472,7 +473,6 @@ Note, lifetime value of -1 means unlimited lifetime.
     "components": [
         {
             "name": "hello",
-            "dependencies": [],
             "state": "STABLE",
             "resource": {
                 "cpus": 1,
@@ -536,7 +536,7 @@ PUT URL - http://localhost:8088/app/v1/services/hello-world
 ##### PUT Request JSON
 ```json
 {
-    "state": "STOPPED"
+  "state": "STOPPED"
 }
 ```
 
@@ -546,17 +546,34 @@ PUT URL - http://localhost:8088/app/v1/services/hello-world
 ##### PUT Request JSON
 ```json
 {
-    "state": "STARTED"
+  "state": "STARTED"
 }
 ```
 
-### Update to flex up/down the no of containers (instances) of a component of a service
+### Update to flex up/down the number of containers (instances) of a component of a service
 PUT URL - http://localhost:8088/app/v1/services/hello-world/components/hello
 
 ##### PUT Request JSON
 ```json
 {
-    "number_of_containers": 3
+  "number_of_containers": 3
+}
+```
+
+Alternatively, you can specify the entire "components" section instead.
+
+PUT URL - http://localhost:8088/app/v1/services/hello-world
+##### PUT Request JSON
+```json
+{
+  "state": "FLEX",
+  "components" :
+    [
+      {
+        "name": "hello",
+        "number_of_containers": 3
+      }
+    ]
 }
 ```
 
@@ -727,6 +744,14 @@ POST URL - http://localhost:8088/app/v1/services
             {
               "type": "ANTI_AFFINITY",
               "scope": "NODE",
+              "node_attributes": {
+                "os": ["centos6", "centos7"],
+                "fault_domain": ["fd1", "fd2"]
+              },
+              "node_partitions": [
+                "gpu",
+                "fast-disk"
+              ],
               "target_tags": [
                 "hello"
               ]
@@ -756,7 +781,6 @@ fulfilled and the service will be in non-STABLE state.
     "components": [
         {
             "name": "hello",
-            "dependencies": [],
             "state": "STABLE",
             "resource": {
                 "cpus": 1,
@@ -767,8 +791,14 @@ fulfilled and the service will be in non-STABLE state.
                 {
                   "type": "ANTI_AFFINITY",
                   "scope": "NODE",
-                  "node_attributes": {},
-                  "node_partitions": [],
+                  "node_attributes": {
+                    "os": ["centos6", "centos7"],
+                    "fault_domain": ["fd1", "fd2"]
+                  },
+                  "node_partitions": [
+                    "gpu",
+                    "fast-disk"
+                  ],
                   "target_tags": [
                     "hello"
                   ]
@@ -858,3 +888,4 @@ POST URL - http://localhost:8088/app/v1/services
     ]
 }
 ```
+