Updated on 2022-07-11 GMT+08:00

REST APIs of Superior Scheduler

Function Description

The REST/HTTP server is part of Superior Scheduler on YARN Resource Manager host and leverage existing YARN resource manager web service port. In the section below, we will denote this YARN address:port as SS_REST_SERVER.

Below uses HTTP as part of URL and only HTTP will be supported.

Superior Scheduler Interfaces

  • Query Application
    • Query *all* application within scheduler engine.
      • URL

        GET http://<SS_REST_SERVER>/ws/v1/sscheduler/applications/list

      • Input

        None.

      • Output
        JSON Response:
        
        {
         "applicationlist": [
          {
           "id": "1020201_0123_12",
           "queue": "root.Q1.Q11",
           "user": "cchen",
           "resource_request": {
               "vcores" : 10,
               "memory" : 100
          },
          "resource_inuse": {
            "vcores" : 100,
            "memory" : 2000
          },
          "number_running_container": 100,
          "number_pending_request": 10
          },
          {
          "id": "1020201_0123_15",
          "queue": "root.Q2.Q21",
          "user": "Jason",
          "resource_request": {
            "vcores" : 4,
            "memory" : 100
          },
          "resource_inuse": {
            "vcores" : 20,
            "memory" : 200
          },
          "resource_reserved": {
            "vcores" : 10, 
            "memory" : 100
          },
          "number_running_container": 20,
          "number_pending_container": 4,
          "number_reserved_container":2
          }
         ]
        }
      Table 1 Parameters of all application

      Attribute

      Type

      Description

      applicationlist

      array

      Array of application IDs.

      queue

      String

      Name of queue application.

      user

      String

      Name of user who submits application.

      resource_request

      object

      Currently requested resource, including vcores, memory, and so on.

      resource_inuse

      object

      Currently resource in use, including vcores, memory, and so on.

      resource_reserved

      object

      Currently reserved resource,

      including vcores, memory, and so on.

      number_running_container

      int

      Total number of running containers. This maps to superior engine decisions.

      number_pending_request

      int

      Total number of pending requests, this is sum of all counts in all demands of allocation.

      number_reserved_container

      int

      Total number of reserved containers, this maps to superior engine decisions.

      id

      String

      Application ID.

    • Query *single* application within scheduler engine.
      • URL

        GET http://<SS_REST_SERVER>/ws/v1/sscheduler/applications/{application_id}

      • Input

        None.

      • Output
        JSON Response:
        
        {
         "applicationlist": [
          {
           "id": "1020201_0123_12",
           "queue": "root.Q1.Q11",
           "user": "cchen",
           "resource_request": {
               "vcores" : 3,
               "memory" : 3072
          },
          "resource_inuse": {
            "vcores" : 100,
            "memory" : 2048
          },
          "number_running_container": 2,
          "number_pending_request": 3,
          "number_reserved_container":1
          "master_container_id": 23402_3420842
          "master_container_resource": node1.domain.com
          "blacklist": [
            {
             "resource": "node5"
            },
            {
             "resource": "node8"
            }
          ],
          "demand": [
            {
             "priority": 1,
             "ismaster": true,
             "capability": {
             "vcores": 2,
              "memory": 2048
            },
            "count": 1,
            "relaxlocality": true,
            "locality": [
             {
              "target": "node1",
              "count": 1,
              "strict": false
             }
            ],
            "resourceselect": "label1",
            "pending_reason": "application limit reached",
            "reserved_resource": {
             "vcores":1,
              "memory":1024
            },
            "reservations":[
            "id": "23402_3420878",
            "resource": "node1.domain.com",
            "reservedAmount": {
             "vcores":1,
             "memory":1024
            } 
           ] 
          },
          {
           "priority": 1,
           "ismaster": false,
           "capability": {
           "vcores": 1,
           "memory": 1024
          },
          "count": 2,
          "relaxlocality": true,
          "locality": [
           {
            "target": "node1",
            "count": 1,
            "strict": false
           },
           {
            "target": "node2",
            "count": 1,
            "strict": false
           },
           {
            "target": "rackA",
            "count": 2,
            "strict": false
           },
          ],
          "resourceselect": "label1",
          "pending_reason": "no available resource"
          }
         ],
         "containers": [
           {
            "id": "23402_3420842",
            "resource": "node1.domain.com",
            "capability": {
             "vcores": 1,
             "memory": 1024
           }
          },
          {
            "id": "23402_3420853",
            "resource": "node2.domain.com",
            "capability": {
             "vcores": 1,
             "memory": 1024
            }
           }
          ]
         }
        }
      • Exceptions

        Application not found.

      Table 2 Parameters of single application

      Attribute

      Type

      Description

      application

      object

      Application object.

      id

      String

      Application ID.

      queue

      String

      Name of queue application.

      user

      String

      Name of user who submits application.

      resource_request

      object

      Currently requested resource, including vcores, memory, and so on.

      resource_inuse

      object

      Currently resource in use, including vcores, memory, and so on.

      resource_reserved

      object

      Currently reserved resource,

      including vcores, memory, and so on.

      number_running_container

      int

      Total number of running containers. This maps to superior engine decisions.

      number_pending_request

      int

      Total number of pending requests, this is sum of all counts in all demands of allocation.

      number_reserved_container

      int

      Total number of reserved containers, this maps to superior engine decisions.

      master_container_id

      String

      The master container ID.

      master_container_resource

      String

      The host name that master container running on.

      demand

      array

      Array of demand objects.

      priority

      int

      Priority of demand.

      ismaster

      boolean

      Is the demand corresponding to "application master".

      capability

      object

      Capability object.

      vcores, memory, ..

      int

      Numeric consumable resource attributes, defining the allocation "unit" for this demand.

      count

      int

      Number of unit required.

      relaxlocality

      boolean

      Locality requirement is preference, and not mandatory if cannot be satisfied.

      locality

      object

      Locality object.

      target

      string

      Locality target's name (that is, node1, rack1..).

      count

      int

      Number of resource "unit" required with this locality requirement.

      strict

      boolean

      If this particular locality is mandatory or not.

      resourceselect

      String

      Resource selection expression for the demand.

      pending_reason

      String

      Reason why this demand is outstanding.

      resource_reserved

      object

      Currently reserved resource for this demand, including vcores, memory, and so on.

      reservations

      array

      Array of reserved container objects.

      reservations:id

      String

      Reserved container ID.

      reservations:resource

      String

      Where the container is allocated.

      reservations:reserveAmount

      object

      The reserved amount of this reservation.

      containers

      array

      Array of allocated container objects.

      containers:id

      String

      Container ID.

      containers:resource

      String

      Where the container is allocated.

      containers:capability

      object

      Capability object.

      containers:vcores,memory…

      int

      Numeric consumable resource attributes allocated to this container.

  • Query Queue
    • Query *all* queues within scheduler engine, including leaf and all middle queues.
      • URL

        GET http://<SS_REST_SERVER>/ws/v1/sscheduler/queues/list

      • Input

        None.

      • Output
        JSON Response:
        
        {
          "queuelist": [
            {
              "name": "root.default",
              "eopen_state": "OPEN",
              "eactive_state": "ACTIVE",
              "open_state": "OPEN",
              "active_state": "ACTIVE",
              "number_pending_application": 2,
              "number_running_application": 10,
              "number_pending_request": 2,
              "number_running_container": 10,
              "number_reserved_container":1,
              "resource_inuse" {
                  "vcores": 10,
                  "memory": 10240
              },
              "resource_request" {
                  "vcores": 2,
                  "memory": 2048
              },
              "resource_reserved" {
                  "vcores": 1
                  "memory": 1024
              }
              
            },
            {
              "name": "root.dev",
              "eopen_state": "OPEN",
              "eactive_state": "INACTIVE",
              "open_state": "OPEN",
              "active_state": "INACTIVE",
              "number_pending_application": 2,
              "number_running_application": 10,
              "number_pending_request": 2,
              "number_running_container": 10,
              "number_reserved_container":0,
              "resource_inuse" {
                  "vcores": 10,
                  "memory": 10240
              },
              "resource_request" {
                  "vcores": 2,
                  "memory": 2048
              },
              "resource_reserved" {
                  "vcores": 0
                  "memory": 0
              }
            },
            {
              "name": "root.qa",
              "eopen_state" : "CLOSED",
              "eactive_state" : "ACTIVE",
              "open_state": "CLOSED",
              "active_state": "ACTIVE",
              "number_pending_application": 2,
              "number_running_application": 10,
              "number_pending_request": 2,
              "number_running_container": 10,
              "number_reserved_container":0,
              "resource_inuse" {
                  "vcores": 10,
                  "memory": 10240
              },
              "resource_request" {
                  "vcores": 2,
                  "memory": 2048
              },
              "resource_reserved" {
                  "vcores": 1
                  "memory": 1024
              }
        
        
            },
            
          ]
        }
      Table 3 Parameters of all queues

      Attribute

      Type

      Description

      queuelist

      array

      Array of queue names.

      name

      String

      Queue name.

      open_state

      String

      This is inner state (self state) of queue. Indicate either "OPEN" or "CLOSED" effective state of the queue. A "CLOSED" queue does not accept any new allocation request.

      eopen_state

      String

      This is outer state of queue. An effective state is combination of queue own state and its ancestors. A "CLOSED" queue does not accept any new allocation request.

      active_state

      String

      This is inner state (self state) of queue. Indicate either "ACTIVE" or "INACTIVE" state of the queue.A "INACTIVE" queue does not schedule any application.

      eactive_state

      String

      This is outer state of queue. An effective state is combination of queue own state and its ancestors. A "INACTIVE" queue does not schedule any application.

      number_pending_application

      int

      Total number of pending applications.

      number_running_application

      int

      Total number of running applications.

      number_pending_request

      int

      Total number of pending request.

      number_running_container

      int

      Total number of running containers.

      numbert_reserved_container

      int

      Total number of reserved containers.

      resource_request

      object

      Pending resource requests within queue in a form of vcores, memory, and so on.

      resource_inuse

      object

      In use resource within queue in a form of vcores, memory, and so on.

      resource_reserved

      object

      Reserved resource within queue in form of vcores, memory, and so on.

      active_state

      String

      Indicate either "ACTIVE" or "INACTIVE" state of the queue. A "INACTIVE" queue does not schedule any allocation.

    • Query *single* queues within scheduler engine, including leaf and all middle queues.
      • URL

        GET http://<SS_REST_SERVER>/ws/v1/sscheduler/queues/{queuename}

      • Input

        None.

      • Output
        JSON Response:
        
        {
          "queue": {
        "name": "root.default",
        "eopen_state": "CLOSED",
        "eactive_state": "INACTIVE",
            "open_state": "CLOSED",
        "active_state": "INACTIVE",
        "leaf_queue" : yes,
            "number_pending_application": 100,
        "number_running_application": 10,
            "number_pending_request": 10,
        "number_running_container": 10,
        "number_reserved_container:1,
        "resource_inuse" {
                  "vcores": 10,
                  "memory": 10240
              },
              "resource_request" {
                  "vcores": 2,
                  "memory": 2048
              },
              "resource_reserved" {
                   "vcores": 1, 
                   "memory": 1024
              }
        
            "configuration": {
          "description": "Production spark queue",
              "max_pending_application": 10000,
              "max_running_application": 1000,
              "allocation_order_policy": "FIFO",
              "default_resource_select": "label1",
              "max_master_share": 10%,
              "max_running_application_per_user": -1,
              "max_allocation_unit": {
                   "vcores": 32,
                   "memory": 128000
              },
              "user_acl": [
                {
                  "user": "user1"
                },
                {
                  "group": "group1"
                }
              ],
              "admin_acl": [
                {
                  "user": "user2"
                },
                {
                  "group": "group2"
                }
              ]
            }
          }
        }
      • Exceptions

        Queue not found.

      Table 4 Parameters of single queues

      Attribute

      Type

      Description

      queue

      object

      A queue object.

      name

      String

      Queue name.

      description

      String

      Purpose of queue.

      open_state

      String

      This is inner state (self state) of queue. Indicate either "OPEN" or "CLOSED" effective state of the queue. A "CLOSED" queue does not accept any new allocation request.

      eopen_state

      String

      This is outer state of queue. An effective state is combination of queue own state and its ancestors. A "CLOSED" queue does not accept any new allocation request.

      active_state

      String

      This is inner state (self state) of queue. Indicate either "ACTIVE" or "INACTIVE" state of the queue.A "INACTIVE" queue does not schedule any application.

      eactive_state

      String

      This is outer state of queue. An effective state is combination of queue own state and its ancestors. A "INACTIVE" queue does not schedule any application.

      leaf_queue

      boolean

      Indicate whether queue is leaf or middle. Yes means leaf queue.

      number_pending_application

      int

      Number of pending application currently.In case of middle or parent queue, this is the aggregated number of all children queue.

      number_running_application

      int

      Number of running application currently. In case of middle or parent queue, this is the aggregated number of all children queue.

      number_pending_request

      int

      Number of pending demand; sum of count from each outstanding demand. In case of middle/parent queue, this is the aggregated number of all children queues.

      number_running_container

      int

      Number of running containers. In case of middle or parent queue, this is the aggregated number of all children queues.

      number_reserved_container

      int

      Number of reserved containers. In case of middle or parent queue, this is the aggregated number of children queues.

      resource_request

      object

      Pending resource requests within queue in a form of vcores, memory etc.

      resource_inuse

      object

      In use resource within queue in a form of vcores and memory etc.

      resource_reserved

      object

      Reserved resources within queue in a form of vcores and memory etc.

      configuration

      object

      A queue configuration object.

      max_pending_application

      int

      Max number of pending application. In case of middle or parent queue, this is the aggregated number or all children queue.

      max_running_application

      int

      Max number of running application. In case of middle/parent queue, this is the aggregated number or all children queue.

      allocation_order_policy

      String

      Allocation policy, can be FIFO, PRIORITY, or FAIR.

      max_running_application_per_user

      int

      Maximum number of running application per user.

      max_master_share

      string

      Percentage of steady share of this queue.

      max_allocation_unit

      object

      Maximum allowed resource per container in vcores and memory format.

      default_resource_select

      String

      Default resource selection expression. It is used when an application does not specify one during its submission.

      user_acl

      array

      Array of user, who have been given "user" rights on this queue.

      admin_acl

      array

      Array of user, who have been given "admin" rights on this queue.

      group

      String

      User group name.

      user

      String

      User name.

  • Query Resource Pool
    • Query *all* resource pools within scheduler engine.
      • URL

        GET http://<SS_REST_SERVER>/ws/v1/sscheduler/resourcepools/list

      • Input

        None.

      • Output
        JSON Response:
        {
          "resourcepool_list": [
            {
              "name": "pool1",
              "description": "resource pool for crc",
              "number_member": 5,
              "members": [
                {
                  "resource": "node1"
                },
                {
                  "resource": "node2"
                },
                {
                  "resource": "node3"
                },
                {
                  "resource": "node4"
                },
                {
                  "resource": "node5"
                }
              ],
              "available_resource": {
                "vcores": 60,
                "memory": 60000
              },
              "total_resource": {
                "vcores": 100,
                "memory": 128000
              },
              "configuration": {
                "resources": [
                  {
                    "resource": "node1"
                  },
                  {
                    "resource": "node[2-5]"
                  }
                ],
                "resource_select": "label1"
              }
            },
            {
              "name": "pool2",
              "description": "resource pool for erc",
              "number_member": 4
              "members": [
                {
                  "resource": "node6"
                },
                {
                  "resource": "node7"
                },
                {
                  "resource": "node8"
                },
                {
                  "resource": "node9"
                }
              ],
              "available_resource": {
                "vcores": 56,
                "memory": 48000
              },
              "total_resource": {
                 "vcores": 100,
                 "memory": 128000
              },
              "configuration": {
                "resources": [
                  {
                    "resource": "node6"
                  },
                  {
                    "resource": "node[7-9]"
                  }
                ],
                "resource_select": "label1"
              }
            },
            {
              "name": "default",
              "description": "system-generated",
              "number_member": 1,
              "members": [
                {
                  "resource": "node0"
                }
              ],
              "available_resource": {
                "vcores": 8,
                "memory": 8192
              },
              "total_resource": {
                "vcores": 16,
                "memory": 12800
              }
            }
          ]
        }
      Table 5 Parameters of all resource pool

      Attribute

      Type

      Description

      resourcepool_list

      array

      Array of resource pool objects.

      name

      String

      Resource pool name.

      number_member

      int

      Number of members in resource pool.

      description

      String

      Description of the resource pool.

      members

      array

      Arrays of resource name, which are current members of the resource pool.

      resource

      String

      Resource name.

      available_resource

      object

      Current available resource in this pool.

      vcores, memory, ..

      int

      Numeric consumable resource attributes, available via this resource pool now.

      total_resource

      object

      total resource in this pool.

      vcores, memory, ..

      int

      Numeric consumable resource attributes, total via this resource pool now.

      configuration

      object

      Configuration object.

      resources

      array

      Array of resource name pattern configured.

      resource

      String

      Resource name pattern.

      resource_select

      String

      Resource select expression.

    • Query *single* resource pools within scheduler engine.
      • URL

        GET http://<SS_REST_SERVER>/ws/v1/sscheduler/resourcepools/{resourcepoolname}

      • Input

        None.

      • Output
        JSON Response:
        {
          "resourcepool": {
            "name": "pool1",
            "description": "resource pool for crc",
            "number_member": 5
            "members": [
              {
                "resource": "node1"
              },
              {
                "resource": "node2"
              },
              {
                "resource": "node3"
              },
              {
                "resource": "node4"
              },
              {
                "resource": "node5"
              }
            ],
            "available_resource": {
              "vcores": 60,
              "memory": 60000
        },
        "total_resource": {
          "vcores": 100,
          "memory": 128000
        },
            "configuration": {
              "resources": [
                {
                  "resource": "node6"
                },
                {
                  "resource": "node[7-9]"
                }
              ],
              "resource_select": "label1"
            }
          }
        }
      • Exceptions

        Resource pool not found.

      Table 6 Parameters of single resource pool

      Attribute

      Type

      Description

      resourcepool

      object

      Resource pool object.

      name

      String

      Resource pool name.

      description

      String

      Description of the resource pool.

      number_member

      int

      Number of members in resource pool.

      members

      array

      Arrays of resource name, which are current members of the resource pool.

      resource

      String

      Resource name.

      available_resource

      object

      Current available resource in this pool.

      vcores, memory, ..

      int

      Numeric consumable resource attributes, available via this resource pool now.

      total_resource

      object

      total resource in this pool.

      vcores, memory, ..

      int

      Numeric consumable resource attributes, total via this resource pool now.

      configuration

      object

      Configuration object.

      resources

      array

      Array of resource name pattern configured.

      resource

      String

      Resource name pattern.

      resource_select

      String

      Resource select expression.

  • Query policiesxmlconf
    • URL

      GET http://<SS_REST_SERVER>/ws/v1/sscheduler/policiesxmlconf/list

    • Input

      None.

    • Output
      <policies>
        <policlist>
          <resourcepool>default</resourcepool>
          <queues>
            <name>default</name>
            <fullname>root.default</fullname>
            <share>20.0</share>
            <reserve>memory 0,vcores 0 : 0.0%</reserve>
            <minimum>memory 0,vcores 0 : 20.0%</minimum>
            <maximum>memory 0,vcores 0 : 100.0%</maximum>
            <defaultuser>
              <maximum>100.0%</maximum>
              <weight>1.0</weight>
            </defaultuser>
          </queues>
        </policlist>
      </policies>