YARN-8821. [YARN-8851] GPU hierarchy/topology scheduling support based on pluggable device framework. Contributed by Zhankun Tang.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePluginScheduler.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin;
 
+import java.util.Map;
 import java.util.Set;
 
 /**
@@ -29,10 +30,15 @@ public interface DevicePluginScheduler {
    * Called when allocating devices. The framework will do all device book
    * keeping and fail recovery. So this hook could be stateless and only do
    * scheduling based on available devices passed in. It could be
-   * invoked multiple times by the framework.
+   * invoked multiple times by the framework. The hint in environment variables
+   * passed in could be potentially used in making better scheduling decision.
+   * For instance, GPU scheduling might support different kind of policy. The
+   * container can set it through environment variables.
    * @param availableDevices Devices allowed to be chosen from.
    * @param count Number of device to be allocated.
+   * @param env Environment variables of the container.
    * @return A set of {@link Device} allocated
    * */
-  Set<Device> allocateDevices(Set<Device> availableDevices, int count);
+  Set<Device> allocateDevices(Set<Device> availableDevices, int count,
+      Map<String, String> env);
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nvidia/NvidiaGPUPluginForRuntimeV2.java

@@ -24,6 +24,7 @@ import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device;
 import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePluginScheduler;
 import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRegisterRequest;
 import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRuntimeSpec;
 import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.YarnRuntimeType;
@@ -32,7 +33,12 @@ import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
@@ -40,8 +46,10 @@ import java.util.TreeSet;
 /**
  * Nvidia GPU plugin supporting both Nvidia container runtime v2 for Docker and
  * non-Docker container.
+ * It has topology aware as well as simple scheduling ability.
  * */
-public class NvidiaGPUPluginForRuntimeV2 implements DevicePlugin {
+public class NvidiaGPUPluginForRuntimeV2 implements DevicePlugin,
+    DevicePluginScheduler {
   public static final Logger LOG = LoggerFactory.getLogger(
       NvidiaGPUPluginForRuntimeV2.class);
 
@@ -69,6 +77,47 @@ public class NvidiaGPUPluginForRuntimeV2 implements DevicePlugin {
   private static final Set<String> DEFAULT_BINARY_SEARCH_DIRS = ImmutableSet.of(
       "/usr/bin", "/bin", "/usr/local/nvidia/bin");
 
+  private boolean topoInitialized = false;
+
+  private Set<Device> lastTimeFoundDevices;
+
+  /**
+   * It caches the combination of different devices and the communication cost.
+   * The key is device count
+   * The value is an ordered list of map entry whose key is device combination,
+   * value is cost. The list is sorted by cost in ascending order.
+   * For instance:
+   * { 2=> [[device1,device2]=>0, [device1,device3]=>10]
+   *   3 => [[device1,device2,device3]=>10, [device2,device3,device5]=>20],
+   * }
+   * */
+  private Map<Integer, List<Map.Entry<Set<Device>, Integer>>> costTable
+      = new HashMap<>();
+
+  /**
+   * The key is a pair of minors. For instance, "0-1" indicates 0 to 1
+   * The value is weight between the two devices.
+   * */
+  private Map<String, Integer> devicePairToWeight = new HashMap<>();
+
+  /**
+   * The container can set this environment variable.
+   * To tell the scheduler what's the policy to use when do scheduling
+   * */
+  public static final String TOPOLOGY_POLICY_ENV_KEY = "NVIDIA_TOPO_POLICY";
+
+  /**
+   * Schedule policy that prefer the faster GPU-GPU communication.
+   * Suitable for heavy GPU computation workload generally.
+   * */
+  public static final String TOPOLOGY_POLICY_PACK = "PACK";
+
+  /**
+   * Schedule policy that prefer the faster CPU-GPU communication.
+   * Suitable for heavy CPU-GPU IO operations generally.
+   * */
+  public static final String TOPOLOGY_POLICY_SPREAD = "SPREAD";
+
   @Override
   public DeviceRegisterRequest getRegisterRequestInfo() throws Exception {
     return DeviceRegisterRequest.Builder.newInstance()
@@ -106,6 +155,8 @@ public class NvidiaGPUPluginForRuntimeV2 implements DevicePlugin {
           id++;
         }
       }
+      // cache it which help to topology scheduling
+      lastTimeFoundDevices = r;
       return r;
     } catch (IOException e) {
       if (LOG.isDebugEnabled()) {
@@ -170,6 +221,422 @@ public class NvidiaGPUPluginForRuntimeV2 implements DevicePlugin {
     return output;
   }
 
+  @Override
+  public Set<Device> allocateDevices(Set<Device> availableDevices, int count,
+      Map<String, String> envs) {
+    Set<Device> allocation = new TreeSet<>();
+    /**
+     * corner cases.
+     * if allocate 1 device or all devices, no topo scheduling needed.
+     * if total available devices is less than 3, no topo scheduling needed.
+     * */
+    if (availableDevices.size() < 3
+        || count == 1
+        || availableDevices.size() == count) {
+      basicSchedule(allocation, count, availableDevices);
+      return allocation;
+    }
+
+    try {
+      if (!topoInitialized) {
+        initCostTable();
+      }
+      // topology aware scheduling
+      topologyAwareSchedule(allocation, count,
+          envs, availableDevices, this.costTable);
+      if (allocation.size() == count) {
+        return allocation;
+      } else {
+        LOG.error("Failed to do topology scheduling. Skip to use basic "
+            + "scheduling");
+      }
+    } catch (IOException e) {
+      LOG.error("Error in getting GPU topology info. "
+          + "Skip topology aware scheduling", e);
+    }
+    // basic scheduling
+    basicSchedule(allocation, count, availableDevices);
+    return allocation;
+  }
+
+  @VisibleForTesting
+  public void initCostTable() throws IOException {
+    // get topology
+    String topo = shellExecutor.getTopologyInfo();
+    // build the graph
+    parseTopo(topo, devicePairToWeight);
+    // build the cost table of different device combinations
+    if (lastTimeFoundDevices == null) {
+      try {
+        getDevices();
+      } catch (Exception e) {
+        LOG.error("Failed to get devices!", e);
+        return;
+      }
+    }
+    buildCostTable(costTable, lastTimeFoundDevices);
+    loggingCostTable(costTable);
+    this.topoInitialized = true;
+  }
+
+  private void loggingCostTable(
+      Map<Integer, List<Map.Entry<Set<Device>, Integer>>> cTable) {
+    if (LOG.isDebugEnabled()) {
+      StringBuilder sb = new StringBuilder("The costTable is:");
+      sb.append("\n{");
+      for (Map.Entry<Integer, List<Map.Entry<Set<Device>, Integer>>> entry
+          : cTable.entrySet()) {
+        sb.append("\n\t")
+            .append(entry.getKey())
+            .append(" => [");
+        for (Map.Entry<Set<Device>, Integer> e : entry.getValue()) {
+          sb.append("\n\t\t").append(e.toString()).append(",\n");
+        }
+        sb.append("\t\t]\n");
+      }
+      sb.append("}\n");
+      LOG.debug(sb.toString());
+    }
+  }
+
+  /**
+   * Generate combination of devices and its cost.
+   * costTable
+   * */
+  private void buildCostTable(
+      Map<Integer, List<Map.Entry<Set<Device>, Integer>>> cTable,
+      Set<Device> ltfDevices) {
+    Device[] deviceList = new Device[ltfDevices.size()];
+    ltfDevices.toArray(deviceList);
+    generateAllDeviceCombination(cTable, deviceList, deviceList.length);
+  }
+
+  /**
+   * For every possible combination of i elements.
+   * We generate a map whose key is the combination, value is cost.
+   */
+  private void generateAllDeviceCombination(
+      Map<Integer, List<Map.Entry<Set<Device>, Integer>>> cTable,
+      Device[] allDevices, int n) {
+    // allocated devices count range from 1 to n-1
+    for (int i = 2; i < n; i++) {
+      Map<Set<Device>, Integer> combinationToCost =
+          new HashMap<>();
+      buildCombination(combinationToCost, allDevices, n, i);
+      // sort the map entry by cost ascending order
+      List<Map.Entry<Set<Device>, Integer>> listSortedByCost =
+          new LinkedList<>(combinationToCost.entrySet());
+      Collections.sort(listSortedByCost,
+          (o1, o2) -> (o1.getValue()).compareTo(o2.getValue()));
+      cTable.put(i, listSortedByCost);
+    }
+  }
+
+  private void buildCombination(Map<Set<Device>, Integer> combinationToCost,
+      Device[] allDevices, int n, int r) {
+    // A temporary list to store all combination one by one
+    Device[] subDeviceList = new Device[r];
+    combinationRecursive(combinationToCost, allDevices, subDeviceList,
+        0, n - 1, 0, r);
+  }
+
+  /**
+   * Populate combination to cost map recursively.
+   *
+   * @param cTc           combinationToCost map.
+   *                      The key is device set, the value is cost
+   * @param allDevices    all devices used to assign value to subDevicelist
+   * @param subDeviceList store a subset of devices temporary
+   * @param start         start index in the allDevices
+   * @param end           last index in the allDevices
+   * @param index         dynamic index in subDeviceList need to be assigned
+   * @param r             the length of the subDeviceList
+   */
+  void combinationRecursive(Map<Set<Device>, Integer> cTc,
+      Device[] allDevices, Device[] subDeviceList,
+      int start, int end, int index, int r) {
+    // sub device list's length is ready to compute the cost
+    if (index == r) {
+      Set<Device> oneSet = new TreeSet<>(Arrays.asList(subDeviceList));
+      int cost = computeCostOfDevices(subDeviceList);
+      cTc.put(oneSet, cost);
+      return;
+    }
+    for (int i = start; i <= end; i++) {
+      subDeviceList[index] = allDevices[i];
+      combinationRecursive(cTc, allDevices, subDeviceList,
+          i + 1, end, index + 1, r);
+    }
+  }
+
+  /**
+   * The cost function used to calculate costs of a sub set of devices.
+   * It calculate link weight of each pair in non-duplicated combination of
+   * devices.
+   */
+  @VisibleForTesting
+  public int computeCostOfDevices(Device[] devices) {
+    int cost = 0;
+    String gpuIndex0;
+    String gpuIndex1;
+    for (int i = 0; i < devices.length; i++) {
+      gpuIndex0 = String.valueOf(devices[i].getMinorNumber());
+      for (int j = i + 1; j < devices.length; j++) {
+        gpuIndex1 = String.valueOf(devices[j].getMinorNumber());
+        cost += this.devicePairToWeight.get(gpuIndex0 + "-" + gpuIndex1);
+      }
+    }
+    return cost;
+  }
+
+  /**
+   * Topology Aware schedule algorithm.
+   * It doesn't consider CPU affinity or NUMA or bus bandwidths.
+   * It support two plicy: "spread" and "pack" which can be set by container's
+   * environment variable. Use pack by default which means prefer the faster
+   * GPU-GPU. "Spread" means prefer the faster CPU-GPU.
+   * It can potentially be extend to take GPU attribute like GPU chip memory
+   * into consideration.
+   * */
+  @VisibleForTesting
+  public void topologyAwareSchedule(Set<Device> allocation, int count,
+      Map<String, String> envs,
+      Set<Device> availableDevices,
+      Map<Integer, List<Map.Entry<Set<Device>, Integer>>> cTable) {
+    int num = 0;
+    String policy = envs.get(TOPOLOGY_POLICY_ENV_KEY);
+    if (policy == null) {
+      policy = TOPOLOGY_POLICY_PACK;
+    }
+
+    /**
+     * Get combinations from costTable given the count of device want to
+     * allocate.
+     * */
+    if (cTable == null) {
+      LOG.error("No cost table initialized!");
+      return;
+    }
+    List<Map.Entry<Set<Device>, Integer>> combinationsToCost =
+        cTable.get(count);
+    Iterator<Map.Entry<Set<Device>, Integer>> iterator =
+        combinationsToCost.iterator();
+    // the container needs spread policy
+    if (policy.equalsIgnoreCase(TOPOLOGY_POLICY_SPREAD)) {
+      // loop from high cost to low cost
+      iterator = ((LinkedList) combinationsToCost).descendingIterator();
+    }
+    while (iterator.hasNext()) {
+      Map.Entry<Set<Device>, Integer> element = iterator.next();
+      if (availableDevices.containsAll(element.getKey())) {
+        allocation.addAll(element.getKey());
+        LOG.info("Topology scheduler allocated: " + allocation);
+        return;
+      }
+    }
+    LOG.error("Unknown error happened in topology scheduler");
+  }
+
+  @VisibleForTesting
+  public void basicSchedule(Set<Device> allocation, int count,
+      Set<Device> availableDevices) {
+    // Basic scheduling
+    // allocate all available
+    if (count == availableDevices.size()) {
+      allocation.addAll(availableDevices);
+      return;
+    }
+    int number = 0;
+    for (Device d : availableDevices) {
+      allocation.add(d);
+      number++;
+      if (number == count) {
+        break;
+      }
+    }
+  }
+
+  /**
+   * A typical sample topo output:
+   *     GPU0  GPU1  GPU2  GPU3  CPU Affinity
+   * GPU0  X  PHB  SOC  SOC  0-31
+   * GPU1 PHB  X   SOC  SOC  0-31
+   * GPU2 SOC SOC  X    PHB  0-31
+   * GPU3 SOC SOC  PHB   X   0-31
+   *
+   *
+   * Legend:
+   *
+   *   X   = Self
+   *   SOC  = Connection traversing PCIe as well as the SMP link between
+   *   CPU sockets(e.g. QPI)
+   *   PHB  = Connection traversing PCIe as well as a PCIe Host Bridge
+   *   (typically the CPU)
+   *   PXB  = Connection traversing multiple PCIe switches
+   *   (without traversing the PCIe Host Bridge)
+   *   PIX  = Connection traversing a single PCIe switch
+   *   NV#  = Connection traversing a bonded set of # NVLinks」
+   * */
+  public void parseTopo(String topo,
+      Map<String, Integer> deviceLinkToWeight) {
+    String[] lines = topo.split("\n");
+    int rowMinor;
+    int colMinor;
+    String legend;
+    String tempType;
+    for (String oneLine : lines) {
+      oneLine = oneLine.trim();
+      if (oneLine.isEmpty()) {
+        continue;
+      }
+      // To the end. No more metrics info
+      if (oneLine.startsWith("Legend")) {
+        break;
+      }
+      // Skip header
+      if (oneLine.contains("Affinity")) {
+        continue;
+      }
+      String[] tokens = oneLine.split(("\\s+"));
+      String name = tokens[0];
+      rowMinor = Integer.parseInt(name.substring(name.lastIndexOf("U") + 1));
+      for (int i = 1; i < tokens.length; i++) {
+        tempType = tokens[i];
+        colMinor = i - 1;
+        // self, skip
+        if (tempType.equals("X")) {
+          continue;
+        }
+        if (tempType.equals("SOC") || tempType.equals("SYS")) {
+          populateGraphEdgeWeight(DeviceLinkType.P2PLinkCrossCPUSocket,
+              rowMinor, colMinor, deviceLinkToWeight);
+          continue;
+        }
+        if (tempType.equals("PHB") || tempType.equals("NODE")) {
+          populateGraphEdgeWeight(DeviceLinkType.P2PLinkSameCPUSocket,
+              rowMinor, colMinor, deviceLinkToWeight);
+          continue;
+        }
+        if (tempType.equals("PXB")) {
+          populateGraphEdgeWeight(DeviceLinkType.P2PLinkMultiSwitch,
+              rowMinor, colMinor, deviceLinkToWeight);
+          continue;
+        }
+        if (tempType.equals("PIX")) {
+          populateGraphEdgeWeight(DeviceLinkType.P2PLinkSingleSwitch,
+              rowMinor, colMinor, deviceLinkToWeight);
+          continue;
+        }
+        if (tempType.equals("NV1")) {
+          populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink1,
+              rowMinor, colMinor, deviceLinkToWeight);
+          continue;
+        }
+        if (tempType.equals("NV2")) {
+          populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink2,
+              rowMinor, colMinor, deviceLinkToWeight);
+          continue;
+        }
+        if (tempType.equals("NV3")) {
+          populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink3,
+              rowMinor, colMinor, deviceLinkToWeight);
+          continue;
+        }
+        if (tempType.equals("NV4")) {
+          populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink4,
+              rowMinor, colMinor, deviceLinkToWeight);
+          continue;
+        }
+        if (tempType.equals("NV5")) {
+          populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink5,
+              rowMinor, colMinor, deviceLinkToWeight);
+          continue;
+        }
+        if (tempType.equals("NV6")) {
+          populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink6,
+              rowMinor, colMinor, deviceLinkToWeight);
+          continue;
+        }
+        if (tempType.equals("NV7")) {
+          populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink7,
+              rowMinor, colMinor, deviceLinkToWeight);
+          continue;
+        }
+        if (tempType.equals("NV8")) {
+          populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink8,
+              rowMinor, colMinor, deviceLinkToWeight);
+          continue;
+        }
+        if (tempType.equals("NV9")) {
+          populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink9,
+              rowMinor, colMinor, deviceLinkToWeight);
+          continue;
+        }
+      } // end one line handling
+    }
+  }
+
+  private void populateGraphEdgeWeight(
+      DeviceLinkType linkType,
+      int leftVertex,
+      int rightVertex,
+      Map<String, Integer> deviceLinkToWeight) {
+    deviceLinkToWeight.put(leftVertex + "-" + rightVertex,
+        linkType.getWeight());
+  }
+
+  /**
+   * Different type of link.
+   * The weight of each link is a relative value.
+   * The higher weight, the higher cost between the GPUs
+   * */
+  public enum DeviceLinkType {
+    /**
+     * For Nvdia GPU NVLink.
+     * */
+    P2PLinkNVLink9(10),
+    P2PLinkNVLink8(20),
+    P2PLinkNVLink7(30),
+    P2PLinkNVLink6(40),
+    P2PLinkNVLink5(50),
+    P2PLinkNVLink4(60),
+    P2PLinkNVLink3(70),
+    P2PLinkNVLink2(80),
+    P2PLinkNVLink1(90),
+
+    /**
+     * Connected to same CPU (Same NUMA node).
+     * */
+    P2PLinkSameCPUSocket(200),
+
+    /**
+     * Cross CPU through socket-level link (e.g. QPI).
+     * Usually cross NUMA node
+     * */
+    P2PLinkCrossCPUSocket(300),
+
+    /**
+     * Just need to traverse one PCIe switch to talk.
+     * */
+    P2PLinkSingleSwitch(600),
+
+    /**
+     * Need to traverse multiple PCIe switch to talk.
+     * */
+    P2PLinkMultiSwitch(1200);
+
+    // A higher link level means slower communication.
+    private int weight;
+
+    public int getWeight() {
+      return weight;
+    }
+
+    DeviceLinkType(int w) {
+      this.weight = w;
+    }
+  }
+
   /**
    * A shell wrapper class easy for test.
    * */
@@ -189,6 +656,13 @@ public class NvidiaGPUPluginForRuntimeV2 implements DevicePlugin {
       return shexec.getOutput();
     }
 
+    // Get the topology metrics info from nvdia-smi
+    public String getTopologyInfo() throws IOException {
+      return Shell.execCommand(environment,
+          new String[]{pathOfGpuBinary, "topo",
+              "-m"}, MAX_EXEC_TIMEOUT_MS);
+    }
+
     public void searchBinary() throws Exception {
       if (pathOfGpuBinary != null) {
         LOG.info("Skip searching, the nvidia gpu binary is already set: "
@@ -228,8 +702,8 @@ public class NvidiaGPUPluginForRuntimeV2 implements DevicePlugin {
   }
 
   @VisibleForTesting
-  public void setPathOfGpuBinary(String pathOfGpuBinary) {
-    this.pathOfGpuBinary = pathOfGpuBinary;
+  public void setPathOfGpuBinary(String pOfGpuBinary) {
+    this.pathOfGpuBinary = pOfGpuBinary;
   }
 
   @VisibleForTesting
@@ -237,4 +711,20 @@ public class NvidiaGPUPluginForRuntimeV2 implements DevicePlugin {
       NvidiaCommandExecutor shellExecutor) {
     this.shellExecutor = shellExecutor;
   }
+
+  @VisibleForTesting
+  public boolean isTopoInitialized() {
+    return topoInitialized;
+  }
+
+  @VisibleForTesting
+  public Map<Integer, List<Map.Entry<Set<Device>, Integer>>> getCostTable() {
+    return costTable;
+  }
+
+  @VisibleForTesting
+  public Map<String, Integer> getDevicePairToWeight() {
+    return devicePairToWeight;
+  }
+
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceMappingManager.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 import org.apache.commons.logging.Log;
@@ -192,7 +193,7 @@ public class DeviceMappingManager {
       DevicePluginScheduler dps = devicePluginSchedulers.get(resourceName);
       // Prefer DevicePluginScheduler logic
       pickAndDoSchedule(allowedDevices, usedDevices, assignedDevices,
-          containerId, requestedDeviceCount, resourceName, dps);
+          container, requestedDeviceCount, resourceName, dps);
 
       // Record in state store if we allocated anything
       if (!assignedDevices.isEmpty()) {
@@ -310,9 +311,11 @@ public class DeviceMappingManager {
    * */
   private void pickAndDoSchedule(Set<Device> allowed,
       Map<Device, ContainerId> used, Set<Device> assigned,
-      ContainerId containerId, int count, String resourceName,
-      DevicePluginScheduler dps) throws ResourceHandlerException {
-
+      Container c, int count, String resourceName,
+      DevicePluginScheduler dps)
+      throws ResourceHandlerException {
+    ContainerId containerId = c.getContainerId();
+    Map<String, String> env = c.getLaunchContext().getEnvironment();
     if (null == dps) {
       if (LOG.isDebugEnabled()) {
         LOG.debug("Customized device plugin scheduler is preferred "
@@ -331,7 +334,8 @@ public class DeviceMappingManager {
       // Pass in unmodifiable set
       Set<Device> dpsAllocated = dps.allocateDevices(
           Sets.difference(allowed, used.keySet()),
-          count);
+          count,
+          ImmutableMap.copyOf(env));
       if (dpsAllocated.size() != count) {
         throw new ResourceHandlerException(dps.getClass()
             + " should allocate " + count

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nvidia/TestNvidiaGPUPluginForRuntimeV2.java

@@ -0,0 +1,848 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.com.nvidia;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRuntimeSpec;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.YarnRuntimeType;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyMap;
+import static org.mockito.ArgumentMatchers.anySet;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test case for NvidiaGPUPluginForRuntimeV2 device plugin.
+ * */
+public class TestNvidiaGPUPluginForRuntimeV2 {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestNvidiaGPUPluginForRuntimeV2.class);
+
+  @Test
+  public void testGetNvidiaDevices() throws Exception {
+    NvidiaGPUPluginForRuntimeV2.NvidiaCommandExecutor mockShell =
+        mock(NvidiaGPUPluginForRuntimeV2.NvidiaCommandExecutor.class);
+    String deviceInfoShellOutput =
+        "0, 00000000:04:00.0\n" +
+        "1, 00000000:82:00.0";
+    String majorMinorNumber0 = "c3:0";
+    String majorMinorNumber1 = "c3:1";
+    when(mockShell.getDeviceInfo()).thenReturn(deviceInfoShellOutput);
+    when(mockShell.getMajorMinorInfo("nvidia0"))
+        .thenReturn(majorMinorNumber0);
+    when(mockShell.getMajorMinorInfo("nvidia1"))
+        .thenReturn(majorMinorNumber1);
+    NvidiaGPUPluginForRuntimeV2 plugin = new NvidiaGPUPluginForRuntimeV2();
+    plugin.setShellExecutor(mockShell);
+    plugin.setPathOfGpuBinary("/fake/nvidia-smi");
+
+    Set<Device> expectedDevices = new TreeSet<>();
+    expectedDevices.add(Device.Builder.newInstance()
+        .setId(0).setHealthy(true)
+        .setBusID("00000000:04:00.0")
+        .setDevPath("/dev/nvidia0")
+        .setMajorNumber(195)
+        .setMinorNumber(0).build());
+    expectedDevices.add(Device.Builder.newInstance()
+        .setId(1).setHealthy(true)
+        .setBusID("00000000:82:00.0")
+        .setDevPath("/dev/nvidia1")
+        .setMajorNumber(195)
+        .setMinorNumber(1).build());
+    Set<Device> devices = plugin.getDevices();
+    Assert.assertEquals(expectedDevices, devices);
+  }
+
+  @Test
+  public void testOnDeviceAllocated() throws Exception {
+    NvidiaGPUPluginForRuntimeV2 plugin = new NvidiaGPUPluginForRuntimeV2();
+    Set<Device> allocatedDevices = new TreeSet<>();
+
+    DeviceRuntimeSpec spec = plugin.onDevicesAllocated(allocatedDevices,
+        YarnRuntimeType.RUNTIME_DEFAULT);
+    Assert.assertNull(spec);
+
+    // allocate one device
+    allocatedDevices.add(Device.Builder.newInstance()
+        .setId(0).setHealthy(true)
+        .setBusID("00000000:04:00.0")
+        .setDevPath("/dev/nvidia0")
+        .setMajorNumber(195)
+        .setMinorNumber(0).build());
+    spec = plugin.onDevicesAllocated(allocatedDevices,
+        YarnRuntimeType.RUNTIME_DOCKER);
+    Assert.assertEquals("nvidia", spec.getContainerRuntime());
+    Assert.assertEquals("0", spec.getEnvs().get("NVIDIA_VISIBLE_DEVICES"));
+
+    // two device allowed
+    allocatedDevices.add(Device.Builder.newInstance()
+        .setId(0).setHealthy(true)
+        .setBusID("00000000:82:00.0")
+        .setDevPath("/dev/nvidia1")
+        .setMajorNumber(195)
+        .setMinorNumber(1).build());
+    spec = plugin.onDevicesAllocated(allocatedDevices,
+        YarnRuntimeType.RUNTIME_DOCKER);
+    Assert.assertEquals("nvidia", spec.getContainerRuntime());
+    Assert.assertEquals("0,1", spec.getEnvs().get("NVIDIA_VISIBLE_DEVICES"));
+  }
+
+  private NvidiaGPUPluginForRuntimeV2 mockEightGPUPlugin() throws IOException {
+    String topoInfo =
+        "\tGPU0\tGPU1\tGPU2\tGPU3\tGPU4\tGPU5\tGPU6\tGPU7\tCPU Affinity\n"
+        + "GPU0\t X \tNV1\tNV1\tNV2\tNV2\tPHB\tPHB\tPHB\t0-63\n"
+        + "GPU1\tNV1\t X \tNV2\tNV1\tPHB\tNV2\tPHB\tPHB\t0-63\n"
+        + "GPU2\tNV1\tNV2\t X \tNV2\tPHB\tPHB\tNV1\tPHB\t0-63\n"
+        + "GPU3\tNV2\tNV1\tNV2\t X \tPHB\tPHB\tPHB\tNV1\t0-63\n"
+        + "GPU4\tNV2\tPHB\tPHB\tPHB\t X \tNV1\tNV1\tNV2\t0-63\n"
+        + "GPU5\tPHB\tNV2\tPHB\tPHB\tNV1\t X \tNV2\tNV1\t0-63\n"
+        + "GPU6\tPHB\tPHB\tNV1\tPHB\tNV1\tNV2\t X \tNV2\t0-63\n"
+        + "GPU7\tPHB\tPHB\tPHB\tNV1\tNV2\tNV1\tNV2\t X \t0-63\n"
+        + "\n"
+        + "Legend:\n"
+        + "\n"
+        + "  X    = Self\n"
+        + "  SYS  = Connection traversing PCIe as well as the SMP interconnect"
+        + " between NUMA nodes (e.g., QPI/UPI)\n"
+        + "  NODE = Connection traversing PCIe as well as the interconnect"
+        + " between PCIe Host Bridges within a NUMA node\n"
+        + "  PHB  = Connection traversing PCIe as well as a PCIe Host Bridge"
+        + " (typically the CPU)\n"
+        + "  PXB  = Connection traversing multiple PCIe switches"
+        + " (without traversing the PCIe Host Bridge)\n"
+        + "  PIX  = Connection traversing a single PCIe switch\n"
+        + "  NV#  = Connection traversing a bonded set of # NVLinks\n";
+
+    String deviceInfoShellOutput = "0, 00000000:04:00.0\n"
+        + "1, 00000000:82:00.0\n"
+        + "2, 00000000:83:00.0\n"
+        + "3, 00000000:84:00.0\n"
+        + "4, 00000000:85:00.0\n"
+        + "5, 00000000:86:00.0\n"
+        + "6, 00000000:87:00.0\n"
+        + "7, 00000000:88:00.0";
+    String majorMinorNumber0 = "c3:0";
+    String majorMinorNumber1 = "c3:1";
+    String majorMinorNumber2 = "c3:2";
+    String majorMinorNumber3 = "c3:3";
+    String majorMinorNumber4 = "c3:4";
+    String majorMinorNumber5 = "c3:5";
+    String majorMinorNumber6 = "c3:6";
+    String majorMinorNumber7 = "c3:7";
+    NvidiaGPUPluginForRuntimeV2.NvidiaCommandExecutor mockShell =
+        mock(NvidiaGPUPluginForRuntimeV2.NvidiaCommandExecutor.class);
+    when(mockShell.getDeviceInfo()).thenReturn(deviceInfoShellOutput);
+    when(mockShell.getMajorMinorInfo("nvidia0"))
+        .thenReturn(majorMinorNumber0);
+    when(mockShell.getMajorMinorInfo("nvidia1"))
+        .thenReturn(majorMinorNumber1);
+    when(mockShell.getMajorMinorInfo("nvidia2"))
+        .thenReturn(majorMinorNumber2);
+    when(mockShell.getMajorMinorInfo("nvidia3"))
+        .thenReturn(majorMinorNumber3);
+    when(mockShell.getMajorMinorInfo("nvidia4"))
+        .thenReturn(majorMinorNumber4);
+    when(mockShell.getMajorMinorInfo("nvidia5"))
+        .thenReturn(majorMinorNumber5);
+    when(mockShell.getMajorMinorInfo("nvidia6"))
+        .thenReturn(majorMinorNumber6);
+    when(mockShell.getMajorMinorInfo("nvidia7"))
+        .thenReturn(majorMinorNumber7);
+    when(mockShell.getTopologyInfo()).thenReturn(topoInfo);
+    when(mockShell.getDeviceInfo()).thenReturn(deviceInfoShellOutput);
+
+    NvidiaGPUPluginForRuntimeV2 plugin = new NvidiaGPUPluginForRuntimeV2();
+    plugin.setShellExecutor(mockShell);
+    plugin.setPathOfGpuBinary("/fake/nvidia-smi");
+    return plugin;
+  }
+
+  private NvidiaGPUPluginForRuntimeV2 mockFourGPUPlugin() throws IOException {
+    String topoInfo = "\tGPU0\tGPU1\tGPU2\tGPU3\tCPU Affinity\n"
+        + "GPU0\t X \tPHB\tSOC\tSOC\t0-31\n"
+        + "GPU1\tPHB\t X \tSOC\tSOC\t0-31\n"
+        + "GPU2\tSOC\tSOC\t X \tPHB\t0-31\n"
+        + "GPU3\tSOC\tSOC\tPHB\t X \t0-31\n"
+        + "\n"
+        + "\n"
+        + " Legend:\n"
+        + "\n"
+        + " X   = Self\n"
+        + " SOC  = Connection traversing PCIe as well as the SMP link between\n"
+        + " CPU sockets(e.g. QPI)\n"
+        + " PHB  = Connection traversing PCIe as well as a PCIe Host Bridge\n"
+        + " (typically the CPU)\n"
+        + " PXB  = Connection traversing multiple PCIe switches\n"
+        + " (without traversing the PCIe Host Bridge)\n"
+        + " PIX  = Connection traversing a single PCIe switch\n"
+        + " NV#  = Connection traversing a bonded set of # NVLinks";
+
+    String deviceInfoShellOutput = "0, 00000000:04:00.0\n"
+        + "1, 00000000:82:00.0\n"
+        + "2, 00000000:83:00.0\n"
+        + "3, 00000000:84:00.0";
+    String majorMinorNumber0 = "c3:0";
+    String majorMinorNumber1 = "c3:1";
+    String majorMinorNumber2 = "c3:2";
+    String majorMinorNumber3 = "c3:3";
+    NvidiaGPUPluginForRuntimeV2.NvidiaCommandExecutor mockShell =
+        mock(NvidiaGPUPluginForRuntimeV2.NvidiaCommandExecutor.class);
+    when(mockShell.getDeviceInfo()).thenReturn(deviceInfoShellOutput);
+    when(mockShell.getMajorMinorInfo("nvidia0"))
+        .thenReturn(majorMinorNumber0);
+    when(mockShell.getMajorMinorInfo("nvidia1"))
+        .thenReturn(majorMinorNumber1);
+    when(mockShell.getMajorMinorInfo("nvidia2"))
+        .thenReturn(majorMinorNumber2);
+    when(mockShell.getMajorMinorInfo("nvidia3"))
+        .thenReturn(majorMinorNumber3);
+    when(mockShell.getTopologyInfo()).thenReturn(topoInfo);
+    when(mockShell.getDeviceInfo()).thenReturn(deviceInfoShellOutput);
+
+    NvidiaGPUPluginForRuntimeV2 plugin = new NvidiaGPUPluginForRuntimeV2();
+    plugin.setShellExecutor(mockShell);
+    plugin.setPathOfGpuBinary("/fake/nvidia-smi");
+    return plugin;
+  }
+
+  @Test
+  public void testTopologySchedulingWithPackPolicy() throws Exception {
+    NvidiaGPUPluginForRuntimeV2 plugin = mockFourGPUPlugin();
+    NvidiaGPUPluginForRuntimeV2 spyPlugin = spy(plugin);
+    // cache the total devices
+    Set<Device> allDevices = spyPlugin.getDevices();
+    // environment variable to use PACK policy
+    Map<String, String> env = new HashMap<>();
+    env.put(NvidiaGPUPluginForRuntimeV2.TOPOLOGY_POLICY_ENV_KEY,
+        NvidiaGPUPluginForRuntimeV2.TOPOLOGY_POLICY_PACK);
+    // Case 0. if available devices is less than 3, no topo scheduling needed
+    Set<Device> copyAvailableDevices = new TreeSet<>(allDevices);
+    Iterator<Device> iterator0 = copyAvailableDevices.iterator();
+    iterator0.next();
+    iterator0.remove();
+    iterator0.next();
+    iterator0.remove();
+    // Case 0. allocate 1 device
+    reset(spyPlugin);
+    Set<Device> allocation = spyPlugin.allocateDevices(copyAvailableDevices,
+        1, env);
+    Assert.assertEquals(allocation.size(), 1);
+    verify(spyPlugin).basicSchedule(anySet(), anyInt(), anySet());
+    Assert.assertFalse(spyPlugin.isTopoInitialized());
+
+    // Case 1. allocate 1 device
+    reset(spyPlugin);
+    allocation = spyPlugin.allocateDevices(allDevices, 1, env);
+    // ensure no topology scheduling needed
+    Assert.assertEquals(allocation.size(), 1);
+    verify(spyPlugin).basicSchedule(anySet(), anyInt(), anySet());
+    reset(spyPlugin);
+    // Case 2. allocate all available
+    allocation = spyPlugin.allocateDevices(allDevices, allDevices.size(), env);
+    Assert.assertEquals(allocation.size(), allDevices.size());
+    verify(spyPlugin).basicSchedule(anySet(), anyInt(), anySet());
+    // Case 3. allocate 2 devices
+    reset(spyPlugin);
+    int count = 2;
+    Map<String, Integer> pairToWeight = spyPlugin.getDevicePairToWeight();
+    allocation = spyPlugin.allocateDevices(allDevices, count, env);
+    Assert.assertEquals(allocation.size(), count);
+    // the costTable should be init and used topology scheduling
+    verify(spyPlugin).initCostTable();
+    Assert.assertTrue(spyPlugin.isTopoInitialized());
+    verify(spyPlugin).topologyAwareSchedule(anySet(), anyInt(), anyMap(),
+        anySet(), anyMap());
+    Assert.assertEquals(allocation.size(), count);
+    Device[] allocatedDevices =
+        allocation.toArray(new Device[count]);
+    // Check weights
+    Assert.assertEquals(NvidiaGPUPluginForRuntimeV2.DeviceLinkType
+        .P2PLinkSameCPUSocket.getWeight(),
+        spyPlugin.computeCostOfDevices(allocatedDevices));
+    // Case 4. allocate 3 devices
+    reset(spyPlugin);
+    count = 3;
+    allocation = spyPlugin.allocateDevices(allDevices, count, env);
+    Assert.assertEquals(allocation.size(), count);
+    // the costTable should be init and used topology scheduling
+    verify(spyPlugin, times(0)).initCostTable();
+    Assert.assertTrue(spyPlugin.isTopoInitialized());
+    verify(spyPlugin).topologyAwareSchedule(anySet(), anyInt(), anyMap(),
+        anySet(), anyMap());
+    Assert.assertEquals(allocation.size(), count);
+    allocatedDevices =
+        allocation.toArray(new Device[count]);
+    // check weights
+    int expectedWeight =
+        NvidiaGPUPluginForRuntimeV2.DeviceLinkType
+            .P2PLinkSameCPUSocket.getWeight()
+            + 2 * NvidiaGPUPluginForRuntimeV2.DeviceLinkType
+            .P2PLinkCrossCPUSocket.getWeight();
+    Assert.assertEquals(expectedWeight,
+        spyPlugin.computeCostOfDevices(allocatedDevices));
+    // Case 5. allocate 2 GPUs from three available devices
+    reset(spyPlugin);
+    Iterator<Device> iterator = allDevices.iterator();
+    iterator.next();
+    // remove GPU0
+    iterator.remove();
+    count = 2;
+    allocation = spyPlugin.allocateDevices(allDevices, count, env);
+    Assert.assertEquals(allocation.size(), count);
+    // the costTable should be init and used topology scheduling
+    verify(spyPlugin, times(0)).initCostTable();
+    Assert.assertTrue(spyPlugin.isTopoInitialized());
+    verify(spyPlugin).topologyAwareSchedule(anySet(), anyInt(), anyMap(),
+        anySet(), anyMap());
+    Assert.assertEquals(allocation.size(), count);
+    allocatedDevices =
+        allocation.toArray(new Device[count]);
+    // check weights
+    Assert.assertEquals(NvidiaGPUPluginForRuntimeV2.DeviceLinkType
+            .P2PLinkSameCPUSocket.getWeight(),
+        spyPlugin.computeCostOfDevices(allocatedDevices));
+    // it should allocate GPU 2 and 3
+    for (Device device : allocation) {
+      if (device.getMinorNumber() == 2) {
+        Assert.assertTrue(true);
+      } else if (device.getMinorNumber() == 3) {
+        Assert.assertTrue(true);
+      } else {
+        Assert.assertTrue("Should allocate GPU 2 and 3", false);
+      }
+    }
+  }
+
+  @Test
+  public void testTopologySchedulingWithSpreadPolicy() throws Exception {
+    NvidiaGPUPluginForRuntimeV2 plugin = mockFourGPUPlugin();
+    NvidiaGPUPluginForRuntimeV2 spyPlugin = spy(plugin);
+    // cache the total devices
+    Set<Device> allDevices = spyPlugin.getDevices();
+    // environment variable to use PACK policy
+    Map<String, String> env = new HashMap<>();
+    env.put(NvidiaGPUPluginForRuntimeV2.TOPOLOGY_POLICY_ENV_KEY,
+        NvidiaGPUPluginForRuntimeV2.TOPOLOGY_POLICY_SPREAD);
+    // Case 1. allocate 1 device
+    Set<Device> allocation = spyPlugin.allocateDevices(allDevices, 1, env);
+    // ensure no topology scheduling needed
+    Assert.assertEquals(allocation.size(), 1);
+    verify(spyPlugin).basicSchedule(anySet(), anyInt(), anySet());
+    reset(spyPlugin);
+    // Case 2. allocate all available
+    allocation = spyPlugin.allocateDevices(allDevices, allDevices.size(), env);
+    Assert.assertEquals(allocation.size(), allDevices.size());
+    verify(spyPlugin).basicSchedule(anySet(), anyInt(), anySet());
+    // Case 3. allocate 2 devices
+    reset(spyPlugin);
+    int count = 2;
+    Map<String, Integer> pairToWeight = spyPlugin.getDevicePairToWeight();
+    allocation = spyPlugin.allocateDevices(allDevices, count, env);
+    Assert.assertEquals(allocation.size(), count);
+    // the costTable should be init and used topology scheduling
+    verify(spyPlugin).initCostTable();
+    Assert.assertTrue(spyPlugin.isTopoInitialized());
+    verify(spyPlugin).topologyAwareSchedule(anySet(), anyInt(), anyMap(),
+        anySet(), anyMap());
+    Assert.assertEquals(allocation.size(), count);
+    Device[] allocatedDevices =
+        allocation.toArray(new Device[count]);
+    // Check weights
+    Assert.assertEquals(NvidiaGPUPluginForRuntimeV2.DeviceLinkType
+        .P2PLinkCrossCPUSocket.getWeight(),
+        spyPlugin.computeCostOfDevices(allocatedDevices));
+    // Case 4. allocate 3 devices
+    reset(spyPlugin);
+    count = 3;
+    allocation = spyPlugin.allocateDevices(allDevices, count, env);
+    Assert.assertEquals(allocation.size(), count);
+    // the costTable should be init and used topology scheduling
+    verify(spyPlugin, times(0)).initCostTable();
+    Assert.assertTrue(spyPlugin.isTopoInitialized());
+    verify(spyPlugin).topologyAwareSchedule(anySet(), anyInt(), anyMap(),
+        anySet(), anyMap());
+    Assert.assertEquals(allocation.size(), count);
+    allocatedDevices =
+        allocation.toArray(new Device[count]);
+    // check weights
+    int expectedWeight =
+        NvidiaGPUPluginForRuntimeV2.DeviceLinkType
+            .P2PLinkSameCPUSocket.getWeight()
+            + 2 * NvidiaGPUPluginForRuntimeV2.DeviceLinkType
+            .P2PLinkCrossCPUSocket.getWeight();
+    Assert.assertEquals(expectedWeight,
+        spyPlugin.computeCostOfDevices(allocatedDevices));
+    // Case 5. allocate 2 GPUs from three available devices
+    reset(spyPlugin);
+    Iterator<Device> iterator = allDevices.iterator();
+    iterator.next();
+    // remove GPU0
+    iterator.remove();
+    count = 2;
+    allocation = spyPlugin.allocateDevices(allDevices, count, env);
+    Assert.assertEquals(allocation.size(), count);
+    // the costTable should be init and used topology scheduling
+    verify(spyPlugin, times(0)).initCostTable();
+    Assert.assertTrue(spyPlugin.isTopoInitialized());
+    verify(spyPlugin).topologyAwareSchedule(anySet(), anyInt(), anyMap(),
+        anySet(), anyMap());
+    Assert.assertEquals(allocation.size(), count);
+    allocatedDevices =
+        allocation.toArray(new Device[count]);
+    // check weights
+    Assert.assertEquals(NvidiaGPUPluginForRuntimeV2.DeviceLinkType
+            .P2PLinkCrossCPUSocket.getWeight(),
+        spyPlugin.computeCostOfDevices(allocatedDevices));
+    // it should allocate GPU 1 and 2
+    for (Device device : allocation) {
+      if (device.getMinorNumber() == 0) {
+        Assert.assertTrue("Shouldn't allocate GPU 0", false);
+      }
+    }
+  }
+
+  @Test
+  public void testCostTableWithNVlink() throws Exception {
+    NvidiaGPUPluginForRuntimeV2 plugin = mockEightGPUPlugin();
+    NvidiaGPUPluginForRuntimeV2 spyPlugin = spy(plugin);
+    // verify the device pair to weight map
+    spyPlugin.initCostTable();
+    Map<String, Integer> devicePairToWeight = spyPlugin.getDevicePairToWeight();
+    // 12 combinations when choose 2 GPUs from 8 respect the order. 8!/6!
+    Assert.assertEquals(56, devicePairToWeight.size());
+    int sameCPUWeight =
+        NvidiaGPUPluginForRuntimeV2.DeviceLinkType
+            .P2PLinkSameCPUSocket.getWeight();
+    int nv1Weight =
+        NvidiaGPUPluginForRuntimeV2.DeviceLinkType
+            .P2PLinkNVLink1.getWeight();
+    int nv2Weight =
+        NvidiaGPUPluginForRuntimeV2.DeviceLinkType
+            .P2PLinkNVLink2.getWeight();
+
+    Assert.assertEquals(nv1Weight, (int)devicePairToWeight.get("0-1"));
+    Assert.assertEquals(nv1Weight, (int)devicePairToWeight.get("1-0"));
+
+    Assert.assertEquals(nv2Weight, (int)devicePairToWeight.get("0-4"));
+    Assert.assertEquals(nv2Weight, (int)devicePairToWeight.get("4-0"));
+
+    Assert.assertEquals(nv2Weight, (int)devicePairToWeight.get("0-3"));
+    Assert.assertEquals(nv2Weight, (int)devicePairToWeight.get("3-0"));
+
+    Assert.assertEquals(sameCPUWeight, (int)devicePairToWeight.get("6-3"));
+    Assert.assertEquals(sameCPUWeight, (int)devicePairToWeight.get("3-6"));
+
+    Assert.assertEquals(nv2Weight, (int)devicePairToWeight.get("6-7"));
+    Assert.assertEquals(nv2Weight, (int)devicePairToWeight.get("7-6"));
+
+    Assert.assertEquals(nv1Weight, (int)devicePairToWeight.get("1-3"));
+    Assert.assertEquals(nv1Weight, (int)devicePairToWeight.get("3-1"));
+
+    // verify cost Table
+    Map<Integer, List<Map.Entry<Set<Device>, Integer>>> costTable =
+        spyPlugin.getCostTable();
+    Assert.assertNull(costTable.get(1));
+    // C8:2 = 8!/2!/6! = 28
+    Assert.assertEquals(28, costTable.get(2).size());
+    // C8:4 = 8!/4!/4! = 70
+    Assert.assertEquals(70, costTable.get(4).size());
+    Assert.assertNull(costTable.get(8));
+
+    Set<Device> allDevices = spyPlugin.getDevices();
+    Map<String, String> env = new HashMap<>();
+    env.put(NvidiaGPUPluginForRuntimeV2.TOPOLOGY_POLICY_ENV_KEY,
+        NvidiaGPUPluginForRuntimeV2.TOPOLOGY_POLICY_PACK);
+    spyPlugin.allocateDevices(allDevices, 3, env);
+    spyPlugin.allocateDevices(allDevices, 2, env);
+  }
+
+  /**
+   * Test the key cost table used for topology scheduling.
+   * */
+  @Test
+  public void testCostTable() throws IOException {
+    NvidiaGPUPluginForRuntimeV2 plugin = mockFourGPUPlugin();
+    NvidiaGPUPluginForRuntimeV2 spyPlugin = spy(plugin);
+    // verify the device pair to weight map
+    spyPlugin.initCostTable();
+    Map<String, Integer> devicePairToWeight = spyPlugin.getDevicePairToWeight();
+    // 12 combinations when choose 2 GPUs from 4 respect the order. 4!/2!
+    Assert.assertEquals(12, devicePairToWeight.size());
+    int sameCPUWeight =
+        NvidiaGPUPluginForRuntimeV2.DeviceLinkType
+            .P2PLinkSameCPUSocket.getWeight();
+    int crossCPUWeight =
+        NvidiaGPUPluginForRuntimeV2.DeviceLinkType
+            .P2PLinkCrossCPUSocket.getWeight();
+    Assert.assertEquals(sameCPUWeight, (int)devicePairToWeight.get("0-1"));
+    Assert.assertEquals(sameCPUWeight, (int)devicePairToWeight.get("1-0"));
+
+    Assert.assertEquals(crossCPUWeight, (int)devicePairToWeight.get("0-2"));
+    Assert.assertEquals(crossCPUWeight, (int)devicePairToWeight.get("2-0"));
+
+    Assert.assertEquals(crossCPUWeight, (int)devicePairToWeight.get("0-3"));
+    Assert.assertEquals(crossCPUWeight, (int)devicePairToWeight.get("3-0"));
+
+    Assert.assertEquals(crossCPUWeight, (int)devicePairToWeight.get("1-2"));
+    Assert.assertEquals(crossCPUWeight, (int)devicePairToWeight.get("2-1"));
+
+    Assert.assertEquals(crossCPUWeight, (int)devicePairToWeight.get("1-3"));
+    Assert.assertEquals(crossCPUWeight, (int)devicePairToWeight.get("3-1"));
+
+    Assert.assertEquals(sameCPUWeight, (int)devicePairToWeight.get("2-3"));
+    Assert.assertEquals(sameCPUWeight, (int)devicePairToWeight.get("3-2"));
+
+    // verify cost Table
+    Map<Integer, List<Map.Entry<Set<Device>, Integer>>> costTable =
+        spyPlugin.getCostTable();
+    Assert.assertNull(costTable.get(1));
+    Assert.assertEquals(6, costTable.get(2).size());
+    Assert.assertEquals(4, costTable.get(3).size());
+    Assert.assertNull(costTable.get(4));
+  }
+  /**
+   * Test GPU topology allocation.
+   * And analysis the GPU allocation's performance against the actual
+   * performance data using tensorflow benchmarks.
+   * https://github.com/tensorflow/benchmarks
+   * */
+  @Test
+  public void testTopologySchedulingPerformanceWithPackPolicyWithNVLink()
+      throws Exception {
+    NvidiaGPUPluginForRuntimeV2 plugin = mockEightGPUPlugin();
+    NvidiaGPUPluginForRuntimeV2 spyPlugin = spy(plugin);
+    Set<Device> allDevices = spyPlugin.getDevices();
+    Map<String, String> env = new HashMap<>();
+    env.put(NvidiaGPUPluginForRuntimeV2.TOPOLOGY_POLICY_ENV_KEY,
+        NvidiaGPUPluginForRuntimeV2.TOPOLOGY_POLICY_PACK);
+
+    /**
+     * Analyze performance against the real data.
+     * Get the topology scheduling algorithm's allocation's
+     * average performance boost against median imagePerSecond and minimum
+     * imagePerSecond in certain model and batch size combinations.
+     * And then calculate the average performance boost.
+     * The average performance boost against
+     * median value means topology scheduler's allocation can stably
+     * outperforms 50% of possible allocations.
+     * The average performance boost against min value means the average boost
+     * comparing to the worst allocations in various scenarios. Which is more
+     * beautiful number for public promotion.
+     * And also the analysis shows the best performance boost against median
+     * and min value.
+     * */
+    ActualPerformanceReport report  = new ActualPerformanceReport();
+    report.readFromFile();
+    ArrayList<ActualPerformanceReport.DataRecord> dataSet =
+        report.getDataSet();
+    Assert.assertEquals(dataSet.size(), 2952);
+    String[] allModels = {"alexnet", "resnet50", "vgg16", "inception3"};
+    int[] batchSizes = {32, 64, 128};
+    int[] gpuCounts = {2, 3, 4, 5, 6, 7};
+    float totalBoostAgainstMedian = 0;
+    int count = 0;
+    float maxBoostAgainstMedian = 0;
+    float totalBoostAgainstMin = 0;
+    float maxBoostAgainstMin = 0;
+    for (String model : allModels) {
+      float totalBoostAgainstMinCertainModel = 0;
+      float totalBoostAgainstMedianCertainModel = 0;
+      float maxBoostAgainstMinCertainModel = 0;
+      float maxBoostAgainstMedianCertainModel = 0;
+      int countOfEachModel = 0;
+      for (int bs : batchSizes) {
+        for (int gpuCount: gpuCounts) {
+          float bstAgainstMedian = calculatePerformanceBoostAgainstMedian(
+              report, model, bs, gpuCount, plugin, allDevices, env);
+          float bstAgainstMinimum = calculatePerformanceBoostAgainstMinimum(
+              report, model, bs, gpuCount, plugin, allDevices, env);
+          totalBoostAgainstMedian += bstAgainstMedian;
+          totalBoostAgainstMin += bstAgainstMinimum;
+          count++;
+          if (maxBoostAgainstMedian < bstAgainstMedian) {
+            maxBoostAgainstMedian = bstAgainstMedian;
+          }
+          if (maxBoostAgainstMin < bstAgainstMinimum) {
+            maxBoostAgainstMin = bstAgainstMinimum;
+          }
+          totalBoostAgainstMinCertainModel += bstAgainstMinimum;
+          totalBoostAgainstMedianCertainModel += bstAgainstMedian;
+          if (maxBoostAgainstMinCertainModel < bstAgainstMinimum) {
+            maxBoostAgainstMinCertainModel = bstAgainstMinimum;
+          }
+          if (maxBoostAgainstMedianCertainModel < bstAgainstMedian) {
+            maxBoostAgainstMedianCertainModel = bstAgainstMedian;
+          }
+          countOfEachModel++;
+        }
+      }
+      LOG.info("Model:{}, The best performance boost against median value is "
+              + "{}", model, maxBoostAgainstMedianCertainModel);
+      LOG.info("Model:{}, The aggregated average performance boost against "
+          + "median value is {}",
+          model, totalBoostAgainstMedianCertainModel/countOfEachModel);
+      LOG.info("Model:{}, The best performance boost against min value is {}",
+          model, maxBoostAgainstMinCertainModel);
+      LOG.info("Model:{}, The aggregated average performance boost against "
+              + "min value is {}",
+          model, totalBoostAgainstMinCertainModel/countOfEachModel);
+    }
+    LOG.info("For all, the best performance boost against median value is "
+        + maxBoostAgainstMedian);
+    LOG.info("For all, the aggregated average performance boost against median "
+        + "value is " + totalBoostAgainstMedian/count);
+    LOG.info("For all, the best performance boost against min value is "
+        + maxBoostAgainstMin);
+    LOG.info("For all, the aggregated average performance boost against min "
+        + "value is " + totalBoostAgainstMin/count);
+  }
+
+  /**
+   * For <code>gpuCount</code> GPUs allocated by the topology algorithm, return
+   * its performance boost against the median value.
+   *
+   * */
+  private float calculatePerformanceBoostAgainstMedian(
+      ActualPerformanceReport report,
+      String model, int bs, int gpuCount,
+      NvidiaGPUPluginForRuntimeV2 plugin, Set<Device> allDevice,
+      Map<String, String> env) {
+    Set<Device> allocation = plugin.allocateDevices(allDevice, gpuCount, env);
+    String gpuAllocationString = convertAllocationToGpuString(allocation);
+    float[] metrics = report.getVariousImagePerSecond(model, bs,
+        gpuCount, gpuAllocationString);
+    return metrics[7];
+  }
+
+  /**
+   * For <code>gpuCount</code> GPUs allocated by the topology algorithm, return
+   * its performance boost against the minimum value.
+   *
+   * */
+  private float calculatePerformanceBoostAgainstMinimum(
+      ActualPerformanceReport report,
+      String model, int bs, int gpuCount,
+      NvidiaGPUPluginForRuntimeV2 plugin, Set<Device> allDevice,
+      Map<String, String> env) {
+    Set<Device> allocation = plugin.allocateDevices(allDevice, gpuCount, env);
+    String gpuAllocationString = convertAllocationToGpuString(allocation);
+    float[] metrics = report.getVariousImagePerSecond(model, bs,
+        gpuCount, gpuAllocationString);
+    return metrics[5];
+  }
+
+  private String convertAllocationToGpuString(Set<Device> allocation) {
+    StringBuilder sb = new StringBuilder();
+    for (Device device : allocation) {
+      sb.append(device.getMinorNumber() + "_");
+    }
+    return sb.toString().substring(0, sb.lastIndexOf("_"));
+  }
+
+  /**
+   * Representation of the performance data report.
+   * */
+  private class ActualPerformanceReport {
+
+    private ArrayList<DataRecord> dataSet = new ArrayList<>();
+
+    public ArrayList<DataRecord> getDataSet() {
+      return dataSet;
+    }
+
+    /**
+     * One line in the report.
+     * */
+    private class DataRecord {
+      DataRecord(String model, int bs, String combination, float fps,
+          int count) {
+        this.batchSize = bs;
+        this.gpuCombination = combination;
+        this.gpuCount = count;
+        this.model = model;
+        this.imagePerSecond = fps;
+      }
+
+      public String getModel() {
+        return model;
+      }
+
+      public int getBatchSize() {
+        return batchSize;
+      }
+
+      public String getGpuCombination() {
+        return gpuCombination;
+      }
+
+      public float getImagePerSecond() {
+        return imagePerSecond;
+      }
+
+      public int getGpuCount() {
+        return gpuCount;
+      }
+
+      private String model;
+      private int batchSize;
+      private String gpuCombination;
+      private float imagePerSecond;
+      private int gpuCount;
+    }
+
+    /**
+     * The file is a real performance report got from a 8 GPUs AWS instance.
+     * It contains every combination GPUs' training performance of Tensorflow
+     * benchmark.
+     * The columns are the model name, batch size, gpu ids and imagesPerSecond
+     * */
+    public void readFromFile() {
+      String csvReportFilePath = getClass().getClassLoader()
+          .getResource("tensorflow-bench-result-for-GPU.csv").getFile();
+      BufferedReader br = null;
+      String line = "";
+      try {
+        br = new BufferedReader(new FileReader(csvReportFilePath));
+        String model;
+        int batchSize;
+        String gpuCombination;
+        float imagePerSecond;
+        int gpuCount;
+        while ((line = br.readLine()) != null) {
+          // skip the licence content
+          if (line.startsWith("#")) {
+            continue;
+          }
+          String[] tokens = line.replaceAll("\"", "").split(",");
+          if (tokens.length != 4) {
+            LOG.error("unexpected performance data format!");
+            break;
+          }
+          model = tokens[0];
+          batchSize = Integer.parseInt(tokens[1].trim());
+          gpuCombination = tokens[2];
+          imagePerSecond = Float.parseFloat(tokens[3]);
+          gpuCount = getGpuCount(gpuCombination);
+          this.dataSet.add(new DataRecord(model, batchSize, gpuCombination,
+              imagePerSecond, gpuCount));
+        }
+      } catch (Exception e) {
+        e.printStackTrace();
+      } finally {
+        if (br != null) {
+          try {
+            br.close();
+          } catch (IOException e) {
+            e.printStackTrace();
+          }
+        }
+      } // end finally
+    }
+
+    /**
+     * Return the maximum, minimum, mean and median performance for model &
+     * bs & gpuCount. And the imagePerSecond for model & bs & gpuCount &
+     * gpuCombinations. And imagePerSecond performance boost comparing to
+     * minimum, mean and media value.
+     * */
+    private float[] getVariousImagePerSecond(String model, int bs,
+        int gpuCount, String gpuCombinations) {
+      float[] result = new float[8];
+      float max = 0;
+      float min = Float.MAX_VALUE;
+      float sum = 0;
+      int count = 0;
+      float wantedImagePerSecond = 0;
+      float currentImagePerSecond;
+      ArrayList<Float> allFps = new ArrayList<>();
+      for (DataRecord dr : getDataSet()) {
+        currentImagePerSecond = dr.getImagePerSecond();
+        if (dr.batchSize == bs
+            && model.equals(dr.getModel())
+            && gpuCount == dr.getGpuCount()) {
+          sum += currentImagePerSecond;
+          count++;
+          if (max < currentImagePerSecond) {
+            max = currentImagePerSecond;
+          }
+          if (min > currentImagePerSecond) {
+            min = currentImagePerSecond;
+          }
+          if (gpuCombinations.equals(dr.getGpuCombination())) {
+            wantedImagePerSecond = dr.getImagePerSecond();
+          }
+          allFps.add(dr.getImagePerSecond());
+        }
+      }
+      float median = getMedian(allFps);
+      float mean = sum/count;
+      result[0] = max;
+      result[1] = min;
+      result[2] = mean;
+      result[3] = median;
+      result[4] = wantedImagePerSecond;
+      result[5] = wantedImagePerSecond/min - 1;
+      result[6] = wantedImagePerSecond/mean - 1;
+      result[7] = wantedImagePerSecond/median - 1;
+      return result;
+    }
+
+    private float getMedian(ArrayList<Float> allFps) {
+      float[] all = ArrayUtils.toPrimitive(allFps.toArray(new Float[0]), 0);
+      Arrays.sort(all);
+      float median;
+      int size = all.length;
+      if (allFps.size() % 2 == 0) {
+        median = (all[size/2] + all[size/2 - 1])/2;
+      } else {
+        median = all[size/2];
+      }
+      return median;
+    }
+
+    private int getGpuCount(String gpuCombination) {
+      String[] tokens = gpuCombination.split("_");
+      return tokens.length;
+    }
+  }
+
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin1.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugi
 
 import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.*;
 
+import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 
@@ -62,7 +63,7 @@ public class FakeTestDevicePlugin1
 
   @Override
   public Set<Device> allocateDevices(Set<Device> availableDevices,
-      int count) {
+      int count, Map<String, String> env) {
     Set<Device> allocated = new TreeSet<Device>();
     int number = 0;
     for (Device d : availableDevices) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDevicePluginAdapter.java

@@ -74,10 +74,11 @@ import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
 
-
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyMap;
+import static org.mockito.ArgumentMatchers.anySet;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.ArgumentMatchers.isA;
@@ -571,7 +572,7 @@ public class TestDevicePluginAdapter {
     adapter.getDeviceResourceHandler().preStart(c1);
     // Use customized scheduler
     verify(spyPlugin, times(1)).allocateDevices(
-        any(Set.class), anyInt());
+        anySet(), anyInt(), anyMap());
     Assert.assertEquals(2,
         dmm.getAvailableDevices(resourceName));
     Assert.assertEquals(1,
@@ -995,7 +996,7 @@ public class TestDevicePluginAdapter {
 
     @Override
     public Set<Device> allocateDevices(Set<Device> availableDevices,
-        int count) {
+        int count, Map<String, String> env) {
       Set<Device> allocated = new TreeSet<>();
       int number = 0;
       for (Device d : availableDevices) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/nvidia/com/TestNvidiaGpuPlugin.java

@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.nvidia.com;
-
-import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device;
-import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRuntimeSpec;
-import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.YarnRuntimeType;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.com.nvidia.NvidiaGPUPluginForRuntimeV2;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.Set;
-import java.util.TreeSet;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * Test case for Nvidia GPU device plugin.
- * */
-public class TestNvidiaGpuPlugin {
-
-  @Test
-  public void testGetNvidiaDevices() throws Exception {
-    NvidiaGPUPluginForRuntimeV2.NvidiaCommandExecutor mockShell =
-        mock(NvidiaGPUPluginForRuntimeV2.NvidiaCommandExecutor.class);
-    String deviceInfoShellOutput =
-        "0, 00000000:04:00.0\n" +
-        "1, 00000000:82:00.0";
-    String majorMinorNumber0 = "c3:0";
-    String majorMinorNumber1 = "c3:1";
-    when(mockShell.getDeviceInfo()).thenReturn(deviceInfoShellOutput);
-    when(mockShell.getMajorMinorInfo("nvidia0"))
-        .thenReturn(majorMinorNumber0);
-    when(mockShell.getMajorMinorInfo("nvidia1"))
-        .thenReturn(majorMinorNumber1);
-    NvidiaGPUPluginForRuntimeV2 plugin = new NvidiaGPUPluginForRuntimeV2();
-    plugin.setShellExecutor(mockShell);
-    plugin.setPathOfGpuBinary("/fake/nvidia-smi");
-
-    Set<Device> expectedDevices = new TreeSet<>();
-    expectedDevices.add(Device.Builder.newInstance()
-        .setId(0).setHealthy(true)
-        .setBusID("00000000:04:00.0")
-        .setDevPath("/dev/nvidia0")
-        .setMajorNumber(195)
-        .setMinorNumber(0).build());
-    expectedDevices.add(Device.Builder.newInstance()
-        .setId(1).setHealthy(true)
-        .setBusID("00000000:82:00.0")
-        .setDevPath("/dev/nvidia1")
-        .setMajorNumber(195)
-        .setMinorNumber(1).build());
-    Set<Device> devices = plugin.getDevices();
-    Assert.assertEquals(expectedDevices, devices);
-  }
-
-  @Test
-  public void testOnDeviceAllocated() throws Exception {
-    NvidiaGPUPluginForRuntimeV2 plugin = new NvidiaGPUPluginForRuntimeV2();
-    Set<Device> allocatedDevices = new TreeSet<>();
-
-    DeviceRuntimeSpec spec = plugin.onDevicesAllocated(allocatedDevices,
-        YarnRuntimeType.RUNTIME_DEFAULT);
-    Assert.assertNull(spec);
-
-    // allocate one device
-    allocatedDevices.add(Device.Builder.newInstance()
-        .setId(0).setHealthy(true)
-        .setBusID("00000000:04:00.0")
-        .setDevPath("/dev/nvidia0")
-        .setMajorNumber(195)
-        .setMinorNumber(0).build());
-    spec = plugin.onDevicesAllocated(allocatedDevices,
-        YarnRuntimeType.RUNTIME_DOCKER);
-    Assert.assertEquals("nvidia", spec.getContainerRuntime());
-    Assert.assertEquals("0", spec.getEnvs().get("NVIDIA_VISIBLE_DEVICES"));
-
-    // two device allowed
-    allocatedDevices.add(Device.Builder.newInstance()
-        .setId(0).setHealthy(true)
-        .setBusID("00000000:82:00.0")
-        .setDevPath("/dev/nvidia1")
-        .setMajorNumber(195)
-        .setMinorNumber(1).build());
-    spec = plugin.onDevicesAllocated(allocatedDevices,
-        YarnRuntimeType.RUNTIME_DOCKER);
-    Assert.assertEquals("nvidia", spec.getContainerRuntime());
-    Assert.assertEquals("0,1", spec.getEnvs().get("NVIDIA_VISIBLE_DEVICES"));
-
-  }
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/tensorflow-bench-result-for-GPU.csv

@@ -0,0 +1,2963 @@
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+"alexnet","32","0_2","2910.87"
+"resnet50","32","0_2","601.82"
+"vgg16","32","0_2","382.83"
+"inception3","32","0_2","392.53"
+"alexnet","64","0_2","4539.27"
+"resnet50","64","0_2","699.90"
+"vgg16","64","0_2","428.95"
+"inception3","64","0_2","435.53"
+"alexnet","128","0_2","6319.08"
+"resnet50","128","0_2","781.39"
+"vgg16","128","0_2","454.85"
+"inception3","128","0_2","456.94"
+"alexnet","32","6_7","3922.30"
+"resnet50","32","6_7","588.72"
+"vgg16","32","6_7","406.52"
+"inception3","32","6_7","380.24"
+"alexnet","64","6_7","5693.85"
+"resnet50","64","6_7","694.92"
+"vgg16","64","6_7","437.44"
+"inception3","64","6_7","429.96"
+"alexnet","128","6_7","7226.03"
+"resnet50","128","6_7","764.63"
+"vgg16","128","6_7","451.29"
+"inception3","128","6_7","454.66"
+"alexnet","32","1_6","1552.58"
+"resnet50","32","1_6","558.30"
+"vgg16","32","1_6","315.01"
+"inception3","32","1_6","375.01"
+"alexnet","64","1_6","2829.13"
+"resnet50","64","1_6","680.00"
+"vgg16","64","1_6","382.46"
+"inception3","64","1_6","427.11"
+"alexnet","128","1_6","4448.61"
+"resnet50","128","1_6","767.72"
+"vgg16","128","1_6","425.83"
+"inception3","128","1_6","453.58"
+"alexnet","32","0_7","1559.17"
+"resnet50","32","0_7","562.46"
+"vgg16","32","0_7","313.64"
+"inception3","32","0_7","379.64"
+"alexnet","64","0_7","2861.61"
+"resnet50","64","0_7","673.91"
+"vgg16","64","0_7","381.18"
+"inception3","64","0_7","424.98"
+"alexnet","128","0_7","4453.04"
+"resnet50","128","0_7","754.64"
+"vgg16","128","0_7","419.01"
+"inception3","128","0_7","447.85"
+"alexnet","32","2_7","1565.90"
+"resnet50","32","2_7","563.24"
+"vgg16","32","2_7","310.85"
+"inception3","32","2_7","377.70"
+"alexnet","64","2_7","2837.95"
+"resnet50","64","2_7","672.73"
+"vgg16","64","2_7","377.73"
+"inception3","64","2_7","428.27"
+"alexnet","128","2_7","4462.88"
+"resnet50","128","2_7","759.73"
+"vgg16","128","2_7","418.70"
+"inception3","128","2_7","453.87"
+"alexnet","32","3_4","1564.28"
+"resnet50","32","3_4","562.02"
+"vgg16","32","3_4","314.93"
+"inception3","32","3_4","379.44"
+"alexnet","64","3_4","2852.29"
+"resnet50","64","3_4","675.38"
+"vgg16","64","3_4","382.05"
+"inception3","64","3_4","429.81"
+"alexnet","128","3_4","4496.05"
+"resnet50","128","3_4","759.31"
+"vgg16","128","3_4","423.16"
+"inception3","128","3_4","455.16"
+"alexnet","32","1_2","3907.10"
+"resnet50","32","1_2","600.94"
+"vgg16","32","1_2","409.16"
+"inception3","32","1_2","390.87"
+"alexnet","64","1_2","5738.23"
+"resnet50","64","1_2","701.38"
+"vgg16","64","1_2","446.87"
+"inception3","64","1_2","437.65"
+"alexnet","128","1_2","7260.26"
+"resnet50","128","1_2","780.78"
+"vgg16","128","1_2","466.06"
+"inception3","128","1_2","464.19"
+"alexnet","32","0_3","3896.77"
+"resnet50","32","0_3","595.41"
+"vgg16","32","0_3","409.77"
+"inception3","32","0_3","390.42"
+"alexnet","64","0_3","5754.24"
+"resnet50","64","0_3","700.06"
+"vgg16","64","0_3","443.89"
+"inception3","64","0_3","435.08"
+"alexnet","128","0_3","7257.56"
+"resnet50","128","0_3","780.39"
+"vgg16","128","0_3","458.60"
+"inception3","128","0_3","457.85"
+"alexnet","32","2_3","3900.87"
+"resnet50","32","2_3","583.55"
+"vgg16","32","2_3","406.56"
+"inception3","32","2_3","380.68"
+"alexnet","64","2_3","5718.20"
+"resnet50","64","2_3","695.97"
+"vgg16","64","2_3","442.44"
+"inception3","64","2_3","431.38"
+"alexnet","128","2_3","7240.75"
+"resnet50","128","2_3","775.64"
+"vgg16","128","2_3","464.23"
+"inception3","128","2_3","457.18"
+"alexnet","32","0_4","3840.64"
+"resnet50","32","0_4","600.71"
+"vgg16","32","0_4","410.09"
+"inception3","32","0_4","390.52"
+"alexnet","64","0_4","5718.76"
+"resnet50","64","0_4","704.05"
+"vgg16","64","0_4","443.72"
+"inception3","64","0_4","435.45"
+"alexnet","128","0_4","7237.24"
+"resnet50","128","0_4","780.98"
+"vgg16","128","0_4","459.58"
+"inception3","128","0_4","456.95"
+"alexnet","32","1_5","3883.02"
+"resnet50","32","1_5","593.46"
+"vgg16","32","1_5","410.63"
+"inception3","32","1_5","388.79"
+"alexnet","64","1_5","5715.07"
+"resnet50","64","1_5","703.81"
+"vgg16","64","1_5","445.57"
+"inception3","64","1_5","436.14"
+"alexnet","128","1_5","7264.90"
+"resnet50","128","1_5","781.55"
+"vgg16","128","1_5","463.32"
+"inception3","128","1_5","462.65"
+"alexnet","32","2_4","1559.84"
+"resnet50","32","2_4","565.44"
+"vgg16","32","2_4","317.92"
+"inception3","32","2_4","379.84"
+"alexnet","64","2_4","2857.86"
+"resnet50","64","2_4","672.61"
+"vgg16","64","2_4","382.30"
+"inception3","64","2_4","429.39"
+"alexnet","128","2_4","4479.28"
+"resnet50","128","2_4","761.21"
+"vgg16","128","2_4","425.00"
+"inception3","128","2_4","455.17"
+"alexnet","32","3_7","2912.16"
+"resnet50","32","3_7","602.43"
+"vgg16","32","3_7","379.59"
+"inception3","32","3_7","391.27"
+"alexnet","64","3_7","4548.95"
+"resnet50","64","3_7","704.63"
+"vgg16","64","3_7","421.16"
+"inception3","64","3_7","437.74"
+"alexnet","128","3_7","6292.67"
+"resnet50","128","3_7","770.04"
+"vgg16","128","3_7","443.30"
+"inception3","128","3_7","459.13"
+"alexnet","32","5_7","2903.06"
+"resnet50","32","5_7","594.05"
+"vgg16","32","5_7","378.23"
+"inception3","32","5_7","388.99"
+"alexnet","64","5_7","4528.86"
+"resnet50","64","5_7","696.01"
+"vgg16","64","5_7","422.22"
+"inception3","64","5_7","432.35"
+"alexnet","128","5_7","6298.93"
+"resnet50","128","5_7","772.53"
+"vgg16","128","5_7","443.27"
+"inception3","128","5_7","460.91"
+"alexnet","32","4_6","2909.87"
+"resnet50","32","4_6","603.09"
+"vgg16","32","4_6","382.92"
+"inception3","32","4_6","391.43"
+"alexnet","64","4_6","4529.73"
+"resnet50","64","4_6","700.33"
+"vgg16","64","4_6","427.43"
+"inception3","64","4_6","431.65"
+"alexnet","128","4_6","6313.76"
+"resnet50","128","4_6","779.77"
+"vgg16","128","4_6","451.37"
+"inception3","128","4_6","455.94"
+"alexnet","32","4_5","2909.74"
+"resnet50","32","4_5","585.69"
+"vgg16","32","4_5","382.49"
+"inception3","32","4_5","379.25"
+"alexnet","64","4_5","4539.81"
+"resnet50","64","4_5","691.43"
+"vgg16","64","4_5","427.21"
+"inception3","64","4_5","428.58"
+"alexnet","128","4_5","6287.64"
+"resnet50","128","4_5","777.12"
+"vgg16","128","4_5","452.45"
+"inception3","128","4_5","449.28"
+"alexnet","32","0_5","1556.36"
+"resnet50","32","0_5","558.62"
+"vgg16","32","0_5","313.98"
+"inception3","32","0_5","380.35"
+"alexnet","64","0_5","2848.95"
+"resnet50","64","0_5","672.35"
+"vgg16","64","0_5","382.56"
+"inception3","64","0_5","426.09"
+"alexnet","128","0_5","4453.50"
+"resnet50","128","0_5","767.17"
+"vgg16","128","0_5","426.69"
+"inception3","128","0_5","446.46"
+"alexnet","32","1_4","1551.61"
+"resnet50","32","1_4","565.30"
+"vgg16","32","1_4","317.38"
+"inception3","32","1_4","380.51"
+"alexnet","64","1_4","2837.42"
+"resnet50","64","1_4","676.73"
+"vgg16","64","1_4","381.76"
+"inception3","64","1_4","433.22"
+"alexnet","128","1_4","4458.53"
+"resnet50","128","1_4","761.51"
+"vgg16","128","1_4","422.85"
+"inception3","128","1_4","459.32"
+"alexnet","32","0_1","2918.37"
+"resnet50","32","0_1","589.76"
+"vgg16","32","0_1","381.40"
+"inception3","32","0_1","386.02"
+"alexnet","64","0_1","4552.76"
+"resnet50","64","0_1","696.71"
+"vgg16","64","0_1","425.30"
+"inception3","64","0_1","426.34"
+"alexnet","128","0_1","6295.42"
+"resnet50","128","0_1","776.13"
+"vgg16","128","0_1","450.08"
+"inception3","128","0_1","453.51"
+"alexnet","32","2_5","1558.34"
+"resnet50","32","2_5","562.50"
+"vgg16","32","2_5","314.97"
+"inception3","32","2_5","377.27"
+"alexnet","64","2_5","2849.74"
+"resnet50","64","2_5","676.88"
+"vgg16","64","2_5","382.05"
+"inception3","64","2_5","426.25"
+"alexnet","128","2_5","4453.20"
+"resnet50","128","2_5","766.82"
+"vgg16","128","2_5","429.61"
+"inception3","128","2_5","452.35"
+"alexnet","32","3_6","1563.11"
+"resnet50","32","3_6","558.04"
+"vgg16","32","3_6","314.65"
+"inception3","32","3_6","377.83"
+"alexnet","64","3_6","2859.37"
+"resnet50","64","3_6","680.86"
+"vgg16","64","3_6","384.07"
+"inception3","64","3_6","426.29"
+"alexnet","128","3_6","4454.93"
+"resnet50","128","3_6","767.93"
+"vgg16","128","3_6","426.04"
+"inception3","128","3_6","456.97"
+"alexnet","32","5_6","3921.50"
+"resnet50","32","5_6","605.61"
+"vgg16","32","5_6","409.56"
+"inception3","32","5_6","390.59"
+"alexnet","64","5_6","5666.52"
+"resnet50","64","5_6","702.71"
+"vgg16","64","5_6","442.91"
+"inception3","64","5_6","434.45"
+"alexnet","128","5_6","7265.78"
+"resnet50","128","5_6","776.79"
+"vgg16","128","5_6","462.54"
+"inception3","128","5_6","463.67"
+"alexnet","32","4_7","3827.15"
+"resnet50","32","4_7","595.29"
+"vgg16","32","4_7","407.49"
+"inception3","32","4_7","392.00"
+"alexnet","64","4_7","5736.73"
+"resnet50","64","4_7","699.92"
+"vgg16","64","4_7","437.92"
+"inception3","64","4_7","433.93"
+"alexnet","128","4_7","7241.76"
+"resnet50","128","4_7","769.96"
+"vgg16","128","4_7","451.85"
+"inception3","128","4_7","461.71"
+"alexnet","32","3_5","1545.52"
+"resnet50","32","3_5","562.12"
+"vgg16","32","3_5","314.16"
+"inception3","32","3_5","377.18"
+"alexnet","64","3_5","2821.83"
+"resnet50","64","3_5","676.41"
+"vgg16","64","3_5","381.80"
+"inception3","64","3_5","427.90"
+"alexnet","128","3_5","4442.79"
+"resnet50","128","3_5","767.63"
+"vgg16","128","3_5","423.39"
+"inception3","128","3_5","454.34"
+"alexnet","32","1_7","1553.76"
+"resnet50","32","1_7","564.63"
+"vgg16","32","1_7","316.17"
+"inception3","32","1_7","377.83"
+"alexnet","64","1_7","2835.24"
+"resnet50","64","1_7","675.65"
+"vgg16","64","1_7","377.14"
+"inception3","64","1_7","430.26"
+"alexnet","128","1_7","4447.93"
+"resnet50","128","1_7","756.78"
+"vgg16","128","1_7","418.48"
+"inception3","128","1_7","451.21"
+"alexnet","32","0_6","1562.12"
+"resnet50","32","0_6","561.32"
+"vgg16","32","0_6","317.29"
+"inception3","32","0_6","379.52"
+"alexnet","64","0_6","2846.76"
+"resnet50","64","0_6","677.32"
+"vgg16","64","0_6","383.31"
+"inception3","64","0_6","425.92"
+"alexnet","128","0_6","4468.00"
+"resnet50","128","0_6","762.83"
+"vgg16","128","0_6","427.33"
+"inception3","128","0_6","447.95"
+"alexnet","32","2_6","2906.72"
+"resnet50","32","2_6","598.11"
+"vgg16","32","2_6","380.97"
+"inception3","32","2_6","389.59"
+"alexnet","64","2_6","4551.68"
+"resnet50","64","2_6","701.91"
+"vgg16","64","2_6","427.35"
+"inception3","64","2_6","434.31"
+"alexnet","128","2_6","6309.10"
+"resnet50","128","2_6","787.22"
+"vgg16","128","2_6","454.29"
+"inception3","128","2_6","466.53"
+"alexnet","32","1_3","2900.56"
+"resnet50","32","1_3","602.97"
+"vgg16","32","1_3","382.61"
+"inception3","32","1_3","388.50"
+"alexnet","64","1_3","4508.73"
+"resnet50","64","1_3","700.32"
+"vgg16","64","1_3","427.10"
+"inception3","64","1_3","438.16"
+"alexnet","128","1_3","6313.84"
+"resnet50","128","1_3","775.58"
+"vgg16","128","1_3","449.99"
+"inception3","128","1_3","462.13"
+"alexnet","32","0_2_4","2079.28"
+"resnet50","32","0_2_4","834.00"
+"vgg16","32","0_2_4","491.30"
+"inception3","32","0_2_4","588.88"
+"alexnet","64","0_2_4","3952.09"
+"resnet50","64","0_2_4","1001.06"
+"vgg16","64","0_2_4","589.62"
+"inception3","64","0_2_4","688.75"
+"alexnet","128","0_2_4","6740.01"
+"resnet50","128","0_2_4","1148.73"
+"vgg16","128","0_2_4","649.41"
+"inception3","128","0_2_4","737.81"
+"alexnet","32","2_4_7","1749.71"
+"resnet50","32","2_4_7","809.35"
+"vgg16","32","2_4_7","487.14"
+"inception3","32","2_4_7","580.21"
+"alexnet","64","2_4_7","3290.68"
+"resnet50","64","2_4_7","993.30"
+"vgg16","64","2_4_7","583.16"
+"inception3","64","2_4_7","683.73"
+"alexnet","128","2_4_7","5716.17"
+"resnet50","128","2_4_7","1123.65"
+"vgg16","128","2_4_7","639.42"
+"inception3","128","2_4_7","734.48"
+"alexnet","32","1_5_6","3357.70"
+"resnet50","32","1_5_6","837.21"
+"vgg16","32","1_5_6","493.92"
+"inception3","32","1_5_6","592.50"
+"alexnet","64","1_5_6","5624.02"
+"resnet50","64","1_5_6","998.43"
+"vgg16","64","1_5_6","589.51"
+"inception3","64","1_5_6","680.17"
+"alexnet","128","1_5_6","8399.25"
+"resnet50","128","1_5_6","1141.93"
+"vgg16","128","1_5_6","648.14"
+"inception3","128","1_5_6","749.69"
+"alexnet","32","0_2_5","1798.27"
+"resnet50","32","0_2_5","800.91"
+"vgg16","32","0_2_5","325.78"
+"inception3","32","0_2_5","570.46"
+"alexnet","64","0_2_5","3366.59"
+"resnet50","64","0_2_5","980.00"
+"vgg16","64","0_2_5","380.18"
+"inception3","64","0_2_5","672.99"
+"alexnet","128","0_2_5","5723.50"
+"resnet50","128","0_2_5","1131.75"
+"vgg16","128","0_2_5","408.50"
+"inception3","128","0_2_5","735.19"
+"alexnet","32","1_2_3","4740.05"
+"resnet50","32","1_2_3","856.45"
+"vgg16","32","1_2_3","580.20"
+"inception3","32","1_2_3","580.64"
+"alexnet","64","1_2_3","7440.68"
+"resnet50","64","1_2_3","1009.10"
+"vgg16","64","1_2_3","648.38"
+"inception3","64","1_2_3","675.79"
+"alexnet","128","1_2_3","10143.99"
+"resnet50","128","1_2_3","1139.79"
+"vgg16","128","1_2_3","682.84"
+"inception3","128","1_2_3","737.75"
+"alexnet","32","1_5_7","3286.33"
+"resnet50","32","1_5_7","837.23"
+"vgg16","32","1_5_7","485.32"
+"inception3","32","1_5_7","588.48"
+"alexnet","64","1_5_7","5774.49"
+"resnet50","64","1_5_7","1010.34"
+"vgg16","64","1_5_7","585.79"
+"inception3","64","1_5_7","678.16"
+"alexnet","128","1_5_7","9127.83"
+"resnet50","128","1_5_7","1123.67"
+"vgg16","128","1_5_7","636.31"
+"inception3","128","1_5_7","738.73"
+"alexnet","32","1_2_6","3292.95"
+"resnet50","32","1_2_6","827.72"
+"vgg16","32","1_2_6","486.64"
+"inception3","32","1_2_6","578.88"
+"alexnet","64","1_2_6","5832.64"
+"resnet50","64","1_2_6","1002.99"
+"vgg16","64","1_2_6","588.80"
+"inception3","64","1_2_6","681.65"
+"alexnet","128","1_2_6","9206.24"
+"resnet50","128","1_2_6","1146.75"
+"vgg16","128","1_2_6","651.40"
+"inception3","128","1_2_6","735.15"
+"alexnet","32","4_5_6","3735.86"
+"resnet50","32","4_5_6","860.70"
+"vgg16","32","4_5_6","577.48"
+"inception3","32","4_5_6","584.22"
+"alexnet","64","4_5_6","6622.13"
+"resnet50","64","4_5_6","1012.54"
+"vgg16","64","4_5_6","642.65"
+"inception3","64","4_5_6","671.19"
+"alexnet","128","4_5_6","9360.12"
+"resnet50","128","4_5_6","1152.27"
+"vgg16","128","4_5_6","680.11"
+"inception3","128","4_5_6","734.46"
+"alexnet","32","2_4_5","1670.03"
+"resnet50","32","2_4_5","798.93"
+"vgg16","32","2_4_5","464.88"
+"inception3","32","2_4_5","566.10"
+"alexnet","64","2_4_5","3169.32"
+"resnet50","64","2_4_5","969.10"
+"vgg16","64","2_4_5","569.22"
+"inception3","64","2_4_5","667.64"
+"alexnet","128","2_4_5","5550.79"
+"resnet50","128","2_4_5","1111.76"
+"vgg16","128","2_4_5","634.23"
+"inception3","128","2_4_5","737.61"
+"alexnet","32","3_5_6","1743.34"
+"resnet50","32","3_5_6","816.58"
+"vgg16","32","3_5_6","490.76"
+"inception3","32","3_5_6","576.92"
+"alexnet","64","3_5_6","3291.97"
+"resnet50","64","3_5_6","991.56"
+"vgg16","64","3_5_6","588.85"
+"inception3","64","3_5_6","682.14"
+"alexnet","128","3_5_6","5785.63"
+"resnet50","128","3_5_6","1133.08"
+"vgg16","128","3_5_6","645.07"
+"inception3","128","3_5_6","737.65"
+"alexnet","32","0_2_3","3958.30"
+"resnet50","32","0_2_3","849.42"
+"vgg16","32","0_2_3","602.72"
+"inception3","32","0_2_3","586.70"
+"alexnet","64","0_2_3","6925.28"
+"resnet50","64","0_2_3","1011.47"
+"vgg16","64","0_2_3","660.90"
+"inception3","64","0_2_3","683.35"
+"alexnet","128","0_2_3","9573.54"
+"resnet50","128","0_2_3","1150.96"
+"vgg16","128","0_2_3","694.72"
+"inception3","128","0_2_3","738.13"
+"alexnet","32","1_4_5","2171.67"
+"resnet50","32","1_4_5","836.67"
+"vgg16","32","1_4_5","564.63"
+"inception3","32","1_4_5","582.91"
+"alexnet","64","1_4_5","4186.81"
+"resnet50","64","1_4_5","1006.95"
+"vgg16","64","1_4_5","638.46"
+"inception3","64","1_4_5","685.53"
+"alexnet","128","1_4_5","7010.65"
+"resnet50","128","1_4_5","1129.47"
+"vgg16","128","1_4_5","677.29"
+"inception3","128","1_4_5","742.54"
+"alexnet","32","3_4_6","1687.37"
+"resnet50","32","3_4_6","811.04"
+"vgg16","32","3_4_6","467.50"
+"inception3","32","3_4_6","584.26"
+"alexnet","64","3_4_6","3146.96"
+"resnet50","64","3_4_6","1000.81"
+"vgg16","64","3_4_6","576.32"
+"inception3","64","3_4_6","680.45"
+"alexnet","128","3_4_6","5577.87"
+"resnet50","128","3_4_6","1128.46"
+"vgg16","128","3_4_6","636.72"
+"inception3","128","3_4_6","743.28"
+"alexnet","32","2_3_7","3319.17"
+"resnet50","32","2_3_7","833.32"
+"vgg16","32","2_3_7","483.87"
+"inception3","32","2_3_7","568.71"
+"alexnet","64","2_3_7","5847.58"
+"resnet50","64","2_3_7","1003.06"
+"vgg16","64","2_3_7","583.99"
+"inception3","64","2_3_7","666.59"
+"alexnet","128","2_3_7","9140.01"
+"resnet50","128","2_3_7","1130.20"
+"vgg16","128","2_3_7","644.36"
+"inception3","128","2_3_7","739.91"
+"alexnet","32","1_6_7","1748.71"
+"resnet50","32","1_6_7","802.21"
+"vgg16","32","1_6_7","485.80"
+"inception3","32","1_6_7","561.86"
+"alexnet","64","1_6_7","3275.24"
+"resnet50","64","1_6_7","971.70"
+"vgg16","64","1_6_7","578.55"
+"inception3","64","1_6_7","664.75"
+"alexnet","128","1_6_7","5612.43"
+"resnet50","128","1_6_7","1108.39"
+"vgg16","128","1_6_7","628.97"
+"inception3","128","1_6_7","725.26"
+"alexnet","32","3_5_7","2133.23"
+"resnet50","32","3_5_7","849.15"
+"vgg16","32","3_5_7","553.56"
+"inception3","32","3_5_7","600.06"
+"alexnet","64","3_5_7","4050.74"
+"resnet50","64","3_5_7","1016.88"
+"vgg16","64","3_5_7","627.91"
+"inception3","64","3_5_7","691.22"
+"alexnet","128","3_5_7","7231.94"
+"resnet50","128","3_5_7","1137.54"
+"vgg16","128","3_5_7","664.70"
+"inception3","128","3_5_7","740.57"
+"alexnet","32","2_3_6","2234.63"
+"resnet50","32","2_3_6","831.59"
+"vgg16","32","2_3_6","480.03"
+"inception3","32","2_3_6","572.81"
+"alexnet","64","2_3_6","4169.79"
+"resnet50","64","2_3_6","1007.82"
+"vgg16","64","2_3_6","586.29"
+"inception3","64","2_3_6","677.11"
+"alexnet","128","2_3_6","6553.91"
+"resnet50","128","2_3_6","1145.83"
+"vgg16","128","2_3_6","652.69"
+"inception3","128","2_3_6","732.84"
+"alexnet","32","0_3_5","1937.14"
+"resnet50","32","0_3_5","806.17"
+"vgg16","32","0_3_5","328.01"
+"inception3","32","0_3_5","570.30"
+"alexnet","64","0_3_5","3578.00"
+"resnet50","64","0_3_5","981.25"
+"vgg16","64","0_3_5","379.53"
+"inception3","64","0_3_5","666.86"
+"alexnet","128","0_3_5","6052.57"
+"resnet50","128","0_3_5","1118.65"
+"vgg16","128","0_3_5","406.99"
+"inception3","128","0_3_5","732.19"
+"alexnet","32","2_3_5","1926.55"
+"resnet50","32","2_3_5","805.04"
+"vgg16","32","2_3_5","326.23"
+"inception3","32","2_3_5","558.61"
+"alexnet","64","2_3_5","3547.90"
+"resnet50","64","2_3_5","977.43"
+"vgg16","64","2_3_5","382.62"
+"inception3","64","2_3_5","663.05"
+"alexnet","128","2_3_5","5981.77"
+"resnet50","128","2_3_5","1134.81"
+"vgg16","128","2_3_5","409.66"
+"inception3","128","2_3_5","741.46"
+"alexnet","32","0_3_4","2226.94"
+"resnet50","32","0_3_4","848.42"
+"vgg16","32","0_3_4","493.18"
+"inception3","32","0_3_4","587.78"
+"alexnet","64","0_3_4","4219.32"
+"resnet50","64","0_3_4","998.77"
+"vgg16","64","0_3_4","592.35"
+"inception3","64","0_3_4","681.22"
+"alexnet","128","0_3_4","6774.41"
+"resnet50","128","0_3_4","1135.61"
+"vgg16","128","0_3_4","645.88"
+"inception3","128","0_3_4","735.48"
+"alexnet","32","2_3_4","1941.78"
+"resnet50","32","2_3_4","804.32"
+"vgg16","32","2_3_4","326.31"
+"inception3","32","2_3_4","557.95"
+"alexnet","64","2_3_4","3556.00"
+"resnet50","64","2_3_4","990.46"
+"vgg16","64","2_3_4","378.75"
+"inception3","64","2_3_4","664.97"
+"alexnet","128","2_3_4","6021.85"
+"resnet50","128","2_3_4","1123.54"
+"vgg16","128","2_3_4","405.19"
+"inception3","128","2_3_4","739.83"
+"alexnet","32","0_3_7","3294.85"
+"resnet50","32","0_3_7","839.61"
+"vgg16","32","0_3_7","486.59"
+"inception3","32","0_3_7","584.84"
+"alexnet","64","0_3_7","5667.63"
+"resnet50","64","0_3_7","1000.57"
+"vgg16","64","0_3_7","584.72"
+"inception3","64","0_3_7","680.12"
+"alexnet","128","0_3_7","9184.33"
+"resnet50","128","0_3_7","1125.96"
+"vgg16","128","0_3_7","633.98"
+"inception3","128","0_3_7","735.43"
+"alexnet","32","4_5_7","3427.28"
+"resnet50","32","4_5_7","868.27"
+"vgg16","32","4_5_7","567.99"
+"inception3","32","4_5_7","578.78"
+"alexnet","64","4_5_7","6198.05"
+"resnet50","64","4_5_7","1002.96"
+"vgg16","64","4_5_7","631.48"
+"inception3","64","4_5_7","682.08"
+"alexnet","128","4_5_7","8907.87"
+"resnet50","128","4_5_7","1136.59"
+"vgg16","128","4_5_7","666.25"
+"inception3","128","4_5_7","732.44"
+"alexnet","32","2_6_7","3219.58"
+"resnet50","32","2_6_7","813.54"
+"vgg16","32","2_6_7","481.11"
+"inception3","32","2_6_7","567.79"
+"alexnet","64","2_6_7","5938.84"
+"resnet50","64","2_6_7","985.45"
+"vgg16","64","2_6_7","580.06"
+"inception3","64","2_6_7","666.98"
+"alexnet","128","2_6_7","9450.72"
+"resnet50","128","2_6_7","1107.06"
+"vgg16","128","2_6_7","637.32"
+"inception3","128","2_6_7","727.22"
+"alexnet","32","1_2_5","2219.28"
+"resnet50","32","1_2_5","833.15"
+"vgg16","32","1_2_5","492.90"
+"inception3","32","1_2_5","583.21"
+"alexnet","64","1_2_5","4231.28"
+"resnet50","64","1_2_5","1009.08"
+"vgg16","64","1_2_5","591.90"
+"inception3","64","1_2_5","685.51"
+"alexnet","128","1_2_5","6729.82"
+"resnet50","128","1_2_5","1139.33"
+"vgg16","128","1_2_5","653.65"
+"inception3","128","1_2_5","746.86"
+"alexnet","32","0_3_6","1948.57"
+"resnet50","32","0_3_6","806.41"
+"vgg16","32","0_3_6","326.84"
+"inception3","32","0_3_6","571.08"
+"alexnet","64","0_3_6","3599.08"
+"resnet50","64","0_3_6","992.91"
+"vgg16","64","0_3_6","379.72"
+"inception3","64","0_3_6","672.61"
+"alexnet","128","0_3_6","6082.66"
+"resnet50","128","0_3_6","1120.49"
+"vgg16","128","0_3_6","407.15"
+"inception3","128","0_3_6","723.71"
+"alexnet","32","0_5_7","1677.06"
+"resnet50","32","0_5_7","815.49"
+"vgg16","32","0_5_7","465.59"
+"inception3","32","0_5_7","577.20"
+"alexnet","64","0_5_7","3110.85"
+"resnet50","64","0_5_7","980.15"
+"vgg16","64","0_5_7","570.96"
+"inception3","64","0_5_7","673.43"
+"alexnet","128","0_5_7","5506.22"
+"resnet50","128","0_5_7","1124.58"
+"vgg16","128","0_5_7","630.76"
+"inception3","128","0_5_7","731.01"
+"alexnet","32","4_6_7","3967.22"
+"resnet50","32","4_6_7","848.63"
+"vgg16","32","4_6_7","597.61"
+"inception3","32","4_6_7","576.29"
+"alexnet","64","4_6_7","6930.99"
+"resnet50","64","4_6_7","1009.36"
+"vgg16","64","4_6_7","649.87"
+"inception3","64","4_6_7","677.69"
+"alexnet","128","4_6_7","9519.65"
+"resnet50","128","4_6_7","1131.50"
+"vgg16","128","4_6_7","674.90"
+"inception3","128","4_6_7","730.49"
+"alexnet","32","1_3_4","1808.34"
+"resnet50","32","1_3_4","809.24"
+"vgg16","32","1_3_4","325.72"
+"inception3","32","1_3_4","575.28"
+"alexnet","64","1_3_4","3371.90"
+"resnet50","64","1_3_4","983.28"
+"vgg16","64","1_3_4","376.01"
+"inception3","64","1_3_4","669.89"
+"alexnet","128","1_3_4","5807.32"
+"resnet50","128","1_3_4","1115.27"
+"vgg16","128","1_3_4","404.52"
+"inception3","128","1_3_4","738.10"
+"alexnet","32","0_1_3","3429.59"
+"resnet50","32","0_1_3","865.48"
+"vgg16","32","0_1_3","575.10"
+"inception3","32","0_1_3","587.80"
+"alexnet","64","0_1_3","6184.53"
+"resnet50","64","0_1_3","1014.01"
+"vgg16","64","0_1_3","639.11"
+"inception3","64","0_1_3","675.40"
+"alexnet","128","0_1_3","8906.14"
+"resnet50","128","0_1_3","1116.76"
+"vgg16","128","0_1_3","675.15"
+"inception3","128","0_1_3","730.13"
+"alexnet","32","0_6_7","1760.17"
+"resnet50","32","0_6_7","805.28"
+"vgg16","32","0_6_7","486.54"
+"inception3","32","0_6_7","565.58"
+"alexnet","64","0_6_7","3305.71"
+"resnet50","64","0_6_7","985.23"
+"vgg16","64","0_6_7","584.18"
+"inception3","64","0_6_7","664.87"
+"alexnet","128","0_6_7","5716.68"
+"resnet50","128","0_6_7","1120.53"
+"vgg16","128","0_6_7","634.96"
+"inception3","128","0_6_7","727.92"
+"alexnet","32","1_3_5","2078.60"
+"resnet50","32","1_3_5","830.27"
+"vgg16","32","1_3_5","489.86"
+"inception3","32","1_3_5","582.43"
+"alexnet","64","1_3_5","3969.36"
+"resnet50","64","1_3_5","1008.78"
+"vgg16","64","1_3_5","589.81"
+"inception3","64","1_3_5","679.00"
+"alexnet","128","1_3_5","6736.64"
+"resnet50","128","1_3_5","1148.53"
+"vgg16","128","1_3_5","651.52"
+"inception3","128","1_3_5","744.32"
+"alexnet","32","2_5_7","1682.88"
+"resnet50","32","2_5_7","815.30"
+"vgg16","32","2_5_7","470.56"
+"inception3","32","2_5_7","575.87"
+"alexnet","64","2_5_7","3114.42"
+"resnet50","64","2_5_7","983.11"
+"vgg16","64","2_5_7","574.22"
+"inception3","64","2_5_7","679.15"
+"alexnet","128","2_5_7","5575.52"
+"resnet50","128","2_5_7","1136.40"
+"vgg16","128","2_5_7","629.84"
+"inception3","128","2_5_7","735.68"
+"alexnet","32","0_5_6","1750.56"
+"resnet50","32","0_5_6","814.72"
+"vgg16","32","0_5_6","488.15"
+"inception3","32","0_5_6","573.77"
+"alexnet","64","0_5_6","3302.69"
+"resnet50","64","0_5_6","986.19"
+"vgg16","64","0_5_6","587.78"
+"inception3","64","0_5_6","672.10"
+"alexnet","128","0_5_6","5770.32"
+"resnet50","128","0_5_6","1117.08"
+"vgg16","128","0_5_6","645.27"
+"inception3","128","0_5_6","735.04"
+"alexnet","32","3_4_5","1685.87"
+"resnet50","32","3_4_5","793.21"
+"vgg16","32","3_4_5","463.05"
+"inception3","32","3_4_5","557.31"
+"alexnet","64","3_4_5","3146.06"
+"resnet50","64","3_4_5","972.87"
+"vgg16","64","3_4_5","571.66"
+"inception3","64","3_4_5","665.56"
+"alexnet","128","3_4_5","5564.64"
+"resnet50","128","3_4_5","1121.20"
+"vgg16","128","3_4_5","635.35"
+"inception3","128","3_4_5","735.31"
+"alexnet","32","1_3_6","1819.65"
+"resnet50","32","1_3_6","807.51"
+"vgg16","32","1_3_6","326.14"
+"inception3","32","1_3_6","575.25"
+"alexnet","64","1_3_6","3359.29"
+"resnet50","64","1_3_6","989.97"
+"vgg16","64","1_3_6","380.75"
+"inception3","64","1_3_6","671.00"
+"alexnet","128","1_3_6","5788.55"
+"resnet50","128","1_3_6","1133.81"
+"vgg16","128","1_3_6","406.23"
+"inception3","128","1_3_6","742.42"
+"alexnet","32","2_5_6","2257.84"
+"resnet50","32","2_5_6","831.32"
+"vgg16","32","2_5_6","563.75"
+"inception3","32","2_5_6","589.81"
+"alexnet","64","2_5_6","4296.99"
+"resnet50","64","2_5_6","1004.35"
+"vgg16","64","2_5_6","638.20"
+"inception3","64","2_5_6","695.85"
+"alexnet","128","2_5_6","7224.86"
+"resnet50","128","2_5_6","1141.31"
+"vgg16","128","2_5_6","681.02"
+"inception3","128","2_5_6","750.51"
+"alexnet","32","1_4_7","1753.42"
+"resnet50","32","1_4_7","819.05"
+"vgg16","32","1_4_7","486.45"
+"inception3","32","1_4_7","579.19"
+"alexnet","64","1_4_7","3261.69"
+"resnet50","64","1_4_7","987.71"
+"vgg16","64","1_4_7","585.32"
+"inception3","64","1_4_7","685.73"
+"alexnet","128","1_4_7","5723.23"
+"resnet50","128","1_4_7","1105.71"
+"vgg16","128","1_4_7","635.61"
+"inception3","128","1_4_7","732.11"
+"alexnet","32","1_3_7","2922.14"
+"resnet50","32","1_3_7","827.05"
+"vgg16","32","1_3_7","484.53"
+"inception3","32","1_3_7","583.22"
+"alexnet","64","1_3_7","5438.01"
+"resnet50","64","1_3_7","999.05"
+"vgg16","64","1_3_7","578.10"
+"inception3","64","1_3_7","678.90"
+"alexnet","128","1_3_7","8173.49"
+"resnet50","128","1_3_7","1117.18"
+"vgg16","128","1_3_7","635.40"
+"inception3","128","1_3_7","732.78"
+"alexnet","32","5_6_7","4765.32"
+"resnet50","32","5_6_7","848.15"
+"vgg16","32","5_6_7","569.61"
+"inception3","32","5_6_7","579.58"
+"alexnet","64","5_6_7","7448.32"
+"resnet50","64","5_6_7","1000.85"
+"vgg16","64","5_6_7","633.16"
+"inception3","64","5_6_7","674.75"
+"alexnet","128","5_6_7","10125.70"
+"resnet50","128","5_6_7","1124.29"
+"vgg16","128","5_6_7","665.93"
+"inception3","128","5_6_7","730.33"
+"alexnet","32","1_4_6","1673.43"
+"resnet50","32","1_4_6","821.14"
+"vgg16","32","1_4_6","466.94"
+"inception3","32","1_4_6","577.34"
+"alexnet","64","1_4_6","3192.23"
+"resnet50","64","1_4_6","989.40"
+"vgg16","64","1_4_6","571.12"
+"inception3","64","1_4_6","681.71"
+"alexnet","128","1_4_6","5599.12"
+"resnet50","128","1_4_6","1138.45"
+"vgg16","128","1_4_6","636.74"
+"inception3","128","1_4_6","736.75"
+"alexnet","32","1_2_4","1943.31"
+"resnet50","32","1_2_4","806.95"
+"vgg16","32","1_2_4","327.60"
+"inception3","32","1_2_4","572.14"
+"alexnet","64","1_2_4","3572.71"
+"resnet50","64","1_2_4","975.24"
+"vgg16","64","1_2_4","380.38"
+"inception3","64","1_2_4","673.11"
+"alexnet","128","1_2_4","6094.53"
+"resnet50","128","1_2_4","1116.32"
+"vgg16","128","1_2_4","408.76"
+"inception3","128","1_2_4","740.87"
+"alexnet","32","1_2_7","1949.16"
+"resnet50","32","1_2_7","809.69"
+"vgg16","32","1_2_7","327.98"
+"inception3","32","1_2_7","571.14"
+"alexnet","64","1_2_7","3584.57"
+"resnet50","64","1_2_7","981.31"
+"vgg16","64","1_2_7","377.28"
+"inception3","64","1_2_7","667.59"
+"alexnet","128","1_2_7","6059.93"
+"resnet50","128","1_2_7","1121.17"
+"vgg16","128","1_2_7","403.07"
+"inception3","128","1_2_7","728.24"
+"alexnet","32","0_1_7","1783.54"
+"resnet50","32","0_1_7","808.88"
+"vgg16","32","0_1_7","323.93"
+"inception3","32","0_1_7","560.39"
+"alexnet","64","0_1_7","3321.22"
+"resnet50","64","0_1_7","973.75"
+"vgg16","64","0_1_7","375.68"
+"inception3","64","0_1_7","659.72"
+"alexnet","128","0_1_7","5771.63"
+"resnet50","128","0_1_7","1106.24"
+"vgg16","128","0_1_7","398.63"
+"inception3","128","0_1_7","721.84"
+"alexnet","32","0_1_2","3752.11"
+"resnet50","32","0_1_2","863.84"
+"vgg16","32","0_1_2","575.96"
+"inception3","32","0_1_2","583.84"
+"alexnet","64","0_1_2","6663.61"
+"resnet50","64","0_1_2","1012.89"
+"vgg16","64","0_1_2","646.64"
+"inception3","64","0_1_2","667.07"
+"alexnet","128","0_1_2","9387.99"
+"resnet50","128","0_1_2","1145.90"
+"vgg16","128","0_1_2","684.89"
+"inception3","128","0_1_2","730.63"
+"alexnet","32","0_1_6","1799.87"
+"resnet50","32","0_1_6","803.27"
+"vgg16","32","0_1_6","325.11"
+"inception3","32","0_1_6","561.79"
+"alexnet","64","0_1_6","3324.30"
+"resnet50","64","0_1_6","990.06"
+"vgg16","64","0_1_6","381.93"
+"inception3","64","0_1_6","660.31"
+"alexnet","128","0_1_6","5719.92"
+"resnet50","128","0_1_6","1131.01"
+"vgg16","128","0_1_6","407.74"
+"inception3","128","0_1_6","732.38"
+"alexnet","32","0_4_5","3271.89"
+"resnet50","32","0_4_5","822.55"
+"vgg16","32","0_4_5","478.86"
+"inception3","32","0_4_5","565.98"
+"alexnet","64","0_4_5","5772.92"
+"resnet50","64","0_4_5","991.99"
+"vgg16","64","0_4_5","586.74"
+"inception3","64","0_4_5","672.47"
+"alexnet","128","0_4_5","8929.89"
+"resnet50","128","0_4_5","1139.48"
+"vgg16","128","0_4_5","647.32"
+"inception3","128","0_4_5","736.26"
+"alexnet","32","0_2_6","2928.63"
+"resnet50","32","0_2_6","830.14"
+"vgg16","32","0_2_6","486.33"
+"inception3","32","0_2_6","583.91"
+"alexnet","64","0_2_6","5401.40"
+"resnet50","64","0_2_6","1011.23"
+"vgg16","64","0_2_6","585.51"
+"inception3","64","0_2_6","675.72"
+"alexnet","128","0_2_6","8149.77"
+"resnet50","128","0_2_6","1116.18"
+"vgg16","128","0_2_6","649.05"
+"inception3","128","0_2_6","739.89"
+"alexnet","32","3_4_7","2257.65"
+"resnet50","32","3_4_7","847.49"
+"vgg16","32","3_4_7","560.35"
+"inception3","32","3_4_7","600.41"
+"alexnet","64","3_4_7","4318.02"
+"resnet50","64","3_4_7","1010.95"
+"vgg16","64","3_4_7","626.70"
+"inception3","64","3_4_7","689.80"
+"alexnet","128","3_4_7","7249.43"
+"resnet50","128","3_4_7","1132.47"
+"vgg16","128","3_4_7","661.96"
+"inception3","128","3_4_7","735.68"
+"alexnet","32","0_1_5","3220.26"
+"resnet50","32","0_1_5","835.88"
+"vgg16","32","0_1_5","490.38"
+"inception3","32","0_1_5","571.67"
+"alexnet","64","0_1_5","5957.92"
+"resnet50","64","0_1_5","1005.80"
+"vgg16","64","0_1_5","591.77"
+"inception3","64","0_1_5","664.13"
+"alexnet","128","0_1_5","9536.47"
+"resnet50","128","0_1_5","1129.10"
+"vgg16","128","0_1_5","650.19"
+"inception3","128","0_1_5","726.53"
+"alexnet","32","3_6_7","2263.52"
+"resnet50","32","3_6_7","828.83"
+"vgg16","32","3_6_7","561.30"
+"inception3","32","3_6_7","578.29"
+"alexnet","64","3_6_7","4302.45"
+"resnet50","64","3_6_7","989.18"
+"vgg16","64","3_6_7","624.44"
+"inception3","64","3_6_7","680.14"
+"alexnet","128","3_6_7","7018.45"
+"resnet50","128","3_6_7","1127.25"
+"vgg16","128","3_6_7","658.21"
+"inception3","128","3_6_7","727.47"
+"alexnet","32","0_4_6","3295.75"
+"resnet50","32","0_4_6","831.08"
+"vgg16","32","0_4_6","484.66"
+"inception3","32","0_4_6","580.10"
+"alexnet","64","0_4_6","5809.04"
+"resnet50","64","0_4_6","1006.98"
+"vgg16","64","0_4_6","587.06"
+"inception3","64","0_4_6","683.48"
+"alexnet","128","0_4_6","9141.93"
+"resnet50","128","0_4_6","1139.15"
+"vgg16","128","0_4_6","648.12"
+"inception3","128","0_4_6","742.75"
+"alexnet","32","0_2_7","1806.62"
+"resnet50","32","0_2_7","808.21"
+"vgg16","32","0_2_7","326.40"
+"inception3","32","0_2_7","567.02"
+"alexnet","64","0_2_7","3417.95"
+"resnet50","64","0_2_7","983.73"
+"vgg16","64","0_2_7","376.15"
+"inception3","64","0_2_7","668.89"
+"alexnet","128","0_2_7","5782.30"
+"resnet50","128","0_2_7","1107.51"
+"vgg16","128","0_2_7","396.52"
+"inception3","128","0_2_7","730.57"
+"alexnet","32","0_1_4","2080.66"
+"resnet50","32","0_1_4","834.74"
+"vgg16","32","0_1_4","488.08"
+"inception3","32","0_1_4","573.27"
+"alexnet","64","0_1_4","3961.03"
+"resnet50","64","0_1_4","1006.86"
+"vgg16","64","0_1_4","588.83"
+"inception3","64","0_1_4","667.29"
+"alexnet","128","0_1_4","6563.84"
+"resnet50","128","0_1_4","1139.71"
+"vgg16","128","0_1_4","646.42"
+"inception3","128","0_1_4","726.07"
+"alexnet","32","2_4_6","2137.17"
+"resnet50","32","2_4_6","847.69"
+"vgg16","32","2_4_6","552.49"
+"inception3","32","2_4_6","594.47"
+"alexnet","64","2_4_6","4040.51"
+"resnet50","64","2_4_6","1010.77"
+"vgg16","64","2_4_6","631.46"
+"inception3","64","2_4_6","691.96"
+"alexnet","128","2_4_6","7225.70"
+"resnet50","128","2_4_6","1151.76"
+"vgg16","128","2_4_6","677.11"
+"inception3","128","2_4_6","750.38"
+"alexnet","32","0_4_7","3346.91"
+"resnet50","32","0_4_7","834.23"
+"vgg16","32","0_4_7","491.41"
+"inception3","32","0_4_7","581.78"
+"alexnet","64","0_4_7","5590.60"
+"resnet50","64","0_4_7","1005.37"
+"vgg16","64","0_4_7","586.35"
+"inception3","64","0_4_7","684.98"
+"alexnet","128","0_4_7","8399.14"
+"resnet50","128","0_4_7","1129.51"
+"vgg16","128","0_4_7","637.06"
+"inception3","128","0_4_7","732.05"
+"alexnet","32","0_1_2_3","4450.67"
+"resnet50","32","0_1_2_3","1103.87"
+"vgg16","32","0_1_2_3","741.17"
+"inception3","32","0_1_2_3","759.75"
+"alexnet","64","0_1_2_3","7978.19"
+"resnet50","64","0_1_2_3","1310.09"
+"vgg16","64","0_1_2_3","848.44"
+"inception3","64","0_1_2_3","858.87"
+"alexnet","128","0_1_2_3","11629.84"
+"resnet50","128","0_1_2_3","1501.38"
+"vgg16","128","0_1_2_3","908.14"
+"inception3","128","0_1_2_3","971.64"
+"alexnet","32","0_3_4_5","2356.73"
+"resnet50","32","0_3_4_5","1046.15"
+"vgg16","32","0_3_4_5","615.97"
+"inception3","32","0_3_4_5","732.68"
+"alexnet","64","0_3_4_5","4499.28"
+"resnet50","64","0_3_4_5","1284.55"
+"vgg16","64","0_3_4_5","763.83"
+"inception3","64","0_3_4_5","868.29"
+"alexnet","128","0_3_4_5","7917.65"
+"resnet50","128","0_3_4_5","1489.12"
+"vgg16","128","0_3_4_5","851.68"
+"inception3","128","0_3_4_5","945.30"
+"alexnet","32","0_2_4_7","2526.65"
+"resnet50","32","0_2_4_7","1065.53"
+"vgg16","32","0_2_4_7","635.49"
+"inception3","32","0_2_4_7","750.58"
+"alexnet","64","0_2_4_7","4827.72"
+"resnet50","64","0_2_4_7","1263.11"
+"vgg16","64","0_2_4_7","771.53"
+"inception3","64","0_2_4_7","879.74"
+"alexnet","128","0_2_4_7","8507.98"
+"resnet50","128","0_2_4_7","1440.52"
+"vgg16","128","0_2_4_7","851.54"
+"inception3","128","0_2_4_7","956.98"
+"alexnet","32","0_2_3_5","2154.12"
+"resnet50","32","0_2_3_5","1037.28"
+"vgg16","32","0_2_3_5","600.36"
+"inception3","32","0_2_3_5","722.83"
+"alexnet","64","0_2_3_5","4093.10"
+"resnet50","64","0_2_3_5","1302.90"
+"vgg16","64","0_2_3_5","741.78"
+"inception3","64","0_2_3_5","852.84"
+"alexnet","128","0_2_3_5","7121.02"
+"resnet50","128","0_2_3_5","1485.92"
+"vgg16","128","0_2_3_5","821.29"
+"inception3","128","0_2_3_5","957.68"
+"alexnet","32","0_1_2_4","2239.12"
+"resnet50","32","0_1_2_4","1084.16"
+"vgg16","32","0_1_2_4","577.55"
+"inception3","32","0_1_2_4","735.85"
+"alexnet","64","0_1_2_4","4239.91"
+"resnet50","64","0_1_2_4","1317.19"
+"vgg16","64","0_1_2_4","739.12"
+"inception3","64","0_1_2_4","865.48"
+"alexnet","128","0_1_2_4","7496.99"
+"resnet50","128","0_1_2_4","1471.40"
+"vgg16","128","0_1_2_4","835.41"
+"inception3","128","0_1_2_4","953.25"
+"alexnet","32","3_5_6_7","2270.18"
+"resnet50","32","3_5_6_7","1074.56"
+"vgg16","32","3_5_6_7","619.66"
+"inception3","32","3_5_6_7","745.42"
+"alexnet","64","3_5_6_7","4304.71"
+"resnet50","64","3_5_6_7","1303.14"
+"vgg16","64","3_5_6_7","755.56"
+"inception3","64","3_5_6_7","872.17"
+"alexnet","128","3_5_6_7","7515.23"
+"resnet50","128","3_5_6_7","1479.46"
+"vgg16","128","3_5_6_7","837.28"
+"inception3","128","3_5_6_7","969.23"
+"alexnet","32","0_3_4_6","2403.86"
+"resnet50","32","0_3_4_6","1066.05"
+"vgg16","32","0_3_4_6","617.40"
+"inception3","32","0_3_4_6","751.26"
+"alexnet","64","0_3_4_6","4605.30"
+"resnet50","64","0_3_4_6","1310.08"
+"vgg16","64","0_3_4_6","767.08"
+"inception3","64","0_3_4_6","871.99"
+"alexnet","128","0_3_4_6","8100.43"
+"resnet50","128","0_3_4_6","1504.77"
+"vgg16","128","0_3_4_6","852.86"
+"inception3","128","0_3_4_6","956.24"
+"alexnet","32","1_2_3_6","2442.09"
+"resnet50","32","1_2_3_6","1060.13"
+"vgg16","32","1_2_3_6","594.30"
+"inception3","32","1_2_3_6","729.95"
+"alexnet","64","1_2_3_6","4632.19"
+"resnet50","64","1_2_3_6","1310.13"
+"vgg16","64","1_2_3_6","739.89"
+"inception3","64","1_2_3_6","876.71"
+"alexnet","128","1_2_3_6","7778.67"
+"resnet50","128","1_2_3_6","1500.83"
+"vgg16","128","1_2_3_6","841.58"
+"inception3","128","1_2_3_6","947.68"
+"alexnet","32","0_1_4_7","2478.37"
+"resnet50","32","0_1_4_7","1065.05"
+"vgg16","32","0_1_4_7","636.10"
+"inception3","32","0_1_4_7","736.89"
+"alexnet","64","0_1_4_7","4722.28"
+"resnet50","64","0_1_4_7","1298.48"
+"vgg16","64","0_1_4_7","771.65"
+"inception3","64","0_1_4_7","876.84"
+"alexnet","128","0_1_4_7","8167.83"
+"resnet50","128","0_1_4_7","1484.56"
+"vgg16","128","0_1_4_7","845.65"
+"inception3","128","0_1_4_7","931.71"
+"alexnet","32","0_3_5_6","1737.61"
+"resnet50","32","0_3_5_6","1064.76"
+"vgg16","32","0_3_5_6","481.60"
+"inception3","32","0_3_5_6","747.24"
+"alexnet","64","0_3_5_6","3347.94"
+"resnet50","64","0_3_5_6","1311.45"
+"vgg16","64","0_3_5_6","635.82"
+"inception3","64","0_3_5_6","898.57"
+"alexnet","128","0_3_5_6","6270.89"
+"resnet50","128","0_3_5_6","1476.25"
+"vgg16","128","0_3_5_6","758.99"
+"inception3","128","0_3_5_6","952.43"
+"alexnet","32","2_3_5_7","1700.29"
+"resnet50","32","2_3_5_7","1029.67"
+"vgg16","32","2_3_5_7","466.90"
+"inception3","32","2_3_5_7","745.50"
+"alexnet","64","2_3_5_7","3293.33"
+"resnet50","64","2_3_5_7","1285.94"
+"vgg16","64","2_3_5_7","636.26"
+"inception3","64","2_3_5_7","886.18"
+"alexnet","128","2_3_5_7","6146.11"
+"resnet50","128","2_3_5_7","1493.41"
+"vgg16","128","2_3_5_7","716.71"
+"inception3","128","2_3_5_7","979.33"
+"alexnet","32","0_4_5_7","2488.34"
+"resnet50","32","0_4_5_7","1094.58"
+"vgg16","32","0_4_5_7","603.17"
+"inception3","32","0_4_5_7","741.65"
+"alexnet","64","0_4_5_7","4807.45"
+"resnet50","64","0_4_5_7","1307.96"
+"vgg16","64","0_4_5_7","755.99"
+"inception3","64","0_4_5_7","877.22"
+"alexnet","128","0_4_5_7","8182.41"
+"resnet50","128","0_4_5_7","1421.20"
+"vgg16","128","0_4_5_7","840.83"
+"inception3","128","0_4_5_7","949.30"
+"alexnet","32","1_3_6_7","1763.40"
+"resnet50","32","1_3_6_7","1033.16"
+"vgg16","32","1_3_6_7","482.33"
+"inception3","32","1_3_6_7","737.95"
+"alexnet","64","1_3_6_7","3397.71"
+"resnet50","64","1_3_6_7","1303.97"
+"vgg16","64","1_3_6_7","637.57"
+"inception3","64","1_3_6_7","880.25"
+"alexnet","128","1_3_6_7","6204.49"
+"resnet50","128","1_3_6_7","1464.87"
+"vgg16","128","1_3_6_7","732.30"
+"inception3","128","1_3_6_7","958.16"
+"alexnet","32","0_1_3_4","2187.20"
+"resnet50","32","0_1_3_4","1089.29"
+"vgg16","32","0_1_3_4","583.58"
+"inception3","32","0_1_3_4","736.17"
+"alexnet","64","0_1_3_4","4198.57"
+"resnet50","64","0_1_3_4","1301.52"
+"vgg16","64","0_1_3_4","736.13"
+"inception3","64","0_1_3_4","869.85"
+"alexnet","128","0_1_3_4","7392.07"
+"resnet50","128","0_1_3_4","1485.39"
+"vgg16","128","0_1_3_4","829.78"
+"inception3","128","0_1_3_4","950.88"
+"alexnet","32","0_2_6_7","2499.71"
+"resnet50","32","0_2_6_7","1044.36"
+"vgg16","32","0_2_6_7","615.11"
+"inception3","32","0_2_6_7","729.64"
+"alexnet","64","0_2_6_7","4795.19"
+"resnet50","64","0_2_6_7","1262.72"
+"vgg16","64","0_2_6_7","758.15"
+"inception3","64","0_2_6_7","872.71"
+"alexnet","128","0_2_6_7","8285.78"
+"resnet50","128","0_2_6_7","1473.78"
+"vgg16","128","0_2_6_7","843.84"
+"inception3","128","0_2_6_7","963.80"
+"alexnet","32","0_1_3_7","3349.51"
+"resnet50","32","0_1_3_7","1080.61"
+"vgg16","32","0_1_3_7","692.24"
+"inception3","32","0_1_3_7","728.45"
+"alexnet","64","0_1_3_7","6260.71"
+"resnet50","64","0_1_3_7","1312.11"
+"vgg16","64","0_1_3_7","805.54"
+"inception3","64","0_1_3_7","868.18"
+"alexnet","128","0_1_3_7","9879.27"
+"resnet50","128","0_1_3_7","1475.91"
+"vgg16","128","0_1_3_7","871.76"
+"inception3","128","0_1_3_7","949.59"
+"alexnet","32","1_2_4_6","1718.99"
+"resnet50","32","1_2_4_6","1081.31"
+"vgg16","32","1_2_4_6","471.40"
+"inception3","32","1_2_4_6","763.19"
+"alexnet","64","1_2_4_6","3334.32"
+"resnet50","64","1_2_4_6","1253.44"
+"vgg16","64","1_2_4_6","630.73"
+"inception3","64","1_2_4_6","901.00"
+"alexnet","128","1_2_4_6","6166.72"
+"resnet50","128","1_2_4_6","1495.90"
+"vgg16","128","1_2_4_6","736.72"
+"inception3","128","1_2_4_6","972.86"
+"alexnet","32","2_3_4_7","1775.71"
+"resnet50","32","2_3_4_7","1068.47"
+"vgg16","32","2_3_4_7","485.57"
+"inception3","32","2_3_4_7","753.00"
+"alexnet","64","2_3_4_7","3446.31"
+"resnet50","64","2_3_4_7","1296.30"
+"vgg16","64","2_3_4_7","633.70"
+"inception3","64","2_3_4_7","876.59"
+"alexnet","128","2_3_4_7","6265.96"
+"resnet50","128","2_3_4_7","1410.33"
+"vgg16","128","2_3_4_7","734.09"
+"inception3","128","2_3_4_7","966.52"
+"alexnet","32","4_5_6_7","4481.74"
+"resnet50","32","4_5_6_7","1109.83"
+"vgg16","32","4_5_6_7","740.63"
+"inception3","32","4_5_6_7","759.92"
+"alexnet","64","4_5_6_7","8018.66"
+"resnet50","64","4_5_6_7","1250.34"
+"vgg16","64","4_5_6_7","836.66"
+"inception3","64","4_5_6_7","884.37"
+"alexnet","128","4_5_6_7","11602.69"
+"resnet50","128","4_5_6_7","1402.23"
+"vgg16","128","4_5_6_7","893.52"
+"inception3","128","4_5_6_7","956.86"
+"alexnet","32","1_2_5_6","2793.13"
+"resnet50","32","1_2_5_6","1096.67"
+"vgg16","32","1_2_5_6","639.02"
+"inception3","32","1_2_5_6","768.34"
+"alexnet","64","1_2_5_6","5397.48"
+"resnet50","64","1_2_5_6","1332.06"
+"vgg16","64","1_2_5_6","780.68"
+"inception3","64","1_2_5_6","884.51"
+"alexnet","128","1_2_5_6","8853.75"
+"resnet50","128","1_2_5_6","1487.90"
+"vgg16","128","1_2_5_6","867.52"
+"inception3","128","1_2_5_6","971.65"
+"alexnet","32","0_2_5_6","1763.37"
+"resnet50","32","0_2_5_6","1075.75"
+"vgg16","32","0_2_5_6","489.44"
+"inception3","32","0_2_5_6","754.01"
+"alexnet","64","0_2_5_6","3438.45"
+"resnet50","64","0_2_5_6","1310.64"
+"vgg16","64","0_2_5_6","640.33"
+"inception3","64","0_2_5_6","897.38"
+"alexnet","128","0_2_5_6","6342.04"
+"resnet50","128","0_2_5_6","1503.36"
+"vgg16","128","0_2_5_6","765.27"
+"inception3","128","0_2_5_6","968.45"
+"alexnet","32","2_3_6_7","2604.63"
+"resnet50","32","2_3_6_7","1059.00"
+"vgg16","32","2_3_6_7","610.98"
+"inception3","32","2_3_6_7","744.27"
+"alexnet","64","2_3_6_7","4986.23"
+"resnet50","64","2_3_6_7","1291.03"
+"vgg16","64","2_3_6_7","754.78"
+"inception3","64","2_3_6_7","879.34"
+"alexnet","128","2_3_6_7","8477.90"
+"resnet50","128","2_3_6_7","1479.41"
+"vgg16","128","2_3_6_7","844.95"
+"inception3","128","2_3_6_7","941.62"
+"alexnet","32","0_5_6_7","2193.29"
+"resnet50","32","0_5_6_7","1049.74"
+"vgg16","32","0_5_6_7","611.72"
+"inception3","32","0_5_6_7","728.77"
+"alexnet","64","0_5_6_7","4177.98"
+"resnet50","64","0_5_6_7","1272.70"
+"vgg16","64","0_5_6_7","722.99"
+"inception3","64","0_5_6_7","867.05"
+"alexnet","128","0_5_6_7","7393.27"
+"resnet50","128","0_5_6_7","1450.39"
+"vgg16","128","0_5_6_7","776.16"
+"inception3","128","0_5_6_7","964.02"
+"alexnet","32","0_2_4_6","2685.07"
+"resnet50","32","0_2_4_6","1084.62"
+"vgg16","32","0_2_4_6","629.81"
+"inception3","32","0_2_4_6","756.66"
+"alexnet","64","0_2_4_6","5171.91"
+"resnet50","64","0_2_4_6","1298.90"
+"vgg16","64","0_2_4_6","765.82"
+"inception3","64","0_2_4_6","895.33"
+"alexnet","128","0_2_4_6","8714.14"
+"resnet50","128","0_2_4_6","1498.72"
+"vgg16","128","0_2_4_6","854.70"
+"inception3","128","0_2_4_6","953.60"
+"alexnet","32","0_2_3_6","2566.31"
+"resnet50","32","0_2_3_6","1073.37"
+"vgg16","32","0_2_3_6","608.60"
+"inception3","32","0_2_3_6","733.45"
+"alexnet","64","0_2_3_6","4792.27"
+"resnet50","64","0_2_3_6","1299.55"
+"vgg16","64","0_2_3_6","758.43"
+"inception3","64","0_2_3_6","862.73"
+"alexnet","128","0_2_3_6","8190.38"
+"resnet50","128","0_2_3_6","1418.46"
+"vgg16","128","0_2_3_6","847.73"
+"inception3","128","0_2_3_6","957.75"
+"alexnet","32","0_1_2_5","2506.08"
+"resnet50","32","0_1_2_5","1074.94"
+"vgg16","32","0_1_2_5","596.66"
+"inception3","32","0_1_2_5","739.29"
+"alexnet","64","0_1_2_5","4602.09"
+"resnet50","64","0_1_2_5","1289.85"
+"vgg16","64","0_1_2_5","743.38"
+"inception3","64","0_1_2_5","878.91"
+"alexnet","128","0_1_2_5","7874.66"
+"resnet50","128","0_1_2_5","1491.51"
+"vgg16","128","0_1_2_5","839.72"
+"inception3","128","0_1_2_5","954.27"
+"alexnet","32","0_3_4_7","2846.93"
+"resnet50","32","0_3_4_7","1086.71"
+"vgg16","32","0_3_4_7","637.60"
+"inception3","32","0_3_4_7","766.88"
+"alexnet","64","0_3_4_7","5427.18"
+"resnet50","64","0_3_4_7","1307.90"
+"vgg16","64","0_3_4_7","780.21"
+"inception3","64","0_3_4_7","890.91"
+"alexnet","128","0_3_4_7","8898.28"
+"resnet50","128","0_3_4_7","1470.08"
+"vgg16","128","0_3_4_7","854.73"
+"inception3","128","0_3_4_7","955.92"
+"alexnet","32","1_2_3_5","2261.27"
+"resnet50","32","1_2_3_5","1078.87"
+"vgg16","32","1_2_3_5","584.85"
+"inception3","32","1_2_3_5","738.49"
+"alexnet","64","1_2_3_5","4293.10"
+"resnet50","64","1_2_3_5","1295.63"
+"vgg16","64","1_2_3_5","735.69"
+"inception3","64","1_2_3_5","885.13"
+"alexnet","128","1_2_3_5","7517.91"
+"resnet50","128","1_2_3_5","1506.67"
+"vgg16","128","1_2_3_5","838.61"
+"inception3","128","1_2_3_5","965.83"
+"alexnet","32","0_1_2_6","3391.07"
+"resnet50","32","0_1_2_6","1082.50"
+"vgg16","32","0_1_2_6","696.72"
+"inception3","32","0_1_2_6","739.59"
+"alexnet","64","0_1_2_6","6253.16"
+"resnet50","64","0_1_2_6","1273.06"
+"vgg16","64","0_1_2_6","817.44"
+"inception3","64","0_1_2_6","863.48"
+"alexnet","128","0_1_2_6","9807.84"
+"resnet50","128","0_1_2_6","1487.96"
+"vgg16","128","0_1_2_6","887.33"
+"inception3","128","0_1_2_6","964.80"
+"alexnet","32","1_3_4_7","1766.26"
+"resnet50","32","1_3_4_7","1073.44"
+"vgg16","32","1_3_4_7","499.08"
+"inception3","32","1_3_4_7","737.80"
+"alexnet","64","1_3_4_7","3455.93"
+"resnet50","64","1_3_4_7","1298.77"
+"vgg16","64","1_3_4_7","629.65"
+"inception3","64","1_3_4_7","885.72"
+"alexnet","128","1_3_4_7","6340.51"
+"resnet50","128","1_3_4_7","1471.51"
+"vgg16","128","1_3_4_7","749.73"
+"inception3","128","1_3_4_7","963.40"
+"alexnet","32","1_4_5_6","3914.54"
+"resnet50","32","1_4_5_6","1086.15"
+"vgg16","32","1_4_5_6","724.87"
+"inception3","32","1_4_5_6","739.18"
+"alexnet","64","1_4_5_6","7366.38"
+"resnet50","64","1_4_5_6","1262.19"
+"vgg16","64","1_4_5_6","833.55"
+"inception3","64","1_4_5_6","875.90"
+"alexnet","128","1_4_5_6","11518.18"
+"resnet50","128","1_4_5_6","1503.92"
+"vgg16","128","1_4_5_6","891.92"
+"inception3","128","1_4_5_6","969.38"
+"alexnet","32","3_4_5_7","2100.80"
+"resnet50","32","3_4_5_7","1060.81"
+"vgg16","32","3_4_5_7","579.13"
+"inception3","32","3_4_5_7","749.35"
+"alexnet","64","3_4_5_7","4042.81"
+"resnet50","64","3_4_5_7","1325.45"
+"vgg16","64","3_4_5_7","724.91"
+"inception3","64","3_4_5_7","888.85"
+"alexnet","128","3_4_5_7","7046.58"
+"resnet50","128","3_4_5_7","1479.38"
+"vgg16","128","3_4_5_7","818.52"
+"inception3","128","3_4_5_7","965.26"
+"alexnet","32","0_1_4_5","2597.13"
+"resnet50","32","0_1_4_5","1068.01"
+"vgg16","32","0_1_4_5","611.08"
+"inception3","32","0_1_4_5","744.05"
+"alexnet","64","0_1_4_5","4983.33"
+"resnet50","64","0_1_4_5","1287.35"
+"vgg16","64","0_1_4_5","759.78"
+"inception3","64","0_1_4_5","881.03"
+"alexnet","128","0_1_4_5","8406.38"
+"resnet50","128","0_1_4_5","1492.98"
+"vgg16","128","0_1_4_5","848.03"
+"inception3","128","0_1_4_5","958.66"
+"alexnet","32","1_5_6_7","2810.66"
+"resnet50","32","1_5_6_7","1061.93"
+"vgg16","32","1_5_6_7","644.96"
+"inception3","32","1_5_6_7","741.90"
+"alexnet","64","1_5_6_7","5355.63"
+"resnet50","64","1_5_6_7","1262.22"
+"vgg16","64","1_5_6_7","781.11"
+"inception3","64","1_5_6_7","883.76"
+"alexnet","128","1_5_6_7","8978.15"
+"resnet50","128","1_5_6_7","1489.93"
+"vgg16","128","1_5_6_7","850.34"
+"inception3","128","1_5_6_7","959.79"
+"alexnet","32","0_1_5_7","2502.73"
+"resnet50","32","0_1_5_7","1039.61"
+"vgg16","32","0_1_5_7","615.96"
+"inception3","32","0_1_5_7","741.89"
+"alexnet","64","0_1_5_7","4828.09"
+"resnet50","64","0_1_5_7","1280.11"
+"vgg16","64","0_1_5_7","756.82"
+"inception3","64","0_1_5_7","864.56"
+"alexnet","128","0_1_5_7","8395.78"
+"resnet50","128","0_1_5_7","1450.85"
+"vgg16","128","0_1_5_7","838.55"
+"inception3","128","0_1_5_7","948.52"
+"alexnet","32","0_4_6_7","2658.20"
+"resnet50","32","0_4_6_7","1064.83"
+"vgg16","32","0_4_6_7","623.62"
+"inception3","32","0_4_6_7","734.78"
+"alexnet","64","0_4_6_7","5079.85"
+"resnet50","64","0_4_6_7","1288.36"
+"vgg16","64","0_4_6_7","760.56"
+"inception3","64","0_4_6_7","879.03"
+"alexnet","128","0_4_6_7","8573.14"
+"resnet50","128","0_4_6_7","1460.30"
+"vgg16","128","0_4_6_7","848.16"
+"inception3","128","0_4_6_7","953.52"
+"alexnet","32","1_4_5_7","3638.31"
+"resnet50","32","1_4_5_7","1060.29"
+"vgg16","32","1_4_5_7","688.71"
+"inception3","32","1_4_5_7","731.76"
+"alexnet","64","1_4_5_7","6753.55"
+"resnet50","64","1_4_5_7","1316.53"
+"vgg16","64","1_4_5_7","807.94"
+"inception3","64","1_4_5_7","881.22"
+"alexnet","128","1_4_5_7","10429.91"
+"resnet50","128","1_4_5_7","1461.66"
+"vgg16","128","1_4_5_7","872.47"
+"inception3","128","1_4_5_7","962.62"
+"alexnet","32","1_3_5_6","2521.01"
+"resnet50","32","1_3_5_6","1072.68"
+"vgg16","32","1_3_5_6","637.64"
+"inception3","32","1_3_5_6","742.43"
+"alexnet","64","1_3_5_6","4803.78"
+"resnet50","64","1_3_5_6","1320.85"
+"vgg16","64","1_3_5_6","776.34"
+"inception3","64","1_3_5_6","878.24"
+"alexnet","128","1_3_5_6","8425.15"
+"resnet50","128","1_3_5_6","1491.99"
+"vgg16","128","1_3_5_6","866.73"
+"inception3","128","1_3_5_6","967.21"
+"alexnet","32","1_2_4_7","1726.44"
+"resnet50","32","1_2_4_7","1052.43"
+"vgg16","32","1_2_4_7","488.92"
+"inception3","32","1_2_4_7","749.79"
+"alexnet","64","1_2_4_7","3385.31"
+"resnet50","64","1_2_4_7","1296.25"
+"vgg16","64","1_2_4_7","650.60"
+"inception3","64","1_2_4_7","887.70"
+"alexnet","128","1_2_4_7","6203.39"
+"resnet50","128","1_2_4_7","1472.73"
+"vgg16","128","1_2_4_7","771.90"
+"inception3","128","1_2_4_7","954.41"
+"alexnet","32","3_4_5_6","2087.69"
+"resnet50","32","3_4_5_6","1063.89"
+"vgg16","32","3_4_5_6","596.13"
+"inception3","32","3_4_5_6","740.96"
+"alexnet","64","3_4_5_6","3973.82"
+"resnet50","64","3_4_5_6","1291.14"
+"vgg16","64","3_4_5_6","718.05"
+"inception3","64","3_4_5_6","885.21"
+"alexnet","128","3_4_5_6","7098.89"
+"resnet50","128","3_4_5_6","1412.25"
+"vgg16","128","3_4_5_6","809.38"
+"inception3","128","3_4_5_6","964.17"
+"alexnet","32","1_3_4_6","1642.05"
+"resnet50","32","1_3_4_6","1035.06"
+"vgg16","32","1_3_4_6","465.32"
+"inception3","32","1_3_4_6","746.58"
+"alexnet","64","1_3_4_6","3254.44"
+"resnet50","64","1_3_4_6","1294.26"
+"vgg16","64","1_3_4_6","633.94"
+"inception3","64","1_3_4_6","892.70"
+"alexnet","128","1_3_4_6","5936.70"
+"resnet50","128","1_3_4_6","1486.92"
+"vgg16","128","1_3_4_6","729.36"
+"inception3","128","1_3_4_6","958.15"
+"alexnet","32","0_1_3_6","2063.19"
+"resnet50","32","0_1_3_6","1048.94"
+"vgg16","32","0_1_3_6","579.06"
+"inception3","32","0_1_3_6","721.00"
+"alexnet","64","0_1_3_6","3954.46"
+"resnet50","64","0_1_3_6","1297.66"
+"vgg16","64","0_1_3_6","715.97"
+"inception3","64","0_1_3_6","871.11"
+"alexnet","128","0_1_3_6","6961.20"
+"resnet50","128","0_1_3_6","1485.55"
+"vgg16","128","0_1_3_6","787.32"
+"inception3","128","0_1_3_6","920.77"
+"alexnet","32","2_3_4_5","1606.35"
+"resnet50","32","2_3_4_5","1017.93"
+"vgg16","32","2_3_4_5","459.36"
+"inception3","32","2_3_4_5","719.65"
+"alexnet","64","2_3_4_5","3152.64"
+"resnet50","64","2_3_4_5","1278.63"
+"vgg16","64","2_3_4_5","616.14"
+"inception3","64","2_3_4_5","874.68"
+"alexnet","128","2_3_4_5","5770.13"
+"resnet50","128","2_3_4_5","1411.25"
+"vgg16","128","2_3_4_5","751.78"
+"inception3","128","2_3_4_5","981.17"
+"alexnet","32","0_3_6_7","1774.75"
+"resnet50","32","0_3_6_7","1061.81"
+"vgg16","32","0_3_6_7","483.00"
+"inception3","32","0_3_6_7","728.37"
+"alexnet","64","0_3_6_7","3439.74"
+"resnet50","64","0_3_6_7","1299.60"
+"vgg16","64","0_3_6_7","620.16"
+"inception3","64","0_3_6_7","876.27"
+"alexnet","128","0_3_6_7","6187.32"
+"resnet50","128","0_3_6_7","1452.60"
+"vgg16","128","0_3_6_7","739.09"
+"inception3","128","0_3_6_7","956.54"
+"alexnet","32","1_3_4_5","1648.82"
+"resnet50","32","1_3_4_5","1053.53"
+"vgg16","32","1_3_4_5","474.88"
+"inception3","32","1_3_4_5","735.47"
+"alexnet","64","1_3_4_5","3223.67"
+"resnet50","64","1_3_4_5","1277.48"
+"vgg16","64","1_3_4_5","631.37"
+"inception3","64","1_3_4_5","878.41"
+"alexnet","128","1_3_4_5","5872.95"
+"resnet50","128","1_3_4_5","1478.75"
+"vgg16","128","1_3_4_5","757.55"
+"inception3","128","1_3_4_5","974.27"
+"alexnet","32","3_4_6_7","2198.00"
+"resnet50","32","3_4_6_7","1086.50"
+"vgg16","32","3_4_6_7","605.22"
+"inception3","32","3_4_6_7","740.45"
+"alexnet","64","3_4_6_7","4182.17"
+"resnet50","64","3_4_6_7","1300.90"
+"vgg16","64","3_4_6_7","741.63"
+"inception3","64","3_4_6_7","883.86"
+"alexnet","128","3_4_6_7","7431.97"
+"resnet50","128","3_4_6_7","1471.75"
+"vgg16","128","3_4_6_7","828.85"
+"inception3","128","3_4_6_7","957.48"
+"alexnet","32","2_5_6_7","3848.35"
+"resnet50","32","2_5_6_7","1064.28"
+"vgg16","32","2_5_6_7","725.85"
+"inception3","32","2_5_6_7","743.96"
+"alexnet","64","2_5_6_7","7084.97"
+"resnet50","64","2_5_6_7","1307.24"
+"vgg16","64","2_5_6_7","834.47"
+"inception3","64","2_5_6_7","870.81"
+"alexnet","128","2_5_6_7","11295.07"
+"resnet50","128","2_5_6_7","1398.24"
+"vgg16","128","2_5_6_7","887.01"
+"inception3","128","2_5_6_7","957.30"
+"alexnet","32","0_2_5_7","1635.41"
+"resnet50","32","0_2_5_7","1055.00"
+"vgg16","32","0_2_5_7","462.32"
+"inception3","32","0_2_5_7","753.23"
+"alexnet","64","0_2_5_7","3200.73"
+"resnet50","64","0_2_5_7","1300.62"
+"vgg16","64","0_2_5_7","629.43"
+"inception3","64","0_2_5_7","889.05"
+"alexnet","128","0_2_5_7","5964.19"
+"resnet50","128","0_2_5_7","1483.04"
+"vgg16","128","0_2_5_7","741.28"
+"inception3","128","0_2_5_7","960.95"
+"alexnet","32","2_4_5_6","2171.39"
+"resnet50","32","2_4_5_6","1070.82"
+"vgg16","32","2_4_5_6","601.90"
+"inception3","32","2_4_5_6","746.16"
+"alexnet","64","2_4_5_6","4150.33"
+"resnet50","64","2_4_5_6","1243.55"
+"vgg16","64","2_4_5_6","742.72"
+"inception3","64","2_4_5_6","896.52"
+"alexnet","128","2_4_5_6","7143.63"
+"resnet50","128","2_4_5_6","1401.09"
+"vgg16","128","2_4_5_6","839.61"
+"inception3","128","2_4_5_6","958.62"
+"alexnet","32","0_2_4_5","2354.15"
+"resnet50","32","0_2_4_5","1032.89"
+"vgg16","32","0_2_4_5","613.87"
+"inception3","32","0_2_4_5","729.32"
+"alexnet","64","0_2_4_5","4495.06"
+"resnet50","64","0_2_4_5","1260.24"
+"vgg16","64","0_2_4_5","765.78"
+"inception3","64","0_2_4_5","876.14"
+"alexnet","128","0_2_4_5","7904.20"
+"resnet50","128","0_2_4_5","1492.16"
+"vgg16","128","0_2_4_5","857.77"
+"inception3","128","0_2_4_5","945.31"
+"alexnet","32","0_2_3_7","3579.71"
+"resnet50","32","0_2_3_7","1074.78"
+"vgg16","32","0_2_3_7","717.17"
+"inception3","32","0_2_3_7","732.96"
+"alexnet","64","0_2_3_7","6539.66"
+"resnet50","64","0_2_3_7","1289.01"
+"vgg16","64","0_2_3_7","827.88"
+"inception3","64","0_2_3_7","880.23"
+"alexnet","128","0_2_3_7","10589.38"
+"resnet50","128","0_2_3_7","1496.59"
+"vgg16","128","0_2_3_7","892.50"
+"inception3","128","0_2_3_7","971.33"
+"alexnet","32","1_2_6_7","2550.26"
+"resnet50","32","1_2_6_7","1036.08"
+"vgg16","32","1_2_6_7","611.47"
+"inception3","32","1_2_6_7","725.47"
+"alexnet","64","1_2_6_7","4903.59"
+"resnet50","64","1_2_6_7","1244.58"
+"vgg16","64","1_2_6_7","756.48"
+"inception3","64","1_2_6_7","863.78"
+"alexnet","128","1_2_6_7","8454.19"
+"resnet50","128","1_2_6_7","1476.63"
+"vgg16","128","1_2_6_7","847.58"
+"inception3","128","1_2_6_7","948.30"
+"alexnet","32","1_2_3_4","2101.41"
+"resnet50","32","1_2_3_4","1052.81"
+"vgg16","32","1_2_3_4","578.65"
+"inception3","32","1_2_3_4","719.45"
+"alexnet","64","1_2_3_4","4015.34"
+"resnet50","64","1_2_3_4","1287.58"
+"vgg16","64","1_2_3_4","717.66"
+"inception3","64","1_2_3_4","863.78"
+"alexnet","128","1_2_3_4","7011.33"
+"resnet50","128","1_2_3_4","1436.30"
+"vgg16","128","1_2_3_4","811.12"
+"inception3","128","1_2_3_4","934.75"
+"alexnet","32","0_2_3_4","2319.82"
+"resnet50","32","0_2_3_4","1057.44"
+"vgg16","32","0_2_3_4","601.04"
+"inception3","32","0_2_3_4","743.73"
+"alexnet","64","0_2_3_4","4336.51"
+"resnet50","64","0_2_3_4","1310.97"
+"vgg16","64","0_2_3_4","751.55"
+"inception3","64","0_2_3_4","866.25"
+"alexnet","128","0_2_3_4","7529.48"
+"resnet50","128","0_2_3_4","1507.12"
+"vgg16","128","0_2_3_4","844.34"
+"inception3","128","0_2_3_4","950.39"
+"alexnet","32","0_1_2_7","2103.28"
+"resnet50","32","0_1_2_7","1043.94"
+"vgg16","32","0_1_2_7","583.05"
+"inception3","32","0_1_2_7","711.30"
+"alexnet","64","0_1_2_7","4000.02"
+"resnet50","64","0_1_2_7","1281.76"
+"vgg16","64","0_1_2_7","719.24"
+"inception3","64","0_1_2_7","848.68"
+"alexnet","128","0_1_2_7","6919.72"
+"resnet50","128","0_1_2_7","1474.99"
+"vgg16","128","0_1_2_7","803.57"
+"inception3","128","0_1_2_7","947.26"
+"alexnet","32","1_2_3_7","3405.12"
+"resnet50","32","1_2_3_7","1083.20"
+"vgg16","32","1_2_3_7","687.88"
+"inception3","32","1_2_3_7","725.02"
+"alexnet","64","1_2_3_7","6251.55"
+"resnet50","64","1_2_3_7","1305.84"
+"vgg16","64","1_2_3_7","808.06"
+"inception3","64","1_2_3_7","856.85"
+"alexnet","128","1_2_3_7","10091.66"
+"resnet50","128","1_2_3_7","1469.37"
+"vgg16","128","1_2_3_7","879.80"
+"inception3","128","1_2_3_7","950.54"
+"alexnet","32","0_1_4_6","2334.40"
+"resnet50","32","0_1_4_6","1057.47"
+"vgg16","32","0_1_4_6","616.21"
+"inception3","32","0_1_4_6","736.38"
+"alexnet","64","0_1_4_6","4448.70"
+"resnet50","64","0_1_4_6","1260.04"
+"vgg16","64","0_1_4_6","761.15"
+"inception3","64","0_1_4_6","872.99"
+"alexnet","128","0_1_4_6","7732.95"
+"resnet50","128","0_1_4_6","1479.43"
+"vgg16","128","0_1_4_6","844.91"
+"inception3","128","0_1_4_6","955.88"
+"alexnet","32","0_3_5_7","1726.13"
+"resnet50","32","0_3_5_7","1074.97"
+"vgg16","32","0_3_5_7","474.72"
+"inception3","32","0_3_5_7","754.76"
+"alexnet","64","0_3_5_7","3343.54"
+"resnet50","64","0_3_5_7","1247.79"
+"vgg16","64","0_3_5_7","632.21"
+"inception3","64","0_3_5_7","894.63"
+"alexnet","128","0_3_5_7","6162.82"
+"resnet50","128","0_3_5_7","1492.81"
+"vgg16","128","0_3_5_7","725.71"
+"inception3","128","0_3_5_7","947.66"
+"alexnet","32","0_1_5_6","2624.90"
+"resnet50","32","0_1_5_6","1059.91"
+"vgg16","32","0_1_5_6","638.85"
+"inception3","32","0_1_5_6","736.08"
+"alexnet","64","0_1_5_6","5040.98"
+"resnet50","64","0_1_5_6","1240.29"
+"vgg16","64","0_1_5_6","778.13"
+"inception3","64","0_1_5_6","860.36"
+"alexnet","128","0_1_5_6","8818.97"
+"resnet50","128","0_1_5_6","1422.96"
+"vgg16","128","0_1_5_6","853.59"
+"inception3","128","0_1_5_6","936.50"
+"alexnet","32","1_3_5_7","2683.02"
+"resnet50","32","1_3_5_7","1096.19"
+"vgg16","32","1_3_5_7","623.13"
+"inception3","32","1_3_5_7","756.16"
+"alexnet","64","1_3_5_7","5108.15"
+"resnet50","64","1_3_5_7","1315.23"
+"vgg16","64","1_3_5_7","759.67"
+"inception3","64","1_3_5_7","877.68"
+"alexnet","128","1_3_5_7","8696.88"
+"resnet50","128","1_3_5_7","1477.01"
+"vgg16","128","1_3_5_7","848.06"
+"inception3","128","1_3_5_7","964.49"
+"alexnet","32","0_4_5_6","2643.23"
+"resnet50","32","0_4_5_6","1055.51"
+"vgg16","32","0_4_5_6","628.09"
+"inception3","32","0_4_5_6","734.57"
+"alexnet","64","0_4_5_6","5098.51"
+"resnet50","64","0_4_5_6","1325.65"
+"vgg16","64","0_4_5_6","772.31"
+"inception3","64","0_4_5_6","866.67"
+"alexnet","128","0_4_5_6","8655.70"
+"resnet50","128","0_4_5_6","1413.59"
+"vgg16","128","0_4_5_6","859.95"
+"inception3","128","0_4_5_6","972.33"
+"alexnet","32","2_4_6_7","3510.08"
+"resnet50","32","2_4_6_7","1067.53"
+"vgg16","32","2_4_6_7","700.53"
+"inception3","32","2_4_6_7","742.03"
+"alexnet","64","2_4_6_7","6411.75"
+"resnet50","64","2_4_6_7","1294.08"
+"vgg16","64","2_4_6_7","815.06"
+"inception3","64","2_4_6_7","867.03"
+"alexnet","128","2_4_6_7","10336.64"
+"resnet50","128","2_4_6_7","1488.99"
+"vgg16","128","2_4_6_7","873.89"
+"inception3","128","2_4_6_7","955.98"
+"alexnet","32","0_1_6_7","1653.86"
+"resnet50","32","0_1_6_7","1041.01"
+"vgg16","32","0_1_6_7","474.83"
+"inception3","32","0_1_6_7","723.42"
+"alexnet","64","0_1_6_7","3218.06"
+"resnet50","64","0_1_6_7","1264.94"
+"vgg16","64","0_1_6_7","630.80"
+"inception3","64","0_1_6_7","870.68"
+"alexnet","128","0_1_6_7","5860.27"
+"resnet50","128","0_1_6_7","1462.48"
+"vgg16","128","0_1_6_7","720.57"
+"inception3","128","0_1_6_7","949.11"
+"alexnet","32","0_1_3_5","2484.45"
+"resnet50","32","0_1_3_5","1073.36"
+"vgg16","32","0_1_3_5","594.68"
+"inception3","32","0_1_3_5","742.14"
+"alexnet","64","0_1_3_5","4642.74"
+"resnet50","64","0_1_3_5","1302.13"
+"vgg16","64","0_1_3_5","744.95"
+"inception3","64","0_1_3_5","876.80"
+"alexnet","128","0_1_3_5","7997.45"
+"resnet50","128","0_1_3_5","1476.91"
+"vgg16","128","0_1_3_5","836.07"
+"inception3","128","0_1_3_5","946.61"
+"alexnet","32","2_4_5_7","2038.98"
+"resnet50","32","2_4_5_7","1041.34"
+"vgg16","32","2_4_5_7","570.29"
+"inception3","32","2_4_5_7","729.67"
+"alexnet","64","2_4_5_7","3893.32"
+"resnet50","64","2_4_5_7","1283.24"
+"vgg16","64","2_4_5_7","689.05"
+"inception3","64","2_4_5_7","882.87"
+"alexnet","128","2_4_5_7","6910.99"
+"resnet50","128","2_4_5_7","1493.99"
+"vgg16","128","2_4_5_7","779.97"
+"inception3","128","2_4_5_7","971.35"
+"alexnet","32","1_4_6_7","2133.74"
+"resnet50","32","1_4_6_7","1065.26"
+"vgg16","32","1_4_6_7","581.22"
+"inception3","32","1_4_6_7","730.57"
+"alexnet","64","1_4_6_7","4053.08"
+"resnet50","64","1_4_6_7","1245.20"
+"vgg16","64","1_4_6_7","700.96"
+"inception3","64","1_4_6_7","880.38"
+"alexnet","128","1_4_6_7","7150.40"
+"resnet50","128","1_4_6_7","1460.20"
+"vgg16","128","1_4_6_7","771.73"
+"inception3","128","1_4_6_7","966.67"
+"alexnet","32","1_2_4_5","1659.42"
+"resnet50","32","1_2_4_5","1068.29"
+"vgg16","32","1_2_4_5","470.06"
+"inception3","32","1_2_4_5","726.71"
+"alexnet","64","1_2_4_5","3233.41"
+"resnet50","64","1_2_4_5","1301.97"
+"vgg16","64","1_2_4_5","646.86"
+"inception3","64","1_2_4_5","861.89"
+"alexnet","128","1_2_4_5","5951.10"
+"resnet50","128","1_2_4_5","1486.53"
+"vgg16","128","1_2_4_5","759.77"
+"inception3","128","1_2_4_5","971.02"
+"alexnet","32","2_3_4_6","1662.92"
+"resnet50","32","2_3_4_6","1066.89"
+"vgg16","32","2_3_4_6","474.97"
+"inception3","32","2_3_4_6","740.54"
+"alexnet","64","2_3_4_6","3234.39"
+"resnet50","64","2_3_4_6","1302.43"
+"vgg16","64","2_3_4_6","631.43"
+"inception3","64","2_3_4_6","878.42"
+"alexnet","128","2_3_4_6","6004.72"
+"resnet50","128","2_3_4_6","1502.46"
+"vgg16","128","2_3_4_6","765.93"
+"inception3","128","2_3_4_6","970.87"
+"alexnet","32","1_2_5_7","2387.33"
+"resnet50","32","1_2_5_7","1086.66"
+"vgg16","32","1_2_5_7","622.22"
+"inception3","32","1_2_5_7","747.42"
+"alexnet","64","1_2_5_7","4572.99"
+"resnet50","64","1_2_5_7","1309.35"
+"vgg16","64","1_2_5_7","762.45"
+"inception3","64","1_2_5_7","896.42"
+"alexnet","128","1_2_5_7","8047.71"
+"resnet50","128","1_2_5_7","1487.27"
+"vgg16","128","1_2_5_7","850.19"
+"inception3","128","1_2_5_7","954.46"
+"alexnet","32","2_3_5_6","1716.47"
+"resnet50","32","2_3_5_6","1073.20"
+"vgg16","32","2_3_5_6","484.66"
+"inception3","32","2_3_5_6","745.13"
+"alexnet","64","2_3_5_6","3327.57"
+"resnet50","64","2_3_5_6","1286.68"
+"vgg16","64","2_3_5_6","649.98"
+"inception3","64","2_3_5_6","889.26"
+"alexnet","128","2_3_5_6","6203.28"
+"resnet50","128","2_3_5_6","1481.07"
+"vgg16","128","2_3_5_6","761.74"
+"inception3","128","2_3_5_6","973.17"
+"alexnet","32","0_3_4_6_7","2096.95"
+"resnet50","32","0_3_4_6_7","1244.62"
+"vgg16","32","0_3_4_6_7","590.19"
+"inception3","32","0_3_4_6_7","890.80"
+"alexnet","64","0_3_4_6_7","4038.73"
+"resnet50","64","0_3_4_6_7","1587.11"
+"vgg16","64","0_3_4_6_7","787.02"
+"inception3","64","0_3_4_6_7","1087.92"
+"alexnet","128","0_3_4_6_7","7385.17"
+"resnet50","128","0_3_4_6_7","1807.63"
+"vgg16","128","0_3_4_6_7","901.62"
+"inception3","128","0_3_4_6_7","1188.98"
+"alexnet","32","1_2_4_5_6","2045.66"
+"resnet50","32","1_2_4_5_6","1233.05"
+"vgg16","32","1_2_4_5_6","733.63"
+"inception3","32","1_2_4_5_6","912.50"
+"alexnet","64","1_2_4_5_6","3937.85"
+"resnet50","64","1_2_4_5_6","1519.26"
+"vgg16","64","1_2_4_5_6","924.42"
+"inception3","64","1_2_4_5_6","1102.64"
+"alexnet","128","1_2_4_5_6","7130.98"
+"resnet50","128","1_2_4_5_6","1749.82"
+"vgg16","128","1_2_4_5_6","1046.62"
+"inception3","128","1_2_4_5_6","1210.15"
+"alexnet","32","0_1_2_5_7","1633.01"
+"resnet50","32","0_1_2_5_7","1294.58"
+"vgg16","32","0_1_2_5_7","593.93"
+"inception3","32","0_1_2_5_7","907.39"
+"alexnet","64","0_1_2_5_7","3217.48"
+"resnet50","64","0_1_2_5_7","1453.33"
+"vgg16","64","0_1_2_5_7","790.96"
+"inception3","64","0_1_2_5_7","1083.60"
+"alexnet","128","0_1_2_5_7","5926.63"
+"resnet50","128","0_1_2_5_7","1820.01"
+"vgg16","128","0_1_2_5_7","928.79"
+"inception3","128","0_1_2_5_7","1210.40"
+"alexnet","32","0_1_2_3_5","2400.05"
+"resnet50","32","0_1_2_3_5","1280.20"
+"vgg16","32","0_1_2_3_5","711.67"
+"inception3","32","0_1_2_3_5","903.58"
+"alexnet","64","0_1_2_3_5","4586.97"
+"resnet50","64","0_1_2_3_5","1512.00"
+"vgg16","64","0_1_2_3_5","905.37"
+"inception3","64","0_1_2_3_5","1084.90"
+"alexnet","128","0_1_2_3_5","8123.87"
+"resnet50","128","0_1_2_3_5","1730.12"
+"vgg16","128","0_1_2_3_5","1025.92"
+"inception3","128","0_1_2_3_5","1200.12"
+"alexnet","32","0_1_3_6_7","1792.58"
+"resnet50","32","0_1_3_6_7","1258.17"
+"vgg16","32","0_1_3_6_7","486.53"
+"inception3","32","0_1_3_6_7","897.38"
+"alexnet","64","0_1_3_6_7","3486.49"
+"resnet50","64","0_1_3_6_7","1567.40"
+"vgg16","64","0_1_3_6_7","690.39"
+"inception3","64","0_1_3_6_7","1076.72"
+"alexnet","128","0_1_3_6_7","6414.55"
+"resnet50","128","0_1_3_6_7","1724.28"
+"vgg16","128","0_1_3_6_7","862.16"
+"inception3","128","0_1_3_6_7","1182.86"
+"alexnet","32","0_1_2_3_6","2747.52"
+"resnet50","32","0_1_2_3_6","1303.52"
+"vgg16","32","0_1_2_3_6","692.67"
+"inception3","32","0_1_2_3_6","904.00"
+"alexnet","64","0_1_2_3_6","5232.22"
+"resnet50","64","0_1_2_3_6","1466.35"
+"vgg16","64","0_1_2_3_6","878.76"
+"inception3","64","0_1_2_3_6","1074.37"
+"alexnet","128","0_1_2_3_6","9005.99"
+"resnet50","128","0_1_2_3_6","1736.32"
+"vgg16","128","0_1_2_3_6","972.20"
+"inception3","128","0_1_2_3_6","1197.24"
+"alexnet","32","0_1_3_4_7","2878.23"
+"resnet50","32","0_1_3_4_7","1279.76"
+"vgg16","32","0_1_3_4_7","605.28"
+"inception3","32","0_1_3_4_7","909.46"
+"alexnet","64","0_1_3_4_7","5433.16"
+"resnet50","64","0_1_3_4_7","1551.98"
+"vgg16","64","0_1_3_4_7","816.51"
+"inception3","64","0_1_3_4_7","1082.58"
+"alexnet","128","0_1_3_4_7","9513.94"
+"resnet50","128","0_1_3_4_7","1808.61"
+"vgg16","128","0_1_3_4_7","917.26"
+"inception3","128","0_1_3_4_7","1181.58"
+"alexnet","32","0_3_4_5_7","2070.60"
+"resnet50","32","0_3_4_5_7","1294.73"
+"vgg16","32","0_3_4_5_7","569.90"
+"inception3","32","0_3_4_5_7","907.48"
+"alexnet","64","0_3_4_5_7","4016.00"
+"resnet50","64","0_3_4_5_7","1482.22"
+"vgg16","64","0_3_4_5_7","778.48"
+"inception3","64","0_3_4_5_7","1095.07"
+"alexnet","128","0_3_4_5_7","7327.48"
+"resnet50","128","0_3_4_5_7","1662.35"
+"vgg16","128","0_3_4_5_7","905.78"
+"inception3","128","0_3_4_5_7","1197.05"
+"alexnet","32","3_4_5_6_7","1551.26"
+"resnet50","32","3_4_5_6_7","1302.50"
+"vgg16","32","3_4_5_6_7","731.90"
+"inception3","32","3_4_5_6_7","908.88"
+"alexnet","64","3_4_5_6_7","3082.40"
+"resnet50","64","3_4_5_6_7","1574.96"
+"vgg16","64","3_4_5_6_7","934.12"
+"inception3","64","3_4_5_6_7","1092.49"
+"alexnet","128","3_4_5_6_7","5733.47"
+"resnet50","128","3_4_5_6_7","1737.14"
+"vgg16","128","3_4_5_6_7","1046.69"
+"inception3","128","3_4_5_6_7","1187.95"
+"alexnet","32","1_3_4_5_7","2006.56"
+"resnet50","32","1_3_4_5_7","1256.90"
+"vgg16","32","1_3_4_5_7","702.96"
+"inception3","32","1_3_4_5_7","912.71"
+"alexnet","64","1_3_4_5_7","3842.77"
+"resnet50","64","1_3_4_5_7","1607.00"
+"vgg16","64","1_3_4_5_7","904.43"
+"inception3","64","1_3_4_5_7","1081.03"
+"alexnet","128","1_3_4_5_7","6934.98"
+"resnet50","128","1_3_4_5_7","1676.79"
+"vgg16","128","1_3_4_5_7","1021.92"
+"inception3","128","1_3_4_5_7","1203.02"
+"alexnet","32","1_3_4_6_7","1501.06"
+"resnet50","32","1_3_4_6_7","1270.40"
+"vgg16","32","1_3_4_6_7","576.07"
+"inception3","32","1_3_4_6_7","891.71"
+"alexnet","64","1_3_4_6_7","2944.82"
+"resnet50","64","1_3_4_6_7","1462.29"
+"vgg16","64","1_3_4_6_7","767.65"
+"inception3","64","1_3_4_6_7","1095.94"
+"alexnet","128","1_3_4_6_7","5532.55"
+"resnet50","128","1_3_4_6_7","1739.56"
+"vgg16","128","1_3_4_6_7","903.32"
+"inception3","128","1_3_4_6_7","1192.67"
+"alexnet","32","0_2_4_5_6","2000.13"
+"resnet50","32","0_2_4_5_6","1239.91"
+"vgg16","32","0_2_4_5_6","581.25"
+"inception3","32","0_2_4_5_6","902.46"
+"alexnet","64","0_2_4_5_6","3880.47"
+"resnet50","64","0_2_4_5_6","1614.45"
+"vgg16","64","0_2_4_5_6","788.24"
+"inception3","64","0_2_4_5_6","1102.10"
+"alexnet","128","0_2_4_5_6","7162.67"
+"resnet50","128","0_2_4_5_6","1717.21"
+"vgg16","128","0_2_4_5_6","916.92"
+"inception3","128","0_2_4_5_6","1216.70"
+"alexnet","32","1_2_3_4_7","1866.07"
+"resnet50","32","1_2_3_4_7","1223.27"
+"vgg16","32","1_2_3_4_7","501.97"
+"inception3","32","1_2_3_4_7","912.99"
+"alexnet","64","1_2_3_4_7","3571.67"
+"resnet50","64","1_2_3_4_7","1475.68"
+"vgg16","64","1_2_3_4_7","695.44"
+"inception3","64","1_2_3_4_7","1096.39"
+"alexnet","128","1_2_3_4_7","6556.04"
+"resnet50","128","1_2_3_4_7","1675.83"
+"vgg16","128","1_2_3_4_7","848.36"
+"inception3","128","1_2_3_4_7","1220.71"
+"alexnet","32","1_3_4_5_6","1969.43"
+"resnet50","32","1_3_4_5_6","1291.81"
+"vgg16","32","1_3_4_5_6","717.73"
+"inception3","32","1_3_4_5_6","918.01"
+"alexnet","64","1_3_4_5_6","3822.32"
+"resnet50","64","1_3_4_5_6","1549.91"
+"vgg16","64","1_3_4_5_6","916.83"
+"inception3","64","1_3_4_5_6","1100.01"
+"alexnet","128","1_3_4_5_6","6957.16"
+"resnet50","128","1_3_4_5_6","1680.00"
+"vgg16","128","1_3_4_5_6","1038.11"
+"inception3","128","1_3_4_5_6","1212.12"
+"alexnet","32","0_1_4_5_6","1942.22"
+"resnet50","32","0_1_4_5_6","1245.53"
+"vgg16","32","0_1_4_5_6","718.38"
+"inception3","32","0_1_4_5_6","911.16"
+"alexnet","64","0_1_4_5_6","3781.73"
+"resnet50","64","0_1_4_5_6","1599.24"
+"vgg16","64","0_1_4_5_6","924.70"
+"inception3","64","0_1_4_5_6","1085.81"
+"alexnet","128","0_1_4_5_6","6931.85"
+"resnet50","128","0_1_4_5_6","1669.00"
+"vgg16","128","0_1_4_5_6","1044.24"
+"inception3","128","0_1_4_5_6","1195.83"
+"alexnet","32","1_2_5_6_7","2101.63"
+"resnet50","32","1_2_5_6_7","1259.52"
+"vgg16","32","1_2_5_6_7","732.89"
+"inception3","32","1_2_5_6_7","898.59"
+"alexnet","64","1_2_5_6_7","4070.02"
+"resnet50","64","1_2_5_6_7","1554.38"
+"vgg16","64","1_2_5_6_7","928.11"
+"inception3","64","1_2_5_6_7","1074.85"
+"alexnet","128","1_2_5_6_7","7376.29"
+"resnet50","128","1_2_5_6_7","1818.20"
+"vgg16","128","1_2_5_6_7","1045.92"
+"inception3","128","1_2_5_6_7","1201.52"
+"alexnet","32","2_3_4_6_7","1947.96"
+"resnet50","32","2_3_4_6_7","1275.36"
+"vgg16","32","2_3_4_6_7","697.03"
+"inception3","32","2_3_4_6_7","896.69"
+"alexnet","64","2_3_4_6_7","3782.46"
+"resnet50","64","2_3_4_6_7","1519.12"
+"vgg16","64","2_3_4_6_7","902.11"
+"inception3","64","2_3_4_6_7","1083.28"
+"alexnet","128","2_3_4_6_7","6839.63"
+"resnet50","128","2_3_4_6_7","1796.80"
+"vgg16","128","2_3_4_6_7","1019.91"
+"inception3","128","2_3_4_6_7","1206.84"
+"alexnet","32","1_2_3_4_5","2260.75"
+"resnet50","32","1_2_3_4_5","1228.11"
+"vgg16","32","1_2_3_4_5","479.68"
+"inception3","32","1_2_3_4_5","898.90"
+"alexnet","64","1_2_3_4_5","4367.22"
+"resnet50","64","1_2_3_4_5","1574.11"
+"vgg16","64","1_2_3_4_5","681.27"
+"inception3","64","1_2_3_4_5","1097.51"
+"alexnet","128","1_2_3_4_5","7818.37"
+"resnet50","128","1_2_3_4_5","1823.85"
+"vgg16","128","1_2_3_4_5","835.01"
+"inception3","128","1_2_3_4_5","1211.19"
+"alexnet","32","1_2_4_6_7","1541.21"
+"resnet50","32","1_2_4_6_7","1260.23"
+"vgg16","32","1_2_4_6_7","701.59"
+"inception3","32","1_2_4_6_7","890.74"
+"alexnet","64","1_2_4_6_7","3007.49"
+"resnet50","64","1_2_4_6_7","1574.34"
+"vgg16","64","1_2_4_6_7","906.03"
+"inception3","64","1_2_4_6_7","1079.29"
+"alexnet","128","1_2_4_6_7","5700.88"
+"resnet50","128","1_2_4_6_7","1668.05"
+"vgg16","128","1_2_4_6_7","1024.56"
+"inception3","128","1_2_4_6_7","1200.23"
+"alexnet","32","2_3_4_5_6","1953.85"
+"resnet50","32","2_3_4_5_6","1304.47"
+"vgg16","32","2_3_4_5_6","574.17"
+"inception3","32","2_3_4_5_6","913.96"
+"alexnet","64","2_3_4_5_6","3763.69"
+"resnet50","64","2_3_4_5_6","1453.61"
+"vgg16","64","2_3_4_5_6","783.19"
+"inception3","64","2_3_4_5_6","1084.33"
+"alexnet","128","2_3_4_5_6","6870.74"
+"resnet50","128","2_3_4_5_6","1812.86"
+"vgg16","128","2_3_4_5_6","921.20"
+"inception3","128","2_3_4_5_6","1215.62"
+"alexnet","32","1_2_4_5_7","2047.07"
+"resnet50","32","1_2_4_5_7","1286.74"
+"vgg16","32","1_2_4_5_7","699.49"
+"inception3","32","1_2_4_5_7","901.24"
+"alexnet","64","1_2_4_5_7","3951.68"
+"resnet50","64","1_2_4_5_7","1608.10"
+"vgg16","64","1_2_4_5_7","903.56"
+"inception3","64","1_2_4_5_7","1093.76"
+"alexnet","128","1_2_4_5_7","7173.19"
+"resnet50","128","1_2_4_5_7","1686.07"
+"vgg16","128","1_2_4_5_7","1022.74"
+"inception3","128","1_2_4_5_7","1183.73"
+"alexnet","32","1_2_3_5_7","2897.10"
+"resnet50","32","1_2_3_5_7","1242.68"
+"vgg16","32","1_2_3_5_7","596.50"
+"inception3","32","1_2_3_5_7","920.73"
+"alexnet","64","1_2_3_5_7","5433.04"
+"resnet50","64","1_2_3_5_7","1597.46"
+"vgg16","64","1_2_3_5_7","797.26"
+"inception3","64","1_2_3_5_7","1100.52"
+"alexnet","128","1_2_3_5_7","9612.76"
+"resnet50","128","1_2_3_5_7","1830.24"
+"vgg16","128","1_2_3_5_7","907.01"
+"inception3","128","1_2_3_5_7","1220.14"
+"alexnet","32","1_4_5_6_7","1603.67"
+"resnet50","32","1_4_5_6_7","1291.48"
+"vgg16","32","1_4_5_6_7","726.37"
+"inception3","32","1_4_5_6_7","911.02"
+"alexnet","64","1_4_5_6_7","3113.77"
+"resnet50","64","1_4_5_6_7","1585.38"
+"vgg16","64","1_4_5_6_7","915.06"
+"inception3","64","1_4_5_6_7","1096.72"
+"alexnet","128","1_4_5_6_7","5931.87"
+"resnet50","128","1_4_5_6_7","1822.17"
+"vgg16","128","1_4_5_6_7","1028.02"
+"inception3","128","1_4_5_6_7","1193.14"
+"alexnet","32","0_1_2_5_6","1813.13"
+"resnet50","32","0_1_2_5_6","1296.12"
+"vgg16","32","0_1_2_5_6","606.32"
+"inception3","32","0_1_2_5_6","910.30"
+"alexnet","64","0_1_2_5_6","3494.02"
+"resnet50","64","0_1_2_5_6","1599.53"
+"vgg16","64","0_1_2_5_6","820.42"
+"inception3","64","0_1_2_5_6","1079.77"
+"alexnet","128","0_1_2_5_6","6392.69"
+"resnet50","128","0_1_2_5_6","1716.36"
+"vgg16","128","0_1_2_5_6","934.58"
+"inception3","128","0_1_2_5_6","1200.88"
+"alexnet","32","0_1_2_3_4","3975.48"
+"resnet50","32","0_1_2_3_4","1253.12"
+"vgg16","32","0_1_2_3_4","701.85"
+"inception3","32","0_1_2_3_4","897.48"
+"alexnet","64","0_1_2_3_4","7339.87"
+"resnet50","64","0_1_2_3_4","1607.19"
+"vgg16","64","0_1_2_3_4","894.14"
+"inception3","64","0_1_2_3_4","1063.87"
+"alexnet","128","0_1_2_3_4","11499.26"
+"resnet50","128","0_1_2_3_4","1772.07"
+"vgg16","128","0_1_2_3_4","1020.44"
+"inception3","128","0_1_2_3_4","1216.59"
+"alexnet","32","0_1_2_3_7","2454.76"
+"resnet50","32","0_1_2_3_7","1306.53"
+"vgg16","32","0_1_2_3_7","862.78"
+"inception3","32","0_1_2_3_7","897.13"
+"alexnet","64","0_1_2_3_7","4557.99"
+"resnet50","64","0_1_2_3_7","1546.31"
+"vgg16","64","0_1_2_3_7","1021.36"
+"inception3","64","0_1_2_3_7","1082.21"
+"alexnet","128","0_1_2_3_7","8140.78"
+"resnet50","128","0_1_2_3_7","1735.50"
+"vgg16","128","0_1_2_3_7","1108.50"
+"inception3","128","0_1_2_3_7","1195.33"
+"alexnet","32","0_2_3_5_6","1716.98"
+"resnet50","32","0_2_3_5_6","1258.91"
+"vgg16","32","0_2_3_5_6","498.76"
+"inception3","32","0_2_3_5_6","922.09"
+"alexnet","64","0_2_3_5_6","3301.12"
+"resnet50","64","0_2_3_5_6","1507.59"
+"vgg16","64","0_2_3_5_6","689.59"
+"inception3","64","0_2_3_5_6","1107.12"
+"alexnet","128","0_2_3_5_6","6003.82"
+"resnet50","128","0_2_3_5_6","1722.08"
+"vgg16","128","0_2_3_5_6","820.35"
+"inception3","128","0_2_3_5_6","1222.28"
+"alexnet","32","0_2_3_5_7","1862.54"
+"resnet50","32","0_2_3_5_7","1219.84"
+"vgg16","32","0_2_3_5_7","480.12"
+"inception3","32","0_2_3_5_7","917.75"
+"alexnet","64","0_2_3_5_7","3588.53"
+"resnet50","64","0_2_3_5_7","1597.62"
+"vgg16","64","0_2_3_5_7","685.61"
+"inception3","64","0_2_3_5_7","1101.13"
+"alexnet","128","0_2_3_5_7","6535.39"
+"resnet50","128","0_2_3_5_7","1824.97"
+"vgg16","128","0_2_3_5_7","847.97"
+"inception3","128","0_2_3_5_7","1201.51"
+"alexnet","32","1_3_5_6_7","2028.19"
+"resnet50","32","1_3_5_6_7","1270.26"
+"vgg16","32","1_3_5_6_7","598.51"
+"inception3","32","1_3_5_6_7","908.74"
+"alexnet","64","1_3_5_6_7","3924.72"
+"resnet50","64","1_3_5_6_7","1583.99"
+"vgg16","64","1_3_5_6_7","808.09"
+"inception3","64","1_3_5_6_7","1074.91"
+"alexnet","128","1_3_5_6_7","7191.10"
+"resnet50","128","1_3_5_6_7","1819.90"
+"vgg16","128","1_3_5_6_7","915.02"
+"inception3","128","1_3_5_6_7","1174.29"
+"alexnet","32","0_1_2_4_7","2225.38"
+"resnet50","32","0_1_2_4_7","1239.60"
+"vgg16","32","0_1_2_4_7","608.73"
+"inception3","32","0_1_2_4_7","903.68"
+"alexnet","64","0_1_2_4_7","4286.77"
+"resnet50","64","0_1_2_4_7","1502.18"
+"vgg16","64","0_1_2_4_7","806.50"
+"inception3","64","0_1_2_4_7","1079.52"
+"alexnet","128","0_1_2_4_7","7752.77"
+"resnet50","128","0_1_2_4_7","1820.19"
+"vgg16","128","0_1_2_4_7","935.89"
+"inception3","128","0_1_2_4_7","1190.31"
+"alexnet","32","0_2_3_4_6","2308.84"
+"resnet50","32","0_2_3_4_6","1291.10"
+"vgg16","32","0_2_3_4_6","575.20"
+"inception3","32","0_2_3_4_6","918.15"
+"alexnet","64","0_2_3_4_6","4453.95"
+"resnet50","64","0_2_3_4_6","1589.10"
+"vgg16","64","0_2_3_4_6","764.42"
+"inception3","64","0_2_3_4_6","1099.41"
+"alexnet","128","0_2_3_4_6","7738.27"
+"resnet50","128","0_2_3_4_6","1850.12"
+"vgg16","128","0_2_3_4_6","911.52"
+"inception3","128","0_2_3_4_6","1220.61"
+"alexnet","32","1_2_3_6_7","1795.92"
+"resnet50","32","1_2_3_6_7","1264.08"
+"vgg16","32","1_2_3_6_7","588.18"
+"inception3","32","1_2_3_6_7","885.45"
+"alexnet","64","1_2_3_6_7","3495.89"
+"resnet50","64","1_2_3_6_7","1566.70"
+"vgg16","64","1_2_3_6_7","793.96"
+"inception3","64","1_2_3_6_7","1086.36"
+"alexnet","128","1_2_3_6_7","6328.07"
+"resnet50","128","1_2_3_6_7","1810.12"
+"vgg16","128","1_2_3_6_7","908.03"
+"inception3","128","1_2_3_6_7","1204.87"
+"alexnet","32","2_4_5_6_7","1532.96"
+"resnet50","32","2_4_5_6_7","1308.29"
+"vgg16","32","2_4_5_6_7","848.21"
+"inception3","32","2_4_5_6_7","888.48"
+"alexnet","64","2_4_5_6_7","2995.40"
+"resnet50","64","2_4_5_6_7","1521.41"
+"vgg16","64","2_4_5_6_7","1010.29"
+"inception3","64","2_4_5_6_7","1065.38"
+"alexnet","128","2_4_5_6_7","5640.42"
+"resnet50","128","2_4_5_6_7","1819.17"
+"vgg16","128","2_4_5_6_7","1103.84"
+"inception3","128","2_4_5_6_7","1188.76"
+"alexnet","32","0_3_4_5_6","2037.81"
+"resnet50","32","0_3_4_5_6","1238.72"
+"vgg16","32","0_3_4_5_6","575.54"
+"inception3","32","0_3_4_5_6","921.91"
+"alexnet","64","0_3_4_5_6","3947.48"
+"resnet50","64","0_3_4_5_6","1581.33"
+"vgg16","64","0_3_4_5_6","785.54"
+"inception3","64","0_3_4_5_6","1113.43"
+"alexnet","128","0_3_4_5_6","7286.87"
+"resnet50","128","0_3_4_5_6","1837.29"
+"vgg16","128","0_3_4_5_6","927.01"
+"inception3","128","0_3_4_5_6","1204.41"
+"alexnet","32","2_3_5_6_7","1929.71"
+"resnet50","32","2_3_5_6_7","1288.97"
+"vgg16","32","2_3_5_6_7","720.06"
+"inception3","32","2_3_5_6_7","903.91"
+"alexnet","64","2_3_5_6_7","3745.20"
+"resnet50","64","2_3_5_6_7","1592.49"
+"vgg16","64","2_3_5_6_7","918.45"
+"inception3","64","2_3_5_6_7","1088.83"
+"alexnet","128","2_3_5_6_7","6819.02"
+"resnet50","128","2_3_5_6_7","1811.84"
+"vgg16","128","2_3_5_6_7","1036.38"
+"inception3","128","2_3_5_6_7","1210.50"
+"alexnet","32","0_2_3_6_7","1801.52"
+"resnet50","32","0_2_3_6_7","1258.58"
+"vgg16","32","0_2_3_6_7","586.08"
+"inception3","32","0_2_3_6_7","912.11"
+"alexnet","64","0_2_3_6_7","3494.92"
+"resnet50","64","0_2_3_6_7","1570.03"
+"vgg16","64","0_2_3_6_7","782.41"
+"inception3","64","0_2_3_6_7","1086.17"
+"alexnet","128","0_2_3_6_7","6350.27"
+"resnet50","128","0_2_3_6_7","1813.90"
+"vgg16","128","0_2_3_6_7","904.44"
+"inception3","128","0_2_3_6_7","1203.68"
+"alexnet","32","0_2_5_6_7","1508.65"
+"resnet50","32","0_2_5_6_7","1262.68"
+"vgg16","32","0_2_5_6_7","722.83"
+"inception3","32","0_2_5_6_7","901.51"
+"alexnet","64","0_2_5_6_7","2953.61"
+"resnet50","64","0_2_5_6_7","1568.43"
+"vgg16","64","0_2_5_6_7","921.60"
+"inception3","64","0_2_5_6_7","1084.05"
+"alexnet","128","0_2_5_6_7","5540.83"
+"resnet50","128","0_2_5_6_7","1661.66"
+"vgg16","128","0_2_5_6_7","1034.61"
+"inception3","128","0_2_5_6_7","1182.24"
+"alexnet","32","0_1_3_4_6","2308.20"
+"resnet50","32","0_1_3_4_6","1269.52"
+"vgg16","32","0_1_3_4_6","580.66"
+"inception3","32","0_1_3_4_6","904.28"
+"alexnet","64","0_1_3_4_6","4373.01"
+"resnet50","64","0_1_3_4_6","1589.51"
+"vgg16","64","0_1_3_4_6","786.99"
+"inception3","64","0_1_3_4_6","1083.95"
+"alexnet","128","0_1_3_4_6","7965.81"
+"resnet50","128","0_1_3_4_6","1664.74"
+"vgg16","128","0_1_3_4_6","941.74"
+"inception3","128","0_1_3_4_6","1192.98"
+"alexnet","32","2_3_4_5_7","1523.02"
+"resnet50","32","2_3_4_5_7","1232.37"
+"vgg16","32","2_3_4_5_7","562.17"
+"inception3","32","2_3_4_5_7","893.30"
+"alexnet","64","2_3_4_5_7","2982.99"
+"resnet50","64","2_3_4_5_7","1600.11"
+"vgg16","64","2_3_4_5_7","747.26"
+"inception3","64","2_3_4_5_7","1091.18"
+"alexnet","128","2_3_4_5_7","5516.58"
+"resnet50","128","2_3_4_5_7","1718.25"
+"vgg16","128","2_3_4_5_7","899.51"
+"inception3","128","2_3_4_5_7","1207.92"
+"alexnet","32","0_2_4_5_7","1969.53"
+"resnet50","32","0_2_4_5_7","1290.92"
+"vgg16","32","0_2_4_5_7","565.11"
+"inception3","32","0_2_4_5_7","903.96"
+"alexnet","64","0_2_4_5_7","3864.06"
+"resnet50","64","0_2_4_5_7","1541.56"
+"vgg16","64","0_2_4_5_7","763.79"
+"inception3","64","0_2_4_5_7","1094.76"
+"alexnet","128","0_2_4_5_7","7134.71"
+"resnet50","128","0_2_4_5_7","1811.45"
+"vgg16","128","0_2_4_5_7","922.02"
+"inception3","128","0_2_4_5_7","1204.41"
+"alexnet","32","0_1_2_4_5","2092.77"
+"resnet50","32","0_1_2_4_5","1258.84"
+"vgg16","32","0_1_2_4_5","587.64"
+"inception3","32","0_1_2_4_5","893.73"
+"alexnet","64","0_1_2_4_5","4131.13"
+"resnet50","64","0_1_2_4_5","1471.17"
+"vgg16","64","0_1_2_4_5","762.08"
+"inception3","64","0_1_2_4_5","1072.60"
+"alexnet","128","0_1_2_4_5","7380.87"
+"resnet50","128","0_1_2_4_5","1794.98"
+"vgg16","128","0_1_2_4_5","883.78"
+"inception3","128","0_1_2_4_5","1205.11"
+"alexnet","32","0_1_2_4_6","2738.88"
+"resnet50","32","0_1_2_4_6","1246.36"
+"vgg16","32","0_1_2_4_6","583.17"
+"inception3","32","0_1_2_4_6","918.12"
+"alexnet","64","0_1_2_4_6","5234.36"
+"resnet50","64","0_1_2_4_6","1474.60"
+"vgg16","64","0_1_2_4_6","798.93"
+"inception3","64","0_1_2_4_6","1087.87"
+"alexnet","128","0_1_2_4_6","9053.17"
+"resnet50","128","0_1_2_4_6","1826.32"
+"vgg16","128","0_1_2_4_6","940.27"
+"inception3","128","0_1_2_4_6","1200.63"
+"alexnet","32","0_2_3_4_5","2166.06"
+"resnet50","32","0_2_3_4_5","1227.28"
+"vgg16","32","0_2_3_4_5","587.18"
+"inception3","32","0_2_3_4_5","898.74"
+"alexnet","64","0_2_3_4_5","4135.93"
+"resnet50","64","0_2_3_4_5","1563.88"
+"vgg16","64","0_2_3_4_5","794.15"
+"inception3","64","0_2_3_4_5","1092.84"
+"alexnet","128","0_2_3_4_5","7566.69"
+"resnet50","128","0_2_3_4_5","1684.63"
+"vgg16","128","0_2_3_4_5","940.31"
+"inception3","128","0_2_3_4_5","1210.55"
+"alexnet","32","0_1_5_6_7","1495.78"
+"resnet50","32","0_1_5_6_7","1244.62"
+"vgg16","32","0_1_5_6_7","589.59"
+"inception3","32","0_1_5_6_7","895.15"
+"alexnet","64","0_1_5_6_7","2946.50"
+"resnet50","64","0_1_5_6_7","1497.50"
+"vgg16","64","0_1_5_6_7","807.82"
+"inception3","64","0_1_5_6_7","1075.77"
+"alexnet","128","0_1_5_6_7","5440.57"
+"resnet50","128","0_1_5_6_7","1802.65"
+"vgg16","128","0_1_5_6_7","907.90"
+"inception3","128","0_1_5_6_7","1192.22"
+"alexnet","32","0_4_5_6_7","1585.16"
+"resnet50","32","0_4_5_6_7","1272.69"
+"vgg16","32","0_4_5_6_7","738.65"
+"inception3","32","0_4_5_6_7","905.05"
+"alexnet","64","0_4_5_6_7","3125.42"
+"resnet50","64","0_4_5_6_7","1539.23"
+"vgg16","64","0_4_5_6_7","932.79"
+"inception3","64","0_4_5_6_7","1078.71"
+"alexnet","128","0_4_5_6_7","5823.83"
+"resnet50","128","0_4_5_6_7","1824.26"
+"vgg16","128","0_4_5_6_7","1054.81"
+"inception3","128","0_4_5_6_7","1183.67"
+"alexnet","32","0_1_3_4_5","2187.28"
+"resnet50","32","0_1_3_4_5","1252.56"
+"vgg16","32","0_1_3_4_5","588.21"
+"inception3","32","0_1_3_4_5","903.21"
+"alexnet","64","0_1_3_4_5","4174.69"
+"resnet50","64","0_1_3_4_5","1554.63"
+"vgg16","64","0_1_3_4_5","781.55"
+"inception3","64","0_1_3_4_5","1069.52"
+"alexnet","128","0_1_3_4_5","7531.90"
+"resnet50","128","0_1_3_4_5","1828.37"
+"vgg16","128","0_1_3_4_5","864.97"
+"inception3","128","0_1_3_4_5","1197.79"
+"alexnet","32","0_1_4_6_7","1952.57"
+"resnet50","32","0_1_4_6_7","1240.84"
+"vgg16","32","0_1_4_6_7","576.15"
+"inception3","32","0_1_4_6_7","891.27"
+"alexnet","64","0_1_4_6_7","3769.86"
+"resnet50","64","0_1_4_6_7","1569.08"
+"vgg16","64","0_1_4_6_7","780.82"
+"inception3","64","0_1_4_6_7","1076.43"
+"alexnet","128","0_1_4_6_7","6910.49"
+"resnet50","128","0_1_4_6_7","1812.87"
+"vgg16","128","0_1_4_6_7","911.72"
+"inception3","128","0_1_4_6_7","1186.93"
+"alexnet","32","0_1_4_5_7","1958.31"
+"resnet50","32","0_1_4_5_7","1248.38"
+"vgg16","32","0_1_4_5_7","699.27"
+"inception3","32","0_1_4_5_7","905.09"
+"alexnet","64","0_1_4_5_7","3787.95"
+"resnet50","64","0_1_4_5_7","1462.07"
+"vgg16","64","0_1_4_5_7","903.41"
+"inception3","64","0_1_4_5_7","1073.80"
+"alexnet","128","0_1_4_5_7","6910.22"
+"resnet50","128","0_1_4_5_7","1833.03"
+"vgg16","128","0_1_4_5_7","1027.51"
+"inception3","128","0_1_4_5_7","1197.57"
+"alexnet","32","0_1_3_5_7","1864.89"
+"resnet50","32","0_1_3_5_7","1300.54"
+"vgg16","32","0_1_3_5_7","591.89"
+"inception3","32","0_1_3_5_7","897.85"
+"alexnet","64","0_1_3_5_7","3596.49"
+"resnet50","64","0_1_3_5_7","1459.54"
+"vgg16","64","0_1_3_5_7","778.05"
+"inception3","64","0_1_3_5_7","1087.32"
+"alexnet","128","0_1_3_5_7","6518.84"
+"resnet50","128","0_1_3_5_7","1644.44"
+"vgg16","128","0_1_3_5_7","919.46"
+"inception3","128","0_1_3_5_7","1164.29"
+"alexnet","32","0_2_4_6_7","2005.30"
+"resnet50","32","0_2_4_6_7","1257.13"
+"vgg16","32","0_2_4_6_7","708.18"
+"inception3","32","0_2_4_6_7","899.03"
+"alexnet","64","0_2_4_6_7","3887.20"
+"resnet50","64","0_2_4_6_7","1516.00"
+"vgg16","64","0_2_4_6_7","911.42"
+"inception3","64","0_2_4_6_7","1074.13"
+"alexnet","128","0_2_4_6_7","7207.91"
+"resnet50","128","0_2_4_6_7","1668.95"
+"vgg16","128","0_2_4_6_7","1034.98"
+"inception3","128","0_2_4_6_7","1197.73"
+"alexnet","32","1_2_3_4_6","1697.41"
+"resnet50","32","1_2_3_4_6","1295.04"
+"vgg16","32","1_2_3_4_6","478.41"
+"inception3","32","1_2_3_4_6","896.31"
+"alexnet","64","1_2_3_4_6","3296.38"
+"resnet50","64","1_2_3_4_6","1482.89"
+"vgg16","64","1_2_3_4_6","680.43"
+"inception3","64","1_2_3_4_6","1102.41"
+"alexnet","128","1_2_3_4_6","6001.96"
+"resnet50","128","1_2_3_4_6","1733.63"
+"vgg16","128","1_2_3_4_6","813.46"
+"inception3","128","1_2_3_4_6","1212.84"
+"alexnet","32","0_1_3_5_6","1675.89"
+"resnet50","32","0_1_3_5_6","1276.26"
+"vgg16","32","0_1_3_5_6","611.55"
+"inception3","32","0_1_3_5_6","901.43"
+"alexnet","64","0_1_3_5_6","3264.97"
+"resnet50","64","0_1_3_5_6","1589.85"
+"vgg16","64","0_1_3_5_6","812.13"
+"inception3","64","0_1_3_5_6","1083.70"
+"alexnet","128","0_1_3_5_6","6061.91"
+"resnet50","128","0_1_3_5_6","1838.97"
+"vgg16","128","0_1_3_5_6","951.21"
+"inception3","128","0_1_3_5_6","1189.52"
+"alexnet","32","0_1_2_6_7","1706.97"
+"resnet50","32","0_1_2_6_7","1258.15"
+"vgg16","32","0_1_2_6_7","590.29"
+"inception3","32","0_1_2_6_7","897.56"
+"alexnet","64","0_1_2_6_7","3346.29"
+"resnet50","64","0_1_2_6_7","1563.33"
+"vgg16","64","0_1_2_6_7","802.02"
+"inception3","64","0_1_2_6_7","1088.17"
+"alexnet","128","0_1_2_6_7","6156.87"
+"resnet50","128","0_1_2_6_7","1798.12"
+"vgg16","128","0_1_2_6_7","930.58"
+"inception3","128","0_1_2_6_7","1190.65"
+"alexnet","32","0_3_5_6_7","1552.67"
+"resnet50","32","0_3_5_6_7","1284.28"
+"vgg16","32","0_3_5_6_7","596.95"
+"inception3","32","0_3_5_6_7","889.79"
+"alexnet","64","0_3_5_6_7","3032.89"
+"resnet50","64","0_3_5_6_7","1571.15"
+"vgg16","64","0_3_5_6_7","781.86"
+"inception3","64","0_3_5_6_7","1085.92"
+"alexnet","128","0_3_5_6_7","5719.18"
+"resnet50","128","0_3_5_6_7","1805.82"
+"vgg16","128","0_3_5_6_7","922.14"
+"inception3","128","0_3_5_6_7","1191.38"
+"alexnet","32","0_2_3_4_7","2894.01"
+"resnet50","32","0_2_3_4_7","1260.65"
+"vgg16","32","0_2_3_4_7","605.23"
+"inception3","32","0_2_3_4_7","925.30"
+"alexnet","64","0_2_3_4_7","5506.22"
+"resnet50","64","0_2_3_4_7","1594.01"
+"vgg16","64","0_2_3_4_7","796.55"
+"inception3","64","0_2_3_4_7","1100.56"
+"alexnet","128","0_2_3_4_7","9759.62"
+"resnet50","128","0_2_3_4_7","1752.98"
+"vgg16","128","0_2_3_4_7","948.43"
+"inception3","128","0_2_3_4_7","1191.96"
+"alexnet","32","1_2_3_5_6","2303.02"
+"resnet50","32","1_2_3_5_6","1307.87"
+"vgg16","32","1_2_3_5_6","595.21"
+"inception3","32","1_2_3_5_6","916.48"
+"alexnet","64","1_2_3_5_6","4401.84"
+"resnet50","64","1_2_3_5_6","1610.75"
+"vgg16","64","1_2_3_5_6","787.11"
+"inception3","64","1_2_3_5_6","1095.32"
+"alexnet","128","1_2_3_5_6","7728.42"
+"resnet50","128","1_2_3_5_6","1750.55"
+"vgg16","128","1_2_3_5_6","927.22"
+"inception3","128","1_2_3_5_6","1219.54"
+"alexnet","32","0_1_2_3_5_6","2136.18"
+"resnet50","32","0_1_2_3_5_6","1477.12"
+"vgg16","32","0_1_2_3_5_6","795.63"
+"inception3","32","0_1_2_3_5_6","1026.29"
+"alexnet","64","0_1_2_3_5_6","4203.41"
+"resnet50","64","0_1_2_3_5_6","1872.17"
+"vgg16","64","0_1_2_3_5_6","1040.37"
+"inception3","64","0_1_2_3_5_6","1269.95"
+"alexnet","128","0_1_2_3_5_6","7960.30"
+"resnet50","128","0_1_2_3_5_6","2077.50"
+"vgg16","128","0_1_2_3_5_6","1185.85"
+"inception3","128","0_1_2_3_5_6","1385.35"
+"alexnet","32","0_1_2_3_6_7","2546.82"
+"resnet50","32","0_1_2_3_6_7","1479.26"
+"vgg16","32","0_1_2_3_6_7","765.25"
+"inception3","32","0_1_2_3_6_7","1013.84"
+"alexnet","64","0_1_2_3_6_7","5013.28"
+"resnet50","64","0_1_2_3_6_7","1853.81"
+"vgg16","64","0_1_2_3_6_7","1008.50"
+"inception3","64","0_1_2_3_6_7","1238.89"
+"alexnet","128","0_1_2_3_6_7","9306.88"
+"resnet50","128","0_1_2_3_6_7","2117.93"
+"vgg16","128","0_1_2_3_6_7","1162.93"
+"inception3","128","0_1_2_3_6_7","1328.98"
+"alexnet","32","0_1_3_4_5_6","2241.22"
+"resnet50","32","0_1_3_4_5_6","1467.36"
+"vgg16","32","0_1_3_4_5_6","535.34"
+"inception3","32","0_1_3_4_5_6","1060.03"
+"alexnet","64","0_1_3_4_5_6","4425.51"
+"resnet50","64","0_1_3_4_5_6","1761.41"
+"vgg16","64","0_1_3_4_5_6","743.15"
+"inception3","64","0_1_3_4_5_6","1248.30"
+"alexnet","128","0_1_3_4_5_6","8614.10"
+"resnet50","128","0_1_3_4_5_6","2048.26"
+"vgg16","128","0_1_3_4_5_6","911.30"
+"inception3","128","0_1_3_4_5_6","1410.49"
+"alexnet","32","0_1_3_5_6_7","2418.94"
+"resnet50","32","0_1_3_5_6_7","1430.26"
+"vgg16","32","0_1_3_5_6_7","660.24"
+"inception3","32","0_1_3_5_6_7","1001.87"
+"alexnet","64","0_1_3_5_6_7","4781.64"
+"resnet50","64","0_1_3_5_6_7","1839.43"
+"vgg16","64","0_1_3_5_6_7","889.03"
+"inception3","64","0_1_3_5_6_7","1242.76"
+"alexnet","128","0_1_3_5_6_7","9201.09"
+"resnet50","128","0_1_3_5_6_7","2111.45"
+"vgg16","128","0_1_3_5_6_7","1007.66"
+"inception3","128","0_1_3_5_6_7","1384.83"
+"alexnet","32","0_2_3_4_6_7","2599.60"
+"resnet50","32","0_2_3_4_6_7","1523.90"
+"vgg16","32","0_2_3_4_6_7","666.28"
+"inception3","32","0_2_3_4_6_7","1035.90"
+"alexnet","64","0_2_3_4_6_7","5097.29"
+"resnet50","64","0_2_3_4_6_7","1843.94"
+"vgg16","64","0_2_3_4_6_7","890.13"
+"inception3","64","0_2_3_4_6_7","1251.51"
+"alexnet","128","0_2_3_4_6_7","9231.44"
+"resnet50","128","0_2_3_4_6_7","2156.50"
+"vgg16","128","0_2_3_4_6_7","1029.38"
+"inception3","128","0_2_3_4_6_7","1395.31"
+"alexnet","32","0_3_4_5_6_7","2356.49"
+"resnet50","32","0_3_4_5_6_7","1488.69"
+"vgg16","32","0_3_4_5_6_7","799.16"
+"inception3","32","0_3_4_5_6_7","983.01"
+"alexnet","64","0_3_4_5_6_7","4698.59"
+"resnet50","64","0_3_4_5_6_7","1779.03"
+"vgg16","64","0_3_4_5_6_7","1037.36"
+"inception3","64","0_3_4_5_6_7","1233.49"
+"alexnet","128","0_3_4_5_6_7","8762.42"
+"resnet50","128","0_3_4_5_6_7","2064.23"
+"vgg16","128","0_3_4_5_6_7","1197.60"
+"inception3","128","0_3_4_5_6_7","1400.64"
+"alexnet","32","2_3_4_5_6_7","2149.95"
+"resnet50","32","2_3_4_5_6_7","1416.79"
+"vgg16","32","2_3_4_5_6_7","646.01"
+"inception3","32","2_3_4_5_6_7","1040.70"
+"alexnet","64","2_3_4_5_6_7","4294.91"
+"resnet50","64","2_3_4_5_6_7","1858.07"
+"vgg16","64","2_3_4_5_6_7","878.64"
+"inception3","64","2_3_4_5_6_7","1245.40"
+"alexnet","128","2_3_4_5_6_7","8208.88"
+"resnet50","128","2_3_4_5_6_7","2093.12"
+"vgg16","128","2_3_4_5_6_7","994.44"
+"inception3","128","2_3_4_5_6_7","1372.45"
+"alexnet","32","0_1_2_4_6_7","2639.36"
+"resnet50","32","0_1_2_4_6_7","1466.42"
+"vgg16","32","0_1_2_4_6_7","629.41"
+"inception3","32","0_1_2_4_6_7","1033.58"
+"alexnet","64","0_1_2_4_6_7","5172.85"
+"resnet50","64","0_1_2_4_6_7","1855.82"
+"vgg16","64","0_1_2_4_6_7","865.84"
+"inception3","64","0_1_2_4_6_7","1231.73"
+"alexnet","128","0_1_2_4_6_7","9318.40"
+"resnet50","128","0_1_2_4_6_7","2057.89"
+"vgg16","128","0_1_2_4_6_7","1009.85"
+"inception3","128","0_1_2_4_6_7","1373.66"
+"alexnet","32","0_2_4_5_6_7","2318.75"
+"resnet50","32","0_2_4_5_6_7","1505.99"
+"vgg16","32","0_2_4_5_6_7","809.82"
+"inception3","32","0_2_4_5_6_7","1010.54"
+"alexnet","64","0_2_4_5_6_7","4624.44"
+"resnet50","64","0_2_4_5_6_7","1868.57"
+"vgg16","64","0_2_4_5_6_7","1046.38"
+"inception3","64","0_2_4_5_6_7","1205.64"
+"alexnet","128","0_2_4_5_6_7","8750.44"
+"resnet50","128","0_2_4_5_6_7","2118.04"
+"vgg16","128","0_2_4_5_6_7","1212.84"
+"inception3","128","0_2_4_5_6_7","1372.23"
+"alexnet","32","0_2_3_5_6_7","2020.79"
+"resnet50","32","0_2_3_5_6_7","1485.21"
+"vgg16","32","0_2_3_5_6_7","669.46"
+"inception3","32","0_2_3_5_6_7","1029.40"
+"alexnet","64","0_2_3_5_6_7","4011.09"
+"resnet50","64","0_2_3_5_6_7","1866.78"
+"vgg16","64","0_2_3_5_6_7","883.43"
+"inception3","64","0_2_3_5_6_7","1232.02"
+"alexnet","128","0_2_3_5_6_7","7835.87"
+"resnet50","128","0_2_3_5_6_7","2060.27"
+"vgg16","128","0_2_3_5_6_7","1011.93"
+"inception3","128","0_2_3_5_6_7","1365.37"
+"alexnet","32","1_2_3_4_5_7","2174.13"
+"resnet50","32","1_2_3_4_5_7","1487.84"
+"vgg16","32","1_2_3_4_5_7","647.28"
+"inception3","32","1_2_3_4_5_7","1051.57"
+"alexnet","64","1_2_3_4_5_7","4298.73"
+"resnet50","64","1_2_3_4_5_7","1833.19"
+"vgg16","64","1_2_3_4_5_7","868.72"
+"inception3","64","1_2_3_4_5_7","1261.89"
+"alexnet","128","1_2_3_4_5_7","8003.76"
+"resnet50","128","1_2_3_4_5_7","2047.40"
+"vgg16","128","1_2_3_4_5_7","1020.25"
+"inception3","128","1_2_3_4_5_7","1395.33"
+"alexnet","32","0_1_4_5_6_7","2771.85"
+"resnet50","32","0_1_4_5_6_7","1470.31"
+"vgg16","32","0_1_4_5_6_7","786.06"
+"inception3","32","0_1_4_5_6_7","1010.25"
+"alexnet","64","0_1_4_5_6_7","5511.62"
+"resnet50","64","0_1_4_5_6_7","1756.03"
+"vgg16","64","0_1_4_5_6_7","1028.21"
+"inception3","64","0_1_4_5_6_7","1219.35"
+"alexnet","128","0_1_4_5_6_7","10000.94"
+"resnet50","128","0_1_4_5_6_7","2056.13"
+"vgg16","128","0_1_4_5_6_7","1183.95"
+"inception3","128","0_1_4_5_6_7","1381.18"
+"alexnet","32","0_1_3_4_6_7","2518.95"
+"resnet50","32","0_1_3_4_6_7","1469.53"
+"vgg16","32","0_1_3_4_6_7","655.05"
+"inception3","32","0_1_3_4_6_7","1006.16"
+"alexnet","64","0_1_3_4_6_7","5024.04"
+"resnet50","64","0_1_3_4_6_7","1855.15"
+"vgg16","64","0_1_3_4_6_7","889.98"
+"inception3","64","0_1_3_4_6_7","1213.38"
+"alexnet","128","0_1_3_4_6_7","9164.86"
+"resnet50","128","0_1_3_4_6_7","2130.45"
+"vgg16","128","0_1_3_4_6_7","1026.17"
+"inception3","128","0_1_3_4_6_7","1390.49"
+"alexnet","32","0_1_2_3_4_5","2343.61"
+"resnet50","32","0_1_2_3_4_5","1450.73"
+"vgg16","32","0_1_2_3_4_5","642.55"
+"inception3","32","0_1_2_3_4_5","999.52"
+"alexnet","64","0_1_2_3_4_5","4589.08"
+"resnet50","64","0_1_2_3_4_5","1828.70"
+"vgg16","64","0_1_2_3_4_5","876.93"
+"inception3","64","0_1_2_3_4_5","1229.16"
+"alexnet","128","0_1_2_3_4_5","8528.08"
+"resnet50","128","0_1_2_3_4_5","2152.71"
+"vgg16","128","0_1_2_3_4_5","1038.26"
+"inception3","128","0_1_2_3_4_5","1396.48"
+"alexnet","32","0_1_2_5_6_7","2224.70"
+"resnet50","32","0_1_2_5_6_7","1431.83"
+"vgg16","32","0_1_2_5_6_7","638.49"
+"inception3","32","0_1_2_5_6_7","1028.90"
+"alexnet","64","0_1_2_5_6_7","4430.08"
+"resnet50","64","0_1_2_5_6_7","1776.42"
+"vgg16","64","0_1_2_5_6_7","859.44"
+"inception3","64","0_1_2_5_6_7","1235.06"
+"alexnet","128","0_1_2_5_6_7","8630.25"
+"resnet50","128","0_1_2_5_6_7","2151.30"
+"vgg16","128","0_1_2_5_6_7","1004.22"
+"inception3","128","0_1_2_5_6_7","1381.84"
+"alexnet","32","0_1_2_4_5_6","2346.21"
+"resnet50","32","0_1_2_4_5_6","1508.11"
+"vgg16","32","0_1_2_4_5_6","659.07"
+"inception3","32","0_1_2_4_5_6","1036.47"
+"alexnet","64","0_1_2_4_5_6","4648.60"
+"resnet50","64","0_1_2_4_5_6","1827.27"
+"vgg16","64","0_1_2_4_5_6","872.66"
+"inception3","64","0_1_2_4_5_6","1273.45"
+"alexnet","128","0_1_2_4_5_6","8477.20"
+"resnet50","128","0_1_2_4_5_6","2056.28"
+"vgg16","128","0_1_2_4_5_6","1029.73"
+"inception3","128","0_1_2_4_5_6","1410.36"
+"alexnet","32","1_2_3_4_5_6","1930.17"
+"resnet50","32","1_2_3_4_5_6","1451.96"
+"vgg16","32","1_2_3_4_5_6","526.10"
+"inception3","32","1_2_3_4_5_6","1031.29"
+"alexnet","64","1_2_3_4_5_6","3786.73"
+"resnet50","64","1_2_3_4_5_6","1887.03"
+"vgg16","64","1_2_3_4_5_6","763.42"
+"inception3","64","1_2_3_4_5_6","1223.04"
+"alexnet","128","1_2_3_4_5_6","7377.01"
+"resnet50","128","1_2_3_4_5_6","2063.40"
+"vgg16","128","1_2_3_4_5_6","902.95"
+"inception3","128","1_2_3_4_5_6","1416.34"
+"alexnet","32","1_2_3_5_6_7","2675.43"
+"resnet50","32","1_2_3_5_6_7","1499.14"
+"vgg16","32","1_2_3_5_6_7","650.54"
+"inception3","32","1_2_3_5_6_7","1047.03"
+"alexnet","64","1_2_3_5_6_7","5282.57"
+"resnet50","64","1_2_3_5_6_7","1881.61"
+"vgg16","64","1_2_3_5_6_7","893.08"
+"inception3","64","1_2_3_5_6_7","1222.94"
+"alexnet","128","1_2_3_5_6_7","9827.73"
+"resnet50","128","1_2_3_5_6_7","2150.98"
+"vgg16","128","1_2_3_5_6_7","1029.29"
+"inception3","128","1_2_3_5_6_7","1381.80"
+"alexnet","32","0_2_3_4_5_6","2274.56"
+"resnet50","32","0_2_3_4_5_6","1445.00"
+"vgg16","32","0_2_3_4_5_6","535.85"
+"inception3","32","0_2_3_4_5_6","1041.61"
+"alexnet","64","0_2_3_4_5_6","4520.16"
+"resnet50","64","0_2_3_4_5_6","1860.40"
+"vgg16","64","0_2_3_4_5_6","770.39"
+"inception3","64","0_2_3_4_5_6","1253.11"
+"alexnet","128","0_2_3_4_5_6","8597.88"
+"resnet50","128","0_2_3_4_5_6","2030.67"
+"vgg16","128","0_2_3_4_5_6","899.52"
+"inception3","128","0_2_3_4_5_6","1376.77"
+"alexnet","32","0_1_2_3_4_6","2425.13"
+"resnet50","32","0_1_2_3_4_6","1469.15"
+"vgg16","32","0_1_2_3_4_6","805.24"
+"inception3","32","0_1_2_3_4_6","1032.27"
+"alexnet","64","0_1_2_3_4_6","4821.86"
+"resnet50","64","0_1_2_3_4_6","1749.76"
+"vgg16","64","0_1_2_3_4_6","1042.25"
+"inception3","64","0_1_2_3_4_6","1268.70"
+"alexnet","128","0_1_2_3_4_6","8864.09"
+"resnet50","128","0_1_2_3_4_6","2169.76"
+"vgg16","128","0_1_2_3_4_6","1198.34"
+"inception3","128","0_1_2_3_4_6","1363.96"
+"alexnet","32","0_1_2_4_5_7","2395.86"
+"resnet50","32","0_1_2_4_5_7","1423.99"
+"vgg16","32","0_1_2_4_5_7","531.80"
+"inception3","32","0_1_2_4_5_7","1042.55"
+"alexnet","64","0_1_2_4_5_7","4771.69"
+"resnet50","64","0_1_2_4_5_7","1786.19"
+"vgg16","64","0_1_2_4_5_7","752.23"
+"inception3","64","0_1_2_4_5_7","1244.40"
+"alexnet","128","0_1_2_4_5_7","8927.83"
+"resnet50","128","0_1_2_4_5_7","2130.18"
+"vgg16","128","0_1_2_4_5_7","942.76"
+"inception3","128","0_1_2_4_5_7","1389.30"
+"alexnet","32","0_1_2_3_5_7","2485.30"
+"resnet50","32","0_1_2_3_5_7","1512.62"
+"vgg16","32","0_1_2_3_5_7","658.25"
+"inception3","32","0_1_2_3_5_7","1043.46"
+"alexnet","64","0_1_2_3_5_7","4861.86"
+"resnet50","64","0_1_2_3_5_7","1855.77"
+"vgg16","64","0_1_2_3_5_7","884.39"
+"inception3","64","0_1_2_3_5_7","1241.13"
+"alexnet","128","0_1_2_3_5_7","9146.08"
+"resnet50","128","0_1_2_3_5_7","2173.70"
+"vgg16","128","0_1_2_3_5_7","1000.91"
+"inception3","128","0_1_2_3_5_7","1393.21"
+"alexnet","32","0_2_3_4_5_7","2584.17"
+"resnet50","32","0_2_3_4_5_7","1496.48"
+"vgg16","32","0_2_3_4_5_7","661.18"
+"inception3","32","0_2_3_4_5_7","1037.55"
+"alexnet","64","0_2_3_4_5_7","5129.75"
+"resnet50","64","0_2_3_4_5_7","1883.96"
+"vgg16","64","0_2_3_4_5_7","896.51"
+"inception3","64","0_2_3_4_5_7","1220.13"
+"alexnet","128","0_2_3_4_5_7","9073.86"
+"resnet50","128","0_2_3_4_5_7","2142.97"
+"vgg16","128","0_2_3_4_5_7","1041.65"
+"inception3","128","0_2_3_4_5_7","1412.48"
+"alexnet","32","0_1_3_4_5_7","2482.18"
+"resnet50","32","0_1_3_4_5_7","1507.03"
+"vgg16","32","0_1_3_4_5_7","658.50"
+"inception3","32","0_1_3_4_5_7","1052.52"
+"alexnet","64","0_1_3_4_5_7","5025.23"
+"resnet50","64","0_1_3_4_5_7","1859.62"
+"vgg16","64","0_1_3_4_5_7","888.07"
+"inception3","64","0_1_3_4_5_7","1263.87"
+"alexnet","128","0_1_3_4_5_7","8985.83"
+"resnet50","128","0_1_3_4_5_7","2149.36"
+"vgg16","128","0_1_3_4_5_7","982.79"
+"inception3","128","0_1_3_4_5_7","1394.34"
+"alexnet","32","0_1_2_3_4_7","2985.66"
+"resnet50","32","0_1_2_3_4_7","1408.04"
+"vgg16","32","0_1_2_3_4_7","660.03"
+"inception3","32","0_1_2_3_4_7","1010.95"
+"alexnet","64","0_1_2_3_4_7","5832.19"
+"resnet50","64","0_1_2_3_4_7","1878.99"
+"vgg16","64","0_1_2_3_4_7","891.55"
+"inception3","64","0_1_2_3_4_7","1258.44"
+"alexnet","128","0_1_2_3_4_7","10751.64"
+"resnet50","128","0_1_2_3_4_7","2164.80"
+"vgg16","128","0_1_2_3_4_7","1013.30"
+"inception3","128","0_1_2_3_4_7","1375.49"
+"alexnet","32","1_3_4_5_6_7","2831.22"
+"resnet50","32","1_3_4_5_6_7","1434.37"
+"vgg16","32","1_3_4_5_6_7","648.39"
+"inception3","32","1_3_4_5_6_7","1002.29"
+"alexnet","64","1_3_4_5_6_7","5525.78"
+"resnet50","64","1_3_4_5_6_7","1852.38"
+"vgg16","64","1_3_4_5_6_7","872.50"
+"inception3","64","1_3_4_5_6_7","1230.71"
+"alexnet","128","1_3_4_5_6_7","10312.83"
+"resnet50","128","1_3_4_5_6_7","2138.01"
+"vgg16","128","1_3_4_5_6_7","981.04"
+"inception3","128","1_3_4_5_6_7","1351.22"
+"alexnet","32","1_2_4_5_6_7","2810.31"
+"resnet50","32","1_2_4_5_6_7","1463.05"
+"vgg16","32","1_2_4_5_6_7","657.05"
+"inception3","32","1_2_4_5_6_7","1029.66"
+"alexnet","64","1_2_4_5_6_7","5538.91"
+"resnet50","64","1_2_4_5_6_7","1884.19"
+"vgg16","64","1_2_4_5_6_7","877.05"
+"inception3","64","1_2_4_5_6_7","1201.52"
+"alexnet","128","1_2_4_5_6_7","10321.55"
+"resnet50","128","1_2_4_5_6_7","2108.87"
+"vgg16","128","1_2_4_5_6_7","996.37"
+"inception3","128","1_2_4_5_6_7","1319.26"
+"alexnet","32","1_2_3_4_6_7","2026.41"
+"resnet50","32","1_2_3_4_6_7","1471.85"
+"vgg16","32","1_2_3_4_6_7","651.58"
+"inception3","32","1_2_3_4_6_7","1004.32"
+"alexnet","64","1_2_3_4_6_7","3984.76"
+"resnet50","64","1_2_3_4_6_7","1851.77"
+"vgg16","64","1_2_3_4_6_7","851.02"
+"inception3","64","1_2_3_4_6_7","1205.94"
+"alexnet","128","1_2_3_4_6_7","7679.94"
+"resnet50","128","1_2_3_4_6_7","2161.22"
+"vgg16","128","1_2_3_4_6_7","1007.43"
+"inception3","128","1_2_3_4_6_7","1399.52"
+"alexnet","32","0_1_2_3_5_6_7","2341.50"
+"resnet50","32","0_1_2_3_5_6_7","1667.38"
+"vgg16","32","0_1_2_3_5_6_7","662.62"
+"inception3","32","0_1_2_3_5_6_7","1155.65"
+"alexnet","64","0_1_2_3_5_6_7","4561.96"
+"resnet50","64","0_1_2_3_5_6_7","2040.63"
+"vgg16","64","0_1_2_3_5_6_7","905.40"
+"inception3","64","0_1_2_3_5_6_7","1378.83"
+"alexnet","128","0_1_2_3_5_6_7","8374.93"
+"resnet50","128","0_1_2_3_5_6_7","2442.46"
+"vgg16","128","0_1_2_3_5_6_7","1107.72"
+"inception3","128","0_1_2_3_5_6_7","1593.34"
+"alexnet","32","0_1_2_4_5_6_7","2167.61"
+"resnet50","32","0_1_2_4_5_6_7","1638.73"
+"vgg16","32","0_1_2_4_5_6_7","557.73"
+"inception3","32","0_1_2_4_5_6_7","1185.76"
+"alexnet","64","0_1_2_4_5_6_7","4301.64"
+"resnet50","64","0_1_2_4_5_6_7","2092.82"
+"vgg16","64","0_1_2_4_5_6_7","818.48"
+"inception3","64","0_1_2_4_5_6_7","1414.97"
+"alexnet","128","0_1_2_4_5_6_7","8082.85"
+"resnet50","128","0_1_2_4_5_6_7","2406.13"
+"vgg16","128","0_1_2_4_5_6_7","1016.84"
+"inception3","128","0_1_2_4_5_6_7","1598.77"
+"alexnet","32","0_1_2_3_4_6_7","1962.71"
+"resnet50","32","0_1_2_3_4_6_7","1638.99"
+"vgg16","32","0_1_2_3_4_6_7","557.16"
+"inception3","32","0_1_2_3_4_6_7","1129.51"
+"alexnet","64","0_1_2_3_4_6_7","3897.41"
+"resnet50","64","0_1_2_3_4_6_7","2045.99"
+"vgg16","64","0_1_2_3_4_6_7","824.24"
+"inception3","64","0_1_2_3_4_6_7","1426.86"
+"alexnet","128","0_1_2_3_4_6_7","7585.63"
+"resnet50","128","0_1_2_3_4_6_7","2342.38"
+"vgg16","128","0_1_2_3_4_6_7","1031.54"
+"inception3","128","0_1_2_3_4_6_7","1584.15"
+"alexnet","32","0_1_2_3_4_5_7","2538.91"
+"resnet50","32","0_1_2_3_4_5_7","1644.99"
+"vgg16","32","0_1_2_3_4_5_7","675.31"
+"inception3","32","0_1_2_3_4_5_7","1208.17"
+"alexnet","64","0_1_2_3_4_5_7","5151.24"
+"resnet50","64","0_1_2_3_4_5_7","2120.26"
+"vgg16","64","0_1_2_3_4_5_7","954.93"
+"inception3","64","0_1_2_3_4_5_7","1447.62"
+"alexnet","128","0_1_2_3_4_5_7","9321.12"
+"resnet50","128","0_1_2_3_4_5_7","2346.78"
+"vgg16","128","0_1_2_3_4_5_7","1127.31"
+"inception3","128","0_1_2_3_4_5_7","1628.69"
+"alexnet","32","0_1_3_4_5_6_7","1998.59"
+"resnet50","32","0_1_3_4_5_6_7","1612.78"
+"vgg16","32","0_1_3_4_5_6_7","552.43"
+"inception3","32","0_1_3_4_5_6_7","1165.81"
+"alexnet","64","0_1_3_4_5_6_7","3885.38"
+"resnet50","64","0_1_3_4_5_6_7","2091.77"
+"vgg16","64","0_1_3_4_5_6_7","800.68"
+"inception3","64","0_1_3_4_5_6_7","1428.81"
+"alexnet","128","0_1_3_4_5_6_7","7403.87"
+"resnet50","128","0_1_3_4_5_6_7","2425.72"
+"vgg16","128","0_1_3_4_5_6_7","974.97"
+"inception3","128","0_1_3_4_5_6_7","1549.14"
+"alexnet","32","0_1_2_3_4_5_6","2549.69"
+"resnet50","32","0_1_2_3_4_5_6","1687.53"
+"vgg16","32","0_1_2_3_4_5_6","685.23"
+"inception3","32","0_1_2_3_4_5_6","1181.25"
+"alexnet","64","0_1_2_3_4_5_6","4965.54"
+"resnet50","64","0_1_2_3_4_5_6","2024.21"
+"vgg16","64","0_1_2_3_4_5_6","953.19"
+"inception3","64","0_1_2_3_4_5_6","1435.29"
+"alexnet","128","0_1_2_3_4_5_6","9345.32"
+"resnet50","128","0_1_2_3_4_5_6","2485.68"
+"vgg16","128","0_1_2_3_4_5_6","1143.87"
+"inception3","128","0_1_2_3_4_5_6","1579.46"
+"alexnet","32","0_2_3_4_5_6_7","2140.24"
+"resnet50","32","0_2_3_4_5_6_7","1650.53"
+"vgg16","32","0_2_3_4_5_6_7","560.76"
+"inception3","32","0_2_3_4_5_6_7","1181.57"
+"alexnet","64","0_2_3_4_5_6_7","4353.79"
+"resnet50","64","0_2_3_4_5_6_7","2038.03"
+"vgg16","64","0_2_3_4_5_6_7","816.42"
+"inception3","64","0_2_3_4_5_6_7","1415.18"
+"alexnet","128","0_2_3_4_5_6_7","8112.01"
+"resnet50","128","0_2_3_4_5_6_7","2385.85"
+"vgg16","128","0_2_3_4_5_6_7","993.87"
+"inception3","128","0_2_3_4_5_6_7","1596.65"
+"alexnet","32","1_2_3_4_5_6_7","2163.43"
+"resnet50","32","1_2_3_4_5_6_7","1631.97"
+"vgg16","32","1_2_3_4_5_6_7","557.85"
+"inception3","32","1_2_3_4_5_6_7","1179.02"
+"alexnet","64","1_2_3_4_5_6_7","4259.44"
+"resnet50","64","1_2_3_4_5_6_7","2097.62"
+"vgg16","64","1_2_3_4_5_6_7","828.59"
+"inception3","64","1_2_3_4_5_6_7","1423.61"
+"alexnet","128","1_2_3_4_5_6_7","8003.65"
+"resnet50","128","1_2_3_4_5_6_7","2406.21"
+"vgg16","128","1_2_3_4_5_6_7","1021.62"
+"inception3","128","1_2_3_4_5_6_7","1595.09"