|
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.sls.appmaster;
|
|
|
|
|
|
import java.io.IOException;
|
|
import java.io.IOException;
|
|
import java.security.PrivilegedExceptionAction;
|
|
import java.security.PrivilegedExceptionAction;
|
|
-import java.text.MessageFormat;
|
|
|
|
import java.util.ArrayList;
|
|
import java.util.ArrayList;
|
|
import java.util.HashMap;
|
|
import java.util.HashMap;
|
|
import java.util.LinkedList;
|
|
import java.util.LinkedList;
|
|
@@ -42,10 +41,10 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
|
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
|
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
|
|
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
|
|
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
|
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
|
-
|
|
|
|
import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
|
|
import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
|
|
import org.apache.hadoop.yarn.sls.SLSRunner;
|
|
import org.apache.hadoop.yarn.sls.SLSRunner;
|
|
-import org.apache.log4j.Logger;
|
|
|
|
|
|
+import org.slf4j.Logger;
|
|
|
|
+import org.slf4j.LoggerFactory;
|
|
|
|
|
|
@Private
|
|
@Private
|
|
@Unstable
|
|
@Unstable
|
|
@@ -111,7 +110,8 @@ public class MRAMSimulator extends AMSimulator {
|
|
// finished
|
|
// finished
|
|
private boolean isFinished = false;
|
|
private boolean isFinished = false;
|
|
|
|
|
|
- public final Logger LOG = Logger.getLogger(MRAMSimulator.class);
|
|
|
|
|
|
+ private static final Logger LOG =
|
|
|
|
+ LoggerFactory.getLogger(MRAMSimulator.class);
|
|
|
|
|
|
@SuppressWarnings("checkstyle:parameternumber")
|
|
@SuppressWarnings("checkstyle:parameternumber")
|
|
public void init(int id, int heartbeatInterval,
|
|
public void init(int id, int heartbeatInterval,
|
|
@@ -135,9 +135,8 @@ public class MRAMSimulator extends AMSimulator {
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
- LOG.info(MessageFormat
|
|
|
|
- .format("Added new job with {0} mapper and {1} reducers",
|
|
|
|
- allMaps.size(), allReduces.size()));
|
|
|
|
|
|
+ LOG.info("Added new job with {} mapper and {} reducers",
|
|
|
|
+ allMaps.size(), allReduces.size());
|
|
|
|
|
|
mapTotal = allMaps.size();
|
|
mapTotal = allMaps.size();
|
|
reduceTotal = allReduces.size();
|
|
reduceTotal = allReduces.size();
|
|
@@ -165,22 +164,21 @@ public class MRAMSimulator extends AMSimulator {
|
|
ContainerId containerId = cs.getContainerId();
|
|
ContainerId containerId = cs.getContainerId();
|
|
if (cs.getExitStatus() == ContainerExitStatus.SUCCESS) {
|
|
if (cs.getExitStatus() == ContainerExitStatus.SUCCESS) {
|
|
if (assignedMaps.containsKey(containerId)) {
|
|
if (assignedMaps.containsKey(containerId)) {
|
|
- LOG.debug(MessageFormat.format("Application {0} has one" +
|
|
|
|
- "mapper finished ({1}).", appId, containerId));
|
|
|
|
|
|
+ LOG.debug("Application {} has one mapper finished ({}).",
|
|
|
|
+ appId, containerId);
|
|
assignedMaps.remove(containerId);
|
|
assignedMaps.remove(containerId);
|
|
mapFinished ++;
|
|
mapFinished ++;
|
|
finishedContainers ++;
|
|
finishedContainers ++;
|
|
} else if (assignedReduces.containsKey(containerId)) {
|
|
} else if (assignedReduces.containsKey(containerId)) {
|
|
- LOG.debug(MessageFormat.format("Application {0} has one" +
|
|
|
|
- "reducer finished ({1}).", appId, containerId));
|
|
|
|
|
|
+ LOG.debug("Application {} has one reducer finished ({}).",
|
|
|
|
+ appId, containerId);
|
|
assignedReduces.remove(containerId);
|
|
assignedReduces.remove(containerId);
|
|
reduceFinished ++;
|
|
reduceFinished ++;
|
|
finishedContainers ++;
|
|
finishedContainers ++;
|
|
} else if (amContainer.getId().equals(containerId)){
|
|
} else if (amContainer.getId().equals(containerId)){
|
|
// am container released event
|
|
// am container released event
|
|
isFinished = true;
|
|
isFinished = true;
|
|
- LOG.info(MessageFormat.format("Application {0} goes to " +
|
|
|
|
- "finish.", appId));
|
|
|
|
|
|
+ LOG.info("Application {} goes to finish.", appId);
|
|
}
|
|
}
|
|
|
|
|
|
if (mapFinished >= mapTotal && reduceFinished >= reduceTotal) {
|
|
if (mapFinished >= mapTotal && reduceFinished >= reduceTotal) {
|
|
@@ -189,16 +187,16 @@ public class MRAMSimulator extends AMSimulator {
|
|
} else {
|
|
} else {
|
|
// container to be killed
|
|
// container to be killed
|
|
if (assignedMaps.containsKey(containerId)) {
|
|
if (assignedMaps.containsKey(containerId)) {
|
|
- LOG.debug(MessageFormat.format("Application {0} has one " +
|
|
|
|
- "mapper killed ({1}).", appId, containerId));
|
|
|
|
|
|
+ LOG.debug("Application {} has one mapper killed ({}).",
|
|
|
|
+ appId, containerId);
|
|
pendingFailedMaps.add(assignedMaps.remove(containerId));
|
|
pendingFailedMaps.add(assignedMaps.remove(containerId));
|
|
} else if (assignedReduces.containsKey(containerId)) {
|
|
} else if (assignedReduces.containsKey(containerId)) {
|
|
- LOG.debug(MessageFormat.format("Application {0} has one " +
|
|
|
|
- "reducer killed ({1}).", appId, containerId));
|
|
|
|
|
|
+ LOG.debug("Application {} has one reducer killed ({}).",
|
|
|
|
+ appId, containerId);
|
|
pendingFailedReduces.add(assignedReduces.remove(containerId));
|
|
pendingFailedReduces.add(assignedReduces.remove(containerId));
|
|
} else if (amContainer.getId().equals(containerId)){
|
|
} else if (amContainer.getId().equals(containerId)){
|
|
- LOG.info(MessageFormat.format("Application {0}'s AM is " +
|
|
|
|
- "going to be killed. Waiting for rescheduling...", appId));
|
|
|
|
|
|
+ LOG.info("Application {}'s AM is " +
|
|
|
|
+ "going to be killed. Waiting for rescheduling...", appId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
@@ -209,8 +207,8 @@ public class MRAMSimulator extends AMSimulator {
|
|
(mapFinished >= mapTotal) &&
|
|
(mapFinished >= mapTotal) &&
|
|
(reduceFinished >= reduceTotal)) {
|
|
(reduceFinished >= reduceTotal)) {
|
|
isAMContainerRunning = false;
|
|
isAMContainerRunning = false;
|
|
- LOG.debug(MessageFormat.format("Application {0} sends out event " +
|
|
|
|
- "to clean up its AM container.", appId));
|
|
|
|
|
|
+ LOG.debug("Application {} sends out event to clean up"
|
|
|
|
+ + " its AM container.", appId);
|
|
isFinished = true;
|
|
isFinished = true;
|
|
break;
|
|
break;
|
|
}
|
|
}
|
|
@@ -219,15 +217,15 @@ public class MRAMSimulator extends AMSimulator {
|
|
for (Container container : response.getAllocatedContainers()) {
|
|
for (Container container : response.getAllocatedContainers()) {
|
|
if (! scheduledMaps.isEmpty()) {
|
|
if (! scheduledMaps.isEmpty()) {
|
|
ContainerSimulator cs = scheduledMaps.remove();
|
|
ContainerSimulator cs = scheduledMaps.remove();
|
|
- LOG.debug(MessageFormat.format("Application {0} starts a " +
|
|
|
|
- "launch a mapper ({1}).", appId, container.getId()));
|
|
|
|
|
|
+ LOG.debug("Application {} starts to launch a mapper ({}).",
|
|
|
|
+ appId, container.getId());
|
|
assignedMaps.put(container.getId(), cs);
|
|
assignedMaps.put(container.getId(), cs);
|
|
se.getNmMap().get(container.getNodeId())
|
|
se.getNmMap().get(container.getNodeId())
|
|
.addNewContainer(container, cs.getLifeTime());
|
|
.addNewContainer(container, cs.getLifeTime());
|
|
} else if (! this.scheduledReduces.isEmpty()) {
|
|
} else if (! this.scheduledReduces.isEmpty()) {
|
|
ContainerSimulator cs = scheduledReduces.remove();
|
|
ContainerSimulator cs = scheduledReduces.remove();
|
|
- LOG.debug(MessageFormat.format("Application {0} starts a " +
|
|
|
|
- "launch a reducer ({1}).", appId, container.getId()));
|
|
|
|
|
|
+ LOG.debug("Application {} starts to launch a reducer ({}).",
|
|
|
|
+ appId, container.getId());
|
|
assignedReduces.put(container.getId(), cs);
|
|
assignedReduces.put(container.getId(), cs);
|
|
se.getNmMap().get(container.getNodeId())
|
|
se.getNmMap().get(container.getNodeId())
|
|
.addNewContainer(container, cs.getLifeTime());
|
|
.addNewContainer(container, cs.getLifeTime());
|
|
@@ -289,17 +287,15 @@ public class MRAMSimulator extends AMSimulator {
|
|
if (!pendingMaps.isEmpty()) {
|
|
if (!pendingMaps.isEmpty()) {
|
|
ask = packageRequests(mergeLists(pendingMaps, scheduledMaps),
|
|
ask = packageRequests(mergeLists(pendingMaps, scheduledMaps),
|
|
PRIORITY_MAP);
|
|
PRIORITY_MAP);
|
|
- LOG.debug(MessageFormat
|
|
|
|
- .format("Application {0} sends out " + "request for {1} mappers.",
|
|
|
|
- appId, pendingMaps.size()));
|
|
|
|
|
|
+ LOG.debug("Application {} sends out request for {} mappers.",
|
|
|
|
+ appId, pendingMaps.size());
|
|
scheduledMaps.addAll(pendingMaps);
|
|
scheduledMaps.addAll(pendingMaps);
|
|
pendingMaps.clear();
|
|
pendingMaps.clear();
|
|
} else if (!pendingFailedMaps.isEmpty()) {
|
|
} else if (!pendingFailedMaps.isEmpty()) {
|
|
ask = packageRequests(mergeLists(pendingFailedMaps, scheduledMaps),
|
|
ask = packageRequests(mergeLists(pendingFailedMaps, scheduledMaps),
|
|
PRIORITY_MAP);
|
|
PRIORITY_MAP);
|
|
- LOG.debug(MessageFormat.format(
|
|
|
|
- "Application {0} sends out " + "requests for {1} failed mappers.",
|
|
|
|
- appId, pendingFailedMaps.size()));
|
|
|
|
|
|
+ LOG.debug("Application {} sends out requests for {} failed mappers.",
|
|
|
|
+ appId, pendingFailedMaps.size());
|
|
scheduledMaps.addAll(pendingFailedMaps);
|
|
scheduledMaps.addAll(pendingFailedMaps);
|
|
pendingFailedMaps.clear();
|
|
pendingFailedMaps.clear();
|
|
}
|
|
}
|
|
@@ -308,17 +304,15 @@ public class MRAMSimulator extends AMSimulator {
|
|
if (!pendingReduces.isEmpty()) {
|
|
if (!pendingReduces.isEmpty()) {
|
|
ask = packageRequests(mergeLists(pendingReduces, scheduledReduces),
|
|
ask = packageRequests(mergeLists(pendingReduces, scheduledReduces),
|
|
PRIORITY_REDUCE);
|
|
PRIORITY_REDUCE);
|
|
- LOG.debug(MessageFormat
|
|
|
|
- .format("Application {0} sends out " + "requests for {1} reducers.",
|
|
|
|
- appId, pendingReduces.size()));
|
|
|
|
|
|
+ LOG.debug("Application {} sends out requests for {} reducers.",
|
|
|
|
+ appId, pendingReduces.size());
|
|
scheduledReduces.addAll(pendingReduces);
|
|
scheduledReduces.addAll(pendingReduces);
|
|
pendingReduces.clear();
|
|
pendingReduces.clear();
|
|
} else if (!pendingFailedReduces.isEmpty()) {
|
|
} else if (!pendingFailedReduces.isEmpty()) {
|
|
ask = packageRequests(mergeLists(pendingFailedReduces, scheduledReduces),
|
|
ask = packageRequests(mergeLists(pendingFailedReduces, scheduledReduces),
|
|
PRIORITY_REDUCE);
|
|
PRIORITY_REDUCE);
|
|
- LOG.debug(MessageFormat.format(
|
|
|
|
- "Application {0} sends out " + "request for {1} failed reducers.",
|
|
|
|
- appId, pendingFailedReduces.size()));
|
|
|
|
|
|
+ LOG.debug("Application {} sends out request for {} failed reducers.",
|
|
|
|
+ appId, pendingFailedReduces.size());
|
|
scheduledReduces.addAll(pendingFailedReduces);
|
|
scheduledReduces.addAll(pendingFailedReduces);
|
|
pendingFailedReduces.clear();
|
|
pendingFailedReduces.clear();
|
|
}
|
|
}
|