org.apache.tez.dag.app.dag.impl

Class DAGImpl

java.lang.Object

org.apache.tez.dag.app.dag.impl.DAGImpl

All Implemented Interfaces:

org.apache.hadoop.yarn.event.EventHandler<DAGEvent>, DAG, DagInfo

public class DAGImpl
extends Object
implements DAG, org.apache.hadoop.yarn.event.EventHandler<DAGEvent>

Implementation of Job interface. Maintains the state machines of Job. The read and write calls use ReadWriteLock for concurrency.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	DAGImpl.OutputKey
static class	DAGImpl.StartTransition

Field Summary

Fields

Modifier and Type	Field and Description
protected static org.apache.hadoop.yarn.state.StateMachineFactory<DAGImpl,DAGState,DAGEventType,DAGEvent>	stateMachineFactory

Constructor Summary

Constructors

Constructor and Description
DAGImpl(TezDAGID dagId, org.apache.hadoop.conf.Configuration amConf, DAGProtos.DAGPlan jobPlan, org.apache.hadoop.yarn.event.EventHandler eventHandler, TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, org.apache.hadoop.security.Credentials dagCredentials, org.apache.hadoop.yarn.util.Clock clock, String appUserName, TaskHeartbeatHandler thh, AppContext appContext)

Method Summary

Modifier and Type	Method and Description
static TezCounters	aggrTaskCounters(TezCounters counters, Collection<Vertex> vertices)
TezCounters	constructFinalFullcounters()
ACLManager	getACLManager()
TezCounters	getAllCounters() Get all the counters of this DAG.
TezCounters	getCachedCounters()
float	getCompletedTaskProgress()
org.apache.hadoop.conf.Configuration	getConf()
org.apache.hadoop.security.Credentials	getCredentials()
DAGScheduler	getDAGScheduler()
DAGStatusBuilder	getDAGStatus(Set<StatusGetOpts> statusOptions)
DAGStatusBuilder	getDAGStatus(Set<StatusGetOpts> statusOptions, long timeoutMillis)
org.apache.hadoop.security.UserGroupInformation	getDagUGI()
Vertex.VertexExecutionContext	getDefaultExecutionContext()
List<String>	getDiagnostics()
org.apache.hadoop.yarn.event.EventHandler	getEventHandler()
TezDAGID	getID()
int	getIndex()
DAGState	getInternalState()
DAGProtos.DAGPlan	getJobPlan()
Map<String,org.apache.hadoop.yarn.api.records.LocalResource>	getLocalResources()
String[]	getLogDirs()
String	getName()
float	getProgress()
DAGReport	getReport()
long	getStartTime()
DAGState	getState()
StateChangeNotifier	getStateChangeNotifier()
protected org.apache.hadoop.yarn.state.StateMachine<DAGState,DAGEventType,DAGEvent>	getStateMachine()
int	getSuccessfulVertices()
TaskImpl	getTask(TezTaskID tId)
TaskAttemptImpl	getTaskAttempt(TezTaskAttemptID taId)
int	getTotalVertices()
String	getUserName()
Vertex	getVertex(String vertexName) Get Vertex by vertex name
Vertex	getVertex(TezVertexID vertexID)
BitSet	getVertexDescendants(int vertexIndex)
Map<String,TezVertexID>	getVertexNameIDMapping()
VertexStatusBuilder	getVertexStatus(String vertexName, Set<StatusGetOpts> statusOptions)
Map<TezVertexID,Vertex>	getVertices()
void	handle(DAGEvent event)
protected void	initializeVerticesAndStart()
boolean	isComplete()
boolean	isUber()
DAGImpl	setLogDirs(String[] logDirs)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

stateMachineFactory

protected static final org.apache.hadoop.yarn.state.StateMachineFactory<DAGImpl,DAGState,DAGEventType,DAGEvent> stateMachineFactory

Constructor Detail

DAGImpl

public DAGImpl(TezDAGID dagId,
               org.apache.hadoop.conf.Configuration amConf,
               DAGProtos.DAGPlan jobPlan,
               org.apache.hadoop.yarn.event.EventHandler eventHandler,
               TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
               org.apache.hadoop.security.Credentials dagCredentials,
               org.apache.hadoop.yarn.util.Clock clock,
               String appUserName,
               TaskHeartbeatHandler thh,
               AppContext appContext)

Method Detail

getStateMachine

protected org.apache.hadoop.yarn.state.StateMachine<DAGState,DAGEventType,DAGEvent> getStateMachine()

getID

public TezDAGID getID()

Specified by:

getID in interface DAG

getLocalResources

public Map<String,org.apache.hadoop.yarn.api.records.LocalResource> getLocalResources()

Specified by:

getLocalResources in interface DAG

getConf

public org.apache.hadoop.conf.Configuration getConf()

Specified by:

getConf in interface DagInfo

getJobPlan

public DAGProtos.DAGPlan getJobPlan()

Specified by:

getJobPlan in interface DAG

getEventHandler

public org.apache.hadoop.yarn.event.EventHandler getEventHandler()

Specified by:

getEventHandler in interface DAG

getVertex

public Vertex getVertex(TezVertexID vertexID)

Specified by:

getVertex in interface DAG

isUber

public boolean isUber()

Specified by:

isUber in interface DAG

getCredentials

public org.apache.hadoop.security.Credentials getCredentials()

Specified by:

getCredentials in interface DagInfo

getDagUGI

public org.apache.hadoop.security.UserGroupInformation getDagUGI()

Specified by:

getDagUGI in interface DAG

getACLManager

public ACLManager getACLManager()

Specified by:

getACLManager in interface DAG

getVertexNameIDMapping

public Map<String,TezVertexID> getVertexNameIDMapping()

Specified by:

getVertexNameIDMapping in interface DAG

getStartTime

public long getStartTime()

Specified by:

getStartTime in interface DAG

getStateChangeNotifier

public StateChangeNotifier getStateChangeNotifier()

Specified by:

getStateChangeNotifier in interface DAG

getDefaultExecutionContext

public Vertex.VertexExecutionContext getDefaultExecutionContext()

Specified by:

getDefaultExecutionContext in interface DAG

getAllCounters

public TezCounters getAllCounters()

Description copied from interface: DAG

Get all the counters of this DAG. This includes job-counters aggregated together with the counters of each task. This creates a clone of the Counters, so use this judiciously.

Specified by:

getAllCounters in interface DAG

Returns:

job-counters and aggregate task-counters

getCachedCounters

public TezCounters getCachedCounters()

Specified by:

getCachedCounters in interface DAG

aggrTaskCounters

public static TezCounters aggrTaskCounters(TezCounters counters,
                                           Collection<Vertex> vertices)

getDiagnostics

public List<String> getDiagnostics()

Specified by:

getDiagnostics in interface DAG

getReport

public DAGReport getReport()

Specified by:

getReport in interface DAG

getProgress

public float getProgress()

Specified by:

getProgress in interface DAG

getCompletedTaskProgress

public float getCompletedTaskProgress()

Specified by:

getCompletedTaskProgress in interface DAG

getVertices

public Map<TezVertexID,Vertex> getVertices()

Specified by:

getVertices in interface DAG

getState

public DAGState getState()

Specified by:

getState in interface DAG

getDAGStatus

public DAGStatusBuilder getDAGStatus(Set<StatusGetOpts> statusOptions)

Specified by:

getDAGStatus in interface DAG

getDAGStatus

public DAGStatusBuilder getDAGStatus(Set<StatusGetOpts> statusOptions,
                                     long timeoutMillis)
                              throws TezException

Specified by:

getDAGStatus in interface DAG

Throws:

TezException

getVertexStatus

public VertexStatusBuilder getVertexStatus(String vertexName,
                                           Set<StatusGetOpts> statusOptions)

Specified by:

getVertexStatus in interface DAG

getTaskAttempt

public TaskAttemptImpl getTaskAttempt(TezTaskAttemptID taId)

getTask

public TaskImpl getTask(TezTaskID tId)

initializeVerticesAndStart

protected void initializeVerticesAndStart()

handle

public void handle(DAGEvent event)

Specified by:

handle in interface org.apache.hadoop.yarn.event.EventHandler<DAGEvent>

getInternalState

@InterfaceAudience.Private
public DAGState getInternalState()

getUserName

public String getUserName()

Specified by:

getUserName in interface DAG

getIndex

public int getIndex()

Specified by:

getIndex in interface DagInfo

getName

public String getName()

Specified by:

getName in interface DAG

Specified by:

getName in interface DagInfo

getTotalVertices

public int getTotalVertices()

Specified by:

getTotalVertices in interface DagInfo

getVertexDescendants

public BitSet getVertexDescendants(int vertexIndex)

Specified by:

getVertexDescendants in interface DagInfo

getSuccessfulVertices

public int getSuccessfulVertices()

Specified by:

getSuccessfulVertices in interface DAG

getVertex

public Vertex getVertex(String vertexName)

Description copied from interface: DAG

Get Vertex by vertex name

Specified by:

getVertex in interface DAG

constructFinalFullcounters

@InterfaceAudience.Private
public TezCounters constructFinalFullcounters()

isComplete

public boolean isComplete()

Specified by:

isComplete in interface DAG

getDAGScheduler

public DAGScheduler getDAGScheduler()

Specified by:

getDAGScheduler in interface DAG

Returns:

the DAGScheduler that will schedule this DAG, null if it doesn't exist

getLogDirs

public String[] getLogDirs()

setLogDirs

public DAGImpl setLogDirs(String[] logDirs)