org.apache.hadoop.mapred
Class ShuffleHandler

java.lang.Object
  org.apache.hadoop.service.AbstractService
      org.apache.hadoop.yarn.server.api.AuxiliaryService
          org.apache.hadoop.mapred.ShuffleHandler

All Implemented Interfaces:

Closeable, org.apache.hadoop.service.Service

public class ShuffleHandler
extends org.apache.hadoop.yarn.server.api.AuxiliaryService

Nested Class Summary

Nested classes/interfaces inherited from interface org.apache.hadoop.service.Service

org.apache.hadoop.service.Service.STATE

Field Summary

static String	CONNECTION_CLOSE
protected static org.apache.hadoop.yarn.server.records.Version	CURRENT_VERSION_INFO
static int	DEFAULT_MAX_SHUFFLE_CONNECTIONS
static int	DEFAULT_MAX_SHUFFLE_THREADS
static int	DEFAULT_SHUFFLE_BUFFER_SIZE
static boolean	DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED
static int	DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT
static boolean	DEFAULT_SHUFFLE_MANAGE_OS_CACHE
static int	DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE
static int	DEFAULT_SHUFFLE_PORT
static int	DEFAULT_SHUFFLE_READAHEAD_BYTES
static boolean	DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED
static int	DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE
static String	MAPREDUCE_SHUFFLE_SERVICEID
static String	MAX_SHUFFLE_CONNECTIONS
static String	MAX_SHUFFLE_THREADS
protected org.apache.hadoop.mapred.ShuffleHandler.HttpPipelineFactory	pipelineFact
static String	SHUFFLE_BUFFER_SIZE
static String	SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED
static String	SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT
static String	SHUFFLE_MANAGE_OS_CACHE
static String	SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE
static String	SHUFFLE_PORT_CONFIG_KEY
static String	SHUFFLE_READAHEAD_BYTES
static String	SHUFFLE_TRANSFERTO_ALLOWED
static String	SUFFLE_SSL_FILE_BUFFER_SIZE_KEY
static boolean	WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED

Constructor Summary

ShuffleHandler()

Method Summary

static int	deserializeMetaData(ByteBuffer meta) A helper function to deserialize the metadata returned by ShuffleHandler.
protected org.apache.hadoop.yarn.server.records.Version	getCurrentVersion()
ByteBuffer	getMetaData()
protected org.apache.hadoop.mapred.ShuffleHandler.Shuffle	getShuffle(org.apache.hadoop.conf.Configuration conf)
void	initializeApplication(org.apache.hadoop.yarn.server.api.ApplicationInitializationContext context)
static ByteBuffer	serializeMetaData(int port) Serialize the shuffle port into a ByteBuffer for use later on.
static ByteBuffer	serializeServiceData(org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier> jobToken) A helper function to serialize the JobTokenIdentifier to be sent to the ShuffleHandler as ServiceData.
protected void	serviceInit(org.apache.hadoop.conf.Configuration conf)
protected void	serviceStart()
protected void	serviceStop()
void	stopApplication(org.apache.hadoop.yarn.server.api.ApplicationTerminationContext context)

Methods inherited from class org.apache.hadoop.yarn.server.api.AuxiliaryService

getRecoveryPath, initializeContainer, setRecoveryPath, stopContainer

Methods inherited from class org.apache.hadoop.service.AbstractService

close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Detail

SHUFFLE_MANAGE_OS_CACHE

public static final String SHUFFLE_MANAGE_OS_CACHE

See Also:

Constant Field Values

DEFAULT_SHUFFLE_MANAGE_OS_CACHE

public static final boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE

See Also:

Constant Field Values

SHUFFLE_READAHEAD_BYTES

public static final String SHUFFLE_READAHEAD_BYTES

See Also:

Constant Field Values

DEFAULT_SHUFFLE_READAHEAD_BYTES

public static final int DEFAULT_SHUFFLE_READAHEAD_BYTES

See Also:

Constant Field Values

CURRENT_VERSION_INFO

protected static final org.apache.hadoop.yarn.server.records.Version CURRENT_VERSION_INFO

pipelineFact

protected org.apache.hadoop.mapred.ShuffleHandler.HttpPipelineFactory pipelineFact

MAPREDUCE_SHUFFLE_SERVICEID

public static final String MAPREDUCE_SHUFFLE_SERVICEID

See Also:

Constant Field Values

SHUFFLE_PORT_CONFIG_KEY

public static final String SHUFFLE_PORT_CONFIG_KEY

See Also:

Constant Field Values

DEFAULT_SHUFFLE_PORT

public static final int DEFAULT_SHUFFLE_PORT

See Also:

Constant Field Values

SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED

public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED

See Also:

Constant Field Values

DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED

public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED

See Also:

Constant Field Values

SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT

public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT

See Also:

Constant Field Values

DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT

public static final int DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT

See Also:

Constant Field Values

SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE

public static final String SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE

See Also:

Constant Field Values

DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE

public static final int DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE

See Also:

Constant Field Values

CONNECTION_CLOSE

public static final String CONNECTION_CLOSE

See Also:

Constant Field Values

SUFFLE_SSL_FILE_BUFFER_SIZE_KEY

public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY

See Also:

Constant Field Values

DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE

public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE

See Also:

Constant Field Values

MAX_SHUFFLE_CONNECTIONS

public static final String MAX_SHUFFLE_CONNECTIONS

See Also:

Constant Field Values

DEFAULT_MAX_SHUFFLE_CONNECTIONS

public static final int DEFAULT_MAX_SHUFFLE_CONNECTIONS

See Also:

Constant Field Values

MAX_SHUFFLE_THREADS

public static final String MAX_SHUFFLE_THREADS

See Also:

Constant Field Values

DEFAULT_MAX_SHUFFLE_THREADS

public static final int DEFAULT_MAX_SHUFFLE_THREADS

See Also:

Constant Field Values

SHUFFLE_BUFFER_SIZE

public static final String SHUFFLE_BUFFER_SIZE

See Also:

Constant Field Values

DEFAULT_SHUFFLE_BUFFER_SIZE

public static final int DEFAULT_SHUFFLE_BUFFER_SIZE

See Also:

Constant Field Values

SHUFFLE_TRANSFERTO_ALLOWED

public static final String SHUFFLE_TRANSFERTO_ALLOWED

See Also:

Constant Field Values

DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED

public static final boolean DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED

See Also:

Constant Field Values

WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED

public static final boolean WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED

See Also:

Constant Field Values

Constructor Detail

ShuffleHandler

public ShuffleHandler()

Method Detail

serializeMetaData

public static ByteBuffer serializeMetaData(int port)
                                    throws IOException

Serialize the shuffle port into a ByteBuffer for use later on.

Parameters:

port - the port to be sent to the ApplciationMaster

Returns:

the serialized form of the port.

Throws:

IOException

deserializeMetaData

public static int deserializeMetaData(ByteBuffer meta)
                               throws IOException

A helper function to deserialize the metadata returned by ShuffleHandler.

Parameters:

meta - the metadata returned by the ShuffleHandler

Returns:

the port the Shuffle Handler is listening on to serve shuffle data.

Throws:

IOException

serializeServiceData

public static ByteBuffer serializeServiceData(org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier> jobToken)
                                       throws IOException

A helper function to serialize the JobTokenIdentifier to be sent to the ShuffleHandler as ServiceData.

Parameters:

jobToken - the job token to be used for authentication of shuffle data requests.

Returns:

the serialized version of the jobToken.

Throws:

IOException

initializeApplication

public void initializeApplication(org.apache.hadoop.yarn.server.api.ApplicationInitializationContext context)

Specified by:

initializeApplication in class org.apache.hadoop.yarn.server.api.AuxiliaryService

stopApplication

public void stopApplication(org.apache.hadoop.yarn.server.api.ApplicationTerminationContext context)

Specified by:

stopApplication in class org.apache.hadoop.yarn.server.api.AuxiliaryService

serviceInit

protected void serviceInit(org.apache.hadoop.conf.Configuration conf)
                    throws Exception

Overrides:

serviceInit in class org.apache.hadoop.service.AbstractService

Throws:

Exception

serviceStart

protected void serviceStart()
                     throws Exception

Overrides:

serviceStart in class org.apache.hadoop.service.AbstractService

Throws:

Exception

serviceStop

protected void serviceStop()
                    throws Exception

Overrides:

serviceStop in class org.apache.hadoop.service.AbstractService

Throws:

Exception

getMetaData

public ByteBuffer getMetaData()

Specified by:

getMetaData in class org.apache.hadoop.yarn.server.api.AuxiliaryService

getShuffle

protected org.apache.hadoop.mapred.ShuffleHandler.Shuffle getShuffle(org.apache.hadoop.conf.Configuration conf)

getCurrentVersion

protected org.apache.hadoop.yarn.server.records.Version getCurrentVersion()