From 8fd1676f782da85311fd0ca22cf46b2663195a0b Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Mon, 26 Jan 2026 10:42:44 +0100 Subject: [PATCH] TEZ-4672: Spotless: cleanup and force removing trailing whitespaces --- pom.xml | 1 + .../apache/tez/client/AMConfiguration.java | 10 +- .../tez/common/ContainerSignatureMatcher.java | 6 +- .../org/apache/tez/common/TezYARNUtils.java | 10 +- .../counters/AggregateFrameworkCounter.java | 6 +- .../common/counters/AggregateTezCounter.java | 2 +- .../counters/AggregateTezCounterDelegate.java | 4 +- .../common/counters/AggregateTezCounters.java | 6 +- .../tez/common/counters/DAGCounter.java | 2 +- .../tez/common/counters/JobCounter.java | 2 +- .../apache/tez/common/counters/Limits.java | 2 +- .../common/security/JobTokenIdentifier.java | 6 +- .../security/JobTokenSecretManager.java | 4 +- .../apache/tez/common/security/Master.java | 2 +- .../tez/common/security/TokenCache.java | 16 +- .../apache/tez/dag/api/DagTypeConverters.java | 10 +- .../tez/dag/api/DataSinkDescriptor.java | 18 +- .../tez/dag/api/DataSourceDescriptor.java | 26 +- .../java/org/apache/tez/dag/api/Edge.java | 10 +- .../apache/tez/dag/api/EdgeManagerPlugin.java | 24 +- .../tez/dag/api/EdgeManagerPluginContext.java | 8 +- .../apache/tez/dag/api/GroupInputEdge.java | 10 +- .../apache/tez/dag/api/OutputDescriptor.java | 2 +- .../org/apache/tez/dag/api/PreWarmVertex.java | 20 +- .../tez/dag/api/ProcessorDescriptor.java | 2 +- .../tez/dag/api/RootInputLeafOutput.java | 4 +- .../java/org/apache/tez/dag/api/Scope.java | 2 +- .../apache/tez/dag/api/TaskLocationHint.java | 6 +- .../tez/dag/api/TezUncheckedException.java | 2 +- .../java/org/apache/tez/dag/api/Vertex.java | 54 ++-- .../org/apache/tez/dag/api/VertexGroup.java | 32 +- .../tez/dag/api/VertexLocationHint.java | 4 +- .../tez/dag/api/VertexManagerPlugin.java | 20 +- .../dag/api/VertexManagerPluginContext.java | 72 ++--- .../api/VertexManagerPluginDescriptor.java | 4 +- .../apache/tez/dag/api/client/DAGClient.java | 8 +- .../tez/dag/api/client/DAGClientInternal.java | 8 +- .../apache/tez/dag/api/client/DAGStatus.java | 4 +- .../apache/tez/dag/api/client/Progress.java | 12 +- .../tez/dag/api/client/VertexStatus.java | 6 +- .../rpc/DAGClientAMProtocolBlockingPB.java | 2 +- .../api/AbstractLogicalIOProcessor.java | 2 +- .../apache/tez/runtime/api/DagIdentifier.java | 2 +- .../apache/tez/runtime/api/InputContext.java | 10 +- .../runtime/api/InputFrameworkInterface.java | 2 +- .../tez/runtime/api/InputInitializer.java | 6 +- .../runtime/api/InputInitializerContext.java | 12 +- .../tez/runtime/api/InputSpecUpdate.java | 14 +- .../tez/runtime/api/InputStatistics.java | 8 +- .../runtime/api/InputStatisticsReporter.java | 4 +- .../LogicalIOProcessorFrameworkInterface.java | 2 +- .../apache/tez/runtime/api/LogicalInput.java | 2 +- .../apache/tez/runtime/api/LogicalOutput.java | 2 +- .../tez/runtime/api/MemoryUpdateCallback.java | 4 +- .../tez/runtime/api/MergedInputContext.java | 8 +- .../tez/runtime/api/MergedLogicalInput.java | 4 +- .../tez/runtime/api/ObjectRegistry.java | 16 +- .../org/apache/tez/runtime/api/Output.java | 12 +- .../tez/runtime/api/OutputCommitter.java | 4 +- .../runtime/api/OutputCommitterContext.java | 2 +- .../apache/tez/runtime/api/OutputContext.java | 4 +- .../runtime/api/OutputFrameworkInterface.java | 2 +- .../tez/runtime/api/OutputStatistics.java | 6 +- .../runtime/api/OutputStatisticsReporter.java | 2 +- .../org/apache/tez/runtime/api/Processor.java | 4 +- .../tez/runtime/api/ProcessorContext.java | 12 +- .../runtime/api/TaskAttemptIdentifier.java | 4 +- .../tez/runtime/api/TaskIdentifier.java | 4 +- .../tez/runtime/api/VertexIdentifier.java | 4 +- .../tez/runtime/api/VertexStatistics.java | 4 +- .../events/CompositeDataMovementEvent.java | 14 +- .../runtime/api/events/DataMovementEvent.java | 6 +- .../InputConfigureVertexTasksEvent.java | 4 +- .../api/events/InputDataInformationEvent.java | 8 +- .../runtime/api/events/InputFailedEvent.java | 10 +- .../api/events/VertexManagerEvent.java | 14 +- .../org/apache/tez/client/TestTezClient.java | 42 +-- .../apache/tez/client/TestTezClientUtils.java | 12 +- .../java/org/apache/tez/dag/api/TestDAG.java | 2 +- .../org/apache/tez/dag/api/TestDAGPlan.java | 8 +- .../tez/dag/api/TestTaskLocationHint.java | 2 +- .../tez/dag/api/client/rpc/TestDAGClient.java | 42 +-- .../apache/tez/common/AsyncDispatcher.java | 30 +- .../tez/common/AsyncDispatcherConcurrent.java | 30 +- .../org/apache/tez/common/GcTimeUpdater.java | 6 +- .../apache/tez/common/TezAbstractEvent.java | 4 +- .../tez/common/TezContainerLogAppender.java | 6 +- .../tez/dag/records/DagIdentifierImpl.java | 8 +- .../records/TaskAttemptIdentifierImpl.java | 10 +- .../tez/dag/records/TaskIdentifierImpl.java | 8 +- .../org/apache/tez/dag/records/TezID.java | 8 +- .../tez/dag/records/TezTaskAttemptID.java | 20 +- .../org/apache/tez/dag/records/TezTaskID.java | 6 +- .../tez/dag/records/VertexIdentifierImpl.java | 10 +- .../tez/dag/utils/RelocalizationUtils.java | 2 +- .../tez/util/TezMxBeanResourceCalculator.java | 2 +- .../tez/common/TestAsyncDispatcher.java | 14 +- .../common/TestAsyncDispatcherConcurrent.java | 28 +- .../org/apache/tez/common/TestTezUtils.java | 8 +- .../tez/dag/api/client/DAGClientHandler.java | 2 +- .../dag/api/oldrecords/TaskAttemptReport.java | 6 +- .../dag/api/oldrecords/TaskAttemptState.java | 6 +- .../org/apache/tez/dag/app/AppContext.java | 4 +- .../org/apache/tez/dag/app/ClusterInfo.java | 2 +- .../apache/tez/dag/app/ContainerContext.java | 4 +- .../dag/app/ContainerHeartbeatHandler.java | 4 +- .../tez/dag/app/HeartbeatHandlerBase.java | 40 +-- .../tez/dag/app/TaskAttemptEventInfo.java | 6 +- .../app/TaskCommunicatorManagerInterface.java | 4 +- .../tez/dag/app/TaskHeartbeatHandler.java | 2 +- .../apache/tez/dag/app/dag/DAGScheduler.java | 14 +- .../org/apache/tez/dag/app/dag/DAGState.java | 2 +- .../tez/dag/app/dag/DAGTerminationCause.java | 12 +- .../tez/dag/app/dag/StateChangeNotifier.java | 20 +- .../java/org/apache/tez/dag/app/dag/Task.java | 12 +- .../apache/tez/dag/app/dag/TaskAttempt.java | 18 +- .../dag/app/dag/TaskAttemptStateInternal.java | 2 +- .../tez/dag/app/dag/TaskTerminationCause.java | 8 +- .../org/apache/tez/dag/app/dag/Vertex.java | 20 +- .../dag/app/dag/VertexTerminationCause.java | 8 +- .../dag/app/dag/event/CallableEventType.java | 2 +- .../app/dag/event/DAGEventCounterUpdate.java | 10 +- .../app/dag/event/DAGEventRecoverEvent.java | 6 +- .../dag/event/DAGEventSchedulerUpdate.java | 8 +- .../dag/app/dag/event/DAGEventStartDag.java | 2 +- .../dag/app/dag/event/SpeculatorEvent.java | 4 +- ...peculatorEventTaskAttemptStatusUpdate.java | 10 +- .../app/dag/event/SpeculatorEventType.java | 2 +- .../dag/app/dag/event/TaskAttemptEvent.java | 4 +- .../event/TaskAttemptEventAttemptFailed.java | 4 +- .../event/TaskAttemptEventAttemptKilled.java | 2 +- .../TaskAttemptEventContainerTerminated.java | 4 +- ...temptEventContainerTerminatedBySystem.java | 2 +- .../TaskAttemptEventContainerTerminating.java | 4 +- .../event/TaskAttemptEventKillRequest.java | 2 +- .../dag/event/TaskAttemptEventNodeFailed.java | 6 +- .../event/TaskAttemptEventOutputFailed.java | 12 +- .../dag/event/TaskAttemptEventSchedule.java | 6 +- .../event/TaskAttemptEventStatusUpdate.java | 10 +- .../app/dag/event/TaskAttemptEventType.java | 6 +- .../tez/dag/app/dag/event/TaskEvent.java | 2 +- .../app/dag/event/TaskEventScheduleTask.java | 4 +- .../tez/dag/app/dag/event/TaskEventType.java | 2 +- .../VertexEventInputDataInformation.java | 4 +- .../VertexEventManagerUserCodeError.java | 4 +- .../event/VertexEventNullEdgeInitialized.java | 6 +- .../dag/event/VertexEventRootInputFailed.java | 6 +- .../VertexEventRootInputInitialized.java | 4 +- .../app/dag/event/VertexEventRouteEvent.java | 2 +- ...VertexEventSourceTaskAttemptCompleted.java | 2 +- .../event/VertexEventSourceVertexStarted.java | 6 +- .../app/dag/event/VertexEventTermination.java | 4 +- .../dag/app/dag/event/VertexEventType.java | 16 +- .../dag/app/dag/impl/AMUserCodeException.java | 8 +- .../app/dag/impl/BroadcastEdgeManager.java | 16 +- .../apache/tez/dag/app/dag/impl/DAGImpl.java | 20 +- .../dag/impl/DAGSchedulerNaturalOrder.java | 16 +- .../org/apache/tez/dag/app/dag/impl/Edge.java | 66 ++--- .../dag/impl/ImmediateStartVertexManager.java | 4 +- .../dag/app/dag/impl/OneToOneEdgeManager.java | 10 +- .../dag/impl/OneToOneEdgeManagerOnDemand.java | 20 +- .../dag/impl/OutputCommitterContextImpl.java | 2 +- .../app/dag/impl/RootInputVertexManager.java | 6 +- .../tez/dag/app/dag/impl/TaskAttemptImpl.java | 70 ++--- .../app/dag/impl/TaskAttemptImplHelpers.java | 4 +- .../TezRootInputInitializerContextImpl.java | 4 +- .../tez/dag/app/dag/impl/VertexImpl.java | 144 ++++----- .../tez/dag/app/dag/impl/VertexManager.java | 80 ++--- .../speculation/legacy/LegacySpeculator.java | 16 +- .../legacy/LegacyTaskRuntimeEstimator.java | 6 +- .../legacy/TaskRuntimeEstimator.java | 2 +- .../tez/dag/app/rm/AMSchedulerEvent.java | 4 +- .../AMSchedulerEventDeallocateContainer.java | 2 +- .../dag/app/rm/AMSchedulerEventTAEnded.java | 4 +- .../rm/AMSchedulerEventTALaunchRequest.java | 2 +- .../tez/dag/app/rm/AMSchedulerEventType.java | 2 +- .../tez/dag/app/rm/TaskSchedulerManager.java | 12 +- .../tez/dag/app/rm/TezAMRMClientAsync.java | 6 +- .../dag/app/rm/YarnTaskSchedulerService.java | 250 ++++++++-------- .../tez/dag/app/rm/container/AMContainer.java | 2 +- .../app/rm/container/AMContainerEvent.java | 8 +- .../container/AMContainerEventAssignTA.java | 4 +- .../container/AMContainerEventCompleted.java | 14 +- .../AMContainerEventLaunchFailed.java | 8 +- .../container/AMContainerEventLaunched.java | 4 +- .../container/AMContainerEventStopFailed.java | 6 +- .../AMContainerEventStopRequest.java | 4 +- .../AMContainerEventTASucceeded.java | 6 +- .../rm/container/AMContainerEventType.java | 12 +- .../app/rm/container/AMContainerHelpers.java | 4 +- .../dag/app/rm/container/AMContainerImpl.java | 24 +- .../app/rm/container/AMContainerState.java | 6 +- .../dag/app/rm/container/AMContainerTask.java | 4 +- .../rm/container/ContainerContextMatcher.java | 2 +- .../apache/tez/dag/app/rm/node/AMNode.java | 2 +- .../node/AMNodeEventContainerAllocated.java | 4 +- .../rm/node/AMNodeEventNodeCountUpdated.java | 4 +- .../app/rm/node/AMNodeEventStateChanged.java | 6 +- .../rm/node/AMNodeEventTaskAttemptEnded.java | 12 +- .../node/AMNodeEventTaskAttemptSucceeded.java | 4 +- .../tez/dag/app/rm/node/AMNodeImpl.java | 2 +- .../tez/dag/app/rm/node/AMNodeTracker.java | 8 +- .../authorize/TezAMPolicyProvider.java | 4 +- .../history/events/ContainerStoppedEvent.java | 8 +- .../history/events/DAGKillRequestEvent.java | 4 +- .../events/TaskAttemptFinishedEvent.java | 18 +- .../events/TaskAttemptStartedEvent.java | 2 +- .../events/VertexConfigurationDoneEvent.java | 2 +- .../events/VertexInitializedEvent.java | 2 +- .../impl/HistoryEventJsonConversion.java | 4 +- .../tez/dag/history/utils/DAGUtils.java | 8 +- .../tez/dag/history/utils/TezEventUtils.java | 2 +- .../apache/tez/dag/utils/TezBuilderUtils.java | 6 +- .../dag/api/client/TestDAGClientHandler.java | 12 +- .../org/apache/tez/dag/app/MockClock.java | 4 +- .../apache/tez/dag/app/MockDAGAppMaster.java | 64 ++-- .../apache/tez/dag/app/MockLocalClient.java | 2 +- .../org/apache/tez/dag/app/MockTezClient.java | 10 +- .../tez/dag/app/TestMemoryWithEvents.java | 26 +- .../tez/dag/app/TestMockDAGAppMaster.java | 130 ++++---- .../apache/tez/dag/app/TestPreemption.java | 50 ++-- .../tez/dag/app/TestRecoveryParser.java | 52 ++-- .../apache/tez/dag/app/TestSpeculation.java | 8 +- .../dag/app/TestTaskCommunicatorManager1.java | 14 +- .../dag/app/dag/TestStateChangeNotifier.java | 6 +- .../app/dag/impl/CallableEventDispatcher.java | 2 +- .../tez/dag/app/dag/impl/TestCommit.java | 24 +- .../tez/dag/app/dag/impl/TestDAGImpl.java | 86 +++--- .../tez/dag/app/dag/impl/TestDAGRecovery.java | 178 +++++------ .../dag/app/dag/impl/TestDAGScheduler.java | 28 +- .../apache/tez/dag/app/dag/impl/TestEdge.java | 22 +- .../impl/TestImmediateStartVertexManager.java | 14 +- .../tez/dag/app/dag/impl/TestTaskAttempt.java | 46 +-- .../tez/dag/app/dag/impl/TestTaskImpl.java | 32 +- .../tez/dag/app/dag/impl/TestVertexImpl.java | 278 +++++++++--------- .../dag/app/dag/impl/TestVertexManager.java | 6 +- .../legacy/TestDataStatistics.java | 6 +- .../tez/dag/app/rm/TestContainerReuse.java | 16 +- .../tez/dag/app/rm/TestTaskScheduler.java | 66 ++--- .../dag/app/rm/TestTaskSchedulerHelpers.java | 8 +- .../dag/app/rm/TestTaskSchedulerManager.java | 22 +- .../dag/app/rm/container/TestAMContainer.java | 18 +- .../dag/app/rm/node/TestAMNodeTracker.java | 8 +- .../TestHistoryEventsProtoConversion.java | 12 +- .../impl/TestHistoryEventJsonConversion.java | 2 +- .../apache/tez/test/EdgeManagerForTest.java | 4 +- .../tez/test/VertexManagerPluginForTest.java | 16 +- .../org/apache/tez/examples/JoinDataGen.java | 6 +- .../org/apache/tez/examples/JoinValidate.java | 2 +- .../apache/tez/examples/OrderedWordCount.java | 20 +- .../tez/examples/SimpleSessionExample.java | 20 +- .../apache/tez/examples/TezExampleBase.java | 4 +- .../org/apache/tez/examples/WordCount.java | 34 +-- .../apache/tez/shufflehandler/IndexCache.java | 6 +- .../tez/shufflehandler/ShuffleHandler.java | 8 +- .../mapred/split/SplitSizeEstimator.java | 2 +- .../hadoop/mapred/split/TezGroupedSplit.java | 24 +- .../split/TezGroupedSplitsInputFormat.java | 28 +- .../mapreduce/split/TezGroupedSplit.java | 28 +- .../split/TezGroupedSplitsInputFormat.java | 32 +- .../split/TezMapReduceSplitsGrouper.java | 4 +- .../tez/mapreduce/client/ClientCache.java | 2 +- .../client/ClientServiceDelegate.java | 4 +- .../mapreduce/client/ResourceMgrDelegate.java | 12 +- .../tez/mapreduce/client/YARNRunner.java | 14 +- .../tez/mapreduce/combine/MRCombiner.java | 42 +-- .../committer/MROutputCommitter.java | 4 +- .../common/MRInputAMSplitGenerator.java | 14 +- .../common/MRInputSplitDistributor.java | 6 +- .../tez/mapreduce/hadoop/DeprecatedKeys.java | 60 ++-- .../tez/mapreduce/hadoop/IDConverter.java | 8 +- .../tez/mapreduce/hadoop/InputSplitInfo.java | 10 +- .../mapreduce/hadoop/InputSplitInfoDisk.java | 4 +- .../mapreduce/hadoop/InputSplitInfoMem.java | 8 +- .../tez/mapreduce/hadoop/MRJobConfig.java | 90 +++--- .../MultiStageMRConfToTezTranslator.java | 4 +- .../hadoop/MultiStageMRConfigUtil.java | 4 +- .../mapreduce/hadoop/TezTypeConverters.java | 8 +- .../hadoop/mapred/JobContextImpl.java | 14 +- .../mapreduce/hadoop/mapred/MRCounters.java | 22 +- .../mapreduce/hadoop/mapred/MRReporter.java | 8 +- .../hadoop/mapred/TaskAttemptContextImpl.java | 12 +- .../hadoop/mapreduce/JobContextImpl.java | 108 +++---- .../hadoop/mapreduce/MapContextImpl.java | 4 +- .../mapreduce/TaskAttemptContextImpl.java | 14 +- .../mapreduce/TaskInputOutputContextImpl.java | 12 +- .../apache/tez/mapreduce/input/MRInput.java | 32 +- .../tez/mapreduce/input/MRInputLegacy.java | 14 +- .../tez/mapreduce/input/MultiMRInput.java | 4 +- .../tez/mapreduce/lib/MRInputUtils.java | 4 +- .../apache/tez/mapreduce/lib/MRReader.java | 10 +- .../tez/mapreduce/lib/MRReaderMapred.java | 4 +- .../apache/tez/mapreduce/output/MROutput.java | 18 +- .../mapreduce/partition/MRPartitioner.java | 2 +- .../tez/mapreduce/processor/MRTask.java | 8 +- .../mapreduce/processor/MRTaskReporter.java | 2 +- .../processor/SimpleMRProcessor.java | 4 +- .../mapreduce/processor/map/MapProcessor.java | 16 +- .../processor/reduce/ReduceProcessor.java | 2 +- .../mapred/split/TestGroupedSplits.java | 62 ++-- .../apache/tez/mapreduce/TezTestUtils.java | 2 +- .../hadoop/TestConfigTranslationMRToTez.java | 8 +- .../tez/mapreduce/hadoop/TestMRHelpers.java | 2 +- .../tez/mapreduce/output/TestMROutput.java | 14 +- .../tez/mapreduce/processor/MapUtils.java | 18 +- .../processor/map/TestMapProcessor.java | 24 +- .../processor/reduce/TestReduceProcessor.java | 14 +- .../tez/auxservices/TestShuffleHandler.java | 2 +- .../parser/datamodel/TaskAttemptInfo.java | 30 +- .../history/parser/datamodel/VertexInfo.java | 14 +- .../tez/history/parser/utils/Utils.java | 4 +- .../apache/tez/history/TestHistoryParser.java | 2 +- .../ats/acls/TestATSHistoryWithACLs.java | 4 +- .../ats/HistoryEventTimelineConversion.java | 2 +- .../TestHistoryEventTimelineConversion.java | 4 +- .../tez/tests/MiniTezClusterWithTimeline.java | 6 +- .../org/apache/tez/common/ContainerTask.java | 6 +- .../apache/tez/common/ProtoConverters.java | 4 +- .../apache/tez/common/TezLocalResource.java | 2 +- .../apache/tez/runtime/InputReadyTracker.java | 4 +- .../LogicalIOProcessorRuntimeTask.java | 12 +- .../org/apache/tez/runtime/RuntimeTask.java | 20 +- .../api/events/TaskStatusUpdateEvent.java | 6 +- .../tez/runtime/api/impl/GroupInputSpec.java | 14 +- .../tez/runtime/api/impl/IOStatistics.java | 10 +- .../apache/tez/runtime/api/impl/TaskSpec.java | 4 +- .../tez/runtime/api/impl/TaskStatistics.java | 4 +- .../api/impl/TezHeartbeatResponse.java | 4 +- .../runtime/api/impl/TezInputContextImpl.java | 10 +- .../api/impl/TezMergedInputContextImpl.java | 2 +- .../api/impl/TezOutputContextImpl.java | 4 +- .../runtime/api/impl/TezTaskContextImpl.java | 10 +- .../objectregistry/ObjectRegistryImpl.java | 2 +- .../common/resources/MemoryDistributor.java | 4 +- .../common/resources/ScalingAllocator.java | 2 +- .../runtime/metrics/TaskCounterUpdater.java | 14 +- .../tez/runtime/task/ContainerReporter.java | 2 +- .../tez/runtime/task/ErrorReporter.java | 2 +- .../org/apache/tez/runtime/task/TezChild.java | 8 +- .../tez/runtime/TestInputReadyTracker.java | 40 +-- .../resources/TestMemoryDistributor.java | 66 ++--- .../tez/runtime/task/TestTaskReporter.java | 8 +- .../InputReadyVertexManager.java | 36 +-- .../vertexmanager/ShuffleVertexManager.java | 40 +-- .../runtime/library/api/KeyValueReader.java | 10 +- .../runtime/library/api/KeyValueWriter.java | 2 +- .../runtime/library/api/KeyValuesReader.java | 8 +- .../tez/runtime/library/api/Partitioner.java | 14 +- .../runtime/library/common/ConfigUtils.java | 10 +- .../tez/runtime/library/common/Constants.java | 16 +- .../common/InputAttemptIdentifier.java | 2 +- .../library/common/InputIdentifier.java | 2 +- .../library/common/ValuesIterator.java | 38 +-- .../library/common/combine/Combiner.java | 4 +- .../common/readers/UnorderedKVReader.java | 24 +- .../common/shuffle/DiskFetchedInput.java | 6 +- .../library/common/shuffle/FetchedInput.java | 12 +- .../common/shuffle/FetchedInputAllocator.java | 4 +- .../common/shuffle/FetchedInputCallback.java | 8 +- .../common/shuffle/FetcherCallback.java | 2 +- .../library/common/shuffle/InputHost.java | 4 +- .../common/shuffle/LocalDiskFetchedInput.java | 2 +- .../common/shuffle/MemoryFetchedInput.java | 4 +- .../library/common/shuffle/ShuffleUtils.java | 6 +- .../impl/ShuffleInputEventHandlerImpl.java | 4 +- .../common/shuffle/impl/ShuffleManager.java | 36 +-- .../impl/SimpleFetchedInputAllocator.java | 14 +- .../shuffle/orderedgrouped/MapHost.java | 12 +- .../shuffle/orderedgrouped/MapOutput.java | 18 +- .../shuffle/orderedgrouped/MergeThread.java | 14 +- .../shuffle/orderedgrouped/ShuffleHeader.java | 14 +- .../common/sort/impl/ExternalSorter.java | 12 +- .../library/common/sort/impl/IFile.java | 2 +- .../common/sort/impl/PipelinedSorter.java | 52 ++-- .../sort/impl/TezRawKeyValueIterator.java | 32 +- .../common/sort/impl/dflt/DefaultSorter.java | 6 +- .../task/local/output/TezTaskOutput.java | 2 +- .../BaseUnorderedPartitionedKVWriter.java | 12 +- .../OrderedPartitionedKVOutputConfig.java | 2 +- .../ConcatenatedMergedKeyValueInput.java | 4 +- .../ConcatenatedMergedKeyValuesInput.java | 6 +- .../library/input/OrderedGroupedKVInput.java | 8 +- .../input/OrderedGroupedMergedKVInput.java | 4 +- .../library/output/UnorderedKVOutput.java | 4 +- .../library/processor/PreWarmProcessor.java | 14 +- .../WeightedScalingMemoryDistributor.java | 2 +- .../TestInputReadyVertexManager.java | 52 ++-- .../TestWeightedScalingMemoryDistributor.java | 6 +- .../library/common/TestInputIdentifiers.java | 4 +- .../library/common/TestValuesIterator.java | 4 +- .../library/common/shuffle/TestFetcher.java | 2 +- .../TestShuffleInputEventHandlerImpl.java | 4 +- .../orderedgrouped/TestMergeManager.java | 2 +- .../orderedgrouped/TestShuffleScheduler.java | 2 +- .../input/TestSortedGroupedMergedInput.java | 4 +- .../output/TestOnFileSortedOutput.java | 6 +- .../output/TestOnFileUnorderedKVOutput.java | 4 +- .../examples/BroadcastAndOneToOneExample.java | 18 +- .../apache/tez/mapreduce/examples/Join.java | 14 +- .../tez/mapreduce/examples/MRRSleepJob.java | 8 +- .../examples/MultipleCommitsExample.java | 16 +- .../mapreduce/examples/RandomTextWriter.java | 56 ++-- .../tez/mapreduce/examples/RandomWriter.java | 58 ++-- .../tez/mapreduce/examples/SecondarySort.java | 40 +-- .../apache/tez/mapreduce/examples/Sort.java | 36 +-- .../examples/TestOrderedWordCount.java | 10 +- .../tez/mapreduce/examples/UnionExample.java | 18 +- .../processor/FilterByWordInputProcessor.java | 2 +- .../FilterByWordOutputProcessor.java | 2 +- .../tez/mapreduce/TestMRRJobsDAGApi.java | 14 +- .../apache/tez/test/AMShutdownController.java | 4 +- .../tez/test/FaultToleranceTestRunner.java | 38 +-- .../org/apache/tez/test/MiniTezCluster.java | 2 +- .../RecoveryServiceWithEventHandlingHook.java | 8 +- .../org/apache/tez/test/SimpleTestDAG.java | 8 +- .../tez/test/SimpleTestDAG3Vertices.java | 8 +- .../org/apache/tez/test/TestAMRecovery.java | 10 +- .../org/apache/tez/test/TestDAGRecovery2.java | 2 +- .../tez/test/TestExceptionPropagation.java | 12 +- .../apache/tez/test/TestFaultTolerance.java | 176 +++++------ .../java/org/apache/tez/test/TestInput.java | 60 ++-- .../java/org/apache/tez/test/TestOutput.java | 6 +- .../org/apache/tez/test/TestProcessor.java | 56 ++-- .../org/apache/tez/test/TestRecovery.java | 8 +- .../java/org/apache/tez/test/TestTezJobs.java | 22 +- .../tez/test/dag/SimpleReverseVTestDAG.java | 6 +- .../apache/tez/test/dag/SimpleVTestDAG.java | 6 +- .../plugins/CriticalPathAnalyzer.java | 92 +++--- .../tez/analyzer/plugins/TezAnalyzerBase.java | 14 +- .../org/apache/tez/analyzer/TestAnalyzer.java | 152 +++++----- 430 files changed, 3250 insertions(+), 3249 deletions(-) diff --git a/pom.xml b/pom.xml index 74943817b0..f86ea7c878 100644 --- a/pom.xml +++ b/pom.xml @@ -1028,6 +1028,7 @@ java,javax,org.apache,com,net,io + diff --git a/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java b/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java index 1b61c45d05..4b0f228a27 100644 --- a/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java @@ -50,15 +50,15 @@ class AMConfiguration { void addAMLocalResources(Map localResources) { this.amLocalResources.putAll(localResources); } - + void clearAMLocalResources() { this.amLocalResources.clear(); } - + void setCredentials(Credentials credentials) { this.credentials = credentials; } - + void setTezConfiguration(TezConfiguration tezConf) { this.tezConf = tezConf; } @@ -78,11 +78,11 @@ TezConfiguration getTezConfiguration() { Credentials getCredentials() { return credentials; } - + void setBinaryConfLR(LocalResource binaryConfLRsrc) { this.binaryConfLRsrc = binaryConfLRsrc; } - + LocalResource getBinaryConfLR() { return binaryConfLRsrc; } diff --git a/tez-api/src/main/java/org/apache/tez/common/ContainerSignatureMatcher.java b/tez-api/src/main/java/org/apache/tez/common/ContainerSignatureMatcher.java index c0a1245aec..44e44499ff 100644 --- a/tez-api/src/main/java/org/apache/tez/common/ContainerSignatureMatcher.java +++ b/tez-api/src/main/java/org/apache/tez/common/ContainerSignatureMatcher.java @@ -33,16 +33,16 @@ public interface ContainerSignatureMatcher { * signature. */ public boolean isSuperSet(Object cs1, Object cs2); - + /** * Checks if the container signatures match exactly * @return true if exact match */ public boolean isExactMatch(Object cs1, Object cs2); - + /** * Gets additional resources specified in lr2, which are not present for lr1 - * + * * @param lr1 * @param lr2 * @return additional resources specified in lr2, which are not present for lr1 diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java index 42e2b45b90..ca51279846 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -162,7 +162,7 @@ public static void appendToEnvFromInputString(Map env, } } } - + public static void setEnvIfAbsentFromInputString(Map env, String envString) { if (envString != null && envString.length() > 0) { @@ -189,7 +189,7 @@ public static void setEnvIfAbsentFromInputString(Map env, } } } - + public static void addToEnvironment( Map environment, String variable, String value, String classPathSeparator) { @@ -199,7 +199,7 @@ public static void addToEnvironment( } else { val = val + classPathSeparator + value; } - environment.put(StringInterner.weakIntern(variable), + environment.put(StringInterner.weakIntern(variable), StringInterner.weakIntern(val)); } diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/AggregateFrameworkCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/AggregateFrameworkCounter.java index aa7d4462cf..8bbc34b286 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/AggregateFrameworkCounter.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/AggregateFrameworkCounter.java @@ -22,7 +22,7 @@ @SuppressWarnings("rawtypes") public class AggregateFrameworkCounter> extends FrameworkCounter implements AggregateTezCounter { - + private long min = Long.MAX_VALUE; private long max = Long.MIN_VALUE; private long count = 0; @@ -36,7 +36,7 @@ public AggregateFrameworkCounter(Enum ref, String groupName) { public void increment(long incr) { throw new IllegalArgumentException("Cannot increment an aggregate counter directly"); } - + @Override public void aggregate(TezCounter other) { final long val = other.getValue(); @@ -71,7 +71,7 @@ public long getMin() { public long getMax() { return max; } - + @SuppressWarnings("unchecked") public FrameworkCounter asFrameworkCounter() { return ((FrameworkCounter)this); diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounter.java index bf711dae86..2bccd154f5 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounter.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounter.java @@ -25,7 +25,7 @@ public interface AggregateTezCounter { public abstract long getMin(); public abstract long getMax(); - + public abstract long getCount(); } \ No newline at end of file diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounterDelegate.java b/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounterDelegate.java index ae2ca7b0c6..80018c3e0a 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounterDelegate.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounterDelegate.java @@ -32,7 +32,7 @@ public class AggregateTezCounterDelegate extends AbstractC public AggregateTezCounterDelegate(T child) { this.child = child; } - + @Override public String getName() { return child.getName(); // this is a pass-through @@ -57,7 +57,7 @@ public void setValue(long value) { public void increment(long incr) { throw new UnsupportedOperationException("Cannot increment an aggregate counter"); } - + /* (non-Javadoc) * @see org.apache.tez.common.counters.AggregateTezCounter#aggregate(org.apache.tez.common.counters.TezCounter) */ diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounters.java b/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounters.java index 332c24a960..86cd7ae528 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounters.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounters.java @@ -19,13 +19,13 @@ package org.apache.tez.common.counters; public class AggregateTezCounters extends TezCounters { - + private static final GroupFactory groupFactory = new GroupFactory(); - + public AggregateTezCounters() { super(groupFactory); } - + // Mix framework group implementation into CounterGroup interface private static class AggregateFrameworkGroupImpl> extends FrameworkCounterGroup implements CounterGroup { diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/DAGCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/DAGCounter.java index ca575d4dfa..a56af4aa94 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/DAGCounter.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/DAGCounter.java @@ -26,7 +26,7 @@ @InterfaceAudience.Public @InterfaceStability.Evolving public enum DAGCounter { - NUM_FAILED_TASKS, + NUM_FAILED_TASKS, NUM_KILLED_TASKS, NUM_SUCCEEDED_TASKS, TOTAL_LAUNCHED_TASKS, diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/JobCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/JobCounter.java index 14e76bc081..8fdbc5c79a 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/JobCounter.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/JobCounter.java @@ -23,7 +23,7 @@ // Per-job counters @Private public enum JobCounter { - NUM_FAILED_MAPS, + NUM_FAILED_MAPS, NUM_FAILED_REDUCES, NUM_KILLED_MAPS, NUM_KILLED_REDUCES, diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java b/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java index a12b0de8b8..0973ea825b 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java @@ -29,7 +29,7 @@ @InterfaceAudience.Private public class Limits { - + private static final Logger LOG = LoggerFactory.getLogger(Limits.class); private static final Configuration DEFAULT_CONFIGURATION = new TezConfiguration(); diff --git a/tez-api/src/main/java/org/apache/tez/common/security/JobTokenIdentifier.java b/tez-api/src/main/java/org/apache/tez/common/security/JobTokenIdentifier.java index 47bf74d2ee..64c79861ab 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/JobTokenIdentifier.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/JobTokenIdentifier.java @@ -37,7 +37,7 @@ public class JobTokenIdentifier extends TokenIdentifier { private Text jobid; public final static Text KIND_NAME = new Text("tez.job"); - + /** * Default constructor */ @@ -58,7 +58,7 @@ public JobTokenIdentifier(Text jobid) { public Text getKind() { return KIND_NAME; } - + /** {@inheritDoc} */ @Override public UserGroupInformation getUser() { @@ -67,7 +67,7 @@ public UserGroupInformation getUser() { } return UserGroupInformation.createRemoteUser(jobid.toString()); } - + /** * Get the jobid * @return the jobid diff --git a/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java b/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java index d4cd2be83e..b46dbf377e 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java @@ -129,7 +129,7 @@ public void removeTokenForJob(String jobId) { currentJobTokens.remove(jobId); } } - + /** * Look up the token password/secret for the given jobId. * @param jobId the jobId to look up @@ -146,7 +146,7 @@ public SecretKey retrieveTokenSecret(String jobId) throws InvalidToken { } return tokenSecret; } - + /** * Look up the token password/secret for the given job token identifier. * @param identifier the job token identifier to look up diff --git a/tez-api/src/main/java/org/apache/tez/common/security/Master.java b/tez-api/src/main/java/org/apache/tez/common/security/Master.java index d0b8d16c47..67d9adb78c 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/Master.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/Master.java @@ -41,7 +41,7 @@ public static String getMasterUserName(Configuration conf) { return conf.get(YarnConfiguration.RM_PRINCIPAL); } - + // This needs to go into YARN public static InetSocketAddress getMasterAddress(Configuration conf) { return conf diff --git a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java index ba3ce5b6d1..c5dd89a06f 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java @@ -42,17 +42,17 @@ * This class provides user facing APIs for transferring secrets from * the job client to the tasks. * The secrets can be stored just before submission of jobs and read during - * the task execution. + * the task execution. */ @InterfaceAudience.Private @InterfaceStability.Unstable public final class TokenCache { - + private static final Logger LOG = LoggerFactory.getLogger(TokenCache.class); private TokenCache() {} - + /** * auxiliary method to get user's secret keys.. * @@ -63,9 +63,9 @@ public static byte[] getSecretKey(Credentials credentials, Text alias) { return null; return credentials.getSecretKey(alias); } - + /** - * Convenience method to obtain delegation tokens from namenodes + * Convenience method to obtain delegation tokens from namenodes * corresponding to the paths passed. * @param credentials credentials * @param ps array of paths @@ -121,7 +121,7 @@ static boolean isTokenRenewalExcluded(FileSystem fs, Configuration conf) { /** * get delegation token for a specific FS */ - static void obtainTokensForFileSystemsInternal(FileSystem fs, + static void obtainTokensForFileSystemsInternal(FileSystem fs, Credentials credentials, Configuration conf) throws IOException { // TODO Change this to use YARN utilities once YARN-1664 is fixed. // RM skips renewing token with empty renewer @@ -149,12 +149,12 @@ static void obtainTokensForFileSystemsInternal(FileSystem fs, * store session specific token */ @InterfaceAudience.Private - public static void setSessionToken(Token t, + public static void setSessionToken(Token t, Credentials credentials) { credentials.addToken(SESSION_TOKEN, t); } /** - * + * * @return session token */ @SuppressWarnings("unchecked") diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java index 1e68226b46..1d292e7ad6 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java @@ -102,7 +102,7 @@ public static PlanLocalResourceVisibility convertToDAGPlan(LocalResourceVisibili default : throw new RuntimeException("unknown 'visibility': " + visibility); } } - + public static List convertToDAGPlan(Map lrs) { List planLrs = Lists.newArrayListWithCapacity(lrs.size()); for (Entry entry : lrs.entrySet()) { @@ -224,7 +224,7 @@ public static VertexLocationHint convertFromDAGPlan( } return VertexLocationHint.create(outputList); } - + public static String convertToDAGPlan(URL resource) { Path p; try { @@ -281,7 +281,7 @@ public static Map createEdgePlanMapFromDAGPlan(List } return edgePlanMap; } - + public static PlanEdgeProperty convertToProto(EdgeProperty prop) { PlanEdgeProperty.Builder edgePropBuilder = PlanEdgeProperty.newBuilder(); edgePropBuilder.setDataMovementType(convertToDAGPlan(prop.getDataMovementType())); @@ -294,10 +294,10 @@ public static PlanEdgeProperty convertToProto(EdgeProperty prop) { edgePropBuilder.setEdgeManager(DagTypeConverters.convertToDAGPlan(prop .getEdgeManagerDescriptor())); } - + return edgePropBuilder.build(); } - + public static EdgeProperty convertFromProto(PlanEdgeProperty edge) { return EdgeProperty.create( (edge.hasEdgeManager() ? diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java index fec2bf6eed..08c8645819 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java @@ -31,14 +31,14 @@ import com.google.common.collect.Sets; /** - * Defines the output and output committer for a data sink + * Defines the output and output committer for a data sink * */ @Public public class DataSinkDescriptor { private final OutputDescriptor outputDescriptor; private final OutputCommitterDescriptor committerDescriptor; - + private final Credentials credentials; private final Collection urisForCredentials = Sets.newHashSet(); @@ -90,7 +90,7 @@ public static DataSinkDescriptor create(OutputDescriptor outputDescriptor, @Nullable Credentials credentials) { return new DataSinkDescriptor(outputDescriptor, committerDescriptor, credentials); } - + /** * Get the {@link OutputDescriptor} for this {@link DataSinkDescriptor} * @return {@link OutputDescriptor} @@ -98,7 +98,7 @@ public static DataSinkDescriptor create(OutputDescriptor outputDescriptor, public OutputDescriptor getOutputDescriptor() { return outputDescriptor; } - + /** * Get the {@link OutputCommitterDescriptor} for this {@link DataSinkDescriptor} * @return {@link OutputCommitterDescriptor} @@ -106,16 +106,16 @@ public OutputDescriptor getOutputDescriptor() { public @Nullable OutputCommitterDescriptor getOutputCommitterDescriptor() { return committerDescriptor; } - - /** + + /** * This method can be used to specify a list of URIs for which Credentials * need to be obtained so that the job can run. An incremental list of URIs * can be provided by making multiple calls to the method. - * + * * Currently, @{link credentials} can only be fetched for HDFS and other * {@link org.apache.hadoop.fs.FileSystem} implementations that support * credentials. - * + * * @param uris * a list of {@link URI}s * @return this @@ -125,7 +125,7 @@ public synchronized DataSinkDescriptor addURIsForCredentials(Collection uri urisForCredentials.addAll(uris); return this; } - + /** * Get the URIs for which credentials will be obtained * @return an unmodifiable list representing the URIs for which credentials diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java index 199474b386..c425688113 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java @@ -36,14 +36,14 @@ import com.google.common.collect.Sets; /** - * Defines the input and input initializer for a data source + * Defines the input and input initializer for a data source * */ @Public public class DataSourceDescriptor { private final InputDescriptor inputDescriptor; private final InputInitializerDescriptor initializerDescriptor; - + private final Credentials credentials; private final int numShards; private final VertexLocationHint locationHint; @@ -126,13 +126,13 @@ public static DataSourceDescriptor create(InputDescriptor inputDescriptor, } /** - * Get the {@link InputDescriptor} for this {@link DataSourceDescriptor} + * Get the {@link InputDescriptor} for this {@link DataSourceDescriptor} * @return {@link InputDescriptor} */ public InputDescriptor getInputDescriptor() { return inputDescriptor; } - + /** * Get the {@link InputInitializerDescriptor} for this {@link DataSourceDescriptor} * @return {@link InputInitializerDescriptor} @@ -140,16 +140,16 @@ public InputDescriptor getInputDescriptor() { public @Nullable InputInitializerDescriptor getInputInitializerDescriptor() { return initializerDescriptor; } - - /** + + /** * This method can be used to specify a list of URIs for which Credentials * need to be obtained so that the job can run. An incremental list of URIs * can be provided by making multiple calls to the method. - * + * * Currently, @{link credentials} can only be fetched for HDFS and other * {@link org.apache.hadoop.fs.FileSystem} implementations that support * credentials. - * + * * @param uris * a list of {@link URI}s * @return this @@ -159,7 +159,7 @@ public synchronized DataSourceDescriptor addURIsForCredentials(Collection u urisForCredentials.addAll(uris); return this; } - + /** * Get the URIs for which credentials will be obtained * @return an unmodifiable list representing the URIs for which credentials @@ -168,10 +168,10 @@ public synchronized DataSourceDescriptor addURIsForCredentials(Collection u public Collection getURIsForCredentials() { return Collections.unmodifiableCollection(urisForCredentials); } - + /** * Number of shards for this data source. If a vertex has only one - * data source this the number of tasks in the vertex should be set to + * data source this the number of tasks in the vertex should be set to * the number of shards * Returns -1 when this is determined at runtime in the AM. * @return number of tasks @@ -180,7 +180,7 @@ public Collection getURIsForCredentials() { public int getNumberOfShards() { return numShards; } - + /** * Returns any credentials needed to access this data source. * Is null when this calculation happens on the AppMaster (default) @@ -190,7 +190,7 @@ public int getNumberOfShards() { public @Nullable Credentials getCredentials() { return credentials; } - + /** * Get the location hints for the tasks in the vertex for this data source. * Is null when shard calculation happens on the AppMaster (default) diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java b/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java index db509f7050..cc76b109b4 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java @@ -23,9 +23,9 @@ /** * Edge defines the connection between a producer and consumer vertex in the DAG. * @link {@link EdgeProperty} defines the relationship between them. The producer - * vertex provides input to the edge and the consumer vertex reads output from the + * vertex provides input to the edge and the consumer vertex reads output from the * edge. - * + * */ @Public public class Edge { @@ -70,7 +70,7 @@ public Vertex getInputVertex() { /** * The @link {@link Vertex} that reads output from the edge - * @return {@link Vertex} + * @return {@link Vertex} */ public Vertex getOutputVertex() { return outputVertex; @@ -83,7 +83,7 @@ public Vertex getOutputVertex() { public EdgeProperty getEdgeProperty() { return edgeProperty; } - + /* * Used to identify the edge in the configuration */ @@ -92,7 +92,7 @@ public String getId() { // ensure it is unique. return String.valueOf(System.identityHashCode(this)); } - + @Override public String toString() { return inputVertex + " -> " + outputVertex + " (" + edgeProperty + ")"; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPlugin.java b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPlugin.java index 4e22f634cd..faebd12273 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPlugin.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPlugin.java @@ -27,8 +27,8 @@ import org.apache.tez.runtime.api.events.InputReadErrorEvent; /** - * This interface defines the routing of the event between tasks of producer and - * consumer vertices. The routing is bi-directional. Users can customize the + * This interface defines the routing of the event between tasks of producer and + * consumer vertices. The routing is bi-directional. Users can customize the * routing by providing an implementation of this interface. */ @Public @@ -41,7 +41,7 @@ public abstract class EdgeManagerPlugin { * Create an instance of the EdgeManagerPlugin. Classes extending this to * create a EdgeManagerPlugin, must provide the same constructor so that Tez * can create an instance of the class at runtime. - * + * * @param context * the context within which this EdgeManagerPlugin will run. Includes * information like configuration which the user may have specified @@ -61,10 +61,10 @@ public EdgeManagerPlugin(EdgeManagerPluginContext context) { * @throws Exception */ public abstract void initialize() throws Exception; - + /** * Get the number of physical inputs on the destination task - * @param destinationTaskIndex Index of destination task for which number of + * @param destinationTaskIndex Index of destination task for which number of * inputs is needed * @return Number of physical inputs on the destination task * @throws Exception @@ -73,20 +73,20 @@ public EdgeManagerPlugin(EdgeManagerPluginContext context) { /** * Get the number of physical outputs on the source task - * @param sourceTaskIndex Index of the source task for which number of outputs + * @param sourceTaskIndex Index of the source task for which number of outputs * is needed * @return Number of physical outputs on the source task * @throws Exception */ public abstract int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) throws Exception; - + /** * Return the routing information to inform consumers about the source task * output that is now available. The return map has the routing information. * The event will be routed to every destination task index in the key of the * map. Every physical input in the value for that task key will receive the * input. - * + * * @param event * Data movement event that contains the output information * @param sourceTaskIndex @@ -110,7 +110,7 @@ public abstract void routeDataMovementEventToDestination(DataMovementEvent event * for that task key will receive the failure notification. This method will * be called once for every source task failure and information for all * affected destinations must be provided in that invocation. - * + * * @param sourceTaskIndex * Source task * @param destinationTaskAndInputIndices @@ -126,16 +126,16 @@ public abstract void routeInputSourceTaskFailedEventToDestination(int sourceTask * @throws Exception */ public abstract int getNumDestinationConsumerTasks(int sourceTaskIndex) throws Exception; - + /** * Return the source task index to which to send the input error event - * + * * @param event * Input read error event. Has more information about the error * @param destinationTaskIndex * Destination task that reported the error * @param destinationFailedInputIndex - * Index of the physical input on the destination task that reported + * Index of the physical input on the destination task that reported * the error * @return Index of the source task that created the unavailable input * @throws Exception diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginContext.java b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginContext.java index ef6925b4ee..705f2023ae 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginContext.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginContext.java @@ -29,19 +29,19 @@ @Public @Unstable public interface EdgeManagerPluginContext { - + /** * Returns the payload specified by the user for the edge. * @return the {@link org.apache.tez.dag.api.UserPayload} specified by the user */ public UserPayload getUserPayload(); - + /** - * Returns the source vertex name + * Returns the source vertex name * @return the source vertex name */ public String getSourceVertexName(); - + /** * Returns the destination vertex name * @return the destination vertex name diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java b/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java index 9eb9a1a935..13934c85e0 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java @@ -22,9 +22,9 @@ import org.apache.tez.runtime.api.MergedLogicalInput; /** - * A composite edge that represents a common connection between a - * group of input vertices and a single output vertex. This can be - * used to perform e.g. an efficient union of the data produced by + * A composite edge that represents a common connection between a + * group of input vertices and a single output vertex. This can be + * used to perform e.g. an efficient union of the data produced by * the input vertices. The output vertex tasks see a unified/merged * view of the data from all the input vertices. */ @@ -88,7 +88,7 @@ public Vertex getOutputVertex() { public EdgeProperty getEdgeProperty() { return edgeProperty; } - + InputDescriptor getMergedInput() { return mergedInput; } @@ -100,7 +100,7 @@ InputDescriptor getMergedInput() { public String getId() { return String.valueOf(this.hashCode()); } - + @Override public String toString() { return inputVertexGroup + " -> " + outputVertex + " (" + edgeProperty + ")"; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java index 58b6ce4630..034270d453 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java @@ -32,7 +32,7 @@ public class OutputDescriptor extends EntityDescriptor { public OutputDescriptor() { super(); } - + private OutputDescriptor(String outputClassName) { super(outputClassName); } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/PreWarmVertex.java b/tez-api/src/main/java/org/apache/tez/dag/api/PreWarmVertex.java index 62979b5088..8522c510fd 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/PreWarmVertex.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/PreWarmVertex.java @@ -58,13 +58,13 @@ private PreWarmVertex(String vertexName, ProcessorDescriptor processorDescriptor Resource taskResource) { super(vertexName, processorDescriptor, parallelism, taskResource); } - + private PreWarmVertex(String vertexName, int parallelism, Resource taskResource) { this(vertexName, ProcessorDescriptor.create( "org.apache.tez.runtime.library.processor.PreWarmProcessor"), parallelism, taskResource); } - + /** * Create a config builder for the @link {@link PreWarmVertex}. This may be used to construct the * pre-warm vertex more flexibly. @@ -128,31 +128,31 @@ public static class PreWarmVertexConfigBuilder { ProcessorDescriptor proc; Resource resource; Configuration conf; - + PreWarmVertexConfigBuilder(Configuration conf) { this.conf = conf; } - + public PreWarmVertexConfigBuilder setName(String name) { this.name = name; return this; } - + public PreWarmVertexConfigBuilder setProcessorDescriptor(ProcessorDescriptor proc) { this.proc = proc; return this; } - + public PreWarmVertexConfigBuilder setResource(Resource resource) { this.resource = resource; return this; } - + public PreWarmVertexConfigBuilder setParallelism(int parallelism) { this.parallelism = parallelism; return this; } - + public PreWarmVertex build() { if (name == null) { name = "_PreWarm_"; @@ -160,7 +160,7 @@ public PreWarmVertex build() { if (parallelism == 0) { parallelism = conf.getInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, -1); if (parallelism == -1) { - throw new TezUncheckedException("Prewarm parallelism must be set or specified in conf via " + throw new TezUncheckedException("Prewarm parallelism must be set or specified in conf via " + TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS); } } @@ -168,7 +168,7 @@ public PreWarmVertex build() { proc = ProcessorDescriptor.create("org.apache.tez.runtime.library.processor.PreWarmProcessor"); } - + return create(name, proc, parallelism, resource); } } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java index 7f088d8caf..8e00a7cf7d 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java @@ -32,7 +32,7 @@ public class ProcessorDescriptor extends EntityDescriptor { public ProcessorDescriptor() { super(); } - + private ProcessorDescriptor(String processorClassName) { super(processorClassName); } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java b/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java index 953a78da6e..f6a8116507 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java @@ -32,7 +32,7 @@ public RootInputLeafOutput(String name, T ioDescriptor, S controllerDescriptor) this.ioDescriptor = ioDescriptor; this.controllerDescriptor = controllerDescriptor; } - + public String getName() { return this.name; } @@ -40,7 +40,7 @@ public String getName() { public T getIODescriptor() { return this.ioDescriptor; } - + public S getControllerDescriptor() { return this.controllerDescriptor; } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Scope.java b/tez-api/src/main/java/org/apache/tez/dag/api/Scope.java index f638e092a5..4288db9189 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/Scope.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/Scope.java @@ -22,7 +22,7 @@ @Private public enum Scope { // DO NOT CHANGE THE ORDER - AM, // can only been set at AM level + AM, // can only been set at AM level DAG, // can been set at AM/DAG level VERTEX, // can been set at AM/DAG/VERTEX level CLIENT, // Client scope - only applicable on client diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java b/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java index ae35d80bb2..0bec4d6470 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java @@ -29,7 +29,7 @@ /** * Describes the placements hints for tasks. - * The system will make a best-effort attempt to run the tasks + * The system will make a best-effort attempt to run the tasks * close to the specified locations. */ @Public @@ -63,7 +63,7 @@ public String toString() { private Set hosts; // Rack names if any to be used private Set racks; - + private TaskBasedLocationAffinity affinitizedTask; private TaskLocationHint(String vertexName, int taskIndex) { @@ -86,7 +86,7 @@ private TaskLocationHint(Set hosts, Set racks) { } /** - * Provide a location hint that affinitizes to the given task in the given vertex. Tez will try + * Provide a location hint that affinitizes to the given task in the given vertex. Tez will try * to run in the same container as the given task or node local to it. Locality may degrade to * rack local or further depending on cluster resource allocations.
* This is expected to be used only during dynamic optimizations via {@link VertexManagerPlugin}s diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezUncheckedException.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezUncheckedException.java index af798b3484..76d01a8abc 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezUncheckedException.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezUncheckedException.java @@ -27,7 +27,7 @@ public class TezUncheckedException extends RuntimeException { private static final long serialVersionUID = -4956339297375386184L; - + public TezUncheckedException(Throwable cause) { super(cause); } public TezUncheckedException(String message) { super(message); } public TezUncheckedException(String message, Throwable cause) { diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java index 309db0a6d9..b02f83c9b5 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java @@ -41,9 +41,9 @@ import com.google.common.collect.Maps; /** - * Defines a vertex in the DAG. It represents the application logic that - * processes and transforms the input data to create the output data. The - * vertex represents the template from which tasks are created to execute + * Defines a vertex in the DAG. It represents the application logic that + * processes and transforms the input data to create the output data. The + * vertex represents the template from which tasks are created to execute * the application in parallel across a distributed execution environment. */ @Public @@ -72,7 +72,7 @@ public class Vertex { private final Map groupInputs = Maps.newHashMap(); private final List dataSources = Lists.newLinkedList(); private final List dataSinks = Lists.newLinkedList(); - + private String taskLaunchCmdOpts = ""; @InterfaceAudience.Private @@ -86,12 +86,12 @@ public class Vertex { private Vertex(String vertexName, ProcessorDescriptor processorDescriptor, int parallelism) { this(vertexName, processorDescriptor, parallelism, null, true); } - + private Vertex(String vertexName, ProcessorDescriptor processorDescriptor) { this(vertexName, processorDescriptor, -1); } - + private Vertex(String vertexName, ProcessorDescriptor processorDescriptor, int parallelism, @@ -201,15 +201,15 @@ public ProcessorDescriptor getProcessorDescriptor() { } /** - * Get the specified number of tasks specified to run in this vertex. It may - * be -1 if the parallelism is defined at runtime. Parallelism may change at + * Get the specified number of tasks specified to run in this vertex. It may + * be -1 if the parallelism is defined at runtime. Parallelism may change at * runtime * @return vertex parallelism */ public int getParallelism() { return parallelism; } - + /** * Set the number of tasks for this vertex * @param parallelism Parallelism for this vertex @@ -227,7 +227,7 @@ public Resource getTaskResource() { } /** - * Specify location hints for the tasks of this vertex. Hints must be specified + * Specify location hints for the tasks of this vertex. Hints must be specified * for all tasks as defined by the parallelism * @param locationHint list of locations for each task in the vertex * @return this Vertex @@ -237,7 +237,7 @@ public Vertex setLocationHint(VertexLocationHint locationHint) { if (locations == null) { return this; } - Preconditions.checkArgument((locations.size() == parallelism), + Preconditions.checkArgument((locations.size() == parallelism), "Locations array length must match the parallelism set for the vertex"); this.locationHint = locationHint; return this; @@ -297,7 +297,7 @@ public Map getConf() { } /** - * Set the command opts for tasks of this vertex. This method should be used + * Set the command opts for tasks of this vertex. This method should be used * when different vertices have different opts. Else, set the launch opts for ' * all vertices via Tezconfiguration#TEZ_TASK_LAUNCH_CMD_OPTS * @param cmdOpts @@ -307,23 +307,23 @@ public Vertex setTaskLaunchCmdOpts(String cmdOpts){ this.taskLaunchCmdOpts = cmdOpts; return this; } - + /** * Specifies an external data source for a Vertex. This is meant to be used * when a Vertex reads Input directly from an external source

- * + * * For vertices which read data generated by another vertex - use the * {@link DAG addEdge} method. - * + * * If a vertex needs to use data generated by another vertex in the DAG and * also from an external source, a combination of this API and the DAG.addEdge * API can be used.

- * + * * Note: If more than one RootInput exists on a vertex, which generates events * which need to be routed, or generates information to set parallelism, a * custom vertex manager should be setup to handle this. Not using a custom * vertex manager for such a scenario will lead to a runtime failure. - * + * * @param inputName * the name of the input. This will be used when accessing the input * in the {@link LogicalIOProcessor} @@ -347,14 +347,14 @@ public Vertex addDataSource(String inputName, DataSourceDescriptor dataSourceDes /** * Specifies an external data sink for a Vertex. This is meant to be used when * a Vertex writes Output directly to an external destination.

- * + * * If an output of the vertex is meant to be consumed by another Vertex in the * DAG - use the {@link DAG addEdge} method. - * + * * If a vertex needs generate data to an external source as well as for * another Vertex in the DAG, a combination of this API and the DAG.addEdge * API can be used. - * + * * @param outputName * the name of the output. This will be used when accessing the * output in the {@link LogicalIOProcessor} @@ -388,7 +388,7 @@ Vertex addAdditionalDataSink(RootInputLeafOutput getGroupInputs() { return groupInputs; } - + void addGroupInput(String groupName, GroupInfo groupInputInfo) { if (groupInputs.put(groupName, groupInputInfo) != null) { throw new IllegalStateException( - "Vertex: " + getName() + + "Vertex: " + getName() + " already has group input with name:" + groupName); } } @@ -633,7 +633,7 @@ void addOutputVertex(Vertex outputVertex, Edge edge) { outputVertices.add(outputVertex); outputEdges.add(edge); } - + /** * Get the input vertices for this vertex * @return List of input vertices @@ -662,7 +662,7 @@ void setTaskResource(Resource resource) { public List getDataSources() { return dataSources; } - + @Private public List getDataSinks() { return dataSinks; @@ -680,7 +680,7 @@ List getInputEdges() { List getOutputEdges() { return outputEdges; } - + List> getInputs() { return Lists.newArrayList(additionalInputs.values()); } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java index 8041b34a9c..15060fcc1e 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java @@ -28,13 +28,13 @@ import com.google.common.collect.Sets; /** - * Represents a virtual collection of vertices whose members can be treated as a single + * Represents a virtual collection of vertices whose members can be treated as a single * named collection for graph operations. Only the following connections are valid. - * A VertexGroup can be connected as an input to a consumer Vertex. The tasks of - * the destination vertex see a single input named after the VertexGroup instead - * multiple inputs from the members of the VertexGroup. + * A VertexGroup can be connected as an input to a consumer Vertex. The tasks of + * the destination vertex see a single input named after the VertexGroup instead + * multiple inputs from the members of the VertexGroup. * An output can be added to a VertexGroup. - * All outgoing edges & outputs of a VertexGroup are automatically transferred to the + * All outgoing edges & outputs of a VertexGroup are automatically transferred to the * member vertices of the VertexGroup. * A VertexGroup is not part of the final DAG. */ @@ -47,7 +47,7 @@ static class GroupInfo { Set outputs = new HashSet(); // destination vertex name to merged input map Map edgeMergedInputs = Maps.newHashMap(); - + GroupInfo(String groupName, Vertex... vertices) { this.groupName = groupName; members = Sets.newHashSetWithExpectedSize(vertices.length); @@ -65,9 +65,9 @@ Set getOutputs() { return outputs; } } - + GroupInfo groupInfo; - + /** * Create an object representing a group of vertices * @param groupName name of the group @@ -86,26 +86,26 @@ Set getOutputs() { public String getGroupName() { return groupInfo.groupName; } - + /** * Add an common data sink to the group of vertices. * Refer to {@link Vertex#addDataSink(String, DataSinkDescriptor)} * @return this object for further chained method calls */ public VertexGroup addDataSink(String outputName, DataSinkDescriptor dataSinkDescriptor) { - RootInputLeafOutput leafOutput = + RootInputLeafOutput leafOutput = new RootInputLeafOutput(outputName, dataSinkDescriptor.getOutputDescriptor(), dataSinkDescriptor.getOutputCommitterDescriptor()); this.groupInfo.outputs.add(outputName); - + // also add output to its members for (Vertex member : getMembers()) { member.addAdditionalDataSink(leafOutput); } - + return this; } - + @Override public String toString() { return "[ VertexGroup: " + groupInfo.getGroupName() + "]" + ", members=" + groupInfo.members; @@ -114,15 +114,15 @@ public String toString() { GroupInfo getGroupInfo() { return groupInfo; } - + Set getMembers() { return groupInfo.members; } - + void addOutputVertex(Vertex outputVertex, GroupInputEdge edge) { this.groupInfo.edgeMergedInputs.put(outputVertex.getName(), edge.getMergedInput()); } - + @Override public int hashCode() { final int prime = 31; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java index c020dd230b..b36b41d314 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java @@ -25,7 +25,7 @@ /** * Describes the placements hints for tasks in a vertex. - * The system will make a best-effort attempt to run the tasks + * The system will make a best-effort attempt to run the tasks * close to the specified locations. */ @Public @@ -33,7 +33,7 @@ public class VertexLocationHint { private final List taskLocationHints; - + private VertexLocationHint(List taskLocationHints) { if (taskLocationHints != null) { this.taskLocationHints = Collections.unmodifiableList(taskLocationHints); diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java index b66a66ac55..0a1ac5d421 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java @@ -31,7 +31,7 @@ import org.apache.tez.runtime.api.events.VertexManagerEvent; /** - * Interface to plugin user logic into the VertexManager to implement runtime + * Interface to plugin user logic into the VertexManager to implement runtime * scheduling optimizations and graph reconfiguration. * The plugin will be notified of interesting events in the vertex execution life * cycle and can respond to them by via the context object @@ -46,7 +46,7 @@ public abstract class VertexManagerPlugin { * Crete an instance of the VertexManagerPlugin. Classes extending this to * create a VertexManagerPlugin, must provide the same constructor so that Tez * can create an instance of the class at runtime. - * + * * @param context * vertex manager plugin context which can be used to access the * payload, vertex properties, etc @@ -56,7 +56,7 @@ public VertexManagerPlugin(VertexManagerPluginContext context) { } /** - * Initialize the plugin. Called when the vertex is initializing. This happens + * Initialize the plugin. Called when the vertex is initializing. This happens * after all source vertices and inputs have initialized * @throws Exception */ @@ -91,10 +91,10 @@ public void onVertexStarted(List completions) throws Exce } onVertexStarted(completionsMap); } - + @Deprecated /** - * This has been replaced by + * This has been replaced by * {@link VertexManagerPlugin#onSourceTaskCompleted(TaskAttemptIdentifier)} * Notification of a source vertex completion. * @param srcVertexName @@ -111,7 +111,7 @@ public void onSourceTaskCompleted(String srcVertexName, Integer taskId) throws E * @throws Exception */ public void onSourceTaskCompleted(TaskAttemptIdentifier attempt) throws Exception { - onSourceTaskCompleted(attempt.getTaskIdentifier().getVertexIdentifier().getName(), + onSourceTaskCompleted(attempt.getTaskIdentifier().getVertexIdentifier().getName(), attempt.getTaskIdentifier().getIdentifier()); } @@ -141,7 +141,7 @@ public abstract void onRootVertexInitialized(String inputName, public final VertexManagerPluginContext getContext() { return this.context; } - + /** * Receive notifications on vertex state changes. *

@@ -150,9 +150,9 @@ public final VertexManagerPluginContext getContext() { * . Notifications will be received for all registered state changes, and not * just for the latest state update. They will be in order in which the state * change occurred. - *


This method may be invoked concurrently with {@link #onVertexStarted(Map)} etc. and + *


This method may be invoked concurrently with {@link #onVertexStarted(Map)} etc. and * multi-threading/concurrency implications must be considered. - * + * * @param stateUpdate * an event indicating the name of the vertex, and it's updated * state. Additional information may be available for specific @@ -160,5 +160,5 @@ public final VertexManagerPluginContext getContext() { * {@link org.apache.tez.dag.api.event.VertexStateUpdate} */ public void onVertexStateUpdated(VertexStateUpdate stateUpdate) throws Exception { - } + } } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java index 564115bfea..814bcc10f0 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java @@ -42,30 +42,30 @@ @Unstable @Public public interface VertexManagerPluginContext { - + public class ScheduleTaskRequest { int taskIndex; TaskLocationHint locationHint; public static ScheduleTaskRequest create(int taskIndex, @Nullable TaskLocationHint locationHint) { - return new ScheduleTaskRequest(taskIndex, locationHint); + return new ScheduleTaskRequest(taskIndex, locationHint); } - + private ScheduleTaskRequest(int taskIndex, @Nullable TaskLocationHint locationHint) { Preconditions.checkState(taskIndex >= 0); this.taskIndex = taskIndex; this.locationHint = locationHint; } - + public int getTaskIndex() { return taskIndex; } - + public TaskLocationHint getTaskLocationHint() { return locationHint; - } + } } - + @Deprecated public class TaskWithLocationHint { Integer taskIndex; @@ -74,30 +74,30 @@ public TaskWithLocationHint(Integer taskIndex, @Nullable TaskLocationHint locati this.taskIndex = Objects.requireNonNull(taskIndex); this.locationHint = locationHint; } - + public Integer getTaskIndex() { return taskIndex; } - + public TaskLocationHint getTaskLocationHint() { return locationHint; } } /** - * Get the edge properties on the input edges of this vertex. The input edge + * Get the edge properties on the input edges of this vertex. The input edge * is represented by the source vertex name * @return Map of source vertex name and edge property */ public Map getInputVertexEdgeProperties(); - + /** - * Get the edge properties on the output edges of this vertex. The output edge + * Get the edge properties on the output edges of this vertex. The output edge * is represented by the destination vertex name * @return Map of destination vertex name and edge property */ public Map getOutputVertexEdgeProperties(); - + /** * Get a {@link VertexStatistics} object to find out execution statistics * about the given {@link Vertex}. @@ -115,45 +115,45 @@ public TaskLocationHint getTaskLocationHint() { * @return Vertex name */ public String getVertexName(); - + /** * Get the payload set for the plugin * @return user payload */ public UserPayload getUserPayload(); - + /** * Get the number of tasks in the given vertex * @param vertexName * @return Total number of tasks in this vertex */ public int getVertexNumTasks(String vertexName); - + /** * Get the resource allocated to a task of this vertex * @return Resource */ Resource getVertexTaskResource(); - + /** - * Get the total resource allocated to this vertex. If the DAG is running in + * Get the total resource allocated to this vertex. If the DAG is running in * a busy cluster then it may have no resources available dedicated to it. The * DAG may divide its available resource among member vertices. * @return Resource */ Resource getTotalAvailableResource(); - + /** * Get the number of nodes in the cluster * @return Number of nodes */ int getNumClusterNodes(); - + /** * Set the new parallelism (number of tasks) of this vertex, * Map of source (input) vertices and edge managers to change the event routing * between the source tasks and the new destination tasks and the number of physical inputs for root inputs. - * This API can change the parallelism only once. Subsequent attempts will be + * This API can change the parallelism only once. Subsequent attempts will be * disallowed * @param parallelism New number of tasks in the vertex * @param locationHint the placement policy for tasks. @@ -184,7 +184,7 @@ public void setVertexParallelism(int parallelism, * times until any of the tasks of the vertex have been scheduled (by invoking * {@link #scheduleTasks(List)}. If needed, the original source edge * properties may be obtained via {@link #getInputVertexEdgeProperties()} - * + * * @param parallelism * New number of tasks in the vertex * @param locationHint @@ -204,7 +204,7 @@ public void reconfigureVertex(int parallelism, @Nullable VertexLocationHint locationHint, @Nullable Map sourceEdgeProperties, @Nullable Map rootInputSpecUpdate); - + /** * API to reconfigure a {@link Vertex} by changing its task parallelism. Task * parallelism is often accompanied by changing the {@link EdgeProperty} of @@ -214,7 +214,7 @@ public void reconfigureVertex(int parallelism, * times until any of the tasks of the vertex have been scheduled (by invoking * {@link #scheduleTasks(List)}. If needed, the original source edge * properties may be obtained via {@link #getInputVertexEdgeProperties()} - * + * * @param parallelism * New number of tasks in the vertex * @param locationHint @@ -222,7 +222,7 @@ public void reconfigureVertex(int parallelism, * {@link VertexLocationHint}s * @param sourceEdgeProperties * Map with Key=name of {@link Edge} to be updated and Value= - * {@link EdgeProperty}. The name of the Edge will be the + * {@link EdgeProperty}. The name of the Edge will be the * corresponding source vertex name. */ public void reconfigureVertex(int parallelism, @@ -256,10 +256,10 @@ public void reconfigureVertex(@Nullable Map rootInputSp /** * Allows a VertexManagerPlugin to assign Events for Root Inputs - * + * * For regular Event Routing changes - the EdgeManager should be configured * via the setVertexParallelism method - * + * * @param inputName * The input name associated with the event * @param events @@ -279,7 +279,7 @@ public void reconfigureVertex(@Nullable Map rootInputSp * @param taskId id of a task of managed vertex */ public void sendEventToProcessor(Collection events, int taskId); - + @Deprecated /** * Replaced by {@link #scheduleTasks(List)} @@ -287,7 +287,7 @@ public void reconfigureVertex(@Nullable Map rootInputSp * @param tasks Indices of the tasks to be started */ public void scheduleVertexTasks(List tasks); - + /** * Notify the vertex to schedule the given tasks * @param tasks Identifier and metadata for the tasks to schedule @@ -295,7 +295,7 @@ public void reconfigureVertex(@Nullable Map rootInputSp public void scheduleTasks(List tasks); /** - * Get the names of the non-vertex inputs of this vertex. These are primary + * Get the names of the non-vertex inputs of this vertex. These are primary * sources of data. * @return Names of inputs to this vertex. Maybe null if there are no inputs */ @@ -304,7 +304,7 @@ public void reconfigureVertex(@Nullable Map rootInputSp /** * Set the placement hint for tasks in this vertex - * + * * @param locationHint */ public void setVertexLocationHint(VertexLocationHint locationHint); @@ -313,7 +313,7 @@ public void reconfigureVertex(@Nullable Map rootInputSp * @return DAG Attempt number */ public int getDAGAttemptNumber(); - + /** * Register to get notifications on updates to the specified vertex. Notifications will be sent * via {@link VertexManagerPlugin#onVertexStateUpdated(org.apache.tez.dag.api.event.VertexStateUpdate)} @@ -324,7 +324,7 @@ public void reconfigureVertex(@Nullable Map rootInputSp * @param stateSet the set of states for which notifications are required. null implies all */ void registerForVertexStateUpdates(String vertexName, @Nullable Set stateSet); - + /** * Optional API. No need to call this when the vertex is not fully defined to * start with. E.g. vertex parallelism is not defined, or edges are not @@ -343,13 +343,13 @@ public void reconfigureVertex(@Nullable Map rootInputSp * reconfiguration. */ public void vertexReconfigurationPlanned(); - + /** - * Optional API. This needs to be called only if {@link #vertexReconfigurationPlanned()} has been + * Optional API. This needs to be called only if {@link #vertexReconfigurationPlanned()} has been * invoked. This must be called after {@link #vertexReconfigurationPlanned()} is called. */ public void doneReconfiguringVertex(); - + /** * Optional API. This API can be invoked to declare that the * {@link VertexManagerPlugin} is done with its work. After this the system diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java index 687f517d99..87d4938caf 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java @@ -22,7 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; /** - * Describes the {@link VertexManagerPlugin} + * Describes the {@link VertexManagerPlugin} */ @Public public class VertexManagerPluginDescriptor extends EntityDescriptor { @@ -31,7 +31,7 @@ public class VertexManagerPluginDescriptor extends EntityDescriptorEnumSet.of(StatusGetOpts.GET_COUNTERS) * @return Final DAG Status * @throws IOException * @throws TezException - * @throws InterruptedException + * @throws InterruptedException */ public abstract DAGStatus waitForCompletionWithStatusUpdates(@Nullable Set statusGetOpts) throws IOException, TezException, InterruptedException; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java index 8346d53da7..ead1134097 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java @@ -104,24 +104,24 @@ public abstract VertexStatus getVertexStatus(String vertexName, /** * Wait for DAG to complete without printing any vertex statuses - * + * * @return Final DAG Status * @throws IOException * @throws TezException - * @throws InterruptedException + * @throws InterruptedException */ public abstract DAGStatus waitForCompletion() throws IOException, TezException, InterruptedException; /** * Wait for DAG to complete and periodically print *all* vertices' status. - * + * * @param statusGetOpts * : status get options. For example, to get counter pass * EnumSet.of(StatusGetOpts.GET_COUNTERS) * @return Final DAG Status * @throws IOException * @throws TezException - * @throws InterruptedException + * @throws InterruptedException */ public abstract DAGStatus waitForCompletionWithStatusUpdates(@Nullable Set statusGetOpts) throws IOException, TezException, InterruptedException; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java index ac42da04cc..cf562a4f46 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java @@ -174,8 +174,8 @@ public boolean equals(Object obj) { && getDiagnostics().equals(other.getDiagnostics()) && getDAGProgress().equals(other.getDAGProgress()) && getVertexProgress().equals(other.getVertexProgress()) - && - ((getDAGCounters() == null && other.getDAGCounters() == null) + && + ((getDAGCounters() == null && other.getDAGCounters() == null) || getDAGCounters().equals(other.getDAGCounters())); } return false; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java index 656838dc5a..82fdadfe8a 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java @@ -28,13 +28,13 @@ @Public @Evolving public class Progress { - + ProgressProtoOrBuilder proxy = null; - + Progress(ProgressProtoOrBuilder proxy) { this.proxy = proxy; } - + public int getTotalTaskCount() { return proxy.getTotalTaskCount(); } @@ -71,7 +71,7 @@ public int getRejectedTaskAttemptCount() { public boolean equals(Object obj) { if (obj instanceof Progress){ Progress other = (Progress)obj; - return getTotalTaskCount() == other.getTotalTaskCount() + return getTotalTaskCount() == other.getTotalTaskCount() && getSucceededTaskCount() == other.getSucceededTaskCount() && getRunningTaskCount() == other.getRunningTaskCount() && getFailedTaskCount() == other.getFailedTaskCount() @@ -114,9 +114,9 @@ public String toString() { sb.append(getSucceededTaskCount()); sb.append(" Running: "); sb.append(getRunningTaskCount()); - sb.append(" Failed: "); + sb.append(" Failed: "); sb.append(getFailedTaskCount()); - sb.append(" Killed: "); + sb.append(" Killed: "); sb.append(getKilledTaskCount()); if (getFailedTaskAttemptCount() > 0) { sb.append(" FailedTaskAttempts: "); diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java index 42edc7f096..4427cfd5fe 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java @@ -118,7 +118,7 @@ public TezCounters getVertexCounters() { countersInitialized.set(true); return vertexCounters; } - + @Override public boolean equals(Object obj) { if (obj instanceof VertexStatus){ @@ -126,8 +126,8 @@ public boolean equals(Object obj) { return getState().equals(other.getState()) && getDiagnostics().equals(other.getDiagnostics()) && getProgress().equals(other.getProgress()) - && - ((getVertexCounters() == null && other.getVertexCounters() == null) + && + ((getVertexCounters() == null && other.getVertexCounters() == null) || getVertexCounters().equals(other.getVertexCounters())); } return false; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java index 9d7ebeefee..252e5752d2 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java @@ -26,7 +26,7 @@ @ProtocolInfo( protocolName = "org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPB", protocolVersion = 1) -public interface DAGClientAMProtocolBlockingPB +public interface DAGClientAMProtocolBlockingPB extends DAGClientAMProtocol.BlockingInterface { } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalIOProcessor.java b/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalIOProcessor.java index 5a4cbe8113..b3ccaa8c9e 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalIOProcessor.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalIOProcessor.java @@ -21,7 +21,7 @@ /** * Abstract representation of the interface {@link LogicalIOProcessor}. - * Implements the base logic of some methods into this class and is expected + * Implements the base logic of some methods into this class and is expected * to be the base class that is derived to implement the user {@link Processor} * */ diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/DagIdentifier.java b/tez-api/src/main/java/org/apache/tez/runtime/api/DagIdentifier.java index dd63b4cfa1..eadcb86e17 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/DagIdentifier.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/DagIdentifier.java @@ -21,6 +21,6 @@ public interface DagIdentifier { public String getName(); - + public int getIdentifier(); } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputContext.java index 6eac2dfc95..ade9f7125e 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputContext.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputContext.java @@ -38,21 +38,21 @@ public interface InputContext extends TaskContext { * @return the convenient string */ String getInputOutputVertexNames(); - + /** - * Get the index of the input in the set of all inputs for the task. The + * Get the index of the input in the set of all inputs for the task. The * index will be consistent and valid only among the tasks of this vertex. * @return index */ public int getInputIndex(); - + /** * Inform the framework that the specific Input is ready for consumption. - * + * * This method can be invoked multiple times. */ public void inputIsReady(); - + /** * Get an {@link InputStatisticsReporter} for this {@link Input} that can * be used to report statistics like data size diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputFrameworkInterface.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputFrameworkInterface.java index 7f0a1d995f..d0c4227714 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputFrameworkInterface.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputFrameworkInterface.java @@ -41,7 +41,7 @@ * This typically means that the Processor will not block when reading from the * corresponding Input. This is done via {@link InputContext}.inputIsReady. * Inputs choose the policy on when they are ready. - * + * * Input implementations are expected to derive from {@link AbstractLogicalInput} */ @Public diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java index ec87fe1e39..24013f2a9b 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java @@ -56,7 +56,7 @@ public InputInitializer(InputInitializerContext initializerContext) { * processing the event being done via this method. The moment this method * returns a list of events, input initialization is considered to be * complete. - * + * * @return a list of events which are eventually routed to a * {@link org.apache.tez.dag.api.VertexManagerPlugin} for routing * @throws Exception @@ -79,7 +79,7 @@ public abstract void handleInputInitializerEvent(List eve /** * Return ahe {@link org.apache.tez.runtime.api.InputInitializerContext} * for this specific instance of the Initializer. - * + * * @return the {@link org.apache.tez.runtime.api.InputInitializerContext} * for the initializer */ @@ -97,7 +97,7 @@ public final InputInitializerContext getContext() { * * Extensive processing should not be performed via this method call. Instead this should just be * used as a notification mechanism to the main initialization, which is via the initialize method. - *
This method may be invoked concurrently with {@link #initialize()} etc. and + *
This method may be invoked concurrently with {@link #initialize()} etc. and * multi-threading/concurrency implications must be considered. * @param stateUpdate an event indicating the name of the vertex, and it's updated state. * Additional information may be available for specific events, Look at the diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java index 680492266c..46786812b4 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java @@ -43,7 +43,7 @@ public interface InputInitializerContext { * @return Application id */ ApplicationId getApplicationId(); - + /** * Get the name of the DAG * @return DAG name @@ -67,20 +67,20 @@ public interface InputInitializerContext { * @return User payload */ UserPayload getInputUserPayload(); - + /** * Get the user payload for the initializer * @return User payload */ UserPayload getUserPayload(); - + /** * Get the number of tasks in this vertex. Maybe -1 if the vertex has not been * initialized with a pre-determined number of tasks. * @return number of tasks */ int getNumTasks(); - + /** * Get the resource allocated to a task of this vertex * @return Resource @@ -93,13 +93,13 @@ public interface InputInitializerContext { int getVertexId(); /** - * Get the total resource allocated to this vertex. If the DAG is running in + * Get the total resource allocated to this vertex. If the DAG is running in * a busy cluster then it may have no resources available dedicated to it. The * DAG may divide its resources among member vertices. * @return Resource */ Resource getTotalAvailableResource(); - + /** * Get the number of nodes in the cluster * @return Number of nodes diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputSpecUpdate.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputSpecUpdate.java index c6bbba7090..736de70a3f 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputSpecUpdate.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputSpecUpdate.java @@ -31,7 +31,7 @@ * Update Input specs for Inputs running in a task. Allows setting the number of physical * inputs for all work units if they have the same number of physical inputs, or individual * numPhysicalInputs for each work unit. - * + * */ @Unstable @Public @@ -41,11 +41,11 @@ public class InputSpecUpdate { private final List numPhysicalInputs; private final static InputSpecUpdate DEFAULT_SINGLE_PHYSICAL_INPUT_SPEC = createAllTaskInputSpecUpdate(1); - + /** * Create an update instance where all work units (typically represented by * {@link InputDataInformationEvent}) will have the same number of physical inputs. - * + * * @param numPhysicalInputs * the number of physical inputs for all work units which will use the LogicalInput * @return a new InputSpecUpdate instance given the parameters @@ -57,7 +57,7 @@ public static InputSpecUpdate createAllTaskInputSpecUpdate(int numPhysicalInputs /** * Create an update instance where all work units (typically represented by * {@link InputDataInformationEvent}) will have the same number of physical inputs. - * + * * @param perWorkUnitNumPhysicalInputs * A list containing one entry per work unit. The order in the list corresponds to task * index or equivalently the order of {@link InputDataInformationEvent}s being sent. @@ -67,7 +67,7 @@ public static InputSpecUpdate createPerTaskInputSpecUpdate( List perWorkUnitNumPhysicalInputs) { return new InputSpecUpdate(perWorkUnitNumPhysicalInputs); } - + public static InputSpecUpdate getDefaultSinglePhysicalInputSpecUpdate() { return DEFAULT_SINGLE_PHYSICAL_INPUT_SPEC; } @@ -90,13 +90,13 @@ public int getNumPhysicalInputsForWorkUnit(int index) { return numPhysicalInputs.get(index); } } - + @Private /* Used for recovery serialization */ public boolean isForAllWorkUnits() { return this.forAllWorkUnits; } - + @Private /* Used for recovery serialization */ public List getAllNumPhysicalInputs() { diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatistics.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatistics.java index 1066dbb861..1cc176027d 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatistics.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatistics.java @@ -30,20 +30,20 @@ @Public @Evolving public interface InputStatistics { - + /** * Returns the data size associated with this logical input *
It is the size of the data read from this input by the vertex. * @return Data size in bytes */ public long getDataSize(); - + /** * Get the numbers of items processed. These could be key-value pairs, table * records etc. - * + * * @return Number of items processed */ public long getItemsProcessed(); - + } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatisticsReporter.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatisticsReporter.java index a85d25b9bd..7c6b163e8b 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatisticsReporter.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatisticsReporter.java @@ -37,11 +37,11 @@ public interface InputStatisticsReporter { /** * Report the number of items processed. These could be key-value pairs, table * records etc. - * + * * @param items * number of items */ public void reportItemsProcessed(long items); - + } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessorFrameworkInterface.java b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessorFrameworkInterface.java index edae2f8b81..336e59725c 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessorFrameworkInterface.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessorFrameworkInterface.java @@ -32,7 +32,7 @@ public interface LogicalIOProcessorFrameworkInterface extends ProcessorFramework /** * Runs the {@link Processor} - * + * * @param inputs * a map of the source vertex name to {@link LogicalInput} - one per * incoming edge. diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInput.java b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInput.java index 9f10d0bef5..f47c92f786 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInput.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInput.java @@ -25,7 +25,7 @@ * edge. A {@link LogicalIOProcessor} sees a single Logical Input per incoming * edge. It provides a unified view that hides the details of the actual physical * inputs - * + * * User implementations are expected to derive from {@link AbstractLogicalInput} */ @Public diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutput.java b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutput.java index 29221895d6..01a8412127 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutput.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutput.java @@ -23,7 +23,7 @@ /** * An @link {@link Output} which handles all outgoing physical connections on an * edge. A {@link LogicalIOProcessor} sees a single Logical Output per outgoing - * edge. It's expected to hide the details of output partitioning and physical + * edge. It's expected to hide the details of output partitioning and physical * outputs from the {@link Processor} * Users are expected to derive from {@link AbstractLogicalOutput} */ diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/MemoryUpdateCallback.java b/tez-api/src/main/java/org/apache/tez/runtime/api/MemoryUpdateCallback.java index 1f304b380f..4eef0c16aa 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/MemoryUpdateCallback.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/MemoryUpdateCallback.java @@ -24,9 +24,9 @@ /** * This interface will be used by Tez to inform components about available * memory. This will typically be implemented by authors of {@link Input}s, - * {@link Output}s and potentially {@link Processor}s in order to request + * {@link Output}s and potentially {@link Processor}s in order to request * for appropriate memory limits for the respective components. - * + * */ @Unstable @Public diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/MergedInputContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/MergedInputContext.java index 65bb087c7a..58edac8b74 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/MergedInputContext.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/MergedInputContext.java @@ -33,19 +33,19 @@ public interface MergedInputContext { * @return {@link UserPayload} */ public UserPayload getUserPayload(); - + /** * Inform the framework that the specific Input is ready for consumption. - * + * * This method can be invoked multiple times. */ public void inputIsReady(); - + /** * Inform the framework that progress has been made */ public void notifyProgress(); - + /** * Get the work directories for the Input * @return an array of work dirs diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/MergedLogicalInput.java b/tez-api/src/main/java/org/apache/tez/runtime/api/MergedLogicalInput.java index d10954c62b..2179654085 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/MergedLogicalInput.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/MergedLogicalInput.java @@ -65,11 +65,11 @@ public MergedLogicalInput(MergedInputContext context, List inputs) { public final List getInputs() { return inputs; } - + public final MergedInputContext getContext() { return context; } - + @Override public final void start() throws Exception { if (!isStarted.getAndSet(true)) { diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/ObjectRegistry.java b/tez-api/src/main/java/org/apache/tez/runtime/api/ObjectRegistry.java index a280942225..0ef5eae029 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/ObjectRegistry.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/ObjectRegistry.java @@ -42,7 +42,7 @@ public interface ObjectRegistry { * as there is only one instance of an Object stored for a given key * irrespective of the life-cycle attached to the Object. The object may stay * in the cache while the Vertex (to which the task belongs) is running. - * + * * @param key * Key to identify the Object * @param value @@ -50,16 +50,16 @@ public interface ObjectRegistry { * @return Previous Object associated with the key attached if present else * null. Could return the same object if the object was associated * with the same key for a different life-cycle. - */ + */ public Object cacheForVertex(String key, Object value); - + /** * Insert or update object into the registry with DAG life-cycle. This will * remove an object associated with the same key with a different life-cycle * as there is only one instance of an Object stored for a given key * irrespective of the life-cycle attached to the Object. The object may stay * in the cache while the DAG (to which the task belongs) is running. - * + * * @param key * Key to identify the Object * @param value @@ -67,16 +67,16 @@ public interface ObjectRegistry { * @return Previous Object associated with the key attached if present else * null. Could return the same object if the object was associated * with the same key for a different life-cycle. - */ + */ public Object cacheForDAG(String key, Object value); - + /** * Insert or update object into the registry with Session life-cycle. This * will remove an object associated with the same key with a different * life-cycle as there is only one instance of an Object stored for a given * key irrespective of the life-cycle attached to the Object. The object may stay * in the cache while the Session (to which the task belongs) is running. - * + * * @param key * Key to identify the Object * @param value @@ -84,7 +84,7 @@ public interface ObjectRegistry { * @return Previous Object associated with the key attached if present else * null. Could return the same object if the object was associated * with the same key for a different life-cycle. - */ + */ public Object cacheForSession(String key, Object value); /** diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/Output.java b/tez-api/src/main/java/org/apache/tez/runtime/api/Output.java index 9125c0d940..e0e75a280c 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/Output.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/Output.java @@ -26,7 +26,7 @@ * * This interface has methods which can be used by a {@link org.apache.tez.runtime.api.Processor} * to control execution of this Output and write data to it. - * + * * Users are expected to derive from {@link AbstractLogicalOutput} * */ @@ -38,16 +38,16 @@ public interface Output { /** * Start any processing that the Output may need to perform. It is the * responsibility of the Processor to start Outputs. - * + * * This typically acts as a signal to Outputs to start any Processing that they * may required. - * + * * This method may be invoked by the framework under certain circumstances, - * and as such requires the implementation to be non-blocking. - * + * and as such requires the implementation to be non-blocking. + * * Outputs must be written to handle multiple start invocations - typically * honoring only the first one. - * + * * @throws Exception */ public void start() throws Exception; diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java index c8f7f5c1f2..b45bca011c 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java @@ -67,7 +67,7 @@ public OutputCommitter(OutputCommitterContext committerContext) { * master process. Based on user configuration, commit is called at the end of * the DAG execution for all outputs or immediately upon completion of all the * tasks that produced the output. This is guaranteed to only be called once. - * + * * @throws java.lang.Exception */ public abstract void commitOutput() throws Exception; @@ -77,7 +77,7 @@ public OutputCommitter(OutputCommitterContext committerContext) { * non-successful state. This is also called to abort a previously committed * output in the case of a post-commit failure. This is called from the * application master process. This may be called multiple times. - * + * * @param finalState * final run-state of the vertex * @throws java.lang.Exception diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java index 5b8906d520..38ea21628c 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java @@ -71,7 +71,7 @@ public interface OutputCommitterContext { * @return User Payload */ public UserPayload getUserPayload(); - + /** * Get Vertex Index in the DAG * @return Vertex index diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputContext.java index f0de897fda..211043863e 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputContext.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputContext.java @@ -42,12 +42,12 @@ public interface OutputContext extends TaskContext { String getInputOutputVertexNames(); /** - * Get the index of the output in the set of all outputs for the task. The + * Get the index of the output in the set of all outputs for the task. The * index will be consistent and valid only among the tasks of this vertex. * @return index */ int getOutputIndex(); - + /** * Get an {@link OutputStatisticsReporter} for this {@link Output} that can * be used to report statistics like data size diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputFrameworkInterface.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputFrameworkInterface.java index 617deee914..71e62df87a 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputFrameworkInterface.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputFrameworkInterface.java @@ -34,7 +34,7 @@ * During initialization, Outputs must specify an initial memory requirement via * {@link OutputContext}.requestInitialMemory *

- * + * * Users are expected to derive from {@link AbstractLogicalOutput} * */ diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatistics.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatistics.java index 2f18a03634..da1f53045a 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatistics.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatistics.java @@ -31,18 +31,18 @@ @Public @Evolving public interface OutputStatistics { - + /** * Returns the data size associated with this logical output *
It is the size of the data written to this output by the vertex. * @return Data size in bytes */ public long getDataSize(); - + /** * Get the numbers of items processed. These could be key-value pairs, table * records etc. - * + * * @return Number of items processed */ public long getItemsProcessed(); diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatisticsReporter.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatisticsReporter.java index 1931e5cae3..a8b0e34623 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatisticsReporter.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatisticsReporter.java @@ -37,7 +37,7 @@ public interface OutputStatisticsReporter { /** * Report the number of items processed. These could be key-value pairs, table * records etc. - * + * * @param items * number of items */ diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/Processor.java b/tez-api/src/main/java/org/apache/tez/runtime/api/Processor.java index 32d786e3d4..749be4d11f 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/Processor.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/Processor.java @@ -22,9 +22,9 @@ /** * {@link Processor} represents the Tez entity responsible for - * consuming {@link Input} and producing {@link Output}. It defines the + * consuming {@link Input} and producing {@link Output}. It defines the * application logic that processes and transforms the data. - * + * * Users are expected to derive from {@link AbstractLogicalIOProcessor} */ @Public diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorContext.java index 3782a8d8d7..14d4424e58 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorContext.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorContext.java @@ -43,7 +43,7 @@ static float preProcessProgress(float progress) { /** * Set the overall progress of this Task Attempt. - * This automatically results in invocation of {@link ProcessorContext#notifyProgress()} + * This automatically results in invocation of {@link ProcessorContext#notifyProgress()} * and so invoking that separately is not required. * @param progress Progress in the range from [0.0 - 1.0f] */ @@ -68,15 +68,15 @@ default void setProgress(float progress) { /** * Blocking call which returns when any of the specified Inputs is ready for * consumption. - * + * * There can be multiple parallel invocations of this function - where each * invocation blocks on the Inputs that it specifies. - * + * * If multiple Inputs are ready, any one of them may be returned by this * method - including an Input which may have been returned in a previous * call. If invoking this method multiple times, it's recommended to remove * previously completed Inputs from the invocation list. - * + * * @param inputs * the list of Inputs to monitor * @return the Input which is ready for consumption @@ -110,10 +110,10 @@ default void setProgress(float progress) { /** * Blocking call which returns only after all of the specified Inputs are * ready for consumption. - * + * * There can be multiple parallel invocations of this function - where each * invocation blocks on the Inputs that it specifies. - * + * * @param inputs * the list of Inputs to monitor * @throws InterruptedException diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/TaskAttemptIdentifier.java b/tez-api/src/main/java/org/apache/tez/runtime/api/TaskAttemptIdentifier.java index 101fa91d36..21cd1c58d7 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/TaskAttemptIdentifier.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/TaskAttemptIdentifier.java @@ -19,8 +19,8 @@ package org.apache.tez.runtime.api; public interface TaskAttemptIdentifier { - + public int getIdentifier(); - + public TaskIdentifier getTaskIdentifier(); } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/TaskIdentifier.java b/tez-api/src/main/java/org/apache/tez/runtime/api/TaskIdentifier.java index 8ef066bb7f..0e9895b01a 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/TaskIdentifier.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/TaskIdentifier.java @@ -19,8 +19,8 @@ package org.apache.tez.runtime.api; public interface TaskIdentifier { - + public int getIdentifier(); - + public VertexIdentifier getVertexIdentifier(); } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/VertexIdentifier.java b/tez-api/src/main/java/org/apache/tez/runtime/api/VertexIdentifier.java index 16e88ad4ec..21d550cd6e 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/VertexIdentifier.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/VertexIdentifier.java @@ -21,8 +21,8 @@ public interface VertexIdentifier { public int getIdentifier(); - + public String getName(); - + public DagIdentifier getDagIdentifier(); } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/VertexStatistics.java b/tez-api/src/main/java/org/apache/tez/runtime/api/VertexStatistics.java index aa526f24b5..0e1f2ba65c 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/VertexStatistics.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/VertexStatistics.java @@ -38,7 +38,7 @@ public interface VertexStatistics { /** * Get statistics about an {@link Edge} input or external input of this * {@link Vertex}.
- * + * * @param inputName * Name of the input {@link Edge} or external input of this vertex * @return {@link InputStatistics} for the given input @@ -48,7 +48,7 @@ public interface VertexStatistics { /** * Get statistics about an {@link Edge} output or external output of this * {@link Vertex}.
- * + * * @param outputName * Name of the output {@link Edge} or external output of this vertex * @return {@link OutputStatistics} for the given output diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java index 32089a99ac..fcda5db769 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java @@ -29,14 +29,14 @@ /** * A convenience class to specify multiple DataMovementEvents which share the * same payload. A contiguous range of srcIndices can be specified. - * + * * This event will NOT be seen by downstream Inputs - instead they will see * {@link DataMovementEvent}s which are generated based on the range specified * in this event. - * + * * This event should be used by an output which has the same payload for all of * the Physical Outputs that it generates. - * + * */ @Public public class CompositeDataMovementEvent extends Event { @@ -67,12 +67,12 @@ public static CompositeDataMovementEvent create(int srcIndexStart, int count, ByteBuffer userPayload) { return new CompositeDataMovementEvent(srcIndexStart, count, userPayload); } - + /** * Expand the {@link CompositeDataMovementEvent} into a routable * {@link DataMovementEvent} by providing the source output index and the * target input index. - * + * * @param sourceIndex * The index of the physical output represented by the * {@link DataMovementEvent} @@ -111,11 +111,11 @@ public int getVersion() { public Iterable getEvents() { return new Iterable() { - + @Override public Iterator iterator() { return new Iterator() { - + int currentPos = sourceIndexStart; @Override diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java index 05c3d3fe4f..800caaa4ed 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java @@ -82,7 +82,7 @@ public static DataMovementEvent create(int sourceIndex, ByteBuffer userPayload) { return new DataMovementEvent(sourceIndex, -1, -1, userPayload); } - + @Private /** * Constructor for Processor-generated User Events @@ -99,11 +99,11 @@ public static DataMovementEvent create(int sourceIndex, ByteBuffer userPayload) { return new DataMovementEvent(sourceIndex, targetIndex, version, userPayload); } - + /** * Make a routable copy of the {@link DataMovementEvent} by adding a target * input index - * + * * @param targetIndex * The index of the physical input to which this * {@link DataMovementEvent} should be routed diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java index 05857d286c..f79119b74b 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java @@ -26,8 +26,8 @@ import org.apache.tez.runtime.api.InputSpecUpdate; /** - * An event typically sent by the {@link InputInitializer} of a vertex - * to configure the tasks of the vertex. It could change the task + * An event typically sent by the {@link InputInitializer} of a vertex + * to configure the tasks of the vertex. It could change the task * placement hints or input specification for the inputs of the tasks */ @Unstable diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java index 8cf0616e6b..cbe43e5ce6 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java @@ -30,14 +30,14 @@ * Events used by {@link InputInitializer} implementations to provide the * user payload for individual tasks running as part of the Vertex for which an * Initial Input has been configured. - * + * * This event is used by InputInitialziers to configure tasks belonging to a * Vertex. The event may be processed by a @link {@link VertexManagerPlugin} * before being sent to tasks. - * + * * A {@link InputInitializer} may send Events with or without a * serialized user payload. - * + * * Events, after being processed by a {@link VertexManagerPlugin}, must * contain the payload in a serialized form. */ @@ -57,7 +57,7 @@ private InputDataInformationEvent(int srcIndex, ByteBuffer userPayload) { this.userPayload = userPayload; this.userPayloadObject = null; } - + private InputDataInformationEvent(int srcIndex, Object userPayloadDeserialized, Object sigChanged) { this.sourceIndex = srcIndex; this.userPayloadObject = userPayloadDeserialized; diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java index 9d8363a785..48ed44f8b4 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java @@ -40,11 +40,11 @@ public class InputFailedEvent extends Event{ * Version number to indicate what attempt generated this Event */ private int version; - + @Private // for Writable public InputFailedEvent() { } - + @Private private InputFailedEvent(int targetIndex, int version) { this.targetIndex = targetIndex; @@ -55,15 +55,15 @@ private InputFailedEvent(int targetIndex, int version) { public static InputFailedEvent create(int targetIndex, int version) { return new InputFailedEvent(targetIndex, version); } - + /** * Create a copy of the {@link InputFailedEvent} by adding a target input * index The index of the physical input to which this event should be routed - * + * * @param targetIndex * The index of the physical input to which this * {@link InputFailedEvent} should be routed - * + * * @return copy of the {@link InputFailedEvent} with the target input index * added */ diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java index 7a450c6cdf..39b59bb55b 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java @@ -38,12 +38,12 @@ public class VertexManagerEvent extends Event { /** - * Vertex to which the event should be sent + * Vertex to which the event should be sent */ private final String targetVertexName; - + private TaskAttemptIdentifier producerAttempt; - + /** * User payload to be sent */ @@ -74,21 +74,21 @@ public static VertexManagerEvent create(String vertexName, ByteBuffer userPayloa public String getTargetVertexName() { return targetVertexName; } - + public ByteBuffer getUserPayload() { return userPayload == null ? null : userPayload.asReadOnlyBuffer(); } - + /** * Get metadata about the task attempt that produced the event. - * This method will provide a valid return value only when invoked in the + * This method will provide a valid return value only when invoked in the * {@link VertexManagerPlugin} * @return attempt metadata */ public TaskAttemptIdentifier getProducerAttemptIdentifier() { return producerAttempt; } - + @Private public void setProducerAttemptIdentifier(TaskAttemptIdentifier producerAttempt) { this.producerAttempt = producerAttempt; diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java index bb2bd98096..68af185cc3 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java @@ -181,7 +181,7 @@ TezClientForTest configureAndCreateTezClient(TezConfiguration conf) throws YarnE IOException { return configureAndCreateTezClient(new HashMap(), true, conf); } - + TezClientForTest configureAndCreateTezClient(Map lrs, boolean isSession, TezConfiguration conf) throws YarnException, IOException, ServiceException { if (conf == null) { @@ -210,12 +210,12 @@ TezClientForTest configureAndCreateTezClient(Map lrs, boo return client; } - + @Test (timeout = 5000) public void testTezClientApp() throws Exception { testTezClient(false, true, "testTezClientApp"); } - + @Test (timeout = 5000) public void testTezClientSession() throws Exception { testTezClient(true, true, "testTezClientSession"); @@ -336,13 +336,13 @@ public void testGetClient() throws Exception { verify(client2.mockYarnClient, times(1)).stop(); /* END reuse of AM from new TezClient */ } - + public TezClientForTest testTezClient(boolean isSession, boolean shouldStop, String dagName) throws Exception { Map lrs = Maps.newHashMap(); String lrName1 = "LR1"; lrs.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); - + TezClientForTest client = configureAndCreateTezClient(lrs, isSession, null); ArgumentCaptor captor = ArgumentCaptor.forClass(ApplicationSubmissionContext.class); @@ -364,7 +364,7 @@ public TezClientForTest testTezClient(boolean isSession, boolean shouldStop, Str } else { verify(client.mockYarnClient, times(0)).submitApplication(captor.capture()); } - + String mockLR1Name = "LR1"; Map lrDAG = Collections.singletonMap(mockLR1Name, LocalResource .newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, @@ -377,7 +377,7 @@ public TezClientForTest testTezClient(boolean isSession, boolean shouldStop, Str .thenReturn(GetAMStatusResponseProto.newBuilder().setStatus(TezAppMasterStatusProto.SHUTDOWN).build()); } DAGClient dagClient = client.submitDAG(dag); - + assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString())); assertEquals(dagClient.getSessionIdentifierString(), client.mockAppId.toString()); @@ -414,7 +414,7 @@ public TezClientForTest testTezClient(boolean isSession, boolean shouldStop, Str dag = DAG.create("DAG-2-" + dagName).addVertex( Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1))); dagClient = client.submitDAG(dag); - + if (isSession) { // same app master verify(client.mockYarnClient, times(1)).submitApplication(captor.capture()); @@ -557,14 +557,14 @@ public void testPreWarm() throws Exception { when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()) .thenReturn(YarnApplicationState.RUNNING); - + when( client.sessionAmProxy.getAMStatus(any(), any())) .thenReturn(GetAMStatusResponseProto.newBuilder().setStatus(TezAppMasterStatusProto.READY).build()); PreWarmVertex vertex = PreWarmVertex.create("PreWarm", 1, Resource.newInstance(1, 1)); client.preWarm(vertex); - + ArgumentCaptor captor1 = ArgumentCaptor.forClass(SubmitDAGRequestProto.class); verify(client.sessionAmProxy, times(1)).submitDAG(any(), captor1.capture()); SubmitDAGRequestProto proto = captor1.getValue(); @@ -698,14 +698,14 @@ public void testMultipleSubmissions() throws Exception { testMultipleSubmissionsJob(false); testMultipleSubmissionsJob(true); } - + public void testMultipleSubmissionsJob(boolean isSession) throws Exception { TezClientForTest client1 = configureAndCreateTezClient(new HashMap(), isSession, null); when(client1.mockYarnClient.getApplicationReport(client1.mockAppId).getYarnApplicationState()) .thenReturn(YarnApplicationState.RUNNING); client1.start(); - + String mockLR1Name = "LR1"; Map lrDAG = Collections.singletonMap(mockLR1Name, LocalResource .newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, @@ -721,20 +721,20 @@ public void testMultipleSubmissionsJob(boolean isSession) throws Exception { // the dag resource will be added to the vertex once client1.submitDAG(dag); - + TezClientForTest client2 = configureAndCreateTezClient(); when(client2.mockYarnClient.getApplicationReport(client2.mockAppId).getYarnApplicationState()) .thenReturn(YarnApplicationState.RUNNING); client2.start(); - + // verify resubmission of same dag to new client (simulates submission error resulting in the // creation of a new client and resubmission of the DAG) client2.submitDAG(dag); - + client1.stop(); client2.stop(); } - + @Test(timeout = 5000) public void testWaitTillReady_Interrupt() throws Exception { final TezClientForTest client = configureAndCreateTezClient(); @@ -760,7 +760,7 @@ public void run() { Assert.assertThat(exceptionReference.get(), CoreMatchers.instanceOf(InterruptedException.class)); client.stop(); } - + @Test(timeout = 5000) public void testWaitTillReadyAppFailed() throws Exception { final TezClientForTest client = configureAndCreateTezClient(); @@ -778,7 +778,7 @@ public void testWaitTillReadyAppFailed() throws Exception { } client.stop(); } - + @Test(timeout = 5000) public void testWaitTillReadyAppFailedNoDiagnostics() throws Exception { final TezClientForTest client = configureAndCreateTezClient(); @@ -793,12 +793,12 @@ public void testWaitTillReadyAppFailedNoDiagnostics() throws Exception { } client.stop(); } - + @Test(timeout = 5000) public void testSubmitDAGAppFailed() throws Exception { final TezClientForTest client = configureAndCreateTezClient(); client.start(); - + client.callRealGetSessionAMProxy = true; String msg = "Application Test Failed"; when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()) @@ -809,7 +809,7 @@ public void testSubmitDAGAppFailed() throws Exception { Vertex vertex = Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1)); DAG dag = DAG.create("DAG-testSubmitDAGAppFailed").addVertex(vertex); - + try { client.submitDAG(dag); fail(); diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java index 56d4192df2..79aa9231ba 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java @@ -79,7 +79,7 @@ import org.junit.Assert; import org.junit.Test; /** - * + * */ public class TestTezClientUtils { private static String TEST_ROOT_DIR = "target" + Path.SEPARATOR @@ -87,7 +87,7 @@ public class TestTezClientUtils { private static final File STAGING_DIR = new File(System.getProperty("test.build.data", "target"), TestTezClientUtils.class.getName()).getAbsoluteFile(); /** - * + * */ @Test (timeout=5000) public void validateSetTezJarLocalResourcesNotDefined() throws Exception { @@ -121,7 +121,7 @@ public void validateSetTezJarLocalResourcesDefinedButEmpty() throws Exception { } /** - * + * */ @Test(expected=FileNotFoundException.class, timeout=5000) public void validateSetTezJarLocalResourcesDefinedNonExistingDirectory() throws Exception { @@ -181,7 +181,7 @@ public void validateSetTezJarLocalResourcesDefinedExistingDirectory() throws Exc assertTrue(resourceNames.contains(innerFile.getName())); assertedDir = true; } - // not supporting deep hierarchies + // not supporting deep hierarchies } } else { assertTrue(resourceNames.contains(file.getName())); @@ -193,7 +193,7 @@ public void validateSetTezJarLocalResourcesDefinedExistingDirectory() throws Exc } /** - * + * * @throws Exception */ @Test (timeout=5000) @@ -214,7 +214,7 @@ public void validateSetTezJarLocalResourcesDefinedExistingDirectoryIgnored() thr } /** - * + * * @throws Exception */ @Test (timeout=20000) diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java index 9880a8c4ba..758117799b 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java @@ -107,7 +107,7 @@ public void testDuplicatedVertexGroup() { e.printStackTrace(); Assert.assertEquals("VertexGroup group_1 already defined!", e.getMessage()); } - // it is possible to create vertex group with same member but different group name + // it is possible to create vertex group with same member but different group name dag.createVertexGroup("group_2", v1, v2); } diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java index 2ee3efeedc..fa5bb1f43c 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java @@ -243,7 +243,7 @@ public void userVertexOrderingIsMaintained() { VertexPlan v3Proto = dagProto.getVertex(2); EdgePlan edgeProto = dagProto.getEdge(0); - // either v1 or v2 will be on top based on topological order + // either v1 or v2 will be on top based on topological order String v1ProtoPayload = new String(v1Proto.getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray()); String v2ProtoPayload = new String(v2Proto.getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray()); assertTrue(v1ProtoPayload.equals("processor1Bytes") || v1ProtoPayload.equals("processor3Bytes")); @@ -306,16 +306,16 @@ public void testCredentialsSerde() { Token token2 = new Token(); dagCredentials.addToken(new Text("Token1"), token1); dagCredentials.addToken(new Text("Token2"), token2); - + dag.setCredentials(dagCredentials); DAGPlan dagProto = dag.createDag(new TezConfiguration(), null, null, null, true); assertTrue(dagProto.hasCredentialsBinary()); - + Credentials fetchedCredentials = DagTypeConverters.convertByteStringToCredentials(dagProto .getCredentialsBinary()); - + assertEquals(2, fetchedCredentials.numberOfTokens()); assertNotNull(fetchedCredentials.getToken(new Text("Token1"))); assertNotNull(fetchedCredentials.getToken(new Text("Token2"))); diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestTaskLocationHint.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestTaskLocationHint.java index 2dada70a95..5b89cea1e1 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/TestTaskLocationHint.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestTaskLocationHint.java @@ -24,7 +24,7 @@ import org.junit.Test; public class TestTaskLocationHint { - + @Test (timeout = 5000) public void testEquality() { TaskLocationHint t1 = TaskLocationHint.createTaskLocationHint("v1", 0); diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java b/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java index 87860447cd..6e979972a7 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java @@ -93,14 +93,14 @@ public class TestDAGClient { private ApplicationReport mockAppReport; private String dagIdStr; private DAGClientAMProtocolBlockingPB mockProxy; - + private VertexStatusProto vertexStatusProtoWithoutCounters; private VertexStatusProto vertexStatusProtoWithCounters; - + private DAGStatusProto dagStatusProtoWithoutCounters; private DAGStatusProto dagStatusProtoWithCounters; - - + + private void setUpData(){ // DAG ProgressProto dagProgressProto = ProgressProto.newBuilder() @@ -110,7 +110,7 @@ private void setUpData(){ .setSucceededTaskCount(2) .setTotalTaskCount(6) .build(); - + TezCountersProto dagCountersProto=TezCountersProto.newBuilder() .addCounterGroups(TezCounterGroupProto.newBuilder() .setName("DAGGroup") @@ -118,7 +118,7 @@ private void setUpData(){ .setDisplayName("dag_counter_1") .setValue(99))) .build(); - + dagStatusProtoWithoutCounters = DAGStatusProto.newBuilder() .addDiagnostics("Diagnostics_0") .setState(DAGStatusStateProto.DAG_RUNNING) @@ -138,11 +138,11 @@ private void setUpData(){ .setKilledTaskCount(1)) ) .build(); - + dagStatusProtoWithCounters = DAGStatusProto.newBuilder(dagStatusProtoWithoutCounters) .setDagCounters(dagCountersProto) .build(); - + // Vertex ProgressProto vertexProgressProto = ProgressProto.newBuilder() .setFailedTaskCount(1) @@ -150,14 +150,14 @@ private void setUpData(){ .setRunningTaskCount(0) .setSucceededTaskCount(1) .build(); - + TezCountersProto vertexCountersProto=TezCountersProto.newBuilder() .addCounterGroups(TezCounterGroupProto.newBuilder() .addCounters(TezCounterProto.newBuilder() .setDisplayName("vertex_counter_1") .setValue(99))) .build(); - + vertexStatusProtoWithoutCounters = VertexStatusProto.newBuilder() .setId("vertex_1") .addDiagnostics("V_Diagnostics_0") @@ -186,12 +186,12 @@ public boolean matches(GetVertexStatusRequestProto requestProto) { && requestProto.getStatusOptionsList().get(0) == StatusGetOptsProto.GET_COUNTERS; } } - + @Before public void setUp() throws YarnException, IOException, TezException, ServiceException{ setUpData(); - + /////////////// mock ////////////////////// mockAppId = mock(ApplicationId.class); mockAppReport = mock(ApplicationReport.class); @@ -202,7 +202,7 @@ public void setUp() throws YarnException, IOException, TezException, ServiceExce .thenReturn(GetDAGStatusResponseProto.newBuilder().setDagStatus(dagStatusProtoWithoutCounters).build()); when(mockProxy.getDAGStatus(isNull(), argThat(new DAGCounterRequestMatcher()))) .thenReturn(GetDAGStatusResponseProto.newBuilder().setDagStatus(dagStatusProtoWithCounters).build()); - + when(mockProxy.getVertexStatus(isNull(), any())) .thenReturn(GetVertexStatusResponseProto.newBuilder().setVertexStatus(vertexStatusProtoWithoutCounters).build()); when(mockProxy.getVertexStatus(isNull(), argThat(new VertexCounterRequestMatcher()))) @@ -215,7 +215,7 @@ public void setUp() throws YarnException, IOException, TezException, ServiceExce realClient.appReport = mockAppReport; realClient.proxy = mockProxy; } - + @Test(timeout = 5000) public void testApp() throws IOException, TezException, ServiceException{ assertTrue(dagClient.getExecutionContext().contains(mockAppId.toString())); @@ -224,7 +224,7 @@ public void testApp() throws IOException, TezException, ServiceException{ DAGClientRPCImpl realClient = (DAGClientRPCImpl)((DAGClientImpl)dagClient).getRealClient(); assertEquals(mockAppReport, realClient.getApplicationReportInternal()); } - + @Test(timeout = 5000) public void testDAGStatus() throws Exception{ DAGStatus resultDagStatus = dagClient.getDAGStatus(null); @@ -232,14 +232,14 @@ public void testDAGStatus() throws Exception{ .setDagId(dagIdStr).setTimeout(0).build()); assertEquals(new DAGStatus(dagStatusProtoWithoutCounters, DagStatusSource.AM), resultDagStatus); System.out.println("DAGStatusWithoutCounter:" + resultDagStatus); - + resultDagStatus = dagClient.getDAGStatus(Sets.newSet(StatusGetOpts.GET_COUNTERS)); verify(mockProxy, times(1)).getDAGStatus(null, GetDAGStatusRequestProto.newBuilder() .setDagId(dagIdStr).setTimeout(0).addStatusOptions(StatusGetOptsProto.GET_COUNTERS).build()); assertEquals(new DAGStatus(dagStatusProtoWithCounters, DagStatusSource.AM), resultDagStatus); System.out.println("DAGStatusWithCounter:" + resultDagStatus); } - + @Test(timeout = 5000) public void testVertexStatus() throws Exception{ VertexStatus resultVertexStatus = dagClient.getVertexStatus("v1", null); @@ -247,7 +247,7 @@ public void testVertexStatus() throws Exception{ .setDagId(dagIdStr).setVertexName("v1").build()); assertEquals(new VertexStatus(vertexStatusProtoWithoutCounters), resultVertexStatus); System.out.println("VertexWithoutCounter:" + resultVertexStatus); - + resultVertexStatus = dagClient.getVertexStatus("v1", Sets.newSet(StatusGetOpts.GET_COUNTERS)); verify(mockProxy).getVertexStatus(null, GetVertexStatusRequestProto.newBuilder() .setDagId(dagIdStr).setVertexName("v1").addStatusOptions(StatusGetOptsProto.GET_COUNTERS) @@ -255,14 +255,14 @@ public void testVertexStatus() throws Exception{ assertEquals(new VertexStatus(vertexStatusProtoWithCounters), resultVertexStatus); System.out.println("VertexWithCounter:" + resultVertexStatus); } - + @Test(timeout = 5000) public void testTryKillDAG() throws Exception{ dagClient.tryKillDAG(); verify(mockProxy, times(1)).tryKillDAG(null, TryKillDAGRequestProto.newBuilder() .setDagId(dagIdStr).build()); } - + @Test(timeout = 5000) public void testWaitForCompletion() throws Exception{ // first time return DAG_RUNNING, second time return DAG_SUCCEEDED @@ -301,7 +301,7 @@ public void testWaitForCompletionWithStatusUpdates() throws Exception{ (DAGStatusProto.newBuilder(dagStatusProtoWithoutCounters) .setState(DAGStatusStateProto.DAG_SUCCEEDED).build()) .build()); - + // first time for getVertexSet // second & third time for check completion ArgumentCaptor rpcControllerArgumentCaptor = diff --git a/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java b/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java index c20869f19c..214aaa9d22 100644 --- a/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java +++ b/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java @@ -72,9 +72,9 @@ public class AsyncDispatcher extends CompositeService implements Dispatcher { private Thread eventHandlingThread; protected final Map, EventHandler> eventHandlers = Maps.newHashMap(); protected final Map, AsyncDispatcher> eventDispatchers = Maps.newHashMap(); - protected final Map, AsyncDispatcherConcurrent> concurrentEventDispatchers = + protected final Map, AsyncDispatcherConcurrent> concurrentEventDispatchers = Maps.newHashMap(); - + private boolean exitOnDispatchException = false; public AsyncDispatcher(String name) { @@ -133,7 +133,7 @@ protected void serviceStart() throws Exception { eventHandlingThread = new Thread(createThread()); eventHandlingThread.setName("Dispatcher thread {" + name + "}"); eventHandlingThread.start(); - + //start all the components super.serviceStart(); } @@ -218,16 +218,16 @@ protected void dispatch(Event event) { } } } - + private void checkForExistingHandler(Class eventType) { EventHandler registeredHandler = (EventHandler) eventHandlers.get(eventType); - Preconditions.checkState(registeredHandler == null, + Preconditions.checkState(registeredHandler == null, "Cannot register same event on multiple dispatchers"); } private void checkForExistingDispatcher(Class eventType) { AsyncDispatcher registeredDispatcher = eventDispatchers.get(eventType); - Preconditions.checkState(registeredDispatcher == null, + Preconditions.checkState(registeredDispatcher == null, "Multiple dispatchers cannot be registered for: " + eventType.getName()); } @@ -236,7 +236,7 @@ private void checkForExistingConcurrentDispatcher(Class eventTyp Preconditions.checkState(concurrentDispatcher == null, "Multiple concurrent dispatchers cannot be registered for: " + eventType.getName()); } - + @VisibleForTesting protected void checkForExistingDispatchers(boolean checkHandler, Class eventType) { if (checkHandler) { @@ -277,14 +277,14 @@ public void register(Class eventType, multiHandler.addHandler(handler); } } - + /** * Add an EventHandler for events handled in their own dispatchers with given name */ public void registerAndCreateDispatcher(Class eventType, EventHandler handler, String dispatcherName) { Preconditions.checkState(getServiceState() == STATE.NOTINITED); - + /* check to see if we have a listener registered */ checkForExistingDispatchers(true, eventType); LOG.info( @@ -294,11 +294,11 @@ public void registerAndCreateDispatcher(Class eventType, eventDispatchers.put(eventType, dispatcher); addIfService(dispatcher); } - + public AsyncDispatcherConcurrent registerAndCreateDispatcher(Class eventType, EventHandler handler, String dispatcherName, int numThreads) { Preconditions.checkState(getServiceState() == STATE.NOTINITED); - + /* check to see if we have a listener registered */ checkForExistingDispatchers(true, eventType); LOG.info( @@ -312,11 +312,11 @@ public AsyncDispatcherConcurrent registerAndCreateDispatcher(Class eventType, EventHandler handler, AsyncDispatcherConcurrent dispatcher) { Preconditions.checkState(getServiceState() == STATE.NOTINITED); - + /* check to see if we have a listener registered */ checkForExistingDispatchers(true, eventType); LOG.info("Registering " + eventType + " with existing concurrent dispatch using: " @@ -355,9 +355,9 @@ public void handle(Event event) { concurrentDispatcher.getEventHandler().handle(event); return; } - + // no registered dispatcher. use internal dispatcher. - + /* all this method does is enqueue all the events onto the queue */ int qSize = eventQueue.size(); if (qSize !=0 && qSize %1000 == 0) { diff --git a/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcherConcurrent.java b/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcherConcurrent.java index 7c0dcced7b..541e35c1dc 100644 --- a/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcherConcurrent.java +++ b/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcherConcurrent.java @@ -43,7 +43,7 @@ import org.slf4j.LoggerFactory; /** - * A dispatcher that can schedule events concurrently. Uses a fixed size threadpool + * A dispatcher that can schedule events concurrently. Uses a fixed size threadpool * to schedule events. Events that have the same serializing hash will get scheduled * on the same thread in the threadpool. This can be used to prevent concurrency issues * for events that may not be independently processed. @@ -74,9 +74,9 @@ public class AsyncDispatcherConcurrent extends CompositeService implements Dispa private ExecutorService execService; private final int numThreads; - + protected final Map, EventHandler> eventHandlers = Maps.newHashMap(); - protected final Map, AsyncDispatcherConcurrent> eventDispatchers = + protected final Map, AsyncDispatcherConcurrent> eventDispatchers = Maps.newHashMap(); private boolean exitOnDispatchException = false; @@ -87,14 +87,14 @@ public class AsyncDispatcherConcurrent extends CompositeService implements Dispa this.eventQueues = Lists.newArrayListWithCapacity(numThreads); this.numThreads = numThreads; } - + class DispatchRunner implements Runnable { final LinkedBlockingQueue queue; - + public DispatchRunner(LinkedBlockingQueue queue) { this.queue = queue; } - + @Override public void run() { while (!stopped && !Thread.currentThread().isInterrupted()) { @@ -124,7 +124,7 @@ public void run() { } } }; - + @Override protected void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); @@ -206,7 +206,7 @@ protected void dispatch(Event event) { private void checkForExistingHandler(Class eventType) { EventHandler registeredHandler = (EventHandler) eventHandlers.get(eventType); - Preconditions.checkState(registeredHandler == null, + Preconditions.checkState(registeredHandler == null, "Cannot register same event on multiple dispatchers"); } @@ -249,15 +249,15 @@ public void register(Class eventType, multiHandler.addHandler(handler); } } - + /** * Add an EventHandler for events handled in their own dispatchers with given name and threads */ - + public AsyncDispatcherConcurrent registerAndCreateDispatcher(Class eventType, EventHandler handler, String dispatcherName, int numThreads) { Preconditions.checkState(getServiceState() == STATE.NOTINITED); - + /* check to see if we have a listener registered */ checkForExistingDispatchers(true, eventType); LOG.info( @@ -268,11 +268,11 @@ public AsyncDispatcherConcurrent registerAndCreateDispatcher(Class eventType, EventHandler handler, AsyncDispatcherConcurrent dispatcher) { Preconditions.checkState(getServiceState() == STATE.NOTINITED); - + /* check to see if we have a listener registered */ checkForExistingDispatchers(true, eventType); LOG.info("Registering " + eventType + " wit existing concurrent dispatch using: " @@ -300,7 +300,7 @@ public void handle(TezAbstractEvent event) { return; } drained = false; - + // offload to specific dispatcher if one exists Class type = event.getType().getDeclaringClass(); AsyncDispatcherConcurrent registeredDispatcher = eventDispatchers.get(type); @@ -308,7 +308,7 @@ public void handle(TezAbstractEvent event) { registeredDispatcher.getEventHandler().handle(event); return; } - + int index = numThreads > 1 ? event.getSerializingHash() % numThreads : 0; // no registered dispatcher. use internal dispatcher. diff --git a/tez-common/src/main/java/org/apache/tez/common/GcTimeUpdater.java b/tez-common/src/main/java/org/apache/tez/common/GcTimeUpdater.java index 4911d0f41a..00241d4bea 100644 --- a/tez-common/src/main/java/org/apache/tez/common/GcTimeUpdater.java +++ b/tez-common/src/main/java/org/apache/tez/common/GcTimeUpdater.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -40,7 +40,7 @@ public GcTimeUpdater(TezCounters counters) { getElapsedGc(); // Initialize 'lastGcMillis' with the current time spent. this.counters = counters; } - + public long getCumulativaGcTime() { long thisGcMillis = 0; for (GarbageCollectorMXBean gcBean : gcBeans) { diff --git a/tez-common/src/main/java/org/apache/tez/common/TezAbstractEvent.java b/tez-common/src/main/java/org/apache/tez/common/TezAbstractEvent.java index b73611202f..ef13d11725 100644 --- a/tez-common/src/main/java/org/apache/tez/common/TezAbstractEvent.java +++ b/tez-common/src/main/java/org/apache/tez/common/TezAbstractEvent.java @@ -21,7 +21,7 @@ /** * Event that allows running in parallel for different instances - * + * * @param * Event type */ @@ -35,7 +35,7 @@ public TezAbstractEvent(TYPE type) { /** * Returning a number that is identical for event instances that need to be * serialized while processing. - * + * * @return Serializing identifier. Not overriding this causes serialization * for all events instances */ diff --git a/tez-common/src/main/java/org/apache/tez/common/TezContainerLogAppender.java b/tez-common/src/main/java/org/apache/tez/common/TezContainerLogAppender.java index 1649a0b318..4d358099ab 100644 --- a/tez-common/src/main/java/org/apache/tez/common/TezContainerLogAppender.java +++ b/tez-common/src/main/java/org/apache/tez/common/TezContainerLogAppender.java @@ -27,13 +27,13 @@ /** * A simple log4j-appender for a tez container's logs. - * + * */ @Unstable public class TezContainerLogAppender extends FileAppender { private String containerLogDir; private String logFileName = TezConstants.TEZ_CONTAINER_LOG_FILE_NAME; - //so that log4j can configure it from the configuration(log4j.properties). + //so that log4j can configure it from the configuration(log4j.properties). @Override public void activateOptions() { @@ -47,7 +47,7 @@ public void activateOptions() { /** * Set the name of the file for logging. This should NOT be an absolute path. * The file will be created within the container's log directory. - * + * * @param fileName * @throws NullPointerException if {@code fileName} is {@code null} * @throws IllegalArgumentException if {@code fileName} is an absolute path diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/DagIdentifierImpl.java b/tez-common/src/main/java/org/apache/tez/dag/records/DagIdentifierImpl.java index 099cb58a13..140d64a39b 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/DagIdentifierImpl.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/DagIdentifierImpl.java @@ -24,7 +24,7 @@ public class DagIdentifierImpl implements DagIdentifier { private final TezDAGID dagId; private final String dagName; - + public DagIdentifierImpl(String dagName, TezDAGID dagId) { this.dagId = dagId; this.dagName = dagName; @@ -39,7 +39,7 @@ public String getName() { public int getIdentifier() { return dagId.getId(); } - + @Override public boolean equals(Object o) { if (this == o) { @@ -56,12 +56,12 @@ public boolean equals(Object o) { return false; } } - + @Override public String toString() { return "Dag: " + dagName + ":[" + getIdentifier() + "]"; } - + @Override public int hashCode() { return dagId.hashCode(); diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptIdentifierImpl.java b/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptIdentifierImpl.java index b834111cb9..3ba1c08ee6 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptIdentifierImpl.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptIdentifierImpl.java @@ -25,7 +25,7 @@ public class TaskAttemptIdentifierImpl implements TaskAttemptIdentifier { private final TaskIdentifier taskIdentifier; private final TezTaskAttemptID attemptId; - + public TaskAttemptIdentifierImpl(String dagName, String vertexName, TezTaskAttemptID attemptId) { this.attemptId = attemptId; this.taskIdentifier = new TaskIdentifierImpl(dagName, vertexName, attemptId.getTaskID()); @@ -35,12 +35,12 @@ public TaskAttemptIdentifierImpl(String dagName, String vertexName, TezTaskAttem public int getIdentifier() { return attemptId.getId(); } - + @Override public TaskIdentifier getTaskIdentifier() { return taskIdentifier; } - + @Override public boolean equals(Object o) { if (this == o) { @@ -57,12 +57,12 @@ public boolean equals(Object o) { return false; } } - + @Override public String toString() { return taskIdentifier.toString() + " Attempt: [" + getIdentifier() + "]"; } - + @Override public int hashCode() { return attemptId.hashCode(); diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TaskIdentifierImpl.java b/tez-common/src/main/java/org/apache/tez/dag/records/TaskIdentifierImpl.java index fb0848aaad..d201af76ef 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/TaskIdentifierImpl.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/TaskIdentifierImpl.java @@ -25,7 +25,7 @@ public class TaskIdentifierImpl implements TaskIdentifier { private final VertexIdentifier vertexIdentifier; private final TezTaskID taskId; - + public TaskIdentifierImpl(String dagName, String vertexName, TezTaskID taskId) { this.taskId = taskId; this.vertexIdentifier = new VertexIdentifierImpl(dagName, vertexName, taskId.getVertexID()); @@ -35,12 +35,12 @@ public TaskIdentifierImpl(String dagName, String vertexName, TezTaskID taskId) { public int getIdentifier() { return taskId.getId(); } - + @Override public VertexIdentifier getVertexIdentifier() { return vertexIdentifier; } - + @Override public boolean equals(Object o) { if (this == o) { @@ -57,7 +57,7 @@ public boolean equals(Object o) { return false; } } - + @Override public String toString() { return vertexIdentifier.toString() + " Task [" + getIdentifier() + "]"; diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezID.java index 7efbd9a889..189032f603 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/TezID.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezID.java @@ -28,9 +28,9 @@ /** * A general identifier, which internally stores the id - * as an integer. This is the super class of {@link TezDAGID}, + * as an integer. This is the super class of {@link TezDAGID}, * {@link TezVertexID}, {@link TezTaskID}, and {@link TezTaskAttemptID}. - * + * * @see TezTaskID * @see TezTaskAttemptID */ @@ -86,7 +86,7 @@ public int compareTo(TezID that) { public void readFields(DataInput in) throws IOException { this.id = in.readInt(); } - + public static int readID(DataInput in) throws IOException { return in.readInt(); } @@ -95,5 +95,5 @@ public static int readID(DataInput in) throws IOException { public void write(DataOutput out) throws IOException { out.writeInt(id); } - + } diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java index fe2b84449f..fca6ddfc97 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java @@ -55,10 +55,10 @@ public class TezTaskAttemptID extends TezID implements TaskIDAware { // Public for Writable serialization. Verify if this is actually required. public TezTaskAttemptID() { } - + /** - * Constructs a TaskAttemptID object from given {@link TezTaskID}. - * @param taskID TaskID that this task belongs to + * Constructs a TaskAttemptID object from given {@link TezTaskID}. + * @param taskID TaskID that this task belongs to * @param id the task attempt number * @throws NullPointerException if {@code taskID} is {@code null} */ @@ -77,7 +77,7 @@ private TezTaskAttemptID(TezTaskID taskId, int id) { public TezTaskID getTaskID() { return taskId; } - + @Override public boolean equals(Object o) { if (!super.equals(o)) @@ -86,7 +86,7 @@ public boolean equals(Object o) { TezTaskAttemptID that = (TezTaskAttemptID)o; return this.taskId.equals(that.taskId); } - + /** * Add the unique string to the StringBuilder * @param builder the builder to append ot @@ -97,12 +97,12 @@ protected StringBuilder appendTo(StringBuilder builder) { builder.append(SEPARATOR); return builder.append(id); } - + @Override public int hashCode() { return taskId.hashCode() * 539501 + id; } - + /**Compare TaskIds by first tipIds, then by task numbers. */ @Override public int compareTo(TezID o) { @@ -114,10 +114,10 @@ public int compareTo(TezID o) { else return tipComp; } @Override - public String toString() { + public String toString() { return appendTo(new StringBuilder(ATTEMPT)).toString(); } - + @Override // Can't do much about this instance if used via the RPC layer. Any downstream // users can however avoid using this method. @@ -125,7 +125,7 @@ public void readFields(DataInput in) throws IOException { taskId = TezTaskID.readTezTaskID(in); super.readFields(in); } - + public static TezTaskAttemptID readTezTaskAttemptID(DataInput in) throws IOException { TezTaskID taskID = TezTaskID.readTezTaskID(in); int attemptIdInt = TezID.readID(in); diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java index 29f896b532..b3631ab7d0 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java @@ -43,7 +43,7 @@ public class TezTaskID extends TezID implements VertexIDAware { public static final String TASK = "task"; private final int serializingHash; - + static final ThreadLocal tezTaskIdFormat = new ThreadLocal() { @Override public FastNumberFormat initialValue() { @@ -72,7 +72,7 @@ private TezTaskID(TezVertexID vertexID, int id) { this.vertexId = vertexID; this.serializingHash = getHashCode(true); } - + public int getSerializingHash() { return serializingHash; } @@ -138,7 +138,7 @@ public void readFields(DataInput in) throws IOException { vertexId = TezVertexID.readTezVertexID(in); super.readFields(in); } - + public static TezTaskID readTezTaskID(DataInput in) throws IOException { TezVertexID vertexID = TezVertexID.readTezVertexID(in); int taskIdInt = TezID.readID(in); diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/VertexIdentifierImpl.java b/tez-common/src/main/java/org/apache/tez/dag/records/VertexIdentifierImpl.java index 83b503203c..2a55c636d5 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/VertexIdentifierImpl.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/VertexIdentifierImpl.java @@ -26,7 +26,7 @@ public class VertexIdentifierImpl implements VertexIdentifier { private final DagIdentifier dagIdentifier; private final TezVertexID vertexId; private final String vertexName; - + public VertexIdentifierImpl(String dagName, String vertexName, TezVertexID vertexId) { this.vertexId = vertexId; this.vertexName = vertexName; @@ -42,12 +42,12 @@ public String getName() { public int getIdentifier() { return vertexId.getId(); } - + @Override public DagIdentifier getDagIdentifier() { return dagIdentifier; } - + @Override public boolean equals(Object o) { if (this == o) { @@ -64,12 +64,12 @@ public boolean equals(Object o) { return false; } } - + @Override public String toString() { return dagIdentifier.toString() + " Vertex: " + vertexName + ":[" + getIdentifier() + "]"; } - + @Override public int hashCode() { return vertexId.hashCode(); diff --git a/tez-common/src/main/java/org/apache/tez/dag/utils/RelocalizationUtils.java b/tez-common/src/main/java/org/apache/tez/dag/utils/RelocalizationUtils.java index 9ccfc76be7..2dd98720f0 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/utils/RelocalizationUtils.java +++ b/tez-common/src/main/java/org/apache/tez/dag/utils/RelocalizationUtils.java @@ -38,7 +38,7 @@ public final class RelocalizationUtils { private RelocalizationUtils() {} - + public static List processAdditionalResources(Map additionalResources, Configuration conf, String destDir) throws IOException { if (additionalResources == null || additionalResources.isEmpty()) { diff --git a/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java b/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java index db804c8152..8f36b8a1ea 100644 --- a/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java +++ b/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java @@ -98,7 +98,7 @@ public float getCpuUsagePercent() { //Returning -1 to indicate, this feature is not yet supported. return -1; } - + private static Method getMxBeanMethod(String methodName) { // New Method to support IBM and Oracle/OpenJDK JDK with OperatingSystemMXBean final String JAVA_VENDOR_NAME = System.getProperty("java.vendor"); diff --git a/tez-common/src/test/java/org/apache/tez/common/TestAsyncDispatcher.java b/tez-common/src/test/java/org/apache/tez/common/TestAsyncDispatcher.java index 2a61a578cb..3390d89e93 100644 --- a/tez-common/src/test/java/org/apache/tez/common/TestAsyncDispatcher.java +++ b/tez-common/src/test/java/org/apache/tez/common/TestAsyncDispatcher.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -76,18 +76,18 @@ public void handle(TestEvent3 event) { handle(); } } - + @SuppressWarnings("unchecked") @Test (timeout=5000) public void testBasic() throws Exception { CountDownLatch latch = new CountDownLatch(4); CountDownEventHandler.latch = latch; - + AsyncDispatcher central = new AsyncDispatcher("Type1"); central.register(TestEventType1.class, new TestEventHandler1()); central.registerAndCreateDispatcher(TestEventType2.class, new TestEventHandler2(), "Type2"); central.registerAndCreateDispatcher(TestEventType3.class, new TestEventHandler3(), "Type3"); - + central.init(new Configuration()); central.start(); central.getEventHandler().handle(new TestEvent1(TestEventType1.TYPE1)); @@ -97,7 +97,7 @@ public void testBasic() throws Exception { latch.await(); central.close(); } - + @Test (timeout=5000) public void testMultipleRegisterFail() throws Exception { AsyncDispatcher central = new AsyncDispatcher("Type1"); @@ -110,7 +110,7 @@ public void testMultipleRegisterFail() throws Exception { } finally { central.close(); } - + central = new AsyncDispatcher("Type1"); try { central.registerAndCreateDispatcher(TestEventType1.class, new TestEventHandler2(), "Type2"); diff --git a/tez-common/src/test/java/org/apache/tez/common/TestAsyncDispatcherConcurrent.java b/tez-common/src/test/java/org/apache/tez/common/TestAsyncDispatcherConcurrent.java index de7e7a7bbd..a25c571038 100644 --- a/tez-common/src/test/java/org/apache/tez/common/TestAsyncDispatcherConcurrent.java +++ b/tez-common/src/test/java/org/apache/tez/common/TestAsyncDispatcherConcurrent.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -38,7 +38,7 @@ static void checkParallelCountersDoneAndFinish() throws Exception { latch.countDown(); latch.await(); } - + public void handle() { latch.countDown(); try { @@ -56,7 +56,7 @@ public TestEvent1(TestEventType1 type, int hash) { super(type); this.hash = hash; } - + @Override public int getSerializingHash() { return hash; @@ -97,12 +97,12 @@ public void handle(TestEvent3 event) { public void testBasic() throws Exception { CountDownLatch latch = new CountDownLatch(4); CountDownEventHandler.init(latch); - + AsyncDispatcher central = new AsyncDispatcher("Type1"); central.register(TestEventType1.class, new TestEventHandler1()); central.registerAndCreateDispatcher(TestEventType2.class, new TestEventHandler2(), "Type2", 1); central.registerAndCreateDispatcher(TestEventType3.class, new TestEventHandler3(), "Type3", 1); - + central.init(new Configuration()); central.start(); // 3 threads in different dispatchers will handle 3 events @@ -113,15 +113,15 @@ public void testBasic() throws Exception { CountDownEventHandler.checkParallelCountersDoneAndFinish(); central.close(); } - + @Test (timeout=5000) public void testMultiThreads() throws Exception { CountDownLatch latch = new CountDownLatch(4); CountDownEventHandler.init(latch); - + AsyncDispatcherConcurrent central = new AsyncDispatcherConcurrent("Type1", 1); central.registerAndCreateDispatcher(TestEventType1.class, new TestEventHandler1(), "Type1", 3); - + central.init(new Configuration()); central.start(); // 3 threads in the same dispatcher will handle 3 events @@ -132,7 +132,7 @@ public void testMultiThreads() throws Exception { CountDownEventHandler.checkParallelCountersDoneAndFinish(); central.close(); } - + @Test (timeout=5000) public void testMultipleRegisterFail() throws Exception { AsyncDispatcher central = new AsyncDispatcher("Type1"); @@ -145,7 +145,7 @@ public void testMultipleRegisterFail() throws Exception { } finally { central.close(); } - + central = new AsyncDispatcher("Type1"); try { central.registerAndCreateDispatcher(TestEventType1.class, new TestEventHandler2(), "Type2", 1); @@ -156,7 +156,7 @@ public void testMultipleRegisterFail() throws Exception { } finally { central.close(); } - + central = new AsyncDispatcher("Type1"); try { central.registerAndCreateDispatcher(TestEventType1.class, new TestEventHandler2(), "Type2", 1); @@ -167,7 +167,7 @@ public void testMultipleRegisterFail() throws Exception { } finally { central.close(); } - + central = new AsyncDispatcher("Type1"); try { central.registerAndCreateDispatcher(TestEventType1.class, new TestEventHandler2(), "Type2"); @@ -178,7 +178,7 @@ public void testMultipleRegisterFail() throws Exception { } finally { central.close(); } - + central = new AsyncDispatcher("Type1"); try { AsyncDispatcherConcurrent concDispatcher = central.registerAndCreateDispatcher( diff --git a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java index 146a2dbcf2..3b9b7cae6e 100644 --- a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java +++ b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -155,7 +155,7 @@ public void testBitSetFromByteArray() { bytes = TezUtilsInternal.toByteArray(bitSet); Assert.assertEquals(TezUtilsInternal.fromByteArray(bytes).cardinality(), bitSet.cardinality()); Assert.assertTrue(TezUtilsInternal.fromByteArray(bytes).equals(bitSet)); - + bitSet = createBitSet(1000); bytes = TezUtilsInternal.toByteArray(bitSet); Assert.assertEquals(TezUtilsInternal.fromByteArray(bytes).cardinality(), bitSet.cardinality()); @@ -167,7 +167,7 @@ public void testBitSetConversion() { for (int i = 0 ; i < 16 ; i++) { BitSet bitSet = createBitSetWithSingleEntry(i); byte[] bytes = TezUtilsInternal.toByteArray(bitSet); - + BitSet deseraialized = TezUtilsInternal.fromByteArray(bytes); Assert.assertEquals(bitSet, deseraialized); Assert.assertEquals(bitSet.cardinality(), deseraialized.cardinality()); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientHandler.java index 098cfe88ab..d30d29436c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientHandler.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientHandler.java @@ -47,7 +47,7 @@ public class DAGClientHandler { private DAGAppMaster dagAppMaster; private final AtomicLong lastHeartbeatTime; - + public DAGClientHandler(DAGAppMaster dagAppMaster) { this.dagAppMaster = dagAppMaster; this.lastHeartbeatTime = new AtomicLong(dagAppMaster.getContext().getClock().getTime()); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java index a2ee7e38ac..699e592fca 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java @@ -52,13 +52,13 @@ public interface TaskAttemptReport { public abstract void setNodeManagerPort(int nmPort); public abstract void setNodeManagerHttpPort(int nmHttpPort); public abstract void setContainerId(ContainerId containerId); - - /** + + /** * Set the shuffle finish time. Applicable only for reduce attempts * @param time the time the shuffle finished. */ public abstract void setShuffleFinishTime(long time); - /** + /** * Set the sort/merge finish time. Applicable only for reduce attempts * @param time the time the shuffle finished. */ diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java index 926835a97d..4b13e098d3 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java @@ -19,9 +19,9 @@ package org.apache.tez.dag.api.oldrecords; public enum TaskAttemptState { - NEW, - STARTING, - RUNNING, + NEW, + STARTING, + RUNNING, SUCCEEDED, FAILED, KILLED diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java index c3258b426a..c60fae15b1 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java @@ -57,9 +57,9 @@ public interface AppContext { ApplicationId getApplicationID(); TezDAGID getCurrentDAGID(); - + long getCumulativeCPUTime(); - + long getCumulativeGCTime(); ApplicationAttemptId getApplicationAttemptId(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/ClusterInfo.java b/tez-dag/src/main/java/org/apache/tez/dag/app/ClusterInfo.java index b30783887a..bd5ca0edcc 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/ClusterInfo.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/ClusterInfo.java @@ -27,7 +27,7 @@ public class ClusterInfo { public ClusterInfo() { } - + public ClusterInfo(Resource maxCapability) { this.maxContainerCapability = maxCapability; initialized = true; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java index 0e1726127a..98bfecf0b0 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java @@ -59,7 +59,7 @@ public ContainerContext(Map localResources, this.javaOpts = javaOpts; this.vertex = null; } - + public ContainerContext(Map localResources, Credentials credentials, Map environment, String javaOpts, @Nullable Vertex vertex) { @@ -114,7 +114,7 @@ public boolean isSuperSet(ContainerContext otherContext) { return isSuperSet(this.environment, otherContext.getEnvironment(), "Environment") && localResourcesCompatible(this.localResources, otherContext.getLocalResources()); } - + /** * @return true if this ContainerContext is an exact match of the specified * container context. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerHeartbeatHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerHeartbeatHandler.java index e494fa6069..4c74f8be04 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerHeartbeatHandler.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerHeartbeatHandler.java @@ -27,8 +27,8 @@ public class ContainerHeartbeatHandler extends HeartbeatHandlerBase { - - public ContainerHeartbeatHandler(AppContext context, + + public ContainerHeartbeatHandler(AppContext context, int expectedConcurrency) { super(context, expectedConcurrency, "ContainerHeartbeatHandler"); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/HeartbeatHandlerBase.java b/tez-dag/src/main/java/org/apache/tez/dag/app/HeartbeatHandlerBase.java index b5ded81500..d7898f8d85 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/HeartbeatHandlerBase.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/HeartbeatHandlerBase.java @@ -35,12 +35,12 @@ public abstract class HeartbeatHandlerBase extends AbstractService { protected int timeOutCheckInterval; protected Thread timeOutCheckerThread; private final String name; - + @SuppressWarnings("rawtypes") protected final EventHandler eventHandler; protected final Clock clock; protected final AppContext appContext; - + private ConcurrentMap runningMap; public HeartbeatHandlerBase(AppContext appContext, int expectedConcurrency, String name) { @@ -73,67 +73,67 @@ public void serviceStop() { timeOutCheckerThread.interrupt(); } } - + protected Runnable createPingChecker() { return new PingChecker(); } protected abstract int getConfiguredTimeout(Configuration conf); protected abstract int getConfiguredTimeoutCheckInterval(Configuration conf); - + public void progressing(T id) { ReportTime time = runningMap.get(id); if (time != null) { time.setLastProgress(clock.getTime()); } } - + public void pinged(T id) { ReportTime time = runningMap.get(id); if (time != null) { time.setLastPing(clock.getTime()); } } - + public void register(T id) { runningMap.put(id, new ReportTime(clock.getTime())); } - + public void unregister(T id) { runningMap.remove(id); } - - - + + + protected static class ReportTime { private long lastPing; private long lastProgress; - + public ReportTime(long time) { setLastProgress(time); } - + public synchronized void setLastPing(long time) { lastPing = time; } - + public synchronized void setLastProgress(long time) { lastProgress = time; lastPing = time; } - + public synchronized long getLastPing() { return lastPing; } - + public synchronized long getLastProgress() { return lastProgress; } } - + protected abstract boolean hasTimedOut(ReportTime report, long currentTime); - + protected abstract void handleTimeOut(T t); - + private class PingChecker implements Runnable { @Override @@ -146,7 +146,7 @@ public void run() { long currentTime = clock.getTime(); while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); + Map.Entry entry = iterator.next(); if(hasTimedOut(entry.getValue(), currentTime)) { // Timed out. Removed from list and send out an event. iterator.remove(); @@ -161,5 +161,5 @@ public void run() { } } } - + } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptEventInfo.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptEventInfo.java index d6b03e4ecd..f1a6d7127c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptEventInfo.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptEventInfo.java @@ -25,17 +25,17 @@ public class TaskAttemptEventInfo { private final int nextFromEventId; private final int nextPreRoutedFromEventId; private final List events; - + public TaskAttemptEventInfo(int nextFromEventId, List events, int nextPreRoutedEventId) { this.nextFromEventId = nextFromEventId; this.nextPreRoutedFromEventId = nextPreRoutedEventId; this.events = events; } - + public int getNextFromEventId() { return nextFromEventId; } - + public int getNextPreRoutedFromEventId() { return nextPreRoutedFromEventId; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManagerInterface.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManagerInterface.java index 93e98493d6..e8a703aac0 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManagerInterface.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManagerInterface.java @@ -35,9 +35,9 @@ public interface TaskCommunicatorManagerInterface { void registerRunningContainer(ContainerId containerId, int taskCommId); void registerTaskAttempt(AMContainerTask amContainerTask, ContainerId containerId, int taskCommId); - + void unregisterRunningContainer(ContainerId containerId, int taskCommId, ContainerEndReason endReason, String diagnostics); - + void unregisterTaskAttempt(TezTaskAttemptID attemptID, int taskCommId, TaskAttemptEndReason endReason, String diagnostics); void dagComplete(DAG dag); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java index 296073a426..ea19e7e4ee 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java @@ -32,7 +32,7 @@ * This class keeps track of tasks that have already been launched. It * determines if a task is alive and running or marks a task as dead if it does * not hear from it for a long time. - * + * */ @SuppressWarnings({"unchecked"}) public class TaskHeartbeatHandler extends HeartbeatHandlerBase { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGScheduler.java index ada7867a6a..06a3b68b6a 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGScheduler.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGScheduler.java @@ -37,9 +37,9 @@ private static class VertexInfo { this.concurrencyLimit = limit; } } - + Map vertexInfo = null; - + public void addVertexConcurrencyLimit(TezVertexID vId, int concurrency) { if (vertexInfo == null) { vertexInfo = new HashMap<>(); @@ -48,7 +48,7 @@ public void addVertexConcurrencyLimit(TezVertexID vId, int concurrency) { vertexInfo.put(vId, new VertexInfo(concurrency)); } } - + public void scheduleTask(DAGEventSchedulerUpdate event) { VertexInfo vInfo = null; if (vertexInfo != null) { @@ -56,7 +56,7 @@ public void scheduleTask(DAGEventSchedulerUpdate event) { } scheduleTaskWithLimit(event, vInfo); } - + private void scheduleTaskWithLimit(DAGEventSchedulerUpdate event, VertexInfo vInfo) { if (vInfo != null) { if (vInfo.concurrency >= vInfo.concurrencyLimit) { @@ -67,7 +67,7 @@ private void scheduleTaskWithLimit(DAGEventSchedulerUpdate event, VertexInfo vIn } scheduleTaskEx(event); } - + public void taskCompleted(DAGEventSchedulerUpdate event) { taskCompletedEx(event); if (vertexInfo != null) { @@ -85,9 +85,9 @@ public void taskCompleted(DAGEventSchedulerUpdate event) { } } } - + public abstract void scheduleTaskEx(DAGEventSchedulerUpdate event); - + public abstract void taskCompletedEx(DAGEventSchedulerUpdate event); /** diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGState.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGState.java index 1acc08db1d..6d5164fb56 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGState.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGState.java @@ -24,7 +24,7 @@ public enum DAGState { SUCCEEDED, FAILED, KILLED, - ERROR, + ERROR, TERMINATING, COMMITTING, } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGTerminationCause.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGTerminationCause.java index b73cbe647b..3b69f7c96c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGTerminationCause.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGTerminationCause.java @@ -20,7 +20,7 @@ /** - * Represents proximate cause of a DAG transition to FAILED or KILLED. + * Represents proximate cause of a DAG transition to FAILED or KILLED. */ public enum DAGTerminationCause { @@ -29,16 +29,16 @@ public enum DAGTerminationCause { /** A service plugin indicated an error */ SERVICE_PLUGIN_ERROR(DAGState.FAILED), - - /** A vertex failed. */ + + /** A vertex failed. */ VERTEX_FAILURE(DAGState.FAILED), - + /** DAG failed due as it had zero vertices. */ ZERO_VERTICES(DAGState.FAILED), - + /** DAG failed during init. */ INIT_FAILURE(DAGState.FAILED), - + /** DAG failed during output commit. */ COMMIT_FAILURE(DAGState.FAILED), diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/StateChangeNotifier.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/StateChangeNotifier.java index 99cb4ac768..753e4e093c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/StateChangeNotifier.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/StateChangeNotifier.java @@ -52,7 +52,7 @@ public class StateChangeNotifier { private static final Logger LOG = LoggerFactory.getLogger(StateChangeNotifier.class); - + private final DAG dag; private final SetMultimap vertexListeners; private final ListMultimap lastKnowStatesMap; @@ -62,20 +62,20 @@ public class StateChangeNotifier { BlockingQueue eventQueue = new LinkedBlockingQueue(); private Thread eventHandlingThread; private volatile boolean stopEventHandling = false; - + private static class NotificationEvent { final VertexStateUpdate update; final VertexStateUpdateListener listener; - + public NotificationEvent(VertexStateUpdate update, VertexStateUpdateListener listener) { this.update = update; this.listener = listener; } - + void sendUpdate() { listener.onStateUpdated(update); } - + @Override public String toString() { return "[ VertexState:(" + update + ") Listener:" + listener + " ]"; @@ -89,7 +89,7 @@ public StateChangeNotifier(DAG dag) { this.lastKnowStatesMap = LinkedListMultimap.create(); startThread(); } - + private void startThread() { this.eventHandlingThread = new Thread("State Change Notifier DAG: " + dag.getID()) { @SuppressWarnings("unchecked") @@ -123,15 +123,15 @@ public void run() { this.eventHandlingThread.setDaemon(true); // dont block exit on this this.eventHandlingThread.start(); } - + @VisibleForTesting protected void processedEventFromQueue() { } - + @VisibleForTesting protected void addedEventToQueue() { } - + public void stop() { this.stopEventHandling = true; if (eventHandlingThread != null) @@ -207,7 +207,7 @@ private void enqueueNotification(NotificationEvent event) { LOG.error("Failed to put event", e); } } - + private final class ListenerContainer { final VertexStateUpdateListener listener; final Set states; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java index cb1bc33866..e9d8667838 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java @@ -52,23 +52,23 @@ public interface Task extends TaskIDAware { * Can the output of the taskAttempt be committed. Note that once the task * gives a go for a commit, further canCommit requests from any other attempts * should return false. - * + * * @param taskAttemptID * @return whether the attempt's output can be committed or not. */ boolean canCommit(TezTaskAttemptID taskAttemptID); - + public Vertex getVertex(); - + public ArrayList getTaskAttemptTezEvents(TezTaskAttemptID attemptID, int fromEventId, int maxEvents); - + public List getDiagnostics(); public void registerTezEvent(TezEvent tezEvent); - + public TaskSpec getBaseTaskSpec(); - + public TaskLocationHint getTaskLocationHint(); long getFirstAttemptStartTime(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java index d8818671fd..7e921bf021 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java @@ -45,7 +45,7 @@ public static class TaskAttemptStatus { public TaskAttemptState state; public float progress; public TezCounters counters; - + public TaskAttemptStatus(TezTaskAttemptID id) { this.id = id; } @@ -82,10 +82,10 @@ public void setCounters(TezCounters counters) { float getProgress(); TaskAttemptState getState(); TaskAttemptState getStateNoLock(); - + void setLastEventSent(TezEvent lastEventSent); - /** + /** * Has attempt reached the final state or not. * @return true if it has finished, else false */ @@ -95,7 +95,7 @@ public void setCounters(TezCounters counters) { * @return the container ID if a container is assigned, otherwise null. */ ContainerId getAssignedContainerID(); - + /** * @return the container if assigned, otherwise null */ @@ -105,23 +105,23 @@ public void setCounters(TezCounters counters) { * @return container mgr address if a container is assigned, otherwise null. */ String getAssignedContainerMgrAddress(); - + /** * @return node's id if a container is assigned, otherwise null. */ NodeId getNodeId(); - + /** * @return node's http address if a container is assigned, otherwise null. */ String getNodeHttpAddress(); - + /** * @return node's rack name if a container is assigned, otherwise null. */ String getNodeRackName(); - /** + /** * @return time at which container is launched. If container is not launched * yet, returns 0. */ @@ -133,7 +133,7 @@ public void setCounters(TezCounters counters) { */ long getScheduleTime(); - /** + /** * @return attempt's finish time. If attempt is not finished * yet, returns 0. */ diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java index 6ddfabb0d0..5da3363fcb 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java @@ -30,7 +30,7 @@ public enum TaskAttemptStateInternal { START_WAIT, SUBMITTED, RUNNING, - KILL_IN_PROGRESS, + KILL_IN_PROGRESS, FAIL_IN_PROGRESS, KILLED, FAILED, diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskTerminationCause.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskTerminationCause.java index e2ae8dadbc..e7e6733017 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskTerminationCause.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskTerminationCause.java @@ -28,10 +28,10 @@ public enum TaskTerminationCause { /** Other vertex failed causing DAG to fail thus killing the parent vertex */ OTHER_VERTEX_FAILURE, - + /** One of the tasks for the source/destination vertex failed. */ - OTHER_TASK_FAILURE, - + OTHER_TASK_FAILURE, + /** One of the tasks of the destination vertex failed. */ OWN_TASK_FAILURE, @@ -39,6 +39,6 @@ public enum TaskTerminationCause { * throw Exception */ AM_USERCODE_FAILURE, - + TASK_RESCHEDULE_IN_COMMITTING, } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java index 49c78aab03..5c0feb5af8 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java @@ -113,15 +113,15 @@ void setParallelism(int parallelism, VertexLocationHint vertexLocationHint, Map sourceEdgeManagers, Map rootInputSpecUpdate, boolean fromVertexManager) throws AMUserCodeException; - + public void reconfigureVertex(int parallelism, @Nullable VertexLocationHint locationHint, @Nullable Map sourceEdgeProperties) throws AMUserCodeException; - + public void reconfigureVertex(@Nullable Map rootInputSpecUpdate, int parallelism, @Nullable VertexLocationHint locationHint) throws AMUserCodeException; - + public void reconfigureVertex(int parallelism, @Nullable VertexLocationHint locationHint, @Nullable Map sourceEdgeProperties, @@ -136,25 +136,25 @@ public void reconfigureVertex(int parallelism, void setOutputVertices(Map outVertices); VertexStatistics getStatistics(); - + Map getInputVertices(); Map getOutputVertices(); - + Map getOutputCommitters(); void setAdditionalInputs(List inputs); void setAdditionalOutputs(List outputs); @Nullable - public Map> + public Map> getAdditionalInputs(); @Nullable - public Map> + public Map> getAdditionalOutputs(); List getInputSpecList(int taskIndex) throws AMUserCodeException; List getOutputSpecList(int taskIndex) throws AMUserCodeException; - + List getGroupInputSpecList(); void addSharedOutputs(Set outputs); Set getSharedOutputs(); @@ -164,10 +164,10 @@ public void reconfigureVertex(int parallelism, void scheduleTasks(List tasks); void scheduleSpeculativeTask(TezTaskID taskId); Resource getTaskResource(); - + public TaskAttemptEventInfo getTaskAttemptTezEvents(TezTaskAttemptID attemptID, int fromEventId, int nextPreRoutedFromEventId, int maxEvents); - + void handleSpeculatorEvent(SpeculatorEvent event); AbstractService getSpeculator(); void initServices(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexTerminationCause.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexTerminationCause.java index 49be74df8d..60fd577c32 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexTerminationCause.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexTerminationCause.java @@ -31,7 +31,7 @@ public enum VertexTerminationCause { /** Initialization failed for one of the root Inputs */ ROOT_INPUT_INIT_FAILURE(VertexState.FAILED), - + /** This vertex failed as its AM usercode (VertexManager/EdgeManager/InputInitializer) * throw Exception */ @@ -54,10 +54,10 @@ public enum VertexTerminationCause { /** This vertex failed during init. */ INIT_FAILURE(VertexState.FAILED), - + INTERNAL_ERROR(VertexState.ERROR), - - /** error when writing recovery log */ + + /** error when writing recovery log */ RECOVERY_ERROR(VertexState.FAILED), /** This vertex failed due to counter limits exceeded. */ diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/CallableEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/CallableEventType.java index e9e93b9869..4d9b308abb 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/CallableEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/CallableEventType.java @@ -19,7 +19,7 @@ package org.apache.tez.dag.app.dag.event; public enum CallableEventType { - + CALLABLE, } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java index 3683a4951b..9cdeea2bc5 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java @@ -26,7 +26,7 @@ public class DAGEventCounterUpdate extends DAGEvent { List counterUpdates = null; - + public DAGEventCounterUpdate(TezDAGID dagId) { super(dagId, DAGEventType.DAG_COUNTER_UPDATE); counterUpdates = new ArrayList<>(); @@ -35,20 +35,20 @@ public DAGEventCounterUpdate(TezDAGID dagId) { public void addCounterUpdate(Enum key, long incrValue) { counterUpdates.add(new CounterIncrementalUpdate(key, incrValue)); } - + public List getCounterUpdates() { return counterUpdates; } - + public static class CounterIncrementalUpdate { Enum key; long incrValue; - + public CounterIncrementalUpdate(Enum key, long incrValue) { this.key = key; this.incrValue = incrValue; } - + public Enum getCounterKey() { return key; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventRecoverEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventRecoverEvent.java index 8e1edf0e43..2667e7ca26 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventRecoverEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventRecoverEvent.java @@ -32,15 +32,15 @@ public DAGEventRecoverEvent(TezDAGID dagId, DAGState desiredState, DAGRecoveryDa this.desiredState = desiredState; this.recoveredDagData = recoveredDagData; } - + public DAGEventRecoverEvent(TezDAGID dagId, DAGRecoveryData recoveredDagData) { this(dagId, null, recoveredDagData); } - + public DAGState getDesiredState() { return desiredState; } - + public boolean hasDesiredState() { return this.desiredState != null; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java index 1dedaecad9..317d50b6e2 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java @@ -28,21 +28,21 @@ public enum UpdateType { TA_SCHEDULE, TA_COMPLETED } - + private final TaskAttempt attempt; private final UpdateType updateType; - + public DAGEventSchedulerUpdate(UpdateType updateType, TaskAttempt attempt) { super(attempt.getDAGID(), DAGEventType.DAG_SCHEDULER_UPDATE); this.attempt = attempt; this.updateType = updateType; } - + public UpdateType getUpdateType() { return updateType; } - + public TaskAttempt getAttempt() { return attempt; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventStartDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventStartDag.java index 9e6d75b019..164fc65f21 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventStartDag.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventStartDag.java @@ -26,7 +26,7 @@ public class DAGEventStartDag extends DAGEvent { private final List additionalUrlsForClasspath; - + public DAGEventStartDag(TezDAGID dagId, List additionalUrlsForClasspath) { super(dagId, DAGEventType.DAG_START); this.additionalUrlsForClasspath = additionalUrlsForClasspath; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEvent.java index 3863a2a6f4..2500b38df4 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEvent.java @@ -23,12 +23,12 @@ public class SpeculatorEvent extends TezAbstractEvent { private final TezVertexID vertexId; - + public SpeculatorEvent(SpeculatorEventType type, TezVertexID vertexId) { super(type); this.vertexId = vertexId; } - + public TezVertexID getVertexId() { return vertexId; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEventTaskAttemptStatusUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEventTaskAttemptStatusUpdate.java index 7ab6141c39..c3325c9386 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEventTaskAttemptStatusUpdate.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEventTaskAttemptStatusUpdate.java @@ -26,12 +26,12 @@ public class SpeculatorEventTaskAttemptStatusUpdate extends SpeculatorEvent { final TaskAttemptState state; final long timestamp; final boolean justStarted; - + public SpeculatorEventTaskAttemptStatusUpdate(TezTaskAttemptID taId, TaskAttemptState state, long timestamp) { this(taId, state, timestamp, false); } - + public SpeculatorEventTaskAttemptStatusUpdate(TezTaskAttemptID taId, TaskAttemptState state, long timestamp, boolean justStarted) { super(SpeculatorEventType.S_TASK_ATTEMPT_STATUS_UPDATE, taId.getVertexID()); @@ -44,15 +44,15 @@ public SpeculatorEventTaskAttemptStatusUpdate(TezTaskAttemptID taId, TaskAttempt public long getTimestamp() { return timestamp; } - + public TezTaskAttemptID getAttemptId() { return id; } - + public boolean hasJustStarted() { return justStarted; } - + public TaskAttemptState getTaskAttemptState() { return state; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEventType.java index f3b5c10503..81673eb5b4 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEventType.java @@ -25,5 +25,5 @@ public enum SpeculatorEventType { //Producer:TaskAttempt S_TASK_ATTEMPT_STATUS_UPDATE, - + } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java index d5d8481a54..489082ae30 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java @@ -29,7 +29,7 @@ public class TaskAttemptEvent extends TezAbstractEvent implements TaskAttemptIDAware { private TezTaskAttemptID attemptID; - + /** * Create a new TaskAttemptEvent. * @param id the id of the task attempt @@ -44,7 +44,7 @@ public TaskAttemptEvent(TezTaskAttemptID id, TaskAttemptEventType type) { public TezTaskAttemptID getTaskAttemptID() { return attemptID; } - + @Override public int getSerializingHash() { return attemptID.getTaskID().getSerializingHash(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptFailed.java index d4c7273c97..677c47b8d1 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptFailed.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptFailed.java @@ -24,7 +24,7 @@ import org.apache.tez.dag.records.TezTaskAttemptID; import org.apache.tez.runtime.api.TaskFailureType; -public class TaskAttemptEventAttemptFailed extends TaskAttemptEvent +public class TaskAttemptEventAttemptFailed extends TaskAttemptEvent implements DiagnosableEvent, TaskAttemptEventTerminationCauseEvent, RecoveryEvent { private final String diagnostics; @@ -56,7 +56,7 @@ public TaskAttemptEventAttemptFailed(TezTaskAttemptID id, public String getDiagnosticInfo() { return diagnostics; } - + @Override public TaskAttemptTerminationCause getTerminationCause() { return errorCause; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptKilled.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptKilled.java index 4642443848..7ee228a9cd 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptKilled.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptKilled.java @@ -43,7 +43,7 @@ public TaskAttemptEventAttemptKilled(TezTaskAttemptID id, this(id, diagnostics, errorCause); this.fromRecovery = fromRecovery; } - + @Override public String getDiagnosticInfo() { return diagnostics; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java index 3db2ffcbb8..5d41176d28 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminatedBySystem.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminatedBySystem.java index 4efbf88948..6d21476928 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminatedBySystem.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminatedBySystem.java @@ -22,7 +22,7 @@ import org.apache.tez.dag.records.TaskAttemptTerminationCause; import org.apache.tez.dag.records.TezTaskAttemptID; -public class TaskAttemptEventContainerTerminatedBySystem extends TaskAttemptEvent +public class TaskAttemptEventContainerTerminatedBySystem extends TaskAttemptEvent implements DiagnosableEvent, TaskAttemptEventTerminationCauseEvent { private final ContainerId containerId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java index 02d04a5dcc..d8a3dadabf 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java index 96cf0e6bdf..5609df0697 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java @@ -20,7 +20,7 @@ import org.apache.tez.dag.records.TaskAttemptTerminationCause; import org.apache.tez.dag.records.TezTaskAttemptID; -public class TaskAttemptEventKillRequest extends TaskAttemptEvent +public class TaskAttemptEventKillRequest extends TaskAttemptEvent implements DiagnosableEvent, TaskAttemptEventTerminationCauseEvent, RecoveryEvent { private final String message; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java index 541ef00418..3ad88dc58f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -20,7 +20,7 @@ import org.apache.tez.dag.records.TaskAttemptTerminationCause; import org.apache.tez.dag.records.TezTaskAttemptID; -public class TaskAttemptEventNodeFailed extends TaskAttemptEvent +public class TaskAttemptEventNodeFailed extends TaskAttemptEvent implements DiagnosableEvent, TaskAttemptEventTerminationCauseEvent { private final String message; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputFailed.java index fbdd2305c7..5696f4ba00 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputFailed.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputFailed.java @@ -22,23 +22,23 @@ import org.apache.tez.dag.records.TezTaskAttemptID; import org.apache.tez.runtime.api.impl.TezEvent; -public class TaskAttemptEventOutputFailed extends TaskAttemptEvent +public class TaskAttemptEventOutputFailed extends TaskAttemptEvent implements TaskAttemptEventTerminationCauseEvent { - + private TezEvent inputFailedEvent; private int consumerTaskNumber; - + public TaskAttemptEventOutputFailed(TezTaskAttemptID sourceTaskAttemptId, TezEvent tezEvent, int numConsumers) { super(sourceTaskAttemptId, TaskAttemptEventType.TA_OUTPUT_FAILED); this.inputFailedEvent = tezEvent; this.consumerTaskNumber = numConsumers; } - + public TezEvent getInputFailedEvent() { return inputFailedEvent; } - + public int getConsumerTaskNumber() { return consumerTaskNumber; } @@ -47,5 +47,5 @@ public int getConsumerTaskNumber() { public TaskAttemptTerminationCause getTerminationCause() { return TaskAttemptTerminationCause.OUTPUT_LOST; } - + } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java index 5c02db9a83..f08faf4a6a 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -33,7 +33,7 @@ public TaskAttemptEventSchedule(TezTaskAttemptID id, int lowLimit, int highLimit public int getPriorityLowLimit() { return priLowLimit; } - + public int getPriorityHighLimit() { return priHighLimit; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java index 458679c74b..eac3f72592 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java @@ -22,24 +22,24 @@ import org.apache.tez.runtime.api.events.TaskStatusUpdateEvent; public class TaskAttemptEventStatusUpdate extends TaskAttemptEvent { - + private TaskStatusUpdateEvent taskAttemptStatus; private boolean readErrorReported = false; - + public TaskAttemptEventStatusUpdate(TezTaskAttemptID id, TaskStatusUpdateEvent statusEvent) { super(id, TaskAttemptEventType.TA_STATUS_UPDATE); this.taskAttemptStatus = statusEvent; } - + public TaskStatusUpdateEvent getStatusEvent() { return this.taskAttemptStatus; } - + public void setReadErrorReported(boolean value) { readErrorReported = value; } - + public boolean getReadErrorReported() { return readErrorReported; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java index 63779fd948..99620f7592 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java @@ -35,7 +35,7 @@ public enum TaskAttemptEventType { TA_FAILED, TA_KILLED, // Generated by TaskCommunicators TA_TIMED_OUT, - + //Producer: Client, Scheduler, On speculation. TA_KILL_REQUEST, @@ -54,8 +54,8 @@ public enum TaskAttemptEventType { // The node running the task attempt failed. TA_NODE_FAILED, - + // Producer: consumer destination vertex TA_OUTPUT_FAILED, - + } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java index 9e741388fa..8db7f109fc 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java @@ -39,7 +39,7 @@ public TaskEvent(TezTaskID taskId, TaskEventType type) { public TezTaskID getTaskID() { return taskId; } - + @Override public int getSerializingHash() { return taskId.getSerializingHash(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventScheduleTask.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventScheduleTask.java index 70d6043f40..dd6bb8046c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventScheduleTask.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventScheduleTask.java @@ -34,11 +34,11 @@ public TaskEventScheduleTask(TezTaskID taskId, TaskSpec baseTaskSpec, TaskLocati this.locationHint = locationHint; this.fromRecovery = fromRecovery; } - + public TaskSpec getBaseTaskSpec() { return baseTaskSpec; } - + public TaskLocationHint getTaskLocationHint() { return locationHint; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventType.java index 726e13e23f..e92a0763ca 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventType.java @@ -25,7 +25,7 @@ public enum TaskEventType { //Producer:Client, Job T_TERMINATE, - + //Producer:Job T_SCHEDULE, diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventInputDataInformation.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventInputDataInformation.java index 6b5cad5ab7..ec73fcf4c3 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventInputDataInformation.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventInputDataInformation.java @@ -27,12 +27,12 @@ public class VertexEventInputDataInformation extends VertexEvent { private final List events; - + public VertexEventInputDataInformation(TezVertexID vertexId, List events) { super(vertexId, VertexEventType.V_INPUT_DATA_INFORMATION); this.events = events; } - + public List getEvents() { return events; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventManagerUserCodeError.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventManagerUserCodeError.java index 022620a064..4b2f0ddba1 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventManagerUserCodeError.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventManagerUserCodeError.java @@ -23,12 +23,12 @@ public class VertexEventManagerUserCodeError extends VertexEvent { final AMUserCodeException e; - + public VertexEventManagerUserCodeError(TezVertexID vertexId, AMUserCodeException e) { super(vertexId, VertexEventType.V_MANAGER_USER_CODE_ERROR); this.e = e; } - + public AMUserCodeException getError() { return e; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventNullEdgeInitialized.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventNullEdgeInitialized.java index bedf28b4fd..f4247d547f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventNullEdgeInitialized.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventNullEdgeInitialized.java @@ -25,17 +25,17 @@ public class VertexEventNullEdgeInitialized extends VertexEvent { final Edge edge; final Vertex vertex; - + public VertexEventNullEdgeInitialized(TezVertexID vertexId, Edge edge, Vertex vertex) { super(vertexId, VertexEventType.V_NULL_EDGE_INITIALIZED); this.edge = edge; this.vertex = vertex; } - + public Edge getEdge() { return edge; } - + public Vertex getVertex() { return vertex; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputFailed.java index b7701d7b03..a6949ffe16 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputFailed.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputFailed.java @@ -22,7 +22,7 @@ import org.apache.tez.dag.records.TezVertexID; public class VertexEventRootInputFailed extends VertexEvent { - + private final String inputName; private final AMUserCodeException error; @@ -31,11 +31,11 @@ public VertexEventRootInputFailed(TezVertexID vertexId, String inputName, AMUser this.inputName = inputName; this.error = error; } - + public AMUserCodeException getError() { return this.error; } - + public String getInputName() { return this.inputName; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputInitialized.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputInitialized.java index 3c5e0be9cf..2383d345f3 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputInitialized.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputInitialized.java @@ -38,11 +38,11 @@ public VertexEventRootInputInitialized(TezVertexID vertexId, String inputName, L this.events = events; } } - + public List getEvents() { return events; } - + public String getInputName() { return this.inputName; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRouteEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRouteEvent.java index 211202ded2..44d4e296d2 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRouteEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRouteEvent.java @@ -24,7 +24,7 @@ import org.apache.tez.runtime.api.impl.TezEvent; public class VertexEventRouteEvent extends VertexEvent { - + final List events; public VertexEventRouteEvent(TezVertexID vertexId, List events) { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java index 6cd38a9b29..e289c9c138 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java @@ -27,7 +27,7 @@ public class VertexEventSourceTaskAttemptCompleted extends VertexEvent { public VertexEventSourceTaskAttemptCompleted( TezVertexID targetVertexId, VertexEventTaskAttemptCompleted completionEvent) { - super(targetVertexId, + super(targetVertexId, VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED); this.completionEvent = completionEvent; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java index 5cc9c238f3..906a9da938 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java @@ -24,7 +24,7 @@ public class VertexEventSourceVertexStarted extends VertexEvent { final int sourceDistanceFromRoot; final TezVertexID sourceVertexId; - + public VertexEventSourceVertexStarted(TezVertexID vertexId, TezVertexID sourceVertexId, int distanceFromRoot) { @@ -32,11 +32,11 @@ public VertexEventSourceVertexStarted(TezVertexID vertexId, this.sourceDistanceFromRoot = distanceFromRoot; this.sourceVertexId = sourceVertexId; } - + public int getSourceDistanceFromRoot() { return sourceDistanceFromRoot; } - + public TezVertexID getSourceVertexId() { return sourceVertexId; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTermination.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTermination.java index 647f6003d8..5e4911d648 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTermination.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTermination.java @@ -23,8 +23,8 @@ /** * Specialized VertexEvent for termination conditions. - * - * The VertexEventType is always V_TERMINATE. + * + * The VertexEventType is always V_TERMINATE. * The terminationCause denotes the specific reason why the vertex is being terminated. */ public class VertexEventTermination extends VertexEvent { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java index ed325290cb..f007c289fb 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java @@ -28,37 +28,37 @@ public enum VertexEventType { //Producer:Job V_INIT, - + //Producer:Vertex V_COMPLETED, V_START, V_SOURCE_TASK_ATTEMPT_COMPLETED, V_SOURCE_VERTEX_STARTED, V_DELETE_SHUFFLE_DATA, - + //Producer:Task V_TASK_COMPLETED, V_TASK_RESCHEDULED, V_TASK_ATTEMPT_COMPLETED, - + //Producer:Any component V_INTERNAL_ERROR, V_MANAGER_USER_CODE_ERROR, - + V_ROUTE_EVENT, - + //Producer: VertexInputInitializer V_ROOT_INPUT_INITIALIZED, V_ROOT_INPUT_FAILED, - + V_INPUT_DATA_INFORMATION, // Recover Event, Producer:DAG V_RECOVER, - + // Producer: Vertex V_READY_TO_INIT, - + // Producer: Edge V_NULL_EDGE_INITIALIZED, diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/AMUserCodeException.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/AMUserCodeException.java index 22b1211544..b313bc06f9 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/AMUserCodeException.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/AMUserCodeException.java @@ -25,7 +25,7 @@ /** * Wrap the exception of user code in AM. Now we only have 3 kinds of user code in AM: *

  • VertexManager
    • - *
  • EdgeManager
    • + *
  • EdgeManager
    • *
  • InputInitializer
    • */ @Private @@ -38,9 +38,9 @@ public static enum Source { EdgeManager, InputInitializer } - + private Source source; - + public AMUserCodeException(Source source, String message, Throwable cause) { super(message, cause); this.source = source; @@ -50,7 +50,7 @@ public AMUserCodeException(Source source, Throwable cause) { super(cause); this.source = source; } - + public Source getSource() { return source; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BroadcastEdgeManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BroadcastEdgeManager.java index 867403fcd5..d60d88eb63 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BroadcastEdgeManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BroadcastEdgeManager.java @@ -38,20 +38,20 @@ public BroadcastEdgeManager(EdgeManagerPluginContext context) { @Override public void initialize() { } - + @Override public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { return getContext().getSourceVertexNumTasks(); } - + @Override public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) { return 1; } - + @Override public void routeDataMovementEventToDestination(DataMovementEvent event, - int sourceTaskIndex, int sourceOutputIndex, + int sourceTaskIndex, int sourceOutputIndex, Map> destinationTaskAndInputIndices) { List inputIndices = Collections.singletonList(sourceTaskIndex); // for each task make the i-th source task as the i-th physical input @@ -59,7 +59,7 @@ public void routeDataMovementEventToDestination(DataMovementEvent event, destinationTaskAndInputIndices.put(i, inputIndices); } } - + @Override public void prepareForRouting() throws Exception { int numSourceTasks = getContext().getSourceVertexNumTasks(); @@ -68,14 +68,14 @@ public void prepareForRouting() throws Exception { commonRouteMeta[i] = EventRouteMetadata.create(1, new int[]{i}, new int[]{0}); } } - + @Override public EventRouteMetadata routeDataMovementEventToDestination( int sourceTaskIndex, int sourceOutputIndex, int destinationTaskIndex) throws Exception { return commonRouteMeta[sourceTaskIndex]; } - + @Override public CompositeEventRouteMetadata routeCompositeDataMovementEventToDestination( int sourceTaskIndex, int destinationTaskIndex) @@ -110,7 +110,7 @@ public int routeInputErrorEventToSource(InputReadErrorEvent event, int destinationTaskIndex, int destinationFailedInputIndex) { return destinationFailedInputIndex; } - + @Override public int getNumDestinationConsumerTasks(int sourceTaskIndex) { return getContext().getDestinationVertexNumTasks(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java index d5c21f5793..6bfca5ebdc 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java @@ -224,7 +224,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG, private final AtomicBoolean internalErrorTriggered = new AtomicBoolean(false); Map localResources; - + long startDAGCpuTime = 0; long startDAGGCTime = 0; @@ -336,7 +336,7 @@ DAGEventType.DAG_TERMINATE, new DAGKilledTransition()) EnumSet.of(DAGState.COMMITTING, DAGState.TERMINATING, DAGState.FAILED, DAGState.SUCCEEDED), DAGEventType.DAG_COMMIT_COMPLETED, COMMIT_COMPLETED_TRANSITION) - .addTransition(DAGState.COMMITTING, DAGState.TERMINATING, + .addTransition(DAGState.COMMITTING, DAGState.TERMINATING, DAGEventType.DAG_TERMINATE, new DAGKilledWhileCommittingTransition()) .addTransition( @@ -572,7 +572,7 @@ public DAGImpl(TezDAGID dagId, } else { defaultExecutionContext = null; } - + this.taskSpecificLaunchCmdOption = new TaskSpecificLaunchCmdOption(dagConf); // This "this leak" is okay because the retained pointer is in an // instance variable. @@ -630,7 +630,7 @@ protected StateMachine getStateMachine() { public TezDAGID getID() { return dagId; } - + @Override public Map getLocalResources() { return localResources; @@ -1040,7 +1040,7 @@ public VertexStatusBuilder getVertexStatus(String vertexName, } return vertex.getVertexStatus(statusOptions); } - + public TaskAttemptImpl getTaskAttempt(TezTaskAttemptID taId) { return (TaskAttemptImpl) getVertex(taId.getVertexID()).getTask(taId.getTaskID()) .getAttempt(taId); @@ -1162,7 +1162,7 @@ public Void call() throws Exception { appContext.getHadoopShim().clearHadoopCallerContext(); } } - + if (!commitEvents.isEmpty()) { try { LOG.info("Start writing dag commit event, " + getID()); @@ -1345,7 +1345,7 @@ static DAGState checkVerticesForCompletion(DAGImpl dag) { + ", numFailedVertices=" + dag.numFailedVertices + ", numKilledVertices=" + dag.numKilledVertices + ", numVertices=" + dag.numVertices - + ", commitInProgress=" + dag.commitFutures.size() + + ", commitInProgress=" + dag.commitFutures.size() + ", terminationCause=" + dag.terminationCause); // log in case of accounting error. @@ -1397,7 +1397,7 @@ static DAGState checkCommitsForCompletion(DAGImpl dag) { + ", numFailedVertices=" + dag.numFailedVertices + ", numKilledVertices=" + dag.numKilledVertices + ", numVertices=" + dag.numVertices - + ", commitInProgress=" + dag.commitFutures.size() + + ", commitInProgress=" + dag.commitFutures.size() + ", terminationCause=" + dag.terminationCause); // continue the commits if DAG#isCommittable return true, otherwise go to TERMINATING or finish dag. @@ -1441,7 +1441,7 @@ private void updateCpuCounters() { dagCounters.findCounter(DAGCounter.AM_CPU_MILLISECONDS).setValue(totalDAGCpuTime); dagCounters.findCounter(DAGCounter.AM_GC_TIME_MILLIS).setValue(totalDAGGCTime); } - + @Override public void incrementDagCounter(DAGCounter counter, int incrValue) { dagCounters.findCounter(counter).increment(incrValue); @@ -2331,7 +2331,7 @@ public DAGState transition(DAGImpl dag, DAGEvent event) { return DAGState.TERMINATING; } } - + } private static class CommitCompletedTransition implements diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java index f98100aac1..036a4f2780 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java @@ -31,18 +31,18 @@ @SuppressWarnings("rawtypes") public class DAGSchedulerNaturalOrder extends DAGScheduler { - - private static final Logger LOG = + + private static final Logger LOG = LoggerFactory.getLogger(DAGSchedulerNaturalOrder.class); private final DAG dag; private final EventHandler handler; - + public DAGSchedulerNaturalOrder(DAG dag, EventHandler dispatcher) { this.dag = dag; this.handler = dispatcher; } - + @Override public void scheduleTaskEx(DAGEventSchedulerUpdate event) { TaskAttempt attempt = event.getAttempt(); @@ -56,17 +56,17 @@ public void scheduleTaskEx(DAGEventSchedulerUpdate event) { LOG.debug("Scheduling " + attempt.getTaskAttemptID() + " between priorityLow: " + priorityLowLimit + " and priorityHigh: " + priorityHighLimit); } - + TaskAttemptEventSchedule attemptEvent = new TaskAttemptEventSchedule( attempt.getTaskAttemptID(), priorityLowLimit, priorityHighLimit); - + sendEvent(attemptEvent); } - + @Override public void taskCompletedEx(DAGEventSchedulerUpdate event) { } - + @SuppressWarnings("unchecked") void sendEvent(TaskAttemptEventSchedule event) { handler.handle(event); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java index 0c1c9a0994..abfc836bb6 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java @@ -120,7 +120,7 @@ public String getSourceVertexName() { public String getDestinationVertexName() { return destinationVertex.getName(); } - + @Override public int getSourceVertexNumTasks() { return sourceVertex.getTotalTasks(); @@ -244,7 +244,7 @@ public void setEdgeProperty(EdgeProperty newEdgeProperty) throws AMUserCodeExcep sourceVertex)); } } - + // Test only method for creating specific scenarios @VisibleForTesting void setCustomEdgeManager(EdgeManagerPluginDescriptor descriptor) @@ -257,7 +257,7 @@ void setCustomEdgeManager(EdgeManagerPluginDescriptor descriptor) edgeProperty.getEdgeDestination()); setEdgeProperty(modifiedEdgeProperty); } - + public void routingToBegin() throws AMUserCodeException { int numDestTasks = edgeManagerContext.getDestinationVertexNumTasks(); synchronized (this) { @@ -282,11 +282,11 @@ public void routingToBegin() throws AMUserCodeException { "Fail to prepareForRouting " + getEdgeInfo(), e); } } - - LOG.info("Routing to begin for edge: " + getEdgeInfo() + ". EdgeProperty: " + edgeProperty + + + LOG.info("Routing to begin for edge: " + getEdgeInfo() + ". EdgeProperty: " + edgeProperty + " onDemandRouting: " + hasOnDemandRouting()); } - + public synchronized boolean hasOnDemandRouting() { return onDemandRouting; } @@ -294,7 +294,7 @@ public synchronized boolean hasOnDemandRouting() { public synchronized EdgeProperty getEdgeProperty() { return this.edgeProperty; } - + public EdgeManagerPlugin getEdgeManager() { return this.edgeManager; } @@ -317,7 +317,7 @@ public void setDestinationVertex(Vertex destinationVertex) { } public InputSpec getDestinationSpec(int destinationTaskIndex) throws AMUserCodeException { - Preconditions.checkState(edgeManager != null, + Preconditions.checkState(edgeManager != null, "Edge Manager must be initialized by this time"); try { int physicalInputCount = edgeManager.getNumDestinationTaskPhysicalInputs(destinationTaskIndex); @@ -335,7 +335,7 @@ public InputSpec getDestinationSpec(int destinationTaskIndex) throws AMUserCodeE } public OutputSpec getSourceSpec(int sourceTaskIndex) throws AMUserCodeException { - Preconditions.checkState(edgeManager != null, + Preconditions.checkState(edgeManager != null, "Edge Manager must be initialized by this time"); try { int physicalOutputCount = edgeManager.getNumSourceTaskPhysicalOutputs( @@ -351,11 +351,11 @@ public OutputSpec getSourceSpec(int sourceTaskIndex) throws AMUserCodeException + sourceTaskIndex + ", " + getEdgeInfo(), e); } } - + public void startEventBuffering() { bufferEvents.set(true); } - + public void stopEventBuffering() throws AMUserCodeException { // assume only 1 entity will start and stop event buffering bufferEvents.set(false); @@ -368,9 +368,9 @@ public void stopEventBuffering() throws AMUserCodeException { } sourceEventBuffer.clear(); } - + public void sendTezEventToSourceTasks(TezEvent tezEvent) throws AMUserCodeException { - Preconditions.checkState(edgeManager != null, + Preconditions.checkState(edgeManager != null, "Edge Manager must be initialized by this time"); if (!bufferEvents.get()) { switch (tezEvent.getEventType()) { @@ -411,7 +411,7 @@ public void sendTezEventToSourceTasks(TezEvent tezEvent) throws AMUserCodeExcept " sourceVertex=" + sourceVertex.getLogIdentifier() + " srcIndex = " + srcTaskIndex + " destAttemptId=" + destAttemptId + - " destIndex=" + destTaskIndex + + " destIndex=" + destTaskIndex + " edgeManager=" + edgeManager.getClass().getName()); } TezTaskID srcTaskId = srcTask.getTaskID(); @@ -428,26 +428,26 @@ public void sendTezEventToSourceTasks(TezEvent tezEvent) throws AMUserCodeExcept sourceEventBuffer.add(tezEvent); } } - + private void handleCompositeDataMovementEvent(TezEvent tezEvent) throws AMUserCodeException { CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent(); EventMetaData srcInfo = tezEvent.getSourceInfo(); - + for (DataMovementEvent dmEvent : compEvent.getEvents()) { TezEvent newEvent = new TezEvent(dmEvent, srcInfo, tezEvent.getEventReceivedTime()); sendTezEventToDestinationTasks(newEvent); } } - + void sendDmEventOrIfEventToTasks(TezEvent tezEvent, int srcTaskIndex, boolean isDataMovementEvent, - Map> taskAndInputIndices) { - Preconditions.checkState(edgeManager != null, + Map> taskAndInputIndices) { + Preconditions.checkState(edgeManager != null, "Edge Manager must be initialized by this time"); Event event = tezEvent.getEvent(); // cache of event object per input index - Map inputIndicesWithEvents = Maps.newHashMap(); + Map inputIndicesWithEvents = Maps.newHashMap(); for (Map.Entry> entry : taskAndInputIndices.entrySet()) { int destTaskIndex = entry.getKey(); List inputIndices = entry.getValue(); @@ -475,15 +475,15 @@ void sendDmEventOrIfEventToTasks(TezEvent tezEvent, int srcTaskIndex, " sourceVertex=" + sourceVertex.getLogIdentifier() + " srcTaskIndex = " + srcTaskIndex + " destVertex=" + destinationVertex.getLogIdentifier() + - " destTaskIndex=" + destTaskIndex + - " destNumTasks=" + destinationVertex.getTotalTasks() + + " destTaskIndex=" + destTaskIndex + + " destNumTasks=" + destinationVertex.getTotalTasks() + " edgeManager=" + edgeManager.getClass().getName()); } sendEventToTask(destTask, tezEventToSend); } } } - + public void sendTezEventToDestinationTasks(TezEvent tezEvent) throws AMUserCodeException { if (!bufferEvents.get()) { boolean isDataMovementEvent = true; @@ -543,27 +543,27 @@ public void sendTezEventToDestinationTasks(TezEvent tezEvent) throws AMUserCodeE destinationEventBuffer.add(tezEvent); } } - + static class PendingEventRouteMetadata { private final EventRouteMetadata routeMeta; private final TezEvent event; private int numEventsRouted; - + public PendingEventRouteMetadata(EventRouteMetadata routeMeta, TezEvent event, int numEventsRouted) { this.routeMeta = routeMeta; this.event = event; this.numEventsRouted = numEventsRouted; } - + public EventRouteMetadata getRouteMeta() { return routeMeta; } - + public TezEvent getTezEvent() { return event; } - + public int getNumEventsRouted() { return numEventsRouted; } @@ -572,11 +572,11 @@ public int getNumEventsRouted() { public PendingEventRouteMetadata removePendingEvents(TezTaskAttemptID attemptID) { return pendingEvents.remove(attemptID); } - + // return false is event could be routed but ran out of space in the list public boolean maybeAddTezEventForDestinationTask(TezEvent tezEvent, TezTaskAttemptID attemptID, - int srcTaskIndex, List listToAdd, int listMaxSize, - PendingEventRouteMetadata pendingRoutes) + int srcTaskIndex, List listToAdd, int listMaxSize, + PendingEventRouteMetadata pendingRoutes) throws AMUserCodeException { if (!routingNeeded) { if (LOG.isDebugEnabled()) { @@ -591,7 +591,7 @@ public boolean maybeAddTezEventForDestinationTask(TezEvent tezEvent, TezTaskAtte switch (tezEvent.getEventType()) { case COMPOSITE_DATA_MOVEMENT_EVENT: { - CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent(); + CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent(); CompositeEventRouteMetadata routeMeta = edgeManagerOnDemand .routeCompositeDataMovementEventToDestination(srcTaskIndex, taskIndex); @@ -686,7 +686,7 @@ public boolean maybeAddTezEventForDestinationTask(TezEvent tezEvent, TezTaskAtte private void sendEventToTask(Task task, TezEvent tezEvent) { task.registerTezEvent(tezEvent); } - + @SuppressWarnings({ "unchecked", "rawtypes" }) private void sendEvent(org.apache.hadoop.yarn.event.Event event) { eventHandler.handle(event); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexManager.java index 4bb5f483f3..4b45171b15 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexManager.java @@ -90,7 +90,7 @@ private void scheduleTasks() { if (!canScheduleTasks()) { return; } - + tasksScheduled = true; List tasksToStart = Lists.newArrayListWithCapacity(managedTasks); for (int i = 0; i < managedTasks; ++i) { @@ -119,7 +119,7 @@ private boolean canScheduleTasks() { return true; } - + @Override public void onVertexStateUpdated(VertexStateUpdate stateUpdate) { Preconditions.checkArgument(stateUpdate.getVertexState() == VertexState.CONFIGURED, diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OneToOneEdgeManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OneToOneEdgeManager.java index 61146ebf26..4fa704a491 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OneToOneEdgeManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OneToOneEdgeManager.java @@ -47,20 +47,20 @@ public void initialize() { public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { return 1; } - + @Override public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) { return 1; } - + @Override public void routeDataMovementEventToDestination(DataMovementEvent event, - int sourceTaskIndex, int sourceOutputIndex, + int sourceTaskIndex, int sourceOutputIndex, Map> destinationTaskAndInputIndices) { checkState(); destinationTaskAndInputIndices.put(sourceTaskIndex, destinationInputIndices); } - + @Override public void routeInputSourceTaskFailedEventToDestination(int sourceTaskIndex, Map> destinationTaskAndInputIndices) { @@ -72,7 +72,7 @@ public int routeInputErrorEventToSource(InputReadErrorEvent event, int destinationTaskIndex, int destinationFailedInputIndex) { return destinationTaskIndex; } - + @Override public int getNumDestinationConsumerTasks(int sourceTaskIndex) { return 1; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OneToOneEdgeManagerOnDemand.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OneToOneEdgeManagerOnDemand.java index ad8904f2b2..1d388df006 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OneToOneEdgeManagerOnDemand.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OneToOneEdgeManagerOnDemand.java @@ -35,8 +35,8 @@ public class OneToOneEdgeManagerOnDemand extends EdgeManagerPluginOnDemand { final List destinationInputIndices = Collections.singletonList(0); final AtomicBoolean stateChecked = new AtomicBoolean(false); - - final EventRouteMetadata commonRouteMeta = + + final EventRouteMetadata commonRouteMeta = EventRouteMetadata.create(1, new int[]{0}, new int[]{0}); final CompositeEventRouteMetadata compositeCommonRouteMeta = @@ -55,25 +55,25 @@ public void initialize() { public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { return 1; } - + @Override public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) { return 1; } - + @Override public void routeDataMovementEventToDestination(DataMovementEvent event, - int sourceTaskIndex, int sourceOutputIndex, + int sourceTaskIndex, int sourceOutputIndex, Map> destinationTaskAndInputIndices) { checkState(); destinationTaskAndInputIndices.put(sourceTaskIndex, destinationInputIndices); } - + @Override public void prepareForRouting() throws Exception { checkState(); } - + @Override public EventRouteMetadata routeDataMovementEventToDestination( int sourceTaskIndex, int sourceOutputIndex, int destinationTaskIndex) @@ -83,7 +83,7 @@ public EventRouteMetadata routeDataMovementEventToDestination( } return null; } - + @Override public @Nullable CompositeEventRouteMetadata routeCompositeDataMovementEventToDestination( int sourceTaskIndex, int destinationTaskIndex) @@ -111,7 +111,7 @@ public int routeInputErrorEventToSource(InputReadErrorEvent event, int destinationTaskIndex, int destinationFailedInputIndex) { return destinationTaskIndex; } - + @Override public int routeInputErrorEventToSource(int destinationTaskIndex, int destinationFailedInputIndex) { return destinationTaskIndex; @@ -121,7 +121,7 @@ public int routeInputErrorEventToSource(int destinationTaskIndex, int destinatio public int getNumDestinationConsumerTasks(int sourceTaskIndex) { return 1; } - + private void checkState() { if (stateChecked.get()) { return; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java index 06be989b9e..6a723ae247 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java @@ -86,7 +86,7 @@ public String getOutputName() { public UserPayload getOutputUserPayload() { return output.getIODescriptor().getUserPayload(); } - + @Override public UserPayload getUserPayload() { return output.getControllerDescriptor().getUserPayload(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java index c3a0ea866f..e33b171513 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java @@ -57,7 +57,7 @@ public class RootInputVertexManager extends VertexManagerPlugin { - private static final Logger LOG = + private static final Logger LOG = LoggerFactory.getLogger(RootInputVertexManager.class); /** @@ -88,7 +88,7 @@ public class RootInputVertexManager extends VertexManagerPlugin { "tez.root-input-vertex-manager.max-src-fraction"; public static final float TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT = 0.75f; - + private String configuredInputName; int totalNumSourceTasks = 0; @@ -319,7 +319,7 @@ public void onRootVertexInitialized(String inputName, InputDescriptor inputDescr + ", VertexName:" + getContext().getVertexName() + ", ConfiguredInput: " + configuredInputName + ", CurrentInput: " + inputName); configuredInputName = inputName; - + InputDataInformationEvent rEvent = (InputDataInformationEvent)event; rEvent.setTargetIndex(rEvent.getSourceIndex()); // 1:1 routing riEvents.add(rEvent); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java index ef2978b2df..4ae718c17e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java @@ -154,7 +154,7 @@ public static DataEventDependencyInfoProto toProto(DataEventDependencyInfo info) } return builder.build(); } - + public static DataEventDependencyInfo fromProto(DataEventDependencyInfoProto proto) { TezTaskAttemptID taId = null; if(proto.hasTaskAttemptId()) { @@ -200,11 +200,11 @@ public static DataEventDependencyInfo fromProto(DataEventDependencyInfoProto pro @VisibleForTesting boolean appendNextDataEvent = true; ArrayList lastDataEvents = Lists.newArrayList(); - + @VisibleForTesting TaskAttemptStatus reportedStatus; private DAGCounter localityCounter; - + org.apache.tez.runtime.api.impl.TaskStatistics statistics; long lastNotifyProgressTimestamp = 0; @@ -222,7 +222,7 @@ public static DataEventDependencyInfo fromProto(DataEventDependencyInfoProto pro private final Resource taskResource; private final ContainerContext containerContext; private final boolean leafVertex; - + private TezTaskAttemptID creationCausalTA; private long creationTime; private long scheduledTime; @@ -232,7 +232,7 @@ public static DataEventDependencyInfo fromProto(DataEventDependencyInfoProto pro protected static final KilledTransitionHelper KILLED_HELPER = new KilledTransitionHelper(); - + private static SingleArcTransition TERMINATED_AFTER_SUCCESS_HELPER = new TerminatedAfterSuccessHelper(KILLED_HELPER); @@ -256,7 +256,7 @@ public static DataEventDependencyInfo fromProto(DataEventDependencyInfoProto pro .addTransition(TaskAttemptStateInternal.NEW, EnumSet.of(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.START_WAIT, TaskAttemptStateInternal.FAILED), TaskAttemptEventType.TA_SCHEDULE, new ScheduleTaskattemptTransition()) - // NEW -> FAILED due to TA_FAILED happens in recovery + // NEW -> FAILED due to TA_FAILED happens in recovery // (No TaskAttemptStartedEvent, but with TaskAttemptFinishedEvent(FAILED) .addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.FAILED, @@ -266,13 +266,13 @@ TaskAttemptEventType.TA_FAILED, new TerminateTransition(FAILED_HELPER)) TaskAttemptEventType.TA_KILL_REQUEST, new TerminateTransition(KILLED_HELPER)) // NEW -> KILLED due to TA_KILLED happens in recovery - // (No TaskAttemptStartedEvent, but with TaskAttemptFinishedEvent(KILLED) + // (No TaskAttemptStartedEvent, but with TaskAttemptFinishedEvent(KILLED) .addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.KILLED, TaskAttemptEventType.TA_KILLED, new TerminateTransition(KILLED_HELPER)) // NEW -> SUCCEEDED due to TA_DONE happens in recovery - // (with TaskAttemptStartedEvent and with TaskAttemptFinishedEvent(SUCCEEDED) + // (with TaskAttemptStartedEvent and with TaskAttemptFinishedEvent(SUCCEEDED) .addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.SUCCEEDED, TaskAttemptEventType.TA_DONE, @@ -407,7 +407,7 @@ TaskAttemptEventType.TA_STARTED_REMOTELY, new StartedTransition()) TaskAttemptStateInternal.RUNNING), TaskAttemptEventType.TA_OUTPUT_FAILED, new OutputReportedFailedTransition()) - // for recovery, needs to log the TA generated events in TaskAttemptFinishedEvent + // for recovery, needs to log the TA generated events in TaskAttemptFinishedEvent .addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.RUNNING, TaskAttemptEventType.TA_TEZ_EVENT_UPDATE, @@ -574,7 +574,7 @@ public TaskAttemptImpl(TezTaskAttemptID attemptId, EventHandler eventHandler, this.containerContext = containerContext; this.leafVertex = leafVertex; this.hungIntervalMax = conf.getLong( - TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS, + TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS, TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS_DEFAULT); this.recoveryData = appContext.getDAGRecoveryData() == null ? null : appContext.getDAGRecoveryData().getTaskAttemptRecoveryData(attemptId); @@ -628,7 +628,7 @@ public List getDiagnostics() { readLock.unlock(); } } - + @Override public TaskAttemptTerminationCause getTerminationCause() { return terminationCause; @@ -659,7 +659,7 @@ public void setCounters(TezCounters counters) { writeLock.unlock(); } } - + TaskStatistics getStatistics() { return this.statistics; } @@ -713,7 +713,7 @@ public ContainerId getAssignedContainerID() { readLock.unlock(); } } - + @Override public Container getAssignedContainer() { readLock.lock(); @@ -806,7 +806,7 @@ public long getCreationTime() { readLock.unlock(); } } - + public TezTaskAttemptID getCreationCausalAttempt() { readLock.lock(); try { @@ -1118,7 +1118,7 @@ private void sendTaskAttemptCleanupEvent() { // TezMRTypeConverter.fromTez(this.attemptId)); // sendEvent(new TaskCleanupEvent(this.attemptId, this.committer, taContext)); } - + private TaskLocationHint getTaskLocationHint() { return locationHint; } @@ -1292,7 +1292,7 @@ public TaskAttemptStateInternal transition(TaskAttemptImpl ta, TaskAttemptEvent ta.sendEvent(new TaskAttemptEvent(ta.getTaskAttemptID(), TaskAttemptEventType.TA_DONE)); break; default: - throw new TezUncheckedException("Invalid state in TaskAttemptFinishedEvent, state=" + throw new TezUncheckedException("Invalid state in TaskAttemptFinishedEvent, state=" + taFinishedEvent.getState() + ", taId=" + ta.getTaskAttemptID()); } return TaskAttemptStateInternal.NEW; @@ -1331,7 +1331,7 @@ public TaskAttemptStateInternal transition(TaskAttemptImpl ta, TaskAttemptEvent } LOG.debug("Asking for container launch with taskAttemptContext: {}", ta.taskSpec); - + // Send out a launch request to the scheduler. int priority; if (ta.isRescheduled && ta.getVertex().getVertexConfig().getTaskRescheduleHigherPriority()) { @@ -1494,7 +1494,7 @@ public void transition(TaskAttemptImpl ta, TaskAttemptEvent taskAttemptEvent) { // only and moves the attempt to a final state, or an initial state. } } - + private boolean isSpeculationEnabled() { return conf.getBoolean(TezConfiguration.TEZ_AM_SPECULATION_ENABLED, TezConfiguration.TEZ_AM_SPECULATION_ENABLED_DEFAULT); @@ -1507,7 +1507,7 @@ public TerminatedBeforeRunningTransition( TerminatedTransitionHelper helper) { super(helper); } - + protected boolean sendSchedulerEvent() { return true; } @@ -1556,7 +1556,7 @@ protected static class ContainerCompletedBeforeRunningTransition extends public ContainerCompletedBeforeRunningTransition() { super(FAILED_HELPER); } - + public ContainerCompletedBeforeRunningTransition(TerminatedTransitionHelper helper) { super(helper); } @@ -1573,7 +1573,7 @@ protected static class StatusUpdaterTransition implements SingleArcTransition { @Override public void transition(TaskAttemptImpl ta, TaskAttemptEvent event) { - TaskAttemptEventStatusUpdate sEvent = (TaskAttemptEventStatusUpdate) event; + TaskAttemptEventStatusUpdate sEvent = (TaskAttemptEventStatusUpdate) event; TaskStatusUpdateEvent statusEvent = sEvent.getStatusEvent(); ta.reportedStatus.state = ta.getState(); ta.reportedStatus.progress = statusEvent.getProgress(); @@ -1590,7 +1590,7 @@ public void transition(TaskAttemptImpl ta, TaskAttemptEvent event) { if (ta.hungIntervalMax > 0 && ta.lastNotifyProgressTimestamp > 0 && currTime - ta.lastNotifyProgressTimestamp > ta.hungIntervalMax) { // task is hung - String diagnostics = "Attempt failed because it appears to make no progress for " + + String diagnostics = "Attempt failed because it appears to make no progress for " + ta.hungIntervalMax + "ms"; LOG.info(diagnostics + " " + ta.getTaskAttemptID()); // send event that will fail this attempt @@ -1598,12 +1598,12 @@ public void transition(TaskAttemptImpl ta, TaskAttemptEvent event) { new TaskAttemptEventAttemptFailed(ta.getTaskAttemptID(), TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, - diagnostics, + diagnostics, TaskAttemptTerminationCause.NO_PROGRESS) ); } } - + if (sEvent.getReadErrorReported()) { // if there is a read error then track the next last data event ta.appendNextDataEvent = true; @@ -1665,10 +1665,10 @@ public void transition(TaskAttemptImpl ta, TaskAttemptEvent event) { // Unregister from the TaskHeartbeatHandler. ta.taskHeartbeatHandler.unregister(ta.attemptId); - + ta.reportedStatus.state = TaskAttemptState.SUCCEEDED; ta.reportedStatus.progress = 1.0f; - + if (ta.isSpeculationEnabled()) { ta.sendEvent(new SpeculatorEventTaskAttemptStatusUpdate(ta.attemptId, TaskAttemptState.SUCCEEDED, ta.clock.getTime())); @@ -1705,7 +1705,7 @@ protected static class ContainerCompletedWhileRunningTransition extends public ContainerCompletedWhileRunningTransition() { super(FAILED_HELPER); } - + public ContainerCompletedWhileRunningTransition(TerminatedTransitionHelper helper) { super(helper); } @@ -1738,7 +1738,7 @@ protected boolean sendSchedulerEvent() { // there is no need to send it again return false; } - + public TerminatedAfterSuccessHelper(TerminatedTransitionHelper helper) { super(helper); } @@ -1767,14 +1767,14 @@ public TaskAttemptStateInternal transition(TaskAttemptImpl attempt, TaskAttemptE return TaskAttemptStateInternal.KILLED; } } - + protected static class OutputReportedFailedTransition implements MultipleArcTransition { @Override public TaskAttemptStateInternal transition(TaskAttemptImpl sourceAttempt, TaskAttemptEvent event) { - TaskAttemptEventOutputFailed outputFailedEvent = + TaskAttemptEventOutputFailed outputFailedEvent = (TaskAttemptEventOutputFailed) event; TezEvent inputFailedEvent = outputFailedEvent.getInputFailedEvent(); TezTaskAttemptID failedDestTaId = inputFailedEvent.getSourceInfo().getTaskAttemptID(); @@ -1897,7 +1897,7 @@ private int getNumNodes(TaskAttemptImpl sourceAttempt) { return numActiveNodes; } } - + @VisibleForTesting protected void sendInputFailedToConsumers() { Vertex vertex = getVertex(); @@ -1905,16 +1905,16 @@ protected void sendInputFailedToConsumers() { if (edges != null && !edges.isEmpty()) { List tezIfEvents = Lists.newArrayListWithCapacity(edges.size()); for (Vertex edgeVertex : edges.keySet()) { - tezIfEvents.add(new TezEvent(new InputFailedEvent(), - new EventMetaData(EventProducerConsumerType.SYSTEM, - vertex.getName(), + tezIfEvents.add(new TezEvent(new InputFailedEvent(), + new EventMetaData(EventProducerConsumerType.SYSTEM, + vertex.getName(), edgeVertex.getName(), getTaskAttemptID()), appContext.getClock().getTime())); } sendEvent(new VertexEventRouteEvent(vertex.getVertexId(), tezIfEvents)); } } - + private void trySetTerminationCause(TaskAttemptTerminationCause err) { // keep only the first error cause if (terminationCause == TaskAttemptTerminationCause.UNKNOWN_ERROR) { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImplHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImplHelpers.java index fba82613dd..e8c1eed3e3 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImplHelpers.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImplHelpers.java @@ -30,7 +30,7 @@ public final class TaskAttemptImplHelpers { private static final Logger LOG = LoggerFactory.getLogger(TaskAttemptImplHelpers.class); private TaskAttemptImplHelpers() {} - + static String[] resolveHosts(String[] src) { String[] result = new String[src.length]; for (int i = 0; i < src.length; i++) { @@ -57,7 +57,7 @@ static String resolveHost(String src) { private static final Pattern ipPattern = // Pattern for matching ip Pattern.compile("\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}"); - + static boolean isIP(String src) { return ipPattern.matcher(src).matches(); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java index 8dfd826e0b..0ad5206658 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java @@ -81,7 +81,7 @@ public String getInputName() { public UserPayload getInputUserPayload() { return this.input.getIODescriptor().getUserPayload(); } - + @Override public UserPayload getUserPayload() { return this.input.getControllerDescriptor().getUserPayload(); @@ -92,7 +92,7 @@ public Configuration getVertexConfiguration() { return vertex.getConf(); } - @Override + @Override public int getNumTasks() { return vertex.getTotalTasks(); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java index 1da67362bd..392739e292 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java @@ -238,7 +238,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl private final VertexRecoveryData recoveryData; private boolean isVertexInitSkipped = false; private List initGeneratedEvents = new ArrayList(); - // set it to be true when setParallelism is called(used for recovery) + // set it to be true when setParallelism is called(used for recovery) private boolean setParallelismCalledFlag = false; private boolean lazyTasksCopyNeeded = false; @@ -257,7 +257,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl // Useful when trying to serialize only the diff from global configs @VisibleForTesting Configuration vertexOnlyConf; - + private final boolean isSpeculationEnabled; @VisibleForTesting @@ -295,7 +295,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl private final List diagnostics = new ArrayList(); protected final StateChangeNotifier stateChangeNotifier; - + //task/attempt related datastructures @VisibleForTesting int numSuccessSourceAttemptCompletions = 0; @@ -509,7 +509,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl VertexState.COMMITTING, VertexState.TERMINATING, VertexEventType.V_TERMINATE, - new VertexKilledWhileCommittingTransition()) + new VertexKilledWhileCommittingTransition()) .addTransition( VertexState.COMMITTING, VertexState.ERROR, @@ -529,7 +529,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl EnumSet.of(VertexState.TERMINATING), VertexEventType.V_MANAGER_USER_CODE_ERROR, new VertexManagerUserCodeErrorTransition()) - + // Transitions from TERMINATING state. .addTransition (VertexState.TERMINATING, @@ -731,7 +731,7 @@ private void augmentStateMachine() { private final String vertexName; private final ProcessorDescriptor processorDescriptor; - + private boolean vertexToBeReconfiguredByManager = false; final AtomicBoolean vmIsInitialized = new AtomicBoolean(false); final AtomicBoolean completelyConfiguredSent = new AtomicBoolean(false); @@ -766,14 +766,14 @@ private void augmentStateMachine() { List pendingTaskEvents = Lists.newLinkedList(); private boolean tasksNotYetScheduled = true; // must be a random access structure - + private final List onDemandRouteEvents = Lists.newArrayListWithCapacity(1000); // Do not send any events if attempt is failed due to INPUT_FAILED_EVENTS. private final Set failedTaskAttemptIDs = Sets.newHashSet(); private final ReadWriteLock onDemandRouteEventsReadWriteLock = new ReentrantReadWriteLock(); private final Lock onDemandRouteEventsReadLock = onDemandRouteEventsReadWriteLock.readLock(); private final Lock onDemandRouteEventsWriteLock = onDemandRouteEventsReadWriteLock.writeLock(); - + List pendingRouteEvents = new LinkedList(); List pendingReportedSrcCompletions = Lists.newLinkedList(); @@ -823,15 +823,15 @@ static class EventInfo { private VertexStatisticsImpl finalStatistics; - - static class IOStatisticsImpl extends org.apache.tez.runtime.api.impl.IOStatistics + + static class IOStatisticsImpl extends org.apache.tez.runtime.api.impl.IOStatistics implements InputStatistics, OutputStatistics { - + @Override public long getDataSize() { return super.getDataSize(); } - + @Override public long getItemsProcessed() { return super.getItemsProcessed(); @@ -1080,7 +1080,7 @@ public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan, // Not sending the notifier a parallelism update since this is the initial parallelism this.dagVertexGroups = dagVertexGroups; - + isSpeculationEnabled = vertexConf.getBoolean(TezConfiguration.TEZ_AM_SPECULATION_ENABLED, TezConfiguration.TEZ_AM_SPECULATION_ENABLED_DEFAULT); @@ -1222,7 +1222,7 @@ public VertexPlan getVertexPlan() { public int getDistanceFromRoot() { return distanceFromRoot; } - + @Override public LinkedHashMap getIOIndices() { return ioIndices; @@ -1353,7 +1353,7 @@ public void addCounters(final TezCounters tezCounters) { @Override public int getMaxTaskConcurrency() { - return vertexConf.getInt(TezConfiguration.TEZ_AM_VERTEX_MAX_TASK_CONCURRENCY, + return vertexConf.getInt(TezConfiguration.TEZ_AM_VERTEX_MAX_TASK_CONCURRENCY, TezConfiguration.TEZ_AM_VERTEX_MAX_TASK_CONCURRENCY_DEFAULT); } @@ -1721,13 +1721,13 @@ public void scheduleSpeculativeTask(TezTaskID taskId) { readLock.unlock(); } } - + void setupEdgeRouting() throws AMUserCodeException { for (Edge e : sourceVertices.values()) { e.routingToBegin(); } } - + private void unsetTasksNotYetScheduled() throws AMUserCodeException { if (tasksNotYetScheduled) { setupEdgeRouting(); @@ -1738,8 +1738,8 @@ private void unsetTasksNotYetScheduled() throws AMUserCodeException { // only now can we be sure of the edge manager type. so until now // we will accumulate pending tasks in case legacy routing gets used. // this is only needed to support mixed mode routing. Else for - // on demand routing events can be directly added to taskEvents when - // they arrive in handleRoutedEvents instead of first caching them in + // on demand routing events can be directly added to taskEvents when + // they arrive in handleRoutedEvents instead of first caching them in // pendingTaskEvents. When legacy routing is removed then pendingTaskEvents // can be removed. if (!pendingTaskEvents.isEmpty()) { @@ -1762,11 +1762,11 @@ private void unsetTasksNotYetScheduled() throws AMUserCodeException { } } } - + TaskSpec createRemoteTaskSpec(int taskIndex) throws AMUserCodeException { return TaskSpec.createBaseTaskSpec(getDAG().getName(), getName(), getTotalTasks(), getProcessorDescriptor(), - getInputSpecList(taskIndex), getOutputSpecList(taskIndex), + getInputSpecList(taskIndex), getOutputSpecList(taskIndex), getGroupInputSpecList(), vertexOnlyConf); } @@ -1816,14 +1816,14 @@ public void scheduleTasks(List tasksToSchedule) { throw new TezUncheckedException(e); } } - + @Override public void reconfigureVertex(int parallelism, @Nullable VertexLocationHint locationHint, @Nullable Map sourceEdgeProperties) throws AMUserCodeException { setParallelismWrapper(parallelism, locationHint, sourceEdgeProperties, null, true); } - + @Override public void reconfigureVertex(@Nullable Map rootInputSpecUpdate, int parallelism, @@ -1838,11 +1838,11 @@ public void reconfigureVertex(int parallelism, @Nullable Map rootInputSpecUpdate) throws AMUserCodeException { setParallelismWrapper(parallelism, locationHint, sourceEdgeProperties, rootInputSpecUpdate, true); } - + @Override public void setParallelism(int parallelism, VertexLocationHint vertexLocationHint, Map sourceEdgeManagers, - Map rootInputSpecUpdates, boolean fromVertexManager) + Map rootInputSpecUpdates, boolean fromVertexManager) throws AMUserCodeException { // temporarily support conversion of edge manager to edge property Map sourceEdgeProperties = Maps.newHashMap(); @@ -1886,8 +1886,8 @@ private void setParallelismWrapper(int parallelism, VertexLocationHint vertexLoc } if (fromVertexManager && canInitVertex()) { - // vertex is fully defined. setParallelism has been called. VertexManager should have - // informed us about this. Otherwise we would have notified listeners that we are fully + // vertex is fully defined. setParallelism has been called. VertexManager should have + // informed us about this. Otherwise we would have notified listeners that we are fully // defined before we are actually fully defined Preconditions .checkState( @@ -1897,7 +1897,7 @@ private void setParallelismWrapper(int parallelism, VertexLocationHint vertexLoc + " context.vertexReconfigurationPlanned() before re-configuring the vertex." + " vertexId=" + logIdentifier); } - + // Input initializer/Vertex Manager/1-1 split expected to set parallelism. if (numTasks == -1) { if (getState() != VertexState.INITIALIZING) { @@ -1955,9 +1955,9 @@ private void setParallelismWrapper(int parallelism, VertexLocationHint vertexLoc // for a vertex to start. Preconditions.checkState(rootInputSpecUpdates == null, "Root Input specs can only be updated when the vertex is configured with -1 tasks"); - + int oldNumTasks = numTasks; - + // start buffering incoming events so that we can re-route existing events for (Edge edge : sourceVertices.values()) { edge.startEventBuffering(); @@ -1980,12 +1980,12 @@ private void setParallelismWrapper(int parallelism, VertexLocationHint vertexLoc } } Preconditions.checkState(this.numTasks == parallelism, getLogIdentifier()); - + // set new vertex location hints setVertexLocationHint(vertexLocationHint); LOG.info("Vertex " + getLogIdentifier() + " parallelism set to " + parallelism + " from " + oldNumTasks); - + // notify listeners stateChangeNotifier.stateChanged(vertexId, new VertexStateUpdateParallelismUpdated(vertexName, numTasks, oldNumTasks)); @@ -2029,7 +2029,7 @@ public void setVertexLocationHint(VertexLocationHint vertexLocationHint) { writeLock.unlock(); } } - + @Override public void vertexReconfigurationPlanned() { writeLock.lock(); @@ -2041,7 +2041,7 @@ public void vertexReconfigurationPlanned() { this.vertexToBeReconfiguredByManager = true; } finally { writeLock.unlock(); - } + } } @Override @@ -2057,10 +2057,10 @@ public void doneReconfiguringVertex() { Preconditions.checkState(getInternalState() == VertexState.INITIALIZING, "Vertex: " + getLogIdentifier()); } - + } finally { writeLock.unlock(); - } + } } @Override @@ -2275,7 +2275,7 @@ public Void run() throws Exception { return null; } }; - ListenableFuture commitFuture = + ListenableFuture commitFuture = vertex.getAppContext().getExecService().submit(commitCallableEvent); Futures.addCallback(commitFuture, commitCallableEvent.getCallback(), GuavaShim.directExecutor()); vertex.commitFutures.put(outputName, commitFuture); @@ -2429,7 +2429,7 @@ void tryEnactKill(VertexTerminationCause trigger, errCause = TaskAttemptTerminationCause.TERMINATED_BY_CLIENT; } if(trySetTerminationCause(trigger)){ - String msg = "Killing tasks in vertex: " + logIdentifier + " due to trigger: " + trigger; + String msg = "Killing tasks in vertex: " + logIdentifier + " due to trigger: " + trigger; LOG.info(msg); for (Task task : tasks.values()) { eventHandler.handle( // attempt was terminated because the vertex is shutting down @@ -2604,7 +2604,7 @@ private boolean initializeVertex() { } // set the vertex services to be initialized. initServices(); - // Only initialize committer when it is in non-recovery mode or vertex is not recovered to completed + // Only initialize committer when it is in non-recovery mode or vertex is not recovered to completed // state in recovery mode if (recoveryData == null || recoveryData.getVertexFinishedEvent() == null) { try { @@ -2667,7 +2667,7 @@ private TaskImpl createTask(int taskIndex) { this.stateChangeNotifier, this); } - + private void createTasks() { for (int i=0; i < this.numTasks; ++i) { TaskImpl task = createTask(i); @@ -2678,7 +2678,7 @@ private void createTasks() { } } } - + private void addTasks(int newNumTasks) { Preconditions.checkArgument(newNumTasks > this.numTasks, getLogIdentifier()); int initialNumTasks = this.numTasks; @@ -2692,7 +2692,7 @@ private void addTasks(int newNumTasks) { } } } - + private void removeTasks(int newNumTasks) { Preconditions.checkArgument(newNumTasks < this.numTasks, getLogIdentifier()); // assign to local variable of LinkedHashMap to make sure that changing @@ -2786,7 +2786,7 @@ private VertexState setupVertex() { } checkTaskLimits(); - // set VertexManager as the last step. Because in recovery case, we may need to restore + // set VertexManager as the last step. Because in recovery case, we may need to restore // some info from last the AM attempt and skip the initialization step. Otherwise numTasks may be // reset to -1 after the restore. try { @@ -2966,8 +2966,8 @@ private void assignVertexManager() throws TezException { } } } - - private static List getTaskAttemptIdentifiers(DAG dag, + + private static List getTaskAttemptIdentifiers(DAG dag, List taIds) { List attempts = new ArrayList(taIds.size()); String dagName = dag.getName(); @@ -2977,8 +2977,8 @@ private static List getTaskAttemptIdentifiers(DAG dag, } return attempts; } - - private static TaskAttemptIdentifier getTaskAttemptIdentifier(String dagName, String vertexName, + + private static TaskAttemptIdentifier getTaskAttemptIdentifier(String dagName, String vertexName, TezTaskAttemptID taId) { return new TaskAttemptIdentifierImpl(dagName, vertexName, taId); } @@ -3045,7 +3045,7 @@ public VertexState transition(VertexImpl vertex, VertexEvent vertexEvent) { return vertex.finished(recoverEvent.getDesiredState()); } } - + public static class InitTransition implements MultipleArcTransition { @@ -3258,8 +3258,8 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { vertex.rootInputInitializerManager.shutdown(); vertex.rootInputInitializerManager = null; } - - // the return of these events from the VM will complete initialization and move into + + // the return of these events from the VM will complete initialization and move into // INITED state if possible via InputDataInformationTransition return vertex.getState(); @@ -3383,7 +3383,7 @@ public void transition(VertexImpl vertex, VertexEvent event) { } } - + public static class StartTransition implements MultipleArcTransition { @@ -3398,7 +3398,7 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { return vertex.startVertex(); } } - + private void maybeSendConfiguredEvent() { // the vertex is fully configured by the time it starts. Always notify completely configured // unless the vertex manager has told us that it is going to reconfigure it further @@ -3435,16 +3435,16 @@ private VertexState startVertex() { } pendingReportedSrcCompletions.clear(); logJobHistoryVertexStartedEvent(); - + // the vertex is fully configured by the time it starts. Always notify completely configured // unless the vertex manager has told us that it is going to reconfigure it further. - // If the vertex was pre-configured then the event would have been sent out earlier. Calling again + // If the vertex was pre-configured then the event would have been sent out earlier. Calling again // would be a no-op. If the vertex was not fully configured and waiting for that to complete then - // we would start immediately after that. Either parallelism updated (now) or IPO changed (future) - // or vertex added (future). Simplify these cases by sending the event now automatically for the + // we would start immediately after that. Either parallelism updated (now) or IPO changed (future) + // or vertex added (future). Simplify these cases by sending the event now automatically for the // user as if they had invoked the planned()/done() API's. maybeSendConfiguredEvent(); - + // TODO: Metrics //job.metrics.runningJob(job); @@ -3464,7 +3464,7 @@ private VertexState startVertex() { eventHandler.handle(new VertexEvent( this.vertexId, VertexEventType.V_COMPLETED)); } - + return VertexState.RUNNING; } @@ -3649,7 +3649,7 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { AMUserCodeException e = errEvent.getError(); String msg = "Exception in " + e.getSource() + ", vertex:" + vertex.getLogIdentifier(); LOG.error(msg, e); - + if (vertex.getState() == VertexState.RUNNING || vertex.getState() == VertexState.COMMITTING) { vertex.addDiagnostic(msg + "," + ExceptionUtils.getStackTrace(e.getCause())); vertex.tryEnactKill(VertexTerminationCause.AM_USERCODE_FAILURE, @@ -3664,7 +3664,7 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { } } } - + /** * Here, the Vertex is being told that one of it's source task-attempts * completed. @@ -3690,7 +3690,7 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { // Inform the vertex manager about the source task completing. TezTaskAttemptID taId = completionEvent.getTaskAttemptId(); vertex.vertexManager.onSourceTaskCompleted( - getTaskAttemptIdentifier(vertex.dag.getName(), + getTaskAttemptIdentifier(vertex.dag.getName(), vertex.dag.getVertex(taId.getVertexID()).getName(), taId)); } catch (AMUserCodeException e) { @@ -3862,7 +3862,7 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { } } - private static class TaskRescheduledWhileCommittingTransition implements + private static class TaskRescheduledWhileCommittingTransition implements SingleArcTransition { @Override @@ -3997,7 +3997,7 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { return vertex.getState(); } } - + @Override public TaskAttemptEventInfo getTaskAttemptTezEvents(TezTaskAttemptID attemptID, int fromEventId, int preRoutedFromEventId, int maxEvents) { @@ -4183,7 +4183,7 @@ private void handleRoutedTezEvents(List tezEvents, boolean isPendingEv if (srcEdge.hasOnDemandRouting()) { processOnDemandEvent(tezEvent, srcEdge, srcTaskIndex); } else { - // send to tasks + // send to tasks srcEdge.sendTezEventToDestinationTasks(tezEvent); } } @@ -4191,14 +4191,14 @@ private void handleRoutedTezEvents(List tezEvents, boolean isPendingEv } break; case ROOT_INPUT_DATA_INFORMATION_EVENT: - { + { checkEventSourceMetadata(this, sourceMeta); if (tasksNotYetScheduled) { // this is only needed to support mixed mode routing. Else for // on demand routing events can be directly added to taskEvents // when legacy routing is removed then pending task events can be // removed. - pendingTaskEvents.add(tezEvent); + pendingTaskEvents.add(tezEvent); } else { InputDataInformationEvent riEvent = (InputDataInformationEvent) tezEvent.getEvent(); Task targetTask = getTask(riEvent.getTargetIndex()); @@ -4279,7 +4279,7 @@ private void handleRoutedTezEvents(List tezEvents, boolean isPendingEv } } } - + private void processOnDemandEvent(TezEvent tezEvent, Edge srcEdge, int srcTaskIndex) { onDemandRouteEventsWriteLock.lock(); try { @@ -4293,7 +4293,7 @@ private void processOnDemandEvent(TezEvent tezEvent, Edge srcEdge, int srcTaskIn onDemandRouteEvents.add(new EventInfo(tezEvent, srcEdge, srcTaskIndex)); if (tezEvent.getEventType() == EventType.INPUT_FAILED_EVENT) { for (EventInfo eventInfo : onDemandRouteEvents) { - if (eventInfo.eventEdge == srcEdge + if (eventInfo.eventEdge == srcEdge && eventInfo.tezEvent.getSourceInfo().getTaskAttemptID().equals( tezEvent.getSourceInfo().getTaskAttemptID()) && (eventInfo.tezEvent.getEventType() == EventType.DATA_MOVEMENT_EVENT @@ -4452,7 +4452,7 @@ public void setAdditionalInputs(List inputs) { this.rootInputSpecs.put(input.getName(), DEFAULT_ROOT_INPUT_SPECS); } } - + // not taking a lock by design. Speculator callbacks to the vertex will take locks if needed @Override public void handleSpeculatorEvent(SpeculatorEvent event) { @@ -4554,7 +4554,7 @@ public Map getInputVertices() { public Map getOutputVertices() { return Collections.unmodifiableMap(this.targetVertices); } - + @Override public VertexStatistics getStatistics() { readLock.lock(); @@ -4601,7 +4601,7 @@ public Resource getTaskResource() { readLock.unlock(); } } - + void addIO(String name) { ioIndices.put(StringInterner.intern(name), ioIndices.size()); } @@ -4800,7 +4800,7 @@ private static void logLocationHints(String vertexName, } /** - * This is for recovery when VertexReconfigureDoneEvent is seen. + * This is for recovery when VertexReconfigureDoneEvent is seen. */ public static class NoOpVertexManager extends VertexManagerPlugin { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java index 6f5d8dfc49..ad6b6ba484 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java @@ -100,7 +100,7 @@ public class VertexManager { final AppContext appContext; final BlockingQueue rootInputInitEventQueue; final StateChangeNotifier stateChangeNotifier; - + private final ListeningExecutorService execService; private final LinkedBlockingQueue eventQueue; private final AtomicBoolean eventInFlight; @@ -125,7 +125,7 @@ private void checkAndThrowIfDone() { throw new TezUncheckedException("Cannot invoke context methods after throwing an exception"); } } - + @Override public synchronized Map getInputVertexEdgeProperties() { checkAndThrowIfDone(); @@ -149,11 +149,11 @@ public synchronized Map getOutputVertexEdgeProperties() { } return vertexEdgeMap; } - + @Override public synchronized VertexStatistics getVertexStatistics(String vertexName) { checkAndThrowIfDone(); - return appContext.getCurrentDAG().getVertex(vertexName).getStatistics(); + return appContext.getCurrentDAG().getVertex(vertexName).getStatistics(); } @Override @@ -180,7 +180,7 @@ public synchronized void setVertexParallelism(int parallelism, VertexLocationHin throw new TezUncheckedException(e); } } - + @Override public synchronized void reconfigureVertex(int parallelism, VertexLocationHint vertexLocationHint, Map sourceEdgeProperties, @@ -193,7 +193,7 @@ public synchronized void reconfigureVertex(int parallelism, VertexLocationHint v throw new TezUncheckedException(e); } } - + @Override public synchronized void reconfigureVertex(int parallelism, @Nullable VertexLocationHint locationHint, @@ -205,7 +205,7 @@ public synchronized void reconfigureVertex(int parallelism, throw new TezUncheckedException(e); } } - + @Override public void reconfigureVertex(@Nullable Map rootInputSpecUpdate, @Nullable VertexLocationHint locationHint, @@ -223,7 +223,7 @@ public synchronized void scheduleTasks(List tasks) { checkAndThrowIfDone(); managedVertex.scheduleTasks(tasks); } - + @Override public synchronized void scheduleVertexTasks(List tasks) { checkAndThrowIfDone(); @@ -365,7 +365,7 @@ private void unregisterForVertexStateUpdates() { } } - + boolean isComplete() { return (isComplete.get() == true); } @@ -404,7 +404,7 @@ public Map> getInputVertexGroups() { @Override public void onStateUpdated(VertexStateUpdate event) { - // this is not called by the vertex manager plugin. + // this is not called by the vertex manager plugin. // no need to synchronize this. similar to other external notification methods enqueueAndScheduleNextEvent(new VertexManagerEventOnVertexStateUpdate(event)); } @@ -424,7 +424,7 @@ public VertexManager(VertexManagerPluginDescriptor pluginDesc, UserGroupInformat this.stateChangeNotifier = stateChangeNotifier; // don't specify the size of rootInputInitEventQueue, otherwise it will fail when addAll this.rootInputInitEventQueue = new LinkedBlockingQueue(); - + pluginContext = new VertexManagerPluginContextImpl(); payload = pluginDesc.getUserPayload(); pluginFailed = new AtomicBoolean(false); @@ -451,7 +451,7 @@ public void initialize() throws AMUserCodeException { throw new AMUserCodeException(Source.VertexManager, e); } } - + private boolean pluginInvocationAllowed(String msg) { if (pluginFailed.get()) { if (LOG.isDebugEnabled()) { @@ -467,7 +467,7 @@ private boolean pluginInvocationAllowed(String msg) { } return true; } - + private void enqueueAndScheduleNextEvent(VertexManagerEvent e) { if (!pluginInvocationAllowed("Dropping event")) { return; @@ -475,7 +475,7 @@ private void enqueueAndScheduleNextEvent(VertexManagerEvent e) { eventQueue.add(e); tryScheduleNextEvent(); } - + private void tryScheduleNextEvent() { if (!pluginInvocationAllowed("Not scheduling")) { return; @@ -491,10 +491,10 @@ private void tryScheduleNextEvent() { ListenableFuture future = execService.submit(e); Futures.addCallback(future, e.getCallback(), GuavaShim.directExecutor()); } else { - // This may happen. Lets say Callback succeeded on threadA. It set eventInFlight to false - // and called tryScheduleNextEvent() and found queue not empty but got paused before it - // could check eventInFlight.compareAndSet(). Another thread managed to dequeue the event - // and schedule a callback. That callback succeeded and set eventInFlight to false, found + // This may happen. Lets say Callback succeeded on threadA. It set eventInFlight to false + // and called tryScheduleNextEvent() and found queue not empty but got paused before it + // could check eventInFlight.compareAndSet(). Another thread managed to dequeue the event + // and schedule a callback. That callback succeeded and set eventInFlight to false, found // the queue empty and completed. Now threadA woke up and successfully did compareAndSet() // tried to dequeue an event and got null. // This could also happen if there is a bug and we manage to schedule for than 1 callback @@ -552,7 +552,7 @@ public void onFailure(Throwable t) { sendInternalError(e); } } - + @Override public void onSuccess(Void result) { try { @@ -563,10 +563,10 @@ public void onSuccess(Void result) { sendInternalError(e); } } - + protected void onSuccessDerived(Void result) { } - + private void sendInternalError(Exception e) { // fail the DAG so that we dont hang // state change must be triggered via an event transition @@ -577,7 +577,7 @@ private void sendInternalError(Exception e) { + ", error=" + ExceptionUtils.getStackTrace(e)))); } } - + private class VertexManagerRootInputInitializedCallback extends VertexManagerCallback { @Override @@ -593,10 +593,10 @@ protected void onSuccessDerived(Void result) { new VertexEventInputDataInformation(managedVertex.getVertexId(), resultEvents)); } } - + class VertexManagerEventOnVertexStateUpdate extends VertexManagerEvent { private final VertexStateUpdate event; - + public VertexManagerEventOnVertexStateUpdate(VertexStateUpdate event) { this.event = event; } @@ -605,56 +605,56 @@ public VertexManagerEventOnVertexStateUpdate(VertexStateUpdate event) { public void invoke() throws Exception { plugin.onVertexStateUpdated(event); } - + } - + class VertexManagerEventOnVertexStarted extends VertexManagerEvent { private final List pluginCompletions; public VertexManagerEventOnVertexStarted(List pluginCompletions) { this.pluginCompletions = pluginCompletions; } - + @Override public void invoke() throws Exception { plugin.onVertexStarted(pluginCompletions); } - + } - + class VertexManagerEventSourceTaskCompleted extends VertexManagerEvent { private final TaskAttemptIdentifier attempt; - + public VertexManagerEventSourceTaskCompleted(TaskAttemptIdentifier attempt) { this.attempt = attempt; } - + @Override public void invoke() throws Exception { - plugin.onSourceTaskCompleted(attempt); + plugin.onSourceTaskCompleted(attempt); } - + } - + class VertexManagerEventReceived extends VertexManagerEvent { private final org.apache.tez.runtime.api.events.VertexManagerEvent vmEvent; - + public VertexManagerEventReceived(org.apache.tez.runtime.api.events.VertexManagerEvent vmEvent) { this.vmEvent = vmEvent; } - + @Override public void invoke() throws Exception { plugin.onVertexManagerEventReceived(vmEvent); } - + } - + class VertexManagerEventRootInputInitialized extends VertexManagerEvent { private final String inputName; private final InputDescriptor inputDescriptor; private final List events; - + public VertexManagerEventRootInputInitialized(String inputName, InputDescriptor inputDescriptor, List events) { super(new VertexManagerRootInputInitializedCallback()); @@ -669,7 +669,7 @@ public void invoke() throws Exception { } } - + abstract class VertexManagerEvent extends CallableEvent { public VertexManagerEvent() { this(VM_CALLBACK); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/LegacySpeculator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/LegacySpeculator.java index b2f9d0aab8..e578cd1fd5 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/LegacySpeculator.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/LegacySpeculator.java @@ -51,17 +51,17 @@ /** * Maintains runtime estimation statistics. Makes periodic updates - * estimates based on progress and decides on when to trigger a - * speculative attempt. Speculation attempts are triggered when the + * estimates based on progress and decides on when to trigger a + * speculative attempt. Speculation attempts are triggered when the * estimated runtime is more than a threshold beyond the mean runtime - * and the original task still has enough estimated runtime left that - * the speculative version is expected to finish sooner than that. If + * and the original task still has enough estimated runtime left that + * the speculative version is expected to finish sooner than that. If * the original is close to completion then we dont start a speculation * because it may be likely a wasted attempt. There is a delay between * successive speculations. */ public class LegacySpeculator extends AbstractService { - + private static final long ON_SCHEDULE = Long.MIN_VALUE; private static final long ALREADY_SPECULATING = Long.MIN_VALUE + 1; private static final long TOO_NEW = Long.MIN_VALUE + 2; @@ -123,7 +123,7 @@ public LegacySpeculator(Configuration conf, AppContext context, Vertex vertex) { public LegacySpeculator(Configuration conf, Clock clock, Vertex vertex) { this(conf, getEstimator(conf, vertex), clock, vertex); } - + static private TaskRuntimeEstimator getEstimator (Configuration conf, Vertex vertex) { TaskRuntimeEstimator estimator; @@ -261,7 +261,7 @@ public void run() { /* ************************************************************* */ public void notifyAttemptStarted(TezTaskAttemptID taId, long timestamp) { - estimator.enrollAttempt(taId, timestamp); + estimator.enrollAttempt(taId, timestamp); } public void notifyAttemptStatusUpdate(TezTaskAttemptID taId, @@ -299,7 +299,7 @@ private void statusUpdate(TezTaskAttemptID attemptID, } } } - + public void handle(SpeculatorEvent event) { SpeculatorEventTaskAttemptStatusUpdate updateEvent = ((SpeculatorEventTaskAttemptStatusUpdate) event); if (updateEvent.hasJustStarted()) { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/LegacyTaskRuntimeEstimator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/LegacyTaskRuntimeEstimator.java index 14d269cc33..c41419dbd5 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/LegacyTaskRuntimeEstimator.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/LegacyTaskRuntimeEstimator.java @@ -29,7 +29,7 @@ /** * Runtime estimator that uses a simple scheme of estimating task attempt - * runtime based on current elapsed runtime and reported progress. + * runtime based on current elapsed runtime and reported progress. */ public class LegacyTaskRuntimeEstimator extends StartEndTimesBase { @@ -41,7 +41,7 @@ public class LegacyTaskRuntimeEstimator extends StartEndTimesBase { @Override public void updateAttempt(TezTaskAttemptID attemptID, TaskAttemptState state, long timestamp) { super.updateAttempt(attemptID, state, timestamp); - + Task task = vertex.getTask(attemptID.getTaskID()); @@ -54,7 +54,7 @@ public void updateAttempt(TezTaskAttemptID attemptID, TaskAttemptState state, lo if (taskAttempt == null) { return; } - + float progress = taskAttempt.getProgress(); Long boxedStart = startTimes.get(attemptID); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/TaskRuntimeEstimator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/TaskRuntimeEstimator.java index 4f747afc4e..6b3870babe 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/TaskRuntimeEstimator.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/TaskRuntimeEstimator.java @@ -26,7 +26,7 @@ /** * Estimate the runtime for tasks of a given vertex. - * + * */ public interface TaskRuntimeEstimator { void enrollAttempt(TezTaskAttemptID id, long timestamp); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEvent.java index dd9d951218..ce9eb33f06 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEvent.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventDeallocateContainer.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventDeallocateContainer.java index 4c9a42d1c0..89cb45ac0c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventDeallocateContainer.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventDeallocateContainer.java @@ -28,7 +28,7 @@ public AMSchedulerEventDeallocateContainer(ContainerId containerId, int schedule super(AMSchedulerEventType.S_CONTAINER_DEALLOCATE, schedulerId); this.containerId = containerId; } - + public ContainerId getContainerId() { return this.containerId; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java index 23a8fb93b0..1f18fa49eb 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java index 0424c97eaf..04007ec4fd 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java @@ -66,7 +66,7 @@ public TezTaskAttemptID getAttemptID() { public Resource getCapability() { return capability; } - + public TaskLocationHint getLocationHint() { return locationHint; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventType.java index cc52ef6bce..bedb702197 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventType.java @@ -31,7 +31,7 @@ public enum AMSchedulerEventType { S_NODE_HEALTHY, // The scheduler should have a way of knowing about unusable nodes. Acting on // this information to change requests etc is scheduler specific. - + // Producer : AMContainer S_CONTAINER_DEALLOCATE } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java index c9c6178732..4d3ac9b1dd 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java @@ -499,9 +499,9 @@ private void handleTaLaunchRequest(AMSchedulerEventTALaunchRequest event) { Vertex vertex = appContext.getCurrentDAG().getVertex(taskAffinity.getVertexName()); Objects.requireNonNull(vertex, "Invalid vertex in task based affinity " + taskAffinity + " for attempt: " + taskAttempt.getTaskAttemptID()); - int taskIndex = taskAffinity.getTaskIndex(); - Preconditions.checkState(taskIndex >=0 && taskIndex < vertex.getTotalTasks(), - "Invalid taskIndex in task based affinity " + taskAffinity + int taskIndex = taskAffinity.getTaskIndex(); + Preconditions.checkState(taskIndex >=0 && taskIndex < vertex.getTotalTasks(), + "Invalid taskIndex in task based affinity " + taskAffinity + " for attempt: " + taskAttempt.getTaskAttemptID()); TaskAttempt affinityAttempt = vertex.getTask(taskIndex).getSuccessfulAttempt(); if (affinityAttempt != null) { @@ -653,7 +653,7 @@ protected void instantiateSchedulers(String host, int port, String trackingUrl, } } - + @Override public synchronized void serviceStart() throws Exception { // clientService is null in case of LocalDAGAppMaster @@ -712,7 +712,7 @@ public void run() { }; this.eventHandlingThread.start(); } - + protected void notifyForTest() { } @@ -834,7 +834,7 @@ public synchronized void appShutdownRequested(int schedulerId) { public synchronized void setApplicationRegistrationData( int schedulerId, Resource maxContainerCapability, - Map appAcls, + Map appAcls, ByteBuffer clientAMSecretKey, String queueName) { this.appContext.getClusterInfo().setMaxContainerCapability( diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java index 525c9e01e1..2052efbaf8 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java @@ -73,19 +73,19 @@ public TezAMRMClientAsync( AMRMClientAsync.CallbackHandler callbackHandler) { super(client, intervalMs, callbackHandler); } - + public synchronized Priority getTopPriority() { if (knownRequestsByPriority.isEmpty()) { return null; } return knownRequestsByPriority.lastKey(); } - + // Remove after YARN-1723 is fixed public synchronized void addNodeToBlacklist(NodeId nodeId) { client.updateBlacklist(Collections.singletonList(nodeId.getHost()), null); } - + //Remove after YARN-1723 is fixed public synchronized void removeNodeFromBlacklist(NodeId nodeId) { client.updateBlacklist(null, Collections.singletonList(nodeId.getHost())); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java index 3495bb3355..01ce0d5ea5 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java @@ -110,7 +110,7 @@ public class YarnTaskSchedulerService extends TaskScheduler Map containerAssignments = new HashMap(); // Remove inUse depending on resolution of TEZ-1129 - Set inUseContainers = Sets.newHashSet(); + Set inUseContainers = Sets.newHashSet(); HashMap releasedContainers = new HashMap(); /** @@ -118,18 +118,18 @@ public class YarnTaskSchedulerService extends TaskScheduler */ Map heldContainers = new HashMap(); - + Set priorityHasAffinity = Sets.newHashSet(); Set blacklistedNodes = Collections .newSetFromMap(new ConcurrentHashMap()); - + Resource totalResources = Resource.newInstance(0, 0); Resource allocatedResources = Resource.newInstance(0, 0); long numHeartbeats = 0; long heartbeatAtLastPreemption = 0; int numHeartbeatsBetweenPreemptions = 0; - + final String appHostName; final int appHostPort; final String appTrackingUrl; @@ -148,12 +148,12 @@ public class YarnTaskSchedulerService extends TaskScheduler int sessionNumMinHeldContainers = 0; int preemptionPercentage = 0; long preemptionMaxWaitTime = 0; - + long highestWaitingRequestWaitStartTime = 0; Priority highestWaitingRequestPriority = null; - + Set sessionMinHeldContainers = Sets.newHashSet(); - + private final Configuration conf; @VisibleForTesting @@ -164,7 +164,7 @@ static class CRCookie { private Object task; private Object appCookie; private Object containerSignature; - + CRCookie(Object task, Object appCookie, Object containerSignature) { this.task = task; this.appCookie = appCookie; @@ -178,7 +178,7 @@ Object getTask() { Object getAppCookie() { return appCookie; } - + Object getContainerSignature() { return containerSignature; } @@ -212,7 +212,7 @@ public CookieContainerRequest( CRCookie getCookie() { return cookie; } - + ContainerId getAffinitizedContainer() { return affinitizedContainerId; } @@ -312,32 +312,32 @@ public synchronized void initialize() { TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS_DEFAULT); Preconditions.checkArgument(idleContainerTimeoutMin >= 0 || idleContainerTimeoutMin == -1, "Idle container release min timeout should be either -1 or >=0"); - + idleContainerTimeoutMax = conf.getLong( TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS_DEFAULT); Preconditions.checkArgument( idleContainerTimeoutMax >= 0 && idleContainerTimeoutMax >= idleContainerTimeoutMin, - "Idle container release max timeout should be >=0 and >= " + + "Idle container release max timeout should be >=0 and >= " + TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS); - - sessionNumMinHeldContainers = conf.getInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, + + sessionNumMinHeldContainers = conf.getInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT); - Preconditions.checkArgument(sessionNumMinHeldContainers >= 0, + Preconditions.checkArgument(sessionNumMinHeldContainers >= 0, "Session minimum held containers should be >=0"); - - preemptionPercentage = conf.getInt(TezConfiguration.TEZ_AM_PREEMPTION_PERCENTAGE, + + preemptionPercentage = conf.getInt(TezConfiguration.TEZ_AM_PREEMPTION_PERCENTAGE, TezConfiguration.TEZ_AM_PREEMPTION_PERCENTAGE_DEFAULT); Preconditions.checkArgument(preemptionPercentage >= 0 && preemptionPercentage <= 100, "Preemption percentage should be between 0-100"); - + numHeartbeatsBetweenPreemptions = conf.getInt( TezConfiguration.TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS, TezConfiguration.TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS_DEFAULT); - Preconditions.checkArgument(numHeartbeatsBetweenPreemptions >= 1, + Preconditions.checkArgument(numHeartbeatsBetweenPreemptions >= 1, "Heartbeats between preemptions should be >=1"); - - preemptionMaxWaitTime = conf.getInt(TezConfiguration.TEZ_AM_PREEMPTION_MAX_WAIT_TIME_MS, + + preemptionMaxWaitTime = conf.getInt(TezConfiguration.TEZ_AM_PREEMPTION_MAX_WAIT_TIME_MS, TezConfiguration.TEZ_AM_PREEMPTION_MAX_WAIT_TIME_MS_DEFAULT); Preconditions.checkArgument(preemptionMaxWaitTime >=0, "Preemption max wait time must be >=0"); @@ -349,7 +349,7 @@ public synchronized void initialize() { "maxRMHeartbeatInterval: " + heartbeatIntervalMax + ", containerReuseEnabled: " + shouldReuseContainers + ", reuseRackLocal: " + reuseRackLocal + - ", reuseNonLocal: " + reuseNonLocal + + ", reuseNonLocal: " + reuseNonLocal + ", localitySchedulingDelay: " + localitySchedulingDelay + ", preemptionPercentage: " + preemptionPercentage + ", preemptionMaxWaitTime: " + preemptionMaxWaitTime + @@ -495,7 +495,7 @@ public void onContainersAllocated(List containers) { super.onContainersAllocated(containers); if (isStopStarted.get()) { - LOG.info("Ignoring container allocations because application is shutting down. Num " + + LOG.info("Ignoring container allocations because application is shutting down. Num " + containers.size()); if (LOG.isDebugEnabled()) { for (Container container : containers) { @@ -587,7 +587,7 @@ public void onContainersAllocated(List containers) { return assignedContainers; } - + @VisibleForTesting long getHeldContainerExpireTime(long startTime) { // expire time is at least extended by min time. @@ -598,7 +598,7 @@ long getHeldContainerExpireTime(long startTime) { long expireTimeMax = startTime + idleContainerTimeoutMax; expireTime = ThreadLocalRandom.current().nextLong(expireTime, expireTimeMax); } - + return expireTime; } @@ -638,28 +638,28 @@ long getHeldContainerExpireTime(long startTime) { // session mode and need to hold onto containers and not done so already determineMinHeldContainers(); } - + heldContainer.resetLocalityMatchLevel(); long currentTime = System.currentTimeMillis(); boolean releaseContainer = false; if (isNew || (heldContainer.getContainerExpiryTime() - currentTime <= 0 && idleContainerTimeoutMin != -1)) { - // container idle timeout has expired or is a new unused container. + // container idle timeout has expired or is a new unused container. // new container is possibly a spurious race condition allocation. if (getContext().isSession() && sessionMinHeldContainers.contains(heldContainer.getContainer().getId())) { // There are no outstanding requests. So its safe to hold new containers. // We may have received more containers than necessary and some are unused // In session mode and container in set of chosen min held containers - // increase the idle container expire time to maintain sanity with + // increase the idle container expire time to maintain sanity with // the rest of the code. heldContainer.setContainerExpiryTime(getHeldContainerExpireTime(currentTime)); } else { - releaseContainer = true; + releaseContainer = true; } } - + if (releaseContainer) { LOG.info("No taskRequests. Container's idle timeout delay expired or is new. " + "Releasing container" @@ -672,7 +672,7 @@ long getHeldContainerExpireTime(long startTime) { + ", delayedContainers=" + delayedContainerManager.delayedContainers.size() + ", isNew=" + isNew); releaseUnassignedContainers( - Collections.singletonList((heldContainer.getContainer()))); + Collections.singletonList((heldContainer.getContainer()))); } else { // no outstanding work and container idle timeout not expired if (LOG.isDebugEnabled()) { @@ -692,7 +692,7 @@ long getHeldContainerExpireTime(long startTime) { // clear min held containers since we need to allocate to tasks if (!sessionMinHeldContainers.isEmpty()) { // update the expire time of min held containers so that they are - // not released immediately, when new requests come in, if they come in + // not released immediately, when new requests come in, if they come in // just before these containers are about to expire (race condition) long currentTime = System.currentTimeMillis(); for (ContainerId minHeldCId : sessionMinHeldContainers) { @@ -770,11 +770,11 @@ long getHeldContainerExpireTime(long startTime) { // Release container if final expiry time is reached // Dont release a new container. The RM may not give us new ones // The assumption is that the expire time is larger than the sum of all - // locality delays. So if we hit the expire time then we have already + // locality delays. So if we hit the expire time then we have already // tried to assign at all locality levels. - // We run the risk of not being able to retain min held containers but - // if we are not being able to assign containers to pending tasks then - // we cannot avoid releasing containers. Or else we may not be able to + // We run the risk of not being able to retain min held containers but + // if we are not being able to assign containers to pending tasks then + // we cannot avoid releasing containers. Or else we may not be able to // get new containers from YARN to match the pending request if (!isNew && heldContainer.getContainerExpiryTime() - currentTime <= 0 && idleContainerTimeoutMin != -1) { @@ -810,7 +810,7 @@ long getHeldContainerExpireTime(long startTime) { hitFinalMatchLevel = false; } } - + if (hitFinalMatchLevel) { boolean safeToRelease = true; Priority topPendingPriority = amRmClient.getTopPriority(); @@ -818,15 +818,15 @@ long getHeldContainerExpireTime(long startTime) { if (isNew && topPendingPriority != null && containerPriority.compareTo(topPendingPriority) < 0) { // this container is of lower priority and given to us by the RM for - // a task that will be matched after the current top priority. Keep + // a task that will be matched after the current top priority. Keep // this container for those pending tasks since the RM is not going // to give this container to us again safeToRelease = false; } - + // Are there any pending requests at any priority? // release if there are tasks or this is not a session - if (safeToRelease && + if (safeToRelease && (!taskRequests.isEmpty() || !getContext().isSession())) { LOG.info("Releasing held container as either there are pending but " + " unmatched requests or this is not a session" @@ -944,7 +944,7 @@ public synchronized void blacklistNode(NodeId nodeId) { amRmClient.addNodeToBlacklist(nodeId); blacklistedNodes.add(nodeId); } - + @Override public synchronized void unblacklistNode(NodeId nodeId) { if (blacklistedNodes.remove(nodeId)) { @@ -952,7 +952,7 @@ public synchronized void unblacklistNode(NodeId nodeId) { amRmClient.removeNodeFromBlacklist(nodeId); } } - + @Override public synchronized void allocateTask( Object task, @@ -972,7 +972,7 @@ public synchronized void allocateTask( addRequestAndTrigger(task, request, hosts, racks); } - + @Override public synchronized void allocateTask( Object task, @@ -994,20 +994,20 @@ public synchronized void allocateTask( priorityHasAffinity.add(priority); } else { LOG.warn("Matching requested to container: " + containerId + - " but requested capability: " + capability + + " but requested capability: " + capability + " does not fit in container resource: " + container.getResource()); } } else { LOG.warn("Matching requested to unknown container: " + containerId); } - + CRCookie cookie = new CRCookie(task, clientCookie, containerSignature); CookieContainerRequest request = new CookieContainerRequest( capability, containerId, hosts, racks, priority, cookie); addRequestAndTrigger(task, request, hosts, racks); } - + private void addRequestAndTrigger(Object task, CookieContainerRequest request, String[] hosts, String[] racks) { addTaskRequest(task, request); @@ -1086,7 +1086,7 @@ public boolean deallocateTask(Object task, boolean taskSucceeded, } return true; } - + @Override public synchronized Object deallocateContainer(ContainerId containerId) { Object task = unAssignContainer(containerId, true); @@ -1134,34 +1134,34 @@ boolean canFit(Resource arg0, Resource arg1) { int mem1 = arg1.getMemory(); int cpu0 = arg0.getVirtualCores(); int cpu1 = arg1.getVirtualCores(); - - if(mem0 <= mem1 && cpu0 <= cpu1) { + + if(mem0 <= mem1 && cpu0 <= cpu1) { return true; } - return false; + return false; } static int scaleDownByPreemptionPercentage(int original, int percent) { return (int) Math.ceil((original * percent)/100.f); } - + private String constructPreemptionPeriodicLog(Resource freeResource) { return "Allocated: " + allocatedResources + " Free: " + freeResource + " pendingRequests: " + taskRequests.size() + " delayedContainers: " + delayedContainerManager.delayedContainers.size() + - " heartbeats: " + numHeartbeats + + " heartbeats: " + numHeartbeats + " lastPreemptionHeartbeat: " + heartbeatAtLastPreemption + - ((highestWaitingRequestPriority != null) ? + ((highestWaitingRequestPriority != null) ? (" highestWaitingRequestWaitStartTime: " + highestWaitingRequestWaitStartTime + " highestWaitingRequestPriority: " + highestWaitingRequestPriority.toString()) : ""); } - + private void resetHighestWaitingPriority(Priority newPri) { highestWaitingRequestPriority = newPri; highestWaitingRequestWaitStartTime = 0; } - + boolean preemptIfNeeded() { if (preemptionPercentage == 0) { // turned off @@ -1179,7 +1179,7 @@ boolean preemptIfNeeded() { } } assert freeResources.getMemory() >= 0; - + CookieContainerRequest highestPriRequest = null; int numHighestPriRequests = 0; for(CookieContainerRequest request : taskRequests.values()) { @@ -1194,28 +1194,28 @@ boolean preemptIfNeeded() { numHighestPriRequests++; } } - + if (highestPriRequest == null) { // nothing pending resetHighestWaitingPriority(null); return true; } - + // reset the wait time when waiting priority changes to prevent carry over of the value if (highestWaitingRequestPriority == null || !highestPriRequest.getPriority().equals(highestWaitingRequestPriority)) { resetHighestWaitingPriority(highestPriRequest.getPriority()); } - + long currTime = System.currentTimeMillis(); if (highestWaitingRequestWaitStartTime == 0) { highestWaitingRequestWaitStartTime = currTime; } - boolean preemptionWaitDeadlineCrossed = + boolean preemptionWaitDeadlineCrossed = (currTime - highestWaitingRequestWaitStartTime) > preemptionMaxWaitTime ? true : false; - if(!preemptionWaitDeadlineCrossed && + if(!preemptionWaitDeadlineCrossed && fitsIn(highestPriRequest.getCapability(), freeResources)) { LOG.debug("{} fits in free resources", highestPriRequest); if (numHeartbeats % 50 == 1) { @@ -1223,7 +1223,7 @@ boolean preemptIfNeeded() { } return true; } - + if (preemptionWaitDeadlineCrossed) { // check if anything lower priority is running - priority inversion // this check could have been done earlier but in the common case @@ -1258,11 +1258,11 @@ boolean preemptIfNeeded() { + " numRequests: " + numHighestPriRequests + ". " + constructPreemptionPeriodicLog(freeResources)); } - + // highest priority request will not fit in existing free resources // free up some more // TODO this is subject to error wrt RM resource normalization - + numPendingRequestsToService = scaleDownByPreemptionPercentage(numHighestPriRequests, preemptionPercentage); @@ -1278,10 +1278,10 @@ boolean preemptIfNeeded() { } int newContainersReleased = 0; for (int i=0; i entry : taskAllocations.entrySet()) { @@ -1403,7 +1403,7 @@ boolean preemptIfNeeded() { // and we get its completed container status } } - + // upcall outside locks if (preemptedContainers != null) { for(int i=0; i> pRequestsList = amRmClient.getMatchingRequestsForTopPriority(location, capability); - if (considerContainerAffinity && + if (considerContainerAffinity && !priorityHasAffinity.contains(amRmClient.getTopPriority())) { considerContainerAffinity = false; } @@ -1482,9 +1482,9 @@ private CookieContainerRequest getMatchingRequestWithoutPriority( CookieContainerRequest firstMatch = null; for (Collection requests : pRequestsList) { for (CookieContainerRequest cookieContainerRequest : requests) { - if (firstMatch == null || // we dont have a match. So look for one + if (firstMatch == null || // we dont have a match. So look for one // we have a match but are looking for a better container level match. - // skip the expensive canAssignTaskToContainer() if the request is + // skip the expensive canAssignTaskToContainer() if the request is // not affinitized to the container container.getId().equals(cookieContainerRequest.getAffinitizedContainer()) ) { @@ -1495,7 +1495,7 @@ private CookieContainerRequest getMatchingRequestWithoutPriority( } ContainerId affCId = cookieContainerRequest.getAffinitizedContainer(); boolean canMatchTaskWithAffinity = true; - if (affCId == null || + if (affCId == null || !heldContainers.containsKey(affCId) || inUseContainers.contains(affCId)) { // affinity not specified @@ -1523,7 +1523,7 @@ private CookieContainerRequest getMatchingRequestWithoutPriority( } } } - + return firstMatch; } @@ -1599,7 +1599,7 @@ private void assignContainer(Object task, assert result == null; inUseContainers.add(container.getId()); containerAssignments.put(container.getId(), task); - HeldContainer heldContainer = heldContainers.get(container.getId()); + HeldContainer heldContainer = heldContainers.get(container.getId()); if (!shouldReuseContainers && heldContainer == null) { heldContainers.put(container.getId(), new HeldContainer(container, -1, -1, assigned, this.containerSignatureMatcher)); @@ -1619,7 +1619,7 @@ private void assignContainer(Object task, heldContainer.setLastTaskInfo(assigned); } } - + private void pushNewContainerToDelayed(List containers){ long expireTime = getHeldContainerExpireTime(System.currentTimeMillis()); @@ -1639,7 +1639,7 @@ private void pushNewContainerToDelayed(List containers){ nextScheduleTime + 1); } } - delayedContainerManager.triggerScheduling(false); + delayedContainerManager.triggerScheduling(false); } private CookieContainerRequest removeTaskRequest(Object task) { @@ -1736,28 +1736,28 @@ private synchronized boolean assignReUsedContainerWithLocation( // nothing left to assign return false; } - - if (topPendingTaskPriority.compareTo(containerPriority) > 0 && + + if (topPendingTaskPriority.compareTo(containerPriority) > 0 && heldContainers.get(container.getId()).isNew()) { - // if the next task to assign is higher priority than the container then + // if the next task to assign is higher priority than the container then // dont assign this container to that task. // if task and container are equal priority - then its first use or reuse // within the same priority - safe to use // if task is lower priority than container then if we use a container that - // is no longer needed by higher priority tasks All those higher pri tasks + // is no longer needed by higher priority tasks All those higher pri tasks // has been assigned resources - safe to use (first use or reuse) - // if task is higher priority than container then we may end up using a - // container that was assigned by the RM for a lower priority pending task + // if task is higher priority than container then we may end up using a + // container that was assigned by the RM for a lower priority pending task // that will be assigned after this higher priority task is assigned. If we - // use that task's container now then we may not be able to match this - // container to that task later on. However the RM has already assigned us - // all containers and is not going to give us new containers. We will get + // use that task's container now then we may not be able to match this + // container to that task later on. However the RM has already assigned us + // all containers and is not going to give us new containers. We will get // stuck for resources. - // the above applies for new containers. If a container has already been + // the above applies for new containers. If a container has already been // re-used then this is not relevant return false; } - + CookieContainerRequest assigned = assigner.assignReUsedContainer(container, honorLocality); if (assigned != null) { @@ -1795,21 +1795,21 @@ private void informAppAboutAssignments( if (blacklistedNodes.contains(container.getNodeId())) { CookieContainerRequest request = entry.getKey(); Object task = getTask(request); - LOG.info("Container: " + container.getId() + - " allocated on blacklisted node: " + container.getNodeId() + + LOG.info("Container: " + container.getId() + + " allocated on blacklisted node: " + container.getNodeId() + " for task: " + task); Object deAllocTask = deallocateContainer(container.getId()); assert deAllocTask.equals(task); // its ok to submit the same request again because the RM will not give us // the bad/unhealthy nodes again. The nodes may become healthy/unblacklisted // and so its better to give the RM the full information. - allocateTask(task, request.getCapability(), - (request.getNodes() == null ? null : - request.getNodes().toArray(new String[request.getNodes().size()])), - (request.getRacks() == null ? null : - request.getRacks().toArray(new String[request.getRacks().size()])), - request.getPriority(), - request.getCookie().getContainerSignature(), + allocateTask(task, request.getCapability(), + (request.getNodes() == null ? null : + request.getNodes().toArray(new String[request.getNodes().size()])), + (request.getRacks() == null ? null : + request.getRacks().toArray(new String[request.getRacks().size()])), + request.getPriority(), + request.getCookie().getContainerSignature(), request.getCookie().getAppCookie()); } else { informAppAboutAssignment(entry.getKey(), container); @@ -1855,7 +1855,7 @@ public void doBookKeepingForAssignedContainer( assignContainer(task, container, assigned); } } - + private class NodeLocalContainerAssigner extends ContainerAssigner { NodeLocalContainerAssigner() { @@ -1946,8 +1946,8 @@ public CookieContainerRequest assignReUsedContainer(Container container, } } - - + + @VisibleForTesting class DelayedContainerManager extends Thread { @@ -1967,7 +1967,7 @@ public int compare(HeldContainer c1, private volatile boolean tryAssigningAll = false; private volatile boolean running = true; private long maxScheduleTimeSeen = -1; - + // used for testing only @VisibleForTesting volatile AtomicBoolean drainedDelayedContainersForTest = null; @@ -1975,7 +1975,7 @@ public int compare(HeldContainer c1, DelayedContainerManager() { super.setName("DelayedContainerManager"); } - + @Override public void run() { try { @@ -2071,7 +2071,7 @@ private void mainLoop() { } releasePendingContainers(); } - + private void doAssignAll() { // The allocatedContainers queue should not be modified in the middle of an iteration over it. // Synchronizing here on TaskScheduler.this to prevent this from happening. @@ -2108,10 +2108,10 @@ private void doAssignAll() { // Inform app informAppAboutAssignments(assignedContainers); } - + /** * Indicate that an attempt should be made to allocate all available containers. - * Intended to be used in cases where new Container requests come in + * Intended to be used in cases where new Container requests come in */ public void triggerScheduling(boolean scheduleAll) { synchronized(this) { @@ -2124,7 +2124,7 @@ public void shutdown() { this.running = false; this.interrupt(); } - + private void releasePendingContainers() { List pendingContainers = Lists.newArrayListWithCapacity( delayedContainers.size()); @@ -2175,17 +2175,17 @@ void removeDelayedContainer(HeldContainer container) { } } } - + synchronized void determineMinHeldContainers() { sessionMinHeldContainers.clear(); if (sessionNumMinHeldContainers <= 0) { return; } - + if (heldContainers.size() <= sessionNumMinHeldContainers) { sessionMinHeldContainers.addAll(heldContainers.keySet()); } - + Map rackHeldNumber = Maps.newHashMap(); Map> nodeHeldContainers = Maps.newHashMap(); for(HeldContainer heldContainer : heldContainers.values()) { @@ -2206,7 +2206,7 @@ synchronized void determineMinHeldContainers() { for (String rack : rackHeldNumber.keySet()) { rackToHoldNumber.put(rack, new AtomicInteger(0)); } - + // distribute evenly across nodes // the loop assigns 1 container per rack over all racks int containerCount = 0; @@ -2222,7 +2222,7 @@ synchronized void determineMinHeldContainers() { } } } - + // distribute containers evenly across nodes while not exceeding rack limit // the loop assigns 1 container per node over all nodes containerCount = 0; @@ -2246,7 +2246,7 @@ synchronized void determineMinHeldContainers() { } } } - + LOG.info("Holding on to " + sessionMinHeldContainers.size() + " containers" + " out of total held containers: " + heldContainers.size()); } @@ -2303,7 +2303,7 @@ enum LocalityMatchLevel { private int numAssignmentAttempts = 0; private Object lastAssignedContainerSignature; final ContainerSignatureMatcher signatureMatcher; - + HeldContainer(Container container, long nextScheduleTime, long containerExpiryTime, @@ -2321,35 +2321,35 @@ enum LocalityMatchLevel { .getNetworkLocation(); this.signatureMatcher = signatureMatcher; } - + boolean isNew() { return lastTaskInfo == null; } - + String getRack() { return this.rack; } - + String getNode() { return this.container.getNodeId().getHost(); } - + int geNumAssignmentAttempts() { return numAssignmentAttempts; } - + void incrementAssignmentAttempts() { numAssignmentAttempts++; } - + public Container getContainer() { return this.container; } - + public long getNextScheduleTime() { return this.nextScheduleTime; } - + public void setNextScheduleTime(long nextScheduleTime) { this.nextScheduleTime = nextScheduleTime; } @@ -2369,7 +2369,7 @@ public Object getLastAssignedContainerSignature() { public CookieContainerRequest getLastTaskInfo() { return this.lastTaskInfo; } - + public void setLastTaskInfo(CookieContainerRequest taskInfo) { // Merge the container signatures to account for any changes to the container // footprint. For example, re-localization of additional resources will diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java index 5f90a89520..f872b2e6af 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java @@ -26,7 +26,7 @@ import org.apache.tez.dag.records.TezTaskAttemptID; public interface AMContainer extends EventHandler{ - + public AMContainerState getState(); public ContainerId getContainerId(); public Container getContainer(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEvent.java index b47ffb0067..946f347c96 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEvent.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -23,12 +23,12 @@ public class AMContainerEvent extends AbstractEvent { private final ContainerId containerId; - + public AMContainerEvent(ContainerId containerId, AMContainerEventType type) { super(type); this.containerId = containerId; } - + public ContainerId getContainerId() { return this.containerId; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java index 682cd02d07..15c2b3c8cd 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java @@ -48,7 +48,7 @@ public AMContainerEventAssignTA(ContainerId containerId, TezTaskAttemptID attemp public TaskSpec getRemoteTaskSpec() { return this.remoteTaskSpec; } - + public Map getRemoteTaskLocalResources() { return this.taskLocalResources; } @@ -56,7 +56,7 @@ public Map getRemoteTaskLocalResources() { public TezTaskAttemptID getTaskAttemptId() { return this.attemptId; } - + public Credentials getCredentials() { return this.credentials; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventCompleted.java index 73fe2fa655..a4301f9217 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventCompleted.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventCompleted.java @@ -29,7 +29,7 @@ public class AMContainerEventCompleted extends AMContainerEvent { private final String diagnostics; private final TaskAttemptTerminationCause errCause; - public AMContainerEventCompleted(ContainerId containerId, + public AMContainerEventCompleted(ContainerId containerId, int exitStatus, String diagnostics, TaskAttemptTerminationCause errCause) { super(containerId, AMContainerEventType.C_COMPLETED); this.exitStatus = exitStatus; @@ -38,26 +38,26 @@ public AMContainerEventCompleted(ContainerId containerId, } public boolean isPreempted() { - return (exitStatus == ContainerExitStatus.PREEMPTED || + return (exitStatus == ContainerExitStatus.PREEMPTED || errCause == TaskAttemptTerminationCause.INTERNAL_PREEMPTION); } - + public boolean isDiskFailed() { return (exitStatus == ContainerExitStatus.DISKS_FAILED); } - + public boolean isSystemAction() { return isPreempted() || isDiskFailed(); } - + public String getDiagnostics() { return diagnostics; } - + public int getContainerExitStatus() { return exitStatus; } - + public TaskAttemptTerminationCause getTerminationCause() { return errCause; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchFailed.java index 7d476d9443..acfef3117f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchFailed.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchFailed.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -22,13 +22,13 @@ public class AMContainerEventLaunchFailed extends AMContainerEvent { private final String message; - + public AMContainerEventLaunchFailed(ContainerId containerId, String message) { super(containerId, AMContainerEventType.C_LAUNCH_FAILED); this.message = message; } - + public String getMessage() { return this.message; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunched.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunched.java index 7a8ecd3035..b646be873e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunched.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunched.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventStopFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventStopFailed.java index 3642edb184..d0a49d5880 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventStopFailed.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventStopFailed.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -23,7 +23,7 @@ public class AMContainerEventStopFailed extends AMContainerEvent { // TODO XXX Not being used for anything. May be useful if we rely less on // the RM informing the job about container failure. - + private final String message; public AMContainerEventStopFailed(ContainerId containerId, String message) { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventStopRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventStopRequest.java index e2930af684..54e634180a 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventStopRequest.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventStopRequest.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java index e0c8fe72fc..2439e09f39 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -29,7 +29,7 @@ public AMContainerEventTASucceeded(ContainerId containerId, super(containerId, AMContainerEventType.C_TA_SUCCEEDED); this.attemptId = attemptId; } - + public TezTaskAttemptID getTaskAttemptId() { return this.attemptId; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventType.java index 330ad57c4e..745265d87e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventType.java @@ -23,7 +23,7 @@ public enum AMContainerEventType { //Producer: Scheduler C_LAUNCH_REQUEST, C_ASSIGN_TA, - + //Producer: NMCommunicator C_LAUNCHED, C_LAUNCH_FAILED, @@ -33,21 +33,21 @@ public enum AMContainerEventType { //Producer: RMCommunicator C_COMPLETED, - + //Producer: RMCommunicator, AMNode C_NODE_FAILED, - + //TODO ZZZ CREUSE: Consider introducing a new event C_NODE_BLACKLISTED -> container can take a call on what to do if this event comes in. - + //Producer: TA-> Scheduler -> Container (in case of failure etc) // Scheduler -> Container (in case of pre-emption etc) // Node -> Container (in case of Node blacklisted etc) C_STOP_REQUEST, - + //Producer: NMCommunicator C_NM_STOP_FAILED, C_NM_STOP_SENT, - + //Producer: ContainerHeartbeatHandler C_TIMED_OUT, } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java index 41efafb2df..c010dd605d 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java @@ -102,12 +102,12 @@ private static ContainerLaunchContext createCommonContainerLaunchContext( Map serviceData = new HashMap<>(); // Tokens - + // Setup up task credentials buffer ByteBuffer containerCredentialsBuffer; try { Credentials containerCredentials = new Credentials(); - + // All Credentials need to be set so that YARN can localize the resources // correctly, even though they may not be used by all tasks which will run // on this container. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java index 986a463e7f..cb79f2c3a3 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java @@ -103,7 +103,7 @@ public class AMContainerImpl implements AMContainer { private long lastTaskFinishTime; private TezDAGID lastTaskDAGID; - + // An assign can happen even during wind down. e.g. NodeFailure caused the // wind down, and an allocation was pending in the AMScheduler. This could // be modelled as a separate state. @@ -124,7 +124,7 @@ public class AMContainerImpl implements AMContainer { private boolean credentialsChanged = false; private boolean completedMessageSent = false; - + // TODO Consider registering with the TAL, instead of the TAL pulling. // Possibly after splitting TAL and ContainerListener. @@ -473,7 +473,7 @@ protected static class LaunchRequestTransition implements @Override public void transition(AMContainerImpl container, AMContainerEvent cEvent) { AMContainerEventLaunchRequest event = (AMContainerEventLaunchRequest) cEvent; - + ContainerContext containerContext = event.getContainerContext(); // Clone - don't use the object that is passed in, since this is likely to // be modified here. @@ -611,7 +611,7 @@ public AMContainerState transition( container.handleExtraTAAssign(event, container.currentAttempt); return AMContainerState.STOP_REQUESTED; } - + Map taskLocalResources = event.getRemoteTaskLocalResources(); Preconditions.checkState(container.additionalLocalResources == null, "No additional resources should be pending when assigning a new task"); @@ -710,7 +710,7 @@ public void transition(AMContainerImpl container, AMContainerEvent cEvent) { container.currentAttempt, errorMessage, // if termination cause is generic exited then replace with specific - (event.getTerminationCause() == TaskAttemptTerminationCause.CONTAINER_EXITED ? + (event.getTerminationCause() == TaskAttemptTerminationCause.CONTAINER_EXITED ? TaskAttemptTerminationCause.CONTAINER_LAUNCH_FAILED : event.getTerminationCause())); } container.registerFailedAttempt(container.currentAttempt); @@ -747,7 +747,7 @@ public void transition(AMContainerImpl container, AMContainerEvent cEvent) { } container.unregisterFromTAListener(ContainerEndReason.OTHER, getMessage(container, cEvent)); container.logStopped(container.currentAttempt == null ? - ContainerExitStatus.SUCCESS + ContainerExitStatus.SUCCESS : ContainerExitStatus.INVALID); container.sendStopRequestToNM(); } @@ -1001,7 +1001,7 @@ public void transition(AMContainerImpl container, AMContainerEvent cEvent) { AMContainerEventCompleted event = (AMContainerEventCompleted) cEvent; String diag = event.getDiagnostics(); for (TezTaskAttemptID taId : container.failedAssignments) { - container.sendTerminatedToTaskAttempt(taId, diag, + container.sendTerminatedToTaskAttempt(taId, diag, TaskAttemptTerminationCause.CONTAINER_EXITED); } if (container.currentAttempt != null) { @@ -1131,13 +1131,13 @@ private void logStopped(int exitStatus) { final Clock clock = appContext.getClock(); final HistoryEventHandler historyHandler = appContext.getHistoryHandler(); ContainerStoppedEvent lEvt = new ContainerStoppedEvent(containerId, - clock.getTime(), - exitStatus, + clock.getTime(), + exitStatus, appContext.getApplicationAttemptId()); historyHandler.handle( new DAGHistoryEvent(appContext.getCurrentDAGID(),lEvt)); } - + protected void deAllocate() { sendEvent(new AMSchedulerEventDeallocateContainer(containerId, schedulerId)); } @@ -1146,7 +1146,7 @@ protected void sendTerminatedToTaskAttempt( TezTaskAttemptID taId, String message, TaskAttemptTerminationCause errCause) { sendEvent(new TaskAttemptEventContainerTerminated(containerId, taId, message, errCause)); } - + protected void sendContainerTerminatedBySystemToTaskAttempt( TezTaskAttemptID taId, String message, TaskAttemptTerminationCause errorCause) { sendEvent(new TaskAttemptEventContainerTerminatedBySystem(containerId, taId, message, errorCause)); @@ -1163,7 +1163,7 @@ protected void maybeSendNodeFailureForFailedAssignment(TezTaskAttemptID taId) { } } - protected void sendNodeFailureToTA(TezTaskAttemptID taId, String message, + protected void sendNodeFailureToTA(TezTaskAttemptID taId, String message, TaskAttemptTerminationCause errorCause) { sendEvent(new TaskAttemptEventNodeFailed(taId, message, errorCause)); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerState.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerState.java index 7d58faac68..76defe7581 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerState.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerState.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -24,7 +24,7 @@ public enum AMContainerState { RUNNING, // indicates a NM stop request has been attempted. This request could fail, in // which case an RM stop request needs to be sent. - STOP_REQUESTED, + STOP_REQUESTED, // A stop request has been registered with YARN STOPPING, diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerTask.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerTask.java index cfc143a402..1592765ba9 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerTask.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerTask.java @@ -50,11 +50,11 @@ public TaskSpec getTask() { public Map getAdditionalResources() { return this.additionalResources; } - + public Credentials getCredentials() { return this.credentials; } - + public boolean haveCredentialsChanged() { return this.credentialsChanged; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerContextMatcher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerContextMatcher.java index a1d3230a4f..be06f0267e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerContextMatcher.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerContextMatcher.java @@ -40,7 +40,7 @@ private void checkArguments(Object cs1, Object cs2) { + cs1.getClass().getName() + " and " + cs2.getClass().getName()); } - + @Override public boolean isSuperSet(Object cs1, Object cs2) { checkArguments(cs1, cs2); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNode.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNode.java index bc01e04081..1134bdd677 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNode.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNode.java @@ -26,7 +26,7 @@ import org.apache.tez.dag.app.dag.DAG; public interface AMNode extends EventHandler { - + public NodeId getNodeId(); public AMNodeState getState(); public List getContainers(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventContainerAllocated.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventContainerAllocated.java index e250f42d3a..23cc054a25 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventContainerAllocated.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventContainerAllocated.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventNodeCountUpdated.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventNodeCountUpdated.java index 3b35daf2a8..63d72834a3 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventNodeCountUpdated.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventNodeCountUpdated.java @@ -21,12 +21,12 @@ public class AMNodeEventNodeCountUpdated extends AMNodeEvent { private final int count; - + public AMNodeEventNodeCountUpdated(int nodeCount, int sourceId) { super(null, sourceId, AMNodeEventType.N_NODE_COUNT_UPDATED); this.count = nodeCount; } - + public int getNodeCount() { return this.count; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventStateChanged.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventStateChanged.java index b371ddde05..2382d2fb26 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventStateChanged.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventStateChanged.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -25,7 +25,7 @@ public class AMNodeEventStateChanged extends AMNodeEvent { public AMNodeEventStateChanged(NodeReport nodeReport, int sourceId) { super(nodeReport.getNodeId(), sourceId, - (nodeReport.getNodeState().isUnusable() ? + (nodeReport.getNodeState().isUnusable() ? AMNodeEventType.N_TURNED_UNHEALTHY : AMNodeEventType.N_TURNED_HEALTHY)); this.nodeReport = nodeReport; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java index 4a4cb6107a..273908bfa4 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -26,7 +26,7 @@ public class AMNodeEventTaskAttemptEnded extends AMNodeEvent { private final boolean failed; private final ContainerId containerId; private final TezTaskAttemptID taskAttemptId; - + public AMNodeEventTaskAttemptEnded(NodeId nodeId, int sourceId, ContainerId containerId, TezTaskAttemptID taskAttemptId, boolean failed) { super(nodeId, sourceId, AMNodeEventType.N_TA_ENDED); @@ -38,15 +38,15 @@ public AMNodeEventTaskAttemptEnded(NodeId nodeId, int sourceId, ContainerId cont public boolean failed() { return failed; } - + public boolean killed() { return !failed; } - + public ContainerId getContainerId() { return this.containerId; } - + public TezTaskAttemptID getTaskAttemptId() { return this.taskAttemptId; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java index 2b8cb7df8e..622229b015 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java index 4d6c5989a1..552538928a 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java @@ -481,7 +481,7 @@ public boolean isBlacklisted() { this.readLock.unlock(); } } - + @Override public boolean isUsable() { return !(isUnhealthy() || isBlacklisted()); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeTracker.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeTracker.java index 548a377acf..9b6286c3b9 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeTracker.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeTracker.java @@ -38,9 +38,9 @@ public class AMNodeTracker extends AbstractService implements EventHandler { - + static final Logger LOG = LoggerFactory.getLogger(AMNodeTracker.class); - + private final ConcurrentMap perSourceNodeTrackers; @SuppressWarnings("rawtypes") @@ -60,11 +60,11 @@ public AMNodeTracker(EventHandler eventHandler, AppContext appContext) { this.eventHandler = eventHandler; this.appContext = appContext; } - + @Override public synchronized void serviceInit(Configuration conf) { this.maxTaskFailuresPerNode = conf.getInt( - TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE, + TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE, TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE_DEFAULT); this.nodeBlacklistingEnabled = conf.getBoolean( TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java b/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java index a212041183..b2353de678 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java @@ -31,8 +31,8 @@ @InterfaceAudience.Private @InterfaceStability.Unstable public class TezAMPolicyProvider extends PolicyProvider { - - private static final Service[] tezApplicationMasterServices = + + private static final Service[] tezApplicationMasterServices = new Service[] { new Service( TezConstants.TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL, diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java index 51109fe18c..85c871f517 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java @@ -40,7 +40,7 @@ public class ContainerStoppedEvent implements HistoryEvent { public ContainerStoppedEvent() { } - + public ContainerStoppedEvent(ContainerId containerId, long stopTime, int exitStatus, @@ -50,7 +50,7 @@ public ContainerStoppedEvent(ContainerId containerId, this.exitStatus = exitStatus; this.applicationAttemptId = applicationAttemptId; } - + @Override public HistoryEventType getEventType() { return HistoryEventType.CONTAINER_STOPPED; @@ -101,7 +101,7 @@ public void fromProtoStream(CodedInputStream inputStream) throws IOException { @Override public String toString() { return "containerId=" + containerId - + ", stoppedTime=" + stopTime + + ", stoppedTime=" + stopTime + ", exitStatus=" + exitStatus; } @@ -112,7 +112,7 @@ public ContainerId getContainerId() { public long getStoppedTime() { return stopTime; } - + public int getExitStatus() { return exitStatus; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java index 20514cfe0e..a77098ce60 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java @@ -85,7 +85,7 @@ public void fromProtoStream(CodedInputStream inputStream) throws IOException { } fromProto(proto); } - + public void fromProto(RecoveryProtos.DAGKillRequestProto proto) { this.dagID = TezDAGID.fromString(proto.getDagId()); this.killRequestTime = proto.getKillRequestTime(); @@ -120,7 +120,7 @@ public boolean writeToRecoveryImmediately() { public TezDAGID getDagID() { return dagID; } - + public long getKillRequestTime() { return killRequestTime; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java index 3021b79c90..9bac94d9fb 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java @@ -84,11 +84,11 @@ public TaskAttemptFinishedEvent(TezTaskAttemptID taId, TaskAttemptState state, @Nullable TaskFailureType taskFailureType, TaskAttemptTerminationCause error, - String diagnostics, TezCounters counters, + String diagnostics, TezCounters counters, List dataEvents, List taGeneratedEvents, - long creationTime, - TezTaskAttemptID creationCausalTA, + long creationTime, + TezTaskAttemptID creationCausalTA, long allocationTime, ContainerId containerId, NodeId nodeId, @@ -133,11 +133,11 @@ public boolean isRecoveryEvent() { public boolean isHistoryEvent() { return true; } - + public List getDataEvents() { return dataEvents; } - + public TaskAttemptFinishedProto toProto() throws IOException { TaskAttemptFinishedProto.Builder builder = TaskAttemptFinishedProto.newBuilder(); @@ -323,7 +323,7 @@ public TezCounters getCounters() { public String getDiagnostics() { return diagnostics; } - + public TaskAttemptTerminationCause getTaskAttemptError() { return error; } @@ -343,15 +343,15 @@ public TaskFailureType getTaskFailureType() { public long getStartTime() { return startTime; } - + public long getCreationTime() { return creationTime; } - + public long getAllocationTime() { return allocationTime; } - + public TezTaskAttemptID getCreationCausalTA() { return creationCausalTA; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java index 2aad49b75a..06d1a36e73 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java @@ -125,7 +125,7 @@ public TezTaskAttemptID getTaskAttemptID() { public long getStartTime() { return launchTime; } - + public ContainerId getContainerId() { return containerId; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java index a5e655d7c1..2362af536c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java @@ -49,7 +49,7 @@ public class VertexConfigurationDoneEvent implements HistoryEvent, VertexIDAware private Map rootInputSpecUpdates; private boolean setParallelismCalledFlag; - public VertexConfigurationDoneEvent() { + public VertexConfigurationDoneEvent() { } public VertexConfigurationDoneEvent(TezVertexID vertexID, diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexInitializedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexInitializedEvent.java index 9427520499..81f20f0bdd 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexInitializedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexInitializedEvent.java @@ -200,7 +200,7 @@ public int getNumTasks() { return numTasks; } - public Map> + public Map> getAdditionalInputs() { return additionalInputs; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java index fc95d33b0c..c3a6dc0d65 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java @@ -221,7 +221,7 @@ private static JSONObject convertAMLaunchedEvent(AMLaunchedEvent event) throws J JSONObject otherInfo = new JSONObject(); otherInfo.put(ATSConstants.APP_SUBMIT_TIME, event.getAppSubmitTime()); jsonObject.put(ATSConstants.OTHER_INFO, otherInfo); - + return jsonObject; } @@ -564,7 +564,7 @@ private static JSONObject convertTaskAttemptFinishedEvent(TaskAttemptFinishedEve otherInfo.put(ATSConstants.COUNTERS, DAGUtils.convertCountersToJSON(event.getCounters())); if (event.getDataEvents() != null && !event.getDataEvents().isEmpty()) { - otherInfo.put(ATSConstants.LAST_DATA_EVENTS, + otherInfo.put(ATSConstants.LAST_DATA_EVENTS, DAGUtils.convertDataEventDependencyInfoToJSON(event.getDataEvents())); } if (event.getNodeId() != null) { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java index ca80d69a47..2588223fe6 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java @@ -109,12 +109,12 @@ public static JSONObject generateSimpleJSONPlan(DAGPlan dagPlan) throws JSONExce } return dagJson; } - + public static JSONObject convertDataEventDependencyInfoToJSON(List info) throws JSONException { return new JSONObject(convertDataEventDependecyInfoToATS(info)); } - + public static Map convertDataEventDependecyInfoToATS(List info) { ArrayList infoList = new ArrayList(); for (DataEventDependencyInfo event : info) { @@ -466,7 +466,7 @@ public static Map convertServicePluginToATSMap( public static Map convertEdgeProperty( EdgeProperty edge) { Map jsonDescriptor = new HashMap(); - + jsonDescriptor.put(DATA_MOVEMENT_TYPE_KEY, edge.getDataMovementType().name()); jsonDescriptor.put(DATA_SOURCE_TYPE_KEY, edge.getDataSourceType().name()); @@ -492,7 +492,7 @@ public static Map convertEdgeProperty( } return jsonDescriptor; } - + public static Map convertEdgeManagerPluginDescriptor( EdgeManagerPluginDescriptor descriptor) { Map jsonDescriptor = new HashMap(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/TezEventUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/TezEventUtils.java index e756325b5b..c113b7c864 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/TezEventUtils.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/TezEventUtils.java @@ -105,7 +105,7 @@ public static TezEvent fromProto(TezEventProto eventProto) throws IOException { tezEvent.setDestinationInfo(destinationInfo); return tezEvent; } - + public static RecoveryProtos.EventMetaDataProto convertEventMetaDataToProto( EventMetaData eventMetaData) { RecoveryProtos.EventMetaDataProto.Builder builder = diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java index 25551a1ccb..2643b722f8 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java @@ -38,13 +38,13 @@ public static TezVertexID newVertexID(TezDAGID dagId, int vertexId) { public static TezTaskAttemptID newTaskAttemptId(TezTaskID taskId, int id) { return TezTaskAttemptID.getInstance(taskId, id); } - + public static DAGReport newDAGReport() { return null; } - public static AMInfo newAMInfo(ApplicationAttemptId appAttemptID, - long startTime, ContainerId containerID, String nmHost, + public static AMInfo newAMInfo(ApplicationAttemptId appAttemptID, + long startTime, ContainerId containerID, String nmHost, int nmPort, int nmHttpPort) { return null; } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/api/client/TestDAGClientHandler.java b/tez-dag/src/test/java/org/apache/tez/dag/api/client/TestDAGClientHandler.java index 34d3f4012b..81461499d2 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/api/client/TestDAGClientHandler.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/api/client/TestDAGClientHandler.java @@ -42,7 +42,7 @@ public class TestDAGClientHandler { - + @Test(timeout = 5000) public void testDAGClientHandler() throws TezException { @@ -77,7 +77,7 @@ public void testDAGClientHandler() throws TezException { } catch (TezException e) { assertTrue(e.getMessage().contains("Unknown dagId")); } - DAGStatus dagStatus = dagClientHandler.getDAGStatus("dag_9999_0001_1", + DAGStatus dagStatus = dagClientHandler.getDAGStatus("dag_9999_0001_1", Sets.newSet(StatusGetOpts.GET_COUNTERS)); assertEquals(mockDagStatusBuilder, dagStatus); @@ -91,8 +91,8 @@ public void testDAGClientHandler() throws TezException { VertexStatus vertexStatus = dagClientHandler.getVertexStatus("dag_9999_0001_1", "v1", Sets.newSet(StatusGetOpts.GET_COUNTERS)); assertEquals(mockVertexStatusBuilder, vertexStatus); - - + + // getTezAppMasterStatus when(mockDagAM.isSession()).thenReturn(false); @@ -103,7 +103,7 @@ public void testDAGClientHandler() throws TezException { assertEquals(TezAppMasterStatus.INITIALIZING, dagClientHandler.getTezAppMasterStatus()); when(mockDagAM.getState()).thenReturn(DAGAppMasterState.ERROR); assertEquals(TezAppMasterStatus.SHUTDOWN, dagClientHandler.getTezAppMasterStatus()); - + // tryKillDAG try{ dagClientHandler.tryKillDAG("dag_9999_0001_2"); @@ -124,7 +124,7 @@ public void testDAGClientHandler() throws TezException { Map localResources = new HashMap(); dagClientHandler.submitDAG(dagPlan, localResources); verify(mockDagAM).submitDAGToAppMaster(dagPlan, localResources); - + // shutdown dagClientHandler.shutdownAM(); verify(mockDagAM).shutdownTezAM(contains("Received message to shutdown AM from")); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockClock.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockClock.java index bce2d4c89a..e8f6c7a93b 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockClock.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockClock.java @@ -24,7 +24,7 @@ import org.apache.hadoop.yarn.util.Clock; public class MockClock implements Clock { - + long time; Collection listeners = new LinkedList<>(); @@ -40,7 +40,7 @@ public MockClock(long initTime) { public long getTime() { return time; } - + public void incrementTime(long inc) { time += inc; for (MockClockListener listener : listeners) { diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java index fbab519376..f9bf0cec14 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java @@ -86,7 +86,7 @@ import org.slf4j.LoggerFactory; public class MockDAGAppMaster extends DAGAppMaster { - + private static final Logger LOG = LoggerFactory.getLogger(MockDAGAppMaster.class); MockContainerLauncher containerLauncher; private final AtomicBoolean launcherGoFlag; @@ -101,23 +101,23 @@ public class MockDAGAppMaster extends DAGAppMaster { boolean doSleep = true; int handlerConcurrency = 1; int numConcurrentContainers = 1; - + ThreadMXBean threadMxBean = ManagementFactory.getThreadMXBean(); AtomicLong heartbeatCpu = new AtomicLong(0); AtomicLong heartbeatTime = new AtomicLong(0); AtomicLong numHearbeats = new AtomicLong(0); - + public static interface StatisticsDelegate { public TaskStatistics getStatistics(TaskSpec taskSpec); } public static interface CountersDelegate { public TezCounters getCounters(TaskSpec taskSpec); } - + public static interface EventsDelegate { public void getEvents(TaskSpec taskSpec, List events, long time); } - + public static interface ContainerDelegate { public void stop(ContainerStopRequest event); public void launch(ContainerLaunchRequest event); @@ -132,20 +132,20 @@ public class MockContainerLauncher extends ContainerLauncher implements Runnable BlockingQueue eventQueue = new LinkedBlockingQueue(); Thread eventHandlingThread; ListeningExecutorService executorService; - + Map containers = Maps.newConcurrentMap(); ArrayBlockingQueue workers; TaskCommunicatorManager taskCommunicatorManager; TezTaskCommunicatorImpl taskCommunicator; - + AtomicBoolean startScheduling = new AtomicBoolean(true); AtomicBoolean goFlag; boolean updateProgress = true; LinkedBlockingQueue containersToProcess = new LinkedBlockingQueue(); - + Map preemptedTasks = Maps.newConcurrentMap(); - + Map tasksWithStatusUpdates = Maps.newConcurrentMap(); public MockContainerLauncher(AtomicBoolean goFlag, @@ -167,17 +167,17 @@ public class ContainerData { boolean completed; String cIdStr; AtomicBoolean remove = new AtomicBoolean(false); - + public ContainerData(ContainerId cId, ContainerLaunchContext context) { this.cId = cId; this.cIdStr = cId.toString(); this.launchContext = context; } - + void remove() { remove.set(true); } - + void clear() { taId = null; vName = null; @@ -191,7 +191,7 @@ void clear() { remove.set(false); } } - + @Override public void start() throws Exception { taskCommunicatorManager = (TaskCommunicatorManager) getTaskCommunicatorManager(); @@ -245,11 +245,11 @@ void waitToGo() { } } } - + public void startScheduling(boolean value) { startScheduling.set(value); } - + public void updateProgress(boolean value) { this.updateProgress = value; } @@ -257,21 +257,21 @@ public void updateProgress(boolean value) { public Map getContainers() { return containers; } - + public void preemptContainerForTask(TezTaskID tId, int uptoVersion) { preemptedTasks.put(tId, uptoVersion); } - + public void preemptContainer(ContainerData cData) { getTaskSchedulerManager().containerCompleted(0, null, ContainerStatus.newInstance(cData.cId, null, "Preempted", ContainerExitStatus.PREEMPTED)); cData.clear(); } - + public void setStatusUpdatesForTask(TezTaskAttemptID tId, int numUpdates) { tasksWithStatusUpdates.put(tId, numUpdates); } - + void stop(ContainerStopRequest event) { // remove from simulated container list containers.remove(event.getContainerId()); @@ -292,20 +292,20 @@ void launch(ContainerLaunchRequest event) { } getContext().containerLaunched(event.getContainerId()); } - + public void waitTillContainersLaunched() throws InterruptedException { while (containers.isEmpty()) { Thread.sleep(50); } } - + void incrementTime(long inc) { Clock clock = MockDAGAppMaster.this.getContext().getClock(); if (clock instanceof MockClock) { ((MockClock) clock).incrementTime(inc); } } - + @Override public void run() { Thread.currentThread().setName("MockLauncher"); @@ -337,7 +337,7 @@ public void run() { LOG.warn("Exception in mock container launcher thread", ie); } } - + private void doHeartbeat(TezHeartbeatRequest request, ContainerData cData) throws Exception { long startTime = System.nanoTime(); long startCpuTime = threadMxBean.getCurrentThreadCpuTime(); @@ -356,7 +356,7 @@ private void doHeartbeat(TezHeartbeatRequest request, ContainerData cData) throw } } } - + class Worker implements Callable { class WorkerCallback implements FutureCallback { @Override @@ -379,15 +379,15 @@ void completeOperation() { volatile ContainerData cData; WorkerCallback callback = new WorkerCallback(); - + WorkerCallback getCallback() { return callback; } - + void setContainerData(ContainerData cData) { this.cData = cData; } - + @Override public Void call() throws Exception { try { @@ -427,7 +427,7 @@ public Void call() throws Exception { cData.numUpdates++; float maxUpdates = (updatesToMake != null) ? updatesToMake.intValue() : 1; float progress = updateProgress ? cData.numUpdates/maxUpdates : 0f; - events.add(new TezEvent(new TaskStatusUpdateEvent(counters, progress, stats, false), + events.add(new TezEvent(new TaskStatusUpdateEvent(counters, progress, stats, false), new EventMetaData( EventProducerConsumerType.SYSTEM, cData.vName, "", cData.taId), MockDAGAppMaster.this.getContext().getClock().getTime())); @@ -450,16 +450,16 @@ public Void call() throws Exception { } } } catch (Exception e) { - // exception from TA listener. Behave like real. Die and continue with others + // exception from TA listener. Behave like real. Die and continue with others LOG.warn("Exception in mock container launcher thread for cId: " + cData.cIdStr, e); cData.remove(); } return null; } - + } } - + public class MockHistoryEventHandler extends HistoryEventHandler { public MockHistoryEventHandler(AppContext context) { @@ -542,7 +542,7 @@ public MockContainerLauncher getContainerLauncher() { public MockDAGAppMasterShutdownHandler getShutdownHandler() { return (MockDAGAppMasterShutdownHandler) this.shutdownHandler; } - + public void clearStats() { heartbeatCpu.set(0); heartbeatTime.set(0); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockLocalClient.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockLocalClient.java index c3355473d1..21a308181a 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockLocalClient.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockLocalClient.java @@ -56,7 +56,7 @@ protected DAGAppMaster createDAGAppMaster(ApplicationAttemptId applicationAttemp String[] localDirs, String[] logDirs, Credentials credentials, String jobUserName) { mockApp = new MockDAGAppMaster(applicationAttemptId, cId, currentHost, nmPort, nmHttpPort, (mockClock!=null ? mockClock : clock), appSubmitTime, isSession, userDir, localDirs, logDirs, - mockAppLauncherGoFlag, initFailFlag, startFailFlag, credentials, jobUserName, + mockAppLauncherGoFlag, initFailFlag, startFailFlag, credentials, jobUserName, concurrency, containers); return mockApp; } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockTezClient.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockTezClient.java index ff6697026c..491732e006 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockTezClient.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockTezClient.java @@ -30,14 +30,14 @@ public class MockTezClient extends TezClient { MockLocalClient client; - + MockTezClient(String name, TezConfiguration tezConf, boolean isSession, Map localResources, Credentials credentials, Clock clock, AtomicBoolean mockAppLauncherGoFlag) { super(name, tezConf, isSession, localResources, credentials); this.client = new MockLocalClient(mockAppLauncherGoFlag, clock); } - + MockTezClient(String name, TezConfiguration tezConf, boolean isSession, Map localResources, Credentials credentials, Clock clock, AtomicBoolean mockAppLauncherGoFlag, @@ -45,20 +45,20 @@ public class MockTezClient extends TezClient { this(name, tezConf, isSession, localResources, credentials, clock, mockAppLauncherGoFlag, initFailFlag, startFailFlag, 1, 1); } - + MockTezClient(String name, TezConfiguration tezConf, boolean isSession, Map localResources, Credentials credentials, Clock clock, AtomicBoolean mockAppLauncherGoFlag, boolean initFailFlag, boolean startFailFlag, int concurrency, int containers) { super(name, tezConf, isSession, localResources, credentials); - this.client = new MockLocalClient(mockAppLauncherGoFlag, clock, initFailFlag, startFailFlag, + this.client = new MockLocalClient(mockAppLauncherGoFlag, clock, initFailFlag, startFailFlag, concurrency, containers); } protected FrameworkClient createFrameworkClient() { return client; } - + public MockLocalClient getLocalClient() { return client; } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMemoryWithEvents.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMemoryWithEvents.java index f913d14183..d1c58f5677 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMemoryWithEvents.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMemoryWithEvents.java @@ -56,14 +56,14 @@ import org.junit.Ignore; import org.junit.Test; -// The objective of these tests is to make sure the large job simulations pass +// The objective of these tests is to make sure the large job simulations pass // within the memory limits set by the junit tests (1GB) -// For large jobs please increase memory limits to account for memory used by the +// For large jobs please increase memory limits to account for memory used by the // simulation code itself public class TestMemoryWithEvents { static Configuration defaultConf; static FileSystem localFs; - + static { try { defaultConf = new Configuration(false); @@ -91,7 +91,7 @@ private void checkMemory(String name, MockDAGAppMaster mockApp) { System.out.println("##### Heap utilization statistics [MB] for " + name); runtime.gc(); - + //Print used memory System.out.println("##### Used Memory:" + (runtime.totalMemory() - runtime.freeMemory()) / mb); @@ -99,25 +99,25 @@ private void checkMemory(String name, MockDAGAppMaster mockApp) { //Print free memory System.out.println("##### Free Memory:" + runtime.freeMemory() / mb); - + //Print total available memory System.out.println("##### Total Memory:" + runtime.totalMemory() / mb); //Print Maximum available memory System.out.println("##### Max Memory:" + runtime.maxMemory() / mb); - + //Print Maximum heartbeat time long numHeartbeats = mockApp.numHearbeats.get(); if (numHeartbeats == 0) { numHeartbeats = 1; } - System.out.println("##### Heartbeat (ms) :" - + " latency avg: " + ((mockApp.heartbeatTime.get() / numHeartbeats) / microsPerMs) + System.out.println("##### Heartbeat (ms) :" + + " latency avg: " + ((mockApp.heartbeatTime.get() / numHeartbeats) / microsPerMs) + " cpu total: " + (mockApp.heartbeatCpu.get() / microsPerMs) + " cpu avg: " + ((mockApp.heartbeatCpu.get() / numHeartbeats) / microsPerMs) + " numHeartbeats: " + mockApp.numHearbeats.get()); } - + private void testMemory(DAG dag, boolean sendDMEvents) throws Exception { StopWatch stopwatch = new StopWatch(); stopwatch.start(); @@ -126,7 +126,7 @@ private void testMemory(DAG dag, boolean sendDMEvents) throws Exception { MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false, numThreads, 1000); tezClient.start(); - + MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); @@ -142,7 +142,7 @@ private void testMemory(DAG dag, boolean sendDMEvents) throws Exception { System.out.println("Time taken(ms): " + stopwatch.now(TimeUnit.MILLISECONDS)); tezClient.stop(); } - + public static class SimulationInitializer extends InputInitializer { public SimulationInitializer(InputInitializerContext initializerContext) { super(initializerContext); @@ -176,7 +176,7 @@ public void testMemoryRootInputEvents() throws Exception { dag.addVertex(vA).addVertex(vB); testMemory(dag, false); } - + @Ignore @Test (timeout = 600000) public void testMemoryOneToOne() throws Exception { @@ -206,7 +206,7 @@ public void testMemoryBroadcast() throws Exception { OutputDescriptor.create("Out"), InputDescriptor.create("In")))); testMemory(dag, true); } - + @Ignore @Test (timeout = 600000) public void testMemoryScatterGather() throws Exception { diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java index 75ac76f609..af05660e00 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java @@ -117,7 +117,7 @@ public class TestMockDAGAppMaster { private static final Log LOG = LogFactory.getLog(TestMockDAGAppMaster.class); static Configuration defaultConf; - static FileSystem localFs; + static FileSystem localFs; static { try { defaultConf = new Configuration(false); @@ -130,7 +130,7 @@ public class TestMockDAGAppMaster { throw new RuntimeException("init failure", e); } } - + static class TestEventsDelegate implements EventsDelegate { @Override public void getEvents(TaskSpec taskSpec, List events, long time) { @@ -146,20 +146,20 @@ public void getEvents(TaskSpec taskSpec, List events, long time) { .getDestinationVertexName(), taskSpec.getTaskAttemptID()), time)); } } - } + } } - + @Test (timeout = 5000) public void testLocalResourceSetup() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); - + MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); tezClient.start(); - + MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); - + Map lrDAG = Maps.newHashMap(); String lrName1 = "LR1"; lrDAG.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), @@ -181,20 +181,20 @@ public void testLocalResourceSetup() throws Exception { // verify tasks are launched with both DAG and task resources. Assert.assertTrue(taskLR.containsKey(lrName1)); Assert.assertTrue(taskLR.containsKey(lrName2)); - + mockLauncher.startScheduling(true); dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); tezClient.stop(); } - + @Test (timeout = 5000) public void testInternalPreemption() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); - + MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); tezClient.start(); - + MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); @@ -208,7 +208,7 @@ public void testInternalPreemption() throws Exception { ContainerData cData = mockLauncher.getContainers().values().iterator().next(); DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG(); mockApp.getTaskSchedulerManager().preemptContainer(0, cData.cId); - + mockLauncher.startScheduling(true); dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); @@ -223,10 +223,10 @@ public void testInternalPreemption() throws Exception { @Test (timeout = 5000) public void testBasicEvents() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); - + MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); tezClient.start(); - + MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); @@ -298,7 +298,7 @@ public void testBasicEvents() throws Exception { tezClient.stop(); } - + public static class LegacyEdgeTestEdgeManager extends EdgeManagerPlugin { List destinationInputIndices = Collections.singletonList(0); public LegacyEdgeTestEdgeManager(EdgeManagerPluginContext context) { @@ -321,7 +321,7 @@ public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) throws Exception @Override public void routeDataMovementEventToDestination(DataMovementEvent event, - int sourceTaskIndex, int sourceOutputIndex, + int sourceTaskIndex, int sourceOutputIndex, Map> destinationTaskAndInputIndices) { destinationTaskAndInputIndices.put(sourceTaskIndex, destinationInputIndices); } @@ -337,20 +337,20 @@ public int routeInputErrorEventToSource(InputReadErrorEvent event, int destinationTaskIndex, int destinationFailedInputIndex) { return destinationTaskIndex; } - + @Override public int getNumDestinationConsumerTasks(int sourceTaskIndex) { return 1; } } - + @Test (timeout = 100000) public void testMixedEdgeRouting() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); - + MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); tezClient.start(); - + MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); @@ -413,23 +413,23 @@ public void testMixedEdgeRouting() throws Exception { tezClient.stop(); } - + @Test (timeout = 100000) public void testConcurrencyLimit() throws Exception { // the test relies on local mode behavior of launching a new container per task. // so task concurrency == container concurrency TezConfiguration tezconf = new TezConfiguration(defaultConf); - + final int concurrencyLimit = 5; MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false, concurrencyLimit*4, 1000); tezClient.start(); - + MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); - + final AtomicInteger concurrency = new AtomicInteger(0); final AtomicBoolean exceededConcurrency = new AtomicBoolean(false); mockApp.containerDelegate = new ContainerDelegate() { @@ -634,10 +634,10 @@ public TezCounters getCounters(TaskSpec taskSpec) { if (vaGrouName != vBGrouName) { Assert.fail("String group name objects dont match despite interning."); } - + tezClient.stop(); } - + @Test (timeout = 10000) public void testBasicStatistics() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); @@ -678,7 +678,7 @@ public void testBasicStatistics() throws Exception { DataOutput outB = new DataOutputStream(bosB); vBStats.write(outB); final byte[] payloadB = bosB.toByteArray(); - + MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); @@ -708,7 +708,7 @@ public TaskStatistics getStatistics(TaskSpec taskSpec) { mockLauncher.startScheduling(true); DAGStatus status = dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState()); - + // verify that the values have been correct aggregated for (org.apache.tez.dag.app.dag.Vertex v : dagImpl.getVertices().values()) { VertexStatistics vStats = v.getStatistics(); @@ -724,34 +724,34 @@ public TaskStatistics getStatistics(TaskSpec taskSpec) { Assert.assertEquals(2, vStats.getOutputStatistics(sinkName).getItemsProcessed()); } } - + tezClient.stop(); } - - private void checkMemory(String name, MockDAGAppMaster mockApp) { - long mb = 1024*1024; - - //Getting the runtime reference from system - Runtime runtime = Runtime.getRuntime(); - - System.out.println("##### Heap utilization statistics [MB] for " + name); - - runtime.gc(); - - //Print used memory - System.out.println("##### Used Memory:" - + (runtime.totalMemory() - runtime.freeMemory()) / mb); - - //Print free memory - System.out.println("##### Free Memory:" - + runtime.freeMemory() / mb); - - //Print total available memory - System.out.println("##### Total Memory:" + runtime.totalMemory() / mb); - - //Print Maximum available memory - System.out.println("##### Max Memory:" + runtime.maxMemory() / mb); - } + + private void checkMemory(String name, MockDAGAppMaster mockApp) { + long mb = 1024*1024; + + //Getting the runtime reference from system + Runtime runtime = Runtime.getRuntime(); + + System.out.println("##### Heap utilization statistics [MB] for " + name); + + runtime.gc(); + + //Print used memory + System.out.println("##### Used Memory:" + + (runtime.totalMemory() - runtime.freeMemory()) / mb); + + //Print free memory + System.out.println("##### Free Memory:" + + runtime.freeMemory() / mb); + + //Print total available memory + System.out.println("##### Total Memory:" + runtime.totalMemory() / mb); + + //Print Maximum available memory + System.out.println("##### Max Memory:" + runtime.maxMemory() / mb); + } @Ignore @Test (timeout = 60000) @@ -763,7 +763,7 @@ public void testBasicCounterMemory() throws Exception { tezClient.start(); final String vAName = "A"; - + DAG dag = DAG.create("testBasicCounterMemory"); Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 10000); dag.addVertex(vA); @@ -797,7 +797,7 @@ public TezCounters getCounters(TaskSpec taskSpec) { checkMemory(dag.getName(), mockApp); tezClient.stop(); } - + @Ignore @Test (timeout = 60000) public void testTaskEventsProcessingSpeed() throws Exception { @@ -809,7 +809,7 @@ public void testTaskEventsProcessingSpeed() throws Exception { tezClient.start(); final String vAName = "A"; - + DAG dag = DAG.create("testTaskEventsProcessingSpeed"); Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 50000); dag.addVertex(vA); @@ -854,7 +854,7 @@ public void testBasicStatisticsMemory() throws Exception { DataOutput outA = new DataOutputStream(bosA); vAStats.write(outA); final byte[] payloadA = bosA.toByteArray(); - + MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); @@ -888,7 +888,7 @@ public TaskStatistics getStatistics(TaskSpec taskSpec) { checkMemory(dag.getName(), mockApp); tezClient.stop(); } - + @Test (timeout = 10000) public void testMultipleSubmissions() throws Exception { Map lrDAG = Maps.newHashMap(); @@ -905,14 +905,14 @@ public void testMultipleSubmissions() throws Exception { dag.addVertex(vA); TezConfiguration tezconf = new TezConfiguration(defaultConf); - + MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); tezClient.start(); DAGClient dagClient = tezClient.submitDAG(dag); dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); tezClient.stop(); - + // submit the same DAG again to verify it can be done. tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); tezClient.start(); @@ -1027,7 +1027,7 @@ public void testCommitOutputOnDAGSuccess() throws Exception { DAGClient dagClient = tezClient.submitDAG(dag1); dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); - + // vertexGroupCommiter fail (uv12) DAG dag2 = createDAG("testDAGVertexGroupCommitFail", true, false); dagClient = tezClient.submitDAG(dag2); @@ -1080,7 +1080,7 @@ public void testCommitOutputOnVertexSuccess() throws Exception { DAGClient dagClient = tezClient.submitDAG(dag1); dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); - + // vertexGroupCommiter fail (uv12) DAG dag2 = createDAG("testDAGVertexGroupCommitFail", true, false); dagClient = tezClient.submitDAG(dag2); @@ -1113,7 +1113,7 @@ public void testCommitOutputOnVertexSuccess() throws Exception { Assert.assertEquals(VertexStatus.State.FAILED, dagClient.getVertexStatus("v3", null).getState()); Assert.assertTrue(StringUtils.join(dagClient.getVertexStatus("v3", null).getDiagnostics(),",") .contains("fail output committer:v3Out")); - + // both committers fail DAG dag4 = createDAG("testDAGBothCommitsFail", true, true); dagClient = tezClient.submitDAG(dag4); @@ -1139,7 +1139,7 @@ public void testCommitOutputOnVertexSuccess() throws Exception { tezClient.stop(); } - + public static class FailingOutputCommitter extends OutputCommitter { boolean failOnCommit = false; diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestPreemption.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestPreemption.java index 5e83369cfe..35e004d7bd 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestPreemption.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestPreemption.java @@ -50,10 +50,10 @@ import org.junit.Test; public class TestPreemption { - + static Configuration defaultConf; static FileSystem localFs; - + static { try { defaultConf = new Configuration(false); @@ -66,25 +66,25 @@ public class TestPreemption { throw new RuntimeException("init failure", e); } } - - MockDAGAppMaster mockApp; + + MockDAGAppMaster mockApp; MockContainerLauncher mockLauncher; - + int dagCount = 0; - + DAG createDAG(DataMovementType dmType) { DAG dag = DAG.create("test-" + dagCount++); Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5); Vertex vB = Vertex.create("B", ProcessorDescriptor.create("Proc.class"), 5); - Edge eAB = Edge.create(vA, vB, + Edge eAB = Edge.create(vA, vB, EdgeProperty.create(dmType, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("O.class"), InputDescriptor.create("I.class"))); - + dag.addVertex(vA).addVertex(vB).addEdge(eAB); return dag; } - + @Test (timeout = 5000) public void testPreemptionWithoutSession() throws Exception { System.out.println("TestPreemptionWithoutSession"); @@ -94,7 +94,7 @@ public void testPreemptionWithoutSession() throws Exception { MockTezClient tezClient = new MockTezClient("testPreemption", tezconf, false, null, null, null, mockAppLauncherGoFlag, false, false, 2, 2); tezClient.start(); - + DAGClient dagClient = tezClient.submitDAG(createDAG(DataMovementType.SCATTER_GATHER)); // now the MockApp has been started. sync with it to get the launcher syncWithMockAppLauncher(false, mockAppLauncherGoFlag, tezClient); @@ -111,19 +111,19 @@ public void testPreemptionWithoutSession() throws Exception { mockLauncher.preemptContainerForTask(taId.getTaskID(), upToTaskVersion); mockLauncher.startScheduling(true); - + dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); for (int i=0; i<=upToTaskVersion; ++i) { - TezTaskAttemptID testTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), i); + TezTaskAttemptID testTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), i); TaskAttemptImpl taImpl = dagImpl.getTaskAttempt(testTaId); Assert.assertEquals(TaskAttemptStateInternal.KILLED, taImpl.getInternalState()); } - + tezClient.stop(); } - + @Test (timeout = 30000) public void testPreemptionWithSession() throws Exception { System.out.println("TestPreemptionWithSession"); @@ -142,7 +142,7 @@ public void testPreemptionWithSession() throws Exception { testPreemptionMultiple(tezClient, createDAG(DataMovementType.ONE_TO_ONE), 1, "1-1"); tezClient.stop(); } - + MockTezClient createTezSession() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); tezconf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 0); @@ -153,8 +153,8 @@ MockTezClient createTezSession() throws Exception { syncWithMockAppLauncher(false, mockAppLauncherGoFlag, tezClient); return tezClient; } - - void syncWithMockAppLauncher(boolean allowScheduling, AtomicBoolean mockAppLauncherGoFlag, + + void syncWithMockAppLauncher(boolean allowScheduling, AtomicBoolean mockAppLauncherGoFlag, MockTezClient tezClient) throws Exception { synchronized (mockAppLauncherGoFlag) { while (!mockAppLauncherGoFlag.get()) { @@ -164,9 +164,9 @@ void syncWithMockAppLauncher(boolean allowScheduling, AtomicBoolean mockAppLaunc mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(allowScheduling); mockAppLauncherGoFlag.notify(); - } + } } - + void testPreemptionSingle(MockTezClient tezClient, DAG dag, int vertexIndex, String info) throws Exception { testPreemptionJob(tezClient, dag, vertexIndex, 0, info + "-Single"); @@ -182,27 +182,27 @@ void testPreemptionJob(MockTezClient tezClient, DAG dag, int vertexIndex, System.out.println("TestPreemption - Running - " + info); TezConfiguration tezconf = new TezConfiguration(defaultConf); tezconf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 0); - + mockLauncher.startScheduling(false); // turn off scheduling to block DAG before submitting it DAGClient dagClient = tezClient.submitDAG(dag); - + DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG(); TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), vertexIndex); TezTaskAttemptID taId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0); mockLauncher.preemptContainerForTask(taId.getTaskID(), upToTaskVersion); mockLauncher.startScheduling(true); - + dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); - + for (int i=0; i<=upToTaskVersion; ++i) { - TezTaskAttemptID testTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), i); + TezTaskAttemptID testTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), i); TaskAttemptImpl taImpl = dagImpl.getTaskAttempt(testTaId); Assert.assertEquals(TaskAttemptStateInternal.KILLED, taImpl.getInternalState()); Assert.assertEquals(TaskAttemptTerminationCause.EXTERNAL_PREEMPTION, taImpl.getTerminationCause()); } - + System.out.println("TestPreemption - Done running - " + info); } } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java index d6cbfa9445..13783fc416 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java @@ -360,7 +360,7 @@ public void testRecoverableSummary_DAGInCommitting() throws IOException { assertTrue(dagData.nonRecoverable); assertTrue(dagData.reason.contains("DAG Commit was in progress")); } - + @Test(timeout=5000) public void testRecoverableSummary_DAGFinishCommitting() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); @@ -456,8 +456,8 @@ public void testRecoverableSummary_VertexFinishCommitting() throws IOException { rService.handle(new DAGHistoryEvent(dagID, new VertexCommitStartedEvent(vertexId, 0L))); rService.handle(new DAGHistoryEvent(dagID, - new VertexFinishedEvent(vertexId, "v1", 10, 0L, 0L, - 0L, 0L, 0L, VertexState.SUCCEEDED, + new VertexFinishedEvent(vertexId, "v1", 10, 0L, 0L, + 0L, 0L, 0L, VertexState.SUCCEEDED, "", null, null, null, null))); rService.stop(); @@ -490,7 +490,7 @@ public void testRecoverableSummary_VertexGroupInCommitting() throws IOException null, "user", new Configuration(), null, null))); // It should be fine to skip other events, just for testing. rService.handle(new DAGHistoryEvent(dagID, - new VertexGroupCommitStartedEvent(dagID, "group_1", + new VertexGroupCommitStartedEvent(dagID, "group_1", Lists.newArrayList(TezVertexID.getInstance(dagID, 0), TezVertexID.getInstance(dagID, 1)), 0L))); rService.stop(); @@ -499,7 +499,7 @@ public void testRecoverableSummary_VertexGroupInCommitting() throws IOException assertTrue(dagData.nonRecoverable); assertTrue(dagData.reason.contains("Vertex Group Commit was in progress")); } - + @Test(timeout=5000) public void testRecoverableSummary_VertexGroupFinishCommitting() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); @@ -526,28 +526,28 @@ public void testRecoverableSummary_VertexGroupFinishCommitting() throws IOExcept TezVertexID v0 = TezVertexID.getInstance(dagID, 0); TezVertexID v1 = TezVertexID.getInstance(dagID, 1); rService.handle(new DAGHistoryEvent(dagID, - new VertexGroupCommitStartedEvent(dagID, "group_1", + new VertexGroupCommitStartedEvent(dagID, "group_1", Lists.newArrayList(v0, v1), 0L))); rService.handle(new DAGHistoryEvent(dagID, - new VertexGroupCommitFinishedEvent(dagID, "group_1", + new VertexGroupCommitFinishedEvent(dagID, "group_1", Lists.newArrayList(v0, v1), 0L))); // also write VertexFinishedEvent, otherwise it is still non-recoverable // when checking with non-summary event rService.handle(new DAGHistoryEvent(dagID, - new VertexFinishedEvent(v0, "v1", 10, 0L, 0L, - 0L, 0L, 0L, VertexState.SUCCEEDED, + new VertexFinishedEvent(v0, "v1", 10, 0L, 0L, + 0L, 0L, 0L, VertexState.SUCCEEDED, "", null, null, null, null))); rService.handle(new DAGHistoryEvent(dagID, - new VertexFinishedEvent(v1, "v1", 10, 0L, 0L, - 0L, 0L, 0L, VertexState.SUCCEEDED, + new VertexFinishedEvent(v1, "v1", 10, 0L, 0L, + 0L, 0L, 0L, VertexState.SUCCEEDED, "", null, null, null, null))); rService.stop(); - + DAGRecoveryData dagData = parser.parseRecoveryData(); assertEquals(dagID, dagData.recoveredDagID); assertFalse(dagData.nonRecoverable); } - + @Test(timeout=5000) public void testRecoverableNonSummary1() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); @@ -576,8 +576,8 @@ public void testRecoverableNonSummary1() throws IOException { rService.handle(new DAGHistoryEvent(dagID, new VertexCommitStartedEvent(vertexId, 0L))); rService.handle(new DAGHistoryEvent(dagID, - new VertexFinishedEvent(vertexId, "v1", 10, 0L, 0L, - 0L, 0L, 0L, VertexState.SUCCEEDED, + new VertexFinishedEvent(vertexId, "v1", 10, 0L, 0L, + 0L, 0L, 0L, VertexState.SUCCEEDED, "", null, null, null, null))); rService.stop(); @@ -585,7 +585,7 @@ public void testRecoverableNonSummary1() throws IOException { assertTrue(dagData.nonRecoverable); assertTrue(dagData.reason.contains("Vertex has been committed, but its full recovery events are not seen")); } - + @Test(timeout=5000) public void testRecoverableNonSummary2() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); @@ -610,10 +610,10 @@ public void testRecoverableNonSummary2() throws IOException { // It should be fine to skip other events, just for testing. TezVertexID vertexId = TezVertexID.getInstance(dagID, 0); rService.handle(new DAGHistoryEvent(dagID, - new VertexGroupCommitStartedEvent(dagID, "group_1", + new VertexGroupCommitStartedEvent(dagID, "group_1", Lists.newArrayList(TezVertexID.getInstance(dagID, 0), TezVertexID.getInstance(dagID, 1)), 0L))); rService.handle(new DAGHistoryEvent(dagID, - new VertexGroupCommitFinishedEvent(dagID, "group_1", + new VertexGroupCommitFinishedEvent(dagID, "group_1", Lists.newArrayList(TezVertexID.getInstance(dagID, 0), TezVertexID.getInstance(dagID, 1)), 0L))); rService.stop(); @@ -715,12 +715,12 @@ public void testRecoveryData() throws IOException { rService.handle(new DAGHistoryEvent(dagID, new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1), null, "user", new Configuration(), null, null))); - DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagID, 100L, + DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagID, 100L, "user", "dagName", null); DAGStartedEvent dagStartedEvent = new DAGStartedEvent(dagID, 0L, "user", "dagName"); rService.handle(new DAGHistoryEvent(dagID, dagInitedEvent)); rService.handle(new DAGHistoryEvent(dagID, dagStartedEvent)); - + // 3 vertices of this dag: v0, v1, v2 TezVertexID v0Id = TezVertexID.getInstance(dagID, 0); TezVertexID v1Id = TezVertexID.getInstance(dagID, 1); @@ -730,8 +730,8 @@ public void testRecoveryData() throws IOException { v0Id, "v0", 200L, 400L, 2, null, null, null, null); rService.handle(new DAGHistoryEvent(dagID, v0InitedEvent)); // v1 VertexFinishedEvent(KILLED) - VertexFinishedEvent v1FinishedEvent = new VertexFinishedEvent(v1Id, "v1", 2, 300L, 400L, - 500L, 600L, 700L, VertexState.KILLED, + VertexFinishedEvent v1FinishedEvent = new VertexFinishedEvent(v1Id, "v1", 2, 300L, 400L, + 500L, 600L, 700L, VertexState.KILLED, "", null, null, null, null); rService.handle(new DAGHistoryEvent(dagID, v1FinishedEvent)); // v2 VertexInitializedEvent -> VertexStartedEvent @@ -766,17 +766,17 @@ public void testRecoveryData() throws IOException { NodeId nodeId = NodeId.newInstance("localhost", 9999); TezTaskAttemptID ta0t0v2Id = TezTaskAttemptID.getInstance(t0v2Id, 0); TaskAttemptStartedEvent ta0t0v2StartedEvent = new TaskAttemptStartedEvent( - ta0t0v2Id, "v1", 0L, containerId, + ta0t0v2Id, "v1", 0L, containerId, nodeId, "", "", ""); rService.handle(new DAGHistoryEvent(dagID, ta0t0v2StartedEvent)); // attempts under t2v2 TezTaskAttemptID ta0t2v2Id = TezTaskAttemptID.getInstance(t2v2Id, 0); TaskAttemptStartedEvent ta0t2v2StartedEvent = new TaskAttemptStartedEvent( - ta0t2v2Id, "v1", 500L, containerId, + ta0t2v2Id, "v1", 500L, containerId, nodeId, "", "", ""); rService.handle(new DAGHistoryEvent(dagID, ta0t2v2StartedEvent)); TaskAttemptFinishedEvent ta0t2v2FinishedEvent = new TaskAttemptFinishedEvent( - ta0t2v2Id, "v1", 500L, 600L, + ta0t2v2Id, "v1", 500L, 600L, TaskAttemptState.SUCCEEDED, null, null, "", null, null, null, 0L, null, 0L, null, null, null, null, null); rService.handle(new DAGHistoryEvent(dagID, ta0t2v2FinishedEvent)); @@ -826,7 +826,7 @@ public void testRecoveryData() throws IOException { assertEquals(ta0t2v2FinishedEvent.getFinishTime(), ta0t2v2Data.getTaskAttemptFinishedEvent().getFinishTime()); } - // Simulate the behavior that summary event is written + // Simulate the behavior that summary event is written // but non-summary is not written to hdfs public static class MockRecoveryService extends RecoveryService{ diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestSpeculation.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestSpeculation.java index b6ec5d7302..66c504e343 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestSpeculation.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestSpeculation.java @@ -277,7 +277,7 @@ void syncWithMockAppLauncher(boolean allowScheduling, AtomicBoolean mockAppLaunc mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(allowScheduling); mockAppLauncherGoFlag.notify(); - } + } } /** @@ -371,7 +371,7 @@ public void testBasicSpeculation(boolean withProgress) throws Exception { Assert.assertEquals(successTaId, task.getSuccessfulAttempt().getTaskAttemptID()); TaskAttempt killedAttempt = task.getAttempt(killedTaId); Joiner.on(",").join(killedAttempt.getDiagnostics()).contains("Killed as speculative attempt"); - Assert.assertEquals(TaskAttemptTerminationCause.TERMINATED_EFFECTIVE_SPECULATION, + Assert.assertEquals(TaskAttemptTerminationCause.TERMINATED_EFFECTIVE_SPECULATION, killedAttempt.getTerminationCause()); if (withProgress) { // without progress updates occasionally more than 1 task speculates @@ -491,7 +491,7 @@ public void testBasicSpeculationNotUseful() throws Exception { dag.addVertex(vA); MockTezClient tezClient = createTezSession(); - + DAGClient dagClient = tezClient.submitDAG(dag); DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG(); TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), 0); @@ -510,7 +510,7 @@ public void testBasicSpeculationNotUseful() throws Exception { Assert.assertEquals(successTaId, task.getSuccessfulAttempt().getTaskAttemptID()); TaskAttempt killedAttempt = task.getAttempt(killedTaId); Joiner.on(",").join(killedAttempt.getDiagnostics()).contains("Killed speculative attempt as"); - Assert.assertEquals(TaskAttemptTerminationCause.TERMINATED_INEFFECTIVE_SPECULATION, + Assert.assertEquals(TaskAttemptTerminationCause.TERMINATED_INEFFECTIVE_SPECULATION, killedAttempt.getTerminationCause()); Assert.assertEquals(1, task.getCounters().findCounter(TaskCounter.NUM_SPECULATIONS) .getValue()); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java index 9ab4485989..f5581d300a 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java @@ -131,14 +131,14 @@ public void setUp() throws TezException { eventHandler = mock(EventHandler.class); MockClock clock = new MockClock(); - + appContext = mock(AppContext.class); doReturn(eventHandler).when(appContext).getEventHandler(); doReturn(dag).when(appContext).getCurrentDAG(); doReturn(appAcls).when(appContext).getApplicationACLs(); doReturn(amContainerMap).when(appContext).getAllContainers(); doReturn(clock).when(appContext).getClock(); - + doReturn(appAttemptId).when(appContext).getApplicationAttemptId(); doReturn(credentials).when(appContext).getAppCredentials(); NodeId nodeId = NodeId.newInstance("localhost", 0); @@ -266,7 +266,7 @@ public void testTaskEventRouting() throws Exception { assertEquals(1, taEvent.getTezEvents().size()); assertEquals(EventType.DATA_MOVEMENT_EVENT, taEvent.getTezEvents().get(0).getEventType()); - + final TaskAttemptEvent taCompleteEvent = (TaskAttemptEvent)argAllValues.get(2); assertEquals(TaskAttemptEventType.TA_DONE, taCompleteEvent.getType()); final VertexEventRouteEvent vertexRouteEvent = (VertexEventRouteEvent)argAllValues.get(3); @@ -274,7 +274,7 @@ public void testTaskEventRouting() throws Exception { assertEquals(EventType.DATA_MOVEMENT_EVENT, vertexRouteEvent.getEvents().get(0).getEventType()); } - + @Test (timeout = 5000) public void testTaskEventRoutingWithReadError() throws Exception { List events = Arrays.asList( @@ -326,13 +326,13 @@ public void testTaskEventRoutingTaskAttemptOnly() throws Exception { assertEquals("only event should be route event", TaskAttemptEventType.TA_DONE, event.getType()); } - + @Test (timeout = 5000) public void testTaskHeartbeatResponse() throws Exception { List events = new ArrayList(); List eventsToSend = new ArrayList(); TaskHeartbeatResponse response = generateHeartbeat(events, 0, 1, 2, eventsToSend); - + assertEquals(2, response.getNextFromEventId()); assertEquals(eventsToSend, response.getEvents()); } @@ -378,7 +378,7 @@ private boolean testPortRange(int port) { boolean succeedToAllocate = true; try { Configuration conf = new Configuration(); - + JobTokenIdentifier identifier = new JobTokenIdentifier(new Text( "fakeIdentifier")); Token sessionToken = new Token(identifier, diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/TestStateChangeNotifier.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/TestStateChangeNotifier.java index 17340c7166..b92789a863 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/TestStateChangeNotifier.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/TestStateChangeNotifier.java @@ -48,7 +48,7 @@ import org.slf4j.LoggerFactory; public class TestStateChangeNotifier { - + // uses the thread based notification code path but effectively blocks update // events till listeners have been notified public static class StateChangeNotifierForTest extends StateChangeNotifier { @@ -64,7 +64,7 @@ public void reset() { count.set(0); totalCount.set(0); } - + @Override protected void processedEventFromQueue() { // addedEventToQueue runs in dispatcher thread while @@ -85,7 +85,7 @@ protected void processedEventFromQueue() { } } } - + @Override protected void addedEventToQueue() { totalCount.incrementAndGet(); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/CallableEventDispatcher.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/CallableEventDispatcher.java index 9484af4a66..a4d8b0372e 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/CallableEventDispatcher.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/CallableEventDispatcher.java @@ -32,5 +32,5 @@ public void handle(CallableEvent event) { event.getCallback().onFailure(e); } } - + } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java index 863abffc26..8438ffd8e4 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java @@ -124,15 +124,15 @@ import org.junit.Test; /** - * + * * The test case of commit here are different from that in TestDAGImpl & * TestVertexImpl in that the commits here are running in separated thread. So * should need to pay some special attention. - * - * 2 kinds of commit + * + * 2 kinds of commit *
  • test XXX_OnDAGSuccess means TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS is true *
  • test XXX_OnVertexSuccess means TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS is false - * + * */ public class TestCommit { @@ -807,7 +807,7 @@ public void testVertexRouteEventErrorWhileCommitting() throws Exception { VertexManagerEvent vmEvent = VertexManagerEvent.create("vertex1", ByteBuffer.wrap(new byte[0])); TezTaskAttemptID taId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v1.getVertexId(), 0), 0); TezEvent tezEvent = new TezEvent(vmEvent, - new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", + new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", null, taId)); v1.handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent))); waitUntil(dag, DAGState.FAILED); @@ -1848,7 +1848,7 @@ public void testVertexGroupCommitFinishedEventFail_OnVertexSuccess() throws Exce false); setupDAG(createDAGPlan(true, true, "testVertexGroupCommitFinishedEventFail_OnVertexSuccess")); historyEventHandler.failVertexGroupCommitFinishedEvent = true; - + initDAG(dag); startDAG(dag); VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1"); @@ -1885,7 +1885,7 @@ public void testVertexGroupCommitFinishedEventFail_OnVertexSuccess() throws Exce Assert.assertEquals(VertexState.KILLED, v3.getState()); Assert.assertEquals(VertexTerminationCause.OTHER_VERTEX_FAILURE, v3.getTerminationCause()); Assert.assertTrue(v3.commitFutures.isEmpty()); - + Assert.assertEquals(1, v12OutputCommitter.initCounter); Assert.assertEquals(1, v12OutputCommitter.setupCounter); Assert.assertEquals(1, v12OutputCommitter.commitCounter); @@ -1903,7 +1903,7 @@ public void testDAGCommitStartedEventFail_OnDAGSuccess() throws Exception { true); setupDAG(createDAGPlan(true, true, "testDAGCommitStartedEventFail_OnDAGSuccess")); historyEventHandler.failDAGCommitStartedEvent = true; - + initDAG(dag); startDAG(dag); VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1"); @@ -1986,7 +1986,7 @@ private void _testCommitCanceled_OnDAGSuccess(DAGTerminationCause terminationCau dag.handle(new DAGEventTerminateDag(dag.getID(), terminationCause, null)); waitUntil(dag, terminationCause.getFinishedState()); - + Assert.assertEquals(terminationCause, dag.getTerminationCause()); // mean the commits have been canceled Assert.assertTrue(dag.commitFutures.isEmpty()); @@ -2010,7 +2010,7 @@ private void _testCommitCanceled_OnDAGSuccess(DAGTerminationCause terminationCau // commit is not started because ControlledThreadPoolExecutor wait before schedule tasks Assert.assertEquals(0, v12OutputCommitter.commitCounter); Assert.assertEquals(1, v12OutputCommitter.abortCounter); - + Assert.assertEquals(1, v3OutputCommitter.initCounter); Assert.assertEquals(1, v3OutputCommitter.setupCounter); // commit is not started because ControlledThreadPoolExecutor wait before schedule tasks @@ -2133,7 +2133,7 @@ public void verifyDAGFinishedEvent(TezDAGID dagId, int expectedTimes) { Assert.assertEquals(expectedTimes, actualTimes); } } - + private static class ControlledThreadPoolExecutor extends ThreadPoolExecutor { public ControlledThreadPoolExecutor(int poolSize) { @@ -2147,7 +2147,7 @@ public ControlledThreadPoolExecutor(int corePoolSize, int maximumPoolSize, } public boolean startFlag = false; - + @Override protected void beforeExecute(Thread t, Runnable r) { while(!startFlag) { diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java index 558bf27e33..7a8b39a445 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java @@ -236,7 +236,7 @@ private DAGImpl chooseDAG(TezDAGID curDAGId) { + ", dagId=" + curDAGId); } } - + private class DagEventDispatcher implements EventHandler { @Override public void handle(DAGEvent event) { @@ -276,7 +276,7 @@ public void handle(TaskAttemptEvent event) { ((EventHandler)ta).handle(event); } } - + private class VertexEventDispatcher implements EventHandler { @@ -885,7 +885,7 @@ public ListenableFuture answer(InvocationOnMock invocation) { return mockFuture; }}) .when(execService).submit((Callable) any()); - + doReturn(execService).when(appContext).getExecService(); historyEventHandler = mock(HistoryEventHandler.class); aclManager = new ACLManager("amUser"); @@ -958,7 +958,7 @@ public ListenableFuture answer(InvocationOnMock invocation) { dispatcher.init(conf); dispatcher.start(); } - + @After public void teardown() { dispatcher.await(); @@ -993,7 +993,7 @@ public void handle(AMSchedulerEvent event) { private void setupDAGWithCustomEdge(ExceptionLocation exLocation) { setupDAGWithCustomEdge(exLocation, false); } - + private void setupDAGWithCustomEdge(ExceptionLocation exLocation, boolean useLegacy) { dagWithCustomEdgeId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 4); dagPlanWithCustomEdge = createDAGWithCustomEdge(exLocation, useLegacy); @@ -1031,7 +1031,7 @@ private void startDAG(DAGImpl impl) { dispatcher.await(); Assert.assertEquals(DAGState.RUNNING, impl.getState()); } - + @Test(timeout = 5000) public void testDAGInit() { initDAG(dag); @@ -1149,7 +1149,7 @@ public void testVertexCompletion() { Assert.assertEquals((float) 2 / 11, dag.getCompletedTaskProgress(), 0.05); } - + @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testEdgeManager_GetNumDestinationTaskPhysicalInputs() { @@ -1182,14 +1182,14 @@ public void testEdgeManager_GetNumSourceTaskPhysicalOutputs() { String diag = StringUtils.join(v1.getDiagnostics(), ","); Assert.assertTrue(diag.contains(ExceptionLocation.GetNumSourceTaskPhysicalOutputs.name())); } - + @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testEdgeManager_RouteDataMovementEventToDestination() { setupDAGWithCustomEdge(ExceptionLocation.RouteDataMovementEventToDestination); dispatcher.getEventHandler().handle( new DAGEvent(dagWithCustomEdge.getID(), DAGEventType.DAG_INIT)); - dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), + dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), null)); dispatcher.await(); Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState()); @@ -1201,7 +1201,7 @@ public void testEdgeManager_RouteDataMovementEventToDestination() { TaskAttemptImpl ta1= (TaskAttemptImpl)t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskID(), 0)); DataMovementEvent daEvent = DataMovementEvent.create(ByteBuffer.wrap(new byte[0])); - TezEvent tezEvent = new TezEvent(daEvent, + TezEvent tezEvent = new TezEvent(daEvent, new EventMetaData(EventProducerConsumerType.INPUT, "vertex1", "vertex2", ta1.getTaskAttemptID())); dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent))); dispatcher.await(); @@ -1213,7 +1213,7 @@ public void testEdgeManager_RouteDataMovementEventToDestination() { String diag = StringUtils.join(v2.getDiagnostics(), ","); Assert.assertTrue(diag.contains(ExceptionLocation.RouteDataMovementEventToDestination.name())); } - + @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testEdgeManager_RouteDataMovementEventToDestinationWithLegacyRouting() { @@ -1234,7 +1234,7 @@ public void testEdgeManager_RouteDataMovementEventToDestinationWithLegacyRouting TaskAttemptImpl ta1= (TaskAttemptImpl)t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskID(), 0)); DataMovementEvent daEvent = DataMovementEvent.create(ByteBuffer.wrap(new byte[0])); - TezEvent tezEvent = new TezEvent(daEvent, + TezEvent tezEvent = new TezEvent(daEvent, new EventMetaData(EventProducerConsumerType.INPUT, "vertex1", "vertex2", ta1.getTaskAttemptID())); dispatcher.getEventHandler().handle( new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent))); @@ -1253,7 +1253,7 @@ public void testEdgeManager_RouteInputSourceTaskFailedEventToDestinationLegacyRo setupDAGWithCustomEdge(ExceptionLocation.RouteInputSourceTaskFailedEventToDestination, true); dispatcher.getEventHandler().handle( new DAGEvent(dagWithCustomEdge.getID(), DAGEventType.DAG_INIT)); - dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), + dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), null)); dispatcher.await(); Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState()); @@ -1265,14 +1265,14 @@ public void testEdgeManager_RouteInputSourceTaskFailedEventToDestinationLegacyRo Task t1= v2.getTask(0); TaskAttemptImpl ta1= (TaskAttemptImpl)t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskID(), 0)); InputFailedEvent ifEvent = InputFailedEvent.create(0, 1); - TezEvent tezEvent = new TezEvent(ifEvent, + TezEvent tezEvent = new TezEvent(ifEvent, new EventMetaData(EventProducerConsumerType.INPUT,"vertex1", "vertex2", ta1.getTaskAttemptID())); dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent))); dispatcher.await(); v2.getTaskAttemptTezEvents(ta1.getTaskAttemptID(), 0, 0, 1000); dispatcher.await(); Assert.assertEquals(VertexState.FAILED, v2.getState()); - + Assert.assertEquals(VertexState.KILLED, v1.getState()); String diag = StringUtils.join(v2.getDiagnostics(), ","); Assert.assertTrue(diag.contains(ExceptionLocation.RouteInputSourceTaskFailedEventToDestination.name())); @@ -1297,25 +1297,25 @@ public void testEdgeManager_GetNumDestinationConsumerTasks() { TaskAttemptImpl ta1= (TaskAttemptImpl)t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskID(), 0)); InputReadErrorEvent ireEvent = InputReadErrorEvent.create("", 0, 0); - TezEvent tezEvent = new TezEvent(ireEvent, + TezEvent tezEvent = new TezEvent(ireEvent, new EventMetaData(EventProducerConsumerType.INPUT,"vertex2", "vertex1", ta1.getTaskAttemptID())); dispatcher.getEventHandler().handle( new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent))); dispatcher.await(); - // + // Assert.assertEquals(VertexState.FAILED, v2.getState()); Assert.assertEquals(VertexState.KILLED, v1.getState()); String diag = StringUtils.join(v2.getDiagnostics(), ","); Assert.assertTrue(diag.contains(ExceptionLocation.GetNumDestinationConsumerTasks.name())); } - + @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testEdgeManager_RouteInputErrorEventToSource() { setupDAGWithCustomEdge(ExceptionLocation.RouteInputErrorEventToSource); dispatcher.getEventHandler().handle( new DAGEvent(dagWithCustomEdge.getID(), DAGEventType.DAG_INIT)); - dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), + dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), null)); dispatcher.await(); Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState()); @@ -1327,17 +1327,17 @@ public void testEdgeManager_RouteInputErrorEventToSource() { Task t1= v2.getTask(0); TaskAttemptImpl ta1= (TaskAttemptImpl)t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskID(), 0)); InputReadErrorEvent ireEvent = InputReadErrorEvent.create("", 0, 0); - TezEvent tezEvent = new TezEvent(ireEvent, + TezEvent tezEvent = new TezEvent(ireEvent, new EventMetaData(EventProducerConsumerType.INPUT,"vertex2", "vertex1", ta1.getTaskAttemptID())); dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent))); dispatcher.await(); - // + // Assert.assertEquals(VertexState.FAILED, v2.getState()); Assert.assertEquals(VertexState.KILLED, v1.getState()); String diag = StringUtils.join(v2.getDiagnostics(), ","); Assert.assertTrue(diag.contains(ExceptionLocation.RouteInputErrorEventToSource.name())); } - + @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testGroupDAGCompletionWithCommitSuccess() { @@ -1354,12 +1354,12 @@ public void testGroupDAGCompletionWithCommitSuccess() { Assert.assertEquals(VertexState.SUCCEEDED, v.getState()); Assert.assertEquals(i+1, groupDag.getSuccessfulVertices()); } - + Assert.assertEquals(3, groupDag.getSuccessfulVertices()); Assert.assertTrue(1.0f == groupDag.getCompletedTaskProgress()); Assert.assertEquals(DAGState.SUCCEEDED, groupDag.getState()); Assert.assertEquals(2, TotalCountingOutputCommitter.totalCommitCounter); - } + } @SuppressWarnings("unchecked") @Test(timeout = 5000) @@ -1426,7 +1426,7 @@ public void testDAGCompletionWithCommitSuccess() { Assert.assertEquals(VertexState.SUCCEEDED, v.getState()); Assert.assertEquals(i+1, mrrDag.getSuccessfulVertices()); } - + // no commit yet for (Vertex v : mrrDag.vertices.values()) { for (OutputCommitter c : v.getOutputCommitters().values()) { @@ -1437,7 +1437,7 @@ public void testDAGCompletionWithCommitSuccess() { Assert.assertEquals(1, committer.setupCounter); } } - + // dag completion and commit Vertex v = mrrDag.getVertex("vertex3"); dispatcher.getEventHandler().handle(new VertexEventTaskCompleted( @@ -1446,7 +1446,7 @@ public void testDAGCompletionWithCommitSuccess() { Assert.assertEquals(VertexState.SUCCEEDED, v.getState()); Assert.assertEquals(3, mrrDag.getSuccessfulVertices()); Assert.assertEquals(DAGState.SUCCEEDED, mrrDag.getState()); - + for (Vertex vertex : mrrDag.vertices.values()) { for (OutputCommitter c : vertex.getOutputCommitters().values()) { CountingOutputCommitter committer= (CountingOutputCommitter) c; @@ -1457,7 +1457,7 @@ public void testDAGCompletionWithCommitSuccess() { } } } - + @SuppressWarnings("unchecked") @Test(timeout=5000) public void testDAGCompletionWithCommitFailure() throws IOException { @@ -1484,7 +1484,7 @@ public void testDAGCompletionWithCommitFailure() throws IOException { ) .build()); badVertex.setAdditionalOutputs(outputs); - + startDAG(mrrDag); dispatcher.await(); for (int i=0; i<2; ++i) { @@ -1495,7 +1495,7 @@ public void testDAGCompletionWithCommitFailure() throws IOException { Assert.assertEquals(VertexState.SUCCEEDED, v.getState()); Assert.assertEquals(i+1, mrrDag.getSuccessfulVertices()); } - + // no commit yet for (Vertex v : mrrDag.vertices.values()) { for (OutputCommitter c : v.getOutputCommitters().values()) { @@ -1506,7 +1506,7 @@ public void testDAGCompletionWithCommitFailure() throws IOException { Assert.assertEquals(1, committer.setupCounter); } } - + // dag completion and commit. Exception causes all outputs to be aborted Vertex v = mrrDag.getVertex("vertex3"); dispatcher.getEventHandler().handle(new VertexEventTaskCompleted( @@ -1516,7 +1516,7 @@ public void testDAGCompletionWithCommitFailure() throws IOException { Assert.assertEquals(3, mrrDag.getSuccessfulVertices()); Assert.assertEquals(DAGState.FAILED, mrrDag.getState()); Assert.assertEquals(DAGTerminationCause.COMMIT_FAILURE, mrrDag.getTerminationCause()); - + for (Vertex vertex : mrrDag.vertices.values()) { for (OutputCommitter c : vertex.getOutputCommitters().values()) { CountingOutputCommitter committer= (CountingOutputCommitter) c; @@ -1526,7 +1526,7 @@ public void testDAGCompletionWithCommitFailure() throws IOException { } } } - + @SuppressWarnings("unchecked") @Test(timeout=5000) public void testDAGErrorAbortAllOutputs() { @@ -1543,7 +1543,7 @@ public void testDAGErrorAbortAllOutputs() { Assert.assertEquals(VertexState.SUCCEEDED, v.getState()); Assert.assertEquals(i+1, mrrDag.getSuccessfulVertices()); } - + // no commit yet for (Vertex v : mrrDag.vertices.values()) { for (OutputCommitter c : v.getOutputCommitters().values()) { @@ -1554,7 +1554,7 @@ public void testDAGErrorAbortAllOutputs() { Assert.assertEquals(1, committer.setupCounter); } } - + // vertex error -> dag error -> abort all outputs Vertex v = mrrDag.getVertex("vertex3"); dispatcher.getEventHandler().handle(new VertexEvent( @@ -1562,7 +1562,7 @@ public void testDAGErrorAbortAllOutputs() { dispatcher.await(); Assert.assertEquals(VertexState.ERROR, v.getState()); Assert.assertEquals(DAGState.ERROR, mrrDag.getState()); - + for (Vertex vertex : mrrDag.vertices.values()) { for (OutputCommitter c : vertex.getOutputCommitters().values()) { CountingOutputCommitter committer= (CountingOutputCommitter) c; @@ -1598,17 +1598,17 @@ public void testDAGErrorAbortNonSuccessfulOutputs() { Assert.assertEquals(1, committer.setupCounter); } } - + // error on vertex -> dag error Vertex errorVertex = mrrDag.getVertex("vertex3"); dispatcher.getEventHandler().handle(new VertexEvent( errorVertex.getVertexId(), VertexEventType.V_INTERNAL_ERROR)); dispatcher.await(); Assert.assertEquals(VertexState.ERROR, errorVertex.getState()); - + dispatcher.await(); Assert.assertEquals(DAGState.ERROR, mrrDag.getState()); - + for (Vertex vertex : mrrDag.vertices.values()) { for (OutputCommitter c : vertex.getOutputCommitters().values()) { CountingOutputCommitter committer= (CountingOutputCommitter) c; @@ -1622,7 +1622,7 @@ public void testDAGErrorAbortNonSuccessfulOutputs() { Assert.assertEquals(1, committer.abortCounter); Assert.assertEquals(1, committer.commitCounter); Assert.assertEquals(1, committer.initCounter); - Assert.assertEquals(1, committer.setupCounter); + Assert.assertEquals(1, committer.setupCounter); } } } @@ -1647,14 +1647,14 @@ public void testVertexReRunning() { Assert.assertEquals(VertexState.SUCCEEDED, v.getState()); Assert.assertEquals(1, dag.getSuccessfulVertices()); Assert.assertEquals(1, dag.numCompletedVertices); - + dispatcher.getEventHandler().handle( new VertexEventTaskReschedule(TezTaskID.getInstance(vId, 0))); dispatcher.await(); Assert.assertEquals(VertexState.RUNNING, v.getState()); Assert.assertEquals(0, dag.getSuccessfulVertices()); Assert.assertEquals(0, dag.numCompletedVertices); - + dispatcher.getEventHandler().handle(new VertexEventTaskCompleted( TezTaskID.getInstance(vId, 0), TaskState.SUCCEEDED)); dispatcher.await(); @@ -1662,7 +1662,7 @@ public void testVertexReRunning() { Assert.assertEquals(VertexState.SUCCEEDED, v.getState()); Assert.assertEquals(1, dag.getSuccessfulVertices()); Assert.assertEquals(1, dag.numCompletedVertices); - + } @SuppressWarnings("unchecked") diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java index 8b82a6aa3e..6682cd17a8 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java @@ -198,7 +198,7 @@ public class TestDAGRecovery { private long t1FinishedTime = t1StartedTime + rand.nextInt(100); private long ta1LaunchTime = t1StartedTime + rand.nextInt(100); private long ta1FinishedTime = ta1LaunchTime + rand.nextInt(100); - + private class DagEventDispatcher implements EventHandler { @Override public void handle(DAGEvent event) { @@ -253,7 +253,7 @@ private class AMSchedulerEventDispatcher implements EventHandler historyEvents = new ArrayList(); @@ -409,7 +409,7 @@ public void commitOutput() throws Exception { @Override public void abortOutput(State finalState) throws Exception { } - + @Override public boolean isTaskRecoverySupported() { return false; @@ -417,9 +417,9 @@ public boolean isTaskRecoverySupported() { } /** - * v1 v2 - * \ / - * \ / + * v1 v2 + * \ / + * \ / * v3 */ private DAGPlan createDAGPlan() { @@ -531,7 +531,7 @@ private DAGPlan createDAGPlan() { return dag; } - + @After public void teardown() { dispatcher.await(); @@ -543,8 +543,8 @@ public void teardown() { } dag = null; } - - + + ////////////////////////////////// DAG Recovery /////////////////////////////////////////////////// /** * RecoveryEvents: SummaryEvent_DAGFinishedEvent(SUCCEEDED) @@ -563,7 +563,7 @@ public void testDAGRecoverFromDesiredSucceeded() { assertEquals(VertexState.SUCCEEDED, dag.getVertex("vertex3").getState()); // DAG#initTime, startTime is not guaranteed to be recovered in this case } - + /** * RecoveryEvents: SummaryEvent_DAGFinishedEvent(FAILED) * Recover dag to FAILED and all of its vertices to FAILED @@ -581,7 +581,7 @@ public void testDAGRecoverFromDesiredFailed() { assertEquals(VertexState.FAILED, dag.getVertex("vertex3").getState()); // DAG#initTime, startTime is not guaranteed to be recovered in this case } - + /** * RecoveryEvents: SummaryEvent_DAGFinishedEvent(KILLED) * Recover dag to KILLED and all of its vertices to KILLED @@ -599,7 +599,7 @@ public void testDAGRecoverFromDesiredKilled() { assertEquals(VertexState.KILLED, dag.getVertex("vertex3").getState()); // DAG#initTime, startTime is not guaranteed to be recovered in this case } - + /** * RecoveryEvents: SummaryEvent_DAGFinishedEvent(ERROR) * Recover dag to ERROR and all of its vertices to ERROR @@ -617,7 +617,7 @@ public void testDAGRecoverFromDesiredError() { assertEquals(VertexState.ERROR, dag.getVertex("vertex3").getState()); // DAG#initTime, startTime is not guaranteed to be recovered in this case } - + /** * RecoveryEvents: DAGSubmittedEvent * Recover it as normal dag execution @@ -630,14 +630,14 @@ public void testDAGRecoverFromNew() { assertEquals(DAGState.RUNNING, dag.getState()); } - + /** * RecoveryEvents: DAGSubmittedEvent, DAGInitializedEvent * Recover it as normal dag execution */ @Test(timeout=5000) public void testDAGRecoverFromInited() { - DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, + DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, "user", "dagName", null); doReturn(dagInitedEvent).when(dagRecoveryData).getDAGInitializedEvent(); DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, dagRecoveryData); @@ -647,10 +647,10 @@ public void testDAGRecoverFromInited() { assertEquals(DAGState.RUNNING, dag.getState()); assertEquals(dagInitedTime, dag.initTime); } - + @Test(timeout=5000) public void testDAGRecoverFromStarted() { - DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, + DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, "user", "dagName", null); doReturn(dagInitedEvent).when(dagRecoveryData).getDAGInitializedEvent(); DAGStartedEvent dagStartedEvent = new DAGStartedEvent(dagId, dagStartedTime, "user", "dagName"); @@ -664,11 +664,11 @@ public void testDAGRecoverFromStarted() { assertEquals(dagInitedTime, dag.initTime); assertEquals(dagStartedTime, dag.startTime); } - + /////////////////////////////// Vertex Recovery ///////////////////////////////////////// - - private void initMockDAGRecoveryDataForVertex() { - DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, + + private void initMockDAGRecoveryDataForVertex() { + DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, "user", "dagName", null); DAGStartedEvent dagStartedEvent = new DAGStartedEvent(dagId, dagStartedTime, "user", "dagName"); doReturn(dagInitedEvent).when(dagRecoveryData).getDAGInitializedEvent(); @@ -677,15 +677,15 @@ private void initMockDAGRecoveryDataForVertex() { /** * RecoveryEvents: - * DAG: DAGInitedEvent -> DAGStartedEvent + * DAG: DAGInitedEvent -> DAGStartedEvent * V1: No any event - * - * Reinitialize V1 again. + * + * Reinitialize V1 again. */ @Test(timeout=5000) public void testVertexRecoverFromNew() { initMockDAGRecoveryDataForVertex(); - + DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, dagRecoveryData); dag.handle(recoveryEvent); dispatcher.await(); @@ -699,25 +699,25 @@ public void testVertexRecoverFromNew() { assertEquals(VertexState.RUNNING, v2.getState()); assertEquals(VertexState.INITED, v3.getState()); } - + /** * RecoveryEvents: - * DAG: DAGInitedEvent -> DAGStartedEvent + * DAG: DAGInitedEvent -> DAGStartedEvent * V1: VertexInitializedEvent - * - * Reinitialize V1 again. + * + * Reinitialize V1 again. */ @Test(timeout=5000) public void testVertexRecoverFromInited() { initMockDAGRecoveryDataForVertex(); List inputGeneratedTezEvents = new ArrayList(); - VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, - "vertex1", 0L, v1InitedTime, + VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, + "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null); VertexRecoveryData vertexRecoveryData = new VertexRecoveryData(v1InitedEvent, null, null, null, null, false); doReturn(vertexRecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id); - + DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, dagRecoveryData); dag.handle(recoveryEvent); dispatcher.await(); @@ -731,27 +731,27 @@ public void testVertexRecoverFromInited() { assertEquals(VertexState.RUNNING, v2.getState()); assertEquals(VertexState.INITED, v3.getState()); } - + /** * RecoveryEvents: - * DAG: DAGInitedEvent -> DAGStartedEvent + * DAG: DAGInitedEvent -> DAGStartedEvent * V1: VertexReconfigrationDoneEvent -> VertexInitializedEvent - * + * * Reinitialize V1 again. */ @Test//(timeout=5000) public void testVertexRecoverFromInitedAndReconfigureDone() { initMockDAGRecoveryDataForVertex(); List inputGeneratedTezEvents = new ArrayList(); - VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, - "vertex1", 0L, v1InitedTime, + VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, + "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null); - VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, + VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, 0L, v1NumTask, null, null, null, false); VertexRecoveryData vertexRecoveryData = new VertexRecoveryData(v1InitedEvent, v1ReconfigureDoneEvent, null, null, new HashMap(), false); doReturn(vertexRecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id); - + DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, dagRecoveryData); dag.handle(recoveryEvent); dispatcher.await(); @@ -772,7 +772,7 @@ public void testVertexRecoverFromInitedAndReconfigureDone() { /** * RecoveryEvents: - * DAG: DAGInitedEvent -> DAGStartedEvent + * DAG: DAGInitedEvent -> DAGStartedEvent * V1: VertexReconfigrationDoneEvent -> VertexInitializedEvent -> VertexStartedEvent * * Reinitialize V1 again. @@ -1059,16 +1059,16 @@ public void testMultipleVertexRecoverFromVertex() { // reinitialize v3 assertEquals(VertexState.INITED, v3.getState()); } - + /////////////////////////////// Task //////////////////////////////////////////////////////////// - + private void initMockDAGRecoveryDataForTask() { List inputGeneratedTezEvents = new ArrayList(); - VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, - "vertex1", 0L, v1InitedTime, + VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, + "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null); Map rootInputSpecs = new HashMap(); - VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, + VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, 0L, v1NumTask, null, null, rootInputSpecs, true); VertexStartedEvent v1StartedEvent = new VertexStartedEvent(v1Id, 0L, v1StartedTime); TaskStartedEvent v1taskStartedEvent = new TaskStartedEvent(t1v1Id, "vertex1", 0L, 0L); @@ -1078,8 +1078,8 @@ private void initMockDAGRecoveryDataForTask() { v1taskRecoveryDataMap.put(t1v1Id, v1taskRecoveryData); VertexRecoveryData v1RecoveryData = new VertexRecoveryData(v1InitedEvent, v1ReconfigureDoneEvent, v1StartedEvent, null, v1taskRecoveryDataMap, false); - - DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, + + DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, "user", "dagName", null); DAGStartedEvent dagStartedEvent = new DAGStartedEvent(dagId, dagStartedTime, "user", "dagName"); doReturn(v1RecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id); @@ -1098,7 +1098,7 @@ public void testTaskRecoverFromKilled() { 0L, 0L, null, TaskState.KILLED, "", null, 4); TaskRecoveryData taskRecoveryData = new TaskRecoveryData(null, taskFinishedEvent, null); doReturn(taskRecoveryData).when(dagRecoveryData).getTaskRecoveryData(t1v1Id); - + dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); @@ -1107,7 +1107,7 @@ public void testTaskRecoverFromKilled() { assertEquals(TaskStateInternal.KILLED, task.getInternalState()); assertEquals(1, vertex1.getCompletedTasks()); } - + /** * RecoveryEvent: TaskStartedEvent * Recover it to Scheduled @@ -1118,15 +1118,15 @@ public void testTaskRecoverFromStarted() { TaskStartedEvent taskStartedEvent = new TaskStartedEvent(t1v1Id, "v1", 0L, 0L); TaskRecoveryData taskRecoveryData = new TaskRecoveryData(taskStartedEvent, null, null); doReturn(taskRecoveryData).when(dagRecoveryData).getTaskRecoveryData(t1v1Id); - + dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); - + VertexImpl vertex1 = (VertexImpl) dag.getVertex(v1Id); TaskImpl task = (TaskImpl)vertex1.getTask(t1v1Id); assertEquals(TaskStateInternal.SCHEDULED, task.getInternalState()); } - + /** * RecoveryEvent: TaskStartedEvent -> TaskFinishedEvent * Recover it to Scheduled @@ -1138,14 +1138,14 @@ public void testTaskRecoverFromSucceeded() { TaskFinishedEvent taskFinishedEvent = new TaskFinishedEvent(t1v1Id, "v1", 0L, 0L, null, TaskState.SUCCEEDED, "", null, 4); TaskAttemptStartedEvent taStartedEvent = new TaskAttemptStartedEvent( - ta1t1v1Id, "v1", 0L, mock(ContainerId.class), + ta1t1v1Id, "v1", 0L, mock(ContainerId.class), mock(NodeId.class), "", "", ""); List taGeneratedEvents = new ArrayList(); EventMetaData metadata = new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex3", ta1t1v2Id); taGeneratedEvents.add(new TezEvent(DataMovementEvent.create(ByteBuffer.wrap(new byte[0])), metadata)); TaskAttemptFinishedEvent taFinishedEvent = new TaskAttemptFinishedEvent( - ta1t1v1Id, "v1", 0L, 0L, + ta1t1v1Id, "v1", 0L, 0L, TaskAttemptState.SUCCEEDED, null, null, "", null, null, taGeneratedEvents, 0L, null, 0L, null, null, null, null, null); TaskAttemptRecoveryData taRecoveryData = new TaskAttemptRecoveryData(taStartedEvent, taFinishedEvent); @@ -1158,7 +1158,7 @@ public void testTaskRecoverFromSucceeded() { dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); - + VertexImpl vertex1 = (VertexImpl) dag.getVertex(v1Id); TaskImpl task = (TaskImpl)vertex1.getTask(t1v1Id); TaskAttemptImpl taskAttempt = (TaskAttemptImpl)task.getAttempt(ta1t1v1Id); @@ -1169,25 +1169,25 @@ public void testTaskRecoverFromSucceeded() { } /////////////////////////////// TaskAttempt Recovery ///////////////////////////////////////////////////// - + private void initMockDAGRecoveryDataForTaskAttempt() { TaskStartedEvent t1StartedEvent = new TaskStartedEvent(t1v1Id, "vertex1", 0L, t1StartedTime); TaskRecoveryData taskRecoveryData = new TaskRecoveryData(t1StartedEvent, null, null); Map taskRecoveryDataMap = new HashMap(); taskRecoveryDataMap.put(t1v1Id, taskRecoveryData); - + List inputGeneratedTezEvents = new ArrayList(); - VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, - "vertex1", 0L, v1InitedTime, + VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, + "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null); Map rootInputSpecs = new HashMap(); - VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, + VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, 0L, v1NumTask, null, null, rootInputSpecs, true); VertexStartedEvent v1StartedEvent = new VertexStartedEvent(v1Id, 0L, v1StartedTime); VertexRecoveryData v1RecoveryData = new VertexRecoveryData(v1InitedEvent, v1ReconfigureDoneEvent, v1StartedEvent, null, taskRecoveryDataMap, false); - - DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, + + DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, "user", "dagName", null); DAGStartedEvent dagStartedEvent = new DAGStartedEvent(dagId, dagStartedTime, "user", "dagName"); doReturn(v1RecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id); @@ -1203,12 +1203,12 @@ private void initMockDAGRecoveryDataForTaskAttempt() { public void testTARecoverFromNewToFailed() { initMockDAGRecoveryDataForTaskAttempt(); TaskAttemptFinishedEvent taFinishedEvent = new TaskAttemptFinishedEvent( - ta1t1v1Id, "v1", ta1LaunchTime, ta1FinishedTime, + ta1t1v1Id, "v1", ta1LaunchTime, ta1FinishedTime, TaskAttemptState.FAILED, TaskFailureType.NON_FATAL, TaskAttemptTerminationCause.CONTAINER_LAUNCH_FAILED, "", null, null, null, 0L, null, 0L, null, null, null, null, null); TaskAttemptRecoveryData taRecoveryData = new TaskAttemptRecoveryData(null, taFinishedEvent); doReturn(taRecoveryData).when(dagRecoveryData).getTaskAttemptRecoveryData(ta1t1v1Id); - + dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); @@ -1223,7 +1223,7 @@ public void testTARecoverFromNewToFailed() { assertEquals(2, task.getAttempts().size()); assertEquals(ta1FinishedTime, taskAttempt.getFinishTime()); } - + /** * RecoveryEvents: TaskAttemptFinishedEvent (KILLED) * Recover it to KILLED @@ -1232,12 +1232,12 @@ public void testTARecoverFromNewToFailed() { public void testTARecoverFromNewToKilled() { initMockDAGRecoveryDataForTaskAttempt(); TaskAttemptFinishedEvent taFinishedEvent = new TaskAttemptFinishedEvent( - ta1t1v1Id, "v1", ta1LaunchTime, ta1FinishedTime, + ta1t1v1Id, "v1", ta1LaunchTime, ta1FinishedTime, TaskAttemptState.KILLED, null, TaskAttemptTerminationCause.TERMINATED_BY_CLIENT, "", null, null, null, 0L, null, 0L, null, null, null, null, null); TaskAttemptRecoveryData taRecoveryData = new TaskAttemptRecoveryData(null, taFinishedEvent); doReturn(taRecoveryData).when(dagRecoveryData).getTaskAttemptRecoveryData(ta1t1v1Id); - + dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); @@ -1250,7 +1250,7 @@ public void testTARecoverFromNewToKilled() { assertEquals(0, task.failedAttempts); assertEquals(ta1FinishedTime, taskAttempt.getFinishTime()); } - + /** * RecoveryEvents: TaskAttemptStartedEvent * Recover it to KILLED @@ -1259,11 +1259,11 @@ public void testTARecoverFromNewToKilled() { public void testTARecoverFromRunning() { initMockDAGRecoveryDataForTaskAttempt(); TaskAttemptStartedEvent taStartedEvent = new TaskAttemptStartedEvent( - ta1t1v1Id, "v1", ta1LaunchTime, mock(ContainerId.class), + ta1t1v1Id, "v1", ta1LaunchTime, mock(ContainerId.class), mock(NodeId.class), "", "", ""); TaskAttemptRecoveryData taRecoveryData = new TaskAttemptRecoveryData(taStartedEvent, null); doReturn(taRecoveryData).when(dagRecoveryData).getTaskAttemptRecoveryData(ta1t1v1Id); - + dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); @@ -1284,7 +1284,7 @@ public void testTARecoverFromRunning() { public void testTARecoverFromSucceeded() { initMockDAGRecoveryDataForTaskAttempt(); TaskAttemptStartedEvent taStartedEvent = new TaskAttemptStartedEvent( - ta1t1v1Id, "v1", ta1LaunchTime, mock(ContainerId.class), + ta1t1v1Id, "v1", ta1LaunchTime, mock(ContainerId.class), mock(NodeId.class), "", "", ""); List taGeneratedEvents = new ArrayList(); EventMetaData sourceInfo = new EventMetaData(EventProducerConsumerType.INPUT, "vertex1", @@ -1292,7 +1292,7 @@ public void testTARecoverFromSucceeded() { taGeneratedEvents.add(new TezEvent(DataMovementEvent.create(ByteBuffer.wrap(new byte[0])), sourceInfo)); TaskAttemptFinishedEvent taFinishedEvent = new TaskAttemptFinishedEvent( - ta1t1v1Id, "v1", ta1LaunchTime, ta1FinishedTime, + ta1t1v1Id, "v1", ta1LaunchTime, ta1FinishedTime, TaskAttemptState.SUCCEEDED, null, null, "", null, null, taGeneratedEvents, 0L, null, 0L, null, null, null, null, null); TaskAttemptRecoveryData taRecoveryData = new TaskAttemptRecoveryData(taStartedEvent, taFinishedEvent); @@ -1300,7 +1300,7 @@ public void testTARecoverFromSucceeded() { dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); - + TaskImpl task = (TaskImpl)dag.getVertex(v1Id).getTask(t1v1Id); TaskAttemptImpl taskAttempt = (TaskAttemptImpl)task.getAttempt(ta1t1v1Id); assertEquals(TaskAttemptStateInternal.SUCCEEDED, taskAttempt.getInternalState()); @@ -1323,32 +1323,32 @@ public void testTARecoverFromSucceeded_OutputCommitterRecoveryNotSupported() thr // t1v2: TaskStartedEvent // v2: VertexInitializedEvent -> VertexConfigurationDoneEvent -> VertexStartedEvent TaskAttemptStartedEvent taStartedEvent = new TaskAttemptStartedEvent( - ta1t1v2Id, "vertex2", ta1LaunchTime, mock(ContainerId.class), + ta1t1v2Id, "vertex2", ta1LaunchTime, mock(ContainerId.class), mock(NodeId.class), "", "", ""); List taGeneratedEvents = new ArrayList(); EventMetaData metadata = new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex3", ta1t1v2Id); taGeneratedEvents.add(new TezEvent(DataMovementEvent.create(ByteBuffer.wrap(new byte[0])), metadata)); TaskAttemptFinishedEvent taFinishedEvent = new TaskAttemptFinishedEvent( - ta1t1v2Id, "vertex2", ta1LaunchTime, ta1FinishedTime, + ta1t1v2Id, "vertex2", ta1LaunchTime, ta1FinishedTime, TaskAttemptState.SUCCEEDED, null, null, "", null, null, taGeneratedEvents, 0L, null, 0L, null, null, null, null, null); TaskAttemptRecoveryData taRecoveryData = new TaskAttemptRecoveryData(taStartedEvent, taFinishedEvent); - doReturn(taRecoveryData).when(dagRecoveryData).getTaskAttemptRecoveryData(ta1t1v2Id); + doReturn(taRecoveryData).when(dagRecoveryData).getTaskAttemptRecoveryData(ta1t1v2Id); Map taRecoveryDataMap = new HashMap(); taRecoveryDataMap.put(ta1t1v2Id, taRecoveryData); - + TaskStartedEvent t1StartedEvent = new TaskStartedEvent(t1v2Id, "vertex2", 0L, t1StartedTime); TaskRecoveryData taskRecoveryData = new TaskRecoveryData(t1StartedEvent, null, taRecoveryDataMap); Map taskRecoveryDataMap = new HashMap(); taskRecoveryDataMap.put(t1v2Id, taskRecoveryData); doReturn(taskRecoveryData).when(dagRecoveryData).getTaskRecoveryData(t1v2Id); - VertexInitializedEvent v2InitedEvent = new VertexInitializedEvent(v2Id, - "vertex2", 0L, v1InitedTime, + VertexInitializedEvent v2InitedEvent = new VertexInitializedEvent(v2Id, + "vertex2", 0L, v1InitedTime, v1NumTask, "", null, null, null); - VertexConfigurationDoneEvent v2ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v2Id, + VertexConfigurationDoneEvent v2ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v2Id, 0L, v1NumTask, null, null, null, false); VertexStartedEvent v2StartedEvent = new VertexStartedEvent(v2Id, 0L, v1StartedTime); VertexRecoveryData v2RecoveryData = new VertexRecoveryData(v2InitedEvent, @@ -1358,7 +1358,7 @@ public void testTARecoverFromSucceeded_OutputCommitterRecoveryNotSupported() thr dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); - + TaskImpl task = (TaskImpl)dag.getVertex(v2Id).getTask(t1v2Id); TaskAttemptImpl taskAttempt = (TaskAttemptImpl)task.getAttempt(ta1t1v2Id); assertEquals(TaskAttemptStateInternal.KILLED, taskAttempt.getInternalState()); @@ -1378,18 +1378,18 @@ public void testTARecoverFromSucceeded_OutputCommitterRecoveryNotSupported() thr public void testTARecoverFromFailed() { initMockDAGRecoveryDataForTaskAttempt(); TaskAttemptStartedEvent taStartedEvent = new TaskAttemptStartedEvent( - ta1t1v1Id, "v1", ta1LaunchTime, mock(ContainerId.class), + ta1t1v1Id, "v1", ta1LaunchTime, mock(ContainerId.class), mock(NodeId.class), "", "", ""); TaskAttemptFinishedEvent taFinishedEvent = new TaskAttemptFinishedEvent( - ta1t1v1Id, "v1", ta1LaunchTime, ta1FinishedTime, + ta1t1v1Id, "v1", ta1LaunchTime, ta1FinishedTime, TaskAttemptState.FAILED, TaskFailureType.NON_FATAL, TaskAttemptTerminationCause.INPUT_READ_ERROR, "", null, null, null, 0L, null, 0L, null, null, null, null, null); TaskAttemptRecoveryData taRecoveryData = new TaskAttemptRecoveryData(taStartedEvent, taFinishedEvent); doReturn(taRecoveryData).when(dagRecoveryData).getTaskAttemptRecoveryData(ta1t1v1Id); - + dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); - + TaskImpl task = (TaskImpl)dag.getVertex(v1Id).getTask(t1v1Id); TaskAttemptImpl taskAttempt = (TaskAttemptImpl)task.getAttempt(ta1t1v1Id); assertEquals(TaskAttemptStateInternal.FAILED, taskAttempt.getInternalState()); @@ -1409,18 +1409,18 @@ public void testTARecoverFromFailed() { public void testTARecoverFromKilled() { initMockDAGRecoveryDataForTaskAttempt(); TaskAttemptStartedEvent taStartedEvent = new TaskAttemptStartedEvent( - ta1t1v1Id, "v1", ta1LaunchTime, mock(ContainerId.class), + ta1t1v1Id, "v1", ta1LaunchTime, mock(ContainerId.class), mock(NodeId.class), "", "", ""); TaskAttemptFinishedEvent taFinishedEvent = new TaskAttemptFinishedEvent( - ta1t1v1Id, "v1", ta1FinishedTime, ta1FinishedTime, + ta1t1v1Id, "v1", ta1FinishedTime, ta1FinishedTime, TaskAttemptState.KILLED, null, TaskAttemptTerminationCause.TERMINATED_BY_CLIENT, "", null, null, null, 0L, null, 0L, null, null, null, null, null); TaskAttemptRecoveryData taRecoveryData = new TaskAttemptRecoveryData(taStartedEvent, taFinishedEvent); doReturn(taRecoveryData).when(dagRecoveryData).getTaskAttemptRecoveryData(ta1t1v1Id); - + dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); - + TaskImpl task = (TaskImpl)dag.getVertex(v1Id).getTask(t1v1Id); TaskAttemptImpl taskAttempt = (TaskAttemptImpl)task.getAttempt(ta1t1v1Id); assertEquals(TaskAttemptStateInternal.KILLED, taskAttempt.getInternalState()); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java index d224377c02..ac6186e732 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java @@ -49,8 +49,8 @@ public void handle(TaskAttemptEventSchedule event) { this.events.add(event); } } - - + + @Test(timeout=5000) public void testDAGSchedulerNaturalOrder() { MockEventHandler mockEventHandler = new MockEventHandler(); @@ -72,10 +72,10 @@ public void testDAGSchedulerNaturalOrder() { .thenReturn(vId1).thenReturn(vId1) .thenReturn(vId2).thenReturn(vId2) .thenReturn(vId3).thenReturn(vId3); - + DAGEventSchedulerUpdate event = new DAGEventSchedulerUpdate( - DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt); - + DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt); + DAGScheduler scheduler = new DAGSchedulerNaturalOrder(mockDag, mockEventHandler); scheduler.scheduleTaskEx(event); @@ -91,7 +91,7 @@ public void testDAGSchedulerNaturalOrder() { Assert.assertEquals(43, mockEventHandler.event.getPriorityHighLimit()); Assert.assertEquals(45, mockEventHandler.event.getPriorityLowLimit()); } - + @Test(timeout=5000) public void testConcurrencyLimit() { MockEventHandler mockEventHandler = new MockEventHandler(); @@ -101,18 +101,18 @@ public void testConcurrencyLimit() { TezVertexID vId1 = TezVertexID.fromString("vertex_1436907267600_195589_1_01"); TezTaskID tId0 = TezTaskID.getInstance(vId0, 0); TezTaskID tId1 = TezTaskID.getInstance(vId1, 0); - + TaskAttempt mockAttempt; Vertex mockVertex = mock(Vertex.class); when(mockDag.getVertex((TezVertexID) any())).thenReturn(mockVertex); when(mockVertex.getDistanceFromRoot()).thenReturn(0); when(mockVertex.getVertexId()).thenReturn(vId0); - + DAGScheduler scheduler = new DAGSchedulerNaturalOrder(mockDag, mockEventHandler); scheduler.addVertexConcurrencyLimit(vId0, 0); // not effective - + // schedule beyond limit and it gets scheduled mockAttempt = mock(TaskAttempt.class); when(mockAttempt.getTaskAttemptID()).thenReturn(TezTaskAttemptID.getInstance(tId0, 0)); @@ -129,14 +129,14 @@ public void testConcurrencyLimit() { scheduler.scheduleTask(new DAGEventSchedulerUpdate( DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt)); Assert.assertEquals(3, mockEventHandler.events.size()); - + mockEventHandler.events.clear(); List mockAttempts = Lists.newArrayList(); int completed = 0; int requested = 0; int scheduled = 0; - scheduler.addVertexConcurrencyLimit(vId1, 2); // effective - + scheduler.addVertexConcurrencyLimit(vId1, 2); // effective + // schedule beyond limit and it gets buffered mockAttempt = mock(TaskAttempt.class); mockAttempts.add(mockAttempt); @@ -147,7 +147,7 @@ public void testConcurrencyLimit() { Assert.assertEquals(mockAttempts.get(scheduled).getTaskAttemptID(), mockEventHandler.events.get(scheduled).getTaskAttemptID()); // matches order scheduled++; - + mockAttempt = mock(TaskAttempt.class); mockAttempts.add(mockAttempt); when(mockAttempt.getTaskAttemptID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++)); @@ -157,7 +157,7 @@ public void testConcurrencyLimit() { Assert.assertEquals(mockAttempts.get(scheduled).getTaskAttemptID(), mockEventHandler.events.get(scheduled).getTaskAttemptID()); // matches order scheduled++; - + mockAttempt = mock(TaskAttempt.class); mockAttempts.add(mockAttempt); when(mockAttempt.getTaskAttemptID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++)); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestEdge.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestEdge.java index 638db8675d..127cec9199 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestEdge.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestEdge.java @@ -99,7 +99,7 @@ public void testOneToOneEdgeManager() { } catch (IllegalStateException e) { Assert.assertTrue(e.getMessage().contains("1-1 source and destination task counts must match")); } - + // now make it consistent when(mockContext.getDestinationVertexNumTasks()).thenReturn(3); manager.routeDataMovementEventToDestination(event, 1, 1, destinationTaskAndInputIndices); @@ -169,25 +169,25 @@ public void testCompositeEventHandling() throws TezException { DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, mock(OutputDescriptor.class), mock(InputDescriptor.class)); Edge edge = new Edge(edgeProp, eventHandler, new TezConfiguration()); - + TezVertexID srcVertexID = createVertexID(1); TezVertexID destVertexID = createVertexID(2); LinkedHashMap srcTasks = mockTasks(srcVertexID, 1); LinkedHashMap destTasks = mockTasks(destVertexID, 5); - + TezTaskID srcTaskID = srcTasks.keySet().iterator().next(); - + Vertex srcVertex = mockVertex("src", srcVertexID, srcTasks); Vertex destVertex = mockVertex("dest", destVertexID, destTasks); - + edge.setSourceVertex(srcVertex); edge.setDestinationVertex(destVertex); edge.initialize(); - + TezTaskAttemptID srcTAID = createTAIDForTest(srcTaskID, 2); // Task0, Attempt 0 - + EventMetaData srcMeta = new EventMetaData(EventProducerConsumerType.OUTPUT, "consumerVertex", "producerVertex", srcTAID); - + // Verification via a CompositeEvent CompositeDataMovementEvent cdmEvent = CompositeDataMovementEvent.create(0, destTasks.size(), ByteBuffer.wrap("bytes".getBytes())); @@ -248,7 +248,7 @@ private LinkedHashMap mockTasks(TezVertexID vertexID, int numTa } return tasks; } - + private Vertex mockVertex(String name, TezVertexID vertexID, LinkedHashMap tasks) { Vertex vertex = mock(Vertex.class); doReturn(vertexID).when(vertex).getVertexId(); @@ -261,13 +261,13 @@ private Vertex mockVertex(String name, TezVertexID vertexID, LinkedHashMap mockInputVertices = + HashMap mockInputVertices = new HashMap(); final String mockSrcVertexId1 = "Vertex1"; EdgeProperty eProp1 = EdgeProperty.create( @@ -74,9 +74,9 @@ public void testBasic() { SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in")); - + final String mockManagedVertexId = "Vertex4"; - + mockInputVertices.put(mockSrcVertexId1, eProp1); mockInputVertices.put(mockSrcVertexId2, eProp2); mockInputVertices.put(mockSrcVertexId3, eProp3); @@ -88,7 +88,7 @@ public void testBasic() { when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(2); when(mockContext.getVertexNumTasks(mockSrcVertexId2)).thenReturn(2); when(mockContext.getVertexNumTasks(mockSrcVertexId3)).thenReturn(2); - + final HashSet scheduledTasks = new HashSet(); doAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { @@ -100,7 +100,7 @@ public Object answer(InvocationOnMock invocation) { } return null; }}).when(mockContext).scheduleTasks(anyList()); - + List emptyCompletions = null; ImmediateStartVertexManager manager = new ImmediateStartVertexManager(mockContext); manager.initialize(); @@ -130,5 +130,5 @@ public Object answer(InvocationOnMock invocation) throws Exception { verify(mockContext, times(2)).scheduleTasks(anyList()); Assert.assertEquals(4, scheduledTasks.size()); } - + } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java index 689b3ff41c..e632fd8a65 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java @@ -145,7 +145,7 @@ public FileStatus getFileStatus(Path f) throws IOException { return new FileStatus(1, false, 1, 1, 1, f); } } - + AppContext appCtx; TezConfiguration vertexConf = new TezConfiguration(); TaskLocationHint locationHint; @@ -158,7 +158,7 @@ public FileStatus getFileStatus(Path f) throws IOException { public static void setup() { MockDNSToSwitchMapping.initializeMockRackResolver(); } - + @Before public void setupTest() { appCtx = mock(AppContext.class); @@ -218,7 +218,7 @@ public void testLocalityRequest() { fail("Second event not of type " + AMSchedulerEventTALaunchRequest.class.getName()); } - + // TODO Move the Rack request check to the client after TEZ-125 is fixed. Set requestedRacks = taImpl.taskRacks; assertEquals(1, requestedRacks.size()); @@ -537,7 +537,7 @@ taListener, taskConf, new SystemClock(), assertEquals(2, taImpl.getDiagnostics().size()); assertEquals("Terminated", taImpl.getDiagnostics().get(1)); - + // check that original error cause is retained assertEquals(TaskAttemptTerminationCause.APPLICATION_ERROR, taImpl.getTerminationCause()); } @@ -721,7 +721,7 @@ taListener, taskConf, new SystemClock(), assertEquals(0, taImpl.getDiagnostics().size()); assertEquals(TaskAttemptTerminationCause.UNKNOWN_ERROR, taImpl.getTerminationCause()); } - + @Test(timeout = 5000) public void testLastDataEventRecording() throws Exception { ApplicationId appId = ApplicationId.newInstance(1, 2); @@ -771,7 +771,7 @@ taListener, taskConf, new SystemClock(), taImpl.handle(new TaskAttemptEventStartedRemotely(taskAttemptID)); assertEquals("Task attempt is not in the RUNNING state", taImpl.getState(), TaskAttemptState.RUNNING); - + long ts1 = 1024; long ts2 = 2048; TezTaskAttemptID mockId1 = mock(TezTaskAttemptID.class); @@ -806,7 +806,7 @@ taListener, taskConf, new SystemClock(), assertEquals(ts2, taImpl.lastDataEvents.get(1).getTimestamp()); assertEquals(mockId2, taImpl.lastDataEvents.get(1).getTaskAttemptId()); // over-write earlier value } - + @Test(timeout = 5000) public void testFailure() throws Exception { ApplicationId appId = ApplicationId.newInstance(1, 2); @@ -864,9 +864,9 @@ taListener, taskConf, new SystemClock(), verifyEventType( arg.getAllValues().subList(0, expectedEventsAtRunning), SpeculatorEventTaskAttemptStatusUpdate.class, 1); - + taImpl.handle(new TaskAttemptEventStatusUpdate(taskAttemptID, new TaskStatusUpdateEvent(null, 0.1f, null, false))); - + taImpl.handle(new TaskAttemptEventAttemptFailed(taskAttemptID, TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, "0", TaskAttemptTerminationCause.APPLICATION_ERROR)); @@ -1271,7 +1271,7 @@ public void testNoProgressFail() throws Exception { assertEquals("Task attempt is not in the RUNNING state", taImpl.getState(), TaskAttemptState.RUNNING); verify(mockHeartbeatHandler).register(taskAttemptID); - + when(mockClock.getTime()).thenReturn(100l); taImpl.handle(new TaskAttemptEventStatusUpdate( taskAttemptID, new TaskStatusUpdateEvent(null, 0.1f, null, true))); @@ -1317,7 +1317,7 @@ public void testEventSerializingHash() throws Exception { TezTaskAttemptID taID11 = TezTaskAttemptID.getInstance(taskID1, 0); TezTaskAttemptID taID12 = TezTaskAttemptID.getInstance(taskID1, 1); TezTaskAttemptID taID21 = TezTaskAttemptID.getInstance(taskID2, 1); - + TaskAttemptEvent taEventFail11 = new TaskAttemptEvent(taID11, TaskAttemptEventType.TA_FAILED); TaskAttemptEvent taEventKill11 = new TaskAttemptEvent(taID11, TaskAttemptEventType.TA_KILL_REQUEST); TaskAttemptEvent taEventKill12 = new TaskAttemptEvent(taID12, TaskAttemptEventType.TA_KILL_REQUEST); @@ -1325,7 +1325,7 @@ public void testEventSerializingHash() throws Exception { TaskEvent tEventKill1 = new TaskEvent(taskID1, TaskEventType.T_ATTEMPT_KILLED); TaskEvent tEventFail1 = new TaskEvent(taskID1, TaskEventType.T_ATTEMPT_FAILED); TaskEvent tEventFail2 = new TaskEvent(taskID2, TaskEventType.T_ATTEMPT_FAILED); - + // all of them should have the same value assertEquals(taEventFail11.getSerializingHash(), taEventKill11.getSerializingHash()); assertEquals(taEventKill11.getSerializingHash(), taEventKill12.getSerializingHash()); @@ -1475,10 +1475,10 @@ taListener, taskConf, new SystemClock(), verifyEventType( arg.getAllValues().subList(0, expectedEventsAtRunning), SpeculatorEventTaskAttemptStatusUpdate.class, 1); - - taImpl.handle(new TaskAttemptEventStatusUpdate(taskAttemptID, + + taImpl.handle(new TaskAttemptEventStatusUpdate(taskAttemptID, new TaskStatusUpdateEvent(null, 0.1f, null, false))); - + taImpl.handle(new TaskAttemptEvent(taskAttemptID, TaskAttemptEventType.TA_DONE)); assertEquals("Task attempt is not in the SUCCEEDED state", taImpl.getState(), @@ -1504,7 +1504,7 @@ taListener, taskConf, new SystemClock(), arg.getAllValues().subList(expectedEventsAtRunning, expectedEventsAfterTerminating), SpeculatorEventTaskAttemptStatusUpdate.class, 2); } - + @Test(timeout = 5000) // Ensure Container Preemption race with task completion is handled correctly by // the TaskAttempt @@ -1704,7 +1704,7 @@ taListener, taskConf, new SystemClock(), "InternalError occurred trying to handle TA_TEZ_EVENT_UPDATE in KILLED state", eventHandler.internalError); } - + @Test(timeout = 5000) // Ensure node failure on Successful Leaf tasks do not cause them to be marked as KILLED public void testNodeFailedLeafVertex() throws Exception { @@ -1783,7 +1783,7 @@ taListener, taskConf, new SystemClock(), expectedEventsAfterTerminating), DAGEventCounterUpdate.class, 1); // Send out a Node Failure. - taImpl.handle(new TaskAttemptEventNodeFailed(taskAttemptID, "NodeDecomissioned", + taImpl.handle(new TaskAttemptEventNodeFailed(taskAttemptID, "NodeDecomissioned", TaskAttemptTerminationCause.NODE_FAILED)); // Verify no additional events @@ -1884,7 +1884,7 @@ taListener, taskConf, new SystemClock(), when(appCtx.getCurrentDAG()).thenReturn(mockDAG); TezEvent tzEvent = new TezEvent(mockReEvent, mockMeta); taImpl.handle(new TaskAttemptEventOutputFailed(taskAttemptID, tzEvent, 11)); - + // failure threshold not met. state is SUCCEEDED assertEquals("Task attempt is not in succeeded state", taImpl.getState(), TaskAttemptState.SUCCEEDED); @@ -1908,7 +1908,7 @@ taListener, taskConf, new SystemClock(), when(destVertex.getRunningTasks()).thenReturn(11); when(mockDAG.getVertex(destVertexID)).thenReturn(destVertex); taImpl.handle(new TaskAttemptEventOutputFailed(taskAttemptID, tzEvent, 11)); - + assertEquals("Task attempt is not in FAILED state", TaskAttemptState.FAILED, taImpl.getState()); assertEquals(TaskAttemptTerminationCause.OUTPUT_LOST, taImpl.getTerminationCause()); // verify unregister is not invoked again @@ -1977,7 +1977,7 @@ taListener, taskConf, new SystemClock(), // verify unregister is not invoked again verify(mockHeartbeatHandler, times(1)).unregister(taskAttemptID2); - Clock mockClock = mock(Clock.class); + Clock mockClock = mock(Clock.class); int readErrorTimespanSec = 1; newVertexConf = new Configuration(vertexConf); @@ -2374,7 +2374,7 @@ public void handle(Event event) { } private class MockTaskAttemptImpl extends TaskAttemptImpl { - + public int taskAttemptStartedEventLogged = 0; public int taskAttemptFinishedEventLogged = 0; public MockTaskAttemptImpl(TezTaskID taskId, int attemptNumber, @@ -2421,7 +2421,7 @@ protected void logJobHistoryAttemptUnsuccesfulCompletion( taskAttemptFinishedEventLogged++; super.logJobHistoryAttemptUnsuccesfulCompletion(state, taskFailureType); } - + @Override protected void sendInputFailedToConsumers() { inputFailedReported = true; diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java index 798b51de66..ebd5c534cc 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java @@ -137,7 +137,7 @@ public class TestTaskImpl { private MockTaskImpl mockTask; private TaskSpec mockTaskSpec; private Vertex mockVertex; - + @SuppressWarnings("rawtypes") class TestEventHandler implements EventHandler { List events = new ArrayList(); @@ -378,7 +378,7 @@ public void testKillScheduledTask() { scheduleTaskAttempt(taskId); killTask(taskId); } - + /** * {@link TaskState#RUNNING}->{@link TaskState#KILLED} */ @@ -484,7 +484,7 @@ public void testKillRunningTaskButAttemptSucceeds() { mockTask.handle(createTaskTASucceededEvent(mockTask.getLastAttempt().getTaskAttemptID())); assertEquals(TaskStateInternal.KILLED, mockTask.getInternalState()); } - + /** * {@link TaskState#RUNNING}->{@link TaskState#KILLED} */ @@ -694,7 +694,7 @@ public void testFailureDuringTaskAttemptCommit() { assertTaskSucceededState(); } - + @Test(timeout = 5000) public void testEventBacklogDuringTaskAttemptCommit() { @@ -724,17 +724,17 @@ public void testChangeCommitTaskAttempt() { launchTaskAttempt(mockTask.getLastAttempt().getTaskAttemptID()); updateAttemptState(mockTask.getLastAttempt(), TaskAttemptState.RUNNING); TezTaskAttemptID lastTAId = mockTask.getLastAttempt().getTaskAttemptID(); - + // Add a speculative task attempt that succeeds mockTask.handle(createTaskTAAddSpecAttempt(mockTask.getLastAttempt().getTaskAttemptID())); launchTaskAttempt(mockTask.getLastAttempt().getTaskAttemptID()); updateAttemptState(mockTask.getLastAttempt(), TaskAttemptState.RUNNING); - + assertEquals(2, mockTask.getAttemptList().size()); - + // previous running attempt should be the casual TA of this speculative attempt Assert.assertEquals(lastTAId, mockTask.getLastAttempt().getSchedulingCausalTA()); - + assertTrue("Second attempt should commit", mockTask.canCommit(mockTask.getAttemptList().get(1).getTaskAttemptID())); assertFalse("First attempt should not commit", @@ -746,7 +746,7 @@ public void testChangeCommitTaskAttempt() { failRunningTaskAttempt(mockTask.getLastAttempt().getTaskAttemptID()); assertEquals(2, mockTask.getAttemptList().size()); - + assertFalse("Second attempt should not commit", mockTask.canCommit(mockTask.getAttemptList().get(1).getTaskAttemptID())); assertTrue("First attempt should commit", @@ -757,7 +757,7 @@ public void testChangeCommitTaskAttempt() { assertTaskSucceededState(); } - + @SuppressWarnings("rawtypes") @Test(timeout = 5000) public void testTaskSucceedAndRetroActiveFailure() { @@ -787,7 +787,7 @@ public void testTaskSucceedAndRetroActiveFailure() { TezEvent mockTezEvent = mock(TezEvent.class); EventMetaData meta = new EventMetaData(EventProducerConsumerType.INPUT, "Vertex", "Edge", mockDestId); when(mockTezEvent.getSourceInfo()).thenReturn(meta); - TaskAttemptEventOutputFailed outputFailedEvent = + TaskAttemptEventOutputFailed outputFailedEvent = new TaskAttemptEventOutputFailed(mockDestId, mockTezEvent, 1); mockTask.handle( createTaskTAFailedEvent(mockTask.getLastAttempt().getTaskAttemptID(), TaskFailureType.NON_FATAL, @@ -799,7 +799,7 @@ public void testTaskSucceedAndRetroActiveFailure() { Assert.assertEquals(AMNodeEventType.N_TA_ENDED, event.getType()); event = eventHandler.events.get(eventHandler.events.size()-1); Assert.assertEquals(VertexEventType.V_TASK_RESCHEDULED, event.getType()); - + // report of output read error should be the causal TA List attempts = mockTask.getAttemptList(); Assert.assertEquals(2, attempts.size()); @@ -841,7 +841,7 @@ public void testDiagnostics_KillNew(){ assertEquals(0, mockTask.taskStartedEventLogged); assertEquals(1, mockTask.taskFinishedEventLogged); } - + @Test(timeout = 5000) public void testDiagnostics_Kill(){ TezTaskID taskId = getNewTaskID(); @@ -1440,7 +1440,7 @@ MockTaskAttemptImpl getLastAttempt() { List getAttemptList() { return taskAttempts; } - + @Override public Vertex getVertex() { return vertex; @@ -1498,12 +1498,12 @@ public void setState(TaskAttemptState state) { public TaskAttemptState getState() { return state; } - + @Override public TaskAttemptState getStateNoLock() { return state; } - + @Override public ContainerId getAssignedContainerID() { return mockContainerId; diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java index d83459f787..833586e57f 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java @@ -255,7 +255,7 @@ public class TestVertexImpl { private boolean useCustomInitializer = false; private InputInitializer customInitializer = null; - + private TezDAGID dagId; private ApplicationAttemptId appAttemptId; private DAGPlan dagPlan; @@ -394,7 +394,7 @@ public byte[] toUserPayload() throws IOException { } } - + private class TaskAttemptEventDispatcher implements EventHandler { @SuppressWarnings("unchecked") @Override @@ -847,7 +847,7 @@ private DAGPlan createDAGPlanWithMixedEdges(String dagName) { EdgeProperty.create(DataMovementType.ONE_TO_ONE, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out.class"), InputDescriptor.create("out.class")))); - + return dag.createDag(conf, null, null, null, true); } @@ -1368,7 +1368,7 @@ private DAGPlan createDAGPlanWithRunningInitializer() { .build(); return dag; } - + private DAGPlan createDAGPlanWithInputDistributor(String initializerClassName) { LOG.info("Setting up invalid dag plan with input distributor"); DAGPlan dag = DAGPlan.newBuilder() @@ -1433,8 +1433,8 @@ private DAGPlan createDAGPlanWithInputDistributor(String initializerClassName) { .build(); return dag; } - - private DAGPlan createDAGPlanForOneToOneSplit(String initializerClassName, + + private DAGPlan createDAGPlanForOneToOneSplit(String initializerClassName, int numTasks, boolean addNullEdge) { VertexPlan.Builder v1Builder = VertexPlan.newBuilder(); v1Builder.setName("vertex1") @@ -1478,7 +1478,7 @@ private DAGPlan createDAGPlanForOneToOneSplit(String initializerClassName, ); } VertexPlan v1Plan = v1Builder.build(); - + VertexPlan.Builder v4Builder = VertexPlan.newBuilder(); v4Builder .setName("vertex4") @@ -1498,7 +1498,7 @@ private DAGPlan createDAGPlanForOneToOneSplit(String initializerClassName, v4Builder.addOutEdgeId("e6"); } VertexPlan v4Plan = v4Builder.build(); - + LOG.info("Setting up one to one dag plan"); DAGPlan.Builder dagBuilder = DAGPlan.newBuilder() .setName("testVertexOneToOneSplit") @@ -1584,7 +1584,7 @@ private DAGPlan createDAGPlanForOneToOneSplit(String initializerClassName, .setId("e4") .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - + ); if (addNullEdge) { dagBuilder.addVertex( @@ -2034,7 +2034,7 @@ private DAGPlan createSamplerDAGPlan(boolean customEdge) { return dag; } - + // Create a plan with 3 vertices: A, B, C. Group(A,B)->C private DAGPlan createVertexGroupDAGPlan() { LOG.info("Setting up group dag plan"); @@ -2131,7 +2131,7 @@ private DAGPlan createVertexGroupDAGPlan() { return dag; } - + private DAGPlan createDAGWithCustomVertexManager() { LOG.info("Setting up custom vertex manager dag plan"); DAGPlan dag = DAGPlan.newBuilder() @@ -2204,7 +2204,7 @@ private DAGPlan createDAGWithCustomVertexManager() { .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) .build() ).build(); - + return dag; } @@ -2820,7 +2820,7 @@ public void setupPostDagCreation(boolean cleanupShuffleDataAtVertexLevel) throws doReturn(dag).when(appContext).getCurrentDAG(); execService = mock(ListeningExecutorService.class); final ListenableFuture mockFuture = mock(ListenableFuture.class); - + doAnswer(new Answer() { public ListenableFuture answer(InvocationOnMock invocation) { Object[] args = invocation.getArguments(); @@ -2829,9 +2829,9 @@ public ListenableFuture answer(InvocationOnMock invocation) { return mockFuture; }}) .when(execService).submit((Callable) any()); - + MockClock clock = new MockClock(); - + doReturn(execService).when(appContext).getExecService(); doReturn(conf).when(appContext).getAMConf(); doReturn(new Credentials()).when(dag).getCredentials(); @@ -2843,7 +2843,7 @@ public ListenableFuture answer(InvocationOnMock invocation) { doReturn(historyEventHandler).when(appContext).getHistoryHandler(); doReturn(dispatcher.getEventHandler()).when(appContext).getEventHandler(); doReturn(clock).when(appContext).getClock(); - + vertexGroups = Maps.newHashMap(); for (PlanVertexGroupInfo groupInfo : dagPlan.getVertexGroupsList()) { vertexGroups.put(groupInfo.getGroupName(), new VertexGroupInfo(groupInfo)); @@ -2886,7 +2886,7 @@ public Vertex answer(InvocationOnMock invocation) throws Throwable { } parseVertexEdges(); - + for (Edge edge : edges.values()) { edge.initialize(); } @@ -3088,7 +3088,7 @@ public void onStateUpdated(VertexStateUpdate event) { events.add(event); } } - + @Test (timeout=5000) public void testVertexConfigureEvent() throws Exception { initAllVertices(VertexState.INITED); @@ -3104,7 +3104,7 @@ public void testVertexConfigureEvent() throws Exception { listener.events.get(0).getVertexState()); updateTracker.unregisterForVertexUpdates("vertex3", listener); } - + @Test (timeout=5000) public void testVertexConfigureEventWithReconfigure() throws Exception { useCustomInitializer = true; @@ -3157,7 +3157,7 @@ public void testVertexConfigureEventBadReconfigure() { Assert.assertTrue(e.getMessage().contains("invoked only after vertexReconfigurationPlanned")); } } - + @Test (timeout=5000) public void testVertexConfigureEventBadSetParallelism() throws Exception { initAllVertices(VertexState.INITED); @@ -3182,7 +3182,7 @@ public void testVertexStart() { VertexImpl v = vertices.get("vertex2"); startVertex(v); } - + @Test (timeout = 5000) public void testVertexGetTAAttempts() throws Exception { initAllVertices(VertexState.INITED); @@ -3192,7 +3192,7 @@ public void testVertexGetTAAttempts() throws Exception { startVertex(v2); VertexImpl v3 = vertices.get("vertex3"); VertexImpl v4 = vertices.get("vertex4"); - + Assert.assertEquals(VertexState.RUNNING, v4.getState()); Assert.assertEquals(1, v4.sourceVertices.size()); Edge e = v4.sourceVertices.get(v3); @@ -3200,10 +3200,10 @@ public void testVertexGetTAAttempts() throws Exception { TezTaskID.getInstance(v3.getVertexId(), 0), 0); TezTaskAttemptID v4TaId = TezTaskAttemptID.getInstance( TezTaskID.getInstance(v4.getVertexId(), 0), 0); - + for (int i=0; i<5; ++i) { v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList( - new TezEvent(DataMovementEvent.create(0, null), + new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId))))); } dispatcher.await(); @@ -3221,13 +3221,13 @@ public void testVertexGetTAAttempts() throws Exception { Assert.assertEquals(5, v4.getOnDemandRouteEvents().size()); for (int i=5; i<11; ++i) { v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList( - new TezEvent(DataMovementEvent.create(0, null), + new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId))))); } dispatcher.await(); // verify all events have been are in taskEvents Assert.assertEquals(11, v4.getOnDemandRouteEvents().size()); - + TaskAttemptEventInfo eventInfo; EdgeManagerPluginOnDemand mockPlugin = mock(EdgeManagerPluginOnDemand.class); EventRouteMetadata mockRoute = EventRouteMetadata.create(1, new int[]{0}); @@ -3237,7 +3237,7 @@ public void testVertexGetTAAttempts() throws Exception { eventInfo = v4.getTaskAttemptTezEvents(v4TaId, 0, 0, 1); Assert.assertEquals(11, eventInfo.getNextFromEventId()); // all events traversed Assert.assertEquals(0, eventInfo.getEvents().size()); // no events - + int fromEventId = 0; // source task id will match. all events will be returned // max events is respected. @@ -3253,14 +3253,14 @@ public void testVertexGetTAAttempts() throws Exception { eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 1); Assert.assertEquals(11, eventInfo.getNextFromEventId()); // all events traversed Assert.assertEquals(0, eventInfo.getEvents().size()); // no events - + // ask for events with sufficient buffer. get all events in a single shot. fromEventId = 0; eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 100); fromEventId = eventInfo.getNextFromEventId(); Assert.assertEquals(11, fromEventId); Assert.assertEquals(11, eventInfo.getEvents().size()); - + // change max events to larger value. max events does not evenly divide total events fromEventId = 0; for (int i=1; i<=2; ++i) { @@ -3272,9 +3272,9 @@ public void testVertexGetTAAttempts() throws Exception { eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5); Assert.assertEquals(11, eventInfo.getNextFromEventId()); // all events traversed Assert.assertEquals(1, eventInfo.getEvents().size()); // remainder events - + // return more events that dont evenly fit in max size - mockRoute = EventRouteMetadata.create(2, new int[]{0, 0}); + mockRoute = EventRouteMetadata.create(2, new int[]{0, 0}); when( mockPlugin.routeDataMovementEventToDestination(anyInt(), anyInt(), anyInt())).thenReturn(mockRoute); @@ -3283,14 +3283,14 @@ public void testVertexGetTAAttempts() throws Exception { for (int i=1; i<=4; ++i) { eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5); fromEventId = eventInfo.getNextFromEventId(); - Assert.assertEquals((i%2 > 0 ? (lastFromEventId+=2) : (lastFromEventId+=3)), fromEventId); + Assert.assertEquals((i%2 > 0 ? (lastFromEventId+=2) : (lastFromEventId+=3)), fromEventId); Assert.assertEquals(5, eventInfo.getEvents().size()); } eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5); Assert.assertEquals(11, eventInfo.getNextFromEventId()); // all events traversed Assert.assertEquals(2, eventInfo.getEvents().size()); // remainder events } - + @Test (timeout = 5000) public void testVertexGetTAAttemptsObsoletion() throws Exception { initAllVertices(VertexState.INITED); @@ -3300,7 +3300,7 @@ public void testVertexGetTAAttemptsObsoletion() throws Exception { startVertex(v2); VertexImpl v3 = vertices.get("vertex3"); VertexImpl v4 = vertices.get("vertex4"); - + List taskList = new LinkedList(); // scheduling start to trigger edge routing to begin for (int i=0; i taskList = new LinkedList(); // scheduling start to trigger edge routing to begin for (int i=0; i edgeManagerDescriptors = Collections.singletonMap( @@ -3497,8 +3497,8 @@ public void testVertexSetParallelismIncrease() throws Exception { EdgeManagerPluginDescriptor mockEdgeManagerDescriptor = EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName()); - EdgeProperty edgeProp = EdgeProperty.create(mockEdgeManagerDescriptor, - DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), + EdgeProperty edgeProp = EdgeProperty.create(mockEdgeManagerDescriptor, + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")); Map edgeManagerDescriptors = Collections.singletonMap( @@ -3509,7 +3509,7 @@ public void testVertexSetParallelismIncrease() throws Exception { EdgeManagerForTest); checkTasks(v3, 10); } - + @Test(timeout = 5000) public void testVertexSetParallelismMultiple() throws Exception { VertexImpl v3 = vertices.get("vertex3"); @@ -3524,7 +3524,7 @@ public void testVertexSetParallelismMultiple() throws Exception { startVertex(v1); v3.reconfigureVertex(10, null, null); checkTasks(v3, 10); - + v3.reconfigureVertex(5, null, null); checkTasks(v3, 5); v3.doneReconfiguringVertex(); @@ -3576,7 +3576,7 @@ public void testVertexSetParallelismMultipleFailAfterSchedule() throws Exception Assert.assertTrue(e.getMessage().contains("setParallelism cannot be called after scheduling")); } } - + @Test(timeout = 5000) public void testVertexScheduleSendEvent() throws Exception { VertexImpl v3 = vertices.get("vertex3"); @@ -3601,7 +3601,7 @@ public void testVertexScheduleSendEvent() throws Exception { Assert.assertNotNull(event.getBaseTaskSpec()); Assert.assertEquals("foobar", event.getBaseTaskSpec().getTaskConf().get("abc")); } - + @Test(timeout = 5000) public void testVertexSetParallelismFailAfterSchedule() throws Exception { VertexImpl v3 = vertices.get("vertex3"); @@ -3622,7 +3622,7 @@ public void testVertexSetParallelismFailAfterSchedule() throws Exception { Assert.assertTrue(e.getMessage().contains("setParallelism cannot be called after scheduling")); } } - + @Test(timeout = 5000) public void testVertexPendingTaskEvents() { // Remove after bulk routing API is removed @@ -3630,9 +3630,9 @@ public void testVertexPendingTaskEvents() { VertexImpl v3 = vertices.get("vertex3"); VertexImpl v2 = vertices.get("vertex2"); VertexImpl v1 = vertices.get("vertex1"); - + startVertex(v1); - + TezTaskID t0_v2 = TezTaskID.getInstance(v2.getVertexId(), 0); TezTaskAttemptID ta0_t0_v2 = TezTaskAttemptID.getInstance(t0_v2, 0); @@ -3675,8 +3675,8 @@ public void testSetCustomEdgeManager() throws Exception { EdgeManagerPluginDescriptor edgeManagerDescriptor = EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName()); edgeManagerDescriptor.setUserPayload(userPayload); - EdgeProperty edgeProp = EdgeProperty.create(edgeManagerDescriptor, - DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), + EdgeProperty edgeProp = EdgeProperty.create(edgeManagerDescriptor, + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")); Vertex v3 = vertices.get("vertex3"); @@ -3935,7 +3935,7 @@ public void testVertexTaskAttemptProcessorFailure() throws Exception { dispatcher.await(); TaskAttemptImpl ta = (TaskAttemptImpl) v.getTask(0).getAttempts().values().iterator().next(); ta.handle(new TaskAttemptEventSchedule(ta.getTaskAttemptID(), 2, 2)); - + NodeId nid = NodeId.newInstance("127.0.0.1", 0); ContainerId contId = ContainerId.newInstance(appAttemptId, 3); Container container = mock(Container.class); @@ -3970,7 +3970,7 @@ public void testVertexTaskAttemptInputFailure() throws Exception { dispatcher.await(); TaskAttemptImpl ta = (TaskAttemptImpl) v.getTask(0).getAttempts().values().iterator().next(); ta.handle(new TaskAttemptEventSchedule(ta.getTaskAttemptID(), 2, 2)); - + NodeId nid = NodeId.newInstance("127.0.0.1", 0); ContainerId contId = ContainerId.newInstance(appAttemptId, 3); Container container = mock(Container.class); @@ -4007,7 +4007,7 @@ public void testVertexTaskAttemptOutputFailure() throws Exception { dispatcher.await(); TaskAttemptImpl ta = (TaskAttemptImpl) v.getTask(0).getAttempts().values().iterator().next(); ta.handle(new TaskAttemptEventSchedule(ta.getTaskAttemptID(), 2, 2)); - + NodeId nid = NodeId.newInstance("127.0.0.1", 0); ContainerId contId = ContainerId.newInstance(appAttemptId, 3); Container container = mock(Container.class); @@ -4031,7 +4031,7 @@ public void testVertexTaskAttemptOutputFailure() throws Exception { Assert.assertEquals(VertexState.RUNNING, v.getState()); Assert.assertEquals(TaskAttemptTerminationCause.OUTPUT_WRITE_ERROR, ta.getTerminationCause()); } - + @Test(timeout = 5000) public void testSourceVertexStartHandling() { LOG.info("Testing testSourceVertexStartHandling"); @@ -4446,14 +4446,14 @@ public void testTaskFailedAfterVertexSuccess() { Assert.assertEquals(0, committer.abortCounter); Assert.assertEquals(1, committer.initCounter); Assert.assertEquals(1, committer.setupCounter); - + dispatcher.getEventHandler().handle( new VertexEventTaskCompleted(t2, TaskState.FAILED)); dispatcher.await(); Assert.assertEquals(VertexState.FAILED, v.getState()); Assert.assertEquals(1, committer.commitCounter); Assert.assertEquals(1, committer.abortCounter); - + } @Test(timeout = 5000) @@ -4548,7 +4548,7 @@ public void testVertexInitWithCustomVertexManager() throws Exception { setupPreDagCreation(); dagPlan = createDAGWithCustomVertexManager(); setupPostDagCreation(false); - + int numTasks = 3; VertexImpl v1 = vertices.get("v1"); VertexImpl v2 = vertices.get("v2"); @@ -4603,39 +4603,39 @@ public void testVertexManagerHeuristic() throws TezException { dagPlan = createDAGPlanWithMixedEdges("testVertexManagerHeuristic"); setupPostDagCreation(false); initAllVertices(VertexState.INITED); - Assert.assertEquals(ImmediateStartVertexManager.class, + Assert.assertEquals(ImmediateStartVertexManager.class, vertices.get("vertex1").getVertexManager().getPlugin().getClass()); - Assert.assertEquals(ShuffleVertexManager.class, + Assert.assertEquals(ShuffleVertexManager.class, vertices.get("vertex2").getVertexManager().getPlugin().getClass()); - Assert.assertEquals(InputReadyVertexManager.class, + Assert.assertEquals(InputReadyVertexManager.class, vertices.get("vertex3").getVertexManager().getPlugin().getClass()); - Assert.assertEquals(ImmediateStartVertexManager.class, + Assert.assertEquals(ImmediateStartVertexManager.class, vertices.get("vertex4").getVertexManager().getPlugin().getClass()); - Assert.assertEquals(ImmediateStartVertexManager.class, + Assert.assertEquals(ImmediateStartVertexManager.class, vertices.get("vertex5").getVertexManager().getPlugin().getClass()); - Assert.assertEquals(InputReadyVertexManager.class, + Assert.assertEquals(InputReadyVertexManager.class, vertices.get("vertex6").getVertexManager().getPlugin().getClass()); } @Test(timeout = 5000) public void testVertexWithOneToOneSplit() throws Exception { - // create a diamond shaped dag with 1-1 edges. + // create a diamond shaped dag with 1-1 edges. // split the source and remaining vertices should split equally // vertex with 2 incoming splits from the same source should split once useCustomInitializer = true; setupPreDagCreation(); dagPlan = createDAGPlanForOneToOneSplit("TestInputInitializer", -1, true); setupPostDagCreation(false); - + int numTasks = 5; VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices .get("vertex1"); VertexImpl v5 = vertices.get("vertex5"); initVertex(v1); - + Assert.assertEquals(VertexState.INITIALIZING, v1.getState()); - + // setting the edge manager should vertex1 should not INIT/START it since // input initialization is not complete. v5 should be inited EdgeManagerPluginDescriptor mockEdgeManagerDescriptor = @@ -4646,7 +4646,7 @@ public void testVertexWithOneToOneSplit() throws Exception { dispatcher.await(); Assert.assertEquals(VertexState.INITIALIZING, v1.getState()); Assert.assertEquals(VertexState.INITED, vertices.get("vertex5").getState()); - + RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager(); List v1Hints = createTaskLocationHints(numTasks); initializerManager1.completeInputInitialization(0, numTasks, v1Hints); @@ -4668,7 +4668,7 @@ public void testVertexWithOneToOneSplit() throws Exception { // v4, v6 still initializing since edge is null Assert.assertEquals(VertexState.INITIALIZING, vertices.get("vertex4").getState()); Assert.assertEquals(VertexState.INITIALIZING, vertices.get("vertex6").getState()); - + Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex1").getState()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex2").getState()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex3").getState()); @@ -4676,7 +4676,7 @@ public void testVertexWithOneToOneSplit() throws Exception { // v4, v6 still initializing since edge is null Assert.assertEquals(VertexState.INITIALIZING, vertices.get("vertex4").getState()); Assert.assertEquals(VertexState.INITIALIZING, vertices.get("vertex6").getState()); - + mockEdgeManagerDescriptor = EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName()); e = vertices.get("vertex6").sourceVertices.get(vertices.get("vertex4")); @@ -4686,18 +4686,18 @@ public void testVertexWithOneToOneSplit() throws Exception { Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex4").getState()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex6").getState()); } - + @Test(timeout = 5000) public void testVertexWithOneToOneSplitWhileRunning() throws Exception { int numTasks = 5; - // create a diamond shaped dag with 1-1 edges. + // create a diamond shaped dag with 1-1 edges. setupPreDagCreation(); dagPlan = createDAGPlanForOneToOneSplit(null, numTasks, false); setupPostDagCreation(false); VertexImpl v1 = vertices.get("vertex1"); v1.vertexReconfigurationPlanned(); initAllVertices(VertexState.INITED); - + // fudge vertex manager so that tasks dont start running v1.vertexManager = new VertexManager( VertexManagerPluginDescriptor.create(VertexManagerPluginForTest.class.getName()), @@ -4725,24 +4725,24 @@ public void testVertexWithOneToOneSplitWhileRunning() throws Exception { Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex3").getState()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex4").getState()); } - + @Test(timeout = 5000) public void testVertexWithOneToOneSplitWhileInited() throws Exception { int numTasks = 5; - // create a diamond shaped dag with 1-1 edges. + // create a diamond shaped dag with 1-1 edges. setupPreDagCreation(); dagPlan = createDAGPlanForOneToOneSplit(null, numTasks, false); setupPostDagCreation(false); VertexImpl v1 = vertices.get("vertex1"); v1.vertexReconfigurationPlanned(); initAllVertices(VertexState.INITED); - + // fudge vertex manager so that tasks dont start running v1.vertexManager = new VertexManager( VertexManagerPluginDescriptor.create(VertexManagerPluginForTest.class.getName()), UserGroupInformation.getCurrentUser(), v1, appContext, mock(StateChangeNotifier.class)); v1.vertexManager.initialize(); - + Assert.assertEquals(numTasks, vertices.get("vertex2").getTotalTasks()); Assert.assertEquals(numTasks, vertices.get("vertex3").getTotalTasks()); Assert.assertEquals(numTasks, vertices.get("vertex4").getTotalTasks()); @@ -4771,13 +4771,13 @@ public void testVertexWithOneToOneSplitWhileInited() throws Exception { @Test(timeout = 5000) public void testVertexVMErrorReport() throws Exception { int numTasks = 5; - // create a diamond shaped dag with 1-1 edges. + // create a diamond shaped dag with 1-1 edges. setupPreDagCreation(); dagPlan = createDAGPlanForOneToOneSplit(null, numTasks, false); setupPostDagCreation(false); VertexImpl v1 = vertices.get("vertex1"); initAllVertices(VertexState.INITED); - + // fudge vertex manager so that tasks dont start running // it is not calling reconfigurtionPlanned() but will call reconfigureVertex(). // the vertex is already fully configured. this causes exception and verify that @@ -4838,7 +4838,7 @@ public void testVertexWithInitializerFailure() throws Exception { .contains(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE.name())); Assert.assertTrue(StringUtils.join(v1.getDiagnostics(), ",") .contains("MockInitializerFailed")); - + VertexImplWithControlledInitializerManager v2 = (VertexImplWithControlledInitializerManager) vertices.get("vertex2"); Assert.assertEquals(VertexState.INITIALIZING, v2.getState()); RootInputInitializerManagerControlled initializerManager2 = v2.getRootInputInitializerManager(); @@ -5783,7 +5783,7 @@ public void testVertexWithInitializerSuccess() throws Exception { Assert.assertEquals(1, inputSpecs.size()); Assert.assertEquals(1, inputSpecs.get(0).getPhysicalEdgeCount()); } - + List taskList = new LinkedList(); // scheduling start to trigger edge routing to begin for (int i=0; i events = Lists.newLinkedList(); TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0); @@ -5817,7 +5817,7 @@ public void testVertexWithInitializerSuccess() throws Exception { dispatcher.getEventHandler().handle( new VertexEventRouteEvent(v2.getVertexId(), events)); dispatcher.await(); - + RootInputInitializerManagerControlled initializerManager2 = v2.getRootInputInitializerManager(); List v2Hints = createTaskLocationHints(10); initializerManager2.completeInputInitialization(0, 10, v2Hints); @@ -5830,7 +5830,7 @@ public void testVertexWithInitializerSuccess() throws Exception { Assert.assertEquals(v2Hints.get(i), v2.getTaskLocationHints()[i]); } Assert.assertEquals(true, initializerManager2.hasShutDown); - + // scheduling start to trigger edge routing to begin taskList = new LinkedList(); // scheduling start to trigger edge routing to begin @@ -5852,7 +5852,7 @@ public void testVertexWithInitializerSuccess() throws Exception { Assert.assertEquals(1, inputSpecs.get(0).getPhysicalEdgeCount()); } } - + @Test(timeout = 5000) public void testVertexWithInitializerSuccessLegacyRouting() throws Exception { // Remove after legacy routing is removed @@ -5886,7 +5886,7 @@ public void testVertexWithInitializerSuccessLegacyRouting() throws Exception { } // task events get buffered Assert.assertEquals(5, v1.pendingTaskEvents.size()); - + VertexImplWithControlledInitializerManager v2 = (VertexImplWithControlledInitializerManager) vertices.get("vertex2"); Assert.assertEquals(VertexState.INITIALIZING, v2.getState()); @@ -5906,7 +5906,7 @@ public void testVertexWithInitializerSuccessLegacyRouting() throws Exception { new VertexEventRouteEvent(v2.getVertexId(), events)); dispatcher.await(); Assert.assertEquals(1, v2.pendingTaskEvents.size()); - + RootInputInitializerManagerControlled initializerManager2 = v2.getRootInputInitializerManager(); List v2Hints = createTaskLocationHints(10); initializerManager2.completeInputInitialization(0, 10, v2Hints); @@ -5928,7 +5928,7 @@ public void testVertexWithInitializerSuccessLegacyRouting() throws Exception { } } - + @Test(timeout = 5000) public void testVertexWithInputDistributor() throws Exception { useCustomInitializer = true; @@ -5963,7 +5963,7 @@ public void testVertexWithInputDistributor() throws Exception { Assert.assertEquals(VertexState.INITED, v1.getState()); Assert.assertEquals(VertexState.INITED, v2.getState()); } - + @Test(timeout = 5000) public void testVertexRootInputSpecUpdateAll() throws Exception { useCustomInitializer = true; @@ -5986,7 +5986,7 @@ public void testVertexRootInputSpecUpdateAll() throws Exception { Assert.assertEquals(RootInputSpecUpdaterVertexManager.class.getName(), v3.getVertexManager() .getPlugin().getClass().getName()); Assert.assertEquals(true, initializerManager1.hasShutDown); - + for (int i = 0; i < expectedNumTasks; i++) { List inputSpecs = v3.getInputSpecList(i); Assert.assertEquals(1, inputSpecs.size()); @@ -6016,7 +6016,7 @@ public void testVertexRootInputSpecUpdatePerTask() throws Exception { Assert.assertEquals(RootInputSpecUpdaterVertexManager.class.getName(), v4.getVertexManager() .getPlugin().getClass().getName()); Assert.assertEquals(true, initializerManager1.hasShutDown); - + for (int i = 0; i < expectedNumTasks; i++) { List inputSpecs = v4.getInputSpecList(i); Assert.assertEquals(1, inputSpecs.size()); @@ -6140,10 +6140,10 @@ protected RootInputInitializerManager createRootInputInitializerManager( @SuppressWarnings("rawtypes") private static class VertexImplWithControlledInitializerManager extends VertexImpl { - + private final DrainDispatcher dispatcher; private RootInputInitializerManagerControlled rootInputInitializerManager; - + public VertexImplWithControlledInitializerManager(TezVertexID vertexId, VertexPlan vertexPlan, String vertexName, Configuration conf, @@ -6176,7 +6176,7 @@ protected RootInputInitializerManager createRootInputInitializerManager( } return rootInputInitializerManager; } - + RootInputInitializerManagerControlled getRootInputInitializerManager() { return rootInputInitializerManager; } @@ -6322,10 +6322,10 @@ public void testVertexGroupInput() throws TezException { dispatcher.getEventHandler().handle(new VertexEvent(vB.getVertexId(), VertexEventType.V_INIT)); dispatcher.await(); - + Assert.assertNull(vA.getGroupInputSpecList()); Assert.assertNull(vB.getGroupInputSpecList()); - + List groupInSpec = vC.getGroupInputSpecList(); Assert.assertEquals(1, groupInSpec.size()); Assert.assertEquals("Group", groupInSpec.get(0).getGroupName()); @@ -6355,7 +6355,7 @@ public void testStartWithUninitializedCustomEdge() throws Exception { Assert.assertEquals(VertexState.INITIALIZING, vA.getState()); Assert.assertEquals(VertexState.INITIALIZING, vB.getState()); Assert.assertEquals(VertexState.INITIALIZING, vC.getState()); - + // setting the edge manager should vA to start EdgeManagerPluginDescriptor mockEdgeManagerDescriptor = EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName()); @@ -6366,9 +6366,9 @@ public void testStartWithUninitializedCustomEdge() throws Exception { Assert.assertEquals(VertexState.RUNNING, vA.getState()); Assert.assertEquals(VertexState.INITIALIZING, vB.getState()); Assert.assertEquals(VertexState.INITIALIZING, vC.getState()); - - EdgeProperty edgeProp = EdgeProperty.create(mockEdgeManagerDescriptor, - DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), + + EdgeProperty edgeProp = EdgeProperty.create(mockEdgeManagerDescriptor, + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")); Map edges = Maps.newHashMap(); edges.put("B", edgeProp); @@ -6390,7 +6390,7 @@ public void testVertexConfiguredDoneByVMBeforeEdgeDefined() throws Exception { setupPreDagCreation(); dagPlan = createSamplerDAGPlan(true); setupPostDagCreation(false); - + VertexImpl vA = vertices.get("A"); VertexImpl vB = vertices.get("B"); VertexImpl vC = vertices.get("C"); @@ -6405,9 +6405,9 @@ public void testVertexConfiguredDoneByVMBeforeEdgeDefined() throws Exception { vB.vertexManager = new VertexManager( VertexManagerPluginDescriptor.create(VertexManagerPluginForTest.class.getName()), UserGroupInformation.getCurrentUser(), vB, appContext, mock(StateChangeNotifier.class)); - + vB.vertexReconfigurationPlanned(); - + dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), VertexEventType.V_INIT)); dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), @@ -6417,7 +6417,7 @@ public void testVertexConfiguredDoneByVMBeforeEdgeDefined() throws Exception { Assert.assertEquals(VertexState.INITIALIZING, vA.getState()); Assert.assertEquals(VertexState.INITIALIZING, vB.getState()); Assert.assertEquals(VertexState.INITIALIZING, vC.getState()); - + // setting the edge manager should vA to start EdgeManagerPluginDescriptor mockEdgeManagerDescriptor = EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName()); @@ -6428,15 +6428,15 @@ public void testVertexConfiguredDoneByVMBeforeEdgeDefined() throws Exception { Assert.assertEquals(VertexState.RUNNING, vA.getState()); Assert.assertEquals(VertexState.INITIALIZING, vB.getState()); Assert.assertEquals(VertexState.INITIALIZING, vC.getState()); - + // vB is not configured yet. Edge to C is not configured. So it should not send configured event // even thought VM says its doneConfiguring vertex vB.doneReconfiguringVertex(); Assert.assertEquals(0, listener.events.size()); - + // complete configuration and verify getting configured signal from vB - EdgeProperty edgeProp = EdgeProperty.create(mockEdgeManagerDescriptor, - DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), + EdgeProperty edgeProp = EdgeProperty.create(mockEdgeManagerDescriptor, + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")); Map edges = Maps.newHashMap(); edges.put("B", edgeProp); @@ -6448,7 +6448,7 @@ public void testVertexConfiguredDoneByVMBeforeEdgeDefined() throws Exception { Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.CONFIGURED, listener.events.get(0).getVertexState()); updateTracker.unregisterForVertexUpdates(vB.getName(), listener); - + Assert.assertEquals(VertexState.RUNNING, vA.getState()); Assert.assertEquals(VertexState.RUNNING, vB.getState()); Assert.assertEquals(VertexState.RUNNING, vC.getState()); @@ -6689,7 +6689,7 @@ public void testExceptionFromVM_Initialize() throws TezException { Assert.assertTrue(diagnostics.contains(VMExceptionLocation.Initialize.name())); Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v1.getTerminationCause()); } - + @Test(timeout = 5000) public void testExceptionFromVM_OnRootVertexInitialized() throws Exception { useCustomInitializer = true; @@ -6713,7 +6713,7 @@ public void testExceptionFromVM_OnRootVertexInitialized() throws Exception { Assert.assertTrue(diagnostics.contains(VMExceptionLocation.OnRootVertexInitialized.name())); Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v1.getTerminationCause()); } - + @Test(timeout = 5000) public void testExceptionFromVM_OnVertexStarted() throws Exception { useCustomInitializer = true; @@ -6733,14 +6733,14 @@ public void testExceptionFromVM_OnVertexStarted() throws Exception { dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_START)); dispatcher.await(); - + Assert.assertEquals(VertexManagerWithException.class, v1.vertexManager.getPlugin().getClass()); Assert.assertEquals(VertexState.FAILED, v1.getState()); String diagnostics = StringUtils.join(v1.getDiagnostics(), ","); Assert.assertTrue(diagnostics.contains(VMExceptionLocation.OnVertexStarted.name())); Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v1.getTerminationCause()); } - + @Test(timeout = 5000) public void testExceptionFromVM_OnSourceTaskCompleted() throws Exception { useCustomInitializer = true; @@ -6776,7 +6776,7 @@ public void testExceptionFromVM_OnSourceTaskCompleted() throws Exception { Assert.assertTrue(diagnostics.contains(VMExceptionLocation.OnSourceTaskCompleted.name())); Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v2.getTerminationCause()); } - + @Test(timeout = 5000) public void testExceptionFromVM_OnVertexManagerEventReceived() throws Exception { useCustomInitializer = true; @@ -6804,7 +6804,7 @@ public void testExceptionFromVM_OnVertexManagerEventReceived() throws Exception String diagnostics = StringUtils.join(v1.getDiagnostics(), ","); Assert.assertTrue(diagnostics.contains(VMExceptionLocation.OnVertexManagerEventReceived.name())); } - + @Test(timeout = 5000) public void testExceptionFromVM_OnVertexManagerVertexStateUpdated() throws Exception { useCustomInitializer = true; @@ -6830,7 +6830,7 @@ public void testExceptionFromVM_OnVertexManagerVertexStateUpdated() throws Excep assertTrue(diagnostics.contains(VMExceptionLocation.OnVertexManagerVertexStateUpdated.name())); Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v2.getTerminationCause()); } - + @Test(timeout = 5000) public void testExceptionFromII_Initialize() throws InterruptedException, TezException { useCustomInitializer = true; @@ -6946,7 +6946,7 @@ public void testExceptionFromII_HandleInputInitializerEvent() throws Exception { dispatcher.getEventHandler() .handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent))); dispatcher.await(); - + // it would cause v2 fail as its II throw exception in handleInputInitializerEvent String diagnostics = StringUtils.join(v2.getDiagnostics(), ","); assertTrue(diagnostics.contains(IIExceptionLocation.HandleInputInitializerEvent.name())); @@ -7209,13 +7209,13 @@ public static enum VMExceptionLocation { OnVertexManagerVertexStateUpdated, Initialize, } - + private VMExceptionLocation exLocation; - + public VertexManagerWithException(VertexManagerPluginContext context) { super(context); } - + @Override public void initialize() { super.initialize(); @@ -7228,7 +7228,7 @@ public void initialize() { getContext().vertexReconfigurationPlanned(); } } - + @Override public void onRootVertexInitialized(String inputName, InputDescriptor inputDescriptor, List events) { @@ -7237,7 +7237,7 @@ public void onRootVertexInitialized(String inputName, } super.onRootVertexInitialized(inputName, inputDescriptor, events); } - + @Override public void onSourceTaskCompleted(TaskAttemptIdentifier attempt) { if (this.exLocation == VMExceptionLocation.OnSourceTaskCompleted) { @@ -7245,7 +7245,7 @@ public void onSourceTaskCompleted(TaskAttemptIdentifier attempt) { } super.onSourceTaskCompleted(attempt); } - + @Override public void onVertexStarted(List completions) { if (this.exLocation == VMExceptionLocation.OnVertexStarted) { @@ -7256,7 +7256,7 @@ public void onVertexStarted(List completions) { getContext().doneReconfiguringVertex(); } } - + @Override public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) { super.onVertexManagerEventReceived(vmEvent); @@ -7264,7 +7264,7 @@ public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) { throw new RuntimeException(this.exLocation.name()); } } - + @Override public void onVertexStateUpdated(VertexStateUpdate stateUpdate) { super.onVertexStateUpdated(stateUpdate); @@ -7274,7 +7274,7 @@ public void onVertexStateUpdated(VertexStateUpdate stateUpdate) { } } } - + public static enum IIExceptionLocation { Initialize, HandleInputInitializerEvent, diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexManager.java index 97ba83652d..abae0c48d0 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexManager.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexManager.java @@ -85,7 +85,7 @@ public class TestVertexManager { Vertex mockVertex; EventHandler mockHandler; ArgumentCaptor requestCaptor; - + @Before public void setup() { mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS); @@ -139,7 +139,7 @@ public void testVertexManagerPluginCtorAccessUserPayload() throws IOException, T mockVertex, mockAppContext, mock(StateChangeNotifier.class)); } - + @Test(timeout = 5000) public void testOnRootVertexInitialized() throws Exception { Configuration conf = new Configuration(); @@ -195,7 +195,7 @@ public void testOnRootVertexInitialized2() throws Exception { verify(mockHandler, times(1)).handle(requestCaptor.capture()); List tezEventsAfterInput1 = requestCaptor.getValue().getEvents(); assertEquals(0, tezEventsAfterInput1.size()); - + InputDescriptor id2 = mock(InputDescriptor.class); List events2 = new LinkedList(); InputDataInformationEvent diEvent2 = diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/speculation/legacy/TestDataStatistics.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/speculation/legacy/TestDataStatistics.java index f4791a99e4..129cde446a 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/speculation/legacy/TestDataStatistics.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/speculation/legacy/TestDataStatistics.java @@ -34,7 +34,7 @@ public void testEmptyDataStatistics() throws Exception { Assert.assertEquals(0, statistics.std(), TOL); Assert.assertEquals(Long.MAX_VALUE, statistics.outlier(1.0f), TOL); } - + @Test(timeout = 5000) public void testSingleEntryDataStatistics() throws Exception { DataStatistics statistics = new DataStatistics(17.29); @@ -44,7 +44,7 @@ public void testSingleEntryDataStatistics() throws Exception { Assert.assertEquals(0, statistics.std(), TOL); Assert.assertEquals(17.29, statistics.outlier(1.0f), TOL); } - + @Test(timeout = 5000) public void testMutiEntryDataStatistics() throws Exception { DataStatistics statistics = new DataStatistics(); @@ -56,7 +56,7 @@ public void testMutiEntryDataStatistics() throws Exception { Assert.assertEquals(6.0, statistics.std(), TOL); Assert.assertEquals(29.0, statistics.outlier(1.0f), TOL); } - + @Test(timeout = 5000) public void testUpdateStatistics() throws Exception { DataStatistics statistics = new DataStatistics(17); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java index 684e397d12..10f840765d 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java @@ -298,7 +298,7 @@ public void testDelayedReuseContainerNotAvailable() .getSpyTaskScheduler(); TaskSchedulerContextDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback(); - + AtomicBoolean drainNotifier = new AtomicBoolean(false); taskScheduler.delayedContainerManager.drainedDelayedContainersForTest = drainNotifier; @@ -940,7 +940,7 @@ public void testReuseAcrossVertices() taskScheduler.shutdown(); taskSchedulerManager.close(); } - + @Test(timeout = 30000l) public void testReuseLocalResourcesChanged() throws IOException, InterruptedException, ExecutionException { LOG.info("Test testReuseLocalResourcesChanged"); @@ -971,7 +971,7 @@ public void testReuseLocalResourcesChanged() throws IOException, InterruptedExce doReturn(true).when(appContext).isSession(); doAnswer(dagIDAnswer).when(appContext).getCurrentDAGID(); doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo(); - + TaskSchedulerManager taskSchedulerManagerReal = new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient, new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf)); TaskSchedulerManager taskSchedulerManager = spy(taskSchedulerManagerReal); @@ -998,12 +998,12 @@ public void testReuseLocalResourcesChanged() throws IOException, InterruptedExce LocalResource lr3 = mock(LocalResource.class); AMContainerEventAssignTA assignEvent = null; - + Map dag1LRs = Maps.newHashMap(); dag1LRs.put(rsrc1, lr1); TezVertexID vertexID11 = TezVertexID.getInstance(dagID1, 1); - + //Vertex 1, Task 1, Attempt 1, host1, lr1 TezTaskAttemptID taID111 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID11, 1), 1); TaskAttempt ta111 = mock(TaskAttempt.class); @@ -1026,7 +1026,7 @@ public void testReuseLocalResourcesChanged() throws IOException, InterruptedExce verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta111), any(), eq(container1)); assignEvent = (AMContainerEventAssignTA) eventHandler.verifyInvocation(AMContainerEventAssignTA.class); assertEquals(1, assignEvent.getRemoteTaskLocalResources().size()); - + // Task assigned to container completed successfully. Container should be re-used. taskSchedulerManager.handleEvent( new AMSchedulerEventTAEnded(ta111, container1.getId(), TaskAttemptState.SUCCEEDED, null, @@ -1054,13 +1054,13 @@ public void testReuseLocalResourcesChanged() throws IOException, InterruptedExce // Setup DAG2 with additional resources. Make sure the container, even without all resources, is reused. TezDAGID dagID2 = TezDAGID.getInstance("0", 2, 0); dagIDAnswer.setDAGID(dagID2); - + Map dag2LRs = Maps.newHashMap(); dag2LRs.put(rsrc2, lr2); dag2LRs.put(rsrc3, lr3); TezVertexID vertexID21 = TezVertexID.getInstance(dagID2, 1); - + //Vertex 2, Task 1, Attempt 1, host1, lr2 TezTaskAttemptID taID211 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID21, 1), 1); TaskAttempt ta211 = mock(TaskAttempt.class); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java index 3afd186e18..e1fc6d6071 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java @@ -279,7 +279,7 @@ public void testTaskSchedulerNoReuse() throws Exception { // no other statuses returned verify(mockApp, times(3)).containerCompleted(any(), (ContainerStatus) any()); verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId) any()); - + // verify blacklisting verify(mockRMClient, times(0)).addNodeToBlacklist((NodeId)any()); String badHost = "host6"; @@ -329,7 +329,7 @@ public void testTaskSchedulerNoReuse() throws Exception { float progress = 0.5f; when(mockApp.getProgress()).thenReturn(progress); Assert.assertEquals(progress, scheduler.getProgress(), 0); - + // check duplicate allocation request scheduler.allocateTask(mockTask1, mockCapability, hosts, racks, mockPriority, null, mockCookie1); @@ -583,7 +583,7 @@ public void testTaskSchedulerWithReuse() throws Exception { AtomicBoolean drainNotifier = new AtomicBoolean(false); scheduler.delayedContainerManager.drainedDelayedContainersForTest = drainNotifier; - + scheduler.onContainersAllocated(containers); TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier); drainableAppCallback.drain(); @@ -684,7 +684,7 @@ public void testTaskSchedulerWithReuse() throws Exception { scheduler.unblacklistNode(badNodeId); verify(mockRMClient, times(1)).removeNodeFromBlacklist(badNodeId); assertEquals(0, scheduler.blacklistedNodes.size()); - + // verify container level matching // add a dummy task to prevent release of allocated containers Object mockTask5 = new MockTask("task5"); @@ -706,7 +706,7 @@ public void testTaskSchedulerWithReuse() throws Exception { drainNotifier.set(false); scheduler.onContainersAllocated(containers); drainableAppCallback.drain(); - verify(mockRMClient, times(5)).releaseAssignedContainer((ContainerId) any()); + verify(mockRMClient, times(5)).releaseAssignedContainer((ContainerId) any()); Object mockTask6 = new MockTask("task6"); Object mockCookie6 = new Object(); // allocate request with container affinity @@ -727,7 +727,7 @@ public void testTaskSchedulerWithReuse() throws Exception { verify(mockRMClient).releaseAssignedContainer(mockCId7); verify(mockRMClient).releaseAssignedContainer(mockCId8); verify(mockRMClient, times(7)).releaseAssignedContainer((ContainerId) any()); - + float progress = 0.5f; when(mockApp.getProgress()).thenReturn(progress); @@ -761,7 +761,7 @@ public void testTaskSchedulerWithReuse() throws Exception { SUCCEED_APP_MESSAGE, DEFAULT_APP_URL); verify(mockRMClient).stop(); } - + @Test (timeout=5000) public void testTaskSchedulerDetermineMinHeldContainers() throws Exception { TezAMRMClientAsync mockRMClient = spy( @@ -776,7 +776,7 @@ public void testTaskSchedulerDetermineMinHeldContainers() throws Exception { scheduler.initialize(); scheduler.start(); - + String rack1 = "r1"; String rack2 = "r2"; String rack3 = "r3"; @@ -839,22 +839,22 @@ public void testTaskSchedulerDetermineMinHeldContainers() throws Exception { scheduler.heldContainers.put(mockCId5, hc5); scheduler.heldContainers.put(mockCId6, hc6); scheduler.heldContainers.put(mockCId7, hc7); - + // test empty case scheduler.sessionNumMinHeldContainers = 0; scheduler.determineMinHeldContainers(); Assert.assertEquals(0, scheduler.sessionMinHeldContainers.size()); - + // test min >= held scheduler.sessionNumMinHeldContainers = 7; scheduler.determineMinHeldContainers(); Assert.assertEquals(7, scheduler.sessionMinHeldContainers.size()); - + // test min < held scheduler.sessionNumMinHeldContainers = 5; scheduler.determineMinHeldContainers(); Assert.assertEquals(5, scheduler.sessionMinHeldContainers.size()); - + Set heldContainers = Sets.newHashSet(); for (ContainerId cId : scheduler.sessionMinHeldContainers) { heldContainers.add(scheduler.heldContainers.get(cId)); @@ -876,7 +876,7 @@ public void testTaskSchedulerDetermineMinHeldContainers() throws Exception { Assert.assertEquals(3, racks.size()); Assert.assertTrue(racks.contains(rack1) && racks.contains(rack2) && racks.contains(rack3)); - + long currTime = System.currentTimeMillis(); heldContainers.clear(); heldContainers.addAll(scheduler.heldContainers.values()); @@ -933,7 +933,7 @@ public void testTaskSchedulerHeldContainersReleaseAfterExpired() throws Exceptio when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.shutdown(); } - + @Test(timeout=5000) public void testTaskSchedulerRandomReuseExpireTime() throws Exception { TezAMRMClientAsync mockRMClient = spy( @@ -965,12 +965,12 @@ public void testTaskSchedulerRandomReuseExpireTime() throws Exception { scheduler1.start(); scheduler2.start(); - + // when min == max the expire time is always min for (int i=0; i<10; ++i) { Assert.assertEquals(minTime, scheduler1.getHeldContainerExpireTime(0)); } - + long lastExpireTime = 0; // when min < max the expire time is random in between min and max for (int i=0; i<10; ++i) { @@ -1147,7 +1147,7 @@ public void testTaskSchedulerPreemption() throws Exception { final ArrayList anyContainers = new ArrayList(); - + Resource taskAsk = Resource.newInstance(1024, 1); scheduler.allocateTask(mockTask1, taskAsk, null, null, pri2, null, null); @@ -1243,7 +1243,7 @@ public List> answer( } }); - + doAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { Object[] args = invocation.getArguments(); @@ -1252,7 +1252,7 @@ public Object answer(InvocationOnMock invocation) { return null; }}) .when(mockApp).preemptContainer((ContainerId)any()); - + scheduler.onContainersAllocated(containers); drainableAppCallback.drain(); Assert.assertEquals(4, scheduler.taskAllocations.size()); @@ -1263,7 +1263,7 @@ public Object answer(InvocationOnMock invocation) { scheduler.taskAllocations.get(mockTask3).getId()); Assert.assertEquals(mockCId3, scheduler.taskAllocations.get(mockTask3KillA).getId()); - // high priority container assigned to lower pri task. This task should still be preempted + // high priority container assigned to lower pri task. This task should still be preempted // because the task priority is relevant for preemption and not the container priority Assert.assertEquals(mockCId4, scheduler.taskAllocations.get(mockTask3KillB).getId()); @@ -1293,7 +1293,7 @@ public Object answer(InvocationOnMock invocation) { Container mockContainer5 = Container.newInstance(mockCId5, host1, null, taskAsk, pri8, null); containers.clear(); containers.add(mockContainer5); - + // new lower pri container added that wont be matched and eventually preempted // Fudge new container being present in delayed allocation list due to race HeldContainer heldContainer = new HeldContainer(mockContainer5, -1, -1, null, @@ -1324,7 +1324,7 @@ public Object answer(InvocationOnMock invocation) { Assert.assertEquals(pri8, reAdded.getPriority()); Assert.assertEquals(taskAsk, reAdded.getCapability()); Assert.assertEquals(mockTaskPri8Cookie, reAdded.getCookie().getAppCookie()); - + // remove fudging. scheduler.delayedContainerManager.delayedContainers.clear(); @@ -1348,7 +1348,7 @@ public Object answer(InvocationOnMock invocation) { drainableAppCallback.drain(); // mockTaskPri3KillB gets preempted to clear 10% of outstanding running preemptable tasks - // this is also a higher priority container than the pending task priority but was running a + // this is also a higher priority container than the pending task priority but was running a // lower priority task. Task priority is relevant for preemption and not container priority as // containers can run tasks of different priorities scheduler.getProgress(); // first heartbeat @@ -1378,14 +1378,14 @@ public Object answer(InvocationOnMock invocation) { scheduler.shutdown(); drainableAppCallback.drain(); } - + @Test (timeout=5000) public void testTaskSchedulerPreemption2() throws Exception { TezAMRMClientAsync mockRMClient = spy( new AMRMClientAsyncForTest(new AMRMClientForTest(), 100)); int waitTime = 1000; - + Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false); conf.setInt(TezConfiguration.TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS, 2); @@ -1422,7 +1422,7 @@ public void testTaskSchedulerPreemption2() throws Exception { final ArrayList anyContainers = new ArrayList(); - + Resource taskAsk = Resource.newInstance(1024, 1); scheduler.allocateTask(mockTask1, taskAsk, null, null, pri4, null, null); @@ -1442,7 +1442,7 @@ public void testTaskSchedulerPreemption2() throws Exception { Container mockContainer1 = Container.newInstance(mockCId1, host1, null, taskAsk, pri4, null); List containers = new ArrayList(); containers.add(mockContainer1); - + doAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { Object[] args = invocation.getArguments(); @@ -1451,7 +1451,7 @@ public Object answer(InvocationOnMock invocation) { return null; }}) .when(mockApp).preemptContainer((ContainerId)any()); - + scheduler.onContainersAllocated(containers); drainableAppCallback.drain(); Assert.assertEquals(1, scheduler.taskAllocations.size()); @@ -1484,10 +1484,10 @@ public Object answer(InvocationOnMock invocation) { Assert.assertEquals(pri2, scheduler.highestWaitingRequestPriority); Assert.assertTrue(scheduler.highestWaitingRequestWaitStartTime >= currTime); Assert.assertEquals(scheduler.numHeartbeats, scheduler.heartbeatAtLastPreemption); - + Thread.sleep(waitTime + 10); long oldStartWaitTime = scheduler.highestWaitingRequestWaitStartTime; - + scheduler.getProgress(); drainableAppCallback.drain(); verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId)any()); @@ -1507,7 +1507,7 @@ public Object answer(InvocationOnMock invocation) { // remove high pri request to test waiting pri change scheduler.deallocateTask(mockTask2, false, null, null); - + scheduler.getProgress(); // waiting value changes Assert.assertEquals(pri6, scheduler.highestWaitingRequestPriority); @@ -1549,7 +1549,7 @@ public void testLocalityMatching() throws Exception { taskScheduler.initialize(); taskScheduler.start(); - + Resource resource = Resource.newInstance(1024, 1); Priority priority = Priority.newInstance(1); @@ -1627,7 +1627,7 @@ public void testLocalityMatching() throws Exception { when(appClient.getFinalAppStatus()).thenReturn(finalStatus); taskScheduler.shutdown(); } - + @Test (timeout=5000) public void testScaleDownPercentage() { Assert.assertEquals(100, YarnTaskSchedulerService.scaleDownByPreemptionPercentage(100, 100)); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java index 277291110c..62f5bf5d1e 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java @@ -141,7 +141,7 @@ RegisterApplicationMasterResponse getRegistrationResponse() { return mockRegResponse; } } - + // Overrides start / stop. Will be controlled without the extra event handling thread. static class TaskSchedulerManagerForTest extends TaskSchedulerManager { @@ -265,7 +265,7 @@ static class TaskSchedulerContextDrainable implements TaskSchedulerContext { private final TaskSchedulerContext real; private final CountingExecutorService countingExecutorService; final AtomicInteger count = new AtomicInteger(0); - + public TaskSchedulerContextDrainable(TaskSchedulerContextImplWrapper real) { countingExecutorService = (CountingExecutorService) real.getExecutorService(); this.real = real; @@ -444,7 +444,7 @@ public Object union(Object cs1, Object cs2) { return cs1; } } - + static class PreemptionMatcher implements ContainerSignatureMatcher { @Override public boolean isSuperSet(Object cs1, Object cs2) { @@ -469,7 +469,7 @@ public Object union(Object cs1, Object cs2) { return cs1; } } - + static void waitForDelayedDrainNotify(AtomicBoolean drainNotifier) throws InterruptedException { diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java index 3b9936d287..ba1666381f 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java @@ -133,7 +133,7 @@ @SuppressWarnings("rawtypes") public class TestTaskSchedulerManager { - + class TestEventHandler implements EventHandler{ List events = Lists.newLinkedList(); @Override @@ -141,11 +141,11 @@ public void handle(Event event) { events.add(event); } } - + class MockTaskSchedulerManager extends TaskSchedulerManager { final AtomicBoolean notify = new AtomicBoolean(false); - + public MockTaskSchedulerManager(AppContext appContext, DAGClientServer clientService, EventHandler eventHandler, ContainerSignatureMatcher containerSignatureMatcher, @@ -162,7 +162,7 @@ protected void instantiateSchedulers(String host, int port, String trackingUrl, taskSchedulerServiceWrappers[0] = new ServicePluginLifecycleAbstractService<>(taskSchedulers[0].getTaskScheduler()); } - + @Override protected void notifyForTest() { synchronized (notify) { @@ -170,7 +170,7 @@ protected void notifyForTest() { notify.notifyAll(); } } - + } AppContext mockAppContext; @@ -342,21 +342,21 @@ public void testTaskBasedAffinity() throws Exception { taskSchedulerManager.notify.wait(); } } - + // verify mockTaskAttempt affinitized to expected affCId verify(mockTaskScheduler, times(1)).allocateTask(mockTaskAttempt, resource, affCId, Priority.newInstance(3), null, event); - + taskSchedulerManager.stop(); taskSchedulerManager.close(); } - + @Test (timeout = 5000) public void testContainerPreempted() throws IOException { Configuration conf = new Configuration(false); taskSchedulerManager.init(conf); taskSchedulerManager.start(); - + String diagnostics = "Container preempted by RM."; TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class); ContainerStatus mockStatus = mock(ContainerStatus.class); @@ -444,13 +444,13 @@ public void testContainerInternalPreemptedAfterContainerCleanup() throws IOExcep taskSchedulerManager.stop(); taskSchedulerManager.close(); } - + @Test (timeout = 5000) public void testContainerDiskFailed() throws IOException { Configuration conf = new Configuration(false); taskSchedulerManager.init(conf); taskSchedulerManager.start(); - + String diagnostics = "NM disk failed."; TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class); ContainerStatus mockStatus = mock(ContainerStatus.class); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java index 9a9c46155b..a44160760f 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java @@ -123,7 +123,7 @@ public void tetSingleSuccessfulTaskFlow() { assertEquals(wc.taskAttemptID, wc.amContainer.getCurrentTaskAttempt()); assertTrue(wc.amContainer.getCurrentTaskAttemptAllocationTime() > 0); assertTrue(wc.amContainer.getCurrentTaskAttemptAllocationTime() >= currTime); - + // Container Launched wc.containerLaunched(); wc.verifyState(AMContainerState.RUNNING); @@ -1235,7 +1235,7 @@ static class WrappedContainer { EventHandler eventHandler; AppContext appContext; - + HistoryEventHandler historyEventHandler; TezDAGID dagID; @@ -1274,7 +1274,7 @@ public WrappedContainer(boolean shouldProfile, String profileString, int cIdInt) vertexID = TezVertexID.getInstance(dagID, 1); taskID = TezTaskID.getInstance(vertexID, 1); taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1); - + eventHandler = mock(EventHandler.class); historyEventHandler = mock(HistoryEventHandler.class); @@ -1328,7 +1328,7 @@ public List verifyCountAndGetOutgoingEvents(int invocations) { verify(eventHandler, times(invocations)).handle(args.capture()); return args.getAllValues(); } - + public void verifyHistoryStopEvent() { ArgumentCaptor args = ArgumentCaptor.forClass(DAGHistoryEvent.class); verify(historyEventHandler, times(1)).handle(args.capture()); @@ -1425,11 +1425,11 @@ public void verifyState(AMContainerState state) { state, amContainer.getState()); } } - + private static class WrappedContainerMultipleDAGs extends WrappedContainer { - + private TezDAGID newDAGID = null; - + @Override protected void mockDAGID() { doAnswer(new Answer() { @@ -1439,7 +1439,7 @@ public TezDAGID answer(InvocationOnMock invocation) throws Throwable { } }).when(appContext).getCurrentDAGID(); } - + void setNewDAGID(TezDAGID newDAGID) { this.newDAGID = newDAGID; } @@ -1482,7 +1482,7 @@ private Event findEventByType(List events, Enum type) { } return null; } - + private LocalResource createLocalResource(String name) { LocalResource lr = LocalResource.newInstance(URL.newInstance(null, "localhost", 2321, name), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, 1, 1000000); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java index 79a7433f60..b714f2ebbe 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java @@ -64,7 +64,7 @@ public class TestAMNodeTracker { DrainDispatcher dispatcher; EventHandler eventHandler; - + @Before public void setup() { dispatcher = new DrainDispatcher(); @@ -72,7 +72,7 @@ public void setup() { dispatcher.start(); eventHandler = dispatcher.getEventHandler(); } - + class TestEventHandler implements EventHandler{ List events = Lists.newLinkedList(); @SuppressWarnings("unchecked") @@ -82,12 +82,12 @@ public void handle(Event event) { eventHandler.handle(event); } } - + @After public void teardown() { dispatcher.stop(); } - + @Test(timeout=5000) public void testHealthUpdateKnownNode() { AppContext appContext = mock(AppContext.class); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java index ec18574849..bb9465cd12 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java @@ -223,7 +223,7 @@ private void testDAGStartedEvent() throws Exception { } private void testDAGKillRequestEvent() throws Exception { - DAGKillRequestEvent event = + DAGKillRequestEvent event = new DAGKillRequestEvent(TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 100334l,false); DAGKillRequestEvent deserializedEvent = (DAGKillRequestEvent) testProtoConversion(event); @@ -328,17 +328,17 @@ private void testVertexReconfigureDoneEvent() throws Exception { Map rootInputSpecUpdates = new HashMap(); rootInputSpecUpdates.put("input1", rootInputSpecUpdateBulk); rootInputSpecUpdates.put("input2", rootInputSpecUpdatePerTask); - + Map sourceEdgeManagers = new HashMap(); // add standard and custom edge - sourceEdgeManagers.put("foo", EdgeProperty.create(DataMovementType.SCATTER_GATHER, - DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, + sourceEdgeManagers.put("foo", EdgeProperty.create(DataMovementType.SCATTER_GATHER, + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out1"), InputDescriptor.create("in1"))); sourceEdgeManagers.put("foo1", EdgeProperty.create(EdgeManagerPluginDescriptor.create("bar1") .setUserPayload( - UserPayload.create(ByteBuffer.wrap(new String("payload").getBytes()), 100)), - DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, + UserPayload.create(ByteBuffer.wrap(new String("payload").getBytes()), 100)), + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out1"), InputDescriptor.create("in1"))); final long reconfigureDoneTime = 100; diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java index 6b16287d1f..ae95aca331 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java @@ -222,7 +222,7 @@ public void testConvertVertexReconfigureDoneEvent() throws JSONException { ApplicationId.newInstance(1l, 1), 1), 1); Map edgeMgrs = new HashMap(); - + edgeMgrs.put("a", EdgeProperty.create(EdgeManagerPluginDescriptor.create("a.class") .setHistoryText("text"), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))); diff --git a/tez-dag/src/test/java/org/apache/tez/test/EdgeManagerForTest.java b/tez-dag/src/test/java/org/apache/tez/test/EdgeManagerForTest.java index 47d133b315..08313a69c4 100644 --- a/tez-dag/src/test/java/org/apache/tez/test/EdgeManagerForTest.java +++ b/tez-dag/src/test/java/org/apache/tez/test/EdgeManagerForTest.java @@ -76,7 +76,7 @@ public int routeInputErrorEventToSource(InputReadErrorEvent event, int destinati @Override public void routeInputSourceTaskFailedEventToDestination(int sourceTaskIndex, - Map> destinationTaskAndInputIndices) { + Map> destinationTaskAndInputIndices) { } @Override @@ -107,7 +107,7 @@ public int routeInputErrorEventToSource(int destinationTaskIndex, int destinatio throws Exception { return 0; } - + // End of overridden methods } \ No newline at end of file diff --git a/tez-dag/src/test/java/org/apache/tez/test/VertexManagerPluginForTest.java b/tez-dag/src/test/java/org/apache/tez/test/VertexManagerPluginForTest.java index 1cdaeca5c4..7a3a92f789 100644 --- a/tez-dag/src/test/java/org/apache/tez/test/VertexManagerPluginForTest.java +++ b/tez-dag/src/test/java/org/apache/tez/test/VertexManagerPluginForTest.java @@ -38,29 +38,29 @@ public class VertexManagerPluginForTest extends VertexManagerPlugin { VertexManagerPluginForTestConfig pluginConfig = new VertexManagerPluginForTestConfig(); - + public static class VertexManagerPluginForTestConfig { Configuration conf = new Configuration(false); static final String RECONFIGURE_ON_START = "reconfigureOnStart"; static final String NUM_TASKS = "numTasks"; - + public void setReconfigureOnStart(boolean value) { conf.setBoolean(RECONFIGURE_ON_START, value); } - + public void setNumTasks(int value) { conf.setInt(NUM_TASKS, value); } - + boolean getReconfigureOnStart() { return conf.getBoolean(RECONFIGURE_ON_START, false); } - + int getNumTasks() { return conf.getInt(NUM_TASKS, 1); } - + public ByteBuffer getPayload() { ByteArrayOutputStream b = new ByteArrayOutputStream(); DataOutputStream out = new DataOutputStream(b); @@ -71,7 +71,7 @@ public ByteBuffer getPayload() { } return ByteBuffer.wrap(b.toByteArray()); } - + void initialize(ByteBuffer buff) { ByteBuffer copy = ByteBuffer.allocate(buff.capacity()); copy.put(buff); @@ -88,7 +88,7 @@ void initialize(ByteBuffer buff) { public VertexManagerPluginForTest(VertexManagerPluginContext context) { super(context); } - + @Override public void initialize() { diff --git a/tez-examples/src/main/java/org/apache/tez/examples/JoinDataGen.java b/tez-examples/src/main/java/org/apache/tez/examples/JoinDataGen.java index b288c13cd0..9c5c944bc4 100644 --- a/tez-examples/src/main/java/org/apache/tez/examples/JoinDataGen.java +++ b/tez-examples/src/main/java/org/apache/tez/examples/JoinDataGen.java @@ -142,13 +142,13 @@ private DAG createDag(TezConfiguration tezConf, Path largeOutPath, Path smallOut GenDataProcessor.class.getName()).setUserPayload( UserPayload.create(ByteBuffer.wrap(GenDataProcessor.createConfiguration(largeOutSizePerTask, smallOutSizePerTask)))), numTasks); - genDataVertex.addDataSink(STREAM_OUTPUT_NAME, + genDataVertex.addDataSink(STREAM_OUTPUT_NAME, MROutput.createConfigBuilder(new Configuration(tezConf), TextOutputFormat.class, largeOutPath.toUri().toString()).build()); - genDataVertex.addDataSink(HASH_OUTPUT_NAME, + genDataVertex.addDataSink(HASH_OUTPUT_NAME, MROutput.createConfigBuilder(new Configuration(tezConf), TextOutputFormat.class, smallOutPath.toUri().toString()).build()); - genDataVertex.addDataSink(EXPECTED_OUTPUT_NAME, + genDataVertex.addDataSink(EXPECTED_OUTPUT_NAME, MROutput.createConfigBuilder(new Configuration(tezConf), TextOutputFormat.class, expectedOutputPath.toUri().toString()).build()); diff --git a/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java b/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java index c99e7c603c..a84e3f76a4 100644 --- a/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java +++ b/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java @@ -125,7 +125,7 @@ protected int runJob(String[] args, TezConfiguration tezConf, } } } - + } @Override diff --git a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java b/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java index fd71d93777..b6b43e4585 100644 --- a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java +++ b/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java @@ -54,7 +54,7 @@ * The example extends WordCount by sorting the words by their count. */ public class OrderedWordCount extends TezExampleBase { - + private static String INPUT = WordCount.INPUT; private static String OUTPUT = WordCount.OUTPUT; private static String TOKENIZER = WordCount.TOKENIZER; @@ -63,8 +63,8 @@ public class OrderedWordCount extends TezExampleBase { private static final Logger LOG = LoggerFactory.getLogger(OrderedWordCount.class); /* - * SumProcessor similar to WordCount except that it writes the count as key and the - * word as value. This is because we can and ordered partitioned key value edge to group the + * SumProcessor similar to WordCount except that it writes the count as key and the + * word as value. This is because we can and ordered partitioned key value edge to group the * words with the same count (as key) and order the counts. */ public static class SumProcessor extends SimpleProcessor { @@ -94,9 +94,9 @@ public void run() throws Exception { } } } - + /** - * No-op sorter processor. It does not need to apply any logic since the ordered partitioned edge + * No-op sorter processor. It does not need to apply any logic since the ordered partitioned edge * ensures that we get the data sorted and grouped by the the sum key. */ public static class NoOpSorter extends SimpleMRProcessor { @@ -120,7 +120,7 @@ public void run() throws Exception { // deriving from SimpleMRProcessor takes care of committing the output } } - + public static DAG createDAG(TezConfiguration tezConf, String inputPath, String outputPath, int numPartitions, boolean disableSplitGrouping, boolean isGenerateSplitInClient, String dagName) throws IOException { @@ -148,8 +148,8 @@ public static DAG createDAG(TezConfiguration tezConf, String inputPath, String o // via an output edge. Vertex summationVertex = Vertex.create(SUMMATION, ProcessorDescriptor.create( SumProcessor.class.getName()), numPartitions); - - // Use IntWritable key and Text value to bring all words with the same count in the same + + // Use IntWritable key and Text value to bring all words with the same count in the same // partition. The data will be ordered by count and words grouped by count. The // setFromConfiguration call is optional and allows overriding the config options with // command line parameters. @@ -166,7 +166,7 @@ public static DAG createDAG(TezConfiguration tezConf, String inputPath, String o sorterVertex.addDataSink(OUTPUT, dataSink); // No need to add jar containing this class as assumed to be part of the tez jars. - + DAG dag = DAG.create(dagName); dag.addVertex(tokenizerVertex) .addVertex(summationVertex) @@ -176,7 +176,7 @@ public static DAG createDAG(TezConfiguration tezConf, String inputPath, String o summationEdgeConf.createDefaultEdgeProperty())) .addEdge( Edge.create(summationVertex, sorterVertex, sorterEdgeConf.createDefaultEdgeProperty())); - return dag; + return dag; } @Override diff --git a/tez-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java b/tez-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java index 432dc06241..bc9b89eeda 100644 --- a/tez-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java +++ b/tez-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java @@ -28,12 +28,12 @@ import org.slf4j.LoggerFactory; /** - * Simple example that shows how Tez session mode can be used to run multiple DAGs in the same - * session for efficiency and performance. Tez session mode enables the execution environment to - * hold onto execution resources so that they can be reused to across multiple DAGs. Typical use + * Simple example that shows how Tez session mode can be used to run multiple DAGs in the same + * session for efficiency and performance. Tez session mode enables the execution environment to + * hold onto execution resources so that they can be reused to across multiple DAGs. Typical use * cases for this would be drill-down queries from a user shell or maintaining a pool of resources * to provide low latency execution of DAGs for an application. - * In this example we will be submitting multiple OrderedWordCount DAGs on different inputs to the + * In this example we will be submitting multiple OrderedWordCount DAGs on different inputs to the * same session. */ public class SimpleSessionExample extends TezExampleBase { @@ -68,19 +68,19 @@ protected int runJob(String[] args, TezConfiguration tezConf, int numPartitions = args.length == 3 ? Integer.parseInt(args[2]) : 1; // Session pre-warming allows the user to hide initial startup, resource acquisition latency etc. - // by pre-allocating execution resources in the Tez session. They can run initialization logic + // by pre-allocating execution resources in the Tez session. They can run initialization logic // in these pre-allocated resources (containers) to pre-warm the containers. // In between DAG executions, the session can hold on to a minimum number of containers. - // Ideally, this would be enough to provide desired balance of efficiency for the application - // and sharing of resources with other applications. Typically, the number of containers to be + // Ideally, this would be enough to provide desired balance of efficiency for the application + // and sharing of resources with other applications. Typically, the number of containers to be // pre-warmed equals the number of containers to be held between DAGs. if (tezConf.getBoolean(enablePrewarmConfig, false)) { // the above parameter is not a Tez parameter. Its only for this example. // In this example we are pre-warming enough containers to run all the sum tasks in parallel. // This means pre-warming numPartitions number of containers. - // We are making the pre-warm and held containers to be the same and using the helper API to - // set up pre-warming. They can be made different and also custom initialization logic can be - // specified using other API's. We know that the OrderedWordCount dag uses default files and + // We are making the pre-warm and held containers to be the same and using the helper API to + // set up pre-warming. They can be made different and also custom initialization logic can be + // specified using other API's. We know that the OrderedWordCount dag uses default files and // resources. Otherwise we would have to specify matching parameters in the preWarm API too. tezConf.setInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, numPartitions); tezClient.preWarm(PreWarmVertex.createConfigBuilder(tezConf).build()); diff --git a/tez-examples/src/main/java/org/apache/tez/examples/TezExampleBase.java b/tez-examples/src/main/java/org/apache/tez/examples/TezExampleBase.java index 5716a0a0f6..3cddf1fe99 100644 --- a/tez-examples/src/main/java/org/apache/tez/examples/TezExampleBase.java +++ b/tez-examples/src/main/java/org/apache/tez/examples/TezExampleBase.java @@ -137,7 +137,7 @@ public final int run(String[] args) throws Exception { * provided configuration. If TezClient is specified, local mode option can not been * specified in arguments, it takes no effect. * @return Zero indicates success, non-zero indicates failure - * @throws Exception + * @throws Exception */ public int run(TezConfiguration conf, String[] args, @Nullable TezClient tezClient) throws Exception { @@ -310,7 +310,7 @@ protected void printExtraOptionsUsage(PrintStream ps) { */ protected abstract int runJob(String[] args, TezConfiguration tezConf, TezClient tezClient) throws Exception; - + @Private @VisibleForTesting public ApplicationId getAppId() { diff --git a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java b/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java index 17cc2b7cd3..a61307e449 100644 --- a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java +++ b/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java @@ -50,7 +50,7 @@ import org.slf4j.LoggerFactory; /** - * Simple example to perform WordCount using Tez API's. WordCount is the + * Simple example to perform WordCount using Tez API's. WordCount is the * HelloWorld program of distributed data processing and counts the number * of occurrences of a word in a distributed text data set. */ @@ -103,11 +103,11 @@ public void run() throws Exception { /* * Example code to write a processor that commits final output to a data sink - * The SumProcessor aggregates the sum of individual word counts generated by + * The SumProcessor aggregates the sum of individual word counts generated by * the TokenProcessor. * The SumProcessor is connected to a DataSink. In this case, its an Output that * writes the data via an OutputFormat to a data sink (typically HDFS). Thats why - * it derives from SimpleMRProcessor that takes care of handling the necessary + * it derives from SimpleMRProcessor that takes care of handling the necessary * output commit operations that makes the final output available for consumers. */ public static class SumProcessor extends SimpleMRProcessor { @@ -121,7 +121,7 @@ public void run() throws Exception { Preconditions.checkArgument(getOutputs().size() == 1); KeyValueWriter kvWriter = (KeyValueWriter) getOutputs().get(OUTPUT).getWriter(); // The KeyValues reader provides all values for a given key. The aggregation of values per key - // is done by the LogicalInput. Since the key is the word and the values are its counts in + // is done by the LogicalInput. Since the key is the word and the values are its counts in // the different TokenProcessors, summing all values per key provides the sum for that word. KeyValuesReader kvReader = (KeyValuesReader) getInputs().get(TOKENIZER).getReader(); while (kvReader.next()) { @@ -140,7 +140,7 @@ public void run() throws Exception { private DAG createDAG(TezConfiguration tezConf, String inputPath, String outputPath, int numPartitions) throws IOException { - // Create the descriptor that describes the input data to Tez. Using MRInput to read text + // Create the descriptor that describes the input data to Tez. Using MRInput to read text // data from the given input path. The TextInputFormat is used to read the text data. DataSourceDescriptor dataSource = MRInput.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, inputPath).groupSplits(!isDisableSplitGrouping()) @@ -151,20 +151,20 @@ private DAG createDAG(TezConfiguration tezConf, String inputPath, String outputP DataSinkDescriptor dataSink = MROutput.createConfigBuilder(new Configuration(tezConf), TextOutputFormat.class, outputPath).build(); - // Create a vertex that reads the data from the data source and tokenizes it using the - // TokenProcessor. The number of tasks that will do the work for this vertex will be decided + // Create a vertex that reads the data from the data source and tokenizes it using the + // TokenProcessor. The number of tasks that will do the work for this vertex will be decided // using the information provided by the data source descriptor. Vertex tokenizerVertex = Vertex.create(TOKENIZER, ProcessorDescriptor.create( TokenProcessor.class.getName())).addDataSource(INPUT, dataSource); - // Create the edge that represents the movement and semantics of data between the producer - // Tokenizer vertex and the consumer Summation vertex. In order to perform the summation in - // parallel the tokenized data will be partitioned by word such that a given word goes to the + // Create the edge that represents the movement and semantics of data between the producer + // Tokenizer vertex and the consumer Summation vertex. In order to perform the summation in + // parallel the tokenized data will be partitioned by word such that a given word goes to the // same partition. The counts for the words should be grouped together per word. To achieve this - // we can use an edge that contains an input/output pair that handles partitioning and grouping + // we can use an edge that contains an input/output pair that handles partitioning and grouping // of key value data. We use the helper OrderedPartitionedKVEdgeConfig to create such an // edge. Internally, it sets up matching Tez inputs and outputs that can perform this logic. - // We specify the key, value and partitioner type. Here the key type is Text (for word), the + // We specify the key, value and partitioner type. Here the key type is Text (for word), the // value type is IntWritable (for count) and we using a hash based partitioner. This is a helper // object. The edge can be configured by configuring the input, output etc individually without // using this helper. The setFromConfiguration call is optional and allows overriding the config @@ -176,23 +176,23 @@ private DAG createDAG(TezConfiguration tezConf, String inputPath, String outputP .build(); // Create a vertex that reads the tokenized data and calculates the sum using the SumProcessor. - // The number of tasks that do the work of this vertex depends on the number of partitions used - // to distribute the sum processing. In this case, its been made configurable via the + // The number of tasks that do the work of this vertex depends on the number of partitions used + // to distribute the sum processing. In this case, its been made configurable via the // numPartitions parameter. Vertex summationVertex = Vertex.create(SUMMATION, ProcessorDescriptor.create(SumProcessor.class.getName()), numPartitions) .addDataSink(OUTPUT, dataSink); - // No need to add jar containing this class as assumed to be part of the Tez jars. Otherwise + // No need to add jar containing this class as assumed to be part of the Tez jars. Otherwise // we would have to add the jars for this code as local files to the vertices. - + // Create DAG and add the vertices. Connect the producer and consumer vertices via the edge DAG dag = DAG.create("WordCount"); dag.addVertex(tokenizerVertex) .addVertex(summationVertex) .addEdge( Edge.create(tokenizerVertex, summationVertex, edgeConf.createDefaultEdgeProperty())); - return dag; + return dag; } @Override diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java index df7df4cd5d..ca6ef2089a 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java @@ -147,9 +147,9 @@ private IndexInformation readIndexFileToCache(Path indexFileName, /** * This method removes the map from the cache if index information for this - * map is loaded(size>0), index information entry in cache will not be - * removed if it is in the loading phrase(size=0), this prevents corruption - * of totalMemoryUsed. It should be called when a map output on this tracker + * map is loaded(size>0), index information entry in cache will not be + * removed if it is in the loading phrase(size=0), this prevents corruption + * of totalMemoryUsed. It should be called when a map output on this tracker * is discarded. * @param mapId The taskID of this map. */ diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java index 02ff08c293..5a3909403e 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java @@ -155,11 +155,11 @@ public class ShuffleHandler { public static final String MAX_SHUFFLE_CONNECTIONS = "mapreduce.shuffle.max.connections"; public static final int DEFAULT_MAX_SHUFFLE_CONNECTIONS = 0; // 0 implies no limit - + public static final String MAX_SHUFFLE_THREADS = "mapreduce.shuffle.max.threads"; // 0 implies Netty default of 2 * number of available processors public static final int DEFAULT_MAX_SHUFFLE_THREADS = 0; - + final boolean connectionKeepAliveEnabled; final int connectionKeepAliveTimeOut; final int mapOutputMetaInfoCacheSize; @@ -327,7 +327,7 @@ public Shuffle(Configuration conf) { indexCache = new IndexCache(conf); this.port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT); } - + public void setPort(int port) { this.port = port; } @@ -598,7 +598,7 @@ protected void verifyRequest(String appid, ChannelHandlerContext ctx, SecureShuffleUtils.verifyReply(urlHashStr, enc_str, tokenSecret); // verification passed - encode the reply String reply = - SecureShuffleUtils.generateHash(urlHashStr.getBytes(Charsets.UTF_8), + SecureShuffleUtils.generateHash(urlHashStr.getBytes(Charsets.UTF_8), tokenSecret); response.headers().set(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH, reply); // Put shuffle version into http header diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/SplitSizeEstimator.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/SplitSizeEstimator.java index a4c0e73d9a..9c0eab840c 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/SplitSizeEstimator.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/SplitSizeEstimator.java @@ -22,7 +22,7 @@ import org.apache.hadoop.mapred.InputSplit; -/* the two big differences between the mapred.* and mapreduce.* split classes +/* the two big differences between the mapred.* and mapreduce.* split classes * is that mapreduce throws InterruptedExceptions */ public interface SplitSizeEstimator { long getEstimatedSize(InputSplit split) throws IOException; diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplit.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplit.java index bc58043303..149e05cabb 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplit.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplit.java @@ -36,7 +36,7 @@ import org.apache.tez.dag.api.TezUncheckedException; /** - * Implements an InputSplit that provides a generic wrapper around + * Implements an InputSplit that provides a generic wrapper around * a group of real InputSplits */ @Public @@ -49,9 +49,9 @@ public class TezGroupedSplit implements InputSplit, Configurable { String rack = null; long length = 0; Configuration conf; - + public TezGroupedSplit() { - + } public TezGroupedSplit(List wrappedSplits, String wrappedInputFormatName, @@ -87,7 +87,7 @@ public void addSplit(InputSplit split) { throw new TezUncheckedException(e); } } - + @Override public void write(DataOutput out) throws IOException { if (wrappedSplits == null) { @@ -101,7 +101,7 @@ public void write(DataOutput out) throws IOException { writeWrappedSplit(split, out); } out.writeLong(length); - + if (locations == null || locations.length == 0) { out.writeInt(0); } else { @@ -126,12 +126,12 @@ public void readFields(DataInput in) throws IOException { } int numSplits = in.readInt(); - + wrappedSplits = new ArrayList(numSplits); for (int i=0; i clazz) + + InputSplit readWrappedSplit(DataInput in, Class clazz) throws IOException { InputSplit split; try { @@ -161,7 +161,7 @@ InputSplit readWrappedSplit(DataInput in, Class clazz) split.readFields(in); return split; } - + @Override public long getLength() throws IOException { return length; @@ -181,7 +181,7 @@ public void setConf(Configuration conf) { public Configuration getConf() { return conf; } - + public String getRack() { return rack; } diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplitsInputFormat.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplitsInputFormat.java index e8845022ba..4cfd128707 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplitsInputFormat.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplitsInputFormat.java @@ -53,9 +53,9 @@ */ @Public @Evolving -public class TezGroupedSplitsInputFormat +public class TezGroupedSplitsInputFormat implements InputFormat, Configurable{ - + private static final Logger LOG = LoggerFactory.getLogger(TezGroupedSplitsInputFormat.class); InputFormat wrappedInputFormat; @@ -64,11 +64,11 @@ public class TezGroupedSplitsInputFormat SplitSizeEstimator estimator; SplitLocationProvider locationProvider; - + public TezGroupedSplitsInputFormat() { - + } - + public void setInputFormat(InputFormat wrappedInputFormat) { this.wrappedInputFormat = wrappedInputFormat; if (LOG.isDebugEnabled()) { @@ -85,13 +85,13 @@ public void setSplitLocationProvider(SplitLocationProvider locationProvider) { this.locationProvider = Objects.requireNonNull(locationProvider); LOG.debug("Split size location provider: {}", locationProvider); } - + public void setDesiredNumberOfSplits(int num) { Preconditions.checkArgument(num >= 0); this.desiredNumSplits = num; LOG.debug("desiredNumSplits: {}", desiredNumSplits); } - + @Override public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { InputSplit[] originalSplits = wrappedInputFormat.getSplits(job, numSplits); @@ -101,7 +101,7 @@ public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { .getGroupedSplits(conf, originalSplits, desiredNumSplits, wrappedInputFormatName, estimator, locationProvider); } - + @Override public RecordReader getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException { @@ -113,11 +113,11 @@ public RecordReader getRecordReader(InputSplit split, JobConf job, } return new TezGroupedSplitsRecordReader(groupedSplit, job, reporter); } - + @SuppressWarnings({ "unchecked", "rawtypes" }) void initInputFormatFromSplit(TezGroupedSplit split) throws TezException { if (wrappedInputFormat == null) { - Class clazz = (Class) + Class clazz = (Class) getClassFromName(split.wrappedInputFormatName); try { wrappedInputFormat = org.apache.hadoop.util.ReflectionUtils.newInstance(clazz, conf); @@ -202,7 +202,7 @@ private void preInitReaders() { public RecordReader getCurReader() { return curReader; } - + @Override public boolean next(K key, V value) throws IOException { @@ -218,17 +218,17 @@ public boolean next(K key, V value) throws IOException { public K createKey() { return curReader.createKey(); } - + @Override public V createValue() { return curReader.createValue(); } - + @Override public float getProgress() throws IOException { return Math.min(1.0f, getPos()/(float)(groupedSplit.getLength())); } - + @Override public void close() throws IOException { if (curReader != null) { diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplit.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplit.java index 2d198ad8d4..b23b553b41 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplit.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplit.java @@ -42,7 +42,7 @@ */ @Public @Evolving -public class TezGroupedSplit extends InputSplit +public class TezGroupedSplit extends InputSplit implements Writable, Configurable { List wrappedSplits = null; @@ -54,7 +54,7 @@ public class TezGroupedSplit extends InputSplit @InterfaceAudience.Private public TezGroupedSplit() { - + } @InterfaceAudience.Private @@ -77,12 +77,12 @@ public TezGroupedSplit(int numSplits, String wrappedInputFormatName, this.locations = locations; this.rack = rack; } - + public TezGroupedSplit(int numSplits, String wrappedInputFormatName, String[] locations) { this(numSplits, wrappedInputFormatName, locations, null); } - + public void addSplit(InputSplit split) { wrappedSplits.add(split); try { @@ -109,7 +109,7 @@ public void write(DataOutput out) throws IOException { writeWrappedSplit(split, out); } out.writeLong(length); - + if (locations == null || locations.length == 0) { out.writeInt(0); } else { @@ -132,14 +132,14 @@ public void readFields(DataInput in) throws IOException { } catch (TezException e) { throw new IOException(e); } - + int numSplits = in.readInt(); - + wrappedSplits = new ArrayList(numSplits); for (int i=0; i clazz) { try { InputSplit split = clazz.newInstance(); @@ -171,13 +171,13 @@ InputSplit readWrappedSplit(DataInput in, Class clazz) { return split; } else { throw new TezUncheckedException( - split.getClass().getName() + " is not a Writable"); + split.getClass().getName() + " is not a Writable"); } } catch (Exception e) { throw new TezUncheckedException(e); } } - + @Override public long getLength() throws IOException, InterruptedException { return length; @@ -187,7 +187,7 @@ public long getLength() throws IOException, InterruptedException { public String[] getLocations() throws IOException, InterruptedException { return locations; } - + @Override public void setConf(Configuration conf) { this.conf = conf; @@ -197,7 +197,7 @@ public void setConf(Configuration conf) { public Configuration getConf() { return conf; } - + public String getRack() { return rack; } diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplitsInputFormat.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplitsInputFormat.java index 89f4357dcf..354c71af34 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplitsInputFormat.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplitsInputFormat.java @@ -47,7 +47,7 @@ @Evolving public class TezGroupedSplitsInputFormat extends InputFormat implements Configurable{ - + private static final Logger LOG = LoggerFactory.getLogger(TezGroupedSplitsInputFormat.class); InputFormat wrappedInputFormat; @@ -55,18 +55,18 @@ public class TezGroupedSplitsInputFormat extends InputFormat Configuration conf; SplitSizeEstimator estimator; SplitLocationProvider locationProvider; - + public TezGroupedSplitsInputFormat() { - + } - + public void setInputFormat(InputFormat wrappedInputFormat) { this.wrappedInputFormat = wrappedInputFormat; if (LOG.isDebugEnabled()) { LOG.debug("wrappedInputFormat: " + wrappedInputFormat.getClass().getName()); } } - + public void setDesiredNumberOfSplits(int num) { Preconditions.checkArgument(num >= 0); this.desiredNumSplits = num; @@ -105,11 +105,11 @@ public RecordReader createRecordReader(InputSplit split, } return new TezGroupedSplitsRecordReader(groupedSplit, context); } - + @SuppressWarnings({ "rawtypes", "unchecked" }) void initInputFormatFromSplit(TezGroupedSplit split) throws TezException { if (wrappedInputFormat == null) { - Class clazz = (Class) + Class clazz = (Class) getClassFromName(split.wrappedInputFormatName); try { wrappedInputFormat = org.apache.hadoop.util.ReflectionUtils.newInstance(clazz, conf); @@ -118,11 +118,11 @@ void initInputFormatFromSplit(TezGroupedSplit split) throws TezException { } } } - + static Class getClassFromName(String name) throws TezException { return ReflectionUtils.getClazz(name); } - + public class TezGroupedSplitsRecordReader extends RecordReader { TezGroupedSplit groupedSplit; @@ -130,13 +130,13 @@ public class TezGroupedSplitsRecordReader extends RecordReader { int idx = 0; long progress; RecordReader curReader; - + public TezGroupedSplitsRecordReader(TezGroupedSplit split, TaskAttemptContext context) throws IOException { this.groupedSplit = split; this.context = context; } - + public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { if (this.groupedSplit != split) { @@ -147,7 +147,7 @@ public void initialize(InputSplit split, } initNextRecordReader(); } - + public boolean nextKeyValue() throws IOException, InterruptedException { while ((curReader == null) || !curReader.nextKeyValue()) { // false return finishes. true return loops back for nextKeyValue() @@ -161,18 +161,18 @@ public boolean nextKeyValue() throws IOException, InterruptedException { public K getCurrentKey() throws IOException, InterruptedException { return curReader.getCurrentKey(); } - + public V getCurrentValue() throws IOException, InterruptedException { return curReader.getCurrentValue(); } - + public void close() throws IOException { if (curReader != null) { curReader.close(); curReader = null; } } - + protected boolean initNextRecordReader() throws IOException { if (curReader != null) { curReader.close(); @@ -203,7 +203,7 @@ protected boolean initNextRecordReader() throws IOException { idx++; return true; } - + /** * return progress based on the amount of data processed so far. */ diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezMapReduceSplitsGrouper.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezMapReduceSplitsGrouper.java index add949451c..fd460dab12 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezMapReduceSplitsGrouper.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezMapReduceSplitsGrouper.java @@ -41,7 +41,7 @@ import org.slf4j.LoggerFactory; /** - * Helper that provides a grouping of input splits based + * Helper that provides a grouping of input splits based * on multiple parameters. It creates {@link TezGroupedSplit} * to wrap the each group of real InputSplits */ @@ -115,7 +115,7 @@ public class TezMapReduceSplitsGrouper extends TezSplitGrouper { * @deprecated See equivalent in {@link TezSplitGrouper} */ @Deprecated - public static final String TEZ_GROUPING_RACK_SPLIT_SIZE_REDUCTION = + public static final String TEZ_GROUPING_RACK_SPLIT_SIZE_REDUCTION = TezSplitGrouper.TEZ_GROUPING_RACK_SPLIT_SIZE_REDUCTION; /** * @deprecated See equivalent in {@link TezSplitGrouper} diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientCache.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientCache.java index f7c8e07629..caf7f31a72 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientCache.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientCache.java @@ -29,7 +29,7 @@ public class ClientCache { private final Configuration conf; private final ResourceMgrDelegate rm; - private Map cache = + private Map cache = new HashMap(); public ClientCache(Configuration conf, ResourceMgrDelegate rm) { diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientServiceDelegate.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientServiceDelegate.java index 1042f2e342..936ef2a1b6 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientServiceDelegate.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientServiceDelegate.java @@ -74,7 +74,7 @@ public String[] getTaskDiagnostics(org.apache.hadoop.mapreduce.TaskAttemptID // FIXME need support to query task diagnostics? return new String[0]; } - + public JobStatus getJobStatus(JobID oldJobID) throws IOException { // handled in YARNRunner throw new UnsupportedOperationException(); @@ -106,5 +106,5 @@ public LogParams getLogFilePath(JobID oldJobID, throws YarnException, IOException { // FIXME logs for an attempt? throw new UnsupportedOperationException(); - } + } } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ResourceMgrDelegate.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ResourceMgrDelegate.java index bec5cc644c..cc273b1ae6 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ResourceMgrDelegate.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ResourceMgrDelegate.java @@ -52,7 +52,7 @@ public class ResourceMgrDelegate { private static final Logger LOG = LoggerFactory.getLogger(ResourceMgrDelegate.class); - + private YarnConfiguration conf; private GetNewApplicationResponse application; private ApplicationId applicationId; @@ -109,7 +109,7 @@ public ClusterMetrics getClusterMetrics() throws IOException, } catch (YarnException e) { throw new IOException(e); } - ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1, + ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1, metrics.getNumNodeManagers() * 10, metrics.getNumNodeManagers() * 2, 1, metrics.getNumNodeManagers(), 0, 0); return oldMetrics; @@ -133,7 +133,7 @@ public String getFilesystemName() throws IOException, InterruptedException { public JobID getNewJobID() throws IOException, InterruptedException { try { - this.application = + this.application = client.createApplication().getNewApplicationResponse(); } catch (YarnException e) { throw new IOException(e); @@ -193,7 +193,7 @@ public QueueInfo[] getChildQueues(String parent) throws IOException, public String getStagingAreaDir() throws IOException, InterruptedException { // Path path = new Path(MRJobConstants.JOB_SUBMIT_DIR); - String user = + String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path path = MRApps.getStagingAreaDir(conf, user); LOG.debug("getStagingAreaDir: dir=" + path); @@ -206,13 +206,13 @@ public String getSystemDir() throws IOException, InterruptedException { //FileContext.getFileContext(conf).delete(sysDir, true); return sysDir.toString(); } - + public long getTaskTrackerExpiryInterval() throws IOException, InterruptedException { return 0; } - + public void setJobPriority(JobID arg0, String arg1) throws IOException, InterruptedException { return; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java index fa6e31f70a..aba66f3817 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java @@ -414,9 +414,9 @@ private Vertex createVertexForStage(Configuration stageConf, Resource taskResource = isMap ? MRHelpers.getResourceForMRMapper(stageConf) : MRHelpers.getResourceForMRReducer(stageConf); - + stageConf.set(MRJobConfig.MROUTPUT_FILE_NAME_PREFIX, "part"); - + UserPayload vertexUserPayload = TezUtils.createUserPayloadFromConf(stageConf); Vertex vertex = Vertex.create(vertexName, ProcessorDescriptor.create(processorName).setUserPayload(vertexUserPayload), @@ -458,7 +458,7 @@ private Vertex createVertexForStage(Configuration stageConf, .addTaskLocalFiles(taskLocalResources) .setLocationHint(VertexLocationHint.create(locations)) .setTaskLaunchCmdOpts(taskJavaOpts); - + if (!isMap) { vertex.setVertexManagerPlugin((ShuffleVertexManager.createConfigBuilder(stageConf).build())); } @@ -623,20 +623,20 @@ public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts) try { dagAMConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, jobSubmitDir); - + // Set Tez parameters based on MR parameters. String queueName = jobConf.get(JobContext.QUEUE_NAME, YarnConfiguration.DEFAULT_QUEUE_NAME); dagAMConf.set(TezConfiguration.TEZ_QUEUE_NAME, queueName); - + int amMemMB = jobConf.getInt(MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB); int amCores = jobConf.getInt(MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES); dagAMConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, amMemMB); dagAMConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, amCores); - dagAMConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, + dagAMConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, jobConf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS)); - + tezClient = new MRTezClient("MapReduce", dagAMConf, false, jobLocalResources, ts); tezClient.start(); dagClient = new MRDAGClient(tezClient.submitDAGApplication(appId, dag)); diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java index 0db9791199..f585f86abc 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java @@ -66,16 +66,16 @@ public class MRCombiner implements Combiner { private static Logger LOG = LoggerFactory.getLogger(MRCombiner.class); - + private final Configuration conf; private final Class keyClass; private final Class valClass; private final RawComparator comparator; private final boolean useNewApi; - + private final TezCounter combineInputRecordsCounter; private final TezCounter combineOutputRecordsCounter; - + private final MRTaskReporter reporter; private final TaskAttemptID mrTaskAttemptID; @@ -103,14 +103,14 @@ public MRCombiner(TaskContext taskContext) throws IOException { combineInputRecordsCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS); combineOutputRecordsCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_OUTPUT_RECORDS); - + boolean isMap = conf.getBoolean(MRConfig.IS_MAP_PROCESSOR,false); this.mrTaskAttemptID = new TaskAttemptID( new TaskID(String.valueOf(taskContext.getApplicationId() .getClusterTimestamp()), taskContext.getApplicationId().getId(), isMap ? TaskType.MAP : TaskType.REDUCE, taskContext.getTaskIndex()), taskContext.getTaskAttemptNumber()); - + LOG.info("Using combineKeyClass: " + keyClass + ", combineValueClass: " + valClass + ", combineComparator: " +comparator + ", useNewApi: " + useNewApi); } @@ -122,16 +122,16 @@ public void combine(TezRawKeyValueIterator rawIter, Writer writer) } else { runOldCombiner(rawIter, writer); } - + } ///////////////// Methods for old API ////////////////////// - + private void runOldCombiner(final TezRawKeyValueIterator rawIter, final Writer writer) throws IOException { Class reducerClazz = (Class) conf.getClass("mapred.combiner.class", null, Reducer.class); - + Reducer combiner = ReflectionUtils.newInstance(reducerClazz, conf); - + OutputCollector collector = new OutputCollector() { @Override public void collect(Object key, Object value) throws IOException { @@ -139,14 +139,14 @@ public void collect(Object key, Object value) throws IOException { combineOutputRecordsCounter.increment(1); } }; - + CombinerValuesIterator values = new CombinerValuesIterator(rawIter, keyClass, valClass, comparator); - + while (values.moveToNext()) { combiner.reduce(values.getKey(), values.getValues().iterator(), collector, reporter); } } - + private final class CombinerValuesIterator extends ValuesIterator { public CombinerValuesIterator(TezRawKeyValueIterator rawIter, Class keyClass, Class valClass, @@ -155,13 +155,13 @@ public CombinerValuesIterator(TezRawKeyValueIterator rawIter, null, combineInputRecordsCounter); } } - + ///////////////// End of methods for old API ////////////////////// - + ///////////////// Methods for new API ////////////////////// - + private void runNewCombiner(final TezRawKeyValueIterator rawIter, final Writer writer) throws InterruptedException, IOException { - + RecordWriter recordWriter = new RecordWriter() { @Override @@ -177,12 +177,12 @@ public void close(TaskAttemptContext context) throws IOException, // Will be closed by whoever invokes the combiner. } }; - + Class reducerClazz = (Class) conf .getClass(MRJobConfig.COMBINE_CLASS_ATTR, null, org.apache.hadoop.mapreduce.Reducer.class); org.apache.hadoop.mapreduce.Reducer reducer = ReflectionUtils.newInstance(reducerClazz, conf); - + org.apache.hadoop.mapreduce.Reducer.Context reducerContext = createReduceContext( conf, @@ -195,7 +195,7 @@ public void close(TaskAttemptContext context) throws IOException, (RawComparator)comparator, keyClass, valClass); - + reducer.run(reducerContext); recordWriter.close(reducerContext); } @@ -250,6 +250,6 @@ public void close() throws IOException { return reducerContext; } - - + + } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java index 1157012aa0..b529f2d612 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java @@ -49,7 +49,7 @@ /** * Implements the {@link OutputCommitter} and provide Map Reduce compatible - * output commit operations for Map Reduce compatible data sinks. + * output commit operations for Map Reduce compatible data sinks. */ @Public public class MROutputCommitter extends OutputCommitter { @@ -75,7 +75,7 @@ public void initialize() throws IOException { jobConf = new JobConf( TezUtils.createConfFromUserPayload(userPayload)); } - + // Read all credentials into the credentials instance stored in JobConf. jobConf.getCredentials().mergeAll(UserGroupInformation.getCurrentUser().getCredentials()); jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java index 3b2f9d3285..3ee94a3e15 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java @@ -50,10 +50,10 @@ import org.slf4j.LoggerFactory; /** - * Implements an {@link InputInitializer} that generates Map Reduce + * Implements an {@link InputInitializer} that generates Map Reduce * splits in the App Master. This may utilizes the up to date cluster - * information to create an optimal distribution of splits. This is the - * recommended {@link InputInitializer} to use when reading Map Reduce + * information to create an optimal distribution of splits. This is the + * recommended {@link InputInitializer} to use when reading Map Reduce * compatible data sources. */ @Public @@ -61,7 +61,7 @@ public class MRInputAMSplitGenerator extends InputInitializer { private boolean sendSerializedEvents; - + private static final Logger LOG = LoggerFactory.getLogger(MRInputAMSplitGenerator.class); public MRInputAMSplitGenerator( @@ -82,7 +82,7 @@ public List initialize() throws Exception { sw.reset().start(); Configuration conf = new JobConf(getContext().getVertexConfiguration()); TezUtils.addToConfFromByteString(conf, userPayloadProto.getConfigurationBytes()); - + sendSerializedEvents = conf.getBoolean( MRJobConfig.MR_TEZ_INPUT_INITIALIZER_SERIALIZE_EVENT_PAYLOAD, MRJobConfig.MR_TEZ_INPUT_INITIALIZER_SERIALIZE_EVENT_PAYLOAD_DEFAULT); @@ -128,7 +128,7 @@ public List initialize() throws Exception { List events = Lists.newArrayListWithCapacity(inputSplitInfo .getNumTasks() + 1); - + InputConfigureVertexTasksEvent configureVertexEvent = InputConfigureVertexTasksEvent.create( inputSplitInfo.getNumTasks(), VertexLocationHint.create(inputSplitInfo.getTaskLocationHints()), @@ -161,7 +161,7 @@ public List initialize() throws Exception { } } } - + return events; } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java index 3534d8ff2e..7e83b9dd7d 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java @@ -48,7 +48,7 @@ import org.slf4j.LoggerFactory; /** - * Implements an {@link InputInitializer} that distributes Map Reduce + * Implements an {@link InputInitializer} that distributes Map Reduce * splits created by the client to tasks in the {@link Vertex} * This can be used when certain reasons (e.g. security) prevent splits * from being produced in the App Master via {@link MRInputAMSplitGenerator} @@ -60,7 +60,7 @@ public class MRInputSplitDistributor extends InputInitializer { private static final Logger LOG = LoggerFactory.getLogger(MRInputSplitDistributor.class); - + private boolean sendSerializedEvents; private MRSplitsProto splitsProto; @@ -89,7 +89,7 @@ public List initialize() throws IOException { LOG.info("Emitting serialized splits: " + sendSerializedEvents); this.splitsProto = userPayloadProto.getSplits(); - + MRInputUserPayloadProto.Builder updatedPayloadBuilder = MRInputUserPayloadProto.newBuilder(userPayloadProto); updatedPayloadBuilder.clearSplits(); diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java index 8712060546..e70141ee34 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java @@ -29,11 +29,11 @@ public final class DeprecatedKeys { - - + + // This could be done via deprecation. /** - * Keys used by the DAG - mainly the AM. + * Keys used by the DAG - mainly the AM. */ private static final Map MR_PARAM_TO_DAG_PARAM_MAP = new HashMap<>(); @@ -43,8 +43,8 @@ public final class DeprecatedKeys { private static final Map MR_PARAM_TO_TEZ_RUNTIME_PARAM_MAP = new HashMap<>(); - - + + static { populateMRToTezRuntimeParamMap(); populateMRToDagParamMap(); @@ -57,7 +57,7 @@ private static void populateMRToDagParamMap() { // TODO Default value handling. MR_PARAM_TO_DAG_PARAM_MAP.put(MRJobConfig.MR_AM_TASK_LISTENER_THREAD_COUNT, TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT); - + MR_PARAM_TO_DAG_PARAM_MAP.put(MRJobConfig.MAX_TASK_FAILURES_PER_TRACKER, TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE); MR_PARAM_TO_DAG_PARAM_MAP.put(MRJobConfig.MR_AM_JOB_NODE_BLACKLISTING_ENABLE, @@ -97,10 +97,10 @@ private static void populateMRToDagParamMap() { // TODO TEZAM4 Sometime, make sure this gets loaded by default. Instead of the current initialization in MRAppMaster, TezChild. // Maybe define in an TEZConfiguration / TEZ JobConf variant. - + public static void init() { } - + private static void populateMRToTezRuntimeParamMap() { registerMRToRuntimeKeyTranslation(MRConfig.MAPRED_IFILE_READAHEAD, TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD); @@ -110,49 +110,49 @@ private static void populateMRToTezRuntimeParamMap() { registerMRToRuntimeKeyTranslation(MRJobConfig.RECORDS_BEFORE_PROGRESS, TezRuntimeConfiguration.TEZ_RUNTIME_RECORDS_BEFORE_PROGRESS); registerMRToRuntimeKeyTranslation(MRJobConfig.IO_SORT_FACTOR, TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR); - + registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_SORT_SPILL_PERCENT, TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT); - + registerMRToRuntimeKeyTranslation(MRJobConfig.IO_SORT_MB, TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB); - + registerMRToRuntimeKeyTranslation(MRJobConfig.INDEX_CACHE_MEMORY_LIMIT, TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES); - + registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_COMBINE_MIN_SPILLS, TezRuntimeConfiguration.TEZ_RUNTIME_COMBINE_MIN_SPILLS); - + registerMRToRuntimeKeyTranslation(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES, Constants.TEZ_RUNTIME_TASK_MEMORY); - + registerMRToRuntimeKeyTranslation(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION); registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_PARALLEL_COPIES, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES); - + registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_FETCH_FAILURES, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT); - + registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_NOTIFY_READERROR, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR); - + registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_CONNECT_TIMEOUT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT); - + registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_READ_TIMEOUT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT); - + registerMRToRuntimeKeyTranslation(MRConfig.SHUFFLE_SSL_ENABLED_KEY, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL); - + registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT); - + registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT); - + registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_MERGE_PERCENT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT); - + registerMRToRuntimeKeyTranslation(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS); - + registerMRToRuntimeKeyTranslation(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM); - + registerMRToRuntimeKeyTranslation(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT); - + registerMRToRuntimeKeyTranslation("map.sort.class", TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS); - + registerMRToRuntimeKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezRuntimeConfiguration.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS); - + registerMRToRuntimeKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezRuntimeConfiguration.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS); - + registerMRToRuntimeKeyTranslation(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, TezConfiguration.TEZ_CREDENTIALS_PATH); registerMRToRuntimeKeyTranslation(MRJobConfig.KEY_COMPARATOR, TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS); @@ -177,7 +177,7 @@ private static void populateMRToTezRuntimeParamMap() { registerMRToRuntimeKeyTranslation(MRJobConfig.MINIMUM_ALLOWED_TASKS, TezConfiguration.TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS); } - + private static void addDeprecatedKeys() { } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java index 8656147e2c..2286262252 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java @@ -38,14 +38,14 @@ public static JobID toMRJobId(TezDAGID dagId) { Long.toString(dagId.getApplicationId().getClusterTimestamp()), dagId.getApplicationId().getId()); } - + public static TaskID toMRTaskId(TezTaskID taskid) { return new TaskID( toMRJobId(taskid.getDAGID()), taskid.getVertexID().getId() == 0 ? TaskType.MAP : TaskType.REDUCE, taskid.getId()); } - + public static TaskID toMRTaskIdForOutput(TezTaskID taskid) { return org.apache.tez.mapreduce.hadoop.mapreduce.TaskAttemptContextImpl .createMockTaskAttemptIDFromTezTaskId(taskid, (taskid.getVertexID().getId() == 0)); @@ -57,7 +57,7 @@ public static TaskAttemptID toMRTaskAttemptId( toMRTaskId(taskAttemptId.getTaskID()), taskAttemptId.getId()); } - + // FIXME hardcoded assumption that one app is one dag public static TezDAGID fromMRJobId( org.apache.hadoop.mapreduce.JobID jobId) { @@ -81,5 +81,5 @@ public static TezTaskAttemptID fromMRTaskAttemptId( fromMRTaskId(taskAttemptId.getTaskID()), taskAttemptId.getId()); } - + } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfo.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfo.java index dfc152dde1..67c60a86ad 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfo.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfo.java @@ -31,10 +31,10 @@ /** * Provides information about input splits.

    - * + * * The get*Path methods are only applicable when generating splits to disk. The * getSplitsProto method is only applicable when generating splits to memory. - * + * */ @Private @Unstable @@ -62,18 +62,18 @@ public enum Type { * Get the splits proto */ public abstract MRSplitsProto getSplitsProto(); - + /** * Get the number of splits that were generated. Same as number of tasks that * should be run for the vertex processing these set of splits. */ public abstract int getNumTasks(); - + /** * Get the {@link Type} of the InputSplitInfo */ public abstract Type getType(); - + /** * Get {@link Credentials} which may be required to access the splits. * @return {@link Credentials} which may be required to access the splits. diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfoDisk.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfoDisk.java index 5cc0d8edc5..36d75e8e6d 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfoDisk.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfoDisk.java @@ -36,7 +36,7 @@ * the vertex and the taskLocationHints define the possible nodes on which the * tasks should be run based on the location of the splits that will be * processed by each task. - * + * * The getSplitsProto method is not supported by this implementation. */ public class InputSplitInfoDisk implements InputSplitInfo { @@ -102,7 +102,7 @@ public MRSplitsProto getSplitsProto() { throw new UnsupportedOperationException("Not supported for Type: " + getType()); } - + @Override public Credentials getCredentials() { return this.credentials; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfoMem.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfoMem.java index eda64a56aa..fbf897c891 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfoMem.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfoMem.java @@ -31,10 +31,10 @@ /** * Represents InputSplitInfo for splits generated to memory.

    - * + * * Since splits are generated in memory, the getSplitsMetaInfoFile and * getSplitsFile are not supported. - * + * */ public class InputSplitInfoMem implements InputSplitInfo { @@ -44,7 +44,7 @@ public class InputSplitInfoMem implements InputSplitInfo { private final Credentials credentials; private final Configuration conf; private final List taskLocationHints; - + private org.apache.hadoop.mapreduce.InputSplit[] newFormatSplits; private org.apache.hadoop.mapred.InputSplit[] oldFormatSplits; @@ -127,7 +127,7 @@ public Credentials getCredentials() { @Override public boolean holdsNewFormatSplits() { - return this.isNewSplit; + return this.isNewSplit; } @Override diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java index 033489c04d..92234b406b 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java @@ -27,13 +27,13 @@ public interface MRJobConfig { @Private static final String MR_TEZ_PREFIX = "mapreduce.tez."; - + // Put all of the attribute names in here so that Job and JobContext are // consistent. public static final String INPUT_FORMAT_CLASS_ATTR = "mapreduce.job.inputformat.class"; - + public static final String NEW_API_MAPPER_CONFIG = "mapred.mapper.new-api"; - + public static final String NEW_API_REDUCER_CONFIG = "mapred.reducer.new-api"; public static final String MAP_CLASS_ATTR = "mapreduce.job.map.class"; @@ -69,7 +69,7 @@ public interface MRJobConfig { public static final String JVM_NUMTASKS_TORUN = "mapreduce.job.jvm.numtasks"; public static final String SPLIT_FILE = "mapreduce.job.splitfile"; - + public static final String SPLIT_METAINFO_MAXSIZE = "mapreduce.job.split.metainfo.maxsize"; public static final long DEFAULT_SPLIT_METAINFO_MAXSIZE = 10000000L; @@ -187,13 +187,13 @@ public interface MRJobConfig { public static final String NUM_REDUCE_PROFILES = "mapreduce.task.profile.reduces"; public static final String TASK_MAP_PROFILE_PARAMS = "mapreduce.task.profile.map.params"; - + public static final String TASK_REDUCE_PROFILE_PARAMS = "mapreduce.task.profile.reduce.params"; - + public static final String TASK_TIMEOUT = "mapreduce.task.timeout"; public static final String TASK_TIMEOUT_CHECK_INTERVAL_MS = "mapreduce.task.timeout.check-interval-ms"; - + public static final String TASK_ID = "mapreduce.task.id"; public static final String TASK_OUTPUT_DIR = "mapreduce.task.output.dir"; @@ -320,11 +320,11 @@ public interface MRJobConfig { public static final String JOB_ACL_MODIFY_JOB = "mapreduce.job.acl-modify-job"; public static final String DEFAULT_JOB_ACL_MODIFY_JOB = " "; - + /* config for tracking the local file where all the credentials for the job * credentials. */ - public static final String MAPREDUCE_JOB_CREDENTIALS_BINARY = + public static final String MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary"; public static final String JOB_SUBMITHOST = @@ -357,23 +357,23 @@ public interface MRJobConfig { public static final String MR_AM_PREFIX = MR_PREFIX + "am."; /** The number of client retires to the AM - before reconnecting to the RM - * to fetch Application State. + * to fetch Application State. */ - public static final String MR_CLIENT_TO_AM_IPC_MAX_RETRIES = + public static final String MR_CLIENT_TO_AM_IPC_MAX_RETRIES = MR_PREFIX + "client-am.ipc.max-retries"; public static final int DEFAULT_MR_CLIENT_TO_AM_IPC_MAX_RETRIES = 3; - + /** * The number of client retries to the RM/HS/AM before throwing exception. */ - public static final String MR_CLIENT_MAX_RETRIES = + public static final String MR_CLIENT_MAX_RETRIES = MR_PREFIX + "client.max-retries"; public static final int DEFAULT_MR_CLIENT_MAX_RETRIES = 3; - + /** The staging directory for map reduce.*/ - public static final String MR_AM_STAGING_DIR = + public static final String MR_AM_STAGING_DIR = MR_AM_PREFIX+"staging-dir"; - public static final String DEFAULT_MR_AM_STAGING_DIR = + public static final String DEFAULT_MR_AM_STAGING_DIR = "/tmp/hadoop-yarn/staging"; /** The amount of memory the MR app master needs.*/ @@ -397,12 +397,12 @@ public interface MRJobConfig { public static final String DEFAULT_MR_AM_ADMIN_COMMAND_OPTS = ""; /** Root Logging level passed to the MR app master.*/ - public static final String MR_AM_LOG_LEVEL = + public static final String MR_AM_LOG_LEVEL = MR_AM_PREFIX+"log.level"; public static final String DEFAULT_MR_AM_LOG_LEVEL = "INFO"; /**The number of splits when reporting progress in MR*/ - public static final String MR_AM_NUM_PROGRESS_SPLITS = + public static final String MR_AM_NUM_PROGRESS_SPLITS = MR_AM_PREFIX+"num-progress-splits"; public static final int DEFAULT_MR_AM_NUM_PROGRESS_SPLITS = 12; @@ -413,7 +413,7 @@ public interface MRJobConfig { public static final String MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT = MR_AM_PREFIX+"containerlauncher.thread-count-limit"; - public static final int DEFAULT_MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT = + public static final int DEFAULT_MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT = 500; /** Number of threads to handle job client RPC requests.*/ @@ -421,15 +421,15 @@ public interface MRJobConfig { MR_AM_PREFIX + "job.client.thread-count"; public static final int DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT = 1; - /** + /** * Range of ports that the MapReduce AM can use when binding. Leave blank * if you want all possible ports. */ - public static final String MR_AM_JOB_CLIENT_PORT_RANGE = + public static final String MR_AM_JOB_CLIENT_PORT_RANGE = MR_AM_PREFIX + "job.client.port-range"; - + /** Enable blacklisting of nodes in the job.*/ - public static final String MR_AM_JOB_NODE_BLACKLISTING_ENABLE = + public static final String MR_AM_JOB_NODE_BLACKLISTING_ENABLE = MR_AM_PREFIX + "job.node-blacklisting.enable"; /** Ignore blacklisting if a certain percentage of nodes have been blacklisted */ @@ -437,21 +437,21 @@ public interface MRJobConfig { MR_AM_PREFIX + "job.node-blacklisting.ignore-threshold-node-percent"; public static final int DEFAULT_MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERCENT = 33; - + /** Enable job recovery.*/ - public static final String MR_AM_JOB_RECOVERY_ENABLE = + public static final String MR_AM_JOB_RECOVERY_ENABLE = MR_AM_PREFIX + "job.recovery.enable"; - /** + /** * Limit on the number of reducers that can be preempted to ensure that at * least one map task can run if it needs to. Percentage between 0.0 and 1.0 */ - public static final String MR_AM_JOB_REDUCE_PREEMPTION_LIMIT = + public static final String MR_AM_JOB_REDUCE_PREEMPTION_LIMIT = MR_AM_PREFIX + "job.reduce.preemption.limit"; public static final float DEFAULT_MR_AM_JOB_REDUCE_PREEMPTION_LIMIT = 0.5f; - + /** AM ACL disabled. **/ - public static final String JOB_AM_ACCESS_DISABLED = + public static final String JOB_AM_ACCESS_DISABLED = "mapreduce.job.am-access-disabled"; public static final boolean DEFAULT_JOB_AM_ACCESS_DISABLED = false; @@ -459,7 +459,7 @@ public interface MRJobConfig { * Limit reduces starting until a certain percentage of maps have finished. * Percentage between 0.0 and 1.0 */ - public static final String MR_AM_JOB_REDUCE_RAMPUP_UP_LIMIT = + public static final String MR_AM_JOB_REDUCE_RAMPUP_UP_LIMIT = MR_AM_PREFIX + "job.reduce.rampup.limit"; public static final float DEFAULT_MR_AM_JOB_REDUCE_RAMP_UP_LIMIT = 0.5f; @@ -472,7 +472,7 @@ public interface MRJobConfig { MR_AM_PREFIX + "job.task.estimator.exponential.smooth.lambda-ms"; - public static final long DEFAULT_MR_AM_TASK_ESTIMATOR_SMOOTH_LAMBDA_MS = + public static final long DEFAULT_MR_AM_TASK_ESTIMATOR_SMOOTH_LAMBDA_MS = 1000L * 60; /** true if the smoothing rate should be exponential.*/ @@ -509,9 +509,9 @@ public interface MRJobConfig { * Set to false for multi-user clusters. This is an internal config that * is set by the MR framework and read by it too. */ - public static final String MR_AM_CREATE_JH_INTERMEDIATE_BASE_DIR = + public static final String MR_AM_CREATE_JH_INTERMEDIATE_BASE_DIR = MR_AM_PREFIX + "create-intermediate-jh-base-dir"; - + public static final String MR_AM_HISTORY_MAX_UNFLUSHED_COMPLETE_EVENTS = MR_AM_PREFIX + "history.max-unflushed-events"; public static final int DEFAULT_MR_AM_HISTORY_MAX_UNFLUSHED_COMPLETE_EVENTS = @@ -531,10 +531,10 @@ public interface MRJobConfig { MR_AM_PREFIX + "history.use-batched-flush.queue-size.threshold"; public static final int DEFAULT_MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD = 50; - + public static final String MR_AM_ENV = MR_AM_PREFIX + "env"; - + public static final String MR_AM_ADMIN_USER_ENV = MR_AM_PREFIX + "admin.user.env"; @@ -591,10 +591,10 @@ public interface MRJobConfig { /** The log directory for the containers */ public static final String TASK_LOG_DIR = MR_PREFIX + "container.log.dir"; - + public static final String TASK_LOG_SIZE = MR_PREFIX + "container.log.filesize"; - - public static final String MAPREDUCE_V2_CHILD_CLASS = + + public static final String MAPREDUCE_V2_CHILD_CLASS = "org.apache.hadoop.mapred.YarnChild"; public static final String APPLICATION_ATTEMPT_ID = @@ -603,17 +603,17 @@ public interface MRJobConfig { /* * MR AM Service Authorization */ - public static final String + public static final String MR_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL = "security.job.task.protocol.acl"; - public static final String + public static final String MR_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT = "security.job.client.protocol.acl"; /** * CLASSPATH for all YARN MapReduce applications. */ - public static final String MAPREDUCE_APPLICATION_CLASSPATH = + public static final String MAPREDUCE_APPLICATION_CLASSPATH = "mapreduce.application.classpath"; /** @@ -633,7 +633,7 @@ public interface MRJobConfig { @InterfaceAudience.Private static final String HS_DELEGATION_TOKEN_REQUIRED = "mapreduce.history.server.delegationtoken.required"; - + public static final String MROUTPUT_FILE_NAME_PREFIX = MR_TEZ_PREFIX + "mroutput.file-name.prefix"; @@ -644,9 +644,9 @@ public interface MRJobConfig { public static final String MRR_INTERMEDIATE_STAGE_PREFIX = "mrr.intermediate.stage."; - + // TODO Move this over to Tez - public static final String MRR_VERTEX_PREFIX = + public static final String MRR_VERTEX_PREFIX = "mrr.vertex."; public static final String VERTEX_NAME = "mapreduce.task.vertex.name"; @@ -658,5 +658,5 @@ public interface MRJobConfig { public static final String MR_TEZ_INPUT_INITIALIZER_SERIALIZE_EVENT_PAYLOAD = MR_TEZ_PREFIX + "input.initializer.serialize.event.payload"; public static final boolean MR_TEZ_INPUT_INITIALIZER_SERIALIZE_EVENT_PAYLOAD_DEFAULT = true; - + } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java index de38766cd3..5487e670fc 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java @@ -45,11 +45,11 @@ public static Configuration[] getStageConfs(Configuration conf) { // Read split info from HDFS conf.setBoolean(MRJobConfig.MR_TEZ_SPLITS_VIA_EVENTS, false); - + // Setup Tez partitioner class conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, MRPartitioner.class.getName()); - + // Setup Tez Combiner class if required. // This would already have been set since the call is via JobClient boolean useNewApi = conf.getBoolean("mapred.mapper.new-api", false); diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java index 23ffd3ef75..91ee805296 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java @@ -45,11 +45,11 @@ public static String getPropertyNameForIntermediateStage( return MRJobConfig.MRR_INTERMEDIATE_STAGE_PREFIX + intermediateStage + "." + originalPropertyName; } - + ////////////////////////////////////////////////////////////////////////////// // Methods based on Vertex Name // ////////////////////////////////////////////////////////////////////////////// - + private static final String INITIAL_MAP_VERTEX_NAME = "initialmap"; private static final String FINAL_REDUCE_VERTEX_NAME = "finalreduce"; private static final String INTERMEDIATE_TASK_VERTEX_NAME_PREFIX = "ivertex"; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java index 0efde4ac70..36a3fd3a1b 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -34,7 +34,7 @@ public final class TezTypeConverters { private TezTypeConverters() {} // TODO Remove unused methods - + // Tez objects will be imported. Others will use the fully qualified name when // required. // All public methods named toYarn / toTez / toMapReduce @@ -44,7 +44,7 @@ public static org.apache.hadoop.mapreduce.JobID toJobID(TezDAGID id) { String.valueOf(id.getApplicationId().getClusterTimestamp()), id.getId()); } - + public static TaskAttemptId toYarn(TezTaskAttemptID taskAttemptId) { TaskAttemptID mrTaskAttemptId = IDConverter .toMRTaskAttemptId(taskAttemptId); diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java index 2f0698699f..ba05c0a937 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java @@ -28,8 +28,8 @@ @InterfaceAudience.Private @InterfaceStability.Unstable -public class JobContextImpl - extends org.apache.hadoop.mapreduce.task.JobContextImpl +public class JobContextImpl + extends org.apache.hadoop.mapreduce.task.JobContextImpl implements JobContext { private JobConf job; private Progressable progress; @@ -44,20 +44,20 @@ public JobContextImpl(JobConf conf, TezDAGID dagId, public JobContextImpl(JobConf conf, TezDAGID dagId) { this(conf, dagId, Reporter.NULL); } - + /** * Get the job Configuration - * + * * @return JobConf */ public JobConf getJobConf() { return job; } - + /** * Get the progress mechanism for reporting progress. - * - * @return progress mechanism + * + * @return progress mechanism */ public Progressable getProgressible() { return progress; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRCounters.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRCounters.java index b6b8e150ea..523cbce9cc 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRCounters.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRCounters.java @@ -30,7 +30,7 @@ public class MRCounters extends org.apache.hadoop.mapred.Counters { private final org.apache.tez.common.counters.TezCounters raw; - + public MRCounters(org.apache.tez.common.counters.TezCounters raw) { this.raw = raw; } @@ -116,7 +116,7 @@ public int size() { public String makeEscapedCompactString() { return toEscapedCompactString(this); } - + public static class MRCounterGroup extends org.apache.hadoop.mapred.Counters.Group { private final org.apache.tez.common.counters.CounterGroup group; public MRCounterGroup(org.apache.tez.common.counters.CounterGroup group) { @@ -204,10 +204,10 @@ public int hashCode() { return result; } } - + public static class MRCounter extends Counter { private final org.apache.tez.common.counters.TezCounter raw; - + public MRCounter(org.apache.tez.common.counters.TezCounter raw) { this.raw = raw; } @@ -294,24 +294,24 @@ static org.apache.tez.common.counters.TezCounter convert( org.apache.hadoop.mapreduce.Counter underlyingCounter = counter.getUnderlyingCounter(); if (underlyingCounter instanceof org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup.FrameworkCounter) { - org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup.FrameworkCounter - real = + org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup.FrameworkCounter + real = (org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup.FrameworkCounter)underlyingCounter; return new org.apache.tez.common.counters.FrameworkCounterGroup.FrameworkCounter( real.getKey(), real.getGroupName()); } else if (underlyingCounter instanceof org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup.FSCounter) { - org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup.FSCounter real = + org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup.FSCounter real = (org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup.FSCounter)underlyingCounter; return new org.apache.tez.common.counters.FileSystemCounterGroup.FSCounter( real.getScheme(), convert(real.getFileSystemCounter())); } else { return new org.apache.tez.common.counters.GenericCounter( - underlyingCounter.getName(), - underlyingCounter.getDisplayName(), + underlyingCounter.getName(), + underlyingCounter.getDisplayName(), underlyingCounter.getValue()); } } - + static org.apache.tez.common.counters.FileSystemCounter convert( org.apache.hadoop.mapreduce.FileSystemCounter c) { switch (c) { @@ -328,6 +328,6 @@ static org.apache.tez.common.counters.FileSystemCounter convert( default: throw new IllegalArgumentException("Unknow FileSystemCounter: " + c); } - + } } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRReporter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRReporter.java index 28c8369166..d5c8dab897 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRReporter.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRReporter.java @@ -29,8 +29,8 @@ public class MRReporter implements Reporter { private TezCounters tezCounters; private InputSplit split; private float progress = 0f; - - + + public MRReporter(TezCounters tezCounters) { this(tezCounters, null); } @@ -39,7 +39,7 @@ public MRReporter(TezCounters tezCounters, InputSplit split) { this.tezCounters = tezCounters; this.split = split; } - + @Override public void progress() { //TODO NEWTEZ @@ -84,7 +84,7 @@ public InputSplit getInputSplit() throws UnsupportedOperationException { public void setProgress(float progress) { this.progress = progress; } - + @Override public float getProgress() { // TODO NEWTEZ This is likely broken. Only set on task complete in Map/ReduceProcessor diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java index fc32825674..e6c628deb8 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java @@ -29,7 +29,7 @@ @InterfaceAudience.Private @InterfaceStability.Unstable public class TaskAttemptContextImpl - extends org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl + extends org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl implements TaskAttemptContext { private MRTaskReporter reporter; @@ -39,24 +39,24 @@ public TaskAttemptContextImpl(JobConf conf, TaskAttemptID taskAttemptId, super(conf, taskAttemptId); this.reporter = reporter; } - + /** * Get the taskAttemptID. - * + * * @return TaskAttemptID */ public TaskAttemptID getTaskAttemptID() { return (TaskAttemptID) super.getTaskAttemptID(); } - + public Progressable getProgressible() { return reporter; } - + public JobConf getJobConf() { return (JobConf) getConfiguration(); } - + @Override public float getProgress() { return reporter.getProgress(); diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java index 40555f4127..2569d4e61f 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java @@ -64,7 +64,7 @@ public class JobContextImpl implements JobContext { public JobContextImpl(Configuration conf, TezDAGID dagId) { this(conf, dagId, MRTaskReporter.NULL); } - + public JobContextImpl(Configuration conf, TezDAGID dagId, Progressable progress) { if (conf instanceof JobConf) { this.conf = (JobConf)conf; @@ -91,26 +91,26 @@ public Configuration getConfiguration() { public JobID getJobID() { return IDConverter.toMRJobId(dagId); } - + /** * Set the JobID. */ public void setJobID(JobID jobId) { this.dagId = IDConverter.fromMRJobId(jobId); } - + /** - * Get configured the number of reduce tasks for this job. Defaults to + * Get configured the number of reduce tasks for this job. Defaults to * 1. * @return the number of reduce tasks for this job. */ public int getNumReduceTasks() { return conf.getNumReduceTasks(); } - + /** * Get the current working directory for the default file system. - * + * * @return the directory name. */ public Path getWorkingDirectory() throws IOException { @@ -124,7 +124,7 @@ public Path getWorkingDirectory() throws IOException { public Class getOutputKeyClass() { return conf.getOutputKeyClass(); } - + /** * Get the value class for job outputs. * @return the value class for job outputs. @@ -147,7 +147,7 @@ public Class getMapOutputKeyClass() { * Get the value class for the map output data. If it is not set, use the * (final) output value class This allows the map output value class to be * different than the final output value class. - * + * * @return the map output value class. */ public Class getMapOutputValueClass() { @@ -155,9 +155,9 @@ public Class getMapOutputValueClass() { } /** - * Get the user-specified job name. This is only used to identify the + * Get the user-specified job name. This is only used to identify the * job to the user. - * + * * @return the job's name, defaulting to "". */ public String getJobName() { @@ -166,79 +166,79 @@ public String getJobName() { /** * Get the {@link InputFormat} class for the job. - * + * * @return the {@link InputFormat} class for the job. */ @SuppressWarnings("unchecked") - public Class> getInputFormatClass() + public Class> getInputFormatClass() throws ClassNotFoundException { - return (Class>) + return (Class>) conf.getClass(INPUT_FORMAT_CLASS_ATTR, TextInputFormat.class); } /** * Get the {@link Mapper} class for the job. - * + * * @return the {@link Mapper} class for the job. */ @SuppressWarnings("unchecked") - public Class> getMapperClass() + public Class> getMapperClass() throws ClassNotFoundException { - return (Class>) + return (Class>) conf.getClass(MAP_CLASS_ATTR, Mapper.class); } /** * Get the combiner class for the job. - * + * * @return the combiner class for the job. */ @SuppressWarnings("unchecked") - public Class> getCombinerClass() + public Class> getCombinerClass() throws ClassNotFoundException { - return (Class>) + return (Class>) conf.getClass(COMBINE_CLASS_ATTR, null); } /** * Get the {@link Reducer} class for the job. - * + * * @return the {@link Reducer} class for the job. */ @SuppressWarnings("unchecked") - public Class> getReducerClass() + public Class> getReducerClass() throws ClassNotFoundException { - return (Class>) + return (Class>) conf.getClass(REDUCE_CLASS_ATTR, Reducer.class); } /** * Get the {@link OutputFormat} class for the job. - * + * * @return the {@link OutputFormat} class for the job. */ @SuppressWarnings("unchecked") - public Class> getOutputFormatClass() + public Class> getOutputFormatClass() throws ClassNotFoundException { - return (Class>) + return (Class>) conf.getClass(OUTPUT_FORMAT_CLASS_ATTR, TextOutputFormat.class); } /** * Get the {@link Partitioner} class for the job. - * + * * @return the {@link Partitioner} class for the job. */ @SuppressWarnings("unchecked") - public Class> getPartitionerClass() + public Class> getPartitionerClass() throws ClassNotFoundException { - return (Class>) + return (Class>) conf.getClass(PARTITIONER_CLASS_ATTR, HashPartitioner.class); } /** * Get the {@link RawComparator} comparator used to compare keys. - * + * * @return the {@link RawComparator} comparator used to compare keys. */ public RawComparator getSortComparator() { @@ -264,44 +264,44 @@ public RawComparator getCombinerKeyGroupingComparator() { return getSortComparator(); } - /** - * Get the user defined {@link RawComparator} comparator for + /** + * Get the user defined {@link RawComparator} comparator for * grouping keys of inputs to the reduce. - * + * * @return comparator set by the user for grouping values. - * @see Job#setGroupingComparatorClass(Class) for details. + * @see Job#setGroupingComparatorClass(Class) for details. */ public RawComparator getGroupingComparator() { return conf.getOutputValueGroupingComparator(); } - + /** - * Get whether job-setup and job-cleanup is needed for the job - * - * @return boolean + * Get whether job-setup and job-cleanup is needed for the job + * + * @return boolean */ public boolean getJobSetupCleanupNeeded() { return conf.getBoolean(MRJobConfig.SETUP_CLEANUP_NEEDED, true); } - + /** - * Get whether task-cleanup is needed for the job - * - * @return boolean + * Get whether task-cleanup is needed for the job + * + * @return boolean */ public boolean getTaskCleanupNeeded() { return conf.getBoolean(MRJobConfig.TASK_CLEANUP_NEEDED, true); } /** - * This method checks to see if symlinks are to be create for the - * localized cache files in the current working directory + * This method checks to see if symlinks are to be create for the + * localized cache files in the current working directory * @return true if symlinks are to be created- else return false */ public boolean getSymlink() { return DistributedCache.getSymlink(conf); } - + /** * Get the archive entries in classpath as an array of Path */ @@ -357,7 +357,7 @@ public Path[] getFileClassPaths() { /** * Parse a list of longs into strings. - * + * * @param timestamps * the list of longs to parse * @return a list of string that were parsed. same length as timestamps. @@ -376,7 +376,7 @@ private static String[] toTimestampStrs(long[] timestamps) { /** * Get the timestamps of the archives. Used by internal * DistributedCache and MapReduce code. - * @return a string array of timestamps + * @return a string array of timestamps * @throws IOException */ public String[] getArchiveTimestamps() { @@ -386,29 +386,29 @@ public String[] getArchiveTimestamps() { /** * Get the timestamps of the files. Used by internal * DistributedCache and MapReduce code. - * @return a string array of timestamps + * @return a string array of timestamps * @throws IOException */ public String[] getFileTimestamps() { return toTimestampStrs(DistributedCache.getFileTimestamps(conf)); } - /** + /** * Get the configured number of maximum attempts that will be made to run a * map task, as specified by the mapred.map.max.attempts * property. If this property is not already set, the default is 4 attempts. - * + * * @return the max number of attempts per map task. */ public int getMaxMapAttempts() { return conf.getMaxMapAttempts(); } - /** + /** * Get the configured number of maximum attempts that will be made to run a * reduce task, as specified by the mapred.reduce.max.attempts * property. If this property is not already set, the default is 4 attempts. - * + * * @return the max number of attempts per reduce task. */ public int getMaxReduceAttempts() { @@ -428,7 +428,7 @@ public boolean getProfileEnabled() { * * The default value for this property is * "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s" - * + * * @return the parameters to pass to the task child to configure profiling */ public String getProfileParams() { @@ -446,7 +446,7 @@ public IntegerRanges getProfileTaskRange(boolean isMap) { /** * Get the reported username for this job. - * + * * @return the username */ public String getUser() { @@ -466,5 +466,5 @@ public JobConf getJobConf() { public Progressable getProgressible() { return progress; } - + } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/MapContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/MapContextImpl.java index 9fde501fd7..9c4271eb8c 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/MapContextImpl.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/MapContextImpl.java @@ -42,8 +42,8 @@ */ @InterfaceAudience.Private @InterfaceStability.Unstable -public class MapContextImpl - extends TaskInputOutputContextImpl +public class MapContextImpl + extends TaskInputOutputContextImpl implements MapContext { private RecordReader reader; private InputSplit split; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskAttemptContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskAttemptContextImpl.java index 941c8732bd..1e79bd6913 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskAttemptContextImpl.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskAttemptContextImpl.java @@ -41,7 +41,7 @@ public class TaskAttemptContextImpl private final TezCounters tezCounters; private final Reporter reporter; - + public static org.apache.hadoop.mapred.TaskAttemptID createMockTaskAttemptID( long clusterId, int vertexIndex, int appId, int taskIndex, int taskAttemptNumber, boolean isMap) { return new org.apache.hadoop.mapred.TaskAttemptID( @@ -50,8 +50,8 @@ public static org.apache.hadoop.mapred.TaskAttemptID createMockTaskAttemptID( isMap ? TaskType.MAP : TaskType.REDUCE, taskIndex), taskAttemptNumber); } - - public static org.apache.hadoop.mapred.TaskAttemptID + + public static org.apache.hadoop.mapred.TaskAttemptID createMockTaskAttemptIDFromTezTaskAttemptId(TezTaskAttemptID tezTaId, boolean isMap) { TezVertexID vId = tezTaId.getVertexID(); ApplicationId appId = vId.getApplicationId(); @@ -61,8 +61,8 @@ public static org.apache.hadoop.mapred.TaskAttemptID createMockTaskAttemptID( isMap ? TaskType.MAP : TaskType.REDUCE, tezTaId.getTaskID().getId()), tezTaId.getId()); } - - public static org.apache.hadoop.mapred.TaskID + + public static org.apache.hadoop.mapred.TaskID createMockTaskAttemptIDFromTezTaskId(TezTaskID tezTaId, boolean isMap) { TezVertexID vId = tezTaId.getVertexID(); ApplicationId appId = vId.getApplicationId(); @@ -81,12 +81,12 @@ public TaskAttemptContextImpl(Configuration conf, TezCounters tezCounters, long } //FIXME we need to use DAG Id but we are using App Id - public TaskAttemptContextImpl(Configuration conf, TaskAttemptID attemptId, + public TaskAttemptContextImpl(Configuration conf, TaskAttemptID attemptId, TezCounters tezCounters, boolean isMap, Reporter reporter) { // TODO NEWTEZ Can the jt Identifier string be taskContext.getUniqueId ? this(conf, attemptId, tezCounters, reporter); } - + public TaskAttemptContextImpl(Configuration conf, TaskAttemptID taId, TezCounters tezCounters, Reporter reporter) { super(conf, taId); this.tezCounters = tezCounters; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskInputOutputContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskInputOutputContextImpl.java index cad73a809f..0f95a39494 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskInputOutputContextImpl.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskInputOutputContextImpl.java @@ -42,8 +42,8 @@ */ @InterfaceAudience.Private @InterfaceStability.Unstable -public abstract class TaskInputOutputContextImpl - extends TaskAttemptContextImpl +public abstract class TaskInputOutputContextImpl + extends TaskAttemptContextImpl implements TaskInputOutputContext { private RecordWriter output; private OutputCommitter committer; @@ -61,16 +61,16 @@ public TaskInputOutputContextImpl(Configuration conf, TaskAttemptID taskid, * Advance to the next key, value pair, returning null if at end. * @return the key object that was read into, or null if no more */ - public abstract + public abstract boolean nextKeyValue() throws IOException, InterruptedException; - + /** * Get the current key. * @return the current key object or null if there isn't one * @throws IOException * @throws InterruptedException */ - public abstract + public abstract KEYIN getCurrentKey() throws IOException, InterruptedException; /** @@ -79,7 +79,7 @@ public TaskInputOutputContextImpl(Configuration conf, TaskAttemptID taskid, * @throws IOException * @throws InterruptedException */ - public abstract VALUEIN getCurrentValue() throws IOException, + public abstract VALUEIN getCurrentValue() throws IOException, InterruptedException; /** diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java index c722789c60..ccc4483b0b 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java @@ -77,9 +77,9 @@ * {@link MRInput} is an {@link Input} which provides key/values pairs * for the consumer. * - * It is compatible with all standard Apache Hadoop MapReduce + * It is compatible with all standard Apache Hadoop MapReduce * {@link InputFormat} implementations. - * + * * This class is not meant to be extended by external projects. */ @Public @@ -154,16 +154,16 @@ public static class MRInputConfigBuilder { initializeInputPath(); } } - + MRInputConfigBuilder setInputClassName(String className) { this.inputClassName = className; return this; } private MRInputConfigBuilder setInputPaths(String inputPaths) { - if (!(org.apache.hadoop.mapred.FileInputFormat.class.isAssignableFrom(inputFormat) || + if (!(org.apache.hadoop.mapred.FileInputFormat.class.isAssignableFrom(inputFormat) || FileInputFormat.class.isAssignableFrom(inputFormat))) { - throw new TezUncheckedException("When setting inputPaths the inputFormat must be " + + throw new TezUncheckedException("When setting inputPaths the inputFormat must be " + "assignable from either org.apache.hadoop.mapred.FileInputFormat or " + "org.apache.hadoop.mapreduce.lib.input.FileInputFormat. " + "Otherwise use the non-path configBuilder." + @@ -272,7 +272,7 @@ public DataSourceDescriptor build() { throw new TezUncheckedException(e); } } - + private DataSourceDescriptor createDistributorDataSource() throws IOException { InputSplitInfo inputSplitInfo; setupBasicConf(conf); @@ -330,7 +330,7 @@ private DataSourceDescriptor createCustomDataSource() throws IOException { private DataSourceDescriptor createGeneratorDataSource() throws IOException { setupBasicConf(conf); MRHelpers.translateMRConfToTez(conf); - + Collection uris = maybeGetURIsForCredentials(); UserPayload payload = MRInputHelpersInternal.createMRInputPayload( @@ -404,7 +404,7 @@ public static MRInputConfigBuilder createConfigBuilder(Configuration conf, } /** - * Create an {@link org.apache.tez.mapreduce.input.MRInput.MRInputConfigBuilder} + * Create an {@link org.apache.tez.mapreduce.input.MRInput.MRInputConfigBuilder} * for {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat} * or {@link org.apache.hadoop.mapred.FileInputFormat} format based InputFormats. *

    @@ -433,7 +433,7 @@ public static MRInputConfigBuilder createConfigBuilder(Configuration conf, } private static final Logger LOG = LoggerFactory.getLogger(MRInput.class); - + private final ReentrantLock rrLock = new ReentrantLock(); private final Condition rrInited = rrLock.newCondition(); @@ -444,7 +444,7 @@ public static MRInputConfigBuilder createConfigBuilder(Configuration conf, protected TaskSplitIndex splitMetaInfo = new TaskSplitIndex(); // Potential counters - #splits, #totalSize, #actualyBytesRead - + @Private volatile boolean splitInfoViaEvents; @@ -476,7 +476,7 @@ void initializeInternal() throws IOException { // Primarily for visibility rrLock.lock(); try { - + if (splitInfoViaEvents) { if (useNewApi) { mrReader = new MRReaderMapReduce(jobConf, getContext().getCounters(), inputRecordCounter, @@ -484,7 +484,7 @@ void initializeInternal() throws IOException { .getTaskVertexIndex(), getContext().getApplicationId().getId(), getContext() .getTaskIndex(), getContext().getTaskAttemptNumber(), getContext()); } else { - mrReader = new MRReaderMapred(jobConf, getContext().getCounters(), inputRecordCounter, + mrReader = new MRReaderMapred(jobConf, getContext().getCounters(), inputRecordCounter, getContext()); } } else { @@ -527,7 +527,7 @@ inputRecordCounter, getContext().getApplicationId().getClusterTimestamp(), } /** - * Returns a {@link KeyValueReader} that can be used to read + * Returns a {@link KeyValueReader} that can be used to read * Map Reduce compatible key value data. An exception will be thrown if next() * is invoked after false, either from the framework or from the underlying InputFormat */ @@ -605,7 +605,7 @@ public List close() throws IOException { * {@link MRInput} sets some additional parameters like split location when using * the new API. This methods returns the list of additional updates, and * should be used by Processors using the old MapReduce API with {@link MRInput}. - * + * * @return the additional fields set by {@link MRInput} */ public Configuration getConfigUpdates() { @@ -638,7 +638,7 @@ void processSplitEvent(InputDataInformationEvent event) rrLock.unlock(); } } - + void checkAndAwaitRecordReaderInitialization() throws IOException { assert rrLock.getHoldCount() == 1; rrLock.lock(); @@ -665,7 +665,7 @@ void initFromEvent(InputDataInformationEvent initEvent) rrLock.unlock(); } } - + private void initFromEventInternal(InputDataInformationEvent initEvent) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug(getContext().getInputOutputVertexNames() + " initializing RecordReader from event"); diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInputLegacy.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInputLegacy.java index 1e976c5401..bd396ce6a2 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInputLegacy.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInputLegacy.java @@ -39,13 +39,13 @@ public class MRInputLegacy extends MRInput { private static final Logger LOG = LoggerFactory.getLogger(MRInputLegacy.class); - + private InputDataInformationEvent initEvent; private volatile boolean inited = false; private ReentrantLock eventLock = new ReentrantLock(); private Condition eventCondition = eventLock.newCondition(); - + /** * Create an {@link org.apache.tez.mapreduce.input.MRInput.MRInputConfigBuilder} * @param conf Configuration for the {@link MRInputLegacy} @@ -68,7 +68,7 @@ public static MRInputConfigBuilder createConfigBuilder(Configuration conf, Class return MRInput.createConfigBuilder(conf, inputFormat, inputPaths).setInputClassName( MRInputLegacy.class.getName()); } - + public MRInputLegacy(InputContext inputContext, int numPhysicalInputs) { super(inputContext, numPhysicalInputs); } @@ -77,11 +77,11 @@ public MRInputLegacy(InputContext inputContext, int numPhysicalInputs) { protected void initializeInternal() throws IOException { LOG.info(getContext().getInputOutputVertexNames() + " MRInputLegacy deferring initialization"); } - + @Private public org.apache.hadoop.mapreduce.InputSplit getNewInputSplit() { return (org.apache.hadoop.mapreduce.InputSplit) mrReader.getSplit(); - } + } @SuppressWarnings("rawtypes") @Unstable @@ -108,13 +108,13 @@ public boolean isUsingNewApi() { public RecordReader getOldRecordReader() { return (RecordReader) mrReader.getRecordReader(); } - + @LimitedPrivate("hive") public void init() throws IOException { super.initializeInternal(); checkAndAwaitRecordReaderInitialization(); } - + @Override void processSplitEvent(InputDataInformationEvent event) { eventLock.lock(); diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java index b831884010..64d97bf1f1 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java @@ -98,10 +98,10 @@ public static MultiMRInputConfigBuilder createConfigBuilder(Configuration conf, @Nullable Class inputFormat) { MultiMRInputConfigBuilder configBuilder = new MultiMRInputConfigBuilder(conf, inputFormat); configBuilder.setInputClassName(MultiMRInput.class.getName()).groupSplits(false); - + return configBuilder; } - + public static class MultiMRInputConfigBuilder extends MRInput.MRInputConfigBuilder { private MultiMRInputConfigBuilder(Configuration conf, Class inputFormat) { super(conf, inputFormat); diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRInputUtils.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRInputUtils.java index 0abc854736..c026762588 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRInputUtils.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRInputUtils.java @@ -61,7 +61,7 @@ public static org.apache.hadoop.mapreduce.InputSplit getNewSplitDetailsFromEvent return MRInputHelpers.createNewFormatSplitFromUserPayload( splitProto, serializationFactory); } - + @SuppressWarnings("unchecked") public static org.apache.hadoop.mapreduce.InputSplit getNewSplitDetailsFromDisk( TaskSplitIndex splitMetaInfo, JobConf jobConf, TezCounter splitBytesCounter) @@ -125,7 +125,7 @@ public static InputSplit getOldSplitDetailsFromDisk(TaskSplitIndex splitMetaInfo inFile.close(); return split; } - + @Private public static InputSplit getOldSplitDetailsFromEvent(MRSplitProto splitProto, Configuration conf) throws IOException { diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReader.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReader.java index aa35fec70c..35999b7798 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReader.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReader.java @@ -26,24 +26,24 @@ @Private public abstract class MRReader extends KeyValueReader { - + private final InputContext context; - + public abstract void setSplit(Object split) throws IOException; public abstract boolean isSetup(); public abstract float getProgress() throws IOException, InterruptedException; public abstract void close() throws IOException; public abstract Object getSplit(); public abstract Object getRecordReader(); - + public MRReader(InputContext context) { this.context = context; } - + protected final void notifyProgress() { context.notifyProgress(); } - + protected final void notifyDone() { context.notifyProgress(); } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReaderMapred.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReaderMapred.java index e4ca0d2c48..1759ef222c 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReaderMapred.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReaderMapred.java @@ -58,7 +58,7 @@ public class MRReaderMapred extends MRReader { private boolean setupComplete = false; - public MRReaderMapred(JobConf jobConf, TezCounters tezCounters, TezCounter inputRecordCounter, + public MRReaderMapred(JobConf jobConf, TezCounters tezCounters, TezCounter inputRecordCounter, InputContext context) throws IOException { this(jobConf, null, tezCounters, inputRecordCounter, context); @@ -143,7 +143,7 @@ public Object getCurrentValue() throws IOException { * {@link MRInput} sets some additional parameters like split location when using the new API. * This methods returns the list of additional updates, and should be used by Processors using the * old MapReduce API with {@link MRInput}. - * + * * @return the additional fields set by {@link MRInput} */ public Configuration getConfigUpdates() { diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java index 08a89c7c80..27a98b2b2d 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java @@ -203,7 +203,7 @@ private void initializeOutputPath() { outputPath = conf.get(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.OUTDIR); } } - + /** * Create the {@link DataSinkDescriptor} * @return {@link DataSinkDescriptor} @@ -246,7 +246,7 @@ public DataSinkDescriptor build() { } return ds; } - + /** * Get the credentials for the output from its {@link FileSystem}s * Use the method to turn this off when not using a {@link FileSystem} @@ -258,7 +258,7 @@ public MROutputConfigBuilder getCredentialsForSinkFileSystem(boolean value) { getCredentialsForSinkFilesystem = value; return this; } - + /** * Disable commit operations for the output (default: true) * If the value is set to false then no {@link org.apache.tez.runtime.api.OutputCommitter} will @@ -358,7 +358,7 @@ public static MROutputConfigBuilder createConfigBuilder(Configuration conf, private final NumberFormat taskNumberFormat = NumberFormat.getInstance(); private final NumberFormat nonTaskNumberFormat = NumberFormat.getInstance(); - + protected JobConf jobConf; boolean useNewApi; protected AtomicBoolean flushed = new AtomicBoolean(false); @@ -427,7 +427,7 @@ protected List initializeBase() throws IOException, InterruptedException jobConf.setInt(JobContext.TASK_PARTITION, taskAttemptId.getTaskID().getId()); jobConf.set(JobContext.ID, taskAttemptId.getJobID().toString()); - + String outputFormatClassName; outputRecordCounter = getContext().getCounters().findCounter( @@ -533,8 +533,8 @@ private TaskAttemptContext createTaskAttemptContext(TaskAttemptID attemptId) { protected String getOutputFileNamePrefix() { String prefix = jobConf.get(MRJobConfig.MROUTPUT_FILE_NAME_PREFIX); if (prefix == null) { - prefix = "part-v" + - nonTaskNumberFormat.format(getContext().getTaskVertexIndex()) + + prefix = "part-v" + + nonTaskNumberFormat.format(getContext().getTaskVertexIndex()) + "-o" + nonTaskNumberFormat.format(getContext().getOutputIndex()); } return prefix; @@ -587,9 +587,9 @@ public synchronized List close() throws IOException { return null; } - + /** - * Call this in the processor before finishing to ensure outputs that + * Call this in the processor before finishing to ensure outputs that * outputs have been flushed. Must be called before commit. * @throws IOException */ diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/partition/MRPartitioner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/partition/MRPartitioner.java index 21e7687a39..0b52d798ed 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/partition/MRPartitioner.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/partition/MRPartitioner.java @@ -32,7 +32,7 @@ /** * Provides an implementation of {@link Partitioner} that is compatible - * with Map Reduce partitioners. + * with Map Reduce partitioners. */ @SuppressWarnings({ "rawtypes", "unchecked" }) @Public diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java index 10b64727b1..c4935dde14 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java @@ -100,7 +100,7 @@ public abstract class MRTask extends AbstractLogicalIOProcessor { protected TaskAttemptID taskAttemptId; protected Progress progress = new Progress(); protected SecretKey jobTokenSecret; - + LogicalInput input; LogicalOutput output; @@ -536,13 +536,13 @@ public void localizeConfiguration(JobConf jobConf) jobConf.setInt(JobContext.TASK_PARTITION, taskAttemptId.getTaskID().getId()); jobConf.set(JobContext.ID, taskAttemptId.getJobID().toString()); - + jobConf.setBoolean(MRJobConfig.TASK_ISMAP, isMap); - + Path outputPath = FileOutputFormat.getOutputPath(jobConf); if (outputPath != null) { if ((committer instanceof FileOutputCommitter)) { - FileOutputFormat.setWorkOutputPath(jobConf, + FileOutputFormat.setWorkOutputPath(jobConf, ((FileOutputCommitter)committer).getTaskAttemptPath(taskAttemptContext)); } else { FileOutputFormat.setWorkOutputPath(jobConf, outputPath); diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTaskReporter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTaskReporter.java index e3fdc27a93..ba34a8375d 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTaskReporter.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTaskReporter.java @@ -55,7 +55,7 @@ public MRTaskReporter(OutputContext context) { this.reporter = new MRReporter(context.getCounters()); this.isProcessorContext = false; } - + public MRTaskReporter(InputContext context) { this.context= context; this.reporter = new MRReporter(context.getCounters()); diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/SimpleMRProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/SimpleMRProcessor.java index 71247db846..565ea6a817 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/SimpleMRProcessor.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/SimpleMRProcessor.java @@ -35,8 +35,8 @@ /** * A {@link SimpleProcessor} that provides Map Reduce specific post - * processing by calling commit (if needed) on all {@link MROutput}s - * connected to this {@link Processor}. + * processing by calling commit (if needed) on all {@link MROutput}s + * connected to this {@link Processor}. */ @Public @Evolving diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java index 0dd218f907..d9ad712ea4 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java @@ -140,7 +140,7 @@ public void run(Map _inputs, done(); } - + /** * Update the job with details about the file split @@ -156,11 +156,11 @@ private void updateJobWithSplit(final JobConf job, InputSplit inputSplit) { } LOG.info("Processing mapred split: " + inputSplit); } - + private void updateJobWithSplit( final JobConf job, org.apache.hadoop.mapreduce.InputSplit inputSplit) { if (inputSplit instanceof org.apache.hadoop.mapreduce.lib.input.FileSplit) { - org.apache.hadoop.mapreduce.lib.input.FileSplit fileSplit = + org.apache.hadoop.mapreduce.lib.input.FileSplit fileSplit = (org.apache.hadoop.mapreduce.lib.input.FileSplit) inputSplit; job.set(JobContext.MAP_INPUT_FILE, fileSplit.getPath().toString()); job.setLong(JobContext.MAP_INPUT_START, fileSplit.getStart()); @@ -180,9 +180,9 @@ void runOldMapper( // Done only for MRInput. // TODO use new method in MRInput to get required info //input.initialize(job, master); - + InputSplit inputSplit = input.getOldInputSplit(); - + updateJobWithSplit(job, inputSplit); RecordReader in = new OldRecordReader(input); @@ -193,7 +193,7 @@ void runOldMapper( (MapRunnable)ReflectionUtils.newInstance(job.getMapRunnerClass(), job); runner.run(in, collector, (Reporter)reporter); - + // Set progress to 1.0f if there was no exception, reporter.setProgress(1.0f); // start the sort phase only if there are reducers @@ -231,7 +231,7 @@ private void runNewMapper(final JobConf job, new NewOutputCollector(out); org.apache.hadoop.mapreduce.InputSplit split = in.getNewInputSplit(); - + updateJobWithSplit(job, split); org.apache.hadoop.mapreduce.MapContext @@ -249,7 +249,7 @@ private void runNewMapper(final JobConf job, mapper.run(mapperContext); // Set progress to 1.0f if there was no exception, reporter.setProgress(1.0f); - + this.statusUpdate(); input.close(); output.close(mapperContext); diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java index 363f372855..9179bec8dc 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java @@ -203,7 +203,7 @@ public void collect(Object key, Object value) // Set progress to 1.0f if there was no exception, reporter.setProgress(1.0f); - + //Clean up: repeated in catch block below reducer.close(); //End of clean up. diff --git a/tez-mapreduce/src/test/java/org/apache/hadoop/mapred/split/TestGroupedSplits.java b/tez-mapreduce/src/test/java/org/apache/hadoop/mapred/split/TestGroupedSplits.java index b3cd162757..6a231e7244 100644 --- a/tez-mapreduce/src/test/java/org/apache/hadoop/mapred/split/TestGroupedSplits.java +++ b/tez-mapreduce/src/test/java/org/apache/hadoop/mapred/split/TestGroupedSplits.java @@ -110,7 +110,7 @@ public void testFormat() throws Exception { // create a combined split for the files TextInputFormat wrappedFormat = new TextInputFormat(); wrappedFormat.configure(job); - TezGroupedSplitsInputFormat format = + TezGroupedSplitsInputFormat format = new TezGroupedSplitsInputFormat(); format.setConf(job); format.setDesiredNumberOfSplits(1); @@ -255,11 +255,11 @@ public void testGzip() throws IOException { FileInputFormat.setInputPaths(job, workDir); TextInputFormat wrappedFormat = new TextInputFormat(); wrappedFormat.configure(job); - TezGroupedSplitsInputFormat format = + TezGroupedSplitsInputFormat format = new TezGroupedSplitsInputFormat(); format.setConf(job); format.setInputFormat(wrappedFormat); - + // TextInputFormat will produce 3 splits for (int j=1; j<=3; ++j) { format.setDesiredNumberOfSplits(j); @@ -270,12 +270,12 @@ public void testGzip() throws IOException { assertEquals("compressed splits == " + j, j, splits.length); } List results = new ArrayList(); - for (int i=0; i read = readSplit(format, splits[i], job); results.addAll(read); } assertEquals("splits length", 11, results.size()); - + final String[] firstList = {"the quick", "brown", "fox jumped", "over", " the lazy", " dog"}; final String[] secondList = {"is", "gzip"}; @@ -303,18 +303,18 @@ private static int testResults(List results, String[] first, int start) { assertEquals("splits["+i+"]", first[i], results.get(start+i).toString()); } return first.length+start; - } - + } + @SuppressWarnings({ "rawtypes", "unchecked" }) @Test(timeout=10000) public void testGroupedSplitSize() throws IOException { JobConf job = new JobConf(defaultConf); InputFormat mockWrappedFormat = mock(InputFormat.class); - TezGroupedSplitsInputFormat format = + TezGroupedSplitsInputFormat format = new TezGroupedSplitsInputFormat(); format.setConf(job); format.setInputFormat(mockWrappedFormat); - + job = (JobConf) TezSplitGrouper.newConfigBuilder(job) .setGroupingSplitSize(50*1000*1000l, 500*1000*1000l) .build(); @@ -327,35 +327,35 @@ public void testGroupedSplitSize() throws IOException { mockSplits[i] = mockSplit1; } when(mockWrappedFormat.getSplits(any(), anyInt())).thenReturn(mockSplits); - - // desired splits not set. We end up choosing min/max split size based on + + // desired splits not set. We end up choosing min/max split size based on // total data and num original splits. In this case, min size will be hit InputSplit[] splits = format.getSplits(job, 0); assertEquals(25, splits.length); - + // split too big. override with max format.setDesiredNumberOfSplits(1); splits = format.getSplits(job, 0); assertEquals(4, splits.length); - + // splits too small. override with min format.setDesiredNumberOfSplits(1000); splits = format.getSplits(job, 0); assertEquals(25, splits.length); - + } - + class TestInputSplit implements InputSplit { long length; String[] locations; int position; - + public TestInputSplit(long length, String[] locations, int position) { this.length = length; this.locations = locations; this.position = position; } - + @Override public void write(DataOutput out) throws IOException { } @@ -373,12 +373,12 @@ public long getLength() throws IOException { public String[] getLocations() throws IOException { return locations; } - + public int getPosition() { return position; } } - + @Test (timeout=5000) public void testMaintainSplitOrdering() throws IOException { int numLocations = 3; @@ -393,14 +393,14 @@ public void testMaintainSplitOrdering() throws IOException { origSplits[pos] = new TestInputSplit(splitLength, splitLoc, pos); } } - + TezMapredSplitsGrouper grouper = new TezMapredSplitsGrouper(); JobConf conf = new JobConf(defaultConf); conf = (JobConf) TezSplitGrouper.newConfigBuilder(conf) .setGroupingSplitSize(splitLength*3, splitLength*3) .setGroupingRackSplitSizeReduction(1) .build(); - + // based on the above settings the 3 nodes will each group 3 splits. // the remainig 3 splits (1 from each node) will be grouped at rack level (default-rack) // all of them will maintain ordering @@ -457,7 +457,7 @@ public void testRepeatableSplits() throws IOException { .setGroupingSplitSize(splitLength*3, splitLength*3) .setGroupingRackSplitSizeReduction(1) .build(); - + // based on the above settings the 3 nodes will each group 3 splits. // the remainig 3 splits (1 from each node) will be grouped at rack level (default-rack) // all of them will maintain ordering @@ -536,17 +536,17 @@ public void testS3Scenario() throws IOException { //splits should be 1 assertEquals(1, groupedSplits.length); } - + @SuppressWarnings({ "rawtypes", "unchecked" }) @Test(timeout=10000) public void testGroupedSplitWithDuplicates() throws IOException { JobConf job = new JobConf(defaultConf); InputFormat mockWrappedFormat = mock(InputFormat.class); - TezGroupedSplitsInputFormat format = + TezGroupedSplitsInputFormat format = new TezGroupedSplitsInputFormat(); format.setConf(job); format.setInputFormat(mockWrappedFormat); - + // put multiple splits with multiple copies in the same location String[] locations = {"common", "common", "common"}; int numSplits = 3; @@ -558,7 +558,7 @@ public void testGroupedSplitWithDuplicates() throws IOException { mockSplits[i] = mockSplit; } when(mockWrappedFormat.getSplits(any(), anyInt())).thenReturn(mockSplits); - + format.setDesiredNumberOfSplits(1); InputSplit[] splits = format.getSplits(job, 1); assertEquals(1, splits.length); @@ -568,17 +568,17 @@ public void testGroupedSplitWithDuplicates() throws IOException { Set splitSet = Sets.newHashSet(split.wrappedSplits); assertEquals(numSplits, splitSet.size()); } - + @SuppressWarnings({ "rawtypes", "unchecked" }) @Test(timeout=10000) public void testGroupedSplitWithBadLocations() throws IOException { JobConf job = new JobConf(defaultConf); InputFormat mockWrappedFormat = mock(InputFormat.class); - TezGroupedSplitsInputFormat format = + TezGroupedSplitsInputFormat format = new TezGroupedSplitsInputFormat(); format.setConf(job); format.setInputFormat(mockWrappedFormat); - + // put multiple splits with multiple copies in the same location int numSplits = 3; InputSplit[] mockSplits = new InputSplit[numSplits]; @@ -596,7 +596,7 @@ public void testGroupedSplitWithBadLocations() throws IOException { mockSplits[2] = mockSplit3; when(mockWrappedFormat.getSplits(any(), anyInt())).thenReturn(mockSplits); - + format.setDesiredNumberOfSplits(1); InputSplit[] splits = format.getSplits(job, 1); assertEquals(1, splits.length); @@ -677,7 +677,7 @@ public void testGroupedSplitWithEstimator() throws IOException { .build(); InputFormat mockWrappedFormat = mock(InputFormat.class); - TezGroupedSplitsInputFormat format = + TezGroupedSplitsInputFormat format = new TezGroupedSplitsInputFormat(); format.setConf(job); format.setInputFormat(mockWrappedFormat); diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java index 0ba5b2cb01..db2aaf447b 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java @@ -46,7 +46,7 @@ public static TezTaskAttemptID getMockTaskAttemptId( taskId) , taskAttemptId); } - + public static TezTaskID getMockTaskId(int jobId, int vertexId, int taskId) { return TezTaskID.getInstance( diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestConfigTranslationMRToTez.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestConfigTranslationMRToTez.java index 3476b2af60..38459c8da2 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestConfigTranslationMRToTez.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestConfigTranslationMRToTez.java @@ -31,17 +31,17 @@ public class TestConfigTranslationMRToTez { @Test(timeout = 5000) - // Tests derived keys - i.e. the actual key is not set, but the value is + // Tests derived keys - i.e. the actual key is not set, but the value is // derived from a fallback key. public void testComplexKeys() { JobConf confVertex1 = new JobConf(); - + confVertex1.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, IntWritable.class.getName()); - + confVertex1.unset(MRJobConfig.KEY_COMPARATOR); confVertex1.unset(MRJobConfig.GROUP_COMPARATOR_CLASS); - + MRHelpers.translateMRConfToTez(confVertex1); assertEquals(IntWritable.Comparator.class.getName(), ConfigUtils diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java index 774d543985..df65df8b8f 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java @@ -142,7 +142,7 @@ private void testCommonEnvSettingsForMRTasks(Map env) { Assert.assertTrue(env.get(Environment.PATH.name()).contains(";%TEZ_ADMIN_ENV%\\bin")); } -// TEZ-273 will reinstate this or similar. +// TEZ-273 will reinstate this or similar. // for (String val : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH) { // Assert.assertTrue(env.get(Environment.CLASSPATH.name()).contains(val)); // } diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutput.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutput.java index 54ba37e4dd..94dbaf81e2 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutput.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutput.java @@ -296,7 +296,7 @@ private OutputContext createMockOutputContext(UserPayload payload, Configuration when(outputContext.getContainerConfiguration()).thenReturn(baseConf); return outputContext; } - + public static LogicalIOProcessorRuntimeTask createLogicalTask( Configuration conf, TezUmbilical umbilical, String dagName, @@ -306,7 +306,7 @@ public static LogicalIOProcessorRuntimeTask createLogicalTask( List outputSpecs = Lists.newLinkedList(); outputSpecs.add(new OutputSpec("Null", MROutput.createConfigBuilder(conf, TestOutputFormat.class).build().getOutputDescriptor(), 1)); - + TaskSpec taskSpec = new TaskSpec( TezTestUtils.getMockTaskAttemptId(0, 0, 0, 0), dagName, vertexName, -1, @@ -353,9 +353,9 @@ public void commitTask(TaskAttemptContext taskContext) throws IOException { @Override public void abortTask(TaskAttemptContext taskContext) throws IOException { } - + } - + public static class TestOutputFormat extends OutputFormat { public static class TestRecordWriter extends RecordWriter { Writer writer; @@ -368,7 +368,7 @@ public static class TestRecordWriter extends RecordWriter { writer = new BufferedWriter(new FileWriter(f)); } } - + @Override public void write(String key, String value) throws IOException, InterruptedException { if (doWrite) { @@ -381,9 +381,9 @@ public void write(String key, String value) throws IOException, InterruptedExcep public void close(TaskAttemptContext context) throws IOException, InterruptedException { writer.close(); } - + } - + @Override public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java index 9fa565e132..26186039f0 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java @@ -77,7 +77,7 @@ public final class MapUtils { private static final Logger LOG = LoggerFactory.getLogger(MapUtils.class); private MapUtils() {} - + public static void configureLocalDirs(Configuration conf, String localDir) throws IOException { String[] localSysDirs = new String[1]; @@ -121,8 +121,8 @@ public static void configureLocalDirs(Configuration conf, String localDir) } conf.set(MRFrameworkConfigs.JOB_LOCAL_DIR, workDir.toString()); } - - private static InputSplit + + private static InputSplit createInputSplit(FileSystem fs, Path workDir, JobConf job, Path file, int numKVs) throws IOException { FileInputFormat.setInputPaths(job, workDir); @@ -146,23 +146,23 @@ public static void configureLocalDirs(Configuration conf, String localDir) } finally { writer.close(); } - + SequenceFileInputFormat format = new SequenceFileInputFormat<>(); InputSplit[] splits = format.getSplits(job, 1); System.err.println("#split = " + splits.length + " ; " + "#locs = " + splits[0].getLocations().length + "; " + - "loc = " + splits[0].getLocations()[0] + "; " + + "loc = " + splits[0].getLocations()[0] + "; " + "off = " + splits[0].getLength() + "; " + "file = " + ((FileSplit)splits[0]).getPath()); return splits[0]; } - + final private static FsPermission JOB_FILE_PERMISSION = FsPermission .createImmutable((short) 0644); // rw-r--r-- // Will write files to PWD, from where they are read. - + private static void writeSplitFiles(FileSystem fs, JobConf conf, InputSplit split) throws IOException { Path jobSplitFile = new Path(conf.get(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR, @@ -200,7 +200,7 @@ public static void generateInputSplit(FileSystem fs, Path workDir, JobConf jobCo InputSplit split = createInputSplit(fs, workDir, jobConf, mapInput, numKVs); writeSplitFiles(fs, jobConf, split); } - + public static LogicalIOProcessorRuntimeTask createLogicalTask(FileSystem fs, Path workDir, JobConf jobConf, int mapId, Path mapInput, TezUmbilical umbilical, String dagName, @@ -211,7 +211,7 @@ public static LogicalIOProcessorRuntimeTask createLogicalTask(FileSystem fs, Pat ProcessorDescriptor mapProcessorDesc = ProcessorDescriptor.create( MapProcessor.class.getName()).setUserPayload( TezUtils.createUserPayloadFromConf(jobConf)); - + Token shuffleToken = new Token<>(); TaskSpec taskSpec = new TaskSpec( diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java index 02972d436c..40091632ef 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java @@ -69,11 +69,11 @@ @SuppressWarnings("deprecation") public class TestMapProcessor { - + private static final Logger LOG = LoggerFactory.getLogger(TestMapProcessor.class); - + private static JobConf defaultConf = new JobConf(); - private static FileSystem localFs = null; + private static FileSystem localFs = null; private static Path workDir = null; static float progressUpdate = 0.0f; static { @@ -89,10 +89,10 @@ public class TestMapProcessor { throw new RuntimeException("init failure", e); } } - - + + public void setUpJobConf(JobConf job) { job.set(TezRuntimeFrameworkConfigs.LOCAL_DIRS, workDir.toString()); @@ -119,7 +119,7 @@ private Path getMapOutputFile(Configuration jobConf, OutputContext outputContext public void cleanup() throws Exception { localFs.delete(workDir, true); } - + @Test(timeout = 5000) public void testMapProcessor() throws Exception { String dagName = "mrdag0"; @@ -134,10 +134,10 @@ public void testMapProcessor() throws Exception { jobConf.set(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR, new Path(workDir, "localized-resources").toUri().toString()); - + Path mapInput = new Path(workDir, "map0"); - - + + MapUtils.generateInputSplit(localFs, workDir, jobConf, mapInput, 10); InputSpec mapInputSpec = new InputSpec("NullSrcVertex", @@ -147,7 +147,7 @@ public void testMapProcessor() throws Exception { .setConfigurationBytes(TezUtils.createByteStringFromConf(jobConf)).build() .toByteArray()))), 1); - OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", + OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1); @@ -166,8 +166,8 @@ public void testMapProcessor() throws Exception { TezTaskOutput mapOutputs = new TezTaskOutputFiles( jobConf, outputContext.getUniqueIdentifier(), outputContext.getDagIdentifier()); - - + + // TODO NEWTEZ FIXME OutputCommitter verification // MRTask mrTask = (MRTask)t.getProcessor(); // Assert.assertEquals(TezNullOutputCommitter.class.getName(), mrTask diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java index c432bf06db..5496ab0f7f 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java @@ -89,7 +89,7 @@ @SuppressWarnings("deprecation") public class TestReduceProcessor { - + private static final Logger LOG = LoggerFactory.getLogger(TestReduceProcessor.class); private static JobConf defaultConf = new JobConf(); @@ -134,14 +134,14 @@ public void testReduceProcessor() throws Exception { String reduceVertexName = MultiStageMRConfigUtil.getFinalReduceVertexName(); JobConf jobConf = new JobConf(defaultConf); setUpJobConf(jobConf); - + MRHelpers.translateMRConfToTez(jobConf); jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 0); jobConf.set(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR, new Path(workDir, "localized-resources").toUri().toString()); jobConf.setBoolean(MRJobConfig.MR_TEZ_SPLITS_VIA_EVENTS, false); - + Path mapInput = new Path(workDir, "map0"); MapUtils.generateInputSplit(localFs, workDir, jobConf, mapInput, 10); @@ -152,7 +152,7 @@ public void testReduceProcessor() throws Exception { .setConfigurationBytes(TezUtils.createByteStringFromConf(jobConf)).build() .toByteArray()))), 1); - OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", + OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName()). setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1); // Run a map @@ -178,9 +178,9 @@ public void testReduceProcessor() throws Exception { Assert.assertEquals(1, cdmEvent.getCount()); DataMovementEvent dme = cdmEvent.getEvents().iterator().next(); dme.setTargetIndex(0); - + LOG.info("Starting reduce..."); - + JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(dagName)); JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(jobConf); Token shuffleToken = new Token(identifier, @@ -195,7 +195,7 @@ public void testReduceProcessor() throws Exception { ProcessorDescriptor reduceProcessorDesc = ProcessorDescriptor.create( ReduceProcessor.class.getName()).setUserPayload( TezUtils.createUserPayloadFromConf(jobConf)); - + InputSpec reduceInputSpec = new InputSpec(mapVertexName, InputDescriptor.create(OrderedGroupedInputLegacy.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1); diff --git a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java index 4c18d71145..253c8421dd 100644 --- a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java +++ b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java @@ -1186,7 +1186,7 @@ public void testRecoveryFromOtherVersions() throws IOException { shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); - + try { shuffle.start(); Assert.fail("Incompatible version, should expect fail here."); diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java index f3d4edcb5a..d5b845dcf7 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java @@ -75,7 +75,7 @@ public class TaskAttemptInfo extends BaseInfo { private TaskInfo taskInfo; private Container container; - + public static class DataDependencyEvent { String taId; long timestamp; @@ -159,7 +159,7 @@ public String getTaskAttemptId() { .intern(otherInfoNode.optString(ATSConstants.TASK_ATTEMPT_ERROR_ENUM)); executionTimeInterval = (endTime > startTime) ? (endTime - startTime) : 0; } - + public static Ordering orderingOnAllocationTime() { return Ordering.from(new Comparator() { @Override @@ -184,19 +184,19 @@ public final long getStartTimeInterval() { public final long getFinishTimeInterval() { return endTime - (getTaskInfo().getVertexInfo().getDagInfo().getStartTime()); } - + public final boolean isSucceeded() { return status.equals(SUCCEEDED); } - + public final List getLastDataEvents() { return lastDataEvents; } - + public final long getExecutionTimeInterval() { return executionTimeInterval; } - + public final long getPostDataExecutionTimeInterval() { if (getStartTime() > 0 && getFinishTime() > 0) { // start time defaults to the actual start time @@ -214,11 +214,11 @@ public final long getPostDataExecutionTimeInterval() { public final long getAllocationToEndTimeInterval() { return (endTime - allocationTime); } - + public final long getAllocationToStartTimeInterval() { return (startTime - allocationTime); } - + public final long getCreationToAllocationTimeInterval() { return (allocationTime - creationTime); } @@ -234,7 +234,7 @@ public final long getFinishTime() { public final long getCreationTime() { return creationTime; } - + public final DataDependencyEvent getLastDataEventInfo(long timeThreshold) { for (int i=lastDataEvents.size()-1; i>=0; i--) { // walk back in time until we get first event that happened before the threshold @@ -245,7 +245,7 @@ public final DataDependencyEvent getLastDataEventInfo(long timeThreshold) { } return null; } - + public final long getTimeTaken() { return getFinishTimeInterval() - getStartTimeInterval(); } @@ -253,7 +253,7 @@ public final long getTimeTaken() { public final long getCreationTimeInterval() { return creationTime - (getTaskInfo().getVertexInfo().getDagInfo().getStartTime()); } - + public final String getCreationCausalTA() { return creationCausalTA; } @@ -261,9 +261,9 @@ public final String getCreationCausalTA() { public final long getAllocationTime() { return allocationTime; } - + public final String getShortName() { - return getTaskInfo().getVertexInfo().getVertexName() + " : " + + return getTaskInfo().getVertexInfo().getVertexName() + " : " + taskAttemptId.substring(taskAttemptId.lastIndexOf('_', taskAttemptId.lastIndexOf('_') - 1) + 1); } @@ -271,7 +271,7 @@ public final String getShortName() { public final String getDiagnostics() { return diagnostics; } - + public final String getTerminationCause() { return terminationCause; } @@ -294,7 +294,7 @@ public final boolean isLocalityInfoAvailable() { } return false; } - + public final String getDetailedStatus() { if (!Strings.isNullOrEmpty(getTerminationCause())) { return getStatus() + ":" + getTerminationCause(); diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java index 6cb883af11..72ecf44183 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java @@ -61,7 +61,7 @@ public class VertexInfo extends BaseInfo { private final long initRequestedTime; private final long startTime; private final long startRequestedTime; - + private final String diagnostics; private final String processorClass; @@ -82,7 +82,7 @@ public class VertexInfo extends BaseInfo { private final List additionalInputInfoList; private final List additionalOutputInfoList; - + private long avgPostDataExecutionTimeInterval = -1; private DagInfo dagInfo; @@ -186,7 +186,7 @@ void setAdditionalInputInfoList(List additionalInp this.additionalInputInfoList.clear(); this.additionalInputInfoList.addAll(additionalInputInfoList); } - + void setAdditionalOutputInfoList(List additionalOutputInfoList) { this.additionalOutputInfoList.clear(); this.additionalOutputInfoList.addAll(additionalOutputInfoList); @@ -234,7 +234,7 @@ public final long getLastTaskFinishTimeInterval() { } return getLastTaskToFinish().getFinishTimeInterval(); } - + public final long getAvgPostDataExecutionTimeInterval() { if (avgPostDataExecutionTimeInterval == -1) { long totalExecutionTime = 0; @@ -268,7 +268,7 @@ public final long getFinishTime() { public final long getInitTime() { return initTime; } - + public final long getInitRequestedTime() { return initRequestedTime; } @@ -276,7 +276,7 @@ public final long getInitRequestedTime() { public final long getStartRequestedTime() { return startRequestedTime; } - + @Override public final long getFinishTimeInterval() { long vertexEndTime = finishTime - (dagInfo.getStartTime()); @@ -298,7 +298,7 @@ public final String getDiagnostics() { public final String getVertexName() { return vertexName; } - + public final String getVertexId() { return vertexId; } diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/utils/Utils.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/utils/Utils.java index 696c56ad30..882321eadf 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/utils/Utils.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/utils/Utils.java @@ -100,8 +100,8 @@ private static void addCounter(CounterGroup group, String counterName, String di LOG.debug("Error finding {} in {} with displayName {}", counterName, group, displayName); } } - - public static List parseDataEventDependencyFromJSON(JSONObject jsonObject) + + public static List parseDataEventDependencyFromJSON(JSONObject jsonObject) throws JSONException { List events = Lists.newArrayList(); JSONArray fields = jsonObject.optJSONArray(Constants.LAST_DATA_EVENTS); diff --git a/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java b/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java index 1c9c1a691c..14401d31bd 100644 --- a/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java +++ b/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java @@ -311,7 +311,7 @@ private void verifyJobSpecificInfo(DagInfo dagInfo) { for (TaskAttemptInfo attempt : attempts) { DataDependencyEvent item = attempt.getLastDataEvents().get(0); assertTrue(item.getTimestamp() > 0); - + if (lastDataEventSourceTA == null) { lastDataEventSourceTA = item.getTaskAttemptId(); } else { diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java index 6b29169cf7..5ec3ff3b93 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java +++ b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java @@ -352,7 +352,7 @@ public void testDAGACls() throws Exception { } /** - * Test Disable Logging for all dags in a session + * Test Disable Logging for all dags in a session * due to failure to create domain in session start * @throws Exception */ @@ -460,7 +460,7 @@ public void testDagLoggingDisabled() throws Exception { .get(ClientResponse.class); assertEquals(404, response.getStatus()); } - + /** * use mini cluster to verify data do push to ats when * the dag logging flag in dagsubmitted event is set on diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java index 6b1c131755..3588c8b9fd 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java +++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java @@ -513,7 +513,7 @@ private static TimelineEntity convertTaskAttemptFinishedEvent(TaskAttemptFinishe atsEntity.addOtherInfo(ATSConstants.COUNTERS, DAGUtils.convertCountersToATSMap(event.getCounters())); if (event.getDataEvents() != null && !event.getDataEvents().isEmpty()) { - atsEntity.addOtherInfo(ATSConstants.LAST_DATA_EVENTS, + atsEntity.addOtherInfo(ATSConstants.LAST_DATA_EVENTS, DAGUtils.convertDataEventDependecyInfoToATS(event.getDataEvents())); } if (event.getNodeId() != null) { diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java index fd23e2f7a6..fc8b0fa499 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java +++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java @@ -636,7 +636,7 @@ public void testConvertTaskAttemptFinishedEvent() { final Map otherInfo = timelineEntity.getOtherInfo(); Assert.assertEquals(17, otherInfo.size()); - Assert.assertEquals(tezTaskAttemptID.toString(), + Assert.assertEquals(tezTaskAttemptID.toString(), timelineEntity.getOtherInfo().get(ATSConstants.CREATION_CAUSAL_ATTEMPT)); Assert.assertEquals(creationTime, timelineEntity.getOtherInfo().get(ATSConstants.CREATION_TIME)); Assert.assertEquals(allocationTime, timelineEntity.getOtherInfo().get(ATSConstants.ALLOCATION_TIME)); @@ -1153,7 +1153,7 @@ public void testConvertVertexReconfigreDoneEvent() { TezVertexID vId = tezVertexID; Map edgeMgrs = new HashMap(); - + edgeMgrs.put("a", EdgeProperty.create(EdgeManagerPluginDescriptor.create("a.class") .setHistoryText("text"), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))); diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/tests/MiniTezClusterWithTimeline.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/tests/MiniTezClusterWithTimeline.java index c71c61535c..869066e0d1 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/tests/MiniTezClusterWithTimeline.java +++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/tests/MiniTezClusterWithTimeline.java @@ -111,7 +111,7 @@ public void serviceInit(Configuration conf) throws Exception { conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(), "apps_staging_dir" + Path.SEPARATOR).getAbsolutePath()); } - + if (conf.get(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC) == null) { // nothing defined. set quick delete value conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0l); @@ -131,7 +131,7 @@ public void serviceInit(Configuration conf) throws Exception { } else { LOG.info("Using Tez AppJar: " + appJarLocalFile.getAbsolutePath()); } - + FileSystem fs = FileSystem.get(conf); Path testRootDir = fs.makeQualified(new Path("target", getName() + "-tmpDir")); Path appRemoteJar = new Path(testRootDir, "TezAppJar.jar"); @@ -309,7 +309,7 @@ public boolean apply(ApplicationReport appReport) { yarnClient.stop(); } } - + public Path getConfigFilePath() { return confFilePath; } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/common/ContainerTask.java b/tez-runtime-internals/src/main/java/org/apache/tez/common/ContainerTask.java index c0a2574f64..53290c2b5b 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/common/ContainerTask.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/common/ContainerTask.java @@ -56,15 +56,15 @@ public TaskSpec getTaskSpec() { public boolean shouldDie() { return shouldDie; } - + public Map getAdditionalResources() { return this.additionalResources; } - + public Credentials getCredentials() { return this.credentials; } - + public boolean haveCredentialsChanged() { return this.credentialsChanged; } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java b/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java index e95d7eab56..77b9cbe8fb 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java @@ -112,7 +112,7 @@ public static CompositeDataMovementEvent convertCompositeDataMovementEventFromPr proto.getCount(), proto.hasUserPayload() ? proto.getUserPayload().asReadOnlyByteBuffer() : null); } - + public static EventProtos.VertexManagerEventProto convertVertexManagerEventToProto( VertexManagerEvent event) { EventProtos.VertexManagerEventProto.Builder vmBuilder = VertexManagerEventProto.newBuilder(); @@ -122,7 +122,7 @@ public static EventProtos.VertexManagerEventProto convertVertexManagerEventToPro } return vmBuilder.build(); } - + public static VertexManagerEvent convertVertexManagerEventFromProto( EventProtos.VertexManagerEventProto vmProto) { return VertexManagerEvent.create(vmProto.getTargetVertexName(), diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/common/TezLocalResource.java b/tez-runtime-internals/src/main/java/org/apache/tez/common/TezLocalResource.java index 3a8358bdca..eb46fc6704 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/common/TezLocalResource.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/common/TezLocalResource.java @@ -33,7 +33,7 @@ public class TezLocalResource implements Writable { public TezLocalResource() { } - + public TezLocalResource(URI uri, long size, long timestamp) { this.uri = uri; this.size = size; diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/InputReadyTracker.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/InputReadyTracker.java index 315efd0cd0..217cb9ebd7 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/InputReadyTracker.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/InputReadyTracker.java @@ -40,9 +40,9 @@ public class InputReadyTracker { private final ConcurrentMap readyInputs; - + private ConcurrentMap> inputToGroupMap; - + private final ReentrantLock lock = new ReentrantLock(); private final Condition condition = lock.newCondition(); diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java index d6973bcb05..9ce0373977 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java @@ -138,7 +138,7 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask { /** Maps which will be provided to the processor run method */ final LinkedHashMap runInputMap; final LinkedHashMap runOutputMap; - + private final Map serviceConsumerMetadata; private final Map envMap; @@ -153,7 +153,7 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask { private final int appAttemptNumber; private volatile InputReadyTracker inputReadyTracker; - + private volatile ObjectRegistry objectRegistry; private final ExecutionContext ExecutionContext; private final long memAvailable; @@ -981,7 +981,7 @@ private void cleanupStructures() { inputContextMap.clear(); outputContextMap.clear(); - // only clean up objects in non-local mode, because local mode share the same + // only clean up objects in non-local mode, because local mode share the same // taskSpec in AM rather than getting it through RPC in non-local mode /** Put other objects here when they are shared between AM & TezChild in local mode **/ if (!tezConf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT)) { @@ -1025,13 +1025,13 @@ void printThreads() { } } } - + @Private @VisibleForTesting public Collection getInputContexts() { return this.inputContextMap.values(); } - + @Private @VisibleForTesting public Collection getOutputContexts() { @@ -1043,7 +1043,7 @@ public Collection getOutputContexts() { public ProcessorContext getProcessorContext() { return this.processorContext; } - + @Private @VisibleForTesting public LogicalIOProcessor getProcessor() { diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java index bde3059221..17e69c8bf3 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java @@ -51,7 +51,7 @@ public abstract class RuntimeTask { protected float progress; protected final TezCounters tezCounters; private final Map counterMap = Maps.newConcurrentMap(); - + protected final TaskSpec taskSpec; protected final Configuration tezConf; protected final TezUmbilical tezUmbilical; @@ -102,11 +102,11 @@ public TezCounters addAndGetTezCounter(String name) { counterMap.put(name, counter); return counter; } - + public boolean hasInitialized() { return EnumSet.of(State.RUNNING, State.CLOSED).contains(state.get()); } - + public String getVertexName() { return taskSpec.getVertexName(); } @@ -114,11 +114,11 @@ public String getVertexName() { public void registerError() { errorReported.set(true); } - + public final void notifyProgressInvocation() { progressNotified.lazySet(true); } - + public boolean getAndClearProgressNotification() { boolean retVal = progressNotified.getAndSet(false); return retVal; @@ -160,19 +160,19 @@ public TezTaskAttemptID getTaskAttemptID() { public int getEventCounter() { return eventCounter.get(); } - + public int getNextFromEventId() { return nextFromEventId.get(); } - + public int getNextPreRoutedEventId() { return nextPreRoutedEventId.get(); } - + public void setNextFromEventId(int nextFromEventId) { this.nextFromEventId.set(nextFromEventId); } - + public void setNextPreRoutedEventId(int nextPreRoutedEventId) { this.nextPreRoutedEventId.set(nextPreRoutedEventId); } @@ -180,7 +180,7 @@ public void setNextPreRoutedEventId(int nextPreRoutedEventId) { public boolean isTaskDone() { return taskDone.get(); } - + public void setFrameworkCounters() { if (counterUpdater != null) { this.counterUpdater.updateCounters(); diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskStatusUpdateEvent.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskStatusUpdateEvent.java index 518cbf4c1b..7c8c8606f9 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskStatusUpdateEvent.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskStatusUpdateEvent.java @@ -37,7 +37,7 @@ public class TaskStatusUpdateEvent extends Event implements Writable { public TaskStatusUpdateEvent() { } - public TaskStatusUpdateEvent(TezCounters tezCounters, float progress, TaskStatistics statistics, + public TaskStatusUpdateEvent(TezCounters tezCounters, float progress, TaskStatistics statistics, boolean progressNotified) { this.tezCounters = tezCounters; this.progress = progress; @@ -52,11 +52,11 @@ public TezCounters getCounters() { public float getProgress() { return progress; } - + public TaskStatistics getStatistics() { return statistics; } - + public boolean getProgressNotified() { return progressNotified; } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/GroupInputSpec.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/GroupInputSpec.java index 0177bc84fa..c1123bc000 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/GroupInputSpec.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/GroupInputSpec.java @@ -35,29 +35,29 @@ public class GroupInputSpec implements Writable{ private String groupName; private List groupVertices; private InputDescriptor mergedInputDescriptor; - + public GroupInputSpec() { // for Writable } - + public String getGroupName() { return groupName; } - + public List getGroupVertices() { return groupVertices; } - + public InputDescriptor getMergedInputDescriptor() { return mergedInputDescriptor; } - + public GroupInputSpec(String groupName, List groupVertices, InputDescriptor inputDescriptor) { this.groupName = StringInterner.intern(groupName); this.groupVertices = groupVertices; this.mergedInputDescriptor = inputDescriptor; } - + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, groupName); @@ -79,7 +79,7 @@ public void readFields(DataInput in) throws IOException { mergedInputDescriptor = new InputDescriptor(); mergedInputDescriptor.readFields(in); } - + @Override public String toString() { StringBuffer sb = new StringBuffer(); diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/IOStatistics.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/IOStatistics.java index 8f2806266f..d605728d5d 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/IOStatistics.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/IOStatistics.java @@ -26,23 +26,23 @@ public class IOStatistics implements Writable { private volatile long dataSize = 0; private volatile long numItems = 0; - + public void setDataSize(long size) { this.dataSize = size; } - + public long getDataSize() { return dataSize; } - + public void setItemsProcessed(long items) { this.numItems = items; } - + public long getItemsProcessed() { return numItems; } - + public void mergeFrom(org.apache.tez.runtime.api.impl.IOStatistics other) { this.setDataSize(this.getDataSize() + other.getDataSize()); this.setItemsProcessed(this.getItemsProcessed() + other.getItemsProcessed()); diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskSpec.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskSpec.java index 15e756b853..cfa62cad4f 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskSpec.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskSpec.java @@ -50,7 +50,7 @@ public class TaskSpec implements Writable, TaskAttemptIDAware { public TaskSpec() { } - + public static TaskSpec createBaseTaskSpec(String dagName, String vertexName, int vertexParallelism, ProcessorDescriptor processorDescriptor, List inputSpecList, List outputSpecList, @@ -156,7 +156,7 @@ public List getInputs() { public List getOutputs() { return outputSpecList; } - + public List getGroupInputs() { return groupInputSpecList; } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskStatistics.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskStatistics.java index 414b78c741..57db928c0b 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskStatistics.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskStatistics.java @@ -37,12 +37,12 @@ public class TaskStatistics implements Writable { public void addIO(String edgeName) { addIO(edgeName, new IOStatistics()); } - + public void addIO(String edgeName, IOStatistics stats) { Objects.requireNonNull(stats, edgeName); ioStatistics.put(StringInterner.intern(edgeName), stats); } - + public Map getIOStatistics() { return ioStatistics; } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezHeartbeatResponse.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezHeartbeatResponse.java index 0aa4db4ec5..271b03c8d9 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezHeartbeatResponse.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezHeartbeatResponse.java @@ -53,7 +53,7 @@ public boolean shouldDie() { public long getLastRequestId() { return lastRequestId; } - + public int getNextFromEventId() { return nextFromEventId; } @@ -73,7 +73,7 @@ public void setLastRequestId(long lastRequestId ) { public void setShouldDie() { this.shouldDie = true; } - + public void setNextFromEventId(int nextFromEventId) { this.nextFromEventId = nextFromEventId; } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java index f218ee3aab..29c5df2936 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java @@ -62,7 +62,7 @@ public class TezInputContextImpl extends TezTaskContextImpl private final Map inputs; private volatile InputReadyTracker inputReadyTracker; private final InputStatisticsReporterImpl statsReporter; - + class InputStatisticsReporterImpl implements InputStatisticsReporter { @Override @@ -76,15 +76,15 @@ public synchronized void reportDataSize(long size) { public void reportItemsProcessed(long items) { // this is a concurrent map. Plus we are not adding/deleting entries runtimeTask.getTaskStatistics().getIOStatistics().get(sourceVertexName) - .setItemsProcessed(items); + .setItemsProcessed(items); } - + } @Private public TezInputContextImpl(Configuration conf, String[] workDirs, int appAttemptNumber, - TezUmbilical tezUmbilical, String dagName, + TezUmbilical tezUmbilical, String dagName, String taskVertexName, String sourceVertexName, int vertexParallelism, TezTaskAttemptID taskAttemptID, int inputIndex, @Nullable UserPayload userPayload, @@ -142,7 +142,7 @@ public void sendEvents(List events) { public UserPayload getUserPayload() { return userPayload; } - + @Override public int getInputIndex() { return inputIndex; diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java index df7aa80327..8042a23e2c 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java @@ -61,7 +61,7 @@ public TezMergedInputContextImpl(@Nullable UserPayload userPayload, String group public UserPayload getUserPayload() { return userPayload; } - + @Override public void inputIsReady() { inputReadyTracker.setInputIsReady(groupInputsMap.get(groupInputName)); diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java index c9392495cf..4c9617a084 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java @@ -87,7 +87,7 @@ public void reportItemsProcessed(long items) { runtimeTask.getTaskStatistics().getIOStatistics().get(destinationVertexName) .setItemsProcessed(items); } - + } @Private @@ -102,7 +102,7 @@ public TezOutputContextImpl(Configuration conf, String[] workDirs, int appAttemp Map auxServiceEnv, MemoryDistributor memDist, OutputDescriptor outputDescriptor, ObjectRegistry objectRegistry, ExecutionContext executionContext, long memAvailable, TezExecutors sharedExecutor) { - super(conf, workDirs, appAttemptNumber, dagName, taskVertexName, + super(conf, workDirs, appAttemptNumber, dagName, taskVertexName, vertexParallelism, taskAttemptID, wrapCounters(runtimeTask, taskVertexName, destinationVertexName, conf), runtimeTask, tezUmbilical, serviceConsumerMetadata, diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezTaskContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezTaskContextImpl.java index 518b717924..1d76906e80 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezTaskContextImpl.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezTaskContextImpl.java @@ -74,7 +74,7 @@ public abstract class TezTaskContextImpl implements TaskContext, Closeable { @Private public TezTaskContextImpl(Configuration conf, String[] workDirs, int appAttemptNumber, - String dagName, String taskVertexName, int vertexParallelism, + String dagName, String taskVertexName, int vertexParallelism, TezTaskAttemptID taskAttemptID, TezCounters counters, LogicalIOProcessorRuntimeTask runtimeTask, TezUmbilical tezUmbilical, Map serviceConsumerMetadata, Map auxServiceEnv, MemoryDistributor memDist, @@ -179,7 +179,7 @@ public String[] getWorkDirs() { public String getUniqueIdentifier() { return uniqueIdentifier; } - + @Override public ObjectRegistry getObjectRegistry() { return objectRegistry; @@ -189,7 +189,7 @@ public ObjectRegistry getObjectRegistry() { public final void notifyProgress() { runtimeTask.notifyProgressInvocation(); } - + @Override public ByteBuffer getServiceConsumerMetaData(String serviceName) { return (ByteBuffer) serviceConsumerMetadata.get(serviceName) @@ -213,7 +213,7 @@ public void requestInitialMemory(long size, MemoryUpdateCallback callbackHandler callbackHandler = new MemoryUpdateCallback() { @Override public void memoryAssigned(long assignedSize) { - + } }; } @@ -224,7 +224,7 @@ public void memoryAssigned(long assignedSize) { public long getTotalMemoryAvailableToTask() { return memAvailable; } - + protected void signalFatalError(Throwable t, String message, EventMetaData sourceInfo) { signalFailure(TaskFailureType.NON_FATAL, t, message, sourceInfo); } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/objectregistry/ObjectRegistryImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/objectregistry/ObjectRegistryImpl.java index eccb71c1dd..b870d817cd 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/objectregistry/ObjectRegistryImpl.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/objectregistry/ObjectRegistryImpl.java @@ -27,7 +27,7 @@ import org.apache.tez.runtime.api.ObjectRegistry; public class ObjectRegistryImpl implements ObjectRegistry { - + public enum ObjectLifeCycle { /** Objects are valid for the lifetime of the Tez JVM/Session */ diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java index 59d58eb91b..5686144f53 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java @@ -57,7 +57,7 @@ public class MemoryDistributor { private final int numTotalInputs; private final int numTotalOutputs; private final Configuration conf; - + private AtomicInteger numInputsSeen = new AtomicInteger(0); private AtomicInteger numOutputsSeen = new AtomicInteger(0); @@ -103,7 +103,7 @@ public MemoryDistributor(int numTotalInputs, int numTotalOutputs, Configuration } - + /** * Used by the Tez framework to request memory on behalf of user requests. */ diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java index 6fa7b0ef3b..98e1fbb979 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java @@ -85,7 +85,7 @@ public Long apply(InitialMemoryRequestContext requestContext) { } else { long allocated = (long) ((requestedSize / (double) totalRequested) * availableForAllocation); allocations.add(allocated); - LOG.debug("Scaling requested: {} to allocated: {}", requestedSize, allocated); + LOG.debug("Scaling requested: {} to allocated: {}", requestedSize, allocated); } } return allocations; diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java index 427098d1fa..b91de9d56b 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java @@ -57,10 +57,10 @@ public class TaskCounterUpdater { private ResourceCalculatorProcessTree pTree; private long initCpuCumulativeTime = 0; private final String pid; - + public TaskCounterUpdater(TezCounters counters, Configuration conf, String pid) { this.tezCounters = counters; - this.conf = conf; + this.conf = conf; this.gcUpdater = new GcTimeUpdater(tezCounters); this.pid = pid; initResourceCalculatorPlugin(); @@ -85,14 +85,14 @@ public void updateCounters() { gcUpdater.incrementGcCounter(); updateResourceCounters(); } - + private void recordInitialCpuStats() { if (pTree != null) { pTree.updateProcessTree(); initCpuCumulativeTime = pTree.getCumulativeCpuTime(); } } - + /** * Update resource information counters */ @@ -114,7 +114,7 @@ void updateResourceCounters() { tezCounters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).setValue(pMem); tezCounters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).setValue(vMem); } - + /** * Updates the {@link TaskCounter#COMMITTED_HEAP_BYTES} counter to reflect the * current total committed heap space usage of this JVM. @@ -124,12 +124,12 @@ private void updateHeapUsageCounter() { tezCounters.findCounter(TaskCounter.COMMITTED_HEAP_BYTES) .setValue(currentHeapUsage); } - + private void initResourceCalculatorPlugin() { Class clazz = this.conf.getClass( TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, TezMxBeanResourceCalculator.class, - ResourceCalculatorProcessTree.class); + ResourceCalculatorProcessTree.class); pTree = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(pid, clazz, conf); diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ContainerReporter.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ContainerReporter.java index fa5155eaa5..a7192f4051 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ContainerReporter.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ContainerReporter.java @@ -31,7 +31,7 @@ /** * Responsible for communication between a running Container and the ApplicationMaster. The main * functionality is to poll for new tasks. - * + * */ public class ContainerReporter extends CallableWithNdc { diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ErrorReporter.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ErrorReporter.java index 1146ce468f..7f5be97d52 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ErrorReporter.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ErrorReporter.java @@ -21,6 +21,6 @@ public interface ErrorReporter { void reportError(Throwable t); - + void shutdownRequested(); } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java index 2e965491a3..60faec56fa 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java @@ -210,7 +210,7 @@ public TezTaskUmbilicalProtocol run() throws Exception { } TezCommonUtils.logCredentials(LOG, credentials, "tezChildInit"); } - + public ContainerExecutionResult run() throws IOException, InterruptedException, TezException { ContainerContext containerContext = new ContainerContext(containerIdString); @@ -333,7 +333,7 @@ public ContainerExecutionResult run() throws IOException, InterruptedException, /** * Setup - * + * * @param containerTask * the new task specification. Must be a valid task * @param childUGI @@ -362,7 +362,7 @@ UserGroupInformation handleNewTaskCredentials(ContainerTask containerTask, /** * Handles any additional resources to be localized for the new task - * + * * @param containerTask * @throws IOException * @throws TezException @@ -399,7 +399,7 @@ public URI apply(TezLocalResource input) { /** * Cleans entries from the object registry, and resets the startedInputsMap if required - * + * * @param containerTask * the new task specification. Must be a valid task */ diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestInputReadyTracker.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestInputReadyTracker.java index 00228fc2a3..ae7a759aa6 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestInputReadyTracker.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestInputReadyTracker.java @@ -46,7 +46,7 @@ public class TestInputReadyTracker { private static final long SLEEP_TIME = 200l; - + @Test(timeout = 20000) public void testWithoutGrouping1() throws InterruptedException { InputReadyTracker inputReadyTracker = new InputReadyTracker(); @@ -65,7 +65,7 @@ public void testWithoutGrouping1() throws InterruptedException { assertTrue(input1.isReady); assertFalse(input2.isReady); assertEquals(input1, readyInput); - + startTime = System.nanoTime(); setDelayedInputReady(input2); assertFalse(inputReadyTracker.waitForAllInputsReady(requestList, 0)); @@ -90,12 +90,12 @@ public void testWithoutGrouping2() throws InterruptedException { List requestList; long startTime = 0l; long readyTime = 0l; - + requestList = new ArrayList(); requestList.add(input1); requestList.add(input2); requestList.add(input3); - + startTime = System.nanoTime(); setDelayedInputReady(input2); Input readyInput = inputReadyTracker.waitForAnyInputReady(requestList); @@ -107,7 +107,7 @@ public void testWithoutGrouping2() throws InterruptedException { assertTrue(readyTime >= startTime + SLEEP_TIME); assertFalse(input1.isReady); assertFalse(input3.isReady); - + requestList = new ArrayList(); requestList.add(input1); requestList.add(input3); @@ -122,7 +122,7 @@ public void testWithoutGrouping2() throws InterruptedException { assertTrue(readyTime >= startTime + SLEEP_TIME); assertTrue(input2.isReady); assertFalse(input3.isReady); - + requestList = new ArrayList(); requestList.add(input3); startTime = System.nanoTime(); @@ -146,16 +146,16 @@ public void testGrouped() throws InterruptedException { ImmediatelyReadyInputForTest input1 = new ImmediatelyReadyInputForTest(inputReadyTracker); ControlledReadyInputForTest input2 = new ControlledReadyInputForTest(inputReadyTracker); - + ImmediatelyReadyInputForTest input3 = new ImmediatelyReadyInputForTest(inputReadyTracker); ControlledReadyInputForTest input4 = new ControlledReadyInputForTest(inputReadyTracker); - - + + List group1Inputs = new ArrayList(); group1Inputs.add(input1); group1Inputs.add(input2); - + List group2Inputs = new ArrayList(); group2Inputs.add(input3); group2Inputs.add(input4); @@ -186,12 +186,12 @@ public void testGrouped() throws InterruptedException { assertTrue(input1.isReady); assertFalse(input2.isReady); assertEquals(group1, readyInput); - - + + requestList = new ArrayList(); requestList.add(group2); - - + + startTime = System.nanoTime(); setDelayedInputReady(input4); inputReadyTracker.waitForAllInputsReady(requestList); @@ -202,9 +202,9 @@ public void testGrouped() throws InterruptedException { assertTrue(input3.isReady); assertTrue(input4.isReady); assertTrue(readyTime >= startTime + SLEEP_TIME); - + } - + private long setDelayedInputReady(final ControlledReadyInputForTest input) { long startTime = System.nanoTime(); new Thread() { @@ -223,7 +223,7 @@ public void run() { private static class ImmediatelyReadyInputForTest extends AbstractLogicalInput { private volatile boolean isReady = false; - + ImmediatelyReadyInputForTest(InputReadyTracker inputReadyTracker) { super(null, 0); isReady = true; @@ -253,12 +253,12 @@ public List close() throws Exception { return null; } } - + private static class ControlledReadyInputForTest extends AbstractLogicalInput { private volatile boolean isReady = false; private InputReadyTracker inputReadyTracker; - + ControlledReadyInputForTest(InputReadyTracker inputReadyTracker) { super(null, 0); this.inputReadyTracker = inputReadyTracker; @@ -270,7 +270,7 @@ public List initialize() throws Exception { } @Override - public void start() throws Exception { + public void start() throws Exception { } @Override diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java index ee1a518cfc..58c1a118da 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java @@ -40,18 +40,18 @@ public class TestMemoryDistributor { protected Configuration conf = new Configuration(); - + @Before public void setup() { conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ENABLED, true); conf.set(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ALLOCATOR_CLASS, ScalingAllocator.class.getName()); } - + @Test(timeout = 5000) public void testScalingNoProcessor() throws TezException { MemoryDistributor dist = new MemoryDistributor(2, 1, conf); - + dist.setJvmMemory(10000l); // First request @@ -59,21 +59,21 @@ public void testScalingNoProcessor() throws TezException { InputContext e1InputContext1 = createTestInputContext(); InputDescriptor e1InDesc1 = createTestInputDescriptor(); dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1); - + // Second request MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest(); InputContext e2InputContext2 = createTestInputContext(); InputDescriptor e2InDesc2 = createTestInputDescriptor(); dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2); - + // Third request - output MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest(); OutputContext e3OutputContext1 = createTestOutputContext(); OutputDescriptor e3OutDesc2 = createTestOutputDescriptor(); dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc2); - + dist.makeInitialAllocations(); - + // Total available: 70% of 10K = 7000 // 3 requests - 10K, 10K, 5K // Scale down to - 2800, 2800, 1400 @@ -81,12 +81,12 @@ public void testScalingNoProcessor() throws TezException { assertEquals(2800, e2Callback.assigned); assertEquals(1400, e3Callback.assigned); } - + @Test(timeout = 5000) public void testScalingNoProcessor2() throws TezException { // Real world values MemoryDistributor dist = new MemoryDistributor(2, 0, conf); - + dist.setJvmMemory(209715200l); // First request @@ -94,23 +94,23 @@ public void testScalingNoProcessor2() throws TezException { InputContext e1InputContext1 = createTestInputContext(); InputDescriptor e1InDesc1 = createTestInputDescriptor(); dist.requestMemory(104857600l, e1Callback, e1InputContext1, e1InDesc1); - + // Second request MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest(); InputContext e2InputContext2 = createTestInputContext(); InputDescriptor e2InDesc2 = createTestInputDescriptor(); dist.requestMemory(157286400l, e2Callback, e2InputContext2, e2InDesc2); - + dist.makeInitialAllocations(); assertEquals(58720256l, e1Callback.assigned); assertEquals(88080384l, e2Callback.assigned); } - + @Test(timeout = 5000) public void testScalingProcessor() throws TezException { MemoryDistributor dist = new MemoryDistributor(2, 1, conf); - + dist.setJvmMemory(10000l); // First request @@ -118,28 +118,28 @@ public void testScalingProcessor() throws TezException { InputContext e1InputContext1 = createTestInputContext(); InputDescriptor e1InDesc1 = createTestInputDescriptor(); dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1); - + // Second request MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest(); InputContext e2InputContext2 = createTestInputContext(); InputDescriptor e2InDesc2 = createTestInputDescriptor(); dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2); - + // Third request - output MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest(); OutputContext e3OutputContext1 = createTestOutputContext(); OutputDescriptor e3OutDesc1 = createTestOutputDescriptor(); dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc1); - + // Fourth request - processor MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest(); ProcessorContext e4ProcessorContext1 = createTestProcessortContext(); ProcessorDescriptor e4ProcessorDesc1 = createTestProcessorDescriptor(); dist.requestMemory(5000, e4Callback, e4ProcessorContext1, e4ProcessorDesc1); - - + + dist.makeInitialAllocations(); - + // Total available: 70% of 10K = 7000 // 4 requests - 10K, 10K, 5K, 5K // Scale down to - 2333.33, 2333.33, 1166.66, 1166.66 @@ -148,14 +148,14 @@ public void testScalingProcessor() throws TezException { assertTrue(e3Callback.assigned >= 1166 && e3Callback.assigned <= 1167); assertTrue(e4Callback.assigned >= 1166 && e4Callback.assigned <= 1167); } - + @Test(timeout = 5000) public void testScalingDisabled() throws TezException { // Real world values Configuration conf = new Configuration(this.conf); conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ENABLED, false); MemoryDistributor dist = new MemoryDistributor(2, 0, conf); - + dist.setJvmMemory(207093760l); // First request @@ -163,25 +163,25 @@ public void testScalingDisabled() throws TezException { InputContext e1InputContext1 = createTestInputContext(); InputDescriptor e1InDesc1 = createTestInputDescriptor(); dist.requestMemory(104857600l, e1Callback, e1InputContext1, e1InDesc1); - + // Second request MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest(); InputContext e2InputContext2 = createTestInputContext(); InputDescriptor e2InDesc2 = createTestInputDescriptor(); dist.requestMemory(144965632l, e2Callback, e2InputContext2, e2InDesc2); - + dist.makeInitialAllocations(); assertEquals(104857600l, e1Callback.assigned); assertEquals(144965632l, e2Callback.assigned); } - + @Test(timeout = 5000) public void testReserveFractionConfigured() throws TezException { Configuration conf = new Configuration(this.conf); conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION, 0.5d); MemoryDistributor dist = new MemoryDistributor(2, 1, conf); - + dist.setJvmMemory(10000l); // First request @@ -189,21 +189,21 @@ public void testReserveFractionConfigured() throws TezException { InputContext e1InputContext1 = createTestInputContext(); InputDescriptor e1InDesc1 = createTestInputDescriptor(); dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1); - + // Second request MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest(); InputContext e2InputContext2 = createTestInputContext(); InputDescriptor e2InDesc2 = createTestInputDescriptor(); dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2); - + // Third request - output MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest(); OutputContext e3OutputContext1 = createTestOutputContext(); OutputDescriptor e3OutDesc2 = createTestOutputDescriptor(); dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc2); - + dist.makeInitialAllocations(); - + // Total available: 50% of 10K = 7000 // 3 requests - 10K, 10K, 5K // Scale down to - 2000, 2000, 1000 @@ -211,8 +211,8 @@ public void testReserveFractionConfigured() throws TezException { assertEquals(2000, e2Callback.assigned); assertEquals(1000, e3Callback.assigned); } - - + + private static class MemoryUpdateCallbackForTest extends MemoryUpdateCallback { long assigned = -1000; @@ -247,14 +247,14 @@ protected InputContext createTestInputContext() { doReturn("task").when(context).getTaskVertexName(); return context; } - + protected OutputContext createTestOutputContext() { OutputContext context = mock(OutputContext.class); doReturn("output").when(context).getDestinationVertexName(); doReturn("task").when(context).getTaskVertexName(); return context; } - + protected ProcessorContext createTestProcessortContext() { ProcessorContext context = mock(ProcessorContext.class); doReturn("task").when(context).getTaskVertexName(); diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskReporter.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskReporter.java index c5c75de228..6e42de3f80 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskReporter.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskReporter.java @@ -132,7 +132,7 @@ public Object answer(InvocationOnMock invocation) throws Throwable { } } - + @Test(timeout = 10000) public void testEventThrottling() throws Exception { TezTaskAttemptID mockTaskAttemptId = mock(TezTaskAttemptID.class); @@ -179,20 +179,20 @@ public void testStatusUpdateAfterInitializationAndCounterFlag() { boolean progressNotified = false; doReturn(progressNotified).when(mockTask).getAndClearProgressNotification(); TezTaskUmbilicalProtocol mockUmbilical = mock(TezTaskUmbilicalProtocol.class); - + float progress = 0.5f; TaskStatistics stats = new TaskStatistics(); TezCounters counters = new TezCounters(); doReturn(progress).when(mockTask).getProgress(); doReturn(stats).when(mockTask).getTaskStatistics(); doReturn(counters).when(mockTask).getCounters(); - + // Setup the sleep time to be way higher than the test timeout TaskReporter.HeartbeatCallable heartbeatCallable = new TaskReporter.HeartbeatCallable(mockTask, mockUmbilical, 100000, 100000, 5, new AtomicLong(0), "containerIdStr"); - + // task not initialized - nothing obtained from task doReturn(false).when(mockTask).hasInitialized(); TaskStatusUpdateEvent event = heartbeatCallable.getStatusUpdateEvent(true); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java index c1caa5251d..edba2109f5 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java @@ -47,7 +47,7 @@ @Private public class InputReadyVertexManager extends VertexManagerPlugin { - private static final Logger LOG = + private static final Logger LOG = LoggerFactory.getLogger(InputReadyVertexManager.class); Map srcVertexInfo = Maps.newHashMap(); @@ -69,7 +69,7 @@ static class SourceVertexInfo { int numTasks; int numFinishedTasks; Boolean taskIsFinished[]; - + SourceVertexInfo(int numTasks, EdgeProperty edgeProperty) { this.numTasks = numTasks; this.numFinishedTasks = 0; @@ -77,13 +77,13 @@ static class SourceVertexInfo { this.taskIsFinished = new Boolean[numTasks]; } } - + private void configure() { Preconditions.checkState(!configured.get(), "Vertex: " + getContext().getVertexName()); int numManagedTasks = getContext().getVertexNumTasks(getContext().getVertexName()); LOG.info("Managing " + numManagedTasks + " tasks for vertex: " + getContext().getVertexName()); - // find out about all input edge types. If there is a custom edge then + // find out about all input edge types. If there is a custom edge then // TODO Until TEZ-1013 we cannot handle custom input formats Map edges = getContext().getInputVertexEdgeProperties(); int oneToOneSrcTaskCount = 0; @@ -113,21 +113,21 @@ private void configure() { } srcVertexInfo.put(srcVertex, new SourceVertexInfo(numSrcTasks, edgeProp)); } - + if (numOneToOneEdges > 0) { Preconditions .checkState(oneToOneSrcTaskCount >= 0, "Vertex: " + getContext().getVertexName()); if (oneToOneSrcTaskCount != numManagedTasks) { numManagedTasks = oneToOneSrcTaskCount; // must change parallelism to make them the same - LOG.info("Update parallelism of vertex: " + getContext().getVertexName() + + LOG.info("Update parallelism of vertex: " + getContext().getVertexName() + " to " + oneToOneSrcTaskCount + " to match source 1-1 vertices."); getContext().reconfigureVertex(oneToOneSrcTaskCount, null, null); } oneToOneSrcTasksDoneCount = new int[oneToOneSrcTaskCount]; oneToOneLocationHints = new TaskLocationHint[oneToOneSrcTaskCount]; } - + Preconditions.checkState(numManagedTasks >=0, "Vertex: " + getContext().getVertexName()); taskIsStarted = new boolean[numManagedTasks]; @@ -136,11 +136,11 @@ private void configure() { getContext().doneReconfiguringVertex(); trySchedulingPendingCompletions(); } - + private boolean readyToSchedule() { return (configured.get() && started.get()); } - + private void trySchedulingPendingCompletions() { if (readyToSchedule() && !pendingCompletions.isEmpty()) { for (TaskAttemptIdentifier attempt : pendingCompletions) { @@ -148,7 +148,7 @@ private void trySchedulingPendingCompletions() { } } } - + @Override public void initialize() { // this will prevent vertex from starting until we notify we are done @@ -162,7 +162,7 @@ public void initialize() { getContext().registerForVertexStateUpdates(entry, EnumSet.of(VertexState.CONFIGURED)); } } - + @Override public synchronized void onVertexStateUpdated(VertexStateUpdate stateUpdate) throws Exception { numConfiguredSources++; @@ -184,7 +184,7 @@ public synchronized void onVertexStarted(List completions // allow scheduling started.set(true); - + trySchedulingPendingCompletions(); } @@ -208,7 +208,7 @@ public synchronized void onVertexManagerEventReceived(VertexManagerEvent vmEvent public synchronized void onRootVertexInitialized(String inputName, InputDescriptor inputDescriptor, List events) { } - + void handleSourceTaskFinished(String vertex, Integer taskId) { SourceVertexInfo srcInfo = srcVertexInfo.get(vertex); if (srcInfo.taskIsFinished[taskId.intValue()] == null) { @@ -222,7 +222,7 @@ void handleSourceTaskFinished(String vertex, Integer taskId) { oneToOneLocationHints[taskId.intValue()] = TaskLocationHint.createTaskLocationHint(vertex, taskId); } } - + // custom edge needs to tell us which of our tasks its connected to // for now only-built in edges supported // Check if current source task's vertex is completed. @@ -231,7 +231,7 @@ void handleSourceTaskFinished(String vertex, Integer taskId) { // we depend on all tasks to finish. So nothing to do now. return; } - + // currently finished vertex task may trigger us to schedule for (SourceVertexInfo vInfo : srcVertexInfo.values()) { if (vInfo.edgeProperty.getDataMovementType() != DataMovementType.ONE_TO_ONE) { @@ -242,7 +242,7 @@ void handleSourceTaskFinished(String vertex, Integer taskId) { } } } - + // all source vertices will full dependencies are done List tasksToStart = null; if (numOneToOneEdges == 0) { @@ -271,11 +271,11 @@ void handleSourceTaskFinished(String vertex, Integer taskId) { } } } - + if (tasksToStart != null && !tasksToStart.isEmpty()) { getContext().scheduleTasks(tasksToStart); } - + } } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java index 6513966a02..c4e2fe2a46 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java @@ -55,8 +55,8 @@ import org.slf4j.LoggerFactory; /** - * Starts scheduling tasks when number of completed source tasks crosses - * slowStartMinSrcCompletionFraction and schedules all tasks + * Starts scheduling tasks when number of completed source tasks crosses + * slowStartMinSrcCompletionFraction and schedules all tasks * when slowStartMaxSrcCompletionFraction is reached */ @Public @@ -176,7 +176,7 @@ public static class CustomShuffleEdgeManager extends EdgeManagerPluginOnDemand { int basePartitionRange; int remainderRangeForLastShuffler; int numSourceTasks; - + int[][] sourceIndices; int[][] targetIndices; @@ -223,10 +223,10 @@ public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) { return numSourceTaskOutputs; } - + @Override public void routeDataMovementEventToDestination(DataMovementEvent event, - int sourceTaskIndex, int sourceOutputIndex, + int sourceTaskIndex, int sourceOutputIndex, Map> destinationTaskAndInputIndices) { int sourceIndex = event.getSourceIndex(); int destinationTaskIndex = sourceIndex/basePartitionRange; @@ -238,8 +238,8 @@ public void routeDataMovementEventToDestination(DataMovementEvent event, } // all inputs from a source task are next to each other in original order - int targetIndex = - sourceTaskIndex * partitionRange + int targetIndex = + sourceTaskIndex * partitionRange + sourceIndex % partitionRange; destinationTaskAndInputIndices.put( @@ -260,15 +260,15 @@ public EventRouteMetadata routeDataMovementEventToDestination( } else { partitionRange = remainderRangeForLastShuffler; } - + // all inputs from a source task are next to each other in original order - int targetIndex = - sourceTaskIndex * partitionRange + int targetIndex = + sourceTaskIndex * partitionRange + sourceIndex % partitionRange; return EventRouteMetadata.create(1, new int[]{targetIndex}); } - + @Override public void prepareForRouting() throws Exception { // target indices derive from num src tasks @@ -278,7 +278,7 @@ public void prepareForRouting() throws Exception { targetIndices[srcTaskIndex] = createIndices(basePartitionRange, srcTaskIndex, basePartitionRange); } - + // source indices derive from num dest tasks (==partitions) int numTargetTasks = getContext().getDestinationVertexNumTasks(); sourceIndices = new int[numTargetTasks][]; @@ -298,7 +298,7 @@ private int[] createTargetIndicesForRemainder(int srcTaskIndex) { return createIndices(remainderRangeForLastShuffler, srcTaskIndex, remainderRangeForLastShuffler); } - + @Override public @Nullable CompositeEventRouteMetadata routeCompositeDataMovementEventToDestination( int sourceTaskIndex, int destinationTaskIndex) @@ -317,7 +317,7 @@ private int[] createTargetIndicesForRemainder(int srcTaskIndex) { partitionRange = basePartitionRange; } - return CompositeEventRouteMetadata.create(partitionRange, targetIndicesToSend[0], + return CompositeEventRouteMetadata.create(partitionRange, targetIndicesToSend[0], sourceIndices[destinationTaskIndex][0]); } @@ -328,7 +328,7 @@ public EventRouteMetadata routeInputSourceTaskFailedEventToDestination( if (destinationTaskIndex == (numDestinationTasks-1)) { partitionRange = remainderRangeForLastShuffler; } - int startOffset = sourceTaskIndex * partitionRange; + int startOffset = sourceTaskIndex * partitionRange; int[] targetIndices = new int[partitionRange]; for (int i=0; i> destinationTaskAndInputIndices) { if (remainderRangeForLastShuffler < basePartitionRange) { int startOffset = sourceTaskIndex * basePartitionRange; @@ -349,8 +349,8 @@ public void routeInputSourceTaskFailedEventToDestination(int sourceTaskIndex, for (int i=0; i allIndices = Lists.newArrayListWithCapacity(basePartitionRange); for (int i=0; i * while (kvReader.next()) { @@ -45,7 +45,7 @@ public abstract class KeyValueReader extends Reader { /** * Moves to the next key/values(s) pair - * + * * @return true if another key/value(s) pair exists, false if there are no more. * @throws IOException * if an error occurs @@ -53,14 +53,14 @@ public abstract class KeyValueReader extends Reader { */ public abstract boolean next() throws IOException; - + /** * Returns the current key * @return the current key */ public abstract Object getCurrentKey() throws IOException; - - + + /** * Returns the current value * @return the current value diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValueWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValueWriter.java index b5c4294a13..512659f31c 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValueWriter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValueWriter.java @@ -32,7 +32,7 @@ public abstract class KeyValueWriter extends Writer { /** * Writes a key/value pair. - * + * * @param key * the key to write * @param value diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValuesReader.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValuesReader.java index 7760818876..98f5489577 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValuesReader.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValuesReader.java @@ -26,7 +26,7 @@ /** * A key/value(s) pair based {@link Reader}. - * + * * Example usage * * while (kvReader.next()) { @@ -45,7 +45,7 @@ public abstract class KeyValuesReader extends Reader { /** * Moves to the next key/values(s) pair - * + * * @return true if another key/value(s) pair exists, false if there are no more. * @throws IOException * if an error occurs @@ -53,13 +53,13 @@ public abstract class KeyValuesReader extends Reader { */ public abstract boolean next() throws IOException; - + /** * Returns the current key * @return the current key */ public abstract Object getCurrentKey() throws IOException; - + /** * Returns an Iterable view of the values associated with the current key * @return an Iterable view of the values associated with the current key diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/Partitioner.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/Partitioner.java index aa73e3bd4c..e551136401 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/Partitioner.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/Partitioner.java @@ -24,25 +24,25 @@ /** * {@link Partitioner} is used by the TEZ framework to partition output * key/value pairs. - * + * * Partitioner Initialization

    The Partitioner class is picked up * using the TEZ_RUNTIME_PARTITIONER_CLASS attribute in {@link TezRuntimeConfiguration} - * + * * TODO NEWTEZ Change construction to first check for a Constructor with a bytep[] payload - * + * * Partitioners need to provide a single argument ({@link Configuration}) * constructor or a 0 argument constructor. If both exist, preference is given * to the single argument constructor. This is primarily for MR support. - * + * * If using the configuration constructor, TEZ_RUNTIME_NUM_EXPECTED_PARTITIONS * will be set in the configuration, to indicate the max number of expected * partitions. - * + * */ @Public @Evolving public interface Partitioner { - + /** * Get partition for given key/value. * @param key key @@ -51,5 +51,5 @@ public interface Partitioner { * @return partition for the given key/value */ int getPartition(Object key, Object value, int numPartitions); - + } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java index 3bb85e33c4..dc739b755a 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java @@ -59,7 +59,7 @@ public static Class getIntermediateOutputCompressorC } // TODO Move defaults over to a constants file. - + public static boolean shouldCompressIntermediateOutput(Configuration conf) { return conf.getBoolean( TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false); @@ -70,7 +70,7 @@ public static Class getIntermediateOutputValueClass(Configuration conf) { TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, null, Object.class); } - + public static Class getIntermediateInputValueClass(Configuration conf) { return (Class) conf.getClass( TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, null, @@ -109,8 +109,8 @@ public static RawComparator getIntermediateInputKeyComparator(Configurati WritableComparable.class), conf); } - - + + // TODO Fix name public static RawComparator getInputKeySecondaryGroupingComparator( Configuration conf) { @@ -124,7 +124,7 @@ public static RawComparator getInputKeySecondaryGroupingComparator( return ReflectionUtils.newInstance(theClass, conf); } - + public static boolean useNewApi(Configuration conf) { return conf.getBoolean("mapred.mapper.new-api", false); } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/Constants.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/Constants.java index e6cf73944f..a68eaab7ee 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/Constants.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/Constants.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -23,7 +23,7 @@ public final class Constants { // TODO NEWTEZ Check which of these constants are expecting specific pieces of information which are being removed - like taskAttemptId - + public static final String TEZ = "tez"; public static final String DAG_PREFIX = "dag_"; @@ -37,7 +37,7 @@ public final class Constants { public static final long DEFAULT_COMBINE_RECORDS_BEFORE_PROGRESS = 10000; // TODO NEWTEZ Remove this constant once the old code is removed. - public static final String TEZ_RUNTIME_TASK_ATTEMPT_ID = + public static final String TEZ_RUNTIME_TASK_ATTEMPT_ID = "tez.runtime.task.attempt.id"; public static final String TEZ_RUNTIME_TASK_OUTPUT_FILENAME_STRING = @@ -47,7 +47,7 @@ public final class Constants { ".index"; public static final String TEZ_RUNTIME_TASK_INPUT_FILE_FORMAT_STRING = - "%s/task_%d.out"; + "%s/task_%d.out"; public static final String TEZ_RUNTIME_JOB_CREDENTIALS = "tez.runtime.job.credentials"; @@ -58,10 +58,10 @@ public final class Constants { @Private public static final String TEZ_RUNTIME_TASK_MEMORY = "tez.runtime.task.memory"; - + public static final String TEZ_RUNTIME_TASK_OUTPUT_DIR = "output"; - - public static final String TEZ_RUNTIME_TASK_OUTPUT_MANAGER = + + public static final String TEZ_RUNTIME_TASK_OUTPUT_MANAGER = "tez.runtime.task.local.output.manager"; private Constants() {} diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/InputAttemptIdentifier.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/InputAttemptIdentifier.java index d1d5aeda1a..8b84bca280 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/InputAttemptIdentifier.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/InputAttemptIdentifier.java @@ -81,7 +81,7 @@ public int getInputIdentifier() { public int getAttemptNumber() { return attemptNumber; } - + public String getPathComponent() { return pathComponent; } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/InputIdentifier.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/InputIdentifier.java index 095c03f7f4..cea5017401 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/InputIdentifier.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/InputIdentifier.java @@ -24,7 +24,7 @@ public class InputIdentifier { private final int inputIndex; - + public InputIdentifier(int srcInputIndex) { this.inputIndex = srcInputIndex; } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ValuesIterator.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ValuesIterator.java index 78defee4d5..5cc9548b94 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ValuesIterator.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ValuesIterator.java @@ -35,10 +35,10 @@ /** * Iterates values while keys match in sorted input. - * + * * This class is not thread safe. Accessing methods from multiple threads will * lead to corrupt data. - * + * */ @Private @@ -56,15 +56,15 @@ public class ValuesIterator { private DataInputBuffer valueIn = new DataInputBuffer(); private TezCounter inputKeyCounter; private TezCounter inputValueCounter; - + private int keyCtr = 0; private boolean hasMoreValues; // For the current key. private boolean isFirstRecord = true; private boolean completedProcessing; - - public ValuesIterator (TezRawKeyValueIterator in, - RawComparator comparator, + + public ValuesIterator (TezRawKeyValueIterator in, + RawComparator comparator, Class keyClass, Class valClass, Configuration conf, TezCounter inputKeyCounter, @@ -86,7 +86,7 @@ public ValuesIterator (TezRawKeyValueIterator in, /** * Move to the next K-Vs pair * @return true if another pair exists, otherwise false. - * @throws IOException + * @throws IOException */ public boolean moveToNext() throws IOException { if (isFirstRecord) { @@ -105,22 +105,22 @@ public boolean moveToNext() throws IOException { } /** The current key. */ - public KEY getKey() { - return key; + public KEY getKey() { + return key; } - + // TODO NEWTEZ Maybe add another method which returns an iterator instead of iterable - + public Iterable getValues() { return new Iterable() { @Override public Iterator iterator() { - + return new Iterator() { private final int keyNumber = keyCtr; - + @Override public boolean hasNext() { return hasMoreValues; @@ -135,7 +135,7 @@ public VALUE next() { .checkState( keyNumber == keyCtr, "Cannot use values iterator on the previous K-V pair after moveToNext has been invoked to move to the next K-V pair"); - + try { readNextValue(); readNextKey(); @@ -154,13 +154,13 @@ public void remove() { } }; } - - + + /** Start processing next unique key. */ private void nextKey() throws IOException { // read until we find a new key - while (hasMoreValues) { + while (hasMoreValues) { readNextKey(); } @@ -171,12 +171,12 @@ private void nextKey() throws IOException { hasMoreValues = more; } - /** + /** * read the next key - which may be the same as the current key. */ private void readNextKey() throws IOException { more = in.next(); - if (more) { + if (more) { DataInputBuffer nextKeyBytes = in.getKey(); if (!in.isSameKey()) { keyIn.reset(nextKeyBytes.getData(), nextKeyBytes.getPosition(), diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/combine/Combiner.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/combine/Combiner.java index 963ea69505..a7ac659baf 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/combine/Combiner.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/combine/Combiner.java @@ -29,8 +29,8 @@ /** *Combiner Initialization

    The Combiner class is picked up * using the TEZ_RUNTIME_COMBINER_CLASS attribute in {@link TezRuntimeConfiguration} - * - * + * + * * Partitioners need to provide a single argument ({@link TezRawKeyValueIterator}) * constructor. */ diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/readers/UnorderedKVReader.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/readers/UnorderedKVReader.java index 94ba6d8ee1..6211fdbe57 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/readers/UnorderedKVReader.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/readers/UnorderedKVReader.java @@ -48,10 +48,10 @@ public class UnorderedKVReader extends KeyValueReader { private static final Logger LOG = LoggerFactory.getLogger(UnorderedKVReader.class); - + private final ShuffleManager shuffleManager; private final CompressionCodec codec; - + private final Class keyClass; private final Class valClass; private final Deserializer keyDeserializer; @@ -62,18 +62,18 @@ public class UnorderedKVReader extends KeyValueReader { private final boolean ifileReadAhead; private final int ifileReadAheadLength; private final int ifileBufferSize; - + private final TezCounter inputRecordCounter; private final InputContext context; - + private K key; private V value; - + private FetchedInput currentFetchedInput; private IFile.Reader currentReader; - + // TODO Remove this once per I/O counters are separated properly. Relying on - // the counter at the moment will generate aggregate numbers. + // the counter at the moment will generate aggregate numbers. private int numRecordsRead = 0; private final AtomicLong totalBytesRead = new AtomicLong(0); private final AtomicLong totalFileBytes = new AtomicLong(0); @@ -106,16 +106,16 @@ public UnorderedKVReader(ShuffleManager shuffleManager, Configuration conf, } // TODO NEWTEZ Maybe add an interface to check whether next will block. - + /** * Moves to the next key/values(s) pair - * + * * @return true if another key/value(s) pair exists, false if there are no * more. * @throws IOException * if an error occurs */ - @Override + @Override public boolean next() throws IOException { if (readNextFromCurrentReader()) { inputRecordCounter.increment(1); @@ -180,11 +180,11 @@ private boolean readNextFromCurrentReader() throws IOException { return false; } } - + /** * Moves to the next available input. This method may block if the input is not ready yet. * Also takes care of closing the previous input. - * + * * @return true if the next input exists, false otherwise * @throws IOException */ diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/DiskFetchedInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/DiskFetchedInput.java index dad9f8c85c..ca2bf72953 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/DiskFetchedInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/DiskFetchedInput.java @@ -36,7 +36,7 @@ public class DiskFetchedInput extends FetchedInput { private static final Logger LOG = LoggerFactory.getLogger(DiskFetchedInput.class); - + private final FileSystem localFS; private final Path tmpOutputPath; private final Path outputPath; @@ -85,7 +85,7 @@ public final Path getInputPath() { } return this.tmpOutputPath; } - + @Override public void commit() throws IOException { if (isState(State.PENDING)) { @@ -104,7 +104,7 @@ public void abort() throws IOException { notifyFetchFailure(); } } - + @Override public void free() { Preconditions.checkState( diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInput.java index 8982c27cad..3270c2979e 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInput.java @@ -28,14 +28,14 @@ @Private public abstract class FetchedInput { - + public static enum Type { WAIT, // TODO NEWTEZ Implement this, only if required. MEMORY, DISK, DISK_DIRECT } - + protected static enum State { PENDING, COMMITTED, ABORTED, FREED } @@ -99,7 +99,7 @@ public InputAttemptIdentifier getInputAttemptIdentifier() { public void notifyFetchComplete() { this.callback.fetchComplete(this); } - + /** * Inform the Allocator about a failed resource. * This should be called by abort @@ -107,7 +107,7 @@ public void notifyFetchComplete() { public void notifyFetchFailure() { this.callback.fetchFailed(this); } - + /** * Inform the Allocator about a completed resource being released. * This should be called by free @@ -115,7 +115,7 @@ public void notifyFetchFailure() { public void notifyFreedResource() { this.callback.freeResources(this); } - + /** * Returns the output stream to be used to write fetched data. Users are * expected to close the OutputStream when they're done @@ -144,7 +144,7 @@ public void notifyFreedResource() { * reclaimed. */ public abstract void free(); - + @Override public int hashCode() { return id; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInputAllocator.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInputAllocator.java index 74fa26a454..3c4ae7e726 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInputAllocator.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInputAllocator.java @@ -27,8 +27,8 @@ public interface FetchedInputAllocator { public FetchedInput allocate(long actualSize, long compresedSize, InputAttemptIdentifier inputAttemptIdentifier) throws IOException; - + public FetchedInput allocateType(Type type, long actualSize, long compresedSize, InputAttemptIdentifier inputAttemptIdentifier) throws IOException; - + } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInputCallback.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInputCallback.java index a89604169d..441b5ac8b3 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInputCallback.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInputCallback.java @@ -19,11 +19,11 @@ package org.apache.tez.runtime.library.common.shuffle; public interface FetchedInputCallback { - + public void fetchComplete(FetchedInput fetchedInput); - + public void fetchFailed(FetchedInput fetchedInput); - + public void freeResources(FetchedInput fetchedInput); - + } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetcherCallback.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetcherCallback.java index b751fb9ce0..28e01d6fba 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetcherCallback.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetcherCallback.java @@ -27,7 +27,7 @@ public interface FetcherCallback { public void fetchSucceeded(String host, InputAttemptIdentifier srcAttemptIdentifier, FetchedInput fetchedInput, long fetchedBytes, long decompressedLength, long copyDuration) throws IOException; - + public void fetchFailed(String host, InputAttemptFetchFailure srcAttemptFetchFailure, boolean connectFailed); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java index 16f33a63be..4b753956c4 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java @@ -29,7 +29,7 @@ /** * Represents a Host with respect to the MapReduce ShuffleHandler. - * + * */ public class InputHost extends HostPort { @@ -122,7 +122,7 @@ public String toDetailedString() { return "HostPort=" + super.toString() + ", InputDetails=" + partitionToInputs; } - + @Override public String toString() { return "HostPort=" + super.toString() + ", PartitionIds=" + diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/LocalDiskFetchedInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/LocalDiskFetchedInput.java index 64cc4c4117..8159865e0a 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/LocalDiskFetchedInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/LocalDiskFetchedInput.java @@ -91,7 +91,7 @@ public void abort() { notifyFetchFailure(); } } - + @Override public void free() { Preconditions.checkState( diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/MemoryFetchedInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/MemoryFetchedInput.java index b6681a1552..1021712779 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/MemoryFetchedInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/MemoryFetchedInput.java @@ -63,7 +63,7 @@ public InputStream getInputStream() { public byte[] getBytes() { return byteArray; } - + @Override public void commit() { if (isState(State.PENDING)) { @@ -79,7 +79,7 @@ public void abort() { notifyFetchFailure(); } } - + @Override public void free() { Preconditions.checkState( diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java index 6d68f424f1..fdc15dd85e 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java @@ -144,7 +144,7 @@ public static void shuffleToMemory(byte[] shuffleData, } } } - + public static void shuffleToDisk(OutputStream output, String hostIdentifier, InputStream input, long compressedLength, long decompressedLength, Logger LOG, InputAttemptIdentifier identifier, boolean ifileReadAhead, int ifileReadAheadLength, boolean verifyChecksum) throws IOException { @@ -188,8 +188,8 @@ public static void shuffleToDisk(OutputStream output, String hostIdentifier, if (bytesLeft != 0) { throw new IOException("Incomplete map output received for " + identifier + " from " + - hostIdentifier + " (" + - bytesLeft + " bytes missing of " + + hostIdentifier + " (" + + bytesLeft + " bytes missing of " + compressedLength + ")"); } } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java index ef646e238a..d46cd86269 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java @@ -57,7 +57,7 @@ public class ShuffleInputEventHandlerImpl implements ShuffleEventHandler { private static final Logger LOG = LoggerFactory.getLogger(ShuffleInputEventHandlerImpl.class); - + private final ShuffleManager shuffleManager; //TODO: unused. Consider removing later? private final FetchedInputAllocator inputAllocator; @@ -97,7 +97,7 @@ public void handleEvents(List events) throws IOException { handleEvent(event); } } - + private void handleEvent(Event event) throws IOException { if (event instanceof DataMovementEvent) { numDmeEvents.incrementAndGet(); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java index 9d2c02b7d1..34112814ae 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java @@ -153,7 +153,7 @@ public class ShuffleManager implements FetcherCallback { private final BlockingQueue pendingHosts; private final Set obsoletedInputs; private Set runningFetchers; - + private final AtomicInteger numCompletedInputs = new AtomicInteger(0); private final AtomicInteger numFetchedSpills = new AtomicInteger(0); @@ -164,10 +164,10 @@ public class ShuffleManager implements FetcherCallback { // Required to be held when manipulating pendingHosts private final ReentrantLock lock = new ReentrantLock(); private final Condition wakeLoop = lock.newCondition(); - + private final int numFetchers; private final boolean asyncHttp; - + // Parameters required by Fetchers private final JobTokenSecretManager jobTokenSecretMgr; private final CompressionCodec codec; @@ -185,7 +185,7 @@ public class ShuffleManager implements FetcherCallback { * Holds the time to wait for failures to batch them and send less events. */ private final int maxTimeToWaitForReportMillis; - + private final String sourceDestNameTrimmed; private final int maxTaskOutputAtOnce; @@ -202,10 +202,10 @@ public class ShuffleManager implements FetcherCallback { private final TezCounter bytesShuffledToDiskCounter; private final TezCounter bytesShuffledToMemCounter; private final TezCounter bytesShuffledDirectDiskCounter; - + private volatile Throwable shuffleError; private final HttpConnectionParams httpConnectionParams; - + private final LocalDirAllocator localDirAllocator; private final RawLocalFileSystem localFs; @@ -222,7 +222,7 @@ public class ShuffleManager implements FetcherCallback { final Map shuffleInfoEventsMap; // TODO More counters - FetchErrors, speed? - + public ShuffleManager(InputContext inputContext, Configuration conf, int numInputs, int bufferSize, boolean ifileReadAheadEnabled, int ifileReadAheadLength, CompressionCodec codec, FetchedInputAllocator inputAllocator) throws IOException { @@ -260,7 +260,7 @@ public ShuffleManager(InputContext inputContext, Configuration conf, int numInpu this.firstEventReceived = inputContext.getCounters().findCounter(TaskCounter.FIRST_EVENT_RECEIVED); this.lastEventReceived = inputContext.getCounters().findCounter(TaskCounter.LAST_EVENT_RECEIVED); this.compositeFetch = ShuffleUtils.isTezShuffleHandler(conf); - + this.enableFetcherTestingErrors = conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_ENABLE_TESTING_ERRORS, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_ENABLE_TESTING_ERRORS_DEFAULT); @@ -279,11 +279,11 @@ public ShuffleManager(InputContext inputContext, Configuration conf, int numInpu obsoletedInputs = Collections.newSetFromMap(new ConcurrentHashMap()); runningFetchers = Collections.newSetFromMap(new ConcurrentHashMap()); - int maxConfiguredFetchers = + int maxConfiguredFetchers = conf.getInt( TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES_DEFAULT); - + this.numFetchers = Math.min(maxConfiguredFetchers, numInputs); final ExecutorService fetcherRawExecutor; @@ -301,7 +301,7 @@ public ShuffleManager(InputContext inputContext, Configuration conf, int numInpu .setDaemon(true).setNameFormat("ShuffleRunner {" + sourceDestNameTrimmed + "}").build()); this.schedulerExecutor = MoreExecutors.listeningDecorator(schedulerRawExecutor); this.schedulerCallable = new RunShuffleCallable(conf); - + this.startTime = System.currentTimeMillis(); this.lastProgressTime = startTime; @@ -621,9 +621,9 @@ Fetcher constructFetcherForHost(InputHost inputHost, Configuration conf) { } return fetcherBuilder.build(); } - + /////////////////// Methods for InputEventHandler - + public void addKnownInput(String hostName, int port, CompositeInputAttemptIdentifier srcAttemptIdentifier, int srcPhysicalIndex) { HostPort identifier = new HostPort(hostName, port); @@ -1176,17 +1176,17 @@ public void onFailure(Throwable t) { inputContext.reportFailure(TaskFailureType.NON_FATAL, t, "Shuffle Scheduler Failed"); } } - + } - + private class FetchFutureCallback implements FutureCallback { private final Fetcher fetcher; - + public FetchFutureCallback(Fetcher fetcher) { this.fetcher = fetcher; } - + private void doBookKeepingForFetcherComplete() { lock.lock(); try { @@ -1196,7 +1196,7 @@ private void doBookKeepingForFetcherComplete() { lock.unlock(); } } - + @Override public void onSuccess(FetchResult result) { fetcher.shutdown(); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/SimpleFetchedInputAllocator.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/SimpleFetchedInputAllocator.java index 7eaff42a16..aaa9014b8e 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/SimpleFetchedInputAllocator.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/SimpleFetchedInputAllocator.java @@ -51,7 +51,7 @@ public class SimpleFetchedInputAllocator implements FetchedInputAllocator, FetchedInputCallback { private static final Logger LOG = LoggerFactory.getLogger(SimpleFetchedInputAllocator.class); - + private final Configuration conf; private final TezTaskOutputFiles fileNameAllocator; @@ -66,7 +66,7 @@ public class SimpleFetchedInputAllocator implements FetchedInputAllocator, private final long initialMemoryAvailable; private final String srcNameTrimmed; - + private volatile long usedMemory = 0; public SimpleFetchedInputAllocator(String srcNameTrimmed, @@ -75,14 +75,14 @@ public SimpleFetchedInputAllocator(String srcNameTrimmed, long maxTaskAvailableMemory, long memoryAvailable) { this.srcNameTrimmed = srcNameTrimmed; - this.conf = conf; + this.conf = conf; this.maxAvailableTaskMemory = maxTaskAvailableMemory; this.initialMemoryAvailable = memoryAvailable; - + this.fileNameAllocator = new TezTaskOutputFiles(conf, uniqueIdentifier, dagID); this.localDirAllocator = new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS); - + // Setup configuration final float maxInMemCopyUse = conf.getFloat( TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, @@ -92,10 +92,10 @@ public SimpleFetchedInputAllocator(String srcNameTrimmed, + TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT + ": " + maxInMemCopyUse); } - + long memReq = (long) (conf.getLong(Constants.TEZ_RUNTIME_TASK_MEMORY, Math.min(maxAvailableTaskMemory, Integer.MAX_VALUE)) * maxInMemCopyUse); - + if (memReq <= this.initialMemoryAvailable) { this.memoryLimit = memReq; } else { diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapHost.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapHost.java index 4746306950..23b9d3a1c7 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapHost.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapHost.java @@ -89,7 +89,7 @@ public String toString() { private final int partitionCount; // Tracks attempt IDs private List maps = new ArrayList(); - + public MapHost(String host, int port, int partition, int partitionCount) { this.host = host; this.port = port; @@ -133,15 +133,15 @@ public synchronized List getAndClearKnownMaps() { maps = new ArrayList(); return currentKnownMaps; } - + public synchronized void markBusy() { state = State.BUSY; } - + public synchronized void markPenalized() { state = State.PENALIZED; } - + public synchronized int getNumKnownMapOutputs() { return maps.size(); } @@ -158,12 +158,12 @@ public synchronized State markAvailable() { } return state; } - + @Override public String toString() { return getHostIdentifier(); } - + /** * Mark the host as penalized */ diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java index 249d0adc28..893cbefa17 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java @@ -36,7 +36,7 @@ abstract class MapOutput { private static final Logger LOG = LoggerFactory.getLogger(MapOutput.class); private static AtomicInteger ID = new AtomicInteger(0); - + public enum Type { WAIT, MEMORY, @@ -119,7 +119,7 @@ public FileChunk getOutputPath() { public byte[] getMemory() { return null; } - + public OutputStream getDisk() { return null; } @@ -136,27 +136,27 @@ public long getSize() { public void commit() throws IOException { } - + public void abort() { } - + public String toString() { return "MapOutput( AttemptIdentifier: " + attemptIdentifier + ", Type: " + getType() + ")"; } - - public static class MapOutputComparator + + public static class MapOutputComparator implements Comparator { public int compare(MapOutput o1, MapOutput o2) { - if (o1.id == o2.id) { + if (o1.id == o2.id) { return 0; } - + if (o1.getSize() < o2.getSize()) { return -1; } else if (o1.getSize() > o2.getSize()) { return 1; } - + if (o1.id < o2.id) { return -1; } else { diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeThread.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeThread.java index c0af90fde0..d969aeeb8f 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeThread.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeThread.java @@ -27,7 +27,7 @@ import org.slf4j.LoggerFactory; abstract class MergeThread extends Thread { - + private static final Logger LOG = LoggerFactory.getLogger(MergeThread.class); private volatile boolean inProgress = false; @@ -38,14 +38,14 @@ abstract class MergeThread extends Thread { private final int mergeFactor; private Thread shuffleSchedulerThread; - + public MergeThread(MergeManager manager, int mergeFactor, ExceptionReporter reporter) { this.manager = manager; this.mergeFactor = mergeFactor; this.reporter = reporter; } - + public synchronized void close() throws InterruptedException { closed = true; if (!Thread.currentThread().isInterrupted()) { @@ -69,7 +69,7 @@ public void setParentThread(Thread shuffleSchedulerThread) { public synchronized boolean isInProgress() { return inProgress; } - + public synchronized void startMerge(Set inputs) { if (!closed) { this.inputs.clear(); @@ -79,7 +79,7 @@ public synchronized void startMerge(Set inputs) { this.inputs.add(iter.next()); iter.remove(); } - LOG.info(getName() + ": Starting merge with " + this.inputs.size() + + LOG.info(getName() + ": Starting merge with " + this.inputs.size() + " segments, while ignoring " + inputs.size() + " segments"); notifyAll(); } @@ -122,14 +122,14 @@ public void run() { synchronized (this) { // Clear inputs inputs.clear(); - inProgress = false; + inProgress = false; notifyAll(); } } } } - public abstract void merge(List inputs) + public abstract void merge(List inputs) throws IOException, InterruptedException; public abstract void cleanup(List inputs, boolean deleteData) diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleHeader.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleHeader.java index f074e897e8..67ac2385bb 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleHeader.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleHeader.java @@ -28,14 +28,14 @@ import org.apache.hadoop.io.WritableUtils; /** - * Shuffle Header information that is sent by the TaskTracker and + * Shuffle Header information that is sent by the TaskTracker and * deciphered by the Fetcher thread of Reduce task * */ @InterfaceAudience.Private @InterfaceStability.Stable public class ShuffleHeader implements Writable { - + /** Header info of the shuffle http request/response */ public static final String HTTP_HEADER_NAME = "name"; public static final String DEFAULT_HTTP_HEADER_NAME = "mapreduce"; @@ -51,9 +51,9 @@ public class ShuffleHeader implements Writable { long uncompressedLength; long compressedLength; int forReduce; - + public ShuffleHeader() { } - + public ShuffleHeader(String mapId, long compressedLength, long uncompressedLength, int forReduce) { this.mapId = mapId; @@ -61,15 +61,15 @@ public ShuffleHeader(String mapId, long compressedLength, this.uncompressedLength = uncompressedLength; this.forReduce = forReduce; } - + public String getMapId() { return this.mapId; } - + public int getPartition() { return this.forReduce; } - + public long getUncompressedLength() { return uncompressedLength; } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java index 268e78865d..b99f06d9dc 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java @@ -111,7 +111,7 @@ public void progress() { protected final SerializationContext serializationContext; protected final Serializer keySerializer; protected final Serializer valSerializer; - + protected final boolean ifileReadAhead; protected final int ifileReadAheadLength; protected final int ifileBufferSize; @@ -159,7 +159,7 @@ public void progress() { // additional spills. Compressed size - so may not represent the size in the // sort buffer) protected final TezCounter additionalSpillBytesWritten; - + protected final TezCounter additionalSpillBytesRead; // Number of spills written & consumed by the same task to generate the final file protected final TezCounter numAdditionalSpills; @@ -213,7 +213,7 @@ public ExternalSorter(OutputContext outputContext, Configuration conf, int numOu + ", serialization=" + conf.get(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY) + ", reportPartitionStats=" + reportPartitionStats); - // counters + // counters mapOutputByteCounter = outputContext.getCounters().findCounter(TaskCounter.OUTPUT_BYTES); mapOutputRecordCounter = outputContext.getCounters().findCounter(TaskCounter.OUTPUT_RECORDS); outputBytesWithOverheadCounter = outputContext.getCounters().findCounter(TaskCounter.OUTPUT_BYTES_WITH_OVERHEAD); @@ -240,7 +240,7 @@ public ExternalSorter(OutputContext outputContext, Configuration conf, int numOu this.ifileBufferSize = conf.getInt("io.file.buffer.size", TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_BUFFER_SIZE_DEFAULT); - + // Task outputs mapOutputFile = TezRuntimeUtils.instantiateTaskOutputManager(conf, outputContext); @@ -328,9 +328,9 @@ public ShuffleHeader getShuffleHeader(int reduce) { } public static long getInitialMemoryRequirement(Configuration conf, long maxAvailableTaskMemory) { - int initialMemRequestMb = + int initialMemRequestMb = conf.getInt( - TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, + TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB_DEFAULT); long reqBytes = ((long) initialMemRequestMb) << 20; //Higher bound checks are done in individual sorter implementations diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java index 2ff4b2a078..56fe0db644 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java @@ -58,7 +58,7 @@ * IFile is the simple format * for the intermediate map-outputs in Map-Reduce. * - * There is a Writer to write out map-outputs in this format and + * There is a Writer to write out map-outputs in this format and * a Reader to read files of this format. */ @InterfaceAudience.Private diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java index 3e9a163289..35836fb50f 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java @@ -72,9 +72,9 @@ @SuppressWarnings({"unchecked", "rawtypes"}) public class PipelinedSorter extends ExternalSorter { - + private static final Logger LOG = LoggerFactory.getLogger(PipelinedSorter.class); - + /** * The size of each record in the index file for the map-outputs. */ @@ -83,7 +83,7 @@ public class PipelinedSorter extends ExternalSorter { private final static int APPROX_HEADER_LENGTH = 150; private final int partitionBits; - + private static final int PARTITION = 0; // partition offset in acct private static final int KEYSTART = 1; // key offset in acct private static final int VALSTART = 2; // val offset in acct @@ -93,7 +93,7 @@ public class PipelinedSorter extends ExternalSorter { private final int minSpillsForCombine; private final ProxyComparator hasher; - // SortSpans + // SortSpans private SortSpan span; //total memory capacity allocated to sorter @@ -103,7 +103,7 @@ public class PipelinedSorter extends ExternalSorter { private int bufferOverflowRecursion; // Merger - private final SpanMerger merger; + private final SpanMerger merger; private final ExecutorService sortmaster; private final ArrayList indexCacheList = @@ -228,7 +228,7 @@ public PipelinedSorter(OutputContext outputContext, Configuration conf, int numO span = new SortSpan(buffers.get(bufferIndex), 1024 * 1024, 16, this.comparator); merger = new SpanMerger(); // SpanIterators are comparable - final int sortThreads = + final int sortThreads = this.conf.getInt( TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS, TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS_DEFAULT); @@ -321,7 +321,7 @@ private int bitcount(int n) { } return bit; } - + public void sort() throws IOException { SortSpan newSpan = span.next(); @@ -424,7 +424,7 @@ synchronized void collect(Object key, Object value, final int partition int valend = -1; try { keySerializer.serialize(key); - valstart = span.kvbuffer.position(); + valstart = span.kvbuffer.position(); valSerializer.serialize(value); valend = span.kvbuffer.position(); } catch(BufferOverflowException overflow) { @@ -602,7 +602,7 @@ public boolean spill(boolean ignoreEmptySpills) throws IOException { while (kvIter.next()) { writer.append(kvIter.getKey(), kvIter.getValue()); } - } else { + } else { if (hasNext) { runCombineProcessor(kvIter, writer); } @@ -880,7 +880,7 @@ private static class BufferStreamWrapper extends OutputStream public BufferStreamWrapper(ByteBuffer out) { this.out = out; } - + @Override public void write(int b) throws IOException { out.put((byte)b); } @Override @@ -890,7 +890,7 @@ public BufferStreamWrapper(ByteBuffer out) { } private static final class InputByteBuffer extends DataInputBuffer { - private byte[] buffer = new byte[256]; + private byte[] buffer = new byte[256]; private ByteBuffer wrapped = ByteBuffer.wrap(buffer); private void resize(int length) { if(length > buffer.length || (buffer.length > 10 * (1+length))) { @@ -1015,7 +1015,7 @@ public int compare(final int mi, final int mj) { final int kvj = offsetFor(mj); final int kvip = kvmeta.get(kvi + PARTITION); final int kvjp = kvmeta.get(kvj + PARTITION); - // sort by partition + // sort by partition if (kvip != kvjp) { return kvip - kvjp; } @@ -1102,11 +1102,11 @@ public int compareInternal(final DataInputBuffer needle, final int needlePart, f } return cmp; } - + public long getEq() { return eq; } - + @Override public String toString() { return String.format("Span[%d,%d]", NMETA*kvmeta.capacity(), kvbuffer.limit()); @@ -1151,7 +1151,7 @@ public DataInputBuffer getValue() { } public boolean next() { - // caveat: since we use this as a comparable in the merger + // caveat: since we use this as a comparable in the merger if(kvindex == maxindex) return false; kvindex += 1; if(kvindex % 100 == 0) { @@ -1168,7 +1168,7 @@ public boolean hasNext() { public void close() { } - public Progress getProgress() { + public Progress getProgress() { return progress; } @@ -1208,7 +1208,7 @@ public String toString() { * bisect returns the next insertion point for a given raw key, skipping keys * which are <= needle using a binary search instead of a linear comparison. * This is massively efficient when long strings of identical keys occur. - * @param needle + * @param needle * @param needlePart * @return */ @@ -1225,8 +1225,8 @@ int bisect(DataInputBuffer needle, int needlePart) { if(span.compareInternal(needle, needlePart, start) > 0) { return kvindex; } - - // bail out early if we haven't got a min run + + // bail out early if we haven't got a min run if(span.compareInternal(needle, needlePart, start+minrun) > 0) { return 0; } @@ -1234,9 +1234,9 @@ int bisect(DataInputBuffer needle, int needlePart) { if(span.compareInternal(needle, needlePart, end) < 0) { return end - kvindex; } - + boolean found = false; - + // we sort 100k items, the max it can do is 20 loops, but break early for(int i = 0; start < end && i < 16; i++) { mid = start + (end - start)/2; @@ -1245,7 +1245,7 @@ int bisect(DataInputBuffer needle, int needlePart) { start = mid; found = true; } else if(cmp < 0) { - start = mid; + start = mid; found = true; } if(cmp > 0) { @@ -1295,7 +1295,7 @@ public boolean isSameKey() throws IOException { } public boolean next() throws IOException { - if(dirty || iter.next()) { + if(dirty || iter.next()) { int prefix = iter.getPartition(); if((prefix >>> (32 - partitionBits)) == partition) { @@ -1342,7 +1342,7 @@ public SpanHeap() { super(256); } /** - * {@link PriorityQueue}.poll() by a different name + * {@link PriorityQueue}.poll() by a different name * @return */ public SpanIterator pop() { @@ -1370,7 +1370,7 @@ private final class SpanMerger implements PartitionedRawKeyValueIterator { private SpanIterator horse; private long total = 0; private long eq = 0; - + public SpanMerger() { // SpanIterators are comparable partIter = new PartitionFilter(this); @@ -1432,7 +1432,7 @@ private SpanIterator pop() { horse = current; return current; } - + public boolean needsRLE() { return (eq > 0.1 * total); } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezRawKeyValueIterator.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezRawKeyValueIterator.java index 683c9b9ce9..4de70b0456 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezRawKeyValueIterator.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezRawKeyValueIterator.java @@ -26,32 +26,32 @@ /** * TezRawKeyValueIterator is an iterator used to iterate over - * the raw keys and values during sort/merge of intermediate data. + * the raw keys and values during sort/merge of intermediate data. */ @InterfaceAudience.Private @InterfaceStability.Unstable public interface TezRawKeyValueIterator { - /** + /** * Gets the current raw key. - * + * * @return Gets the current raw key as a DataInputBuffer * @throws IOException */ DataInputBuffer getKey() throws IOException; - - /** + + /** * Gets the current raw value. - * - * @return Gets the current raw value as a DataInputBuffer + * + * @return Gets the current raw value as a DataInputBuffer * @throws IOException */ DataInputBuffer getValue() throws IOException; - - /** + + /** * Sets up the current key and value (for getKey and getValue). - * - * @return true if there exists a key/value, - * false otherwise. + * + * @return true if there exists a key/value, + * false otherwise. * @throws IOException */ boolean next() throws IOException; @@ -65,14 +65,14 @@ public interface TezRawKeyValueIterator { */ boolean hasNext() throws IOException; - /** + /** * Closes the iterator so that the underlying streams can be closed. - * + * * @throws IOException */ void close() throws IOException; - - /** Gets the Progress object; this has a float (0.0 - 1.0) + + /** Gets the Progress object; this has a float (0.0 - 1.0) * indicating the bytes processed by the iterator so far */ Progress getProgress(); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java index 0ab16c14ed..5faaab7233 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java @@ -68,11 +68,11 @@ @SuppressWarnings({"unchecked", "rawtypes"}) public final class DefaultSorter extends ExternalSorter implements IndexedSortable { - + private static final Logger LOG = LoggerFactory.getLogger(DefaultSorter.class); // TODO NEWTEZ Progress reporting to Tez framework. (making progress vs %complete) - + /** * The size of each record in the index file for the map-outputs. */ @@ -440,7 +440,7 @@ int distanceTo(final int i, final int j, final int mod) { */ int offsetFor(int metapos) { return (metapos % maxRec) * NMETA; - + } /** diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutput.java index 414f3d060f..c88cde83bc 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutput.java @@ -67,7 +67,7 @@ public TezTaskOutput(Configuration conf, String uniqueId, int dagID) { /** * Create a local output file name. This method is meant to be used *only* if * the size of the file is not know up front. - * + * * @return path the path to write to * @throws IOException */ diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/BaseUnorderedPartitionedKVWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/BaseUnorderedPartitionedKVWriter.java index e33db95532..0ebf24ddc8 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/BaseUnorderedPartitionedKVWriter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/BaseUnorderedPartitionedKVWriter.java @@ -48,7 +48,7 @@ public abstract class BaseUnorderedPartitionedKVWriter extends KeyValuesWriter { private static final Logger LOG = LoggerFactory.getLogger(BaseUnorderedPartitionedKVWriter.class); - + protected final OutputContext outputContext; protected final Configuration conf; protected final RawLocalFileSystem localFs; @@ -63,7 +63,7 @@ public abstract class BaseUnorderedPartitionedKVWriter extends KeyValuesWriter { protected final int numPartitions; protected final CompressionCodec codec; protected final TezTaskOutput outputFileHandler; - + protected final boolean ifileReadAhead; protected final int ifileReadAheadLength; protected final int ifileBufferSize; @@ -83,7 +83,7 @@ public abstract class BaseUnorderedPartitionedKVWriter extends KeyValuesWriter { * meta-information. */ protected final TezCounter outputBytesWithOverheadCounter; - + /** * Represents the final output size, with file format overheads and compression factored in. * Does not consider spills. @@ -122,7 +122,7 @@ public BaseUnorderedPartitionedKVWriter(OutputContext outputContext, Configurati throw new RuntimeException(e); } this.numPartitions = numOutputs; - + // k/v serialization keyClass = ConfigUtils.getIntermediateOutputKeyClass(this.conf); valClass = ConfigUtils.getIntermediateOutputValueClass(this.conf); @@ -131,7 +131,7 @@ public BaseUnorderedPartitionedKVWriter(OutputContext outputContext, Configurati valSerialization = serializationFactory.getSerialization(valClass); keySerializer = keySerialization.getSerializer(keyClass); valSerializer = valSerialization.getSerializer(valClass); - + outputRecordBytesCounter = outputContext.getCounters().findCounter(TaskCounter.OUTPUT_BYTES); outputRecordsCounter = outputContext.getCounters().findCounter(TaskCounter.OUTPUT_RECORDS); outputBytesWithOverheadCounter = outputContext.getCounters().findCounter(TaskCounter.OUTPUT_BYTES_WITH_OVERHEAD); @@ -161,7 +161,7 @@ public BaseUnorderedPartitionedKVWriter(OutputContext outputContext, Configurati } this.ifileBufferSize = conf.getInt("io.file.buffer.size", TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_BUFFER_SIZE_DEFAULT); - + LOG.info("Instantiating Partitioner: [" + conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS) + "]"); try { this.partitioner = TezRuntimeUtils.instantiatePartitioner(this.conf); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java index 95684cfda3..92ef593b18 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java @@ -44,7 +44,7 @@ @InterfaceStability.Evolving /** * Configure {@link org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput}

    - * + * * Values will be picked up from tez-site if not specified, otherwise defaults from * {@link org.apache.tez.runtime.library.api.TezRuntimeConfiguration} will be used. */ diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValueInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValueInput.java index ef247e8424..542b44a8ce 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValueInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValueInput.java @@ -33,7 +33,7 @@ /** * Implements a {@link MergedLogicalInput} that merges the incoming inputs - * (e.g. from a {@link GroupInputEdge} and provide a unified view of the + * (e.g. from a {@link GroupInputEdge} and provide a unified view of the * input. It concatenates all the inputs to provide a unified view */ @Public @@ -96,7 +96,7 @@ public float getProgress() throws IOException, InterruptedException { } /** - * Provides a {@link KeyValueReader} that iterates over the + * Provides a {@link KeyValueReader} that iterates over the * concatenated input data */ @Override diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValuesInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValuesInput.java index 0034425cc1..aceb489bb5 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValuesInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValuesInput.java @@ -33,7 +33,7 @@ /** * Implements a {@link MergedLogicalInput} that merges the incoming inputs - * (e.g. from a {@link GroupInputEdge} and provide a unified view of the + * (e.g. from a {@link GroupInputEdge} and provide a unified view of the * input. It concatenates all the inputs to provide a unified view */ @@ -96,9 +96,9 @@ public float getProgress() throws IOException, InterruptedException { return (1.0f)*(currentReaderIndex + 1)/getInputs().size(); } } - + /** - * Provides a {@link KeyValuesReader} that iterates over the + * Provides a {@link KeyValuesReader} that iterates over the * concatenated input data */ @Override diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java index 3908bd038c..dd19031009 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java @@ -62,8 +62,8 @@ * {@link OrderedGroupedKVInput} in a {@link AbstractLogicalInput} which shuffles * intermediate sorted data, merges them and provides key/ to the * consumer. This is typically used to bring one partition of a set of partitioned - * distributed data to one consumer. The shuffle operation brings all partitions - * to one place. These partitions are assumed to be sorted and are merged sorted to + * distributed data to one consumer. The shuffle operation brings all partitions + * to one place. These partitions are assumed to be sorted and are merged sorted to * merge them into a single input view. * * The Copy and Merge will be triggered by the initialization - which is handled @@ -198,14 +198,14 @@ public synchronized List close() throws IOException { if (shuffle != null) { shuffle.shutdown(); } - + long dataSize = getContext().getCounters() .findCounter(TaskCounter.SHUFFLE_BYTES_DECOMPRESSED).getValue(); getContext().getStatisticsReporter().reportDataSize(dataSize); long inputRecords = getContext().getCounters() .findCounter(TaskCounter.REDUCE_INPUT_RECORDS).getValue(); getContext().getStatisticsReporter().reportItemsProcessed(inputRecords); - + return Collections.emptyList(); } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedMergedKVInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedMergedKVInput.java index b85d5563a2..a3bf658d26 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedMergedKVInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedMergedKVInput.java @@ -42,7 +42,7 @@ * A {@link MergedLogicalInput} which merges multiple * {@link OrderedGroupedKVInput}s and returns a single view of these by merging * values which belong to the same key. - * + * * Combiners and Secondary Sort are not implemented, so there is no guarantee on * the order of values. */ @@ -85,7 +85,7 @@ private static class OrderedGroupedMergedKeyValuesReader extends KeyValuesReader private Object currentKey; private final MergedInputContext context; - public OrderedGroupedMergedKeyValuesReader(List inputs, MergedInputContext context) + public OrderedGroupedMergedKeyValuesReader(List inputs, MergedInputContext context) throws Exception { keyComparator = ((OrderedGroupedKVInput) inputs.get(0)) .getInputKeyComparator(); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java index 4e7fbd0c5d..5351bc8d8d 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java @@ -66,7 +66,7 @@ public class UnorderedKVOutput extends AbstractLogicalOutput { @VisibleForTesting Configuration conf; - + private MemoryUpdateCallbackHandler memoryUpdateCallbackHandler; private final AtomicBoolean isStarted = new AtomicBoolean(false); @@ -95,7 +95,7 @@ public synchronized List initialize() UnorderedPartitionedKVWriter.getInitialMemoryRequirement(conf, getContext() .getTotalMemoryAvailableToTask()) : 0; getContext().requestInitialMemory(memRequestSize, memoryUpdateCallbackHandler); - + return Collections.emptyList(); } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/PreWarmProcessor.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/PreWarmProcessor.java index 4103078dd1..a03efc7421 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/PreWarmProcessor.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/PreWarmProcessor.java @@ -43,17 +43,17 @@ public void run() throws Exception { preWarmTezCode(); preWarmUserCode(); } - + /** * Pre-warm Tez code. Users can override this with an empty method * to not pre-warm Tez code if they want to. */ protected void preWarmTezCode() { // Do nothing. Can potentially pre-warm Tez library components - + // Currently, must sleep for some time so that container re-use - // can be prevented from kicking in. This will allow sufficient - // time to obtain containers from YARN as long as those resources + // can be prevented from kicking in. This will allow sufficient + // time to obtain containers from YARN as long as those resources // are available try { Thread.sleep(3000); @@ -61,10 +61,10 @@ protected void preWarmTezCode() { e.printStackTrace(); } } - + /** - * Pre-warm user code. Users can override this - * to pre-warm their own code if they want to. + * Pre-warm user code. Users can override this + * to pre-warm their own code if they want to. */ protected void preWarmUserCode() { // Do nothing diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/resources/WeightedScalingMemoryDistributor.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/resources/WeightedScalingMemoryDistributor.java index 9b6f21754d..fe90ff2573 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/resources/WeightedScalingMemoryDistributor.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/resources/WeightedScalingMemoryDistributor.java @@ -100,7 +100,7 @@ public Iterable assignMemory(long availableForAllocation, int numTotalInpu } if (numRequestsScaled == 0) { - // Fall back to regular scaling. e.g. BROADCAST : SHUFFLE = 0:1. + // Fall back to regular scaling. e.g. BROADCAST : SHUFFLE = 0:1. // i.e. if Shuffle present, Broadcast gets nothing, but otherwise it // should get an allocation numRequestsScaled = numRequests; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java index 578dd14c46..0f5bcd8224 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java @@ -48,10 +48,10 @@ @SuppressWarnings("unchecked") public class TestInputReadyVertexManager { - + @Captor ArgumentCaptor> requestCaptor; - + @Before public void init() { MockitoAnnotations.initMocks(this); @@ -59,7 +59,7 @@ public void init() { @Test (timeout=5000) public void testBasicScatterGather() throws Exception { - HashMap mockInputVertices = + HashMap mockInputVertices = new HashMap(); String mockSrcVertexId1 = "Vertex1"; EdgeProperty eProp1 = EdgeProperty.create( @@ -68,9 +68,9 @@ public void testBasicScatterGather() throws Exception { SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in")); - + String mockManagedVertexId = "Vertex"; - + VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class); when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices); when(mockContext.getVertexName()).thenReturn(mockManagedVertexId); @@ -96,10 +96,10 @@ public void testBasicScatterGather() throws Exception { verify(mockContext, times(1)).scheduleTasks(requestCaptor.capture()); Assert.assertEquals(2, requestCaptor.getValue().size()); } - + @Test (timeout=5000) public void testBasicOneToOne() throws Exception { - HashMap mockInputVertices = + HashMap mockInputVertices = new HashMap(); String mockSrcVertexId1 = "Vertex1"; EdgeProperty eProp1 = EdgeProperty.create( @@ -108,16 +108,16 @@ public void testBasicOneToOne() throws Exception { SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in")); - + String mockManagedVertexId = "Vertex"; - + VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class); when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices); when(mockContext.getVertexName()).thenReturn(mockManagedVertexId); when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(3); when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(3); mockInputVertices.put(mockSrcVertexId1, eProp1); - + InputReadyVertexManager manager = new InputReadyVertexManager(mockContext); manager.initialize(); verify(mockContext, times(1)).vertexReconfigurationPlanned(); @@ -153,10 +153,10 @@ public void testBasicOneToOne() throws Exception { Assert.assertEquals(2, requestCaptor.getValue().get(0) .getTaskLocationHint().getAffinitizedTask().getTaskIndex()); } - + @Test (timeout=5000) public void testDelayedConfigureOneToOne() throws Exception { - HashMap mockInputVertices = + HashMap mockInputVertices = new HashMap(); String mockSrcVertexId1 = "Vertex1"; EdgeProperty eProp1 = EdgeProperty.create( @@ -165,16 +165,16 @@ public void testDelayedConfigureOneToOne() throws Exception { SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in")); - + String mockManagedVertexId = "Vertex"; - + VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class); when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices); when(mockContext.getVertexName()).thenReturn(mockManagedVertexId); when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(3); when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(3); mockInputVertices.put(mockSrcVertexId1, eProp1); - + InputReadyVertexManager manager = new InputReadyVertexManager(mockContext); manager.initialize(); verify(mockContext, times(1)).vertexReconfigurationPlanned(); @@ -190,7 +190,7 @@ public void testDelayedConfigureOneToOne() throws Exception { // then source vertex configured. now we start manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED)); verify(mockContext, times(1)).doneReconfiguringVertex(); - + verify(mockContext, times(2)).scheduleTasks(requestCaptor.capture()); manager.onSourceTaskCompleted( TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 2)); @@ -205,7 +205,7 @@ public void testDelayedConfigureOneToOne() throws Exception { @Test (timeout=5000) public void testComplex() throws Exception { - HashMap mockInputVertices = + HashMap mockInputVertices = new HashMap(); String mockSrcVertexId1 = "Vertex1"; EdgeProperty eProp1 = EdgeProperty.create( @@ -228,7 +228,7 @@ public void testComplex() throws Exception { SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in")); - + String mockManagedVertexId = "Vertex"; Container mockContainer2 = mock(Container.class); ContainerId mockCId2 = mock(ContainerId.class); @@ -236,7 +236,7 @@ public void testComplex() throws Exception { Container mockContainer3 = mock(Container.class); ContainerId mockCId3 = mock(ContainerId.class); when(mockContainer3.getId()).thenReturn(mockCId3); - + VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class); when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices); when(mockContext.getVertexName()).thenReturn(mockManagedVertexId); @@ -246,9 +246,9 @@ public void testComplex() throws Exception { mockInputVertices.put(mockSrcVertexId1, eProp1); mockInputVertices.put(mockSrcVertexId2, eProp2); mockInputVertices.put(mockSrcVertexId3, eProp3); - + List initialCompletions = Lists.newArrayList(); - + // 1-1 sources do not match managed tasks. setParallelism called to make them match when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(4); InputReadyVertexManager manager = new InputReadyVertexManager(mockContext); @@ -260,7 +260,7 @@ public void testComplex() throws Exception { verify(mockContext, times(1)).reconfigureVertex(3, null, null); verify(mockContext, times(1)).doneReconfiguringVertex(); manager.onVertexStarted(initialCompletions); - + // 1-1 sources do not match when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(3); when(mockContext.getVertexNumTasks(mockSrcVertexId3)).thenReturn(4); @@ -276,9 +276,9 @@ public void testComplex() throws Exception { e.getMessage().contains("1-1 source vertices must have identical concurrency"); } verify(mockContext, times(1)).reconfigureVertex(anyInt(), any(), any()); // not invoked - + when(mockContext.getVertexNumTasks(mockSrcVertexId3)).thenReturn(3); - + initialCompletions.add(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 0)); initialCompletions.add(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId2, 0)); manager = new InputReadyVertexManager(mockContext); @@ -333,11 +333,11 @@ public void testComplex() throws Exception { .getTaskLocationHint().getAffinitizedTask().getVertexName()); Assert.assertEquals(2, requestCaptor.getValue().get(0) .getTaskLocationHint().getAffinitizedTask().getTaskIndex()); // affinity to last completion - + // no more starts manager.onSourceTaskCompleted( TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId3, 2)); verify(mockContext, times(3)).scheduleTasks(anyList()); - + } } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java index 7349810819..aa07fd41d2 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java @@ -44,7 +44,7 @@ import org.junit.Test; public class TestWeightedScalingMemoryDistributor extends TestMemoryDistributor { - + @Override public void setup() { conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ENABLED, true); @@ -53,7 +53,7 @@ public void setup() { conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION, 0.3d); conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_PER_IO, 0.0d); } - + @Test(timeout = 5000) public void testSimpleWeightedScaling() throws TezException { Configuration conf = new Configuration(this.conf); @@ -147,7 +147,7 @@ public void testAdditionalReserveFractionWeightedScaling() throws TezException { assertEquals(500, e3Callback.assigned); assertEquals(1500, e4Callback.assigned); } - + @Test(timeout = 5000) public void testWeightedScalingNonConcurrent() throws TezException { Configuration conf = new Configuration(this.conf); diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestInputIdentifiers.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestInputIdentifiers.java index 5eb3b5030a..0a2b1086be 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestInputIdentifiers.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestInputIdentifiers.java @@ -25,7 +25,7 @@ import org.junit.Test; public class TestInputIdentifiers { - + @Test(timeout = 5000) public void testInputAttemptIdentifier() { Set set = new HashSet(); @@ -33,7 +33,7 @@ public void testInputAttemptIdentifier() { InputAttemptIdentifier i2 = new InputAttemptIdentifier(1, 1, null); InputAttemptIdentifier i3 = new InputAttemptIdentifier(1, 0, null); InputAttemptIdentifier i4 = new InputAttemptIdentifier(0, 1, null); - + Assert.assertTrue(set.add(i1)); Assert.assertFalse(set.add(i1)); Assert.assertFalse(set.add(i2)); diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestValuesIterator.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestValuesIterator.java index eb342dd4c4..92c09938fa 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestValuesIterator.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestValuesIterator.java @@ -256,9 +256,9 @@ comparator, new ProgressReporter(), new GenericCounter("readsCounter", "y"), /** * Tests whether data in valuesIterator matches with sorted input data set. - * + * * Returns a list of value counts for each key. - * + * * @param valuesIterator * @return List * @throws IOException diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestFetcher.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestFetcher.java index bba5225d5a..d38ccb7fdf 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestFetcher.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestFetcher.java @@ -150,7 +150,7 @@ public void testLocalFetchModeSetting() throws Exception { @Test(timeout = 3000) public void testSetupLocalDiskFetch() throws Exception { - + CompositeInputAttemptIdentifier[] srcAttempts = { new CompositeInputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_0", 1), new CompositeInputAttemptIdentifier(1, 2, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1", 1), diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandlerImpl.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandlerImpl.java index 5b489f20e1..955b1e86a5 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandlerImpl.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandlerImpl.java @@ -173,7 +173,7 @@ public void testMultipleEvents1() throws IOException { Event dme1 = createDataMovementEvent(0, taskIndex1, createEmptyPartitionByteString(0)); int taskIndex2 = 2; Event dme2 = createDataMovementEvent(0, taskIndex2, null); - + List eventList = new LinkedList(); eventList.add(dme1); eventList.add(dme2); @@ -427,7 +427,7 @@ private Event createDataMovementEvent(boolean addSpillDetails, int srcIdx, int t ByteBuffer payload = payloadBuilder.build().toByteString().asReadOnlyByteBuffer(); return DataMovementEvent.create(srcIdx, targetIdx, attemptNum, payload); } - + private Event createDataMovementEvent(int srcIndex, int targetIndex, ByteString emptyPartitionByteString) { DataMovementEventPayloadProto.Builder builder = DataMovementEventPayloadProto.newBuilder(); diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestMergeManager.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestMergeManager.java index 42ad051afe..62c36bb13e 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestMergeManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestMergeManager.java @@ -846,7 +846,7 @@ public void testOnDiskMergerFilenames() throws IOException, InterruptedException assertEquals(m1Prefix, m2Prefix); assertNotEquals(m1Prefix, m3Prefix); assertNotEquals(m2Prefix, m3Prefix); - + verify(inputContext, atLeastOnce()).notifyProgress(); } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleScheduler.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleScheduler.java index 9c4e9668f3..1707e7f74e 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleScheduler.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleScheduler.java @@ -191,7 +191,7 @@ public Void call() throws Exception { scheduler.copySucceeded(identifiers[i], mapHosts[i], 20, 25, 100, mapOutput, false); scheduler.freeHost(mapHosts[i]); } - + verify(inputContext, atLeast(numInputs)).notifyProgress(); // Ensure the executor exits, and without an error. diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/input/TestSortedGroupedMergedInput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/input/TestSortedGroupedMergedInput.java index 585522b804..2fc7c52a4d 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/input/TestSortedGroupedMergedInput.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/input/TestSortedGroupedMergedInput.java @@ -45,7 +45,7 @@ public class TestSortedGroupedMergedInput { MergedInputContext createMergedInputContext() { return mock(MergedInputContext.class); } - + @Test(timeout = 5000) public void testSimple() throws Exception { SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, @@ -297,7 +297,7 @@ public void testSkippedKey2() throws Exception { } getNextFromFinishedReader(kvsReader); } - + // Reads all values for a key, but doesn't trigger the last hasNext() call. @Test(timeout = 5000) public void testSkippedKey3() throws Exception { diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java index 4d32d14c4c..8735a9a948 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java @@ -90,7 +90,7 @@ public class TestOnFileSortedOutput { //For sorter (pipelined / Default) private SorterImpl sorterImpl; private int sorterThreads; - + final AtomicLong outputSize = new AtomicLong(); final AtomicLong numRecords = new AtomicLong(); @@ -383,7 +383,7 @@ private OutputContext createTezOutputContext() throws IOException { serviceProviderMetaData.writeInt(PORT); TezCounters counters = new TezCounters(); - + OutputStatisticsReporter reporter = mock(OutputStatisticsReporter.class); doAnswer(new Answer() { @Override public Object answer(InvocationOnMock invocation) throws Throwable { @@ -398,7 +398,7 @@ private OutputContext createTezOutputContext() throws IOException { } }).when(reporter).reportItemsProcessed(anyLong()); - + OutputContext context = mock(OutputContext.class); doReturn(localConf).when(context).getContainerConfiguration(); doReturn(counters).when(context).getCounters(); diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java index 7c57d215eb..727407349c 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java @@ -242,7 +242,7 @@ private OutputContext createOutputContext(Configuration payloadConf, Configurati TezTaskID taskID = TezTaskID.getInstance(vertexID, 1); TezTaskAttemptID taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1); UserPayload userPayload = TezUtils.createUserPayloadFromConf(payloadConf); - + TaskSpec mockSpec = mock(TaskSpec.class); when(mockSpec.getInputs()).thenReturn(Collections.singletonList(mock(InputSpec.class))); when(mockSpec.getOutputs()).thenReturn(Collections.singletonList(mock(OutputSpec.class))); @@ -251,7 +251,7 @@ private OutputContext createOutputContext(Configuration payloadConf, Configurati new DefaultHadoopShim(), sharedExecutor); LogicalIOProcessorRuntimeTask runtimeTask = spy(task); - + Map auxEnv = new HashMap(); ByteBuffer bb = ByteBuffer.allocate(4); bb.putInt(shufflePort); diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java index 2e6232b995..0127778080 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java @@ -106,10 +106,10 @@ public void run() throws Exception { boolean doLocalityCheck = getContext().getUserPayload().getPayload().get(0) > 0 ? true : false; int broadcastSum = getContext().getUserPayload().getPayload().get(1); int expectedSum = broadcastSum + getContext().getTaskIndex(); - System.out.println("Index: " + getContext().getTaskIndex() + + System.out.println("Index: " + getContext().getTaskIndex() + " sum: " + sum + " expectedSum: " + expectedSum + " broadcastSum: " + broadcastSum); - Preconditions.checkState((sum == expectedSum), "Sum = " + sum); - + Preconditions.checkState((sum == expectedSum), "Sum = " + sum); + if (doLocalityCheck) { ObjectRegistry objectRegistry = getContext().getObjectRegistry(); String index = (String) objectRegistry.get(String.valueOf(getContext().getTaskIndex())); @@ -149,7 +149,7 @@ private DAG createDAG(FileSystem fs, TezConfiguration tezConf, Vertex broadcastVertex = Vertex.create("Broadcast", ProcessorDescriptor.create( InputProcessor.class.getName()), numBroadcastTasks); - + Vertex inputVertex = Vertex.create("Input", ProcessorDescriptor.create( InputProcessor.class.getName()).setUserPayload(procPayload), numOneToOneTasks); @@ -174,7 +174,7 @@ private DAG createDAG(FileSystem fs, TezConfiguration tezConf, edgeConf.createDefaultBroadcastEdgeProperty())); return dag; } - + public boolean run(Configuration conf, boolean doLocalityCheck) throws Exception { System.out.println("Running BroadcastAndOneToOneExample"); // conf and UGI @@ -195,7 +195,7 @@ public boolean run(Configuration conf, boolean doLocalityCheck) throws Exception Path stagingDir = new Path(stagingDirStr); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr); stagingDir = fs.makeQualified(stagingDir); - + // No need to add jar containing this class as assumed to be part of // the tez jars. @@ -226,7 +226,7 @@ public boolean run(Configuration conf, boolean doLocalityCheck) throws Exception tezSession.stop(); } } - + @Override public int run(String[] args) throws Exception { boolean doLocalityCheck = true; @@ -252,12 +252,12 @@ public int run(String[] args) throws Exception { boolean status = run(getConf(), doLocalityCheck); return status ? 0 : 1; } - + private static void printUsage() { System.err.println("broadcastAndOneToOneExample " + skipLocalityCheck); ToolRunner.printGenericCommandUsage(System.err); } - + static String skipLocalityCheck = "-skipLocalityCheck"; public static void main(String[] args) throws Exception { diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Join.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Join.java index 3c0dcf77d6..2c7ab2d15c 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Join.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Join.java @@ -48,24 +48,24 @@ /** * Given a set of sorted datasets keyed with the same class and yielding - * equal partitions, it is possible to effect a join of those datasets + * equal partitions, it is possible to effect a join of those datasets * prior to the map. The example facilitates the same. * * To run: bin/hadoop jar build/hadoop-examples.jar join * [-r reduces] - * [-inFormat input format class] - * [-outFormat output format class] - * [-outKey output key class] - * [-outValue output value class] + * [-inFormat input format class] + * [-outFormat output format class] + * [-outKey output key class] + * [-outValue output value class] * [-joinOp <inner|outer|override>] - * [in-dir]* in-dir out-dir + * [in-dir]* in-dir out-dir */ public class Join extends Configured implements Tool { public static final String REDUCES_PER_HOST = "mapreduce.join.reduces_per_host"; static int printUsage() { System.out.println("join [-r ] " + "[-inFormat ] " + - "[-outFormat ] " + + "[-outFormat ] " + "[-outKey ] " + "[-outValue ] " + "[-joinOp ] " + diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java index abf0364042..405e82022e 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java @@ -398,7 +398,7 @@ public static void main(String[] args) throws Exception { int res = ToolRunner.run(new Configuration(), new MRRSleepJob(), args); System.exit(res); } - + private Credentials credentials = new Credentials(); public DAG createDAG(Configuration conf, Path stagingDir, @@ -520,7 +520,7 @@ public DAG createDAG(Configuration conf, Path stagingDir, List vertices = new ArrayList(); - + UserPayload mapUserPayload = TezUtils.createUserPayloadFromConf(mapStageConf); int numTasks = generateSplitsInAM ? -1 : numMapper; @@ -538,7 +538,7 @@ public DAG createDAG(Configuration conf, Path stagingDir, vertices.add(mapVertex); if (iReduceStagesCount > 0 - && numIReducer > 0) { + && numIReducer > 0) { for (int i = 0; i < iReduceStagesCount; ++i) { Configuration iconf = intermediateReduceStageConfs[i]; @@ -710,7 +710,7 @@ else if (args[i].equals("-generateSplitsInAM")) { } splitsOptionFound = true; generateSplitsInAM = Boolean.parseBoolean(args[++i]); - + } else if (args[i].equals("-writeSplitsToDfs")) { if (splitsOptionFound) { diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MultipleCommitsExample.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MultipleCommitsExample.java index d716ecad1a..cd8be927f3 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MultipleCommitsExample.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MultipleCommitsExample.java @@ -59,7 +59,7 @@ * v1 -> v3
    * v2 -> v3
    * (v1,v2) is connected to v3 as vertex group.
    - * (v1,v2) have multiple shared outputs, each of them have its own multiple outputs. + * (v1,v2) have multiple shared outputs, each of them have its own multiple outputs. * And v3 also has multiple outputs.
    */ public class MultipleCommitsExample extends TezExampleBase { @@ -104,7 +104,7 @@ public void initialize() throws Exception { super.initialize(); config = MultipleOutputProcessorConfig.fromUserPayload(getContext().getUserPayload()); } - + @Override public void run() throws Exception { for (int i=0;i < config.outputNum;++i) { @@ -118,18 +118,18 @@ public void run() throws Exception { writer.write(NullWritable.get(), new Text("dummy")); } } - + public static class MultipleOutputProcessorConfig implements Writable { - + String outputNamePrefix; int outputNum; String sharedOutputNamePrefix = null; int sharedOutputNum; public MultipleOutputProcessorConfig(){ - + } - + public MultipleOutputProcessorConfig(String outputNamePrefix, int outputNum) { this.outputNamePrefix = outputNamePrefix; this.outputNum = outputNum; @@ -171,7 +171,7 @@ public void readFields(DataInput in) throws IOException { sharedOutputNum = in.readInt(); } } - + public UserPayload toUserPayload() throws IOException { NonSyncByteArrayOutputStream out = new NonSyncByteArrayOutputStream(); this.write(new NonSyncDataOutputStream(out)); @@ -202,7 +202,7 @@ protected int runJob(String[] args, TezConfiguration tezConf, return runDag(dag, false, LOG); } - private DAG createDAG(TezConfiguration tezConf, + private DAG createDAG(TezConfiguration tezConf, String v1OutputPathPrefix, int v1OutputNum, String v2OutputPathPrefix, int v2OutputNum, String uv12OutputPathPrefix, int uv12OutputNum, String v3OutputPathPrefix, int v3OutputNum, boolean commitOnVertexSuccess) throws IOException { diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java index a1e6dbf06a..581596114e 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java @@ -69,47 +69,47 @@ * 1099511627776 * * - * + * * Equivalently, {@link RandomTextWriter} also supports all the above options * and ones supported by {@link Tool} via the command-line. - * + * * To run: bin/hadoop jar hadoop-${version}-examples.jar randomtextwriter - * [-outFormat output format class] output + * [-outFormat output format class] output */ public class RandomTextWriter extends Configured implements Tool { - public static final String TOTAL_BYTES = + public static final String TOTAL_BYTES = "mapreduce.randomtextwriter.totalbytes"; - public static final String BYTES_PER_MAP = + public static final String BYTES_PER_MAP = "mapreduce.randomtextwriter.bytespermap"; - public static final String MAPS_PER_HOST = + public static final String MAPS_PER_HOST = "mapreduce.randomtextwriter.mapsperhost"; public static final String MAX_VALUE = "mapreduce.randomtextwriter.maxwordsvalue"; public static final String MIN_VALUE = "mapreduce.randomtextwriter.minwordsvalue"; public static final String MIN_KEY = "mapreduce.randomtextwriter.minwordskey"; public static final String MAX_KEY = "mapreduce.randomtextwriter.maxwordskey"; - + static int printUsage() { System.out.println("randomtextwriter " + - "[-outFormat ] " + + "[-outFormat ] " + ""); ToolRunner.printGenericCommandUsage(System.out); return 2; } - + /** * User counters */ static enum Counters { RECORDS_WRITTEN, BYTES_WRITTEN } static class RandomTextMapper extends Mapper { - + private long numBytesToWrite; private int minWordsInKey; private int wordsInKeyRange; private int minWordsInValue; private int wordsInValueRange; private Random random = new Random(); - + /** * Save the configuration value that we need to write the data. */ @@ -122,7 +122,7 @@ public void setup(Context context) { minWordsInValue = conf.getInt(MIN_VALUE, 10); wordsInValueRange = (conf.getInt(MAX_VALUE, 100) - minWordsInValue); } - + /** * Given an output filename, write a bunch of random records to it. */ @@ -130,31 +130,31 @@ public void map(Text key, Text value, Context context) throws IOException,InterruptedException { int itemCount = 0; while (numBytesToWrite > 0) { - // Generate the key/value - int noWordsKey = minWordsInKey + + // Generate the key/value + int noWordsKey = minWordsInKey + (wordsInKeyRange != 0 ? random.nextInt(wordsInKeyRange) : 0); - int noWordsValue = minWordsInValue + + int noWordsValue = minWordsInValue + (wordsInValueRange != 0 ? random.nextInt(wordsInValueRange) : 0); Text keyWords = generateSentence(noWordsKey); Text valueWords = generateSentence(noWordsValue); - - // Write the sentence + + // Write the sentence context.write(keyWords, valueWords); - + numBytesToWrite -= (keyWords.getLength() + valueWords.getLength()); - + // Update counters, progress etc. context.getCounter(Counters.BYTES_WRITTEN).increment( keyWords.getLength() + valueWords.getLength()); context.getCounter(Counters.RECORDS_WRITTEN).increment(1); if (++itemCount % 200 == 0) { - context.setStatus("wrote record " + itemCount + ". " + + context.setStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left."); } } context.setStatus("done with " + itemCount + " records."); } - + private Text generateSentence(int noWords) { StringBuffer sentence = new StringBuffer(); String space = " "; @@ -165,20 +165,20 @@ private Text generateSentence(int noWords) { return new Text(sentence.toString()); } } - + /** * This is the main routine for launching a distributed random write job. * It runs 10 maps/node and each node writes 1 gig of data to a DFS file. * The reduce doesn't do anything. - * - * @throws IOException + * + * @throws IOException */ @SuppressWarnings("deprecation") - public int run(String[] args) throws Exception { + public int run(String[] args) throws Exception { if (args.length == 0) { - return printUsage(); + return printUsage(); } - + Configuration conf = getConf(); try (JobClient client = new JobClient(conf)) { return run(client, conf, args); @@ -251,7 +251,7 @@ private int run(JobClient client, Configuration conf, String[] args) return ret; } - + public static void main(String[] args) throws Exception { int res = ToolRunner.run(getTezDecoratedConfiguration(), new RandomTextWriter(), args); System.exit(res); diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java index 4b7a4e5726..2a0471d1b8 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java @@ -74,43 +74,43 @@ * 1099511627776 * * - * + * * Equivalently, {@link RandomWriter} also supports all the above options * and ones supported by {@link GenericOptionsParser} via the command-line. */ public class RandomWriter extends Configured implements Tool { public static final String TOTAL_BYTES = "mapreduce.randomwriter.totalbytes"; - public static final String BYTES_PER_MAP = + public static final String BYTES_PER_MAP = "mapreduce.randomwriter.bytespermap"; - public static final String MAPS_PER_HOST = + public static final String MAPS_PER_HOST = "mapreduce.randomwriter.mapsperhost"; public static final String MAX_VALUE = "mapreduce.randomwriter.maxvalue"; public static final String MIN_VALUE = "mapreduce.randomwriter.minvalue"; public static final String MIN_KEY = "mapreduce.randomwriter.minkey"; public static final String MAX_KEY = "mapreduce.randomwriter.maxkey"; - + /** * User counters */ static enum Counters { RECORDS_WRITTEN, BYTES_WRITTEN } - + /** * A custom input format that creates virtual inputs of a single string * for each map. */ static class RandomInputFormat extends InputFormat { - /** + /** * Generate the requested number of file splits, with the filename * set to the filename of the output file. */ public List getSplits(JobContext job) throws IOException { List result = new ArrayList(); Path outDir = FileOutputFormat.getOutputPath(job); - int numSplits = + int numSplits = job.getConfiguration().getInt(MRJobConfig.NUM_MAPS, 1); for(int i=0; i < numSplits; ++i) { - result.add(new FileSplit(new Path(outDir, "dummy-split-" + i), 0, 1, + result.add(new FileSplit(new Path(outDir, "dummy-split-" + i), 0, 1, (String[])null)); } return result; @@ -127,13 +127,13 @@ static class RandomRecordReader extends RecordReader { public RandomRecordReader(Path p) { name = p; } - + public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { - + } - + public boolean nextKeyValue() { if (name != null) { key = new Text(); @@ -143,15 +143,15 @@ public boolean nextKeyValue() { } return false; } - + public Text getCurrentKey() { return key; } - + public Text getCurrentValue() { return value; } - + public void close() {} public float getProgress() { @@ -167,7 +167,7 @@ public RecordReader createRecordReader(InputSplit split, static class RandomMapper extends Mapper { - + private long numBytesToWrite; private int minKeySize; private int keySizeRange; @@ -176,22 +176,22 @@ static class RandomMapper extends Mapper= offset; --i) { data[i] = (byte) random.nextInt(256); } } - + /** * Given an output filename, write a bunch of random records to it. */ - public void map(WritableComparable key, + public void map(WritableComparable key, Writable value, Context context) throws IOException,InterruptedException { int itemCount = 0; while (numBytesToWrite > 0) { - int keyLength = minKeySize + + int keyLength = minKeySize + (keySizeRange != 0 ? random.nextInt(keySizeRange) : 0); randomKey.setSize(keyLength); randomizeBytes(randomKey.getBytes(), 0, randomKey.getLength()); @@ -204,13 +204,13 @@ public void map(WritableComparable key, context.getCounter(Counters.BYTES_WRITTEN).increment(keyLength + valueLength); context.getCounter(Counters.RECORDS_WRITTEN).increment(1); if (++itemCount % 200 == 0) { - context.setStatus("wrote record " + itemCount + ". " + + context.setStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left."); } } context.setStatus("done with " + itemCount + " records."); } - + /** * Save the values out of the configuaration that we need to write * the data. @@ -221,29 +221,29 @@ public void setup(Context context) { numBytesToWrite = conf.getLong(BYTES_PER_MAP, 1*1024*1024*1024); minKeySize = conf.getInt(MIN_KEY, 10); - keySizeRange = + keySizeRange = conf.getInt(MAX_KEY, 1000) - minKeySize; minValueSize = conf.getInt(MIN_VALUE, 0); - valueSizeRange = + valueSizeRange = conf.getInt(MAX_VALUE, 20000) - minValueSize; } } - + /** * This is the main routine for launching a distributed random write job. * It runs 10 maps/node and each node writes 1 gig of data to a DFS file. * The reduce doesn't do anything. - * - * @throws IOException + * + * @throws IOException */ @SuppressWarnings("deprecation") - public int run(String[] args) throws Exception { + public int run(String[] args) throws Exception { if (args.length == 0) { System.out.println("Usage: writer "); ToolRunner.printGenericCommandUsage(System.out); return 2; } - + Path outDir = new Path(args[0]); Configuration conf = getConf(); try (JobClient client = new JobClient(conf)) { @@ -297,7 +297,7 @@ private int run(JobClient client, Configuration conf, Path outDir) return ret; } - + public static void main(String[] args) throws Exception { int res = ToolRunner.run(getTezDecoratedConfiguration(), new RandomWriter(), args); System.exit(res); diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java index d767a640b5..7706a956c9 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java @@ -45,23 +45,23 @@ /** * This is an example Hadoop Map/Reduce application. * It reads the text input files that must contain two integers per a line. - * The output is sorted by the first and second number and grouped on the + * The output is sorted by the first and second number and grouped on the * first number. * * To run: bin/hadoop jar build/hadoop-examples.jar secondarysort - * in-dir out-dir + * in-dir out-dir */ public class SecondarySort extends Configured implements Tool { - + /** * Define a pair of integers that are writable. * They are serialized in a byte comparable format. */ - public static class IntPair + public static class IntPair implements WritableComparable { private int first = 0; private int second = 0; - + /** * Set the left and right values. */ @@ -76,7 +76,7 @@ public int getSecond() { return second; } /** - * Read the two integers. + * Read the two integers. * Encoded as: MIN_VALUE -> 0, 0 -> -MIN_VALUE, MAX_VALUE-> -1 */ @Override @@ -102,7 +102,7 @@ public boolean equals(Object right) { return false; } } - /** A Comparator that compares serialized IntPair. */ + /** A Comparator that compares serialized IntPair. */ public static class Comparator extends WritableComparator { public Comparator() { super(IntPair.class); @@ -129,13 +129,13 @@ public int compareTo(IntPair o) { } } } - + /** * Partition based on the first part of the pair. */ public static class FirstPartitioner extends Partitioner{ @Override - public int getPartition(IntPair key, IntWritable value, + public int getPartition(IntPair key, IntWritable value, int numPartitions) { return Math.abs(key.getFirst() * 127) % numPartitions; } @@ -145,11 +145,11 @@ public int getPartition(IntPair key, IntWritable value, * Compare only the first part of the pair, so that reduce is called once * for each value of the first part. */ - public static class FirstGroupingComparator + public static class FirstGroupingComparator implements RawComparator { @Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { - return WritableComparator.compareBytes(b1, s1, Integer.SIZE/8, + return WritableComparator.compareBytes(b1, s1, Integer.SIZE/8, b2, s2, Integer.SIZE/8); } @@ -165,14 +165,14 @@ public int compare(IntPair o1, IntPair o2) { * Read two integers from each line and generate a key, value pair * as ((left, right), right). */ - public static class MapClass + public static class MapClass extends Mapper { - + private final IntPair key = new IntPair(); private final IntWritable value = new IntWritable(); - + @Override - public void map(LongWritable inKey, Text inValue, + public void map(LongWritable inKey, Text inValue, Context context) throws IOException, InterruptedException { StringTokenizer itr = new StringTokenizer(inValue.toString()); int left = 0; @@ -188,16 +188,16 @@ public void map(LongWritable inKey, Text inValue, } } } - + /** * A reducer class that just emits the sum of the input values. */ - public static class Reduce + public static class Reduce extends Reducer { - private static final Text SEPARATOR = + private static final Text SEPARATOR = new Text("------------------------------------------------"); private final Text first = new Text(); - + @Override public void reduce(IntPair key, Iterable values, Context context @@ -236,7 +236,7 @@ public int run(String[] args) throws Exception { // the reduce output is Text, IntWritable job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); - + FileInputFormat.addInputPath(job, new Path(otherArgs[0])); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); return job.waitForCompletion(true) ? 0 : 1; diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Sort.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Sort.java index 3f4664ad92..d038b76502 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Sort.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Sort.java @@ -50,22 +50,22 @@ * * To run: bin/hadoop jar build/hadoop-examples.jar sort * [-r reduces] - * [-inFormat input format class] - * [-outFormat output format class] - * [-outKey output key class] - * [-outValue output value class] + * [-inFormat input format class] + * [-outFormat output format class] + * [-outKey output key class] + * [-outValue output value class] * [-totalOrder pcnt num samples max splits] - * in-dir out-dir + * in-dir out-dir */ public class Sort extends Configured implements Tool { - public static final String REDUCES_PER_HOST = + public static final String REDUCES_PER_HOST = "mapreduce.sort.reducesperhost"; private Job job = null; static int printUsage() { System.out.println("sort [-r ] " + "[-inFormat ] " + - "[-outFormat ] " + + "[-outFormat ] " + "[-outKey ] " + "[-outValue ] " + "[-totalOrder ] " + @@ -89,12 +89,12 @@ public int run(String[] args) throws Exception { int num_reduces = (int) (cluster.getMaxReduceTasks() * 0.9); String sort_reduces = conf.get(REDUCES_PER_HOST); if (sort_reduces != null) { - num_reduces = cluster.getTaskTrackers() * + num_reduces = cluster.getTaskTrackers() * Integer.parseInt(sort_reduces); } - Class inputFormatClass = + Class inputFormatClass = SequenceFileInputFormat.class; - Class outputFormatClass = + Class outputFormatClass = SequenceFileOutputFormat.class; Class outputKeyClass = BytesWritable.class; Class outputValueClass = BytesWritable.class; @@ -105,16 +105,16 @@ public int run(String[] args) throws Exception { if ("-r".equals(args[i])) { num_reduces = Integer.parseInt(args[++i]); } else if ("-inFormat".equals(args[i])) { - inputFormatClass = + inputFormatClass = Class.forName(args[++i]).asSubclass(InputFormat.class); } else if ("-outFormat".equals(args[i])) { - outputFormatClass = + outputFormatClass = Class.forName(args[++i]).asSubclass(OutputFormat.class); } else if ("-outKey".equals(args[i])) { - outputKeyClass = + outputKeyClass = Class.forName(args[++i]).asSubclass(WritableComparable.class); } else if ("-outValue".equals(args[i])) { - outputValueClass = + outputValueClass = Class.forName(args[++i]).asSubclass(Writable.class); } else if ("-totalOrder".equals(args[i])) { double pcnt = Double.parseDouble(args[++i]); @@ -140,7 +140,7 @@ public int run(String[] args) throws Exception { job.setJobName("sorter"); job.setJarByClass(Sort.class); - job.setMapperClass(Mapper.class); + job.setMapperClass(Mapper.class); job.setReducerClass(Reducer.class); job.setNumReduceTasks(num_reduces); @@ -159,7 +159,7 @@ public int run(String[] args) throws Exception { } FileInputFormat.setInputPaths(job, otherArgs.get(0)); FileOutputFormat.setOutputPath(job, new Path(otherArgs.get(1))); - + if (sampler != null) { System.out.println("Sampling input to effect total-order sort..."); job.setPartitionerClass(TotalOrderPartitioner.class); @@ -175,7 +175,7 @@ public int run(String[] args) throws Exception { System.out.println("Running on " + cluster.getTaskTrackers() + - " nodes to sort from " + + " nodes to sort from " + FileInputFormat.getInputPaths(job)[0] + " into " + FileOutputFormat.getOutputPath(job) + " with " + num_reduces + " reduces."); @@ -184,7 +184,7 @@ public int run(String[] args) throws Exception { int ret = job.waitForCompletion(true) ? 0 : 1; Date end_time = new Date(); System.out.println("Job ended: " + end_time); - System.out.println("The job took " + + System.out.println("The job took " + (end_time.getTime() - startTime.getTime()) /1000 + " seconds."); return ret; } diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java index 0db7aec575..176f2e7f39 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java @@ -442,7 +442,7 @@ public int run(String[] args) throws Exception { TestOrderedWordCount instance = new TestOrderedWordCount(); String stagingDirStr = conf.get(TezConfiguration.TEZ_AM_STAGING_DIR, - TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT) + Path.SEPARATOR + + TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT) + Path.SEPARATOR + Long.toString(System.currentTimeMillis()); Path stagingDir = new Path(stagingDirStr); FileSystem pathFs = stagingDir.getFileSystem(tezConf); @@ -450,7 +450,7 @@ public int run(String[] args) throws Exception { tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr); stagingDir = pathFs.makeQualified(new Path(stagingDirStr)); - + TokenCache.obtainTokensForNamenodes(instance.credentials, new Path[] {stagingDir}, conf); TezClientUtils.ensureStagingDirExists(tezConf, stagingDir); @@ -459,7 +459,7 @@ public int run(String[] args) throws Exception { // TEZ-674 Obtain tokens based on the Input / Output paths. For now assuming staging dir // is the same filesystem as the one used for Input/Output. - + if (useTezSession) { LOG.info("Creating Tez Session"); tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true); @@ -510,7 +510,7 @@ public int run(String[] args) throws Exception { Map localResources = new TreeMap(); - + DAG dag = instance.createDAG(fs, tezConf, localResources, stagingDir, dagIndex, inputPath, outputPath, generateSplitsInClient, useMRSettings, intermediateNumReduceTasks, @@ -538,7 +538,7 @@ public int run(String[] args) throws Exception { preWarmVertex.addTaskLocalFiles(dag.getVertex("initialmap").getTaskLocalFiles()); preWarmVertex.setTaskEnvironment(dag.getVertex("initialmap").getTaskEnvironment()); preWarmVertex.setTaskLaunchCmdOpts(dag.getVertex("initialmap").getTaskLaunchCmdOpts()); - + tezSession.preWarm(preWarmVertex); } diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java index 8525756c42..7b9ea1867d 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java @@ -167,7 +167,7 @@ private DAG createDAG(FileSystem fs, TezConfiguration tezConf, Map localResources, Path stagingDir, String inputPath, String outputPath) throws IOException { DAG dag = DAG.create("UnionExample"); - + int numMaps = -1; Configuration inputConf = new Configuration(tezConf); inputConf.setBoolean("mapred.mapper.new-api", false); @@ -194,14 +194,14 @@ private DAG createDAG(FileSystem fs, TezConfiguration tezConf, outputConf.set(FileOutputFormat.OUTDIR, outputPath); DataSinkDescriptor od = MROutput.createConfigBuilder(outputConf, null).build(); checkerVertex.addDataSink("union", od); - + Configuration allPartsConf = new Configuration(tezConf); DataSinkDescriptor od2 = MROutput.createConfigBuilder(allPartsConf, TextOutputFormat.class, outputPath + "-all-parts").build(); checkerVertex.addDataSink("all-parts", od2); - Configuration partsConf = new Configuration(tezConf); + Configuration partsConf = new Configuration(tezConf); DataSinkDescriptor od1 = MROutput.createConfigBuilder(partsConf, TextOutputFormat.class, outputPath + "-parts").build(); VertexGroup unionVertex = dag.createVertexGroup("union", mapVertex1, mapVertex2); @@ -221,7 +221,7 @@ private DAG createDAG(FileSystem fs, TezConfiguration tezConf, GroupInputEdge.create(unionVertex, checkerVertex, edgeConf.createDefaultEdgeProperty(), InputDescriptor.create( ConcatenatedMergedKeyValuesInput.class.getName()))); - return dag; + return dag; } private static void printUsage() { @@ -244,18 +244,18 @@ public boolean run(String inputPath, String outputPath, Configuration conf) thro FileSystem fs = FileSystem.get(tezConf); String stagingDirStr = Path.SEPARATOR + "user" + Path.SEPARATOR + user + Path.SEPARATOR+ ".staging" + Path.SEPARATOR - + Path.SEPARATOR + Long.toString(System.currentTimeMillis()); + + Path.SEPARATOR + Long.toString(System.currentTimeMillis()); Path stagingDir = new Path(stagingDirStr); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr); stagingDir = fs.makeQualified(stagingDir); - + // No need to add jar containing this class as assumed to be part of // the tez jars. // TEZ-674 Obtain tokens based on the Input / Output paths. For now assuming staging dir // is the same filesystem as the one used for Input/Output. - + TezClient tezSession = TezClient.create("UnionExampleSession", tezConf); tezSession.start(); @@ -269,10 +269,10 @@ public boolean run(String inputPath, String outputPath, Configuration conf) thro throw new FileAlreadyExistsException("Output directory " + outputPath + " already exists"); } - + Map localResources = new TreeMap(); - + DAG dag = createDAG(fs, tezConf, localResources, stagingDir, inputPath, outputPath); diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordInputProcessor.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordInputProcessor.java index adafb2db6c..aa37751d33 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordInputProcessor.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordInputProcessor.java @@ -93,7 +93,7 @@ public void run(Map _inputs, if (_outputs.size() != 1) { throw new IllegalStateException("FilterByWordInputProcessor processor can only work with a single output"); } - + for (LogicalInput input : _inputs.values()) { input.start(); } diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordOutputProcessor.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordOutputProcessor.java index ccccc7eec5..b9f368626c 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordOutputProcessor.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordOutputProcessor.java @@ -57,7 +57,7 @@ public void close() throws Exception { @Override public void run() throws Exception { - + if (inputs.size() != 1) { throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with a single input"); } diff --git a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java index 6bcbe3b46b..e1717c9c94 100644 --- a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java +++ b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java @@ -159,7 +159,7 @@ public static void setup() throws IOException { } catch (IOException io) { throw new RuntimeException("problem starting mini dfs cluster", io); } - + if (mrrTezCluster == null) { mrrTezCluster = new MiniTezCluster(TestMRRJobsDAGApi.class.getName(), 1, 1, 1); @@ -185,7 +185,7 @@ public static void tearDown() { } // TODO Add cleanup code. } - + @Test(timeout = 60000) public void testSleepJob() throws TezException, IOException, InterruptedException { SleepProcessorConfig spConf = new SleepProcessorConfig(1); @@ -616,7 +616,7 @@ public State testMRRSleepJobDagSubmitCore( UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf); UserPayload stage2Payload = TezUtils.createUserPayloadFromConf(stage2Conf); UserPayload stage3Payload = TezUtils.createUserPayloadFromConf(stage3Conf); - + DAG dag = DAG.create("testMRRSleepJobDagSubmit-" + random.nextInt(1000)); Class inputInitializerClazz = @@ -789,7 +789,7 @@ private static LocalResource createLocalResource(FileSystem fc, Path file, return LocalResource.newInstance(resourceURL, type, visibility, resourceSize, resourceModificationTime); } - + @Test(timeout = 60000) public void testVertexGroups() throws Exception { LOG.info("Running Group Test"); @@ -803,7 +803,7 @@ public void testVertexGroups() throws Exception { writer.write("efgh "); writer.close(); out.close(); - + UnionExample job = new UnionExample(); if (job.run(inPath.toString(), outPath.toString(), mrrTezCluster.getConfig())) { LOG.info("Success VertexGroups Test"); @@ -811,7 +811,7 @@ public void testVertexGroups() throws Exception { throw new TezUncheckedException("VertexGroups Test Failed"); } } - + @Test(timeout = 60000) public void testBroadcastAndOneToOne() throws Exception { LOG.info("Running BroadcastAndOneToOne Test"); @@ -853,7 +853,7 @@ public List initialize() throws Exception { return super.initialize(); } } - + private static void createTestJar(OutputStream outStream, String dummyClassName) throws URISyntaxException, IOException { JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); diff --git a/tez-tests/src/test/java/org/apache/tez/test/AMShutdownController.java b/tez-tests/src/test/java/org/apache/tez/test/AMShutdownController.java index 4baf6de98d..952b8ef1c4 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/AMShutdownController.java +++ b/tez-tests/src/test/java/org/apache/tez/test/AMShutdownController.java @@ -27,10 +27,10 @@ public abstract class AMShutdownController { private List historyEvents = new ArrayList(); - + protected AppContext appContext; protected RecoveryService recoveryService; - + public AMShutdownController(AppContext appContext, RecoveryService recoveryService) { this.appContext = appContext; this.recoveryService = recoveryService; diff --git a/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java b/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java index 49764e8ce2..3e2e1beb44 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java +++ b/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java @@ -42,13 +42,13 @@ * then executed in the session. Returns success if DAG succeeds. */ public class FaultToleranceTestRunner { - + static String DEFAULT_FT_STAGING_DIR = "tmp"; static String FT_STAGING_DIR = "tez.test-fault-tolerance.staging-dir"; Configuration conf = null; TezClient tezSession = null; Resource defaultResource = Resource.newInstance(100, 0); - + void setup() throws Exception { TezConfiguration tezConf = null; if (conf == null) { @@ -69,44 +69,44 @@ void setup() throws Exception { Path remoteStagingDir = defaultFs.makeQualified(new Path(testRootDir, String .valueOf(new Random().nextInt(100000)))); TezClientUtils.ensureStagingDirExists(tezConf, remoteStagingDir); - + tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString()); tezSession = TezClient.create("FaultToleranceTestRunner", tezConf); tezSession.start(); } - + void tearDown() throws Exception { if (tezSession != null) { tezSession.stop(); } } - + DAG getDAG(String className, String confFilePath) throws Exception { Class clazz = Class.forName(className); Method method = clazz.getMethod("createDAG", Configuration.class); - + Configuration testConf = new Configuration(false); if (confFilePath != null) { Path confPath = new Path(confFilePath); testConf.addResource(confPath); } - + DAG dag = (DAG) method.invoke(null, testConf); - + return dag; } - + boolean run(Configuration conf, String className, String confFilePath) throws Exception { this.conf = conf; setup(); - + try { tezSession.waitTillReady(); - + DAG dag = getDAG(className, confFilePath); - + DAGClient dagClient = tezSession.submitDAG(dag); DAGStatus dagStatus = dagClient.getDAGStatus(null); while (!dagStatus.isCompleted()) { @@ -117,24 +117,24 @@ boolean run(Configuration conf, String className, String confFilePath) throws Ex Thread.sleep(500); dagStatus = dagClient.getDAGStatus(null); } - + if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) { return true; } - + } finally { tearDown(); } - + return false; } - + static void printUsage() { System.err.println( "Usage: " + " FaultToleranceTestRunner [generic options] "); GenericOptionsParser.printGenericCommandUsage(System.err); } - + public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); @@ -149,13 +149,13 @@ public static void main(String[] args) throws Exception { printUsage(); System.exit(1); } - + FaultToleranceTestRunner job = new FaultToleranceTestRunner(); if (job.run(conf, className, confFilePath)) { System.out.println("Succeeded."); } else { System.out.println("Failed."); System.exit(2); - } + } } } diff --git a/tez-tests/src/test/java/org/apache/tez/test/MiniTezCluster.java b/tez-tests/src/test/java/org/apache/tez/test/MiniTezCluster.java index 0751ddda28..d7c5df6206 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/MiniTezCluster.java +++ b/tez-tests/src/test/java/org/apache/tez/test/MiniTezCluster.java @@ -311,7 +311,7 @@ public boolean apply(ApplicationReport appReport) { yarnClient.stop(); } } - + public Path getConfigFilePath() { return confFilePath; } diff --git a/tez-tests/src/test/java/org/apache/tez/test/RecoveryServiceWithEventHandlingHook.java b/tez-tests/src/test/java/org/apache/tez/test/RecoveryServiceWithEventHandlingHook.java index a50b4aee1e..d895f7e1de 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/RecoveryServiceWithEventHandlingHook.java +++ b/tez-tests/src/test/java/org/apache/tez/test/RecoveryServiceWithEventHandlingHook.java @@ -70,7 +70,7 @@ public void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); String clazz = conf.get(AM_RECOVERY_SERVICE_HOOK_CLASS); Preconditions.checkArgument(clazz != null, "RecoveryServiceHook class is not specified"); - this.hook = ReflectionUtils.createClazzInstance(clazz, + this.hook = ReflectionUtils.createClazzInstance(clazz, new Class[]{RecoveryServiceWithEventHandlingHook.class, AppContext.class}, new Object[]{this, super.appContext}); } @@ -181,7 +181,7 @@ public void postHandleRecoveryEvent(DAGHistoryEvent event) recoveryService.shutdown(); } } - + @Override public void preHandleSummaryEvent(HistoryEventType eventType, SummaryEvent summaryEvent) throws IOException { @@ -195,7 +195,7 @@ public void postHandleSummaryEvent(HistoryEventType eventType, } /** - * + * * Shutdown AM based on one recovery event if it is matched. * This would be serialized as property of TezConfiguration and deserialized at runtime. */ @@ -378,7 +378,7 @@ public boolean match(HistoryEvent incomingEvent) { return false; } - + public HistoryEventType getEventType() { return event.getEventType(); } diff --git a/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG.java b/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG.java index 486f5007be..ae1c950829 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG.java +++ b/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG.java @@ -32,7 +32,7 @@ /** * Simple Test DAG with 2 vertices using TestProcessor/TestInput/TestOutput. - * + * * v1 * | * v2 @@ -45,8 +45,8 @@ public final class SimpleTestDAG { public static int TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT = 2; private SimpleTestDAG() {} - - public static DAG createDAG(String name, + + public static DAG createDAG(String name, Configuration conf) throws Exception { UserPayload payload = UserPayload.create(null); int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT; @@ -65,7 +65,7 @@ public static DAG createDAG(String name, TestInput.getInputDesc(payload)))); return dag; } - + public static DAG createDAG(Configuration conf) throws Exception { return createDAG("SimpleTestDAG", conf); } diff --git a/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG3Vertices.java b/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG3Vertices.java index 38c0b23b35..b52709628d 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG3Vertices.java +++ b/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG3Vertices.java @@ -32,7 +32,7 @@ /** * Simple Test DAG with 3 vertices using TestProcessor/TestInput/TestOutput. - * + * * v1 * | * v2 @@ -47,8 +47,8 @@ public final class SimpleTestDAG3Vertices { public static int TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT = 2; private SimpleTestDAG3Vertices() {} - - public static DAG createDAG(String name, + + public static DAG createDAG(String name, Configuration conf) throws Exception { UserPayload payload = null; int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT; @@ -74,7 +74,7 @@ public static DAG createDAG(String name, TestInput.getInputDesc(payload)))); return dag; } - + public static DAG createDAG(Configuration conf) throws Exception { return createDAG("SimpleTestDAG3Vertices", conf); } diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java b/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java index 087c67782f..fc7ad2a03d 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java @@ -628,7 +628,7 @@ public void onSourceTaskCompleted(TaskAttemptIdentifier attempt) { } } - + /** * VertexManager which control schedule only one task when it is test case of partially-finished. * @@ -673,19 +673,19 @@ public void onVertexStarted(List completions) @Override public void onSourceTaskCompleted(TaskAttemptIdentifier attempt) throws Exception { - + } @Override public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) throws Exception { - + } @Override public void onRootVertexInitialized(String inputName, InputDescriptor inputDescriptor, List events) throws Exception { - + } } @@ -753,7 +753,7 @@ public DoNothingProcessor(ProcessorContext context) { @Override public void run() throws Exception { - // Sleep 3 second in vertex2 to avoid that vertex2 completed + // Sleep 3 second in vertex2 to avoid that vertex2 completed // before vertex2 get the SourceVertexTaskAttemptCompletedEvent. // SourceVertexTaskAttemptCompletedEvent will been ingored if vertex in SUCCEEDED, // so AM won't been killed in the VM of vertex2 diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java index 06c14c1a38..1731b1d637 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java @@ -134,7 +134,7 @@ public void setup() throws Exception { TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir); TezConfiguration tezConf = createSessionConfig(remoteStagingDir); - + tezSession = TezClient.create("TestDAGRecovery2", tezConf); tezSession.start(); } diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java b/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java index eaed04701f..eea5307515 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java @@ -212,9 +212,9 @@ private void stopNonSessionClient() { /** * verify the diagnostics in DAGStatus is correct in session mode, using local * mode for fast speed - * + * * @throws Exception - * + * */ @Test(timeout = 600000) public void testExceptionPropagationSession() throws Exception { @@ -242,7 +242,7 @@ public void testExceptionPropagationSession() throws Exception { * verify the diagnostics in {@link DAGStatus} is correct in non-session mode, * and also verify that diagnostics from {@link DAGStatus} should match that * from {@link ApplicationReport} - * + * * @throws Exception */ @Test(timeout = 120000) @@ -328,7 +328,7 @@ public static enum ExceptionLocation { /** * create a DAG with 2 vertices (v1 --> v2), set payload on Input/Output/Processor/VertexManagerPlugin to * control where throw exception - * + * * @param exLocation * @return * @throws IOException @@ -347,7 +347,7 @@ private DAG createDAG(ExceptionLocation exLocation) throws IOException { v1.setVertexManagerPlugin(RootInputVertexManagerWithException .getVMDesc(exLocation)); - Vertex v2 = + Vertex v2 = Vertex.create("v2", DoNothingProcessor.getProcDesc(), 1); v2.addDataSource("input2", DataSourceDescriptor.create(InputDescriptor.create(NoOpInput.class.getName()), @@ -848,7 +848,7 @@ public void routeDataMovementEventToDestination(DataMovementEvent event, super.routeDataMovementEventToDestination(event, sourceTaskIndex, sourceOutputIndex, destinationTaskAndInputIndices); } - + @Override public void prepareForRouting() throws Exception { if (exLocation == ExceptionLocation.EM_PrepareForRouting) { diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestFaultTolerance.java b/tez-tests/src/test/java/org/apache/tez/test/TestFaultTolerance.java index a7e6d96221..bcffd406cf 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestFaultTolerance.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestFaultTolerance.java @@ -67,7 +67,7 @@ public class TestFaultTolerance { private static String TEST_ROOT_DIR = "target" + Path.SEPARATOR + TestFaultTolerance.class.getName() + "-tmpDir"; protected static MiniDFSCluster dfsCluster; - + private static TezClient tezSession = null; private static TezConfiguration tezConf; @@ -90,11 +90,11 @@ public static void setup() throws Exception { miniTezconf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS miniTezCluster.init(miniTezconf); miniTezCluster.start(); - + Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String .valueOf(new Random().nextInt(100000)))); TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir); - + tezConf = new TezConfiguration(miniTezCluster.getConfig()); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString()); @@ -137,8 +137,8 @@ void runDAGAndVerify(DAG dag, DAGStatus.State finalState) throws Exception { void runDAGAndVerify(DAG dag, DAGStatus.State finalState, int checkFailedAttempts) throws Exception { runDAGAndVerify(dag, finalState, checkFailedAttempts, null); } - - void runDAGAndVerify(DAG dag, DAGStatus.State finalState, int checkFailedAttempts, + + void runDAGAndVerify(DAG dag, DAGStatus.State finalState, int checkFailedAttempts, String diagnostics) throws Exception { tezSession.waitTillReady(); DAGClient dagClient = tezSession.submitDAG(dag); @@ -153,7 +153,7 @@ void runDAGAndVerify(DAG dag, DAGStatus.State finalState, int checkFailedAttempt } Assert.assertEquals(finalState, dagStatus.getState()); - + if (checkFailedAttempts > 0) { Assert.assertEquals(checkFailedAttempts, dagStatus.getDAGProgress().getFailedTaskAttemptCount()); @@ -203,7 +203,7 @@ public void testBasicSuccessScatterGather() throws Exception { DAG dag = SimpleTestDAG.createDAG("testBasicSuccessScatterGather", null); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + @Test (timeout=60000) public void testBasicSuccessBroadcast() throws Exception { DAG dag = DAG.create("testBasicSuccessBroadcast"); @@ -219,7 +219,7 @@ public void testBasicSuccessBroadcast() throws Exception { TestInput.getInputDesc(null)))); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + @Test (timeout=60000) public void testBasicTaskFailure() throws Exception { Configuration testConf = new Configuration(false); @@ -229,14 +229,14 @@ public void testBasicTaskFailure() throws Exception { TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v1"), "0"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v1"), 0); - + //verify value at v2 task1 testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "1"); //value of v2 task1 is 4. - //v1 attempt0 has value of 1 (attempt index + 1). + //v1 attempt0 has value of 1 (attempt index + 1). //v1 attempt1 has value of 2 (attempt index + 1). - //v3 attempt0 verifies value of 1 + 2 (values from input vertices) + //v3 attempt0 verifies value of 1 + 2 (values from input vertices) // + 1 (attempt index + 1) = 4 testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 4); @@ -244,7 +244,7 @@ public void testBasicTaskFailure() throws Exception { DAG dag = SimpleTestDAG.createDAG("testBasicTaskFailure", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED, 1); } - + @Test (timeout=60000) public void testTaskMultipleFailures() throws Exception { Configuration testConf = new Configuration(false); @@ -254,18 +254,18 @@ public void testTaskMultipleFailures() throws Exception { TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v1"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v1"), 1); - + //v1 task0,1 attempt 2 succeed. Input sum = 6. Plus one (v2 attempt0). //ending sum is 7. testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 7); - + DAG dag = SimpleTestDAG.createDAG("testTaskMultipleFailures", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED, 4); } - + @Test (timeout=60000) public void testTaskMultipleFailuresDAGFail() throws Exception { Configuration testConf = new Configuration(false); @@ -275,11 +275,11 @@ public void testTaskMultipleFailuresDAGFail() throws Exception { TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v1"), "0"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v1"), -1); - + DAG dag = SimpleTestDAG.createDAG("testTaskMultipleFailuresDAGFail", testConf); runDAGAndVerify(dag, DAGStatus.State.FAILED); } - + @Test (timeout=60000) public void testBasicInputFailureWithExit() throws Exception { Configuration testConf = new Configuration(false); @@ -293,9 +293,9 @@ public void testBasicInputFailureWithExit() throws Exception { TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v2"), "0"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "0"); - + //v2 task1 attempt0 index0 fails and exits. - //v1 task0 attempt1 reruns. + //v1 task0 attempt1 reruns. //v2 task1 attempt1 has: // v1 task0 attempt1 (value = 2) + v1 task1 attempt0 (value = 1) // + its own value, attempt + 1 (value = 2). Total is 5. @@ -303,11 +303,11 @@ public void testBasicInputFailureWithExit() throws Exception { TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 5); - + DAG dag = SimpleTestDAG.createDAG("testBasicInputFailureWithExit", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + @Test (timeout=60000) public void testBasicInputFailureWithoutExit() throws Exception { Configuration testConf = new Configuration(false); @@ -319,7 +319,7 @@ public void testBasicInputFailureWithoutExit() throws Exception { TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v2"), "0"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "0"); - + testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( @@ -329,7 +329,7 @@ public void testBasicInputFailureWithoutExit() throws Exception { DAG dag = SimpleTestDAG.createDAG("testBasicInputFailureWithoutExit", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + @Test (timeout=60000) public void testBasicInputFailureWithoutExitDeadline() throws Exception { Configuration testConf = new Configuration(false); @@ -342,12 +342,12 @@ public void testBasicInputFailureWithoutExitDeadline() throws Exception { TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v2"), "0"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "0"); - + DAG dag = SimpleTestDAG.createDAG("testBasicInputFailureWithoutExitDeadline", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + @Test (timeout=60000) public void testMultipleInputFailureWithoutExit() throws Exception { Configuration testConf = new Configuration(false); @@ -359,7 +359,7 @@ public void testMultipleInputFailureWithoutExit() throws Exception { TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v2"), "0"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "-1"); - + //v2 task0 attempt0 input0,1 fails. wait. //v1 task0 attempt1 reruns. v1 task1 attempt1 reruns. //2 + 2 + 1 = 5 @@ -370,11 +370,11 @@ public void testMultipleInputFailureWithoutExit() throws Exception { TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 5); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 5); - + DAG dag = SimpleTestDAG.createDAG("testMultipleInputFailureWithoutExit", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + @Test (timeout=60000) public void testMultiVersionInputFailureWithoutExit() throws Exception { Configuration testConf = new Configuration(false); @@ -388,22 +388,22 @@ public void testMultiVersionInputFailureWithoutExit() throws Exception { TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "0"); testConf.setInt(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v2"), 1); - + //v2 task1 attempt0 input0 input-attempt0 fails. Wait. v1 task0 attempt1 reruns. //v2 task1 attempt0 input0 input-attempt1 fails. Wait. v1 task0 attempt2 reruns. //v2 task1 attempt0 input0 input-attempt2 succeeds. - //input values (3 + 1) + 1 = 5 + //input values (3 + 1) + 1 = 5 testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 5); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 3); - + DAG dag = SimpleTestDAG.createDAG("testMultiVersionInputFailureWithoutExit", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + @Test (timeout=60000) public void testTwoLevelsFailingDAGSuccess() throws Exception { Configuration testConf = new Configuration(); @@ -411,7 +411,7 @@ public void testTwoLevelsFailingDAGSuccess() throws Exception { .withName("testTwoLevelsFailingDAGSuccess").withConf(testConf).build(); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + @Test (timeout=60000) public void testThreeLevelsFailingDAGSuccess() throws Exception { Configuration testConf = new Configuration(); @@ -419,7 +419,7 @@ public void testThreeLevelsFailingDAGSuccess() throws Exception { .withName("testThreeLevelsFailingDAGSuccess").withConf(testConf).build(); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + @Test (timeout=60000) public void testSixLevelsFailingDAGSuccess() throws Exception { Configuration testConf = new Configuration(); @@ -427,7 +427,7 @@ public void testSixLevelsFailingDAGSuccess() throws Exception { .withName("testSixLevelsFailingDAGSuccess").withConf(testConf).build(); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + @Test (timeout=60000) public void testThreeLevelsFailingDAG2VerticesHaveFailedAttemptsDAGSucceeds() throws Exception { Configuration testConf = new Configuration(); @@ -441,7 +441,7 @@ public void testThreeLevelsFailingDAG2VerticesHaveFailedAttemptsDAGSucceeds() th //3 attempts fail testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "l2v1"), 2); - + //l3v1 failure testConf.setBoolean(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "l3v1"), true); @@ -450,11 +450,11 @@ public void testThreeLevelsFailingDAG2VerticesHaveFailedAttemptsDAGSucceeds() th //3 attempts fail testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "l3v1"), 2); - - //l2v1: task0 attempt0 succeeds. task1 attempt3 succeeds. + + //l2v1: task0 attempt0 succeeds. task1 attempt3 succeeds. //l3v1 finally task0 attempt3 will succeed. //l1v1 outputs 1. l1v2 outputs 2. - //l2v1 task0 attempt0 output = 2. + //l2v1 task0 attempt0 output = 2. //l2v2 output: attempt0 (l1v2+self = 2+1) * 3 tasks = 9 //l3v1 task0 attempt3 = l2v1 (2) + l2v2 (9) + self (4) = 15 testConf.set(TestProcessor.getVertexConfName( @@ -466,19 +466,19 @@ public void testThreeLevelsFailingDAG2VerticesHaveFailedAttemptsDAGSucceeds() th .withName("testThreeLevelsFailingDAG2VerticesHaveFailedAttemptsDAGSucceeds").withConf(testConf).build(); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + /** * Test input failure. * v1-task0 v1-task1 * | \ / | * v2-task0 v2-task1 - * + * * Use maximum allowed failed attempt of 4 (default value during session creation). * v1-task1-attempt0 fails. Attempt 1 succeeds. - * v2-task0-attempt0 runs. Its input1-inputversion0 fails. + * v2-task0-attempt0 runs. Its input1-inputversion0 fails. * This will trigger rerun of v1-task1. * v1-task1-attempt2 is re-run and succeeds. - * v2-task0-attempt0 (no attempt bump) runs. Check its input1. + * v2-task0-attempt0 (no attempt bump) runs. Check its input1. * The input version is now 2. The attempt will now succeed. * @throws Exception */ @@ -493,8 +493,8 @@ public void testInputFailureCausesRerunAttemptWithinMaxAttemptSuccess() throws E testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v1"), 0); //at v2, task 0 attempt 0 input 1 input-version 0 fails. - //This will trigger re-run of v1's task 1. - //At v1, attempt 2 will kicks off. This attempt is still ok because + //This will trigger re-run of v1's task 1. + //At v1, attempt 2 will kicks off. This attempt is still ok because //failed attempt so far at v1-task1 is 1 (not greater than 4). testConf.setBoolean(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v2"), true); @@ -508,9 +508,9 @@ public void testInputFailureCausesRerunAttemptWithinMaxAttemptSuccess() throws E //at v2-task0-attempt0/1-input1 has input failure at input version 0 only. testConf.setInt(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v2"), 1); - + //v2-task1-attempt0 takes v1-task0-attempt0 input and v1-task1-attempt1 input. - //v2-task1 does not take v1-task1-attempt2 (re-run caused by input failure + //v2-task1 does not take v1-task1-attempt2 (re-run caused by input failure //triggered by v2-task0) output. //1 + 2 + 1 = 4 testConf.set(TestProcessor.getVertexConfName( @@ -520,20 +520,20 @@ public void testInputFailureCausesRerunAttemptWithinMaxAttemptSuccess() throws E // Work-around till TEZ-877 gets fixed //testConf.setInt(TestProcessor.getVertexConfName( // TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 4); - + DAG dag = SimpleTestDAG.createDAG( "testInputFailureCausesRerunAttemptWithinMaxAttemptSuccess", testConf); //Job should succeed. runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + /** * Sets configuration for cascading input failure tests that * use SimpleTestDAG3Vertices. * @param testConf configuration - * @param failAndExit whether input failure should trigger attempt exit + * @param failAndExit whether input failure should trigger attempt exit */ - private void setCascadingInputFailureConfig(Configuration testConf, + private void setCascadingInputFailureConfig(Configuration testConf, boolean failAndExit) { // v2 attempt0 succeeds. // v2 task0 attempt1 input0 fails up to version 0. @@ -566,7 +566,7 @@ private void setCascadingInputFailureConfig(Configuration testConf, TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v3"), 0); } - + /** * Test cascading input failure without exit. Expecting success. * v1 -- v2 -- v3 @@ -574,7 +574,7 @@ private void setCascadingInputFailureConfig(Configuration testConf, * v2 task0 attempt1 input0 fails. Wait. Triggering v1 rerun. * v1 attempt1 rerun and succeeds. v2 accepts v1 attempt1 output. v2 attempt1 succeeds. * v3 attempt0 accepts v2 attempt1 output. - * + * * AM vertex succeeded order is v1, v2, v1, v2, v3. * @throws Exception */ @@ -582,7 +582,7 @@ private void setCascadingInputFailureConfig(Configuration testConf, public void testCascadingInputFailureWithoutExitSuccess() throws Exception { Configuration testConf = new Configuration(false); setCascadingInputFailureConfig(testConf, false); - + //v2 task0 attempt1 value = v1 task0 attempt1 (2) + v1 task1 attempt0 (1) + 2 = 5 //v3 all-tasks attempt0 takes v2 task0 attempt1 value (5) + v2 task1 attempt0 (3) + 1 = 9 testConf.set(TestProcessor.getVertexConfName( @@ -591,12 +591,12 @@ public void testCascadingInputFailureWithoutExitSuccess() throws Exception { TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 9); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 1), 9); - + DAG dag = SimpleTestDAG3Vertices.createDAG( "testCascadingInputFailureWithoutExitSuccess", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + /** * Test cascading input failure with exit. Expecting success. * v1 -- v2 -- v3 @@ -604,7 +604,7 @@ public void testCascadingInputFailureWithoutExitSuccess() throws Exception { * v2 task0 attempt1 input0 fails. v2 attempt1 exits. Triggering v1 rerun. * v1 attempt1 rerun and succeeds. v2 accepts v1 attempt1 output. v2 attempt2 succeeds. * v3 attempt1 accepts v2 attempt2 output. - * + * * AM vertex succeeded order is v1, v2, v3, v1, v2, v3. * @throws Exception */ @@ -612,7 +612,7 @@ public void testCascadingInputFailureWithoutExitSuccess() throws Exception { public void testCascadingInputFailureWithExitSuccess() throws Exception { Configuration testConf = new Configuration(false); setCascadingInputFailureConfig(testConf, true); - + //v2 task0 attempt2 value = v1 task0 attempt1 (2) + v1 task1 attempt0 (1) + 3 = 6 //v3 all-tasks attempt1 takes v2 task0 attempt2 value (6) + v2 task1 attempt0 (3) + 2 = 11 testConf.set(TestProcessor.getVertexConfName( @@ -621,18 +621,18 @@ public void testCascadingInputFailureWithExitSuccess() throws Exception { TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 11); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 1), 11); - + DAG dag = SimpleTestDAG3Vertices.createDAG( "testCascadingInputFailureWithExitSuccess", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + /** - * Input failure of v3 causes rerun of both both v1 and v2 vertices. + * Input failure of v3 causes rerun of both both v1 and v2 vertices. * v1 v2 * \ / * v3 - * + * * @throws Exception */ @Test (timeout=60000) @@ -650,7 +650,7 @@ public void testInputFailureCausesRerunOfTwoVerticesWithoutExit() throws Excepti TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v3"), "-1"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v3"), "1"); - + //v3 attempt0: //v1 task0,1 attempt2 = 6. v2 task0,1 attempt2 = 6. //total = 6 + 6 + 1 = 13 @@ -658,32 +658,32 @@ public void testInputFailureCausesRerunOfTwoVerticesWithoutExit() throws Excepti TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 13); - + DAG dag = SimpleVTestDAG.createDAG( "testInputFailureCausesRerunOfTwoVerticesWithoutExit", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + /** - * Downstream(v3) attempt failure of a vertex connected with - * 2 upstream vertices.. + * Downstream(v3) attempt failure of a vertex connected with + * 2 upstream vertices.. * v1 v2 * \ / * v3 - * + * * @throws Exception */ @Test (timeout=60000) public void testAttemptOfDownstreamVertexConnectedWithTwoUpstreamVerticesFailure() throws Exception { Configuration testConf = new Configuration(false); - + testConf.setBoolean(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "v3"), true); testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v3"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v3"), 1); - + //v1 input = 2. v2 input = 2 //v3 attempt2 value = 2 + 2 + 3 = 7 testConf.set(TestProcessor.getVertexConfName( @@ -692,19 +692,19 @@ public void testAttemptOfDownstreamVertexConnectedWithTwoUpstreamVerticesFailure TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 7); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 1), 7); - + DAG dag = SimpleVTestDAG.createDAG( "testAttemptOfDownstreamVertexConnectedWithTwoUpstreamVerticesFailure", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + /** - * Input failure of v2,v3 trigger v1 rerun. - * Reruns can send output to 2 downstream vertices. + * Input failure of v2,v3 trigger v1 rerun. + * Reruns can send output to 2 downstream vertices. * v1 * / \ - * v2 v3 - * + * v2 v3 + * * Also covers multiple consumer vertices report failure against same producer task. * @throws Exception */ @@ -723,7 +723,7 @@ public void testInputFailureRerunCanSendOutputToTwoDownstreamVertices() throws E TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "-1"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v2"), "0"); - + testConf.setBoolean(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v3"), true); testConf.setBoolean(TestInput.getVertexConfName( @@ -736,7 +736,7 @@ public void testInputFailureRerunCanSendOutputToTwoDownstreamVertices() throws E TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v3"), "-1"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v3"), "0"); - + //both vertices trigger v1 rerun. v1 attempt1 output is 2 * 2 tasks = 4. //v2 attempt0 = 4 + 1 = 5 testConf.set(TestProcessor.getVertexConfName( @@ -748,17 +748,17 @@ public void testInputFailureRerunCanSendOutputToTwoDownstreamVertices() throws E TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 5); - + DAG dag = SimpleReverseVTestDAG.createDAG( "testInputFailureRerunCanSendOutputToTwoDownstreamVertices", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - - + + /** * SimpleTestDAG (v1,v2) has v2 task0/1 input failures triggering v1 rerun * upto version 1. - * + * * v1 attempt0 succeeds. * v2-task0-attempt0 rejects v1 version0/1. Trigger v1 attempt1. * v2-task1-attempt0 rejects v1 version0/1. Trigger v1 attempt2. @@ -780,7 +780,7 @@ public void testTwoTasksHaveInputFailuresSuccess() throws Exception { TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "0"); testConf.setInt(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v2"), 1); - + //v2 task0 accepts v1 task0 attempt2(3) and v1 task1 attempt0(1) = 4 //v2 task0 attempt0 = 1 //total = 5 @@ -791,11 +791,11 @@ public void testTwoTasksHaveInputFailuresSuccess() throws Exception { //similarly for v2 task1 testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 5); - + DAG dag = SimpleTestDAG.createDAG("testTwoTasksHaveInputFailuresSuccess", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + @Test (timeout=240000) public void testRandomFailingTasks() throws Exception { Configuration testConf = new Configuration(false); @@ -805,7 +805,7 @@ public void testRandomFailingTasks() throws Exception { .withName("testRandomFailingTasks").withConf(testConf).build(); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + @Ignore @Test (timeout=240000) public void testRandomFailingInputs() throws Exception { @@ -816,7 +816,7 @@ public void testRandomFailingInputs() throws Exception { .withName("testRandomFailingInputs").withConf(testConf).build(); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + @Test (timeout=240000) public void testNoProgress() throws Exception { Configuration testConf = new Configuration(false); @@ -827,7 +827,7 @@ public void testNoProgress() throws Exception { Vertex hung = dag.getVertex("v1"); hung.setConf(TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS, Long.toString(1000)); hung.setConf(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, Integer.toString(2)); - + // dag will fail with 2 attempts failing from vertex v1 runDAGAndVerify(dag, DAGStatus.State.FAILED, 2, "no progress"); } diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestInput.java b/tez-tests/src/test/java/org/apache/tez/test/TestInput.java index 32a4f91489..f8b1317a47 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestInput.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestInput.java @@ -58,7 +58,7 @@ public class TestInput extends AbstractLogicalInput { private static final Logger LOG = LoggerFactory .getLogger(TestInput.class); - + public static final String COUNTER_NAME = "TestInput"; Configuration conf; @@ -67,7 +67,7 @@ public class TestInput extends AbstractLogicalInput { AtomicInteger inputReady = new AtomicInteger(-1); int lastInputReadyValue = -1; int failingInputUpto = 0; - + boolean doFail = false; boolean doRandomFail = false; float randomFailProbability = 0.0f; @@ -77,7 +77,7 @@ public class TestInput extends AbstractLogicalInput { Set failingInputIndices = Sets.newHashSet(); Integer failAll = new Integer(-1); int[] inputValues; - + /** * Enable failure for this logical input */ @@ -94,7 +94,7 @@ public class TestInput extends AbstractLogicalInput { public static String TEZ_FAILING_INPUT_RANDOM_FAIL_PROBABILITY = "tez.failing-input.random-fail-probability"; /** - * Logical input will exit (and cause task failure) after reporting failure to + * Logical input will exit (and cause task failure) after reporting failure to * read. */ public static String TEZ_FAILING_INPUT_DO_FAIL_AND_EXIT = @@ -106,13 +106,13 @@ public class TestInput extends AbstractLogicalInput { public static String TEZ_FAILING_INPUT_FAILING_INPUT_INDEX = "tez.failing-input.failing-input-index"; /** - * Up to which version of the above physical inputs to fail. 0 will fail the + * Up to which version of the above physical inputs to fail. 0 will fail the * first version. 1 will fail the first and second versions. And so on. */ public static String TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT = "tez.failing-input.failing-upto-input-attempt"; /** - * Indices of the tasks in the first for which this input will fail. Comma + * Indices of the tasks in the first for which this input will fail. Comma * separated list of +ve integers. -1 means all tasks. E.g. 0 means the first * task in the vertex will have failing inputs. */ @@ -152,7 +152,7 @@ public int doRead() { synchronized (inputReady) { while (inputReady.get() <= lastInputReadyValue) { try { - LOG.info("Waiting for inputReady: " + inputReady.get() + + LOG.info("Waiting for inputReady: " + inputReady.get() + " last: " + lastInputReadyValue); inputReady.wait(); } catch (InterruptedException e) { @@ -168,13 +168,13 @@ public int doRead() { if ( (failingTaskIndices.contains(failAll) || failingTaskIndices.contains(getContext().getTaskIndex())) && - (failingTaskAttempts.contains(failAll) || + (failingTaskAttempts.contains(failAll) || failingTaskAttempts.contains(getContext().getTaskAttemptNumber())) && (lastInputReadyValue <= failingInputUpto)) { List events = Lists.newLinkedList(); if (failingInputIndices.contains(failAll)) { for (int i=0; i= getNumPhysicalInputs()) { - throwException("InputIndex: " + index.intValue() + + throwException("InputIndex: " + index.intValue() + " should be less than numInputs: " + getNumPhysicalInputs()); } if (completedInputVersion[index.intValue()] < lastInputReadyValue) { continue; // dont fail a previous version now. } - String msg = ("FailingInput: " + getContext().getUniqueIdentifier() + + String msg = ("FailingInput: " + getContext().getUniqueIdentifier() + " index: " + index.intValue() + " version: " + lastInputReadyValue); events.add(InputReadErrorEvent.create(msg, index.intValue(), lastInputReadyValue, 1, false, false, "localhost")); @@ -205,10 +205,10 @@ public int doRead() { // keep sending input read error until we receive the new input // this check breaks the loop when we see a new input version // thus, when multiple input versions arrive, this methods gets triggered - // for each version via wait-notify. But all events may have been processed in - // handleEvents() before the code reaches this point. Having this loop, makes - // it quickly exit for an older version if a newer version has been seen. - // however, if a newer version is not seen then it keeps sending input error + // for each version via wait-notify. But all events may have been processed in + // handleEvents() before the code reaches this point. Having this loop, makes + // it quickly exit for an older version if a newer version has been seen. + // however, if a newer version is not seen then it keeps sending input error // indefinitely, by design. while (lastInputReadyValue == inputReady.get()) { // keep sending events @@ -233,7 +233,7 @@ public int doRead() { } } } - if (previousAttemptReadFailed && + if (previousAttemptReadFailed && (lastInputReadyValue <= failingInputUpto)) { // if any previous attempt has failed then dont be done when we see // a previously failed input @@ -241,7 +241,7 @@ public int doRead() { done = false; } } - + } } else { // random fail @@ -249,20 +249,20 @@ public int doRead() { for (int index=0; index initialize() throws Exception { getVertexConfName(TEZ_FAILING_INPUT_DO_FAIL_AND_EXIT, vName), false); LOG.info("doFail: " + doFail + " doFailAndExit: " + doFailAndExit); if (doFail) { - for (String failingIndex : + for (String failingIndex : conf.getTrimmedStringCollection( getVertexConfName(TEZ_FAILING_INPUT_FAILING_TASK_INDEX, vName))) { LOG.info("Adding failing task index: " + failingIndex); failingTaskIndices.add(Integer.valueOf(failingIndex)); } - for (String failingIndex : + for (String failingIndex : conf.getTrimmedStringCollection( getVertexConfName(TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, vName))) { LOG.info("Adding failing task attempt: " + failingIndex); @@ -322,7 +322,7 @@ public List initialize() throws Exception { failingInputUpto = conf.getInt( getVertexConfName(TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, vName), 0); LOG.info("Adding failing input upto: " + failingInputUpto); - for (String failingIndex : + for (String failingIndex : conf.getTrimmedStringCollection( getVertexConfName(TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, vName))) { LOG.info("Adding failing input index: " + failingIndex); @@ -360,7 +360,7 @@ public void handleEvents(List inputEvents) throws Exception { " numInputs: " + getNumPhysicalInputs() + " numCompletedInputs: " + numCompletedInputs); this.completedInputVersion[dmEvent.getTargetIndex()] = dmEvent.getVersion(); - this.inputValues[dmEvent.getTargetIndex()] = + this.inputValues[dmEvent.getTargetIndex()] = dmEvent.getUserPayload().getInt(); } else if (event instanceof InputFailedEvent) { InputFailedEvent ifEvent = (InputFailedEvent) event; @@ -374,7 +374,7 @@ public void handleEvents(List inputEvents) throws Exception { } } if (numCompletedInputs == getNumPhysicalInputs()) { - int maxInputVersionSeen = -1; + int maxInputVersionSeen = -1; for (int i=0; i initialize() throws Exception { getContext().requestInitialMemory(0l, null); //Mandatory call return Collections.emptyList(); } - + void write(int value) { this.output = value; } diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java b/tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java index 8b58973237..649e4c8890 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java @@ -50,9 +50,9 @@ public class TestProcessor extends AbstractLogicalIOProcessor { private static final Logger LOG = LoggerFactory .getLogger(TestProcessor.class); - + Configuration conf; - + boolean doFail = false; boolean doRandomFail = false; float randomFailProbability = 0.0f; @@ -60,10 +60,10 @@ public class TestProcessor extends AbstractLogicalIOProcessor { Set failingTaskIndices = Sets.newHashSet(); int failingTaskAttemptUpto = 0; Integer failAll = new Integer(-1); - + int verifyValue = -1; Set verifyTaskIndices = Sets.newHashSet(); - + /** * Enable failure for this processor */ @@ -77,14 +77,14 @@ public class TestProcessor extends AbstractLogicalIOProcessor { * Probability to random fail a task attempt. Range is 0 to 1. The number is set per DAG. */ public static String TEZ_FAILING_PROCESSOR_RANDOM_FAIL_PROBABILITY = "tez.failing-processor.random-fail-probability"; - + /** * Time to sleep in the processor in milliseconds. */ public static String TEZ_FAILING_PROCESSOR_SLEEP_MS = "tez.failing-processor.sleep-ms"; /** - * The indices of tasks in the vertex for which the processor will fail. This + * The indices of tasks in the vertex for which the processor will fail. This * is a comma-separated list of +ve integeres. -1 means all fail. */ public static String TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX = @@ -95,10 +95,10 @@ public class TestProcessor extends AbstractLogicalIOProcessor { */ public static String TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT = "tez.failing-processor.failing-upto-task-attempt"; - - public static String TEZ_FAILING_PROCESSOR_VERIFY_VALUE = + + public static String TEZ_FAILING_PROCESSOR_VERIFY_VALUE = "tez.failing-processor.verify-value"; - + public static String TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX = "tez.failing-processor.verify-task-index"; @@ -128,12 +128,12 @@ void throwException(String msg) { public static String getVertexConfName(String confName, String vertexName) { return confName + "." + vertexName; } - + public static String getVertexConfName(String confName, String vertexName, int taskIndex) { return confName + "." + vertexName + "." + String.valueOf(taskIndex); } - + @Override public void initialize() throws Exception { if (getContext().getUserPayload() != null && getContext().getUserPayload().hasPayload()) { @@ -165,10 +165,10 @@ public void initialize() throws Exception { } failingTaskAttemptUpto = conf.getInt( getVertexConfName(TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, vName), 0); - LOG.info("Adding failing attempt : " + failingTaskAttemptUpto + + LOG.info("Adding failing attempt : " + failingTaskAttemptUpto + " dag: " + getContext().getDAGName()); } - + doRandomFail = conf .getBoolean(TEZ_FAILING_PROCESSOR_DO_RANDOM_FAIL, false); randomFailProbability = conf.getFloat(TEZ_FAILING_PROCESSOR_RANDOM_FAIL_PROBABILITY, 0.0f); @@ -198,16 +198,16 @@ public void run(Map inputs, } Thread.sleep(sleepMs); - + if (!doRandomFail) { // not random fail if (doFail) { if ( (failingTaskIndices.contains(failAll) || failingTaskIndices.contains(getContext().getTaskIndex())) && - (failingTaskAttemptUpto == failAll.intValue() || + (failingTaskAttemptUpto == failAll.intValue() || failingTaskAttemptUpto >= getContext().getTaskAttemptNumber())) { - String msg = "FailingProcessor: " + getContext().getUniqueIdentifier() + + String msg = "FailingProcessor: " + getContext().getUniqueIdentifier() + " dag: " + getContext().getDAGName() + " taskIndex: " + getContext().getTaskIndex() + " taskAttempt: " + getContext().getTaskAttemptNumber(); @@ -219,17 +219,17 @@ public void run(Map inputs, // random fail // If task attempt number is below limit, try to randomly fail the attempt. int taskAttemptNumber = getContext().getTaskAttemptNumber(); - int maxFailedAttempt = conf.getInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, + int maxFailedAttempt = conf.getInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS_DEFAULT); if (taskAttemptNumber < maxFailedAttempt - 1) { float rollNumber = (float) Math.random(); - String msg = "FailingProcessor random fail turned on." + - " Do a roll: " + getContext().getUniqueIdentifier() + + String msg = "FailingProcessor random fail turned on." + + " Do a roll: " + getContext().getUniqueIdentifier() + " dag: " + getContext().getDAGName() + " taskIndex: " + getContext().getTaskIndex() + " taskAttempt: " + taskAttemptNumber + " maxFailedAttempt: " + maxFailedAttempt + - " rollNumber: " + rollNumber + + " rollNumber: " + rollNumber + " randomFailProbability " + randomFailProbability; LOG.info(msg); if (rollNumber < randomFailProbability) { @@ -240,9 +240,9 @@ public void run(Map inputs, } } } - + if (inputs.entrySet().size() > 0) { - String msg = "Reading input of current FailingProcessor: " + getContext().getUniqueIdentifier() + + String msg = "Reading input of current FailingProcessor: " + getContext().getUniqueIdentifier() + " dag: " + getContext().getDAGName() + " vertex: " + getContext().getTaskVertexName() + " taskIndex: " + getContext().getTaskIndex() + @@ -267,9 +267,9 @@ public void run(Map inputs, LOG.info("Reading input: " + entry.getKey() + " inputValue= " + inputValue); sum += inputValue; } - + if (outputs.entrySet().size() > 0) { - String msg = "Writing output of current FailingProcessor: " + getContext().getUniqueIdentifier() + + String msg = "Writing output of current FailingProcessor: " + getContext().getUniqueIdentifier() + " dag: " + getContext().getDAGName() + " vertex: " + getContext().getTaskVertexName() + " taskIndex: " + getContext().getTaskIndex() + @@ -286,8 +286,8 @@ public void run(Map inputs, TestOutput output = (TestOutput) entry.getValue(); output.write(sum); } - - LOG.info("Output for DAG: " + getContext().getDAGName() + + LOG.info("Output for DAG: " + getContext().getDAGName() + " vertex: " + getContext().getTaskVertexName() + " task: " + getContext().getTaskIndex() + " attempt: " + getContext().getTaskAttemptNumber() @@ -296,8 +296,8 @@ public void run(Map inputs, .contains(new Integer(getContext().getTaskIndex()))) { if (verifyValue != -1 && verifyValue != sum) { // expected output value set and not equal to observed value - String msg = "Expected output mismatch of current FailingProcessor: " - + getContext().getUniqueIdentifier() + + String msg = "Expected output mismatch of current FailingProcessor: " + + getContext().getUniqueIdentifier() + " dag: " + getContext().getDAGName() + " vertex: " + getContext().getTaskVertexName() + " taskIndex: " + getContext().getTaskIndex() + diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestRecovery.java b/tez-tests/src/test/java/org/apache/tez/test/TestRecovery.java index b29f74f57e..c81a50c136 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestRecovery.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestRecovery.java @@ -151,7 +151,7 @@ public void testRecovery_OrderedWordCount() throws Exception { ContainerId containerId = ContainerId.newInstance( ApplicationAttemptId.newInstance(appId, 1), 1); NodeId nodeId = NodeId.newInstance("localhost", 10); - + List initGeneratedEvents = Lists.newArrayList( new TezEvent(InputDataInformationEvent.createWithObjectPayload(0, new Object()), null)); @@ -178,7 +178,7 @@ public void testRecovery_OrderedWordCount() throws Exception { new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId0, 0L, 2, null, null, null, true)), - + new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId1, 0L, 2, null, null, null, true)), @@ -394,7 +394,7 @@ public void testRecovery_HashJoin() throws Exception { new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId0, 0L, 2, null, null, null, true)), - + new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId1, 0L, 2, null, null, null, true)), @@ -402,7 +402,7 @@ public void testRecovery_HashJoin() throws Exception { new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId2, 0L, 2, null, null, null, true)), - + new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent( vertexId0, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java index 5c0f2a2b9d..bd3223490e 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java @@ -119,7 +119,7 @@ /** * Tests for Tez example jobs - * + * */ public class TestTezJobs { @@ -802,7 +802,7 @@ public static void verifyOrderedWordCountOutput(Path resultFile, FileSystem fs) Assert.assertEquals(0, currentCounter); } - + public static void verifyOutput(Path outputDir, FileSystem fs) throws IOException { FileStatus[] fileStatuses = fs.listStatus(outputDir); Path resultFile = null; @@ -831,7 +831,7 @@ public static void verifyOutput(Path outputDir, FileSystem fs) throws IOExceptio assertTrue(foundSuccessFile); verifyOrderedWordCountOutput(resultFile, fs); } - + @Test(timeout = 60000) public void testOrderedWordCount() throws Exception { String inputDirStr = "/tmp/owc-input/"; @@ -862,7 +862,7 @@ public void testOrderedWordCount() throws Exception { } } - + @Test(timeout = 60000) public void testOrderedWordCountDisableSplitGrouping() throws Exception { String inputDirStr = TEST_ROOT_DIR + "/tmp/owc-input/"; @@ -911,7 +911,7 @@ public void testSimpleSessionExample() throws Exception { Path inputDir = new Path(inputDirStr); remoteFs.mkdirs(inputDir); generateOrderedWordCountInput(inputDir, remoteFs); - String outputDirStr = "/tmp/owc-output-" + i + "/"; + String outputDirStr = "/tmp/owc-output-" + i + "/"; outputPaths[i] = outputDirStr; Path outputDir = new Path(outputDirStr); outputDirs[i] = outputDir; @@ -923,10 +923,10 @@ public void testSimpleSessionExample() throws Exception { YarnClient yarnClient = YarnClient.createYarnClient(); try { - + yarnClient.init(mrrTezCluster.getConfig()); yarnClient.start(); - + List apps = yarnClient.getApplications(); int appsBeforeCount = apps != null ? apps.size() : 0; @@ -940,10 +940,10 @@ public void testSimpleSessionExample() throws Exception { for (int i=0; i notes = Lists.newLinkedList(); - + public CriticalPathStep(TaskAttemptInfo attempt, EntityType type) { this.type = type; this.attempt = attempt; @@ -105,9 +105,9 @@ public List getNotes() { return notes; } } - + List criticalPath = Lists.newLinkedList(); - + Map attempts = Maps.newHashMap(); int maxConcurrency = 0; @@ -121,7 +121,7 @@ public CriticalPathAnalyzer(Configuration conf) { super(conf); } - @Override + @Override public void analyze(DagInfo dagInfo) throws TezException { // get all attempts in the dag and find the last failed/succeeded attempt. // ignore killed attempt to handle kills that happen upon dag completion @@ -129,7 +129,7 @@ public void analyze(DagInfo dagInfo) throws TezException { long lastAttemptFinishTime = 0; for (VertexInfo vertex : dagInfo.getVertices()) { for (TaskInfo task : vertex.getTasks()) { - for (TaskAttemptInfo attempt : task.getTaskAttempts()) { + for (TaskAttemptInfo attempt : task.getTaskAttempts()) { attempts.put(attempt.getTaskAttemptId(), attempt); if (attempt.getStatus().equals(SUCCEEDED_STATE) || attempt.getStatus().equals(FAILED_STATE)) { @@ -141,25 +141,25 @@ public void analyze(DagInfo dagInfo) throws TezException { } } } - + if (lastAttempt == null) { LOG.info("Cannot find last attempt to finish in DAG " + dagInfo.getDagId()); return; } - + createCriticalPath(dagInfo, lastAttempt, lastAttemptFinishTime, attempts); - + analyzeCriticalPath(dagInfo); if (getConf().getBoolean(DRAW_SVG, true)) { saveCriticalPathAsSVG(dagInfo); } } - + public List getCriticalPath() { return criticalPath; } - + private void saveCriticalPathAsSVG(DagInfo dagInfo) { SVGUtils svg = new SVGUtils(); String outputDir = getOutputDir(); @@ -170,7 +170,7 @@ private void saveCriticalPathAsSVG(DagInfo dagInfo) { LOG.info("Writing output to: " + outputFileName); svg.saveCriticalPathAsSVG(dagInfo, outputFileName, criticalPath); } - + static class TimeInfo implements Comparable { long timestamp; int count; @@ -179,17 +179,17 @@ static class TimeInfo implements Comparable { this.timestamp = timestamp; this.start = start; } - + @Override public int compareTo(TimeInfo o) { return Long.compare(this.timestamp, o.timestamp); } - + @Override public int hashCode() { return (int)((timestamp >> 32) ^ timestamp); } - + @Override public boolean equals(Object o) { if (this == o) { @@ -207,7 +207,7 @@ public boolean equals(Object o) { } } } - + private void determineConcurrency(DagInfo dag) { ArrayList timeInfo = Lists.newArrayList(); for (VertexInfo v : dag.getVertices()) { @@ -221,7 +221,7 @@ private void determineConcurrency(DagInfo dag) { } } Collections.sort(timeInfo); - + int concurrency = 0; TimeInfo lastTimeInfo = null; for (TimeInfo t : timeInfo) { @@ -230,7 +230,7 @@ private void determineConcurrency(DagInfo dag) { if (lastTimeInfo == null || lastTimeInfo.timestamp < t.timestamp) { lastTimeInfo = t; lastTimeInfo.count = concurrency; - concurrencyByTime.add(lastTimeInfo); + concurrencyByTime.add(lastTimeInfo); } else { // lastTimeInfo.timestamp == t.timestamp lastTimeInfo.count = concurrency; @@ -240,7 +240,7 @@ private void determineConcurrency(DagInfo dag) { // System.out.println(t.timestamp + " " + t.count); // } } - + private int getIntervalMaxConcurrency(long begin, long end) { int concurrency = 0; for (TimeInfo timeInfo : concurrencyByTime) { @@ -256,7 +256,7 @@ private int getIntervalMaxConcurrency(long begin, long end) { } return concurrency; } - + private void analyzeAllocationOverhead(DagInfo dag) { List preemptedAttempts = Lists.newArrayList(); for (VertexInfo v : dag.getVertices()) { @@ -276,7 +276,7 @@ private void analyzeAllocationOverhead(DagInfo dag) { if (step.getType() != EntityType.ATTEMPT) { continue; } - + long creationTime = attempt.getCreationTime(); long allocationTime = attempt.getAllocationTime(); long finishTime = attempt.getFinishTime(); @@ -302,7 +302,7 @@ private void analyzeAllocationOverhead(DagInfo dag) { } if (containerAttempt.getTaskInfo().getVertexInfo().getVertexId().equals( attempt.getTaskInfo().getVertexInfo().getVertexId())) { - // another task from the same vertex ran in this container. So there are multiple + // another task from the same vertex ran in this container. So there are multiple // waves for this vertex on this container. reUsesForVertex++; } @@ -312,14 +312,14 @@ private void analyzeAllocationOverhead(DagInfo dag) { // for containerAttempts that used the container while this attempt was waiting // add up time container was allocated to containerAttempt. Account for allocations // that started before this attempt was created. - containerPreviousAllocatedTime += + containerPreviousAllocatedTime += (cFinishTime - (cAllocTime > creationTime ? cAllocTime : creationTime)); } } int numVertexTasks = attempt.getTaskInfo().getVertexInfo().getNumTasks(); int intervalMaxConcurrency = getIntervalMaxConcurrency(creationTime, finishTime); double numWaves = getWaves(numVertexTasks, intervalMaxConcurrency); - + if (reUsesForVertex > 1) { step.notes.add("Container ran multiple tasks for this vertex. "); if (numWaves < 1) { @@ -333,9 +333,9 @@ private void analyzeAllocationOverhead(DagInfo dag) { if (containerPreviousAllocatedTime >= attempt.getCreationToAllocationTimeInterval()) { step.notes.add("Container was fully allocated"); } else { - step.notes.add("Container in use for " + + step.notes.add("Container in use for " + SVGUtils.getTimeStr(containerPreviousAllocatedTime) + " out of " + - SVGUtils.getTimeStr(attempt.getCreationToAllocationTimeInterval()) + + SVGUtils.getTimeStr(attempt.getCreationToAllocationTimeInterval()) + " of allocation wait time"); } } @@ -355,13 +355,13 @@ private void analyzeAllocationOverhead(DagInfo dag) { } } } - + private double getWaves(int numTasks, int concurrency) { double numWaves = (numTasks*1.0) / concurrency; numWaves = (double)Math.round(numWaves * 10d) / 10d; // convert to 1 decimal place return numWaves; } - + private void analyzeWaves(DagInfo dag) { for (int i = 0; i < criticalPath.size(); ++i) { CriticalPathStep step = criticalPath.get(i); @@ -390,7 +390,7 @@ private void analyzeWaves(DagInfo dag) { } } } - + private void analyzeStragglers(DagInfo dag) { long dagStartTime = dag.getStartTime(); long dagTime = dag.getFinishTime() - dagStartTime; @@ -413,9 +413,9 @@ private void analyzeStragglers(DagInfo dag) { long attemptExecTime = attempt.getPostDataExecutionTimeInterval(); if (avgPostDataExecutionTime * 1.25 < attemptExecTime) { step.notes - .add("Potential straggler. Post Data Execution time " + + .add("Potential straggler. Post Data Execution time " + SVGUtils.getTimeStr(attemptExecTime) - + " compared to vertex average of " + + + " compared to vertex average of " + SVGUtils.getTimeStr(avgPostDataExecutionTime)); } } @@ -423,7 +423,7 @@ private void analyzeStragglers(DagInfo dag) { LOG.debug("DAG time taken: " + dagTime + " TotalAttemptTime: " + totalAttemptCriticalTime + " DAG finish time: " + dag.getFinishTime() + " DAG start time: " + dagStartTime); } - + private void analyzeCriticalPath(DagInfo dag) { if (!criticalPath.isEmpty()) { determineConcurrency(dag); @@ -432,7 +432,7 @@ private void analyzeCriticalPath(DagInfo dag) { analyzeAllocationOverhead(dag); } } - + private void createCriticalPath(DagInfo dagInfo, TaskAttemptInfo lastAttempt, long lastAttemptFinishTime, Map attempts) { List tempCP = Lists.newLinkedList(); @@ -456,11 +456,11 @@ private void createCriticalPath(DagInfo dagInfo, TaskAttemptInfo lastAttempt, Preconditions.checkState(currentAttempt != null); Preconditions.checkState(currentAttemptStopCriticalPathTime > 0); LOG.debug("Step: " + tempCP.size() + " Attempt: " + currentAttempt.getTaskAttemptId()); - + currentStep = new CriticalPathStep(currentAttempt, EntityType.ATTEMPT); currentStep.stopCriticalPathTime = currentAttemptStopCriticalPathTime; - // consider the last data event seen immediately preceding the current critical path + // consider the last data event seen immediately preceding the current critical path // stop time for this attempt long currentStepLastDataEventTime = 0; String currentStepLastDataTA = null; @@ -489,14 +489,14 @@ private void createCriticalPath(DagInfo dagInfo, TaskAttemptInfo lastAttempt, } tempCP.add(currentStep); - + // find the next attempt on the critical path boolean dataDependency = false; // find out predecessor dependency if (currentStepLastDataEventTime > currentAttempt.getCreationTime()) { dataDependency = true; } - + long startCriticalPathTime = 0; String nextAttemptId = null; CriticalPathDependency reason = null; @@ -583,17 +583,17 @@ private void createCriticalPath(DagInfo dagInfo, TaskAttemptInfo lastAttempt, currentStep.startCriticalPathTime = startCriticalPathTime; currentStep.reason = reason; - + Preconditions.checkState(currentStep.stopCriticalPathTime >= currentStep.startCriticalPathTime); - + if (Strings.isNullOrEmpty(nextAttemptId)) { Preconditions.checkState(reason.equals(CriticalPathDependency.INIT_DEPENDENCY)); Preconditions.checkState(startCriticalPathTime == 0); // no predecessor attempt found. this is the last step in the critical path - // assume attempts start critical path time is when its scheduled. before that is + // assume attempts start critical path time is when its scheduled. before that is // vertex initialization time currentStep.startCriticalPathTime = currentStep.attempt.getCreationTime(); - + // add vertex init step long initStepStopCriticalTime = currentStep.startCriticalPathTime; currentStep = new CriticalPathStep(currentAttempt, EntityType.VERTEX_INIT); @@ -601,7 +601,7 @@ private void createCriticalPath(DagInfo dagInfo, TaskAttemptInfo lastAttempt, currentStep.startCriticalPathTime = dagInfo.getStartTime(); currentStep.reason = CriticalPathDependency.INIT_DEPENDENCY; tempCP.add(currentStep); - + if (!tempCP.isEmpty()) { for (int i=tempCP.size() - 1; i>=0; --i) { criticalPath.add(tempCP.get(i)); @@ -609,16 +609,16 @@ private void createCriticalPath(DagInfo dagInfo, TaskAttemptInfo lastAttempt, } return; } - + currentAttempt = attempts.get(nextAttemptId); currentAttemptStopCriticalPathTime = startCriticalPathTime; } } } - + @Override public CSVResult getResult() throws TezException { - String[] headers = { "Entity", "PathReason", "Status", "CriticalStartTime", + String[] headers = { "Entity", "PathReason", "Status", "CriticalStartTime", "CriticalStopTime", "Notes" }; CSVResult csvResult = new CSVResult(headers); @@ -626,8 +626,8 @@ public CSVResult getResult() throws TezException { String entity = (step.getType() == EntityType.ATTEMPT ? step.getAttempt().getTaskAttemptId() : (step.getType() == EntityType.VERTEX_INIT ? step.attempt.getTaskInfo().getVertexInfo().getVertexName() : "DAG COMMIT")); - String [] record = {entity, step.getReason().name(), - step.getAttempt().getDetailedStatus(), String.valueOf(step.getStartCriticalTime()), + String [] record = {entity, step.getReason().name(), + step.getAttempt().getDetailedStatus(), String.valueOf(step.getStartCriticalTime()), String.valueOf(step.getStopCriticalTime()), Joiner.on(";").join(step.getNotes())}; csvResult.addRecord(record); diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java index b585ec050f..a39f7e13bf 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java @@ -53,7 +53,7 @@ public abstract class TezAnalyzerBase extends Configured implements Tool, Analyzer { private static final Logger LOG = LoggerFactory.getLogger(TezAnalyzerBase.class); - + private static final String EVENT_FILE_NAME = "eventFileName"; private static final String OUTPUT_DIR = "outputDir"; private static final String SAVE_RESULTS = "saveResults"; @@ -88,7 +88,7 @@ private static Options buildOptions() { (EVENT_FILE_NAME) .desc("File with event data for the DAG").hasArg() .required(false).build(); - + Option fromSimpleHistoryOption = Option.builder().argName(FROM_SIMPLE_HISTORY).longOpt (FROM_SIMPLE_HISTORY) .desc("Event data from Simple History logging. Must also specify event file") @@ -114,11 +114,11 @@ private static Options buildOptions() { opts.addOption(help); return opts; } - + protected String getOutputDir() { return outputDir; } - + private void printUsage() { System.err.println("Analyzer base options are"); Options options = buildOptions(); @@ -139,7 +139,7 @@ public int run(String[] args) throws Exception { return -1; } saveResults = cmdLine.hasOption(SAVE_RESULTS); - + if(cmdLine.hasOption(HELP)) { printUsage(); return 0; @@ -167,7 +167,7 @@ public int run(String[] args) throws Exception { } DagInfo dagInfo = null; - + if (files.isEmpty()) { if (cmdLine.hasOption(FROM_SIMPLE_HISTORY)) { System.err.println("Event file name must be specified when using simple history"); @@ -194,7 +194,7 @@ public int run(String[] args) throws Exception { files.add(new File(outputDir + Path.SEPARATOR + dagId + ".zip")); } - + Preconditions.checkState(!files.isEmpty()); if (cmdLine.hasOption(FROM_SIMPLE_HISTORY)) { SimpleHistoryParser parser = new SimpleHistoryParser(files); diff --git a/tez-tools/analyzers/job-analyzer/src/test/java/org/apache/tez/analyzer/TestAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/test/java/org/apache/tez/analyzer/TestAnalyzer.java index ae2af65178..223fe12e64 100644 --- a/tez-tools/analyzers/job-analyzer/src/test/java/org/apache/tez/analyzer/TestAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/test/java/org/apache/tez/analyzer/TestAnalyzer.java @@ -85,9 +85,9 @@ public class TestAnalyzer { private static Configuration conf = new Configuration(); private static FileSystem fs; - + private static TezClient tezSession = null; - + private boolean usingATS = true; private boolean downloadedSimpleHistoryFile = false; private static String yarnTimelineAddress; @@ -105,7 +105,7 @@ public static void setupClass() throws Exception { setupTezCluster(); } - + @AfterClass public static void tearDownClass() throws Exception { LOG.info("Stopping mini clusters"); @@ -118,7 +118,7 @@ public static void tearDownClass() throws Exception { dfsCluster = null; } } - + private CriticalPathAnalyzer setupCPAnalyzer() { Configuration analyzerConf = new Configuration(false); analyzerConf.setBoolean(CriticalPathAnalyzer.DRAW_SVG, false); @@ -145,15 +145,15 @@ private static void setupTezCluster() throws Exception { private TezConfiguration createCommonTezLog() throws Exception { TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig()); - + tezConf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100); Path remoteStagingDir = dfsCluster.getFileSystem().makeQualified(new Path(TEST_ROOT_DIR, String .valueOf(new Random().nextInt(100000)))); - + tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString()); tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false); - + return tezConf; } @@ -165,10 +165,10 @@ private void createTezSessionATS() throws Exception { tezConf.setBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, true); tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName()); - + Path remoteStagingDir = dfsCluster.getFileSystem().makeQualified(new Path(TEST_ROOT_DIR, String .valueOf(new Random().nextInt(100000)))); - + tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString()); tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false); @@ -176,7 +176,7 @@ private void createTezSessionATS() throws Exception { tezSession = TezClient.create("TestAnalyzer", tezConf, true); tezSession.start(); } - + private void createTezSessionSimpleHistory() throws Exception { TezConfiguration tezConf = createCommonTezLog(); tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, @@ -186,7 +186,7 @@ private void createTezSessionSimpleHistory() throws Exception { Path remoteStagingDir = dfsCluster.getFileSystem().makeQualified(new Path(TEST_ROOT_DIR, String .valueOf(new Random().nextInt(100000)))); - + tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString()); tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false); @@ -194,23 +194,23 @@ private void createTezSessionSimpleHistory() throws Exception { tezSession = TezClient.create("TestFaultTolerance", tezConf, true); tezSession.start(); } - + private StepCheck createStep(String attempt, CriticalPathDependency reason) { return createStep(attempt, reason, null, null); } - - private StepCheck createStep(String attempt, CriticalPathDependency reason, + + private StepCheck createStep(String attempt, CriticalPathDependency reason, TaskAttemptTerminationCause errCause, List notes) { return new StepCheck(attempt, reason, errCause, notes); } - + private class StepCheck { String attempt; // attempt is the TaskAttemptInfo short name with regex CriticalPathDependency reason; TaskAttemptTerminationCause errCause; List notesStr; - StepCheck(String attempt, CriticalPathDependency reason, + StepCheck(String attempt, CriticalPathDependency reason, TaskAttemptTerminationCause cause, List notes) { this.attempt = attempt; this.reason = reason; @@ -230,7 +230,7 @@ List getNotesStr() { return notesStr; } } - + private void runDAG(DAG dag, DAGStatus.State finalState) throws Exception { tezSession.waitTillReady(); LOG.info("ABC Running DAG name: " + dag.getName()); @@ -247,24 +247,24 @@ private void runDAG(DAG dag, DAGStatus.State finalState) throws Exception { Assert.assertEquals(finalState, dagStatus.getState()); } - + private void verify(ApplicationId appId, int dagNum, List steps) throws Exception { String dagId = TezDAGID.getInstance(appId, dagNum).toString(); DagInfo dagInfo = getDagInfo(dagId); - + verifyCriticalPath(dagInfo, steps); } - + private DagInfo getDagInfo(String dagId) throws Exception { // sleep for a bit to let ATS events be sent from AM DagInfo dagInfo = null; if (usingATS) { //Export the data from ATS String[] args = { "--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR, "--yarnTimelineAddress=" + yarnTimelineAddress }; - + int result = ATSImportTool.process(args); assertTrue(result == 0); - + //Parse ATS data and verify results //Parse downloaded contents File downloadedFile = new File(DOWNLOAD_DIR @@ -282,7 +282,7 @@ private DagInfo getDagInfo(String dagId) throws Exception { + SIMPLE_HISTORY_DIR + HISTORY_TXT + "." + applicationAttemptId); FileSystem fs = historyPath.getFileSystem(miniTezCluster.getConfig()); - + Path localPath = new Path(DOWNLOAD_DIR, HISTORY_TXT); fs.copyToLocalFile(historyPath, localPath); } @@ -294,17 +294,17 @@ private DagInfo getDagInfo(String dagId) throws Exception { } return dagInfo; } - + private void verifyCriticalPath(DagInfo dagInfo, List stepsOptions) throws Exception { CriticalPathAnalyzer cp = setupCPAnalyzer(); cp.analyze(dagInfo); - + List criticalPath = cp.getCriticalPath(); for (CriticalPathStep step : criticalPath) { LOG.info("ABC Step: " + step.getType()); if (step.getType() == EntityType.ATTEMPT) { - LOG.info("ABC Attempt: " + step.getAttempt().getShortName() + LOG.info("ABC Attempt: " + step.getAttempt().getShortName() + " " + step.getAttempt().getDetailedStatus()); } LOG.info("ABC Reason: " + step.getReason()); @@ -324,7 +324,7 @@ private void verifyCriticalPath(DagInfo dagInfo, List stepsOptions) StepCheck check = steps[i-1]; CriticalPathStep step = criticalPath.get(i); Assert.assertEquals(CriticalPathStep.EntityType.ATTEMPT, step.getType()); - Assert.assertTrue(check.getAttemptDetail(), + Assert.assertTrue(check.getAttemptDetail(), step.getAttempt().getShortName().matches(check.getAttemptDetail())); Assert.assertEquals(steps[i-1].getReason(), step.getReason()); if (check.getErrCause() != null) { @@ -338,31 +338,31 @@ private void verifyCriticalPath(DagInfo dagInfo, List stepsOptions) } } } - + Assert.assertEquals(CriticalPathStep.EntityType.DAG_COMMIT, criticalPath.get(criticalPath.size() - 1).getType()); break; } } - + Assert.assertTrue(foundMatchingLength); - + } - + @Test (timeout=300000) public void testWithATS() throws Exception { usingATS = true; createTezSessionATS(); runTests(); } - + @Test (timeout=300000) public void testWithSimpleHistory() throws Exception { usingATS = false; createTezSessionSimpleHistory(); runTests(); } - + private void runTests() throws Exception { ApplicationId appId = tezSession.getAppMasterApplicationId(); List> stepsOptions = Lists.newArrayList(); @@ -380,23 +380,23 @@ private void runTests() throws Exception { stepsOptions.add(testInputFailureRerunCanSendOutputToTwoDownstreamVertices()); stepsOptions.add(testCascadingInputFailureWithExitSuccess()); stepsOptions.add(testInternalPreemption()); - + // close session to flush if (tezSession != null) { tezSession.stop(); } Thread.sleep((TezConstants.TEZ_DAG_SLEEP_TIME_BEFORE_EXIT*3)/2); - + // verify all dags for (int i=0; i testBasicSuccessScatterGather() throws Exception { Configuration testConf = new Configuration(false); testConf.setInt(SimpleTestDAG.TEZ_SIMPLE_DAG_NUM_TASKS, 1); - StepCheck[] check = { + StepCheck[] check = { createStep("v1 : 000000_0", CriticalPathDependency.INIT_DEPENDENCY), createStep("v2 : 000000_0", CriticalPathDependency.DATA_DEPENDENCY) }; @@ -404,7 +404,7 @@ private List testBasicSuccessScatterGather() throws Exception { runDAG(dag, DAGStatus.State.SUCCEEDED); return Collections.singletonList(check); } - + private List testBasicTaskFailure() throws Exception { Configuration testConf = new Configuration(false); testConf.setInt(SimpleTestDAG.TEZ_SIMPLE_DAG_NUM_TASKS, 1); @@ -424,7 +424,7 @@ private List testBasicTaskFailure() throws Exception { runDAG(dag, DAGStatus.State.SUCCEEDED); return Collections.singletonList(check); } - + private List testTaskMultipleFailures() throws Exception { Configuration testConf = new Configuration(false); testConf.setInt(SimpleTestDAG.TEZ_SIMPLE_DAG_NUM_TASKS, 1); @@ -440,13 +440,13 @@ private List testTaskMultipleFailures() throws Exception { createStep("v1 : 000000_1", CriticalPathDependency.RETRY_DEPENDENCY), createStep("v1 : 000000_2", CriticalPathDependency.RETRY_DEPENDENCY), createStep("v2 : 000000_0", CriticalPathDependency.DATA_DEPENDENCY), - }; - + }; + DAG dag = SimpleTestDAG.createDAG("testTaskMultipleFailures", testConf); runDAG(dag, DAGStatus.State.SUCCEEDED); return Collections.singletonList(check); } - + private List testBasicInputFailureWithExit() throws Exception { Configuration testConf = new Configuration(false); testConf.setInt(SimpleTestDAG.TEZ_SIMPLE_DAG_NUM_TASKS, 1); @@ -467,12 +467,12 @@ private List testBasicInputFailureWithExit() throws Exception { createStep("v1 : 000000_1", CriticalPathDependency.OUTPUT_RECREATE_DEPENDENCY), createStep("v2 : 000000_1", CriticalPathDependency.DATA_DEPENDENCY), }; - + DAG dag = SimpleTestDAG.createDAG("testBasicInputFailureWithExit", testConf); runDAG(dag, DAGStatus.State.SUCCEEDED); return Collections.singletonList(check); } - + private List testBasicInputFailureWithoutExit() throws Exception { Configuration testConf = new Configuration(false); testConf.setInt(SimpleTestDAG.TEZ_SIMPLE_DAG_NUM_TASKS, 1); @@ -554,14 +554,14 @@ private List testMultiVersionInputFailureWithoutExit() throws Excep runDAG(dag, DAGStatus.State.SUCCEEDED); return Collections.singletonList(check); } - + /** * Sets configuration for cascading input failure tests that * use SimpleTestDAG3Vertices. * @param testConf configuration - * @param failAndExit whether input failure should trigger attempt exit + * @param failAndExit whether input failure should trigger attempt exit */ - private void setCascadingInputFailureConfig(Configuration testConf, + private void setCascadingInputFailureConfig(Configuration testConf, boolean failAndExit, int numTasks) { // v2 attempt0 succeeds. @@ -596,7 +596,7 @@ private void setCascadingInputFailureConfig(Configuration testConf, TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v3"), 0); } - + /** * Test cascading input failure without exit. Expecting success. * v1 -- v2 -- v3 @@ -604,7 +604,7 @@ private void setCascadingInputFailureConfig(Configuration testConf, * v2 task0 attempt1 input0 fails. Wait. Triggering v1 rerun. * v1 attempt1 rerun and succeeds. v2 accepts v1 attempt1 output. v2 attempt1 succeeds. * v3 attempt0 accepts v2 attempt1 output. - * + * * AM vertex succeeded order is v1, v2, v1, v2, v3. * @throws Exception */ @@ -621,13 +621,13 @@ private List testCascadingInputFailureWithoutExitSuccess() throws E createStep("v2 : 000000_1", CriticalPathDependency.DATA_DEPENDENCY), createStep("v3 : 000000_0", CriticalPathDependency.DATA_DEPENDENCY), }; - + DAG dag = SimpleTestDAG3Vertices.createDAG( "testCascadingInputFailureWithoutExitSuccess", testConf); runDAG(dag, DAGStatus.State.SUCCEEDED); return Collections.singletonList(check); } - + /** * Test cascading input failure with exit. Expecting success. * v1 -- v2 -- v3 @@ -635,14 +635,14 @@ private List testCascadingInputFailureWithoutExitSuccess() throws E * v2 task0 attempt1 input0 fails. v2 attempt1 exits. Triggering v1 rerun. * v1 attempt1 rerun and succeeds. v2 accepts v1 attempt1 output. v2 attempt2 succeeds. * v3 attempt1 accepts v2 attempt2 output. - * + * * AM vertex succeeded order is v1, v2, v3, v1, v2, v3. * @throws Exception */ private List testCascadingInputFailureWithExitSuccess() throws Exception { Configuration testConf = new Configuration(false); setCascadingInputFailureConfig(testConf, true, 1); - + StepCheck[] check = { createStep("v1 : 000000_0", CriticalPathDependency.INIT_DEPENDENCY), createStep("v2 : 000000_0", CriticalPathDependency.DATA_DEPENDENCY), @@ -658,9 +658,9 @@ private List testCascadingInputFailureWithExitSuccess() throws Exce runDAG(dag, DAGStatus.State.SUCCEEDED); return Collections.singletonList(check); } - + /** - * 1 NM is running and can run 4 containers based on YARN mini cluster defaults and + * 1 NM is running and can run 4 containers based on YARN mini cluster defaults and * Tez defaults for AM/task memory * v3 task0 reports read errors against both tasks of v2. This re-starts both of them. * Now all 4 slots are occupied 1 AM + 3 tasks @@ -677,7 +677,7 @@ private List testInternalPreemption() throws Exception { StepCheck[] check = { createStep("v1 : 00000[01]_0", CriticalPathDependency.INIT_DEPENDENCY), createStep("v2 : 00000[01]_0", CriticalPathDependency.DATA_DEPENDENCY), - createStep("v3 : 000000_0", CriticalPathDependency.DATA_DEPENDENCY, + createStep("v3 : 000000_0", CriticalPathDependency.DATA_DEPENDENCY, TaskAttemptTerminationCause.INTERNAL_PREEMPTION, null), createStep("v2 : 00000[01]_1", CriticalPathDependency.OUTPUT_RECREATE_DEPENDENCY), createStep("v1 : 000000_1", CriticalPathDependency.OUTPUT_RECREATE_DEPENDENCY, @@ -685,19 +685,19 @@ private List testInternalPreemption() throws Exception { createStep("v2 : 00000[01]_[12]", CriticalPathDependency.DATA_DEPENDENCY), createStep("v3 : 000000_1", CriticalPathDependency.DATA_DEPENDENCY) }; - + DAG dag = SimpleTestDAG3Vertices.createDAG( "testInternalPreemption", testConf); runDAG(dag, DAGStatus.State.SUCCEEDED); return Collections.singletonList(check); } - + /** - * Input failure of v3 causes rerun of both both v1 and v2 vertices. + * Input failure of v3 causes rerun of both both v1 and v2 vertices. * v1 v2 * \ / * v3 - * + * * @throws Exception */ private List testInputFailureCausesRerunOfTwoVerticesWithoutExit() throws Exception { @@ -715,7 +715,7 @@ private List testInputFailureCausesRerunOfTwoVerticesWithoutExit() TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v3"), "-1"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v3"), "1"); - + StepCheck[] check = { // use regex for either vertices being possible on the path createStep("v[12] : 000000_0", CriticalPathDependency.INIT_DEPENDENCY), @@ -735,17 +735,17 @@ private List testInputFailureCausesRerunOfTwoVerticesWithoutExit() runDAG(dag, DAGStatus.State.SUCCEEDED); return Collections.singletonList(check); } - + /** - * Downstream(v3) attempt failure of a vertex connected with - * 2 upstream vertices.. + * Downstream(v3) attempt failure of a vertex connected with + * 2 upstream vertices.. * v1 v2 * \ / * v3 - * + * * @throws Exception */ - private List testAttemptOfDownstreamVertexConnectedWithTwoUpstreamVerticesFailure() + private List testAttemptOfDownstreamVertexConnectedWithTwoUpstreamVerticesFailure() throws Exception { Configuration testConf = new Configuration(false); testConf.setInt(SimpleVTestDAG.TEZ_SIMPLE_V_DAG_NUM_TASKS, 1); @@ -755,7 +755,7 @@ private List testAttemptOfDownstreamVertexConnectedWithTwoUpstreamV TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v3"), "0"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v3"), 1); - + StepCheck[] check = { // use regex for either vertices being possible on the path createStep("v[12] : 000000_0", CriticalPathDependency.INIT_DEPENDENCY), @@ -769,17 +769,17 @@ private List testAttemptOfDownstreamVertexConnectedWithTwoUpstreamV runDAG(dag, DAGStatus.State.SUCCEEDED); return Collections.singletonList(check); } - + /** * Input failure of v2,v3 trigger v1 rerun. * Both v2 and v3 report error on v1 and dont exit. So one of them triggers next * version of v1 and also consume the output of the next version. While the other - * consumes the output of the next version of v1. - * Reruns can send output to 2 downstream vertices. + * consumes the output of the next version of v1. + * Reruns can send output to 2 downstream vertices. * v1 * / \ - * v2 v3 - * + * v2 v3 + * * Also covers multiple consumer vertices report failure against same producer task. * @throws Exception */ @@ -798,7 +798,7 @@ private List testInputFailureRerunCanSendOutputToTwoDownstreamVerti TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "-1"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v2"), "0"); - + testConf.setBoolean(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v3"), true); testConf.setBoolean(TestInput.getVertexConfName( @@ -811,7 +811,7 @@ private List testInputFailureRerunCanSendOutputToTwoDownstreamVerti TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v3"), "-1"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v3"), "0"); - + StepCheck[] check = { // use regex for either vertices being possible on the path createStep("v1 : 000000_0", CriticalPathDependency.INIT_DEPENDENCY), @@ -824,5 +824,5 @@ private List testInputFailureRerunCanSendOutputToTwoDownstreamVerti runDAG(dag, DAGStatus.State.SUCCEEDED); return Collections.singletonList(check); } - + }