+ jn ^RIt^RIt^RIt^RIt^RIt^RIHtHt^RIH t H t ^RI H t ^RI Ht^RIHt^RIHt^RIHtHtHtHtHt^RIt^RIHt^R IHt^R IH t H!t!^R I"H#t#^R I$H%t%^R I&H't'H(t(H)t)^RI*H+t+H,t,H-t-H.t.^RI/H0t0H1t1H2t2.RSOt3]Ph!]54t6!RR]7]4t8]8Prt9]8Ptt:]8Pvt;]8Pxt<]8Pzt=]8P|t>]8P~t?]8Pt@]8PtA]8PtB]8PtC]8PtD].!^4tE]9tF]:tG];tH]BtI]P!R4tK!RR]4tL]RRl4tMRTR R!lltN!R"R#]4tORTR$R%lltPRTR&R'lltQR(R)ltR!R*R]O4tS!R+R,]S4tT!R-R]S4tU!R.R]S4tVR/R0ltWR1R2ltXRUR3R4lltYR5R6ltZR7R8lt[R9R:lt\R;R<lt]!R=R]O4t^]!R>R?44t_!R@RA]4t`!RBRC]^4ta!RDR]a4tbRVREltcRWRFRGlltd!RHR]a4te!RIR]a4tf!RJR]a4tg!RKR]a4thRLRMltiRNROltjRPtkRQRRltlR#)XN)ABCabstractmethod)Counter defaultdict)Callable) dataclass)Enum) lru_cache)AnycastLiteral NamedTupleProtocol)OptimizedModule) FSDPModule UnshardHandle)_Loss)record_function)generate_rank_to_stage_mappinggenerate_stage_to_rank_mapping InferenceMode) _split_tensor merge_chunkssplit_args_kwargs_into_chunksTensorChunkSpec)_PipelineStageBase _RecvInfo PipelineStagePipelineScheduleSinglePipelineScheduleMulti Schedule1F1B ScheduleGPipeScheduleInterleaved1F1BScheduleLoopedBFSScheduleInterleavedZeroBubbleScheduleZBVZeroBubbleScheduleDualPipeVcla]tRt^6toRtRtRtRtRtRt Rt Rt R t R t R tR t]V3R lRl4tRtVtR#)_ComputationTypeFIWUNSHARDRESHARDSEND_FRECV_FSEND_BRECV_BB OVERLAP_F_B REDUCE_GRADc$<V^8dQhRS[RR/#)actionreturnr)str)format __classdict__s"g/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torch/distributed/pipelining/schedules.py __annotate___ComputationType.__annotate__Fs NNN!3Nc^\V4# \dp\RT 24ThRp?ii;i)Invalid computation type N)r) ValueError RuntimeError)r8excs& r>from_str_ComputationType.from_strEs: N#F+ + N!:6(CD# M Ns ,',N)__name__ __module__ __qualname____firstlineno__FORWARDBACKWARD_INPUTBACKWARD_WEIGHTr-r.r/r0r1r2 FULL_BACKWARDr4r5 staticmethodrG__static_attributes____classdictcell__r=s@r>r)r)6sUGNOGG F F F FMKKNNrAr)zK(\d+)(F|I|B|W|UNSHARD|RESHARD|REDUCE_GRAD|SEND_F|RECV_F|SEND_B|RECV_B)(\d*)cxa]tRt^mtoRtRtRtRt]V3RlRl4t ] V3RlRl4t V3Rlt R t VtR#) _ActionNc"VP4#N)__repr__selfs&r>__str___Action.__str__ss}}rAcVPePVPUu.uFp\V4NK ppRRPV4 RVPP 2#\ VP 4pW0PP, pVPeV\ VP4, pV#uupi)N(;)) sub_actionsreprjoincomputation_typevaluer; stage_indexmicrobatch_index)r\ sub_actionsub_action_reprsrepr_strs& r>rZ_Action.__repr__vs    'CGCSCSTCSZZ 0CS Tsxx 012!D4I4I4O4O3PQ Q4++,H --33 3H$$0C 5 566O UsB;c <V^8dQhRS[/#)r7r9)bool)r<r=s"r>r?_Action.__annotate__s  t rAcRVP\\\\\ 39#rY)rfrNrQrOrPr4r[s&r> is_compute_op_Action.is_compute_ops)$$      )   rAc <V^8dQhRS[/#)r7 action_stringr:)r<r=s"r>r?rps3 3 3 rAc VP4pVR8XdR#VPR4'dRV9dVPR4pV^VpVV^,Rp.pVP4'dSVPR4F=p\P VP44pVfK,VP V4K? \ R \P V4RV'd\V4R7#RR7#\PV4;p'd]VP4wrp \ \V4\P V 4\V 4'd\V 44#R4#VR8XdR#\RV R24h) z~ Reverse of __repr__ String should be formatted as [stage][action type][(microbatch)] e.g. `2F0`, `1UNSHARD`, `3SEND_F1` Nr`rbra)rhrfrirczInvalid action string: zD, should be formatted as [stage][action type][(microbatch)] e.g. 2F0)strip startswithfindsplitrWrGappendr)tuple _action_regexmatchgroupsintlenrE) ru bracket_endsub_partcomputation_type_partrcsub_strrjrrhrfris & r>rG_Action.from_strs&++- B   # #C ( (SM-A',,S1K$+H%2a!% ! K~~'~~c2G!(!1!1'--/!BJ!-#**:6 3!1!:!:;P!Q!%2=E+.  DH  "'' 6 65 6>Clln ;K+;K  ))*:;),-=)>)>$% EI  b %m_4x y  rAct<V^8dQh/S[;R&S[;R&S[R,;R&S[R,R,;R&#)r7rhrfNrirc)rW.)rr)r~)r<r=s"r>r?rpmsC&&Dj'~&-4 rArI)rJrKrLrMrircr]rZpropertyrrrRrG__annotate_func__rSrTrUs@r>rWrWmsL$(04K   3 3 CrArWc0V^8dQhR\R\/#r7r8r9)rWr;)r<s"r>r?r?sCrAcR\V4 2#)zPP:r:)r8s&r>_get_profiler_function_namers V rAcV^8dQhR\\\\R,,3,R\R,R\/#)r7pipeline_orderNerror_step_numberr9)dictrlistrWr;)r<s"r>r?r?s=<<d7T>223<Tz< <rAc a\P!V4pVFBp\\W,44F!pW,V,eKRW,V&K# KD \ RVP 444p\V4Uu.uF=pR\ V4P\\ V^, 444,NK? pp\V4Uu.uFq`PVR.V,4NK pp\\P!VRR/4p\V4p \V 4Uu.uFpR\ V4,NK p p\V.VO5!U u.uFp \ RV 44NK up oR\V^,4^,,RPV3Rl\V 444,p \WX4U Uu.uFewrV R 2RPV3R l\V444,Ve(\!V P#4^,4V8XdR MR,NKg pp pV R ,R PV4,R ,pV#uupiuupiuupiuup iuupp i) z Formats the pipeline order in a timestep (row) x rank (column) grid of actions and returns the formatted string. If `error_step_number` is passed in, an additional label will be added to signify which step that it is erroring on. rwc38"TFp\V4xK R#5irY)r).0actionss& r> )_format_pipeline_order..sH0GWCLL0GzStep fillvalueRank c3T"TFqe\\V44M^xK R#5irY)rr;)ritems& r>rrs F#$.CD NA 5#s&( c3D<"TFwrVRSV, 2 xK R#5irrs+<5K5;q>"" #$5K z: c3V<"TFwr\V4RSV, 2 xK R#5irr:)rrrrs& r>rrs+R>c$i+a.!1123>s&)z <-- ERROR HERE )copydeepcopyrangermaxvaluesr;zfillsortedgetr itertools zip_longestzipre enumeraterr|)rrrankr num_steps step_labelskey rank_actionstransposed_actions num_ranks rank_labelscol header_rowrrowformatted_rowsformatted_tablers&& @r>_format_pipeline_orderrsj]]>2Ns>/01A#A&.*,$Q'2H0E0E0GHHIAFyAQAQA#a&,,s3y1}#56777AQ >DN=S=Sc3y 01=S i33\PRPQN#I-29-=>-=7SV##-=K>{8%788C F# FF8K KN+a/0388<5>{5K<4Jk> ?JE ' ((R9S>R R S!,EKKM!$%)::    ? !4'$))N*CCdJO I? s AI1"I6:I;&JA+JcTa]tRtRtoR$V3RlRlltR%RltRtRtV3R lR ltR%V3R lR llt ] R&V3R lRll4t ] RRRRRRRRRRRRRR/V3RlRll4t RRRRRRRRRR/V3RlRllt R'V3RlRlltR%RltR%V3RlRlltV3RlR ltV3R!lR"ltR#tVtR#)(_PipelineScheduleiNTc <V^8dQhRS[RS[RS[P3,R,RS[S[R3,R,RS[S[S[3,R,RS[S[S[3,S[S[,,R,RS[ /#) r7n_microbatchesloss_fn.Nargs_chunk_speckwargs_chunk_specoutput_merge_spec scale_grads) rrtorchTensorr~rrr;r ro)r<r=s"r>r?_PipelineSchedule.__annotate__ s 9 9 9#u||+,t3 934t; 9  _ 45< 9  S>E#J6= 9 9rAcWnW nW`nW0nW@nWPnVPRJVn.Vn\PRVPP4R#)NzUsing %s) _n_microbatches_loss_fnr_args_chunk_spec_kwargs_chunk_spec_output_merge_spec _has_backward_internal_lossesloggerinfo __class__rJ)r\rrrrrrs&&&&&&&r>__init___PipelineSchedule.__init__ se . '!0"3"3 "]]$657 J 7 78rAcVP'dGVPe7VPW#V,V4pVPP V4R#R#R#rY)is_lastr _compute_lossrr})r\stageoutput target_mbsmb_index loss_kwargslosss&&&&&& r>_maybe_compute_loss%_PipelineSchedule._maybe_compute_loss+sH ===T]]6%%f.BKPD  ! ! ( ( .7=rAcH^Tu;8*;'d\VP48MupVP'd*VPeV'dVPV,#\VP4^8wd$V'g\ RV RVP 24hR#)rNzLoss for microbatch z6 is not available. Available losses for microbatches: )rrrrrE)r\rr valid_indexs&&& r>_maybe_get_loss!_PipelineSchedule._maybe_get_loss2s8@@c$*?*?&@@ ===T]]6;((2 2 && '1 ,[&xj166:6K6K5LN  rAc \V\4'gV.p\;QJdRV4F 'gK RM RM !RV44pV'dVe~\VP4VP 8wd/\ RVP R\VP4 24hVP4VPVP4VPP4R#)z2 Update the losses to those in the internal state c38"TFqPxK R#5irYrrrs& r>r3_PipelineSchedule._update_losses..Es!DVE--VrTFN Expecting z losses but got ) isinstanceranyrrrrEclearextend)r\stageslossescontains_last_stages&&& r>_update_losses _PipelineSchedule._update_losses>s &$''XF!c!DV!Dccc!DV!DD 6#54(()T-A-AA" !5!5 66Fs4K`K`GaFbc LLN MM$// 0 ##%rAc@<V^8dQhRS[S[,RS[RS[RR/#)r7r has_backwardp2p_doner9N)rrro)r<r=s"r>r?rUs?N1N1'(N1N1 N1  N1rAc \;QJdRV4F 'dK RM RM !RV44'Ed\\\,V4p\;QJdRV4F 'gK RM RM !RV44pV'd\;QJdRV4F 'gK RM RM !RV44'dVFXp\ P !VPV4'd\RVP R24h\ PVn KZ \PR\V44R #R p\\\,V4FpVPW'R 7pK Tp R p \!\\\,V44F_pVP#V R 7p V f \R 4hV P%4^8Xd\ PM\ P&p Wn Ka \PR V 'd V P(MR\V44R #V'gA.p VF"pV P+VP-44K$ \/\1V 44R #R #)uFRun the P2P warm-up protocol for the given stages. For ``PipelineStage`` instances this executes the forward/backward vote protocol (which warms up 2-rank sub-communicators) and sets each stage's ``_inference_mode``. For other stage types it falls back to the legacy ``_get_init_p2p_neighbors_ops`` + ``_batch_p2p`` path. Args: stages: The pipeline stages owned by this rank. has_backward: Whether the schedule includes a backward pass. p2p_done: ``True`` if P2P neighbours have already been initialised (avoids redundant init on eval↔train mode switches). c3B"TFp\V\4xK R#5irY)rrrs& r>r0_PipelineSchedule._warmup_p2p..hsDVEz%//VsFTc32"TFpVP'*;'d0VPVP^, 4'*;'gAVP'*;'d(VPVP^,4'*xK R#5iN)is_first _is_same_rankrhrrsts& r>rrrsx!$B[[MM)9)9"..1:L)M%MQQ NOO2+;+;BNNQrrws'&?H  *f4ys02zStage z requires dynamic shape inference, which is not supported with a fake process group. Provide complete static metadata (inputs/outputs, plus input_grads/output_grads for DTensors with backward) to the PipelineStage constructor.zUFake process group detected; set inference_mode=static for %d stage(s) without votingN) received_acc)received_resultzP2P warm-up voting failedz1Rank determined inference_mode=%s for %d stage(s)None)allr rrrr needs_dynamic _user_metarErhSTATIC_inference_moderdebugr_warmup_forward_votereversed_warmup_backward_resultrDYNAMICrgr_get_init_p2p_neighbors_ops_wait_batch_p2p _batch_p2p) r\rrr pp_stageshas_cross_rankraccrresultdetermined_modeall_opss &&&& r> _warmup_p2p_PipelineSchedule._warmup_p2pUs& 3DVD333DVD D DT-0&9I S!$!SSS!$!N ##&?H&###&?H&##$B$222==,OO*$R^^$45++*7)=)=B&$ 5K '+Cd=16:000P;*-F48O!$tM':F"CD66v6N>&'BCC{{})"((&.. )8%E LLC)8%%fF   (*Gu@@BC Jw/ 0 rAc<V^8dQhRS[S[,RS[S[R3,S[,RS[S[S[3,R,RS[RS[RS[R S[S[S[3,R,R S[S[S[3,/#) r7rargs.kwargsNtargetfwd_initializedbwd_initializedrr9)rrr~r rr;ro)r<r=s"r>r?rsW0W0'(W0CHo#W0S#X% W0  W0  W0W0#s(^d*W0 tTz W0rAc  aTpV'dVPV8wdRpRpV'*p VP;'dV'*p V 'g V 'gWV3#V 'dVPWPV4\VU u0uFTp \P!V P4P R8wgK4\P!V P4kKV up 4p V 'dV ^,P MRoSei\ ;QJdV3RlV 4F 'dK RM RM!V3RlV 44'g)V U u0uFqP kK pp \RV 24hVU u.uFp \V \4'gKV NK pp VFp V P4K \PPV SR7;_uu_4V 'dMRpVFBp V P'dTMTpV PVPVVVPR7pKD RpV 'dARp\!V4F-p V P#VPVP$VVVR 7pK/ RpRRR4VFp V P'4K WV3#uup iuup iuup i +'giL;;i TFp T P'4K i;i) uCommon stage initialization shared by Single and Multi schedules. Handles mode-change detection (eval↔train), P2P warm-up, RNG forking, forward / backward metadata inference, and FSDP cleanup. Returns the updated ``(fwd_initialized, bwd_initialized)`` flags. FcpuNc3@<"TFqPS8HxK R#5irY)type)rdevice device_types& r>r:_PipelineSchedule._initialize_pp_stages..s/ 5<6KK; &WsTzOAll stages must have the same device type for RNG forking. Found device types: )devicesr0)r)rr(received_grad_metar)rr#rrr/r.rAssertionErrorrr_pre_metadata_inference_backuprandomfork_rngr_prepare_forward_infrarr_prepare_backward_infrar _post_metadata_inference_cleanup)r\rr&r'r(r)r*rr needs_fwd needs_bwdrr2r/ device_typespipeline_stagesnext_stage_args stage_argsprev_stage_grad_metar0s&&&&&&&& @r>_initialize_pp_stages'_PipelineSchedule._initialize_pp_stagess(#  2 2o E#O#O'' &&>>+> "3 3    V%7%7 B$ #E<< -22e;+ U\\*#  *1gajood  "33/ 5</ 333/ 5</ , , 7>>gFKKgL> ''3n6  & %eE=)IEEv  %E  0 0 2% 9&&wK&PP+/O!'-2^^^T */*F*F 00&")-);); +G+"('+O04(!)&!1/4/L/L 00$(MM#)/C(3 0M0,"2'+O/Q2)668)//g ?  QP2)668)sJ1J9$J J1J$J$0)J<AJ)6AJ)6J<) J9 4J<<Kc<V^8dQhRS[R,RS[R,RS[R,RS[R,RS[RS[S[S[3,R,/#r7arg_mbsN kwarg_mbsrrreturn_outputsrrrorr;r )r<r=s"r>r?r sd"""$;"4K " t "  "#s(^d*"rAc \h)aX Run one iteration of the pipeline schedule with list of microbatches. Will go through all the microbatches according to the schedule implementation. Args: microbatches: list of microbatch args. return_outputs: whether to return the outputs from the last stage. loss_kwargs: extra keyword arguments forwarded to the loss function. NotImplementedError)r\rFrGrrrHrs&&&&&&&r>_step_microbatches$_PipelineSchedule._step_microbatches s ("!rAr(rrHrrFrGrc j<V^8dQhRS[R,RS[S[S[3,R,RS[RS[RS[/#)r7rNrrFrGr)rrr;r )r<r=s"r>r?r!sP " "t " #s(^d* " " " "rAc \h)z+Run one iteration of the pipeline schedule.rK) r\r(rrHrrFrGrr&r's &$$$$$$$*,r>step_PipelineSchedule.step s "!rAc@<V^8dQhRS[R,RS[RS[RS[/#)r7rNrFrGrrr )r<r=s"r>r?r0s?J7J7t J7  J7  J7J7rAc  VPpRVnVP!VRVRVRVRVRV/VBWn# Yni;i)aURun one forward-only iteration of the pipeline schedule. ``eval`` uses the same input contract as ``step``, but temporarily disables backward execution. By default, ``args``, ``kwargs``, and ``target`` are full-batch values that the schedule splits into microbatches. When the caller has already split inputs into microbatches, pass them through ``arg_mbs``, ``kwarg_mbs``, and ``target_mbs`` instead. Args: \*args (Any): Whole-batch positional root inputs when this rank owns the first pipeline stage. Do not pass positional inputs with pre-split inputs. target (Any, optional): Whole-batch target for loss computation. When passing pre-split inputs, pass targets through ``target_mbs`` instead. Default: ``None``. losses (list, optional): Mutable list populated with one loss per microbatch when this schedule owns the last stage and a ``loss_fn`` was configured. Default: ``None``. arg_mbs (list[tuple], optional): Pre-split positional inputs, one tuple per microbatch. Default: ``None``. kwarg_mbs (list[dict], optional): Pre-split keyword inputs, one dict per microbatch. Default: ``None``. target_mbs (list, optional): Pre-split targets, one entry per microbatch. Default: ``None``. \*\*kwargs (Any): Whole-batch keyword root inputs when this rank owns the first pipeline stage. Do not pass keyword inputs with pre-split inputs. Returns: Any or None: The merged output from the last stage when this rank owns the last stage; otherwise ``None``. Raises: TypeError: If a pre-split microbatch container has the wrong type. ValueError: If whole-batch and pre-split inputs are mixed, or if a pre-split container does not have one entry per microbatch. Examples:: >>> # xdoctest: +SKIP("requires a constructed distributed pipeline schedule") >>> output = schedule.eval(x, mask=mask) >>> arg_mbs = [(x0,), (x1,)] >>> kwarg_mbs = [{"mask": mask0}, {"mask": mask1}] >>> output = schedule.eval( ... arg_mbs=arg_mbs, ... kwarg_mbs=kwarg_mbs, ... ) Fr(rrFrGr)rrQ) r\r(rrFrGrr&r'original_has_backwards &$$$$$*, r>eval_PipelineSchedule.eval0swx!% 2 2 7!&D 99  $  & "7 !6 s"7?c <V^8dQhRS[R,RS[R,RS[R,RS[R,RS[S[S[3,/#)r7rFNrGrrr9)rr~)r<r=s"r>r?r|sV$"$"$"$;$"4K $" t $" tTz  $"rAc aRV3RllpVe V!VR4MR.SP,pVe V!VR4M/.SP,pVe V!VR4Ve.\V\4'g\R\ V4 24hW3#)z Pre-process/check inputs c$V^8dQhR\/#)r7namer:)r<s"r>r?5_PipelineSchedule._check_inputs..__annotate__s  # rAc <\V\4'g\V R\V4 24h\ V4SP 8wd(\ RSP RV R\ V4 24hR#)z must be a list but got a rrz but got N)rr TypeErrorr.rrrD)mbsr\r\s&&r>check_type_and_len;_PipelineSchedule._check_inputs..check_type_and_lenspc4((4&(B49+ NOO3x4///  !5!5 6avYs3xjQ0rArFrGrz losses must be a list but got a rI)rrrr_r.)r\rFrGrrrasf&&&& r> _check_inputs_PipelineSchedule._check_inputs|s     w 2dT111G  y+ 6t333I  ! z< 8  fd++"B4<. QRR!!rAc<VP!W3/T;'g/B#rY)r)r\rr(rs&&&&r>r_PipelineSchedule._compute_losss}}VC 0A0ArCCrAc^<V^8dQhRS[S[R3,RS[S[S[3,R,/#)r7r&.r'N)r~r rr;)r<r=s"r>r?rs8LLCHoLS#X%LrAc V'g V'd3\VVVPVPVP4wr4W43#R.VP,/.VP,3#)zR Splits a full-batch input into chunks (i.e. microbatches) and returns the chunks rI)rrrr)r\r&r' args_split kwargs_splits&&& r> _split_inputs_PipelineSchedule._split_inputssi 6'D$$%%'' ( $J+ +4$...t7K7K0KK KrAc<V^8dQhRS[S[R3,RS[S[S[3,RS[RS[RS[RS[RS[S[R ,S[R ,S[R ,3,/#) r7r&.r'r(rFrGrr9N)r~r rr;r)r<r=s"r>r?rsy7.7.CHo7.S#X7. 7.  7.  7.7. td{D4K4 57.rAc \;QJdRWEV34F 'gK RM RM!RWEV344pV'gBVPW4wrVe%\\V\VP 44MRp WV 3#V'd \ R4hV'd*RP\V44p \ RV R24hVe \ R4hVPWEV4wrE\\WERR 74Fhwp wr\V \4'g\R V R \V 4 24h\V\ 4'dKP\R V R \V4 24h WEV3#) c3("TFqRJxK R#5irYrI)rr`s& r>r;_PipelineSchedule._get_microbatch_inputs..sT3SC43SsTFNzmWhen using pre-split inputs, pass pre-split positional inputs through arg_mbs=... instead of positional args., z4Unexpected keyword arguments with pre-split inputs: z5. Pass pre-split keyword inputs through kwarg_mbs=...z`When using pre-split inputs, pass pre-split targets through target_mbs=... instead of target=...)strictz.arg_mbs must be a list of tuples, but arg_mbs[z] is a z1kwarg_mbs must be a list of dicts, but kwarg_mbs[)rrkrr_TARGET_CHUNK_SPECrrDrerrcrrrr~r_r.r)r\r&r'r(rFrGr pre_splitrirj targets_splitnamesrarg_mbkwarg_mbs&&&&&&& r>_get_microbatch_inputs(_PipelineSchedule._get_microbatch_inputssCTG 3STCCCTG 3STT '+'9'9$'G $J%]6+=t?S?STU  ]: : B  IIfVn-EFugNFF   7  "//JO,5 4 0- (H(vfe,,'jV ~? h--!!) '$x.1AC- :--rAc6<V^8dQhRS[S[,RS[/#)r7 output_chunksr9rT)r<r=s"r>r?rs  DI # rAc .\VVP4#)z Merge output chunks back to a batch state. If output_merge_spec is None, the utility will merge output chunks by dimension 0 (batch dim). )rr)r\r|s&&r>_merge_outputs _PipelineSchedule._merge_outputss    # #  rA)rrrrrrrrNNNNTrYNNNNTN)NNNN)rJrKrLrMrrrrr#rBrrMrQrWrcrrkryr~rSrTrUs@r>rrs' 9 9D/ &.N1N1xW0W0r"""* " "# "  " .2 " " " " " "J7J7# J7  J7  J7J7J7X$"$"LDLL.7.7.r  rArcV^8dQhR\\P,R\R,R\\P,/#r7p2p_opsdescNr9)rr P2POpr;Work)r<s"r>r?r?s6.+.+TZZ(.+d .+d499o.+rAc \V4^8Xd.#V'dV R2MRp\PRW 4/pVF8pVPVPP .4P V4K: \V4^8d:.p\VP44FwrgV\WqR7, pK V#VUu0uFqDPkK ppV\P08XddVUu.uF@pVPVPVPVPVPR7NKB p pV U u.uF qfKV NK up #V\P 08XddVUu.uF@pVPVPVPVPVPR7NKB p pV U u.uF qfKV NK up #\P"!V4#uupiuupiuup iuupiuup i)a3 Wrapper over batch_isend_irecv that avoids coalescing for homogeneous batches (all-send or all-recv). Coalescing serializes ops on a single CUDA stream, which causes head-of-line blocking when independent P2P ops could otherwise overlap. Mixed batches still use batch_isend_irecv for deadlock avoidance. rqrwzbatch_p2p %s%sr)r tag group_dst)r r group_src)rrr setdefaultr  group_namer}ritemsropr isendtensorr group_peerirecvbatch_isend_irecv) rrdesc_str ops_by_grouppworks_ group_opsop_types send_workswork recv_workss && r>rrs 7|q "$r{H LL!8513L  2 2B7>>qA <1!# #<#5#5#78LA Z 5 5E9 %&ggH&DJJ<  DDaeeq||D L  ",@@@DJJ<  DDaeeq||D L  ",@@@  ! !' **' A As,G#5AG(G- G-.AG2:G7G7c V^8dQhR\\P,R\R,R\\ \\P ,3,/#r)rr rr;rrr)r<s"r>r?r?1sC $** %(4Z #tDII rAc\\4p/p\V4^8XdV#VF$pW$P,P V4K& \ VP 44FwrV\WaR7W5&K V#)z Sorts the list of P2P ops by the peer rank, and then calls batch_isend_irecv. Return a dictionary of works by peer rank. This function helps us avoid hangs in case of skip connections. r)rrrpeerr}rrr)rr ops_by_peer work_by_peerrropss&& r>_sorted_batch_p2pr1sw0;4/@K/1L 7|qGG##B'K--/0 '7 1 rAcNV^8dQhR\\P,/#)r7r)rr r)r<s"r>r?r?Ls$tyy/rAc8VFpVP4K R#)zP Waits for a list of dist.Work (typically from _batch_p2p / _sorted_batch_p2p). N)wait)rws& r>rrLs rAcaa]tRtRtoRtRV3RlV3RllltRRltRRR RR RR RR RR RRR/V3RlRlltV3RlRltRt Vt V;t #)riTax Base class for single-stage schedules. Implements the `step` method. Derived classes should implement `_step_microbatches`. Gradients are scaled by num_microbatches depending on the `scale_grads` argument, defaulting to True. This setting should match the configuration of your loss_fn, which may either average losses (scale_grads=True) or sum losses (scale_grads=False). NTc<V^8dQhRS[RS[RS[R,RS[S[R3,R,RS[S[S[3,R,RS[S[S[3,S[S[,,R,R S[/# r7rrrNr.rrr rrrr~rrr;r ro)r<r=s"r>r?#PipelineScheduleSingle.__annotate___s  !  D  34t;   _ 45<   S>E#J6=  rAc <\SV`VVVVVVR7WnVPVnRVnRVnVP4VnR#)rrrrrrFN) superr_stage num_stages _num_stages_stage_forward_initialized_stage_backward_initialized_get_pipeline_orderr r\rrrrrrrrs &&&&&&&&r>rPipelineScheduleSingle.__init___sd )+//#    ++*/'+0(  $ $ & rAc VPVP.VVVVPVPVR7wVnVnR#rN)rBrrr)r\r&r'r(rs&&&&&r>_initialize_stage(PipelineScheduleSingle._initialize_stage|sO  & & [[M     + +  , ,# '   +  ,rAr(rrHrrFrGrc p<V^8dQhRS[R,RS[RS[S[S[3,R,RS[RS[RS[/#r7rNrHrrFrGrrI)r<r=s"r>r?rsbggt g  g #s(^d* ggggrAc  VP'd'\P!4'g \R4hVPVPnVPP 4VPVV VVVV4wrp VPV V V VVVR7VPP'd.V'd&VPVPP4#R#)a Run one training iteration of a single-stage pipeline schedule. By default, ``args``, ``kwargs``, and ``target`` are full-batch values that the schedule splits into microbatches. When the caller has already split inputs into microbatches, pass them through ``arg_mbs``, ``kwarg_mbs``, and ``target_mbs`` instead. Args: \*args (Any): Whole-batch positional inputs for the first pipeline stage. Do not pass positional inputs with pre-split inputs. target (Any, optional): Whole-batch target for loss computation. When passing pre-split inputs, pass targets through ``target_mbs`` instead. Default: ``None``. losses (list, optional): Mutable list populated with one loss per microbatch when this schedule owns the last stage and a ``loss_fn`` was configured. Default: ``None``. return_outputs (bool, optional): Whether to merge and return output chunks on the last stage. Default: ``True``. loss_kwargs (dict, optional): Extra keyword arguments forwarded to the configured ``loss_fn``. Default: ``None``. arg_mbs (list[tuple], optional): Pre-split positional inputs, one tuple per microbatch. Default: ``None``. kwarg_mbs (list[dict], optional): Pre-split keyword inputs, one dict per microbatch. Default: ``None``. target_mbs (list, optional): Pre-split targets, one entry per microbatch. Default: ``None``. \*\*kwargs (Any): Whole-batch keyword inputs for the first pipeline stage. Do not pass keyword inputs with pre-split inputs. Returns: Any or None: The merged output from the last stage when this rank owns the last stage and ``return_outputs=True``; otherwise ``None``. Raises: RuntimeError: If backward computation is enabled and ``step`` is called under ``torch.no_grad()``. Use :meth:`eval` for forward-only execution. TypeError: If a pre-split microbatch container has the wrong type. ValueError: If whole-batch and pre-split inputs are mixed, or if a pre-split container does not have one entry per microbatch. Examples:: >>> # xdoctest: +SKIP("requires a constructed distributed pipeline schedule") >>> output = schedule.step(x, mask=mask, target=target) >>> arg_mbs = [(x0,), (x1,)] >>> kwarg_mbs = [{"mask": mask0}, {"mask": mask1}] >>> target_mbs = [target0, target1] >>> output = schedule.step( ... arg_mbs=arg_mbs, ... kwarg_mbs=kwarg_mbs, ... target_mbs=target_mbs, ... ) step() requires gradients to be enabled for backward computation; it should not be used under torch.no_grad() context. Please call eval() instead.rN) rris_grad_enabledrErrclear_runtime_statesryrMrr~r|) r\r(rrHrrFrGrr&r'rirjrus &$$$$$$$*, r>rQPipelineScheduleSingle.stepsF    e&;&;&=&=. $(#5#5   ((*262M2M       3 / -      #   ;;   >&&t{{'@'@A ArAcb<V^8dQhRS[S[S[S[R,,3,R,/#r7r9NrrrrW)r<r=s"r>r?rs,T#tGdN/C*C%Dt%KrAc R#)a\ Returns the pipeline execution order as a schedule IR. The returned IR is a dictionary mapping rank IDs to lists of actions. Each action is either an _Action object representing computation to perform, or None representing a deliberate idle step. The None values are used to represent pipeline bubbles where a rank must wait for dependencies from other ranks before proceeding. However during execution, with the _PipelineScheduleRuntime, these Nones are skipped since the relevant communication (send/recv) will be scheduled and waited on. Returns: A dictionary mapping rank -> list of actions NrIr[s&r>r*PipelineScheduleSingle._get_pipeline_orders rA)rrrrrrNN) rJrKrLrM__doc__rrrQrrSrT __classcell__rr=s@@r>rrTs  :  gg# g $ g .2 gggggRrAc:a]tRtRtoRtRV3RlRlltRtVtR#)_ScheduleForwardOnlyizc The forward-only schedule. Will go through all the microbatches and perform only the forward pass Nc<V^8dQhRS[R,RS[R,RS[R,RS[R,RS[RS[S[S[3,R,/#rErI)r<r=s"r>r?!_ScheduleForwardOnly.__annotate__ sd,",","$;,"4K ," t ,"  ,"#s(^d*,"rAc  JVfVe \R4hVPWW44wrVe V^,MRpVPV^,V^,V4.p\VP4EFp \ RV 24;_uu_4VP PV 4p \V RR7p V P4Fp \V 4K VP PWV ,W),4VP PV 4p \V RR7p VPV P44RRR4\PRVP P V 4EK VFp \V 4K R# +'giLV;i)z, Run one iteration of the pipeline schedule Nz7Forward-only schedule does not support loss computationForward fwd_recvrfwd_send[%s] Forwarded microbatch %s)rErcrrrrrget_fwd_recv_opsrrrforward_one_chunkget_fwd_send_opsrrrrh) r\rFrGrrrHrmaybe_first_targetfwd_sends_to_waitrrrrs &&&&&&& r>rM'_ScheduleForwardOnly._step_microbatches sZ  !V%7I "//JW.8.DZ]$ wqz9Q<9KL46t++,A 8A300kk2215)#J?!LLND#D)+ --aY\Jkk2215)#J?!((81 LL79P9PRS T-$&D D !&#10s B;F F" rIr)rJrKrLrMrrMrSrTrUs@r>rrs ,",","rArcLa]tRtRtoRtR V3RlRlltV3RlRltRtVtR#) r"i;zR The GPipe schedule. Will go through all the microbatches in a fill-drain manner. Nc<V^8dQhRS[R,RS[R,RS[R,RS[R,RS[RS[S[S[3,R,/#rErI)r<r=s"r>r?ScheduleGPipe.__annotate__AstRYRYRY$;RY4K RY t RY  RY#s(^d*RYrAc  VPWW44wrVe V^,MRpVPV^,V^,Wv4.p\VP4EF(p \ RV 24;_uu_4VP P V 4p \V RR7p V P4Fp \V 4K VP PWV ,W),VR7p VP PV 4p \V RR7p VPV P44RRR4\PRVP PV 4VP!VP X W9V4EK+ VFp \V 4K .p\VP4EF,p \ RV 24;_uu_4VP P#V 4p \V R R7p V P4Fp \V 4K VP%VP V 4pVP P'V VWP^, 8HR 7VP P)V 4p \V R R7p VPV P44RRR4\PR VP PV 4EK/ VFp \V 4K VP+VP V4VP P-VP.'dVP4R#^4R# +'giEL,;i +'giL;i) a Run one iteration of the pipeline schedule with list of microbatches. Will go through all the microbatches according to the GPipe schedule. Args: microbatches: list of microbatch args. return_outputs: whether to return the outputs from the last stage. Nrrrsave_forward_outputrrz Backward bwd_recvr last_backwardbwd_sendz[%s] Backwarded microbatch %s)rcrrrrrrrrrrrrrrrhrget_bwd_recv_opsrbackward_one_chunkget_bwd_send_opsrperform_reduce_gradr)r\rFrGrrrHrrrrrrrrbwd_sends_to_waitrs&&&&&&& r>rM ScheduleGPipe._step_microbatchesAs""//JW.8.DZ]$  AJ ! &8 46t++,A 8A300kk2215)#J?!LLND#D)+66qz9<^7kk2215)#J?!((81 LL79P9PRS T  $ $T[[&* U#-,&D D !& 46t++,A 9QC11kk2215)#J?!LLND#D)+++DKK; .."#';';a'?"?/ kk2215)#J?!((82" LL8$++:Q:QST U%-*&D D !& DKK0 ''@P@P@P(<(<XVWXm100821s6B=MCM* M' * M: cb<V^8dQhRS[S[S[S[R,,3,R,/#rr)r<r=s"r>r?rs,T#tGdN/C*C%Dt%KrAc  &/pVPp\V4Fp.pTpVPR.V,4\VP4F-pVP \ V\ PV44K/ ^V^, V, ,pVPR.V,4\VP4F-pVP \ V\ PV44K/ \W@P4W&K V#)z Returns the pipeline order for GPipe schedule. See base method in PipelineScheduleSingle for details on the schedule IR format. N) rrrrr}rWr)rNrQ_add_reduce_grad)r\r pp_group_sizerr warmup_delaymb_idxbackward_delays& r>r!ScheduleGPipe._get_pipeline_orders (( -(D,.G L NND6L0 1  4 45wt-=-E-EvNO6-!"3d":;N NND6N2 3  4 45wt-=-K-KVTU6$4G=Q=Q#RN ')*rArIr) rJrKrLrMrrMrrSrTrUs@r>r"r";s# RYRYhrAcnaa]tRtRtoRtR V3RlV3RllltR V3RlRlltV3RlRltR tVt V;t #) r!izc The 1F1B schedule. Will perform one forward and one backward on the microbatches in steady state. c<V^8dQhRS[RS[RS[R,RS[S[R3,R,RS[S[S[3,R,RS[S[S[3,S[S[,,R,R S[/#rr)r<r=s"r>r?Schedule1F1B.__annotate__s!D  34t;   _ 45<  S>E#J6=rAc <\SV`VVVVVVVR7W P8d\RV RVP R24hR#))rrrrrrrzNumber of microbatches (z9) must be greater than or equal to the number of stages (z).N)rrrrDrs &&&&&&&&r>rSchedule1F1B.__init__sl )+//#   ,, ,*>*:;##'#3#3"4B8  -rAc<V^8dQhRS[R,RS[R,RS[R,RS[R,RS[RS[S[S[3,R,/#rErI)r<r=s"r>r?rstWYWYWY$;WY4K WY t WY  WY#s(^d*WYrAc VPWW44wrVe V^,MRpVPV^,V^,Wv4\VPVPVP P , 4p^p ^p .p .p \V4Fp VP PV 4p\\VRR74VP PV W,W),VR7p\V 4VP PV 4p W^, 8wd\V RR7p VPVP WW4V ^, p K VP PV 4p\\V V,RR74VPVP V 4pVP P!V VWP^, 8HR7VP P#V 4pV ^, p WP8XdMVP PV 4p\\VV,RR74VP PV W,W),VR7pVPVP WW4VP PV 4p V ^, p EKb\VR R7p WP8dVP PV 4p\\VR R74VPVP V 4pVP P!V VWP^, 8HR7\V 4VP P#V 4p\VR R7p V ^, p K\V 4VP%VP V4VP P'VP('dVP4R#^4R#) aN Run one iteration of the pipeline schedule with list of microbatches. Will go through all the microbatches according to the 1F1B schedule. Args: microbatches: list of microbatch args. return_outputs: whether to return the outputs from the last stage. loss_kwargs: extra keyword arguments forwarded to the loss function. Nrrrrfwd_send_bwd_recvrbwd_send_fwd_recvrr)rcrminrrrrhrrrrrrrrrrrrrr)r\rFrGrrrHrr warmup_chunks fwd_mb_index bwd_mb_index send_work fwd_sendsr fwd_recvsr bwd_recvsr bwd_sendss&&&&&&& r>rMSchedule1F1B._step_microbatchess$"//JW.8.DZ]$  AJ ! &8     t{{66 6   &(  }%A 44\BI JyzB C[[22%'$2 3F I & 44\BIq00&yzB  $ $ V  A L?&H 44\BI Jy9'I A L  " DKK0 ''@P@P@P(<(<XVWXrAcb<V^8dQhRS[S[S[S[R,,3,R,/#rr)r<r=s"r>r?rns/BBT#tGdN/C*C%Dt%KBrAc  V/pVPp\V4EF p.pVPR.V,4V^, V, p^p\V4F/pVP\ V\ P V44TpK1 \^^V^, V, ,4pVPR.V,4^p VPV, p V ^8drV^, pVP\ V\ P V44V ^,p VP\ V\ PV 44V ^, p KxVPV , p V ^8dW#, ^8dYVPR4V ^8d?VP\ V\ PV 44V ^, p V ^,p KjKlVP\ V\ PV 44V ^, p V ^,p K\W@P4W&EK V#)z Returns the pipeline order for 1F1B schedule. See base method in PipelineScheduleSingle for details on the schedule IR format. N) rrrr}rWr)rNrrrQr) r\rrrr num_forward forward_mbr wait_for_1f1b backward_mbremaining_forwardremaining_backwards & r>r Schedule1F1B._get_pipeline_orderns (( -(D,.G NND6D= ))1,4KJ;'wt-=-E-EqIJ (  1 (9D(@#ABM NND6M1 2K $ 4 4{ B #a'a wt-=-E-EzRS!Q&!D"2"@"@+Nq "&!5!5 !C $q("(A-NN4()A-#D*:*H*H+V$q( *a/* .NN&6&D&DkR 1$K&!+G=Q=Q#RN q)rrArIrr) rJrKrLrMrrrMrrSrTrrs@@r>r!r!s5 2WYWYrBBBrAc0V^8dQhR\R\/#)r7 action_typer9)r)ro)r<s"r>r?r?s!!'7!D!rAc V\\39#rY)r,r3)rs&r>_requires_reduce_gradrs 1a&  rAcV^8dQhR\\R,,R\R\\R,,/#)r7rNrr9rrWr)r<s"r>r?r?s6$$ 'D. !$36$ 'D.$rAc ^aa.p\\4oVV3RlpVFpVfK VPV4.pVP\8Xd+VP eVP F pV!We4K MV!WE4VF#pVP\ V\R44K% K V#)z REDUCE_GRAD refers to joint across minibatches grad reduction. reduce_grad frees memory and we want to schedule it just after the last "backward"-like stage. c<\VP4'dEVPpSV;;,^, uu&SV,S8XdVPV4R#R#R#r)rrfrhr})a to_schedulerhcntrs&& r> _leaf_action&_add_reduce_grad.._leaf_actionsR !3!3 4 4--K   ! ;>1"";/2 5rAN)rrr}rfr4rcrWr5) rractions_with_reduce_gradrrschedule_reduce_grad_stage_idxsrj stage_idxrs &f @r>rrs68%c*C0 9  ''*57'   ,1Jmm ZI,  <8I $ + +GI{D,Q R9 $#rAcvV^8dQhR\\R,,R\R\\,/#)r7compute_actionsNmax_active_stagesr9r)r<s"r>r?r?s8PP'D.)PP ']PrAca a a RRlp\4o .o RV V 3RllpRV V 3Rllp\V4F{wrVVfK V!WVR4o \\V 3RlS 44p\\V 3R lS 44pVF p V!V 4K VF p V!V 4K S P V4K} \S 4F p V!V 4K S #) aAGiven a basic schedule involving only compute actions (F,B,W,OVERLAP_F_B), add UNSHARD/RESHARD actions for FSDP. UNSHARD refers to fetching the full contents of an FSDP-sharded layer, requiring an all-gather operation. RESHARD does the opposite, releasing memory (but doing no communication) We abandon the "timestep lock" during lowering max_active_stages controls how many prefetches we allow. It should be measured in mb and tuneable but in practice 3 stages is probably the thing we want? (to account for having one f and one b active, and something else prefetching?) cvV^8dQhR\R\\R,,R\\,/#)r7count next_actionsNr9)rrrW)r<s"r>r?*_add_unshard_reshard..__annotate__s/#T'D.5IdSVirAc\4p.pVFpVfK VP\8XdVPerVPFLpVPV9gKVP VP4VP VP4KN \V4V8dV#KVPV9gKVP VP4VP VP4\V4V8XgKV# V#)zdRemove duplicates (same stage, different microbatch), find next 'count' stages that will do compute.)setrfr4rcrhaddr}r)r(r)seenretrrjs&& r>next_stage_indices0_add_unshard_reshard..next_stage_indicessA}%%49R&'mm %11= HHZ%;%;<JJz'='=>'43x5( )}}D0/ 1==1s8u,! #" rAc$V^8dQhR\/#r7rhr)r<s"r>r?r*GGcGrAcj<SPV4SP\V\R44R#rY)r-r}rWr-rh active_stagesfsdp_aware_actionss&r>_unshard&_add_unshard_reshard.._unshards(+&!!'+w"EFrAc$V^8dQhR\/#r3r4)r<s"r>r?r*r5rAcj<SPV4SP\V\R44R#rY)remover}rWr.r7s&r>_reshard&_add_unshard_reshard.._reshards([)!!'+w"EFrANc<VS9#rYrI)sr8s&r>&_add_unshard_reshard..s a}&rCrDs avorA)r,rrfilterr}) r$r%r0r:r?rr8fetchevictrr8r9rFs && @@@r>_add_unshard_reshardrJs 0"eM(*GGGG/  > $$5qr7JKVr?r?+s)##'D.)# ']#rAc^.pV'Ed"VP^4pVfK \V4'dV^,fVP^4K/\V4^8d V^,MRpVP\8XdVeVP\8XdVP VP 8XddVP VP 8XdIVP\VP \VP 44VP^4EKVPV4EK*V#)a)Given a basic schedule involving only compute actions (F,I,W), merge adjacent I and W ops into B ops. (note: I = BACKWARD_INPUT, W = BACKWARD_WEIGHT, B = FULL_BACKWARD) B refers to running the whole backward (not separating grad_input and grad_weight), which can be more efficient in some cases. N) poprrfrOrPrhrir}rWrQ)r$merged_actionsr8 next_actions& r> _merge_bwrQ+sN / $$Q' > /""q'9'A    "-0,@1,Doa($   # #~ 5',,?""k&=&=='';+G+GG  ! !**M6;R;RS     "  ! !& ) rAc V^8dQhR\\\\,3,R\\.\3,R\R\\\\,3,/#)r7r$ stage_to_rankrr9rrrrWr)r<s"r>r?r?Qs]tt#tG},-tSE3J'tt #tG}  trAc aaaa a aVUu/uFq3.bK ppVUu/uF q3\4bK upoRVV3Rllo RV 3RllpRV3Rllo V'EdRp\V4EFo\VS,4^8g$\RS: R \VS,4: 24hVS,^,pVeVPeVPpMV3p\ ;QJd!V VV3R lV4F 'dK RM R M!V VV3R lV44'gKVeVS,P V4VFp SS,PV 4S !V 4'gK+V!V 4wrVS,P V 4SS,PV 4VS!V P4,P V 4SS!V P4,PV 4K VS,P^4\VS,4^8XdVSR pEK V'dEK\R 4hV#uupiuupi) z Transforms a compute-only schedule into a complete schedule with communication actions. For actions with sub-actions (OVERLAP_F_B) we ensure that all the subactions have been computed and the communication is ready c0V^8dQhR\R\/#rrWro)r<s"r>r?$_add_send_recv..__annotate___s  7 t rAc<VP\8XdKVPS^, 8g;'d-S!VP^,4S!VP48g#VP\\39dDVP^8g;'d-S!VP^, 4S!VP48g#R#)rF)rfr*rhrOrQ)r8rrSs&r> _has_comms"_add_send_recv.._has_comms_s  " "a '%%a733M""Q&=v112=3 3 $ $(G G%%*33}""Q&0v11203 3rAcRV^8dQhR\R\\\3,/#r)rWr~)r<s"r>r?rXjs$  7 uWg-='> rAcH<S!V4'g\V R24hVPpVPpVPp\ Y\ 8Xd\ M\V4pV\ 8Xd V^,MV^, p\ YR\ 8Xd\M\V4pWF3#)z is not a valid comm action) r4rhrfrirWr*r/r1r0r2)r8r"ctypersendrecv_stage_idxrecvrZs& r> _get_comms"_add_send_recv.._get_commsjs&!! F8+F!GH H&& ''((yQJ&FFK*/1*Q)a-~!vPzrAc`V^8dQhR\R,R\\,R\/#)r7r8N prev_actionsr9)rWr,ro)r<s"r>r?rXus*((7T>(W(RV(rAc<VfR#VP\8XdsVP^8wdb\VP\VP 4V9dR#\VP^, \VP 4V9dR#R#VP\ \39dVPS^, 8wd\VP\VP 4V9dR#\VP^,\ VP 4V9dR#\VP^,\VP 4V9dR#R#R#)zWe don't put our own recv ops in the schedule, we let a sender on another rank put our recv ops in place. This helps ensure a sane (non-hanging) ordering of sends and recvs. But it also means we might not be able to schedule our next compute action yet. TF) rfr*rhrWr0rirOrQr2)r8rers&&r>_ready_to_schedule*_add_send_recv.._ready_to_scheduleus  >  $ $ )f.@.@A.E**FF4K4KL **Q.63J3JK   # # 'F F""j1n4**FF4K4KL **Q.@W@WX **Q. v?V?VW rAFrank=z, len(compute_actions[rank])=c3D<"TFpS!VSS,4xK R#5irYrI)rrrgrers& r>r!_add_send_recv..s!V+Q)!\$-?@@+rTz6Malformed compute schedule, can't schedule sends/recvs) r,rrr4rcrr}r-rhrN)r$rSrr comm_actionsrbprogressr8 all_actionsrr_rarZrgres&ff` @@@r>_add_send_recvroQsDS-S?4Bh?L-SET,U_T35[_,UL    ((T /?+D-.2$w.L_T5J1K0M%NOO$T*1-F!f&8&8&D$00 %i 3V+V333V+VVV!T"))&1$A &**1-!!}}%/] %T*11$7$T*..t4$]43C3C%DELLTR$]43C3C%DEII$O% D ! % %a (?4()Q.#D)H=,>x H  S.T,Us IIc V^8dQhR\\\\,3,R\\.\3,R\\\\,3,/#)r7rrSr9rT)r<s"r>r?r?sR[[ #tG}$ %[SE3J'[ #tG} [rAc aaa\Po\Pp\Po\PpRVV3RllpRVV3Rllp/pVP 4EFwrx.p /p VEFp V P SV39d*V PV P V P3p WV &K@V P SV39djV!V 4p W}8d\V P 4UUu.uFwrV!V4V 8XgKVNK pppVF#p V PV PV 44K% V Pe V PMV 3pVFpVP \8XdDVPSVP3p W9d#V PV PV 44KYK[VP \\39gKxVPVVP3p W9gKV PV PV 44K V PV 4EK V 'd*\RV R\!V P#44 R24hWV&EK V#uuppi)a_ Defers RECV operations to reduce interference with unrelated compute ops, while maintaining deadlock-safe ordering via rank-parity P2P ordering. By default, the schedule places RECV ops as early as possible (ASAP) to overlap P2P communication with computation. However, on some platforms (e.g., AMD ROCm), a pending RECV can block unrelated compute ops that also use the communication fabric (e.g., FSDP allgather inside a forward pass), creating pipeline bubbles. This function defers each RECV to as late as possible, subject to: 1. A RECV must appear before the compute op that consumes its data. 2. Deadlock avoidance via rank-parity ordering (see pytorch/pytorch#172668): - When current rank > peer rank: SEND before RECV is safe, so deferred RECVs from that peer are NOT flushed before SENDs to that peer. - When current rank < peer rank: RECV before SEND is required, so deferred RECVs from that peer ARE flushed before SENDs to that peer. This breaks circular waits: the lower-ranked side always posts RECV first, providing the matching target for the higher-ranked side's SEND. c0V^8dQhR\R\/#rrWr)r<s"r>r?%_defer_recv_ops..__annotate__999C9rAc<VPS8XdS!VP^, 4#S!VP^,4#rrfrh)r8r0rSs&r>_recv_peer_rank(_defer_recv_ops.._recv_peer_rank=  " "f , !3!3a!78 8 !3!3a!78 8rAc0V^8dQhR\R\/#rrs)r<s"r>r?rtrurAc<VPS8XdS!VP^,4#S!VP^, 4#rwrx)r8r/rSs&r>_send_peer_rank(_defer_recv_ops.._send_peer_rankr{rAz!Malformed input schedule on rank z,: leftover RECV ops with no consumer found: z9. Every RECV must be consumed by a downstream compute op.)r)r0r2r/r1rrfrhrir}rNrcrNrQrOr4rr)rrSr2r1ryr~r r action_list new_actionsdeferredr8rrkvto_flush consumerssubr0r/s&f @@r>_defer_recv_opsrsK0 $ $F  $ $F  $ $F  $ $F99 99 (*F$]]_%' LN!F&&66*::&&++++ !' &&66*::&v.;&.nn&6 &6da/!:LPT:T&6  (#**8<<+<= ( '-&8&8&D""6) !''72??FC4H4HIC#**8<<+<='))m^-LL??FC4H4HIC#**8<<+<=!   v &G"J  3D6:++/0A+B*CDJJ  #t a-b M= s *I I c V^8dQhR\\\\R,,3,R\R\R\R\\\3,/#)r7rNrrnum_microbatchesr9r)r<s"r>r?r?&s\'' #tGdN++ ,''' '  #s(^ 'rAc haaa\S4V8Xg\RV R\S4 24h\V4FpVS9gK \RV 24h \V4Uu/uF>pV\\ 4\ \ 4\ \ 4\\ 4/bK@ upo/oRVVV3RllpSFp\SV,4FkwrxVfK \V\4'g\RV RV RV R24hVPeVPF p V!WV4K KbV!WV4Km K SFp \SV ,\,4p \SV ,\ ,4p \SV ,\ ,4p \SV ,\,4pW8Xg\R V R \ R V R V 24hW8Xg\R V RV RV 24hWV,^,,V8XdK\R V RV RV RV RV 2 4h S#uupi)z2Schedule has incorrect number of ranks - expected z , actual z%Schedule is missing actions for rank c<V^8dQhR\R\R\/#)r7r8rrQrs)r<s"r>r?(_validate_schedule..__annotate__As!:::s:#:rAc <VPpVPpVPpV\8Xd&S V,\,P V4EMaV\ 8XdlVS V,\,9d/RV RV RV RV R2 p\ S VR7pV RV 2p\V4hS V,\ ,P V4MV\8XdlVS V,\,9d/RV RV RV RV R2 p\ S VR7pV RV 2p\V4hS V,\,P V4MuV\8XdkVS V,\,9d/RV RV R V RV R 2 p\ S VR7pV RV 2p\V4hS V,\,P V4VS 9dVS V&R#S V,p W8Xg\RV RV R V R V R V 2 4hR#)zPProcess a single action and update stage_actions and stage_index_to_rank_mappingr, step z": Running Full Backward for stage z , microbatch z without first running Forwardrz Full pipeline schedule: z#: Running Backward Input for stage z$: Running Backward Weight for stage z% without first running Backward Inputz: Stage z is assigned to both rank z and rank N) rhrfrir*r-r3rr4r+r,) r8rrQs_idr^mb_id error_msgformatted_schedulefull_error_msg existing_rankr stage_actionsstage_index_to_rank_mappings &&& r>_process_action+_validate_schedule.._process_actionAsf!!'''' A: $  " & &u - aZM$/22D6.PQUPVW""'(FH&<t&"!k!>?Q>RS%^44 $  " & &u - aZM$/22D6.QRVQWX""'(FH&<t&"!k!>?Q>RS%^44 $  " & &u - aZM$/22D6.RSWRXY""'(MO&<t&"!k!>?Q>RS%^44 $  " & &u - 2 204 ' -7=M)$D6htf_validate_scheduler&sp LM )@yY\]dYeXf g  m$ w  #H!OP P%j)=*H  su su su su   *=M#%::x%gdm4LD~fg..$D6.GxOmn !!-"("4"4J#Jd;#5 d35$=&q)*=&q)*=&q)*=&q)*( tfAaS 8kJZI[\   :4&AfD(  t ))-== :4& L\K]^fDd4&2 %, '&_=s"AH/caa]tRtRtoRtRV3RlV3RllltRV3RlRlltV3R lR ltV3R lR ltR t RV3RlRllt RRRRRRRRRRRRRR/V3RlRllt RV3RlRllt Rt VtV;t#) r ia@ Base class for multi-stage schedules. Implements the `step` method. Gradients are scaled by num_microbatches depending on the `scale_grads` argument, defaulting to True. This setting should match the configuration of your loss_fn, which may either average losses (scale_grads=True) or sum losses (scale_grads=False). NTc<V^8dQhRS[S[,RS[RS[R,RS[S[R3,R,RS[S[S[3,R,RS[S[S[3,S[S[,,R,R S[ R,R S[ R S[ / #) r7rrrNr.rruse_full_backwardrbackward_requires_autograd rrrrr~rrr;r ro)r<r=s"r>r?"PipelineScheduleMulti.__annotate__s44'(44D 4 34t; 4  _ 45< 4 S>E#J6=4 $;44%)4rAc <a \S V`VVVVVVR7WnV^,PVnV^,P VnV^,PVn\VP VP4Vn VPFp VPV n K RVn RVn VPRJo V 3RlVn/VnWnVe\"P%R4R#R#)rFNc0<VP;'dS#rYr)rhas_losss&r>rC0PipelineScheduleMulti.__init__..s%--2L2LH2LrAzDeprecation warning: 'use_full_backward' is no longer supported. Simply stop passing it, and everything should still work fine.)rr_stagesrr group_sizer group_rankrrstage_index_to_group_rank_stages_forward_initialized_stages_backward_initializedr_should_compute_lossr_backward_requires_autogradrwarning) r\rrrrrrrrrrrrs &&&&&&&&&& @r>rPipelineScheduleMulti.__init__s )+//#   !!9//#AY111I(( )G    0 0* &\\E.2.L.LE +",1(,1)d2$L!@B ,F(  ( NNQ  )rAc4<V^8dQhRS[S[R3,/#)r7r&.)r~r )r<r=s"r>r?rs::#s(O:rAc jVP;'dVPVP8gpVPpVPVPVVVVPVPVR7wVnVnVP'd%V'd V'dVP 4R#R#R#r)rrrrBr&_validate_adjacent_stage_communication)r\r&r'r(rreinit_for_mode_switchforward_initialized_befores&&&&& r>_initialize_stages(PipelineScheduleMulti._initialize_stagess"&!A!A" "   $"C"C C &*%E%E"  & & LL     , ,  - -# '   ,  -  + + +*.D  7 7 9/E ,rAc<V^8dQhRR/#rrI)r<r=s"r>r?rs333rAc JRRlpVPEFpVPpVP^,Uu0uF7p\V\4'gKVP fK+VP kK9 ppVP 'd \4M V^, 0pV!VRVV4VPP4UUu0uFpVF pVfK VkK K p ppVP'd \4M V^,0p V!VRV V 4EK R#uupiuuppi)zGValidate that stage communication follows adjacent-stage topology only.c xV^8dQhR\R\R\\,R\\,RR/#)r7r" directionactual_stage_indicesexpected_stage_indicesr9N)rr;r,)r<s"r>r?RPipelineScheduleMulti._validate_adjacent_stage_communication..__annotate__sB    #&c( %(H     rAcW#, pV'd7\RV RV R\V4 R\V4 R\V4 R2 4hR#)zLPipelineScheduleMulti only supports adjacent-stage communication, but stage z has z stages z with non-adjacent stages z (allowed adjacent stages: z\). This commonly indicates skip connections, which are unsupported in this schedule runtime.N)rEr)r"rrrnon_adjacent_stage_indicess&&&& r>_check_stage_indicesZPipelineScheduleMulti._validate_adjacent_stage_communication.._check_stage_indicessq *>)V &)"009{% {K$%9:;<++12L+M*NO14567- -  *rANz forward recvz forward send) rrhargs_recv_inforrsourcerr, act_send_inforr) r\rrr"ractual_fwd_recv_sourcesexpected_fwd_recv_sourcesdstsdstactual_fwd_send_destsexpected_fwd_send_destss & r>rr?r6s4 M MCgn!556 M  MrAc \VVPVPVP4VnVP FpVPVnK R#)zM Allocates the stage index to rank mapping which is needed for communication N)rrrrrr)r\rrs&& r>_validate_and_set_stage_mapping5PipelineScheduleMulti._validate_and_set_stage_mapping6sM *<         * & \\E.2.L.LE +"rAc \VRRR7;_uu_4p\P!V4pVPF%pVP VPV,4K' RRR4R# +'giR#;i)QDump a CSV representation of the schedule into a file with the provided filename.rrwnewlineN)opencsvwriterrwriterow)r\filenamecsvfilerrs&& r> _dump_csvPipelineScheduleMulti._dump_csvEsW (C , ,ZZ(F++ 3 3D 9:,- , , ,s A A-- A> c4<V^8dQhRS[RS[R,/#r7rr< compute_only compute_commsr;r )r<r=s"r>r?rLs)BBB78BrAc  VR8wd\RV 24h\VRR7;_uu_4p\P!V4p\ V4F7wrVVUu.uFp\ P V4NK upVPV&K9 RRR4VPVP4R#uupi +'giL2;i)zLoad a CSV representation of the schedule from a file with the provided filename. This API will most likely get renamed/refactored so is marked as internal for now. format must be "compute_only" for PipelineScheduleMulti. rz#format must be "compute_only", got rwrN) r4rrreaderrrWrGrr)r\rr<rrrrrBs&&& r> _load_csvPipelineScheduleMulti._load_csvLs ^ # #Fvh!OP P (B ' '7ZZ(F&v. JM,N#QW-=-=a-@#,N##D)/( ,,T-@-@A -O( 's+B2B- 4B2-B22 C r(rrHrrFrGrc p<V^8dQhRS[R,RS[RS[S[S[3,R,RS[RS[RS[/#rrI)r<r=s"r>r?rasbppt p  p #s(^d* pppprAc  VP'd9VP'd'\P!4'g \ R4hVP Fp VPV nK VP Fp V P4K VPVV VVVV4wrp VPV V V VVVR7VP F<p V P'gKV'gK!VPV P4u# R#)a Run one training iteration of a multi-stage pipeline schedule. By default, ``args``, ``kwargs``, and ``target`` are full-batch values that the schedule splits into microbatches. When the caller has already split inputs into microbatches, pass them through ``arg_mbs``, ``kwarg_mbs``, and ``target_mbs`` instead. Args: \*args (Any): Whole-batch positional root inputs when this rank owns the first pipeline stage. Do not pass positional inputs with pre-split inputs. target (Any, optional): Whole-batch target for loss computation. When passing pre-split inputs, pass targets through ``target_mbs`` instead. Default: ``None``. losses (list, optional): Mutable list populated with one loss per microbatch when this schedule owns the last stage and a ``loss_fn`` was configured. Default: ``None``. return_outputs (bool, optional): Whether to merge and return output chunks on the last stage. Default: ``True``. loss_kwargs (dict, optional): Extra keyword arguments forwarded to the configured ``loss_fn``. Default: ``None``. arg_mbs (list[tuple], optional): Pre-split positional inputs, one tuple per microbatch. Default: ``None``. kwarg_mbs (list[dict], optional): Pre-split keyword inputs, one dict per microbatch. Default: ``None``. target_mbs (list, optional): Pre-split targets, one entry per microbatch. Default: ``None``. \*\*kwargs (Any): Whole-batch keyword root inputs when this rank owns the first pipeline stage. Do not pass keyword inputs with pre-split inputs. Returns: Any or None: The merged output from the last stage when this rank owns the last stage and ``return_outputs=True``; otherwise ``None``. Raises: RuntimeError: If backward computation is enabled and ``step`` is called under ``torch.no_grad()``. Use :meth:`eval` for forward-only execution. TypeError: If a pre-split microbatch container has the wrong type. ValueError: If whole-batch and pre-split inputs are mixed, or if a pre-split container does not have one entry per microbatch. Examples:: >>> # xdoctest: +SKIP("requires a constructed distributed pipeline schedule") >>> output = schedule.step(x, mask=mask, target=target) >>> arg_mbs = [(x0,), (x1,)] >>> kwarg_mbs = [{"mask": mask0}, {"mask": mask1}] >>> target_mbs = [target0, target1] >>> output = schedule.step( ... arg_mbs=arg_mbs, ... kwarg_mbs=kwarg_mbs, ... target_mbs=target_mbs, ... ) rrN) rrrrrErrrryrMrr~r|)r\r(rrHrrFrGrr&r'rrirjrus&$$$$$$$*, r>rQPipelineScheduleMulti.stepasL    000))++. \\E!%!3!3E "\\E  & & ("372M2M       3 / -      #  \\E}}}**5+>+>??"rAc<V^8dQhRS[R,RS[R,RS[R,RS[R,RS[RS[S[S[3,R,/#rErI)r<r=s"r>r?rslE2E2E2$;E24K E2 t E2  E2#s(^d*E2rAc  \ VPWW44wrVe V^,MRpVPV^,V^,Wv4VPUu/uFqPVbK p p\ 4p \ 4p V Fup V ^8d*V P VP V ^, ,4WP^, 8gKLV P VP V ^,,4Kw \4p \VPVP,4EFewr.pVEeyVPpVPpVPp Vf \R4hV\P 8XdcW,pVP#VVV,VV,VR7pVP%VVVVV4VP'VP)V44EMV\P*8XdW,pVP-VV4pW;;,^, uu&W,VP.8HpVP0'd VP.M^pVP3VVRVR7V'dVP1V4VP'VP5V44EMV\P68XdQW,pVP-VV4pVP3VVRRR7VP'VP5V44MV\P88XdW,pW;;,^, uu&W,VP.8HpVP0'd VP.M^pVP;VVR7V'dVP1V4M\=RV 24hV FpVPV,pRpV\?V48d VV,pVfK7VPpVPpVPp Vf \R4hV\P 8XdBV ^,V 9d2W^,,pVP'VPAV44KKV\*\6\839dK\=RV 24h V FpVPV,pRpV\?V48d VV,pVfK7VPpVPpVPp Vf \R4hV\ \839dK}V\6\*39dBV ^, V 9d2W^, ,pVP'VPCV44KK\=RV 24h \E\GV44EKh VPUVPV4R#uupi \Hdkp\JPMR TPTPNPPTTT4\JPMR \STPTR 74ThRp?ii;i) Operate on the microbatches for looped schedules (multiple stages on each rank). TODO: Does not use sorted_batch_isend_irecv(). As a result, this schedule does not support models with skip connections. NzCAll currently supported action types require valid microbatch_indexrTr full_backwardrFrzUnknown computation type zi[Rank %s] pipeline schedule %s caught the following exception '%s' at time_step %s when running action %sz%sr)+rcrrrhr,r-rrrrrrrfrir4r)rNrrrrrQrrrrrrOrPbackward_weight_one_chunkrDrrrrr ExceptionrerrorrrJrr)r\rFrGrrrHrrrstage_index_to_stageall_prev_ranksall_next_ranksrhbackward_counter time_stepr8rrfrrrrgrad_scale_factor prev_rank prev_rank_opsprev_rank_action next_rank next_rank_opsnext_rank_actiones&&&&&&& r>rM(PipelineScheduleMulti._step_microbatchess"//JW.8.DZ]$  AJ ! &8 37,,? 2>  u $, ? $'5#&5/KQ""4#A#A+PQ/#RS--11""4#A#A+PQ/#RS 0*1!*4+>+>tyy+I!J I[ (*%'-'>'>$%66H"("4"4K',a(+;+C+CC 4 A!&!8!8$#H-%h/0> "9" 00!6:x 5#9#9(#CD)-=-K-KK 4 A#33E8D(5:5,9T=Q=QQ&594D4D4DD00!*00$!%*.*7 1 )!--.?@ 5#9#9(#CD)-=-L-LL 4 A#33E8D00$!%*/*/ 1  5#9#9(#CD)-=-M-MM 4 A(5:5,9T=Q=QQ&594D4D4DD00!*77$*78)!--.?@(+DEUDV)WXX"0I$($7$7 $BM'+$ 3}#55+8+C('3+;+L+L(#3#D#D&6&B&B #+"0 e#,/?/G/GG*Q2FF)=1_(M # 5+A+A(+K L G .)*+2 !",";r r s44l::233j M M;BB*pp# p $ p .2 pppppdE2E2E2rAc@a]tRtRtoRtRtRtRtRtV3Rlt Rt Vt R#)_PipelineContexti z=Context passed to custom functions during pipeline execution.Nc<V^8dQh/S[;R&S[S[,R,;R&S[S[,R,;R&S[R,;R&S[R,;R&#)r7 schedule_refNrFrGrr)rrr~r)r<r=s"r>r?_PipelineContext.__annotate__ s`$#  %[4 &  DzD ' t " 4KrArI) rJrKrLrMrrFrGrrrrSrTrUs@r>rr s%G#'G#'I"JFrArc2a]tRtRtoV3RlRltRtVtR#)_CustomFunctionProtocoli c*<V^8dQhRS[RS[RR/#)r7r8ctxr9N)rWr)r<r=s"r>r?$_CustomFunctionProtocol.__annotate__ sKKwK-=K$KrAcR#rYrI)r\r8rs&&&r>__call__ _CustomFunctionProtocol.__call__ srArIN)rJrKrLrMrrSrTrUs@r>r r  s KKrAr caa]tRtRtoRtV3RltV3RlRltRV3RlV3RllltRV3RlV3R llltRV3R lR llt R t V3R lRlt RV3RlRllt Rt VtV;t#)_PipelineScheduleRuntimei a Provides a simple runtime that requires a 'schedule IR' including specified communication operations. Can be instantiated directly by creating _PipelineScheduleRuntime and calling load_csv, or can be subclassed and the subclass can be responsible for creating a schedule IR. c<VPRR4Vn\SV` !V/VB/Vn\ 4Vn/Vn/Vn\\4Vn \4Vn R#) defer_pp_recvFN)rN_defer_pp_recvrr_comp_type_to_function_maprr bwd_recv_ops fwd_recv_opsrr unshard_opsr,unsharded_stages)r\r&r'rs&*,r>r!_PipelineScheduleRuntime.__init__ sh$*JJ$F $)&)LN'.5iEGDFr?%_PipelineScheduleRuntime.__annotate__ s.%L%L*%L1%L  %LrAc  V\\\\\\ \ \39d\RV R24hWP9d\PRV4W PV&R#)aN Register a custom function to be executed for a specific computation type. Args: computation_type: The computation type for which to register the custom function custom_function: The function to execute when this computation type is encountered. Must have signature: (action: _Action, ctx: _PipelineContext) -> None rCz. Only FORWARD, FULL_BACKWARD, BACKWARD_INPUT, BACKWARD_WEIGHT, OVERLAP_F_B, UNSHARD, RESHARD and REDUCE_GRAD are supported.zTComputation type %s is already registered. Overwriting the existing custom function.N) rNrQrOrPr4r-r.r5rDrrr)r\rfrs&&&r>register_custom_function1_PipelineScheduleRuntime.register_custom_function s          $ +,<+=>no  >> > NN<   =L''(89rAch<V^8dQhRS[S[S[S[R,,3,RS[R,/#)r7rNr<r)rrrrWr )r<r=s"r>r?r  s<:G:Gc4$//0:G78:GrAc  h<a\SS`V4/SnVR8XdgVF^p.SPV&W,F@pVf\R\ V4 24hSPV,P V4KB K` R#VR8XEdVP 4FFwr5\V4F2wrdVfK VP'dK\RV RV RV R24h KH VFUp\W,4SPV&\SPV,SP4SPV&KW \SPV3R lSPR 7SnSP'd#\!SPV3R lR 7SnR#R#\#R V: R24h)z Given an in-memory representation for a simple compute-only schedule, lower it to a complex schedule including communication actions. Stores the schedule in self, and must be called before running step_mo() rNz$Expected action to be not None, got rz?Expected compute-only schedule but found communication action 'z ' at rank z , position ze. Communication actions (e.g. SEND_F, RECV_F, etc.) should not be present when format='compute_only'.c*<SPV,#rYrrBr\s&r>rCG_PipelineScheduleRuntime._prepare_schedule_with_comms.. s(F(Fq(IrA)rSrc*<SPV,#rYr'r(s&r>rCr) sD,J,J1,MrA)rSformat= is not implemented)rrpipeline_order_with_commsr4r.r}rrrrrDrJrrrorrrrL)r\rr<rr8rrrsf&& r>_prepare_schedule_with_comms5_PipelineScheduleRuntime._prepare_schedule_with_comms s /8CE& _ $79..t4%mmF~,B4<.Q2248??G , ~ %%,]]_!!*;!7IA)%333",#$$*8:dV;qcJT!U#"8&5 7KM8..t48H2248((8..t4  .<..I++.D * """1@22"M2.# & 1D&EF FrAc4<V^8dQhRS[RS[R,/#rr)r<r=s"r>r?r % s)GGG78GrAc  <VR8Xd-\S V`V4VPVP4R#VR8Xd/p\ VRR7;_uu_4p\ P !V4p\V4F,wrgVUu.uFp\PV4NK upW6&K. VPW2R7RRR4R#\RV: R24huupi +'giR#;i) zLoads a csv in simple format and then lowers it to include communication actions format must be either "compute_only" or "compute_comms". If compute_only, the lowering passes will automatically be run to generate a compute_comms schedule. rrrwr)r<Nr+r,) rrr.rrrrrrWrGrL) r\rr<rrrrrrBrs &&& r>r"_PipelineScheduleRuntime._load_csv% s ^ # G h '  - -d.A.A B  &Gh++wG,!*6!2IDBE$F#QW%5%5a%8#$FGM"311'1I ,+ & 1D&EF F%G,++s+C>C CC C' c4<V^8dQhRS[RS[R,/#rr)r<r=s"r>r?r > s)JJJ78JrAc VR8XdVPf \R4h\VRRR7;_uu_4p\P!V4pVPF%pVP VPV,4K' RRR4R#VR8XdVP f \R4h\VRRR7;_uu_4p\P!V4pVP F%pVP VP V,4K' RRR4R#R# +'giR#;i +'giR#;i) rrNz'Compute only schedule must be availablerrwrrz6Must initialize compute_comms schedule before dump_csv)rr4rrrrr-)r\rr<rrrs&&& r>r"_PipelineScheduleRuntime._dump_csv> s ^ #""*$%NOOhR00GG, //DOOD$7$7$=>010 &--5$LhR00GG, ::DOOD$B$B4$HI;10 ' 100100sA DA D/ D, / E cNa\SPV3RlSP4#)c*<SPV,#rYr'r(s&r>rC4_PipelineScheduleRuntime._simulate..X sd44Q7rA)_simulate_comms_computer-rr[sfr> _simulate"_PipelineScheduleRuntime._simulateU s%&  * * 7     rAc <V^8dQhRS[/#)r7r)r)r<r=s"r>r?r \ s V V'9 VrAc h\VP\4pV'dVPpW0P9dSVPV,FpVP 4K VPVVP PV4W0P 9d\RV: 24hR#R#)zQIf an unshard is active for `stage_idx`, wait() it and mark `stage_idx` unshared.z*Attempted to compute on sharded stage_idx=N) rsubmodrrhrrrr-r4)r\rstage_uses_fsdpr"rs&& r>_assert_unsharded*_PipelineScheduleRuntime._assert_unsharded\ s$U\\:> ))I,,,**955BGGI6$$Y/%%)))4 5 55$'R |%TUU6 rAc<V^8dQhRS[R,RS[R,RS[R,RS[R,RS[RS[S[S[3,R,/#rErI)r<r=s"r>r?r i sl~2~2~2$;~24K ~2 t ~2  ~2#s(^d*~2rAc  <aaaaaaaaSPSSSV4wooSe S^,MRpSPS^,S^,VS4SPUu/uFqPVbK upoSPf \ R4h.oRVVVVVVVV3Rllp SP P4\SPSP,4Fwr\PRV V 4\\V 44;_uu_4V PSP9d4\!SSSSV4p SPV P,!W4MRV P\"8Xd6V P$f \ R4hV P$F p V !V 4K MV !V 4RRR4K S'd\-SP/44K#\1SP24^8wd \ R 4hSP5SPV4R#uupi +'giEKc;i \&dIp\P)RT T 4\P)\+SPT R74ThRp?ii;i) rNzLMust call _prepare_schedule_with_comms() before calling _step_microbatches()c(V^8dQhR\RR/#)r7r8r9N)rW)r<s"r>r?A_PipelineScheduleRuntime._step_microbatches..__annotate__ s"g Ig IGg Ig IrAc <VPpVPe VPMRpV^8g'V\\\39g\ RV: R24hVP pSV,p\VP\4pV^,S9pV^, S9pV\8Xd,SP\VPV444R#V\8Xd,SP\VPV444R#V\ 8XdQW23SP"9d\ RV: RV: R24h\VP%V44SP"W23&R#V\&8XdQW23SP(9d\ RV: RV: R24h\VP+V44SP(W23&R#V\8XdV'dVSP,9dVSP.9g\ RV: R24hVPP14F]p\V\4'gK\3\4VP7R R 74p SP.V,PV 4K_ R#R#V\8XdV'dVSP,9d\ R V: R 24hVSP.9d\ R V: R 24hVPP14F+p\V\4'gKVP94K- SP,P;V4R#R#V\<8XdSP?V4VP@'gPV'gHW23SP"9d\ RV: R24h\CSP"PEW2344VPGVSV,SV,SR7p SPIWJSVS4V'd"SV^,,PKW4R#R#V\L8XEdSP?V4VPN'gPV'gHW23SP(9d\ RV: R24h\CSP(PEW2344SPQWB4p SPRV;;,^, uu&SPRV,SPT8Hp VPWVV R V R7V'd2SV^, ,PYVP[V4V4R#R#V\\8XdSP?V4VPN'gPV'gHW23SP(9d\ RV: R24h\CSP(PEW2344SPQWB4p VPWVV RRR7V'd2SV^, ,PYVP[V4V4R#R#V\^8XdfSP?V4SPRV;;,^, uu&SPRV,SPT8Hp VPaVV R7R#V\8Xd4SPb'd SPTM^p VPeV 4R#\gRV: R24h)Nzaction=z missing mb_indexzRecv twice for stage_idx=z mb_index=z without executing forwardz without executing backwardzUnsharding the same stage_idx=z twiceT)async_opzResharding stage_idx=z without unshardingz before finishing unshardzComputing action=z before receiving inputrz Attempted to run compute action=rFrz is unknown or unsupportedrx)4rfrir-r.r5r4rhrr>rr/r}rrr1rr0rrr2rrrrmodulesr runshardreshardr>rNr@rrrNrrset_local_fwd_inputrQrrrrrset_local_bwd_inputget_local_bwd_outputrOrPrrrrD)r8 comp_typerr"rr?is_next_stage_on_this_rankis_prev_stage_on_this_rank submodulehandlerrrrrFrGrrHr\send_opsrrs& r>_perform_actionD_PipelineScheduleRuntime._step_microbatches.._perform_action sP//I+1+B+B+N''TV A %y0A%BCC**I(3E(zBO)2Q:N)N &)2Q:N)N &F" 5+A+A(+K LMf$ 5+A+A(+K LMf$(D,=,==(4)[xkA[\)>>%T-=-==,/NIrM+_PipelineScheduleRuntime._step_microbatchesi sZ"//JPVW.8.DZ]$  AJ ! &8+ 37,,? 2>  u $,?   ) ) 1 ^  +-g Ig IT ##%!*4+I+I$))+T!U I LLJ  $%@%HII..$2Q2QQ. #%&" 778O8OP" 00K?!--5"01J"KK%+%7%7E+E2&8(/%JI"VR HLLN + t A % !<= = DLL&1O? |JII&  r  *66*3  s>H.?IB!H3<I3 I >II JAJJ)rrrrrr-rrr)rr)rJrKrLrMrrr"r.rrr:r@rMrSrTrrs@@r>rr sd& %L%LN:G:GxGG2JJ.  V V~2~2~2rArcLaa]tRtRtoRtRV3RlV3RllltRtRtVtV;t #)r$ij aM Breadth-First Pipeline Parallelism. See https://arxiv.org/abs/2211.05953 for details. Similar to Interleaved 1F1B, Looped BFS supports multiple stages per rank. What is different is that when microbatches are ready for multiple local stages, Loops BFS will prioritizes the earlier stage, running all available microbatches at once. c<V^8dQhRS[S[,RS[RS[S[,R,RS[S[S[3,S[S[,,R,RS[ RS[ RS[ /#) r7rrrNrrrr) rrrrrrr;r r~ro)r<r=s"r>r?ScheduleLoopedBFS.__annotate__t sw??'(??E!D( ?  S>E#J6= ?  ?%)??rAc <\S V`VVVVVVVR7/Vn\VP4F"pVP V4p WPV&K$ VP VP4R#))rrrrrrrN)rrrrr!_calculate_single_rank_operationsr.) r\rrrrrrrrrank_opsrs &&&&&&&& r>rScheduleLoopedBFS.__init__t s )/#'A'  @B$,,-D==dCH(0   %. ))$*=*=>rAc 2a\VP4p\WPV,VP4p\V4Uu.uFpRNK ppVF1oVP V3Rl\VP 444K3 ^VP^, V, ,pVP R.V,4\ V4F:oVP V3Rl\ \VP 4444K< V#uupi)Nc3Z<"TF p\S\PV4xK" R#5irY)rWr)rNrrrhs& r>rFScheduleLoopedBFS._calculate_single_rank_operations.. s+ ;H %5%=%=xHH ;(+c3Z<"TF p\S\PV4xK" R#5irY)rWr)rQras& r>rrb s+ EH %5%C%CXNN Erc)rrrrrrr)r\rn_local_stages stage_indicesrr]post_warmup_opsrhs&& @r>r\3ScheduleLoopedBFS._calculate_single_rank_operations sT\\* $$~5t7I7I 9>d )D 1$ )D(K OO %d&:&: ; )t11A5<=01#M2K OO (t/C/C)D E 3 %*Es D)r)NNTTF rJrKrLrMrrr\rSrTrrs@@r>r$r$j s??@rAc f\\4p \\4p \\4p \V4U u.uFp RNK pp W,^V^, V, ,,W%,, pV 'dW, ^, pW#,V,p.p^pV 'd\M\p\V4EFtpVV8dsV!V4pV V,;p^,V V&VP \ V\PV44VV^, 8XdVPR.V,4K{K}VTu;8:dW#,8Ed MEMV!V4pV V,;p^,V V&VP \ V\PV44V!V4pV V,;p^,V V&VP \ VVV44VP V4V 'dlVV, V8d[V!VV,4pV V,;p^,V V&VP \ V\PV44V^, pEKEKEKV 'gVP R4V!V4pV V,;p^,V V&VP \ VVV44VP V4V 'gEK VV, V8gEKV!VV,4pV V,;p^,V V&VP \ V\PV44V^, pEKw V 'djV\V48dZV!VV,4pV V,;p^,V V&VP \ V\PV44V^, pKqV#uup irY) rrrrOrQr}rWr)rNrrPr)rer warmup_ops fwd_bwd_ops cooldown_opsrforward_stage_indexbackward_stage_indexnum_1f1b_microbatchesenable_zero_bubblefwd_stage_mb_indexbwd_stage_mb_indexweight_stage_mb_indexrr]rg total_opsbackward_op_idsweight_op_countFULL_BACKWARD_OR_BACKWARD_INPUTrfwd_stage_indexrrbwd_stage_indexrweight_stage_indexweight_mb_indexs&&&&&&&&&& r>_get_1f1b_rank_opsr} s*5S)9)4S)9,7,<5:$K%@KqdKH%@ &ma.?$.F)GG  O'.2(<7IOO--$I ?1"5O/??3  / OO)9)A)A8L Z!^# 89$2 8 8 8 81"5O 2? CC 3  / OO)9)A)A<P 326O 2? CC 3  / OO)H,W   " "2 &!b:o9N&N%9#O4&"(==O'PPO=%&89*(88' 1$'O!&&%226O 2? CC 3  / OO)H,W   " "2 &!!b:o9N&N%9#O4&"(==O'PPO=%&89*(88' 1$[^ 33G!G1//2RS45GH HO 501  "$4$D$Do  1 Og&As N.clV^8dQhR\R\R\R\R\R\R\/#)r7rremicrobatches_per_roundrrmultiply_factorr9r4)r<s"r>r?r?: sN<< << < <  <  < _get_warmup_opsr: s=-q04JJ'ma>OSW=W*XXJ zN: ;;rAcXaa]tRtRtoRtRV3RlV3RllltV3RlRltRtVtV;t #) r#iM aQ The Interleaved 1F1B schedule. See https://arxiv.org/pdf/2104.04473 for details. Will perform one forward and one backward on the microbatches in steady state and supports multiple stages per rank. When microbatches are ready for multiple local stages, Interleaved 1F1B prioritizes the earlier microbatch (also called "depth first"). This schedule is mostly similar to the original paper. It differs by being relaxing the requirement of num_microbatch % pp_size == 0. Using the flex_pp schedule, we will have num_rounds = max(1, n_microbatches // pp_group_size) and it works as long as n_microbatches % num_rounds is 0. As a few examples, support 1. pp_group_size = 4, n_microbatches = 10. We will have num_rounds = 2 and n_microbatches % 2 is 0. 2. pp_group_size = 4, n_microbatches = 3. We will have num_rounds = 1 and n_microbatches % 1 is 0. c<V^8dQhRS[S[,RS[RS[R,RS[S[R3,R,RS[S[S[3,R,RS[S[S[3,S[S[,,R,R S[ R S[ R S[ / # r7rrrNr.rrrrrr)r<r=s"r>r?$ScheduleInterleaved1F1B.__annotate___ s+?+?'(+?+?D +? 34t; +?  _ 45< +? S>E#J6=+?+?%)+?+?rAc H<V^,PVn\S V` VVVVVVVVV R7 \ V4VnV^,P Vn\^W P,4Vn W P,Vn W P,^8wd\RVP RV R24h/Vn \VP4F"p VPV 4p WPV &K$ VPVP4R#)r rrrrrrrrrz_Interleaved 1F1B requires the number of microbatches to be a multiple of the number of rounds ( ), but got .N)rrrrrrerrrnumber_of_roundsrrDrrr\r. r\rrrrrrrrrrr]rs &&&&&&&&&& r>r ScheduleInterleaved1F1B.__init___ s#$AY11 )+//#'A'  "&k1I(( #A~9K9K'K L&48M8M&M# 11 1Q 65595J5J4KL)*!- @B$,,-D==dCH(0   %. ))$*=*=>rAc><V^8dQhRS[S[R,,/#rrL)r<r=s"r>r?r s2 2 gn9M2 rAc aaa\SSPSPSPSP^R7oSPSP,pVS, pW#, pSV,V,p\ P RSSVVV4VV3RlpVVV3Rlp\SPSPSVVSVV4#)r7r=rank %s, warmup_ops %s, 1f1b %s, cooldown_ops %s total_ops %sc<VSP,SP,pVSP,S,#rYrrerrQ local_indexrr\s& r>rnVScheduleInterleaved1F1B._calculate_single_rank_operations..forward_stage_index 44#>#>>$BUBUUK$"4"44< roWScheduleInterleaved1F1B._calculate_single_rank_operations..backward_stage_index W##:%$*E*EE%%&&   $"4"44< r\9ScheduleInterleaved1F1B._calculate_single_rank_operations s$      ' '       ,,t/C/CC$z1 %3 ,|;  K        =  ="              rArrerrrrNNNNTTFrirs@@r>r#r#M s$"+?+?Z2 2 2 rAc^aa]tRtRtoRtR V3RlV3RllltV3RlRltRtRtVt V;t #) r%i a[ The Interleaved Zero Bubble schedule. See https://arxiv.org/pdf/2401.10241 for details. Will perform one forward and one backward on inputs for the microbatches in steady state and supports multiple stages per rank. Uses the backward for weights to fill in the pipeline bubble. In particular this is implementing the ZB1P schedule in the paper. c<V^8dQhRS[S[,RS[RS[R,RS[S[R3,R,RS[S[S[3,R,RS[S[S[3,S[S[,,R,R S[ R S[ R S[ / #rr)r<r=s"r>r?*ScheduleInterleavedZeroBubble.__annotate__ s4?4?'(4?4?D 4? 34t; 4?  _ 45< 4? S>E#J6=4?4?%)4?4?rAc <\WPP4V^,PVn\ S V`VVVVVVVVV R7 \V4VnV^,PVn \^W P,4Vn W P,Vn W P,^8wd\RVP RV R24h/Vn\!VP4F"p VP#V 4p WPV &K$ VP%VPVP,4VnVP'VP4R#)rrzZZero bubble requires the number of microbatches to be a multiple of the number of rounds (rrN)"_check_torch_compile_compatibilityrrJrrrrrrerrrrrrDrrr\_add_bubbles_to_actionsr.rs &&&&&&&&&& r>r&ScheduleInterleavedZeroBubble.__init__ s^ +6>>3J3JK#AY11 )+//#'A'  "&k1I(( #A~9K9K'K L&48M8M&M# 11 1Q 65595J5J4KL)*!- @B$,,-D==dCH(0   %.#::   $"4"4 4  ))$*=*=>rAc><V^8dQhRS[S[R,,/#rrL)r<r=s"r>r?r s7 7 gn9M7 rAc aaa \SSPSPSPSP^R7o SPSP,pVS , pW#, pS V,V,p\ P RSS VVV4VV3RlpVVV 3RlpSp\SPSPS VVSVVVRR7 #)rrrc<VSP,SP,pVSP,S,#rYrrs& r>rn\ScheduleInterleavedZeroBubble._calculate_single_rank_operations..forward_stage_index rrAc<SP^, VS, SP,SP,, pVSP,S,#rwrrs& r>ro]ScheduleInterleavedZeroBubble._calculate_single_rank_operations..backward_stage_index# rrAT)rqr) r\rrrlrmrurnrorprks ff @r>r\?ScheduleInterleavedZeroBubble._calculate_single_rank_operations s$      ' '       ,,t/C/CC$z1 %3 ,|;  K        =  =!%!            !#  rAcVPpRp\4p/p/p/p^p\VP4Fp .WY&^Wi&^Wy&K Rp \4p \VP4EF p Wi,p V \ W),48dK$Rp W),V ,eW),V ,p V f\ R\ V 4 24hV wrppV!WVW4'gRWY,PW),V ,4VeV PWV34Wi;;,^, uu&KWY,PR4Wy;;,^, uu&KWi;;,^, uu&WY,PR4EK# VPV 4V 'gEKcT^8d\PRTT4T#)cV\P8XdV^8wdV^, W3V9dR#R#V\P8Xd2W^, 8XdV\PV3V9#V^,W3V9#R#)rTF)r)rNrQ)rr microbatchnum_stages_globalseen_opss&&&&&r> need_bubbleJScheduleInterleavedZeroBubble._add_bubbles_to_actions..need_bubble> s}%---A:519b"=X"M  '55511!#3#;#;ZHPXXX 22(BBrATFNz)Expected temp_action to be not None, got z?Non zero bubbles added: total_bubbles_added=%s bubbles_added=%s) rr,rrrr4r.r}r-updaterr)r\rrrrr  next_pointer bubbles_addedtotal_bubbles_addedr should_stop temp_seen_ops timestamp temp_actionrhrrrs&& r>r5ScheduleInterleavedZeroBubble._add_bubbles_to_actions; s%% r%r% s,4?4?l7 7 rCCrAcXaa]tRtRtoRtRV3RlV3RllltV3RlRltRtVtV;t #) r&i a{ The Zero Bubble schedule (ZBV variant). See https://arxiv.org/pdf/2401.10241 Section 6 for details. This schedules requires exactly two stages per rank. This schedule will perform one forward and one backward on inputs for the microbatches in steady state and supports multiple stages per rank. Uses backward with respect to weights to fill in the pipeline bubble. This ZB-V schedule would have the "zero bubble" property only if time forward == time backward input == time backward weights. In practice, this is not likely true for real models so alternatively a greedy scheduler could be implemented for unequal/unbalanced time. c<V^8dQhRS[S[,RS[RS[R,RS[S[R3,R,RS[S[S[3,R,RS[S[S[3,S[S[,,R,R S[ R S[ R S[ / #rr)r<r=s"r>r?"ScheduleZBVZeroBubble.__annotate__ s3?3?'(3?3?D 3? 34t; 3?  _ 45< 3? S>E#J6=3?3?%)3?3?rAc <\WPP4V^,PVn\ S V`VVVVVVVVV R7 \VPVPRR7Vn VPFp VPV n K \V4Vn VP^8wd\RVP R24hV^,PVnV^,P Vn/Vn\%VP4F"p VP'V 4p WP"V &K$ VP)VP"4R#)rrrstyle0ZBV requires exactly 2 stages per rank, but got rNrrrJrrrrrrrrrrerDrrrrrr\r.r\rrrrrrrrrrrr]rs&&&&&&&&&& r>rScheduleZBVZeroBubble.__init__ sO +6>>3J3JK#AY11 )+//#'A'  *H    0 0* &\\E.2.L.LE +""&k   ! #B&&'q*  1I((  ).. @B$,,-D==dCH(0   %. ))$*=*=>rAc><V^8dQhRS[S[R,,/#rrL)r<r=s"r>r?r s zzgn9MzrAc 4 \^VP,^, VP4p\V4Uu.uFpRNK ppR wrVrx^VPV, ,^, p Tp VP^, V, p \V 4F-pVP \ V \VR74V^, pK/ Tp \V 4FWpVP \ V \VR74V^, pVP \ V \VR74V^, pKY VPV, p \V 4FxpVP \ V \VR74V^, pVP \ V \VR74VP \ V \VR74V^, pKz We8gWR8dWR8d+VP \ V \VR74V^, pVP \ V \VR74VP \ V \VR74V^, pVP \ V \VR74V^, pVP \ V \VR74VP \ V \VR74V^, pKYxrTp\V4FWpVP \ V \VR74V^, pVP \ V \VR74V^, pKY VPV, p\V4FWpVP \ V \VR74V^, pVP \ V \VR74V^, pKY W8d-VP \ V \VR74V^, pK2W8d-VP \ V \VR74V^, pK2W8XdWu8Xg\RV RV RV 24hW8XdW8Xg\RV RV RV 24hVUu.uF4pVe,VPeVPVP8dTMRNK6 ppV#uupiuupi) r7N)rfriz0Expected w0_cnt == b0_cnt == f0_cnt, got w0_cnt=z , b0_cnt=z , f0_cnt=z0Expected w1_cnt == b1_cnt == f1_cnt, got w1_cnt=z , b1_cnt=z , f1_cnt=)rrrr) rrrrrr}rWr*r+r,r4ri)r\rn_microrr]f0_cntf1_cntb0_cntb1_cnt warmup_n1stage_id_chunk0stage_id_chunk1 warmup_n2 warmup_n3w0_cntw1_cnt cooldown_n1 cooldown_n2r8s&& r>r\7ScheduleZBVZeroBubble._calculate_single_rank_operations sLa$,,,q0$2F2FG8=d )D 1$ )D*4&++d23a7 //A-4y!A OO!fU  aKF "  y!A OO!fU  aKF OO!fU  aKF"&&- y!A OO!fU  aKF OO!fU  OO!fU  aKF"o!1'!f !  OO!fU  OO!fU  aKF OO!fU  aKF OO!fU  OO!fU  aKF {#A OO!fU  aKF OO!fU  aKF$((4/ {#A OO!fU  aKF OO!fU  aKF$o OO!fU  aKFo OO!fU  aKF V%5 B6()TZS[[dekdlm  V%5 B6()TZS[[dekdlm #  # %++7++d.B.BB  #   m*EX  s R:Rrerrrrrrrirs@@r>r&r& s' 3?3?jzzzrAcXaa]tRtRtoRtRV3RlV3RllltV3RlRltRtVtV;t #) r'iCz The DualPipeV schedule. A more efficient schedule variant based on the DualPipe schedule introduced by DeepSeek in https://arxiv.org/pdf/2412.19437 Based on the open sourced code from https://github.com/deepseek-ai/DualPipe c<V^8dQhRS[S[,RS[RS[R,RS[S[R3,R,RS[S[S[3,R,RS[S[S[3,S[S[,,R,R S[ R S[ R S[ / #rr)r<r=s"r>r?ScheduleDualPipeV.__annotate__Ks8?8?'(8?8?D 8? 34t; 8?  _ 45< 8? S>E#J6=8?8?%)8?8?rAc "<\WPP4V^,PVn\ S V`VVVVVVVVV R7 \VPVPRR7Vn VPFp VPV n K \V4Vn VP^8wd\RVP R24hW P8d\RV RVP R24hV^,PVnV^,P Vn/Vn\%VP4F"p VP'V 4p WP"V &K$ VP)VP"4R #) rrrrrrzDDualPipeV requires at least as many microbatches as stages, but got z microbatches and z stages.Nrrs&&&&&&&&&& r>rScheduleDualPipeV.__init__Ks +6>>3J3JK#AY11 )+//#'A'  *H    0 0* &\\E.2.L.LE +""&k   ! #B&&'q*  ,, ,V!""4T5E5E4FhP  1I((  ).. @B$,,-D==dCH(0   %. ))$*=*=>rAc><V^8dQhRS[S[R,,/#rrL)r<r=s"r>r?rs ^^gn9M^rAcaaa.p/o.oVPpVPp\W3^,RR7pWQ,wrgRV3RlloRVV3RllpRVVV3Rllp R VV3R llp W1, ^, ^,p \V 4Fp V !W&\4K V^,p \V 4Fp V !W&\4V !W'\4K W1, ^, p\V4F%p V !W'\ 4V !V4V !W'\4K' WC^,, V,^,p\V4FHpV^8Xd)W^, 8XdV !W&\4V !W'\ 4M V!VVVR 7V!VVVR 7KJ W1, ^, p\V4Fp V !W'\ 4V!VVVR 7K V^,pR p\V4FypVV^,8XdV^,^8XdR pV'd\ M\ pV !W'V4VV^,8XdV^,^8XdR pV'd\ M\ pV !W&V4K{ W1, ^, p\V4F(p V !V4V'd\ M\ pV !W&V4K* V^,p\V4F p V !V4K V#)r7rrc$V^8dQhR\/#r3r4)r<s"r>r?IScheduleDualPipeV._calculate_single_rank_operations..__annotate__s C C3 CrAc<V\3pV\3pSPV^4^,SV&SPV^4^,SV&R#)zbHelper method to increment BACKWARD_INPUT and BACKWARD_WEIGHT counters when FULL_BACKWARD is used.N)rOrPr)rh input_key weight_keycounterss& r>increment_backward_countsVScheduleDualPipeV._calculate_single_rank_operations..increment_backward_countssK$n5I%7J"*,,y!"#BHZ rAc<V^8dQhR\R\R\/#)r7r forward_stagebackward_stage)rr)r<s"r>r?rs& 6 6 6 6  6rAc<V\3pV\3pSPV^4pSPV^4p\V\V4\V\V43pVP \R\ RV44V^,SV&S !V4R#)zYHelper method to add an overlapped forward+backward action which tracks microbatch index.Nrx)rNrOrrWrQr}r4) rrr forward_key backward_keyr rrcrrs &&& r>add_overlap_f_bLScheduleDualPipeV._calculate_single_rank_operations..add_overlap_f_bs)'2K*N;L!k15J",,|Q7K w ; {CK NN72{D+F G%/NH[ ! %n 5rAc<V^8dQhR\R\R\/#)r7rrhrf)rrr))r<s"r>r?rs& - - - -/ -rAc <V\8wdW3MV\3pSPV^4pVP\ WV44V\8Xd S!V4R#V\8XdSPW34V^,SV&R#)rN)rQrOrr}rW)rrhrfrrrr weight_queues&&& r> add_actionGScheduleDualPipeV._calculate_single_rank_operations..add_actions$}4/!>2   ||C+H NN7;(K L =0)+6$~5 ''(?@ (1  rAc$V^8dQhR\/#)r7r)r)r<s"r>r?rs C C$ CrAc<S'gR#SP^4wrVP\V\V44V\3pSP V^4^,SV&R#)z4Helper method to add a weight action from the queue.N)rNr}rWrPr)ractual_stage_indexr|rrrs& r>add_weight_action_if_pendingYScheduleDualPipeV._calculate_single_rank_operations..add_weight_action_if_pendings`2>2B2B12E /  NN&## -o>J#+<< A#>#BHZ rA)rrFT)rrrrrNrOrQ)r\rrr num_chunksrank_to_stages stage0_index stage1_indexrrrstep_1rstep_2step_3step_4rstep_5step_6 enable_zbrNstep_7step_8rrrs&& @@@r>r\3ScheduleDualPipeV._calculate_single_rank_operationss(*   && )) 7 1}C &4%9"  C C 6 6. - -0 C C$"Q&!+vA wg 6vA wg 6 wg 6 !A%vA wn = ( 1 wg 6 !m+d2Q6vAAv$a-/7':7-@".#/ *+ $!A%vA wm < *+  vAFaKD1HM *3I wi 8FaKD1HM *3I wi 8!A%vA ( 1*3I wi 8 vA ( 1rArrrirs@@r>r'r'Cs'8?8?t^^^rAc$V^8dQhR\/#)r7 schedule_namer:)r<s"r>r?r?&sAAcArAcTR\R\R\R\R\R\ R\ R\R \/ pVUu/uFq"P4VbK ppVP4pWC9d)\R V R \VP44 24hWV,,#uupi) z Maps a schedule name (case insensitive) to its corresponding class object. Args: schedule_name (str): The name of the schedule. 1F1BInterleaved1F1BGPipe LoopedBFSInterleavedZeroBubblerr ZBVZeroBubble DualPipeVzUnknown schedule name 'z'. The valid options are ) r!r#r"r$r%rr r&r'lowerrDrkeys)r  schedule_maprlowercase_keyslowercase_schedule_names& r>get_schedule_classr&s  2&!> "8!6.& L-99LqggilLN9+1134%m_4MdS_SdSdSfNgMh i   '>? @@ :sB%cTV^8dQhR\\.\3,R\/#)r7rSr)rr)r<s"r>r?r?As+BB#+SE3J#7BEHBrAc aaa a \V4UUu/uF!pY0V,Uu.uF qDfKVNK upbK# ppp\V4Uu/uFq3.bK upo S Uu/uF q3\4bK upo RV V 3RllpRV VV3RllpV'EdRp\V4Fhp\W,4^8XdKW,^,pV!V4'd(Ve V!W84W,P^4RpK_V!VR4Kj \VRR7Fp \W ,4^8XgKW K \V4Fp\W,4^8XdKS V,R ,eK/W,^,pV!V4'gKNVe%VS V,R &S V,P V4W,P^4K \VRR7Fp \W ,4^8XgKW K V'dEK\ R \ S 44VF"p\ R V: R W,^, 24K$ \R 4hS #uupiuuppiuupiuupi)aThis function dry-run simulates the actions in the schedule from the perspective of all ranks, and flags any deadlocks caused by missing or misordered communications. It also simulates any bubbles in time where a rank can not execute any action due to waiting for unmet dependencies. The total number of simulator steps can be used as a metric for unit tests involving IR optimization passes as reordering and merging of IR can reduce the number of simulated steps. The simulation is not high-fidelity and does not model overlapping of compute and communication, or cuda streams. Future work may be to enhance this and model the compute time, comms overlap, and even memory. Nc>V^8dQhR\R\R,/#)r7rr8N)rrW)r<s"r>r?-_simulate_comms_compute..__annotate__Ws--c-7T>-rAct<SV,PV4VeSV,PV4R#R#rY)r}r-)rr8_prev_ops_rank _schedules&&r>add_to_schedule0_simulate_comms_compute..add_to_scheduleWs3$v&   4 $ $V , rAc>V^8dQhR\R,R\/#)r7r8Nr9rW)r<s"r>r?r\s"8B8B7T>8Bd8BrAc<VfR#VPpSS !V4,pVP\8XduVP^8XdR#\VP\VP 4V9dR#\VP^, \VP 4V9dR#R#VP\ \39dVPS ^, 8XdR#\VP\VP 4V9dR#\VP^,\ VP 4V9dR#\VP^,\VP 4V9dR#R#VP\8XdR#VP\8Xd*\VP\VP 4pW29#VP\8Xd7V^, p\V\VP 4pVSS !V4,9#VP\8Xd[\VP\ VP 4p\VP\VP 4pWb9;'gWr9#VP\8Xd7V^,p\V\VP 4pVSS !V4,9#\RV 24h)NTFzUnsupported action type ) rhrfr*rWr0rirOrQr2rPr/r1rD) r8r"prev_ops expected_fpeer_stage_idx expected_send expected_b expected_bwr!rrSs & r>rg3_simulate_comms_compute.._ready_to_schedule\sa >&& !- ":;  " "a '!!Q&**FF4K4KLPXX**Q.63J3JKxW  $ $(G G!!Z!^3v))663J3JKxW**Q.@W@WX**Q. v?V?VW  $ $ 7  $ $ . !3!3Q8O8OPJ) )  $ $ .&]N#NFFr9r9AsP>**D .@.Qqq.@@* $N323Tb32IHQ.QytSU{y.QN-- 8B8Bt .>*D>'(A-#)!,F!&))%#D1$((+d++5A>$%*"%6 >*D>'(A-r".#)!,F!&))%*0IdOB'"4(,,V4$((++5A>$%*"%6x )+A)+L M&~n.B1.E-FGH':; ; kA2/Rs&IIII I!I&Ic.p\V4Fmp\W,4FUwrEVfK TPR\V4RVP\ \ \39dRMRRRRVR VR VR ^/4KW Ko ^RIp\VR 4;_uu_4pVPR V/V4RRR4R# +'giR#;i)a This function dumps a schedule IR into a chrometrace format so it can be visualized. It is currently very basic and only serves as a graphical alternative to dumping the schedule IR as text. As future work we may extend this to include more accurate heuristics for durations, or let users input durations, add 'flow events' to let the UI show the connection between sends and recvs, and model cuda streams for comm/compute as separate streams on the chrometrace view. Nr\cat computation communicationphXpidtidtsdurr traceEvents) rrr}r;rfr*r3r,jsonrdump)schedulereventsrtimestepr8r;fs&& r>_dump_chrometracerAsFx  )(. 9 H~ MMCK!22q!Qi?&,#44(1  !:!& h   =&)1-    s B66 C cFV^8dQhR\\,R\/#)r7rr )rrr;)r<s"r>r?r?s" # $58rAc :VFp\VP\PP4'gK9VPP 4F=p\V\ 4'gK\RV R\V4P 24h K R#)z Check if the schedule is compatible with torch.compile. Args: stages: List of pipeline stages to check schedule_name: Name of the schedule for error message Raises: RuntimeError: If any stage uses torch.compile zThe ze schedule is not supported with stage modules that have used torch.compile. Found OptimizedModule in N) rr>rnnModulerHrrEr.rJ)rr rmodules&& r>rrs}%,,88 ll**,F&/22"=/*004V 0E0E/FH- rA) rrr r!r"r#r$r%r&r'rY))rF)r7)mrrrloggingreabcrr collectionsrrcollections.abcr dataclassesrenumr functoolsr typingr r r rrrtorch.distributed distributedr  torch._dynamortorch.distributed.fsdprrtorch.nn.modules.lossrtorch.profilerr_utilsrrrrrrrrrrrr__all__ getLoggerrJrr;r)rNrOrPr-r.r/r0r1r2rQr4r5rsr*r+r,r3compilerrWrrrrrrrrr"r!rrrJrQrorrr rr rr$r}rr#r%r&r'rr9rArrIrAr>r[s  #,$!;; )<'*  @?    8 $NsDN.  " "!00"22  " "  " "         .. ** ** %Q'  R T jT n  <~u u p.+b6o.od2"12"jx*xvz)zz!$BPf#Ltn[|'Dp2-p2f  LhL}24}2@D0DNFR<&q 6q h}$<}@4D`0`FA6BJ!.HrA