+
    œ°³j0  ã                  ó8  € ^ RI Ht ^ RIt^ RIHt ^ RIHt ^RIHt ^RI	H
t
Ht ]'       d   ^ RIHt ]
! 4       '       d   ^ RIt^ RIHt ]! 4       '       d   ^ RIHt R R lt ! R	 R
]P&                  4      t ! R R4      tR R ltR R ltR R ltR R ltR# )é    )ÚannotationsN)Úwraps)ÚTYPE_CHECKING)ÚCausalLMOutputWithPast)Úis_torch_availableÚis_torch_distributed_availablec               ó(   € V ^8„  d   QhRRRRRRRR/# )é   Úforward_signatureúinspect.SignatureÚargsÚtupleÚkwargsÚdictÚreturn© )Úformats   "Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/distributed/pipeline_parallel.pyÚ__annotate__r   #   s+   € ÷ !ñ !Ð,=ð !ÀUð !ÐTXð !Ð]añ !ó    c                óp   € V P                   ! V/ VB pVP                  4        \        VP                  4      # ©N)Úbind_partialÚapply_defaultsr   Ú	arguments)r   r   r   Úbounds   &&& r   Ú_bind_forward_kwargsr   #   s2   € Ø×*Ò*¨DÐ;°FÑ;€EØ	×ÑÔÜ�—‘Ó Ð r   c                  ó2   a € ] tR t^)tRtV 3R ltR tRtV ;t# )ÚPipelineIdentityLayerzLAn identity layer replacing modules not owned by the current pipeline stage.c                	ó$   <€ \         SV `  4        R # r   )ÚsuperÚ__init__)Úselfr   r   Ú	__class__s   &*,€r   r"   ÚPipelineIdentityLayer.__init__,   s   ø€ Ü‰ÑÖr   c                óh   € V'       d
   V^ ,          # \        \        VP                  4       4      4      # )zQReturn the first arg from args or the first value from kwargs as nothing is used.)ÚnextÚiterÚvalues)r#   r   r   s   &*,r   ÚforwardÚPipelineIdentityLayer.forward/   s#   € çˆt�A�wÐ?¤D¬¨f¯m©m«oÓ)>Ó$?Ð?r   r   )	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r"   r*   Ú__static_attributes__Ú__classcell__)r$   s   @r   r   r   )   s   ø† ÙVõ÷@ð @r   r   c                  ób   € ] tR t^4tRtR R ltRRRR/R R lltR	 R
 ltR R ltR R lt	Rt
R# )ÚPipelineStagezCPipeline-parallel stage metadata derived from a 1-D PP device mesh.c               ó   € V ^8„  d   QhRR/# )r
   Úpp_meshú(torch.distributed.device_mesh.DeviceMeshr   )r   s   "r   r   ÚPipelineStage.__annotate__7   s   € ÷ Eñ EÐ Hñ Er   c                	ó  € VP                  4       V n        VP                  4       V n        VP	                  4       V n        V P                  ^ 8H  V n        V P                  V P                  ^,
          8H  V n        V P                  ^ 8”  d   V P                  ^,
          MRV n        V P                  V P                  ^,
          8  d   V P                  ^,           MRV n	        \        P                  ! V P
                  4      R8H  V n        R# )r   NÚgloo)Úget_local_rankÚpp_rankÚsizeÚpp_sizeÚ	get_groupÚpp_groupÚpp_is_first_stageÚpp_is_last_stageÚpp_prev_rankÚpp_next_rankÚdistÚget_backendÚcomm_on_cpu)r#   r6   s   &&r   r"   ÚPipelineStage.__init__7   s¼   € Ø×-Ñ-Ó/ˆŒØ—|‘|“~ˆŒØ×)Ñ)Ó+ˆŒØ!%§¡°Ñ!2ˆÔØ $§¡°·±¸qÕ0@Ñ @ˆÔØ04·±¸qÔ0@˜DŸL™L¨1Ö,ÀdˆÔØ04·±¸t¿|¹|ÈaÕ?OÔ0O˜DŸL™L¨1Ö,ÐUYˆÔÜ×+Ò+¨D¯M©MÓ:¸fÑDˆÖr   ÚtensorNÚshapec               ó0   € V ^8„  d   QhRRRRRRRRR	R
RR/# )r
   Ú	operationÚstrÚdeviceútorch.deviceÚdtypeútorch.dtyperI   útorch.Tensor | NonerJ   ztuple[int, ...] | Noner   r   )r   s   "r   r   r8   A   sN   € ÷ )Jñ )Jàð)Jð ð	)Jð
 ð)Jð $ð)Jð &ð)Jð 
ñ)Jr   c               óF  € V P                   '       d   \        P                  ! R4      MTpR;rxVR8X  d:   V P                  '       d   R# V P                  p\        P
                  ! WSVR7      pMVVR8X  dB   V P                  '       d   R# V P                  pVP                  WcR7      P                  4       pM\        RV 24      hVP                  R4      p	V	'       d   TMTp
\        P                  ! V	'       d   \        P                  M\        P                  WJV P                   R	7      p\        P"                  ! V.4       F  pVP%                  4        K  	  VP&                  R
8X  d   \        P(                  P+                  4        V	'       d   R# VP                  W#R7      # )z>Point-to-point pipeline communication between adjacent stages.ÚcpuNÚrecv_forward©rP   rN   Úsend_forward©rN   rP   z.Unsupported pipeline communication operation: Úsend)ÚgroupÚcuda)rG   ÚtorchrN   rA   rC   ÚemptyrB   rD   ÚtoÚ
contiguousÚ
ValueErrorÚ
startswithrE   ÚP2POpÚisendÚirecvr@   Úbatch_isend_irecvÚwaitÚtyper[   Úsynchronize)r#   rL   rN   rP   rI   rJ   Úcomm_deviceÚsrcÚdestÚis_sendÚ	peer_rankÚopÚreqs   &&$$$$       r   ÚcommunicateÚPipelineStage.communicateA   s?  € ð .2×-=×-=Ð-=”e—l’l 5Ô)À6ˆØÐˆà˜Ô&Ø×%×%Ð%Ùà×#Ñ#ˆCä—[’[ ¸KÔH‰Fà˜.Ô(Ø×$×$Ð$Ùà×$Ñ$ˆDØ—Y‘Y k�YÓ?×JÑJÓL‰Fô ÐMÈiÈ[ÐYÓZÐZð ×&Ñ& vÓ.ˆß#‘D¨ˆ	Ü�ZŠZ§gœŸ
š
´4·:±:¸vÐX\×XeÑXeÔfˆä×)Ò)¨2¨$Ö/ˆCØ�H‰HŽJñ 0à×Ñ˜vÔ%Ü�J‰J×"Ñ"Ô$çˆtÐI F§I¡I°V IÓ$IÐIr   c               ó$   € V ^8„  d   QhRRRRRR/# )r
   ÚrankÚintÚ
num_layersr   ztuple[int, int]r   )r   s   "r   r   r8   l   s!   € ÷ &ñ &¨ð &¸#ð &À/ñ &r   c                ó~   € W P                   ,          pW,          pWP                   ^,
          8X  d   TMWC,           pWE3# )zø
Return [start, end) owned by rank.

Example with pp_size=3 and num_layers=32:
    layers_per_rank = 32 // 3 = 10
    rank 0 -> [0, 10)   # layers 0-9
    rank 1 -> [10, 20)  # layers 10-19
    rank 2 -> [20, 32)  # layers 20-31 (10 + 2 remainder)
)r>   )r#   rs   ru   Úlayers_per_rankÚstart_layerÚ	end_layers   &&&   r   Úlayer_range_for_rankÚ"PipelineStage.layer_range_for_rankl   s9   € ð %¯©Õ4ˆØÕ,ˆà"&¯,©,¸Õ*:Ô":‘JÀÕ@]ˆ	ØÐ%Ð%r   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r
   ÚkeyrM   ru   rt   Úbase_model_prefixr   z
int | Noner   )r   s   "r   r   r8   }   s)   € ÷ ñ  Sð °cð Ècð ÐV`ñ r   c                óØ  € V R2pVP                  V R24      '       d   ^ # VP                  V R24      '       g   VP                  R4      '       d   V P                  ^,
          # V R2pVP                  V4      '       g   R# \        VP                  R4      ^,          4      p\	        V P                  4       F,  pV P                  Wr4      w  r‰Y†u;8:  d   V	8  g   K&   Vu #  K.  	  R# )zPReturn the PP rank that owns a checkpoint parameter key, or ``None`` if unknown.Ú.zembed_tokens.znorm.zlm_head.zlayers.N)ra   r>   rt   ÚsplitÚrangerz   )
r#   r}   ru   r~   Úbase_prefixÚlayers_prefixÚ	layer_idxrs   rx   ry   s
   &&&&      r   Úfind_rank_for_keyÚPipelineStage.find_rank_for_key}   sÐ   € à*Ð+¨1Ð-ˆà�>‰>˜[˜M¨Ð7×8Ò8Ùà�>‰>˜[˜M¨Ð/×0Ò0°C·N±NÀ:×4NÒ4NØ—<‘< !Õ#Ð#à&˜- wÐ/ˆØ�~‰~˜m×,Ò,Ùä˜Ÿ	™	 #› qÕ)Ó*ˆ	Ü˜$Ÿ,™,Ö'ˆDØ%)×%>Ñ%>¸tÓ%PÑ"ˆKØÖ3¨)×3Ø’ò 4ñ (ñ
 r   c               ó(   € V ^8„  d   QhRRRRRRRR/# )	r
   rI   rR   rP   rQ   rN   rO   r   útorch.Tensorr   )r   s   "r   r   r8   “   s2   € ÷  5ñ  5à#ð 5ð ð	 5ð
 ð 5ð 
ñ 5r   c               óL  € V P                   ^8:  d   V# V P                   ^,
          pV P                  '       d   \        P                  ! R4      MTp^pV P                  '       d„   VP
                  V8X  g   Q RV RVP
                   24       hVP                  WRR7      P                  4       p\        P                  ! \        VP                  4      \        P                  VR7      pM'\        P                  ! V\        P                  VR7      p\        P                  ! W„V P                  R7       V P                  '       g0   \        P                  ! \!        VP#                  4       4      W%R7      p\        P                  ! XW@P                  R7       VP                  W2R7      # )z­Broadcast logits from the last PP rank to every rank. Only the last stage computes logits, but every rank must return them so `generate()` works without PP-specific changes.rT   zExpected logits with z( dims (batch, seq_len, vocab_size), got rX   rV   )rj   rZ   )r>   rG   r\   rN   rB   Úndimr^   r_   rI   ÚlistrJ   Úlongr]   rE   Ú	broadcastr@   r   Útolist)	r#   rI   rP   rN   Ú	last_rankri   Úlogits_ndimÚlogitsÚ	shape_msgs	   &&$$     r   Úbroadcast_from_lastÚ!PipelineStage.broadcast_from_last“   s5  € ð �<‰<˜1ÔØˆMà—L‘L 1Õ$ˆ	Ø-1×-=×-=Ð-=”e—l’l 5Ô)À6ˆàˆð × × Ð Ø—;‘; +Ô-ð Ø'¨ }Ð4\Ð]c×]hÑ]hÐ\iÐjóÐ-ð —Y‘Y k�YÓ?×JÑJÓLˆFÜŸš¤T¨&¯,©,Ó%7¼u¿z¹zÐR]Ô^‰IäŸš K´u·z±zÈ+ÔVˆIÜ�Š�y°t·}±}ÕEð ×$×$Ð$Ü—[’[¤ y×'7Ñ'7Ó'9Ó!:À%Ô\ˆFÜ�Š�v 9·M±MÕBà�y‰y ˆyÓ4Ð4r   )rG   r@   rA   rB   rD   rC   r<   r>   )r,   r-   r.   r/   r0   r"   rp   rz   r†   r”   r1   r   r   r   r4   r4   4   s<   † ÙMõEð)Jð '+ð)Jð )-÷)JõV&õ"÷, 5ñ  5r   r4   c               ó$   € V ^8„  d   QhRRRRRR/# )r
   Úmodelú	nn.Moduler6   r7   r   r   )r   s   "r   r   r   ¶   s#   € ÷ .ñ . ið .Ð:bð .Ðgpñ .r   c                ó"  a aa€ \        V4      pVS n        \        S S P                  4      pVP                  p\        V4      pVP                  VP                  V4      w  rg\        S P                  RR4      pVP                  ;'       g    T;'       d    VP                  p	V	'       g   \        4       Vn        \        V4       F  p
W¦8  g	   W§8¼  g   K  \        4       WJ&   K  	  VP                  '       g   \        4       Vn        \        4       S n        V'       d   VP                  '       g   / S n        \        S RR4      '       gG   S P"                  o\$        P&                  ! S4      o\)        S4      VV V3R l4       pVS n        RS n        S # )z8Naive even split of `base_model.layers` across PP ranks.Útie_word_embeddingsFÚ_pp_forward_wrappedc                 ó&   <€ \        SSS.V O5/ VB # r   )Úpipeline_parallel_naive_forward)r   r   r   r—   Úoriginal_forwards   *,€€€r   Úpp_naive_forwardÚ4apply_pipeline_parallelism.<locals>.pp_naive_forwardÙ   s    ø€ ä2°5Ð:JÐL]ÐoÐ`dÒoÐhnÑoÐor   T)r4   Ú	_pp_stageÚgetattrr~   ÚlayersÚlenrz   r<   ÚconfigrA   rB   r   Úembed_tokensr‚   ÚnormÚlm_headÚall_tied_weights_keysr*   ÚinspectÚ	signaturer   r›   )r—   r6   ÚstageÚ
base_modelr£   ru   rx   ry   ÚtiedÚkeep_embed_tokensr…   rŸ   r   rž   s   f&          @@r   Úapply_pipeline_parallelismr°   ¶   sW  ú€ ô ˜'Ó"€EØ€E„Oä˜ × 7Ñ 7Ó8€JØ×Ñ€FÜ�V“€Jà"×7Ñ7¸¿¹ÀzÓRÑ€KÜ�5—<‘<Ð!6¸Ó>€Dð ×/Ñ/×TÐT°D×4SÐ4S¸U×=SÑ=SÐßÜ"7Ó"9ˆ
Ôä˜:Ö&ˆ	ØÔ" iÖ&<Ü 5Ó 7ˆFÓñ 'ð ×!×!Ð!Ü/Ó1ˆ
ŒÜ-Ó/ˆŒ÷ �E×*×*Ð*Ø&(ˆÔ#ô �5Ð/°×7Ò7Ø Ÿ=™=ÐÜ#×-Ò-Ð.>Ó?Ðä	ÐÓ	 ö	pó 
!ð	pð )ˆŒØ$(ˆÔ!à€Lr   c               ó$   € V ^8„  d   QhRRRRRR/# )r
   Ú
fwd_kwargsr   Úhidden_sizert   r   ztuple[int, ...]r   )r   s   "r   r   r   ç   s&   € ÷ Wñ W Tð W¸ð WÀñ Wr   c                óÆ   € V P                  R 4      ;pe   \        VP                  4      # V P                  R4      ;pe   . VP                  R,          OVN5# \        R4      h)Úinputs_embedsÚ	input_ids:Nr
   Nz>Cannot determine hidden states shape for pipeline recv_forward)Úgetr   rJ   r`   )r²   r³   rµ   r¶   s   &&  r   Ú_hidden_states_shaper¸   ç   s`   € Ø#Ÿ™¨Ó8Ð8ˆÒEÜ�]×(Ñ(Ó)Ð)Ø—^‘^ KÓ0Ð0ˆ	Ò=Ø2�—‘ Õ$Ð2 kÑ2Ð2Ü
ÐUÓ
VÐVr   c               ó$   € V ^8„  d   QhRRRRRR/# )r
   r²   r   Úhidden_statesr‰   r   r   )r   s   "r   r   r   ï   s"   € ÷ ñ °Dð Èð ÐZ^ñ r   c                ó2   € V P                  RR4       WR&   V # )zQStage 0 uses input_ids; later stages use received hidden states as inputs_embeds.r¶   Nrµ   )Úpop)r²   rº   s   &&r   Ú#_feed_hidden_states_as_input_embedsr½   ï   s   € à‡N�N�; Ô%Ø"/ˆÑØÐr   c               ó    € V ^8„  d   QhRRRR/# )r
   r—   r˜   r   r   r   )r   s   "r   r   r   ö   s    € ÷ $Rñ $RØð$RØ;Lñ$Rr   c                óü  € V P                   p\        V P                  4       4      P                  p\        V P                  4       4      P                  p\        W#V4      pTp	VP                  '       g?   \        W€P                  P                  4      p
VP                  R WgV
R7      p\        W‹4      p	VP                  '       d!   V! R/ V	B pVP                  VP                  ríMv\        W P                   4      pV	P#                  4        UUu/ uF  w  ppVR9  g   K  VVbK  	  pppV! R/ VB pVP                  RWgVP$                  R7       RVP                  ríVP'                  W×VR7      p\)        WÞR7      # u uppi )	rU   )rN   rP   rJ   rW   )rN   rP   rI   NrV   )r’   Úpast_key_valuesr   >   ÚlabelsÚlogits_to_keep)r¡   r'   Ú
parametersrN   rP   r   rA   r¸   r¥   r³   rp   r½   rB   r’   rÀ   r¢   r~   ÚitemsÚlast_hidden_stater”   r   )r—   rž   r   r   r   r¬   rN   rP   Úcaller_kwargsr²   rJ   rº   Úoutputsr’   rÀ   r­   ÚkÚvÚbase_kwargsÚbase_outputss   &&&*,               r   r�   r�   ö   sa  € ð �O‰O€Eä�%×"Ñ"Ó$Ó%×,Ñ,€FÜ�×!Ñ!Ó#Ó$×*Ñ*€Eô )Ð):À&ÓI€MØ€Jð ×"×"Ð"Ü$ ]·L±L×4LÑ4LÓMˆØ×)Ñ)¨.ÀÐ\aÐ)ÓbˆÜ8¸ÓVˆ
à××Ðá"Ñ0 ZÑ0ˆØ")§.¡.°'×2IÑ2I‘ô ˜U×$;Ñ$;Ó<ˆ
Ø(2×(8Ñ(8Ô(:ÔdÑ(:¡  1¸aÐGcÑ>c”t�q˜!’tÑ(:ˆÑdÙ!Ñ0 KÑ0ˆØ×Ñ˜.°È\×MkÑMkÐÔlØ"&¨×(DÑ(D�ð
 ×&Ñ& vÀ6Ð&ÓJ€Fä!¨ÔQÐQùó es   ÄE8ÄE8)Ú
__future__r   rª   Ú	functoolsr   Útypingr   Úmodeling_outputsr   Úutilsr   r   Útorch.nnÚnnr\   Útorch.distributedÚdistributedrE   r   ÚIdentityr   r4   r°   r¸   r½   r�   r   r   r   Ú<module>rÖ      sw   ðõ #ã Ý Ý  å 5ß F÷ Ýá×ÒÛÝá!×#Ò#Ý$õ!ô@˜BŸK™Kô @÷5ñ 5õD.õbWõ÷$Rr   