
    qib                    `   U d dl mZ d dlZd dlZd dlZd dlZd dlmZ ddlm	Z	 ddl
mZmZ ddlmZ ddlmZ  e       r*d dlZd dlmZ d d	lmZ ej                  j+                         Z ej.                  e      Z	 dF	 	 	 dGd
ZdHdZdIdJdZ e       r|ej:                  ej<                  ej>                  ej@                  ejB                  ejD                  ejF                  ejH                  ejJ                  ejL                  ejN                  dZ(dKdZ)d Z*	 dL	 	 	 	 	 	 	 	 	 dMdZ+dNdOdZ,d Z- G d dej\                  j^                        Z0 G d dej\                  j^                        Z1 G d dej\                  j^                        Z2 G d dej\                  j^                        Z3 G d dej\                  j^                        Z4d Z5d Z6d Z7d  Z8d! Z9	 	 	 dF	 	 	 dPd"Z: G d# d$      Z; G d% d&e;      Z< G d' d(e;      Z= G d) d*e;      Z> G d+ d,e;      Z? G d- d.e<      Z@ G d/ d0e?      ZA G d1 d2e;      ZB G d3 d4e;      ZC G d5 d6e;      ZD G d7 d8e;      ZE G d9 d:e;      ZF G d; d<e;      ZG G d= d>e      ZH eH       ZId>eJd?<   	 	 	 	 	 	 	 	 dQd@ZK	 	 	 	 	 	 	 	 dRdAZL	 dNdBZMdC ZNdSdDZOdE ZPy)T    )annotationsN)reduce   )DistributedConfig)is_torch_greater_or_equallogging)GeneralInterface)is_torch_available)nnc                   || t        d      | |t        d      |1t        d      st        d      t        j                  j                         j                  }|dk(  rd}t        t        |      }t        j                  j                         s	 t        t        j                  d         }t        t        j                  d	         }t        t        j                  d
         }ddddd}	|	j                  |      }
t        j                  j                  |
||       t        t        |      }|dk7  r|j                  |       |dk7  r|j                  t        t        j                  d	                |j#                         }t        j$                  ||      }|}|dkD  r^ddl}t)        t        j*                  d      |_        t)        t        j*                  d      |_        nt        j$                  |      }|xs i }||nt        j                  j1                         }t        j                  j3                  |j                  |f      }n{|j4                  dkD  rd|j6                  vrt        d      |d   }|j9                         }t        j$                  |j:                   dt        t        j                  d	                }|||fS # t         $ r}t        d      |d}~ww xY w)z
    Sets up the device mesh and initialized the backend for tensor parallelism.
    This function is called when the model is loaded and the TP plan is set to 'auto'.
    Nz-tp_plan has to be set when tp_size is passed.zY`tp_plan` and `device_map` are mutually exclusive. Choose either one for parallelization.z2.5z3Tensor parallel is only supported for `torch>=2.5`.mpscpuRANK
LOCAL_RANK
WORLD_SIZEncclglooxcclhccl)cudar   xpuhpu)backendrank
world_sizezWe tried to initialize torch.distributed for you, but it failed. Make sure you init torch distributed in your script to use `tp_plan`.r   w   tpzsWhen using `tp_plan` and n-d `device_mesh`, it must contain a 'tp' dimension. Please provide a valid `device_mesh`.:)
ValueErrorr   OSErrortorch_C_get_acceleratortypegetattrdistributedis_initializedintosenvirongetinit_process_group
set_device	Exceptioncurrent_devicedevicesysopendevnullstdoutstderrget_world_sizeinit_device_meshndimmesh_dim_namessizedevice_type)tp_plantp_sizedevice_mesh
device_mapr<   r0   r   
local_rankr   backend_mapr   eindex	tp_devicer2   s                  [/opt/pipecat/venv/lib/python3.12/site-packages/transformers/integrations/tensor_parallel.pyinitialize_tensor_parallelismrG   (   s    wHIIz5tuu(/OPP hh//166%K 4  //12::f-. L!9:
 L!9:
'-fVTZ[%//+6!!44W4\f4g!(!<%'"--j9 %%%c"**\*B&CD"113E[%8I"Jqy!"**c2
!"**c2
 [1I$*J$0'e6G6G6V6V6X''88'Ta;555 <  &d+K""$\\[%<%<$=Qs2::lC[?\>]"^_
{G++G  W s   B4K 	K-K((K-c                2    t        j                  dd |       S )ag  
    Replace the numbers in the `name` by wildcards, only if they are in-between dots (`.`) or if they are between
    a dot (`.`) and the end of the string.
    This matches how modules are named/numbered when using a nn.ModuleList or nn.Sequential, but will NOT match
    numbers in a parameter name itself, e.g. if the param is named `"w1"` or `"w2"`.
    z\.\d+(\.|$)c                *    d| j                  d      z   S )Nz.*r   group)ms    rF   <lambda>z2replace_layer_number_by_wildcard.<locals>.<lambda>w   s    D1771:,=     )resub)names    rF    replace_layer_number_by_wildcardrR   p   s     66."=tDDrN   c                t    t        |       }||v r||   S |r"d|v r|j                  dd      d   x}|v r||   S y)a  
    Get the TP style for a parameter from the TP plan.

    The TP plan is a dictionary that maps parameter names to TP styles.
    The parameter name can be a generic name with wildcards (e.g. "*.weight") or a specific name (e.g. "layer_1.weight").

    The `is_weight` is important because for weights, we want to support `.weights` and `.bias` cases seamlessly! but
    not parent classes for `post_init` calls
    .r   r   N)rR   rsplit)parameter_namer=   	is_weightgeneric_param_namemodule_names        rF   _get_parameter_tp_planrZ   z   s_     :.IW$)**	s00EWE^E^_bdeEfghEi6iknu5u{##rN   )BOOLU8I8I16F16BF16I32F32F64I64F8_E4M3c                    t        |t              r8t        |      }| |z  dk(  sJ d|  d|        | |z  }|D cg c]  }||z  	 c}S | |z  dk(  s
J d|        | |z  }|g|z  S c c}w )a  
    Convert block count or proportions to block sizes.

    This function accepts

    - The number of blocks (int), in which case the block size is
      total_size//blocks; or
    - A list of block sizes (list[int]).

    In the second case, if sum(blocks) < total_size, the ratios between
    the block sizes will be preserved. For instance, if blocks is
    [2, 1, 1] and total_size is 1024, the returned block sizes are
    [512, 256, 256].
    r   zCannot split z in proportional blocks: zPrepacked is not divisible by )
isinstancelistsum)
total_sizeblockstotal_blocks	part_sizeblocksingle_sizes         rF   _blocks_to_block_sizesrp      s     &$6{L(A-lzlJcdjck/ll-,.	/56e	E!66F"a'R+I&)RR' F*}v%%	 7s   A&c                
   | }|j                   |   }|j                         }t        |d      }g }	d}
|D ].  }||z  }||z  }|dz   |z  }|	t        |
|z   |
|z         z  }	|
|z  }
0 |j	                         }d}|dk(  s|dk(  r$|d   j                  t        j                        }d	}|dk(  r||	df   }n6|dk(  s|d
k(  r|dd|	df   }n!|dk(  s|dk(  r|d|	f   }nt        d| d      |r|S |j                  t        |         S )u  
    When weights are packed (gate_up_proj), we need to make sure each shard gets its correct share.
    So if you have: gate_proj       ( 16, 5120, 8190)
    and             up_proj         ( 16, 5120, 8190)
    packed as       gate_up_proj    ( 16, 5120, 2 * 8190)
    And you shard along the last dimension, you need to interleave the gate and up values:

    Now, if we shard along the last dimension across TP_size (Tensor Parallelism size), we must interleave the values from gate and up projections correctly.

    Let's take TP_size = 4 for an example:

    Packed tensor `gate_up_proj`
    ---------------------------------------------------------------
    [ G0  G1  G2  G3 | G4  G5  G6  G7 | ... | U0  U1  U2  U3 | U4  U5  U6  U7 | ... ]
     ↑─────────────↑   ↑─────────────↑        ↑─────────────↑  ↑─────────────↑
       Gate Slice 0      Gate Slice 1            Up Slice 0       Up Slice 1

    Explanation:
    - The first half of the tensor (left of the center) holds the gate_proj values.
    - The second half (right of the center) holds the up_proj values.
    - For TP=4, we divide each half into 4 slices. In this example, we show two slices for brevity.
    - Each shard receives one slice from the gate part and the corresponding slice from the up part.

    For instance:
    • Shard 0 gets: [ Gate Slice 0, Up Slice 0 ] = [ G0, G1, G2, G3, U0, U1, U2, U3 ]
    • Shard 1 gets: [ Gate Slice 1, Up Slice 1 ] = [ G4, G5, G6, G7, U4, U5, U6, U7 ]
    • … and so on.

    This ensures that each shard receives an equal portion of both gate and up projections, maintaining consistency across tensor parallelism.
    r   )rj   rk   r   r   Fre   F8_E5M2.TNzUnsupported dim z", only dim 0, 1 or 2 are supported)
shaper;   rp   range	get_dtypetor"   float16r    str_to_dtype)paramempty_paramr?   r   dimslice_rj   r   block_sizestensors_slicesblock_offset
block_sizeshard_block_sizestartstopslice_dtypecastedtensors                     rF   get_packed_weightsr      sY   > F""3'J!!#J(JqIKNL! #
%3''q,,%u 4lT6IJJ
"# ""$K Fi;)#;.
ax+,	SBY>3./	SBY^+,+C50RSTTyyk233rN   c                   |dk7  rt        d      |dk\  r|n|| j                  z   }| j                  |   }||z  }||z  }| j                  d| }| j                  |dz   d }	 | j                  g |||||	 }
t	        |      }t	        |      dz   }t        t        |
j                              }||   ||   c||<   ||<    |
j                  | }|j                  |       }|S )as  
    Reorders a tensor that was reconstructed from sharded packed weights into its canonical packed format.

    For example, if a weight was packed (e.g., gate_proj and up_proj) and then sharded,
    DTensor.full_tensor() might produce an interleaved layout like [G0, U0, G1, U1, ...]
    along the sharded dimension. This function reorders it to [G0, G1, ..., U0, U1, ...].
    This is an inverse operation to get_packed_weights.

    Args:
        reconstructed_tensor: The tensor reconstructed from DTensor (e.g., via .full_tensor().contiguous()).
        sharded_dim: The dimension index in the reconstructed_tensor that was originally sharded.
        world_size: The tensor parallel world size.
        num_packed_projs: The number of projections that were packed together (e.g., 2 for gate_up_proj).

    Returns:
        The reordered tensor in canonical packed format.
    r   zNum blocks different from 2 is not supported yet. This is most likely a bug in your implementation as we only pack gate and up projections together.r   Nr   )	r    r9   ru   viewlenrh   rv   permute
reshape_as)packed_parametersharded_dimr   
num_blocksactual_sharded_dimtotal_size_on_sharded_dimoriginal_block_size_on_dimshard_chunk_sizeprefix_shapesuffix_shapetensor_viewaxis_ws_absaxis_npp_abspermute_ordertensor_permutedfinal_ordered_tensors                   rF   repack_weightsr      sO   0 Q c
 	
 )4q(8kL\LaLa>a 0 6 67I J!:j!H1Z?#))*=+=>L#))*<q*@*BCL'"'' 	 	 		
 
K l#K|$q(L{//01M>KL>Y[hit[u;M+l ;)k))=9O +556FGrN   c                J   |j                   }|j                  }t        t        j                  |      }t        | t        j                        rt        | j                        n| j                         }	|dk  r||z   }|j                         dk(  r|dk(  rt        |	      dk(  rd}n(|j                         dk(  r|dk(  rt        |	      dk(  rd}t        j                  |	|   |z        }
||
z  }t        ||
z   |	|         }||k\  rt        d| d|       ||k\  rt        d| d|       |_|j                         dk(  rL|dk(  rGt        |	      dk(  r9||cxk  r|k  rn n| d	d	 S t        j                   g t        j"                  |
      S t%        d	      gt        |	      z  }||	|   k  rAt%        ||      ||<   | t'        |         } t        | t              r| D cg c]  }|d	d	 	 } }| S d|	|<   t        j                   t'        |	      t        j"                        S c c}w )a
  
    Generalized tensor sharding across a multi-dimensional device mesh.
    Extract only the fraction of the parameter owned by the given `rank` when the parameter would have gone sharding at provided `dim`.
    Extraction follows the pytorch `Shard` placement so that sharding and materializing back to full tensor follows `Shard` semantics.
    `Shard` follows torch.chunk style sharding of the tensor. We demonstrate some cases below on how sharding happens including some edge cases
    such as some ranks having an empty tensor as shard. Below implementation is robut to all these cases.

    Case (1)
    empty_param                 (16, 5120, 8190)
    dim                         0
    device_mesh.size()          4
    rank 0 gets					(4, 5120, 8190)			 (0 ... 4, 5120, 8190)
    rank 1 gets					(4, 5120, 8190)			 (4 ... 8, 5120, 8190)
    rank 2 gets					(4, 5120, 8190)			 (8 ... 12, 5120, 8190)
    rank 3 gets					(4, 5120, 8190)			 (12 ... 16, 5120, 8190)

    Case (2)
    empty_param                 (16, 5120, 8190)
    dim                         0
    device_mesh.size()          14
    rank 0 gets					(2, 5120, 8190)			 (0 ... 2, 5120, 8190)
    rank 1 gets					(2, 5120, 8190)			 (2 ... 4, 5120, 8190)
    rank 2 gets					(2, 5120, 8190)			 (4 ... 6, 5120, 8190)
    rank 3 gets					(2, 5120, 8190)			 (6 ... 8, 5120, 8190)
    rank 4 gets					(2, 5120, 8190)			 (8 ... 10, 5120, 8190)
    rank 5 gets					(2, 5120, 8190)			 (10 ... 12, 5120, 8190)
    rank 6 gets					(2, 5120, 8190)			 (12 ... 14, 5120, 8190)
    rank 7 gets					(2, 5120, 8190)			 (14 ... 16, 5120, 8190)
    rank 8 gets					(0, 5120, 8190)
    rank 9 gets					(0, 5120, 8190)
    rank 10 gets			    (0, 5120, 8190)
    rank 11 gets				(0, 5120, 8190)
    rank 12 gets				(0, 5120, 8190)
    rank 13 gets				(0, 5120, 8190)

    Case (3)
    empty_param                 (16, 5120, 8190)
    dim                         0
    device_mesh.size()          3
    rank 0 gets					(6, 5120, 8190)			 (0 ... 6, 5120, 8190)
    rank 1 gets					(6, 5120, 8190)			 (6 ... 12, 5120, 8190)
    rank 2 gets					(4, 5120, 8190)			 (12 ... 16, 5120, 8190)

    In case (2), empty shards are returned with appropriate dimension to allow for operations to work smoothly.
    Args:
        param (torch.Tensor): The tensor to shard.
        empty_param (torch.Tensor): A tensor used for shape reference.
        device_mesh (torch.Tensor): Shape [d_0, ..., d_n] representing the mesh.
        rank (int): Global rank of the current process/device.
        dim (int): Dimension along which to shard the tensor.
    r      r   r   zdim z* is out of bounds for tensor of dimension zRank z  is out of bounds for mesh size Ndtyper1   )r   )r9   ru   r   operatormulrg   r"   Tensorrh   	get_shaper}   r   mathceilminr    emptyint64slicetuple)r{   r|   r?   r   r}   
tensor_idx	param_dim
mesh_shaper   param_shape
shard_sizer   endslice_indicesps                  rF   get_tensor_shardr   @  s   h   I""Jj1J'1%'F$u{{#EOOL]K
Qw#oA#(s;/?1/D		a	C1H[1AQ1F;s+j89J:E
ej +c"2
3C
i4u$NykZ[[z5&FzlSTT +//"3q"8SAX#kJZ^_J_J$$8O;;rTBB4[MC$44M{3"5#.ceM*+eT"#()aQqT)E)K;;u[)==	 *s   H c                2    t        j                  | |d      S )z9Split tensor along last dimension into world_size chunks.rt   r}   )r"   chunk)xr   s     rF   _split_along_last_dimr     s    ;;q*"--rN   c                  0    e Zd ZdZed        Zed        Zy)_AllReduceBackwardzRIdentity forward, all-reduce backward. Used before colwise layers (f in Megatron).c                    || _         |S N)r?   ctxr   r?   s      rF   forwardz_AllReduceBackward.forward  s    %rN   c                    | j                   }|j                         dk(  r|d fS |j                         }t        j                  |t        j
                  j                  |j                                |d fS Nr   oprK   )r?   r;   
contiguousdist
all_reduceReduceOpSUM	get_group)r   grad_outputr?   s      rF   backwardz_AllReduceBackward.backward  sc    oo"$$!,,.(9(9AVAVAXYD  rN   N__name__
__module____qualname____doc__staticmethodr   r    rN   rF   r   r     s+    \  ! !rN   r   c                  0    e Zd ZdZed        Zed        Zy)_AllReduceForwardzQAll-reduce forward, identity backward. Used after rowwise layers (g in Megatron).c                    |j                         dk(  r|S t        j                  |t        j                  j                  |j                                |S r   )r;   r   r   r   r   r   r   s      rF   r   z_AllReduceForward.forward  s@    "Hdmm//{7L7L7NOrN   c                
    |d fS r   r   )r   r   s     rF   r   z_AllReduceForward.backward  s    D  rN   Nr   r   rN   rF   r   r     s+    [  ! !rN   r   c                  0    e Zd ZdZed        Zed        Zy)
_AllGatherz<All-gather forward, split backward. Gathers sharded outputs.c                   || _         |j                         }|dk(  r|S |j                         dz
  }|j                         }|j	                         }|j                         }t        |      D cg c]  }t        j                  |       }}|||<   t        j                  |||       t        j                  ||      j                         S c c}w Nr   rJ   r   r?   r;   r}   get_local_rankr   r   rv   r"   
empty_liker   
all_gathercat)	r   r   r?   r   last_dimr   rK   _tensor_lists	            rF   r   z_AllGather.forward  s    % %%'
?H557Q;))+%%'LLN49*4EFqu''*FFDQe4yy(3>>@@ Gs   /Cc                    | j                   }|j                         }|dk(  r|d fS |j                         }t        ||      }||   j	                         d fS Nr   r?   r;   r   r   r   )r   r   r?   r   r   chunkss         rF   r   z_AllGather.backward  s]    oo %%'
?$$))+&{J?d|&&($..rN   Nr   r   rN   rF   r   r     s-    FA A" 	/ 	/rN   r   c                  0    e Zd ZdZed        Zed        Zy)_Splitz>Split forward, all-gather backward. Scatters replicated input.c                    || _         |j                         }|dk(  r|S |j                         }t        ||      }||   j	                         S r   r   )r   r   r?   r   r   r   s         rF   r   z_Split.forward  sP    % %%'
?H))+&q*5d|&&((rN   c                   | j                   }|j                         }|dk(  r|d fS |j                         dz
  }|j                         }|j	                         }|j                         }t        |      D cg c]  }t        j                  |       }}|||<   t        j                  |||       t        j                  ||      j                         d fS c c}w r   r   	r   r   r?   r   r   r   rK   r   r   s	            rF   r   z_Split.backward      oo %%'
?$$??$q())+%%'!,,.>CJ>OPu''4PP'D[>yy(3>>@$FF Q   6CNr   r   rN   rF   r   r     s-    H	) 	) G GrN   r   c                  0    e Zd ZdZed        Zed        Zy)_ReduceScatterzCReduce-scatter forward, all-gather backward. For sequence parallel.c                   || _         |j                         }|dk(  r|S |j                         dz
  }|j                         }t	        |j                  ||            }t	        |j                        }||xx   |z  cc<   t        j                  ||j                  |j                        }t        j                  ||t        j                  j                  |       |S )Nr   r   r   r   )r?   r;   r}   r   rh   r   ru   r"   r   r   r1   r   reduce_scatterr   r   )	r   r   r?   r   r   rK   input_chunksoutput_shapeoutputs	            rF   r   z_ReduceScatter.forward*  s    % %%'
?H557Q;%%'AGGJHG=>AGG}X:-\JFLT]]5F5FeTrN   c                   | j                   }|j                         }|dk(  r|d fS |j                         dz
  }|j                         }|j	                         }|j                         }t        |      D cg c]  }t        j                  |       }}|||<   t        j                  |||       t        j                  ||      j                         d fS c c}w r   r   r   s	            rF   r   z_ReduceScatter.backward=  r   r   Nr   r   rN   rF   r   r   '  s-    M $ G GrN   r   c                .    t         j                  | |      S )zAIdentity forward, all-reduce backward. Use before colwise layers.)r   applyr   r?   s     rF   all_reduce_backwardr   U  s    ##A{33rN   c                .    t         j                  | |      S )z@All-reduce forward, identity backward. Use after rowwise layers.)r   r   r   s     rF   all_reduce_forwardr   Z  s    ""1k22rN   c                .    t         j                  | |      S )z#All-gather forward, split backward.)r   r   r   s     rF   r   r   _  s    A{++rN   c                .    t         j                  | |      S )z#Split forward, all-gather backward.)r   r   r   s     rF   splitr  d  s    <<;''rN   c                .    t         j                  | |      S )z,Reduce-scatter forward, all-gather backward.)r   r   r   s     rF   r   r   i  s    ;//rN   c                h    | j                  fd       | j                  fd       | S )z
    Copy pasted from torch's function but we remove the communications (partitioning)
    as well as buffer registering that is similarly not efficient.
    c                     | |      S r   r   )modinputsr?   input_fns     rF   rM   z#distribute_module.<locals>.<lambda>y  s    Xc6S^=_ rN   c                     | |      S r   r   )r  r  outputsr?   	output_fns      rF   rM   z#distribute_module.<locals>.<lambda>{  s    )CQXZeBf rN   )register_forward_pre_hookregister_forward_hook)moduler?   r  r
  s    ```rF   distribute_moduler  n  s4     (()_`$$%fgMrN   c                  j    e Zd ZdZdZdZdZd	dZed        Z	ed        Z
	 d		 	 	 	 	 d
dZddZddZy)TensorParallelLayerz.General tensor parallel layer for transformersNc                .    || _         || _        || _        y r   )r   r?   r|   )selfr?   r   r|   s       rF   __init__zTensorParallelLayer.__init__  s    	&&rN   c                     y r   r   r  r  r?   s      rF   _prepare_input_fnz%TensorParallelLayer._prepare_input_fn  s    58rN   c                     y r   r   r  r	  r?   s      rF   _prepare_output_fnz&TensorParallelLayer._prepare_output_fn  s    7:rN   c                    t         r   )NotImplementedErrorr  r{   r   r1   r   s        rF   shard_tensorz TensorParallelLayer.shard_tensor  s
     "!rN   c                H    t        ||| j                  | j                         y r   )r  r  r  r  r  r?   kwargss       rF   prepare_module_tpz%TensorParallelLayer.prepare_module_tp  s"    ""##		
rN   c                    t        |      S )z
        Compute the expected shape after TP sharding for a given full shape.

        Args:
            full_shape: The full (unsharded) parameter shape

        Returns:
            The expected sharded shape for this rank
        )r   )r  
full_shapes     rF   get_expected_sharded_shapez.TensorParallelLayer.get_expected_sharded_shape  s     Z  rN   NNNr{   torch.Tensorr   
int | Nonereturnr'  r  	nn.Moduler)  r+  r#  ztuple[int, ...] | torch.Sizer)  ztuple[int, ...])r   r   r   r   r?   r   r|   r  r   r  r  r  r!  r$  r   rN   rF   r  r    se    8KDK'
 8 8: : VZ"!"/9"	"

!rN   r  c                  P     e Zd ZdZdd fdZd Zd Z	 d		 	 	 	 	 d
dZddZ xZ	S )ColwiseParallelz
    Column-wise parallel: weight is sharded on dim -2 (output features).
    Forward: input replicated -> output sharded on last dim.
    If gather_output=True, output is all-gathered to produce full tensor.
    c                2    t        |   di | || _        y Nr   )superr  gather_output)r  r2  r   	__class__s      rF   r  zColwiseParallel.__init__  s    "6"*rN   c                ,    |r|d   n|}t        ||      S Nr   r   r  r  r  r?   input_tensors        rF   r  z!ColwiseParallel._prepare_input_fn  s    $*vay"<==rN   c                6    | j                   rt        ||      S |S r   )r2  r   r  r  r	  r?   s       rF   r  z"ColwiseParallel._prepare_output_fn  s    g{33rN   c                n   t        |t        j                        r|j                         nt	        |j                               }|dk(  r.t        || j                  | j                  | j                  d      }n-t        || j                  | j                  | j                  d      }|j                  ||      S Nr   rt   rs   r1   r   rg   r"   r   r}   r   r   r   r|   r?   r   rx   r  r{   r   r1   r   r}   	parameters          rF   r  zColwiseParallel.shard_tensor  s     (u||<eiik#eooFWBX!8(0@0@$BRBRTXT]T]_abI(0@0@$BRBRTXT]T]_abI||6|77rN   c                8   | j                   j                         }t        |      }t        |      dk(  rdnd}|dk  rt        |      |z   n|}t	        j
                  ||   |z        }| j                  |z  }t        ||z   ||         }||z
  ||<   t        |      S )Nr   rt   rs   r   )	r?   r;   rh   r   r   r   r   r   r   r  r#  r   ru   r}   r   r   r   s           rF   r$  z*ColwiseParallel.get_expected_sharded_shape  s    %%**,
Z J!Ob"%'c%j3sYYuSzJ67
		J&%*$eCj15[c
U|rN   F)r2  boolr%  r&  r,  
r   r   r   r   r  r  r  r  r$  __classcell__r3  s   @rF   r.  r.    s@    +> VZ	8!	8/9	8		8
rN   r.  c                  >    e Zd ZdZed        Zed        ZddZd Zy)ReplicatedWithGradAllReduceaM  
    Replicated parameter with gradient all-reduce.

    For parameters like q_norm/k_norm that sit between colwise and rowwise
    layers. The parameter is replicated (not sharded), but its gradient
    accumulates from local heads only in TP mode. This class registers a
    backward hook to all-reduce the parameter gradient.
    c                    |S r   r   r  s      rF   r  z-ReplicatedWithGradAllReduce._prepare_input_fn  s    rN   c                    |S r   r   r  s      rF   r  z.ReplicatedWithGradAllReduce._prepare_output_fn  s    rN   Nc                .    |d   j                  ||      S N.r=  rx   r  s        rF   r  z(ReplicatedWithGradAllReduce.shard_tensor      Sz}}F%}88rN   c                0    |fd}|j                  |       y )Nc                t    | j                         D ]%  }|j                  t        |j                  |       ' y r   )
parametersgradr   )r  
grad_inputr   meshr{   s        rF   _backward_hookzEReplicatedWithGradAllReduce.prepare_module_tp.<locals>._backward_hook  s0    ) 9::)&uzz489rN   )register_full_backward_hook)r  r  r?   r   rV  s        rF   r!  z-ReplicatedWithGradAllReduce.prepare_module_tp  s     ?J 	9
 	**>:rN   r%  )	r   r   r   r   r   r  r  r  r!  r   rN   rF   rI  rI    s9        9;rN   rI  c                  &    e Zd ZdZd ZddZddZy)MlaKvAProjParallela#  
    For MLA attention used in DeepSeek-V2 style models (deepseek_v2, longcat_flash, glm_moe_dsa, glm4_moe_lite):
    kv_a_proj_with_mqa output is [kv_lora_rank + qk_rope_head_dim] (can have different naming but important thing
    to understand is that it is split)
    Example below (from modeling_longcat_flash.py):

    kv_a_proj_with_mqa
            |
            split
            /            k_pass    k_rot  <-- "bypasses kv_b_proj"
        |          |        (goes straight to attention,
    kv_a_layernorm |         never touches kv_b_proj)
        |          |
    kv_b_proj      |
    (colwise)      |
        |          |
        k_pass     k_rot
            \      /
               cat
                |
            key_states

    k_pass is passed to kv_b_proj (colwise) which has built-in all_reduce_backward so we don't have a partial gradient for it.
    However, k_rot goes straight to attention, never touches kv_b_proj. So we need to average gradient across all ranks otherwise we only get gradient for one rank (partial gradient).
    c                8   t        |j                  d      s"t        dt        |      j                   d      |j                  j
                  }|j                  |j                  d   |z
  |gd      \  }}t        ||      }t        j                  ||gd      S )Nqk_rope_head_dimzConfig for z does not have `qk_rope_head_dim`. MlaKvAProjParallel requires `qk_rope_head_dim` to be defined in the model config. Please add it to the model's config or update the TP plan mapping.rt   r   )hasattrconfigAttributeErrorr%   r   r[  r  ru   r   r"   r   )r  r  r   r?   rope_dimpass_outputrope_outputs          rF   r  z%MlaKvAProjParallel._prepare_output_fn  s    szz#56 d3i001 2U U 
 ::..#)<<b1AH1Lh0W]_<#` [)+{Cyy+{3<<rN   Nc                .    |d   j                  ||      S rM  rN  r  s        rF   r  zMlaKvAProjParallel.shard_tensor   rO  rN   c                B    ||_         t        ||| j                         y )N)r
  )r]  r  r  )r  r  r?   r]  r   s        rF   r!  z$MlaKvAProjParallel.prepare_module_tp#  s    &+9P9PQrN   r%  r   )r   r   r   r   r  r  r!  r   rN   rF   rY  rY    s    6
=9RrN   rY  c                  P     e Zd ZdZdd fdZd Zd Z	 d		 	 	 	 	 d
dZddZ xZ	S )RowwiseParallela  
    Row-wise parallel: weight is sharded on dim -1 (input features).
    Forward: input (optionally split) -> output partial -> all-reduce to replicate.

    Args:
        split_input: If True, splits replicated input before matmul. Use when input
                     comes from a non-parallelizable operation (chunk/slice).
                     Default False (expects pre-sharded input from colwise layer).
    c                2    t        |   di | || _        y r0  )r1  r  split_input)r  rg  r   r3  s      rF   r  zRowwiseParallel.__init__3  s    "6"&rN   c                    t        |d      r$|j                  |j                  |_        d |_        |r|d   n|}| j                  rt	        ||      S |S )Nbiasr   )r\  ri  _biasrg  r  r7  s        rF   r  z!RowwiseParallel._prepare_input_fn7  sP    3CHH$8CICH$*vay{33rN   c                l    t        ||      }t        |d      r|j                  ||j                  z   }|S )Nrj  )r   r\  rj  r:  s       rF   r  z"RowwiseParallel._prepare_output_fnC  s4    $Wk:3 SYY%:		)GrN   c                   t        |t        j                        r|j                         nt	        |j                               }|dk(  r|d   }n-t        || j                  | j                  | j                  d      }|j                  ||      S )Nr   .rt   r=  r>  r?  s          rF   r  zRowwiseParallel.shard_tensorI  ss     (u||<eiik#eooFWBX!8c
I(0@0@$BRBRTXT]T]_abI||6|77rN   c                J   t        |      dk(  rt        |      S | j                  j                         }t	        |      }d}|dk  rt        |      |z   n|}t        j                  ||   |z        }| j                  |z  }t        ||z   ||         }||z
  ||<   t        |      S Nr   rt   r   )	r   r   r?   r;   rh   r   r   r   r   rB  s           rF   r$  z*RowwiseParallel.get_expected_sharded_shapeT  s    z?a$$%%**,
Z "%'c%j3sYYuSzJ67
		J&%*$eCj15[c
U|rN   rC  )rg  rD  r%  r&  r,  rE  rG  s   @rF   re  re  (  s@    '
 VZ	8!	8/9	8		8rN   re  c                  &    e Zd ZdZ	 d	 	 	 	 	 ddZy)PackedColwiseParallelz@Packed column-wise parallel for fused weights like gate_up_proj.Nc                0   t        |t        j                        r|j                         nt	        |j                               }|dk(  r.t        || j                  | j                  | j                  d      }n| j                  | j                  j                        }|t	        |      k  r.t        || j                  | j                  | j                  d      }n-t        || j                  | j                  | j                  d      }|j                  ||      S r<  )rg   r"   r   r}   r   r   r   r|   r?   r   r$  ru   r   rx   )r  r{   r   r1   r   r}   r@  expected_shapes           rF   r  z"PackedColwiseParallel.shard_tensorf  s     (u||<eiik#eooFWBX!8(0@0@$BRBRTXT]T]_abI!<<T=M=M=S=STNS(( -UD4D4DdFVFVX\XaXacef	 /ud6F6FHXHXZ^ZcZcegh	||6|77rN   r%  r&  r   r   r   r   r  r   rN   rF   rp  rp  c  s)    J VZ8!8/98	8rN   rp  c                  &    e Zd ZdZ	 d	 	 	 	 	 ddZy)PackedRowwiseParallelz=Packed row-wise parallel for fused weights like gate_up_proj.Nc                   t        |t        j                        r|j                         nt	        |j                               }|dk(  r|d   }nt        |t        j                        r|j                  n|j                         }| j                  j                         dk\  r| j                  j                  d   nd}t	        |      dk\  r|d   nd}	|	|k  r.t        || j                  | j                  | j                  d      }n-t        || j                  | j                  | j                  d      }|j                  ||      S )Nr   .rt   r   r=  )rg   r"   r   r}   r   r   ru   r|   r   r?   r   r   rx   )
r  r{   r   r1   r   r}   r@  r   expected_packed_dim
actual_dims
             rF   r  z"PackedRowwiseParallel.shard_tensor|  s    (u||<eiik#eooFWBX!8c
I *4E5<<)H%++eooN_K@D@P@P@T@T@VZ[@[$"2"2"8"8"<ab,/,<,ARqJ//,UD4D4DdFVFVX\XaXacef	 /ud6F6FHXHXZ^ZcZcegh	||6|77rN   r%  r&  rs  r   rN   rF   ru  ru  y  s)    G VZ8!8/98	8rN   ru  c                  T     e Zd ZdZddd	 fdZd Zd Z	 d
	 	 	 	 	 ddZddZ xZ	S )EmbeddingParallelzXEmbeddingParallel: shards embedding table, handles masked lookups for vocab parallelism.r   embedding_dim_shardingc               2    t        |   di | || _        y r0  )r1  r  r|  )r  r|  r   r3  s      rF   r  zEmbeddingParallel.__init__  s    "6"&<#rN   c                    |r|d   n|}| j                   dk(  r_|j                         }|j                  j                  d   }||z  }||z   }||k  ||k\  z  }	|	|_        |j                         |z
  }
d|
|	<   |
S |S r5  )r|  r   weightru   _input_maskclone)r  r  r  r?   r8  r   per_partition_sizevocab_start_indexvocab_end_index
input_maskmasked_inputs              rF   r  z#EmbeddingParallel._prepare_input_fn  s    $*vay &&!+--/D
 "%!1!1!!4 $'9 9/2DDO ')::|?^_J(CO (--/2CCL'(L$rN   c                    | j                   dk(  rNt        |d      rB|j                  }|j                  d      j	                  |      }|| j                         z  }|`t        ||      S )Nr   r  rt   )r|  r\  r  	unsqueeze	expand_asfloatr   )r  r  r	  r?   r  mask_expandeds         rF   r  z$EmbeddingParallel._prepare_output_fn  sd    &&!+]0KJ&004>>wGM- 6 6 88G!';77rN   c                   t        |t        j                        r|j                         nt	        |j                               }|dk(  r.t        || j                  | j                  | j                  d      }n7t        || j                  | j                  | j                  | j                        }|j                  ||      S )Nr   rt   r=  )rg   r"   r   r}   r   r   r   r|   r?   r   r|  rx   r?  s          rF   r  zEmbeddingParallel.shard_tensor  s     (u||<eiik#eooFWBX!8(0@0@$BRBRTXT]T]_abI(    		++I ||6|77rN   c                L   | j                   j                         }t        |      }t        |      dk(  rdn| j                  }|dk  rt        |      |z   n|}t        j                  ||   |z        }| j                  |z  }t        ||z   ||         }||z
  ||<   t        |      S rn  )
r?   r;   rh   r   r|  r   r   r   r   r   rB  s           rF   r$  z,EmbeddingParallel.get_expected_sharded_shape  s    %%**,
Z  J!Ob)D)D"%'c%j3sYYuSzJ67
		J&%*$eCj15[c
U|rN   )r|  r)   r%  r&  r,  rE  rG  s   @rF   rz  rz    sA    b89 =4	8 VZ8!8/98	8"rN   rz  c                  H     e Zd ZdZdd fdZd Zd Z	 d	 	 	 	 	 d	dZ xZS )
SequenceParallelzd
    Sequence Parallel: input/output sharded on sequence dimension.
    Weights are replicated.
    c                2    t        |   di | || _        y r0  )r1  r  sequence_dim)r  r  use_local_outputuse_dtensorr   r3  s        rF   r  zSequenceParallel.__init__  s    "6"(rN   c                ,    |r|d   n|}t        ||      S r5  )r   r7  s        rF   r  z"SequenceParallel._prepare_input_fn  s    $*vay ,44rN   c                    t        ||      S r   )r   r:  s       rF   r  z#SequenceParallel._prepare_output_fn  s    g{33rN   c                .    |d   j                  ||      S rM  rN  r  s        rF   r  zSequenceParallel.shard_tensor       Sz}}F%}88rN   )r   FF)r  r)   r  rD  r%  r&  )	r   r   r   r   r  r  r  r  rF  rG  s   @rF   r  r    s;    
)54 VZ9!9/99	9rN   r  c                  @     e Zd ZdZ fdZ	 d	 	 	 	 	 ddZddZ xZS )GroupedGemmParallelzb
    Applies Expert Parallelism to MoE experts by loading the correct experts on each device.
    c                $    t        |   di | y r0  r1  r  r  r   r3  s     rF   r  zGroupedGemmParallel.__init__      "6"rN   c                   | j                   j                  d   }|| j                  j                         z  dk7  r*t	        d| d| j                  j                          d      || j                  j                         z  }|}t        |t        j                        r|j                  |j                  nd}||z  }|dz   |z  }	t        |t        j                        s|j                         n|j                  }
|#||cxk  r|	k  rn n|d d  j                  |      S ||||	 j                  ||      S t        |
      dk\  r|y |d d  j                  ||      S )Nr   zAGlobal number of experts must be divisible by number of devices:  %  != 0r   )r1   r=  )r|   ru   r?   r;   r    rg   r"   r1   rD   r   r   rx   r   )r  r{   r   r1   r   global_num_expertslocal_num_expertsr   r   r   ru   s              rF   r  z GroupedGemmParallel.shard_tensor  sp    "--33A6 0 0 5 5 771<STfSggjkok{k{  lA  lA  lC  kD  DI  J  /$2B2B2G2G2II&
fell+%+\\%=V\\1F#zZ')3E5<<)H!ekk!ez&?C&?8;;f;--s#&&fE&BBZ1_!78;;fE;::rN   c                |    | j                   j                         }t        |      }|d   |z  }||d<   t        |      S r5  )r?   r;   rh   r   )r  r#  r   ru   r  s        rF   r$  z.GroupedGemmParallel.get_expected_sharded_shape  sB    %%**,
Z !!H
2$aU|rN   r%  r&  r,  )r   r   r   r   r  r  r$  rF  rG  s   @rF   r  r    s6    # VZ;!;/9;	;4rN   r  c                  X     e Zd ZdZ fdZed        Zed        Z	 d	 	 	 	 	 ddZ xZ	S )RouterParallelzQ
    Allows to reshape the router scores to support running expert parallel.
    c                $    t        |   di | y r0  r  r  s     rF   r  zRouterParallel.__init__'  r  rN   c                    |r|d   S |S r5  r   r  s      rF   r  z RouterParallel._prepare_input_fn*  s    "vay..rN   c                (   |j                         |j                         }}| j                  |z  dk7  rt        d| j                   d| d      | j                  |z  }|\  }}}t	        j
                  |      j                  d||      }|dd||z  |dz   |z  f   }|j                  ||z  |k7  d      }|dkD  rt	        j                  ||      }n(|j                  |dkD  d      j                  |dk  d      }|j                  |dk(  |      }|||fS )a  
        Imagine if you had 4 tokens, top_k = 4, and 128experts.
        With EP = 8. The num_local_expert should be 128/8 = 16
        Imagine router_indices being:
        [ 52,  42, 119,  67],
        [102,  89,  61,  40],
        [ 82, 103,   4,  34],
        [ 93,  23, 109,  11],

        then you can map which rank should be getting which values

        [3, 2, 7, 4],
        [6, 5, 3, 2],
        [5, 6, 0, 2],
        [5, 1, 6, 0],

        Thus for say rank 0, you fill with 16 (num_local_expert) the index tensor

        [ 16, 16, 16, 16],
        [ 16, 16, 16, 16],
        [ 16, 16, 4, 16],
        [ 16, 16, 16, 11],

        This works well. For another rank you need to make sure you round to num_local_expert
        because the next operation will one hot encode the router index vector.

        This allows us to know directly which local expert is hit.
        Similarly the scores are indexed with something created form
        router_indices.

        The kinda naive training loop that we use for device_map "auto" uses a similar logic.
        Here we are just making each rank believe that he is alone, and he computes his part of the hiddenstates.
        Mask invalid indices with num_local_expert for one-hot encoding, so the computes will skip the masking index.
        r   z>The number of experts must be divisible by number of ep_size: r  r  r   Nrt   )	r   r;   num_expertsr    r"   
zeros_likescatter_masked_fillfmod)	r  r	  r?   ep_rankep_sizenum_local_expertsrouter_logitsrouter_scoresrouter_indicess	            rF   r  z!RouterParallel._prepare_output_fn.  sG   H '5579I9I9K??W$)PQTQ`Q`Paadeldmmrs   OOw67>4}n((7@@NTab%a3D)DRSWhGh)h&hi'33^GX5X]d4dfhiq "ZZ8IJN+778JANZZ[ilm[moqrN'33Nb4HJ[\m^;;rN   c                .    |d   j                  ||      S rM  rN  r  s        rF   r  zRouterParallel.shard_tensord  r  rN   r%  r&  
r   r   r   r   r  r   r  r  r  rF  rG  s   @rF   r  r  "  sZ    # / / 3< 3<l VZ9!9/99	9rN   r  c                  X     e Zd ZdZ fdZed        Zed        Z	 d	 	 	 	 	 ddZ xZ	S )MoeTensorParalellExpertsa7  
    Note: For tensor parallel, the MoEExpertsParallel TP layer handles gradient sync:
        - all_reduce_backward on hidden_states (for colwise gate_up_proj gradient)
        - all_reduce_backward on top_k_weights (for router gradient)
        - all_reduce_forward on output (for partial expert outputs)
    c                $    t        |   di | y r0  r  r  s     rF   r  z!MoeTensorParalellExperts.__init__r  r  rN   c                Z    |d   }|d   }|d   }t        ||      }t        ||      }|||fS )Nr   r   r   r6  )r  r  r?   hidden_statestop_k_indextop_k_weightss         rF   r  z*MoeTensorParalellExperts._prepare_input_fnu  sH     q	Qiq	 ,M;G
 ,M;G{M::rN   c                    t        ||      S r   )r   r  s      rF   r  z+MoeTensorParalellExperts._prepare_output_fn  s     "';77rN   c                .    |d   j                  ||      S rM  rN  r  s        rF   r  z%MoeTensorParalellExperts.shard_tensor  s    
 Sz}}F%}88rN   r%  r&  r  rG  s   @rF   r  r  j  sY    # ; ;  8 8
 VZ9!9/99	9rN   r  c                  .    e Zd ZdZed        ZddZd Zy)MoeIdentityExpertParallelaM  
    TP class for zero/identity experts in MoE layers.

    Under TP, the parent MoeTensorParalellExperts does all_reduce_forward (sum)
    on the expert module output. Identity experts produce the same output on
    every rank, so the sum gives world_size * output. This class divides the
    input by world_size to compensate.
    c                :    |r|d   n|}||j                         z  S r5  )r;   )r  r  r?   r8  s       rF   r  z+MoeIdentityExpertParallel._prepare_input_fn  s#    $*vayk..000rN   Nc                .    |d   j                  ||      S rM  rN  r  s        rF   r  z&MoeIdentityExpertParallel.shard_tensor  rO  rN   c                4    t        ||| j                         y )N)r  )r  r  r  s       rF   r!  z+MoeIdentityExpertParallel.prepare_module_tp  s    &+8N8NOrN   r%  )r   r   r   r   r   r  r  r!  r   rN   rF   r  r    s&     1 1
9PrN   r  c                  &   e Zd ZU  e       r_er] ed       ed       e        e        ed       e        e	        e
        e        e        e        e        e        e       dni Zdddddddd	d	d	d

Zded<   dddd	d	d	d	d	d	d	d

Zded<   y	)ParallelInterfacer   r{  T)r2  )rg  )embedding_rowwisecolwise_gather_outputcolwiserowwiserowwise_split_inputpacked_colwisepacked_rowwisesequence_parallelgrouped_gemm	ep_routermoe_tp_expertsmoe_identity_expertreplicated_with_grad_allreducemla_kv_a_projrs   rt   N)
r  r  r  r  r  r  r  r  r  r  zdict[str, int | None]plan_to_weight_dimplan_to_bias_dim)r   r   r   r
   _torch_distributed_availablerz  r.  re  rp  ru  r  r  r  r  r  rI  rY  _global_mappingr  __annotations__r  r   rN   rF   r  r    s    ( $@ "3!!L%44%H&(&(#2t#D3535!1!3/1')68#<#>.I.K/1	
" % 2 !#!!*.1-  !##!!*./+ rN   r  ALL_PARALLEL_STYLESc                n   |j                         }d|j                  xs i v r|j                  d      nd}|dk  r| j                  |z   }t	        |      D cg c]  }t        j                  |        }}t        j                  || j                         |       t        j                  ||      S c c}w )a~  
    All-gather a sharded tensor along the specified dimension to reconstruct the full tensor.

    Args:
        local_tensor: The local shard of the tensor on this rank
        shard_dim: The dimension along which the tensor was sharded
        device_mesh: The device mesh for distributed communication

    Returns:
        The full reconstructed tensor (same on all ranks)
    r   Nr   rJ   r   )r;   r:   r   r9   rv   r"   r   r   r   r   r   )local_tensor	shard_dimr?   r   process_groupr   gathered_tensorss          rF   gather_full_tensorr    s     !!#J37K<V<V<\Z\3]K))$/cgM 1} %%	1	 AFj@QR1((6RROO$l&=&=&?}U 99%955	 Ss   B2c                   t         j                  }t         j                  }i }| j                         D ]  \  }}d|v r|j	                  dd      d   n|}	d|v r|j	                  dd      d   nd}
t        j                  dd|	      }t        j                  dd|      }d}||v r||   }n,||v r||   }n"d|v r|j	                  dd      d   }||v r||   }|||vr|||<   |
dk(  r|j                  |      }n|j                  |      }||||<   t        |||      }|dv rt        |||d	      }|j                         ||<    |S )
a*  
    Gather sharded tensors to reconstruct full tensors for saving.

    This function all-gathers each sharded tensor along its shard dimension
    to reconstruct the full unsharded tensor for checkpoint saving.

    Args:
        state_dict: The model state dict with local sharded tensors
        tp_plan: The tensor parallel plan mapping layer patterns to shard styles
        device_mesh: The device mesh for distributed communication
        tp_size: The tensor parallel world size

    Returns:
        State dict with full (gathered) tensors
    rT   r   r   N\d+*ri  )r  r  r   )r  r  r  itemsrU   rO   rP   r,   r  r   r   )
state_dictr=   r?   r>   r  r  resultkeyr   
param_name
param_typerX   generic_full_keycurrent_planparent_param_namer  full_tensors                    rF   gather_state_dict_for_saver    s   , -??*;;F!'') (/V.1SjSZZQ'*c
.1SjSZZQ'*d
VVFC<66&#s3 w&"#34L7*"#56L&& 2 9 9#q A! D G+&'89<7I#I F3K (,,\:I*..|<I F3K )KH??(i!LK!,,.sQ(/T MrN   c           	         Bt            }	 |j                  || j                         _        |_        fd_        yy# t        $ r}t	        d| d d|        Y d}~=d}~ww xY w)ai  
    This function is called in `PretrainedModel.post_init()`. It is responsible of adding hooks
    to the modules of the `model`, based on the `PretrainedModel._tp_plan`.

    This is the place where we add the `pre_forward` and `post_forwards` hooks. These are defined
    for each `TensorParallelLayer` as `_prepare_input_fn` and `_prepare_output_fn`.

    N)r]  Trying to prepare 0, but it's not supported. Corresponding module: z Fix it's TP plan: c                 .    j                          d  S )Nz

TP Plan: )__repr__)current_module_planr  s   rF   rM   z5add_tensor_parallel_hooks_to_module.<locals>.<lambda>d  s    V__%6$7{CVBW"X rN   )r  r!  r]  r  print_hf_tp_plan_hf_device_meshr  )	modelr  r=   
layer_namer  r?   rV   tp_layerrC   s	    `  `    rF   #add_tensor_parallel_hooks_to_moduler  N  s     &&':;	&&v{5<<&P 1!,X ' # 	$ZL0`ag`hh{|}{~ 	s   A 	A/A**A/c                   d|v r|j                  dd      n|\  }}	| j                  xs i }
| j                  |      }t        |      }t	        ||
      }t        j                         dk(  r7|t        j                  d| d       nt        j                  d| d|        |G	 t        |   }||_
        ||_        ||_        |j                  |d||      }|r|j                         }n|dd j#                  |      }t%        |t&        j(                  j*                        s/t&        j(                  j+                  ||j-                               }t/        ||	|       |S # t        $ r!}t!        d	| d
| d d|        Y d}~d}~ww xY w)a  
    This function is called in `from_pretrained` when loading a model's checkpoints.
    It receives the pointer to the parameter (or the parameter itself) and takes care of "sharding".
    All process run this function, so they just load the partition of the tensor that they require.

    Main uses cases:
    - column / rowise parallelism, you just shard all the weights of the layer (weight and bias)
    - packed layers: you slice the weights, then shard like above
    - custom operation:
        - you want to add an all-gather at the end of a local layer.
        - you want to have a layer that is isolated from the rest of the world (because torch.DTensor does not work well with `.view` for instance)

    rT   r   r   NzTensor sharding plan for z+ not found, using default 'replicate' plan.z: )r   r   r1   r  r  z" Fix it's TP plan, current layer: z : )requires_grad)rU   r=   get_submoduler)   rZ   r   get_rankloggerinfor  r|   r?   r   r  r   r  r  rx   rg   r"   r   	Parameteris_floating_pointsetattr)r  r{   r|   rV   param_casting_dtypeis_contiguousr   r?   r  r  r=   module_to_tpcurrent_shard_planr  rC   s                  rF   shard_and_distribute_moduler  g  s     ?B^>S^223:YgJ
mm!rG&&z2Lt9D/H}}!%KK3J<?jklKK3J<rBTAUVW%	*+=>H#.H #.H  HM))%DH[dh)iE((* a/0 eUXX//0""58U8U8W"XL*e,L # 	$^$44deqdr  sU  V^  U_  _b  cd  be  f 	s   %AE! !	F*FFc                x   |y| D ch c]  }t        |       }}t        |      }|j                         }|D ]  }d|v r|j                  dd      d   n|}t	        j
                  dd|      }||v r$|j                  |d       |j                  |       ]d|v sb|j                  dd      d   x}|v s||j                  |d       |j                  |        t        |      dkD  rt        j                  d|        t        |      dkD  r(t        j                  dd	j                  |              yyc c}w )
z
    Verify the TP plan of the model, log a warning if the layers that were not sharded and the rules that were not applied.
    NrT   r   r   r  r  z>The following TP rules were not applied on any of the layers: z'The following layers were not sharded: z, )rR   setcopyrU   rO   rP   popdiscardr   r  warningjoin)	expected_keysr=   r  generic_keysunsharded_layersunused_rulesr  rX   r  s	            rF   verify_tp_planr    sJ   
 ERSc4S9SLS<(<<>L 	*.1SjSZZQ'*c
VVFC<(/6$$S)&&ASAZAZ[^`aAbcdAe,e,=jq+q.5$$S)	* <1WXdWefg
q @K[A\@]^_ !# Ts   D7c           	        || _         || _        |6t        |t              rt	        j
                  |      }|| j                  _        t        |t              r|| _        | j                  }|t        r}|j                         D ]  }|t        vst        d| dt                | j                         D ]8  \  }}t        |dd      st        ||d      }	t!        | ||d|	|       d|_        : | S )	z,Distribute a model according to the TP plan.z"Unsupported tensor parallel style z. Supported styles are 
_is_hookedF)rV   r=   rW    )r  r  r=   r  r  r?   T)_tp_size_device_meshrg   dictr   	from_dictr]  distributed_configr=   r  valuesr  r    named_modulesr&   rZ   r  r  )
r  r=   r  r?   r>   
model_planvrQ   r  plans
             rF   distribute_modelr"    s   EN$E%($/!2!<!<=O!P*<''4 J">""$ 	wA++ #EaSH_`s_t!uvv	w "//1 	%LD&6<7-T:afg3!&!(, + !%F	% LrN   r%  )r=   zstr | dict[str, str] | Noner>   r(  )rQ   strr)  r#  )T)rV   r#  r=   dict[str, str]r)  z
str | None)rj   r)   rk   zint | list[int]r)  z	list[int])r   )
r   r'  r   r)   r   r)   r   r)   r)  r'  r   )r   r(  r*  )r  r'  r  r)   r?   zdist.device_mesh.DeviceMeshr)  r'  )r  dict[str, torch.Tensor]r=   r$  r>   r)   r)  r%  )r  z	list[str]r=   zdict[str, str] | None)Q
__future__r   r   r   r*   rO   	functoolsr   r'   r   utilsr   r   utils.genericr	   utils.import_utilsr
   r"   torch.distributedr   r   is_availabler  
get_loggerr   r  rG   rR   rZ   rD  uint8int8int16ry   bfloat16int32float32float64r   float8_e4m3fnrz   rp   r   r   r   r   autogradFunctionr   r   r   r   r   r   r   r   r  r   r  r  r.  rI  rY  re  rp  ru  rz  r  r  r  r  r  r  r  r  r  r  r  r  r  r"  r   rN   rF   <module>r8     s1   #   	 	  + 6 , 3 $ $)#4#4#A#A#C  
		H	% dhE,(E,3=E,PE. 

kkjj{{}}{{}}}}{{&&L&4A4P 	> "> >  >  	> 
 > Be>P.4!00 !$!// !/(( /DGU^^$$ GD&GU^^,, &G\4
3
,
(
0 	
 "*! *!Z)) )X;"5 ;@-R, -R`8) 8v8O 8,8O 84H+ HV9* 92(- (VE9( E9P&92 &9RP 3 P.4( 4n *;)< & <66+.6=X66<D'DD 	D
 DP Z^Y21h`:rN   