qi2dZddlmZddlZddlZddlmZddlmcmZ ddl m Z ddl mZmZmZddlmZddlmZdd lmZmZdd lmZmZdd lmZdd lmZdd lm Z m!Z!m"Z"m#Z#ddl$m%Z%ddl&m'Z'm(Z(ddl)m*Z*m+Z+ddl,m-Z-ddl.m/Z/ddl0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7ddl8m9Z9m:Z:m;Z;meTZUGd?d@eTZVGdAdBeTZWGdCdDeTZXGdEdFeTZYe!dGHGdIdJeTeZZgdKZ[y)Nz>> tokens = torch.tensor([[1, 2, 3], [4, 5, 6]]) >>> hashes = rolling_polynomial_hash(tokens, prime=31) >>> # hash[0] = 1*31^0 + 2*31^1 + 3*31^2 >>> # hash[1] = 4*31^0 + 5*31^1 + 6*31^2 dtypedevicer.dim)torchtensorint64r.arangeshapesum) token_tensorr* prime_tensorpowers prime_powerss U/opt/pipecat/venv/lib/python3.12/site-packages/transformers/models/blt/modular_blt.pyrolling_polynomial_hashr>:sa.<<U[[ATATUL \\,,,R09L9L MF'L 99\L0b 99 token_ids group_sizemax_hashcZtj5|j\}}tj||dz tj|j }tj ||gd}|jd|d}t||} | |z} ddd| S#1swY SxYw)z1Hash token groups and map to range [0, max_hash].r$r,r1N) r3no_gradr7zerosr5r.catunfoldr>) r@rAr*rB batch_sizeseq_lenpadding padded_tokenswindowshashes hash_valuess r=byte_group_hash_functionrOWs  ('oo G++j*q. T]TdTde 7I"6A>  &&q*a8(%8x'  (  ( s BB  B*local_encoder_tokensencoder_hash_tok_embedding$encoder_hash_byte_group_nb_functionsencoder_hash_byte_group_sizeencoder_hash_byte_group_vocabreturncgd}|j|}d}t|D]Y} || t|z} |D]A} t|| | |} | ||zz} ||| j |j z }|dz }C[|S)z=Compute token embeddings enhanced with hash-based embeddings.) ʚ;l21Ai oYlvtl.l}glAul 0lTlAK l|rr$) embed_tokensrangelenrOtor.)rP local_encoderrQrRrSrTprimes embeddings embedding_idxfunc_nbr*rAhash_idsoffset_hash_idss r=compute_hash_embeddingsrcis F++,@AJM=>wV,-6 J/0DjRWYvwH&9V)VVO 4_EHHIZIZ[ [J Q M   r?F patch_ids num_patchessequence_lengthpatches_as_queries cross_attn_kr-cz|j\}}|j}|rp||z} |} tj||j dj dj |||} |j dj |||} no|} ||z} |j dj |||} tj||j dj dj |||} | | k(} |rdnd}| j ||} || | f}| j|k7rtd| jd|| j d} d| j|z }|j|jtjtj|j} | S) aR Prepare cross-attention mask for patch-based attention, following mllama's robust approach. This function creates masks that control which patches can attend to which other patches, with support for query/key role swapping and cross-attention multipliers. Args: patch_ids (torch.Tensor): Tensor of shape [batch_size, seq_len] containing patch ids. num_patches (int): Total number of patches. sequence_length (int): Length of the sequence. patches_as_queries (bool): If True, patches are used as queries, otherwise as keys. cross_attn_k (int): Cross-attention multiplier for repeating patches. dtype (torch.dtype): Data type for the output mask. Returns: Tuple[torch.Tensor, torch.Tensor]: - cross_attention_mask: 4D tensor [batch_size, 1, q_len, kv_len] r0rr/r$r1zCross attention mask shape z doesn't match expected g?) r7r.r3r6 unsqueezeexpandrepeat_interleave ValueErrorr[ masked_fillboolfinfomin)rdrerfrgrhr-rHrIr.q_lenkv_len q_patch_ids kv_patch_idscross_attention_mask repeat_dimexpected_shapeinverted_cross_attn_masks r=#_prepare_patch_cross_attention_maskrzs4$//J   Fl*  LLV 4 Yq\ Yr] VJ W 5  !**1-44ZgV |+))"-44Z+V LLV 4 > >q A K KA N U UV`bikv w  ',6)bJ/AA,T^A_!%0N!!^3)*>*D*D)EE]^l]m n  099!< #%9%<%z(process_patch_lengths..s6&#f+6sr,r1r$)sizeitemdivmodextendappendmaxr3rEr-r. enumerater4rZanyr8r7nonzero) r{r|rH processedseqrlength full_chunks remaindermax_lenpaddedi last_nonzeros r=process_patch_lengthsrs##A&JI!#'l )F[[]F%+F4D%E "K MM+,{: ; i(  )  !6I66G [[*g.m6I6IR_RfRf gFy)t 6 ',||F-BUBU^k^r^r'sF1mF m# $t ! Q##% Q7! ((Q(/779==?DDFJ =L=() Mr?c eZdZy)BltMLPN__name__ __module__ __qualname__r?r=rr r?rc eZdZy) BltRMSNormNrrr?r=rr rr?rcDeZdZejedZy)BltRotaryEmbeddingc|jddddfjj|jddd}|dddddfj}t |j j tr/|j j dk7r|j j nd}t|d5|j|jzjdd}tj|dd }|j|jz}|j|jz} dddj|j  j|j fS#1swYAxYw) Nrr/r$mpscpuF) device_typeenabledrr1)r-)inv_freqfloatrkr7 isinstancer.typestrr transposer3rlcosattention_scalingsinr[r-) selfx position_idsinv_freq_expandedposition_ids_expandedrfreqsembrrs r=forwardzBltRotaryEmbedding.forwards@!MM$4-8>>@GG HZHZ[\H]_acde ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfk  U C 5&,,.1F1L1L1NNYYZ[]^_E))%;C'')d444C'')d444C  5 vvAGGv$cff177f&;;;  5 5s =BFF N)rrrr3rDrrrr?r=rrs$U]]_ < )&*<*<&BUBUV(263E3E6K^K^(_%r?)rrrintr __classcell__rs@r=rr"s`#``r?rc(eZdZdedeffd ZxZS)rrrc&t|||yr)rrrs r=rzBltSelfAttention.__init__-s +r?)rrrr%rrrrs@r=rr,s,y,S,,r?rc eZdZdZd dedededzffd Z d dejdejdzd ejdzd e e fd Z xZ S)BltCrossAttentionzdD]9}t ||d}|t|ds |jj,d}n|y|dz}dD]v}t ||d}|t|ds tj"|jd |d |zd |z t |ddXtj|jxt |dt |dd} | bt| drVtj"| jd |d |zd |z t | ddtj| jyt|t.s|dk(rt |jdd}|7t|jdr!t |jj0dd}|7t|jdr!t |jjdd}d} ||dz} t |dt |dd} t |dd} t |dt |dd} | | fD]}|t|ds| xs|jj,ddz}tj"|jd |d |zd |z t |ddktj|j| t| drt| jj,d}|dz}tj"| jd |d |zd |z t | ddtj| jyt|tj2rg|j4}|dz}tj"|jd |d |zd |z |jtj|jyt|t6r|j8dk7rt:|j8n |j<}||j\}}tj>|j@|tj>|jB|yy)a Initialize BLT weights following the original ByteLatentTransformer: - Most weights are drawn from a truncated normal. - Scale is ~ 1 / sqrt(model_dim) (or 1 / sqrt(hidden_dim) for FFN outputs). - Norm layers are set to weight = 1, bias = 0. RMSNorm LayerNormweightNbiasrencoder_configgrr)meanstdab)r"r)rrrrdenser/)rrrrrr decoder_config gate_projfc1up_proj down_projfc2r$default)"rrrrnnrgetattrinitones_rzeros_r Embeddingrhasattrr embedding_dim trunc_normal_ padding_idxrrrr7rrLinear in_featuresr rope_typer compute_default_rope_parameterscopy_roriginal_inv_freq)rmodule class_namerrr2nameproj proj_namerin_stdrrr  hidden_dimout_stdfan_inrope_fn buffer_valuers r= _init_weightsz BltPreTrainedModel._init_weightsts%%..  fz2<<8 9Y*=TXcgqXqvx.: 6==)vvt,8 FKK(  fbll +!$++}dCK"wt{{  f/1BC D W I $++}d;C{wv}=(({MD"646D'GD(,C"kk//3  {t)C< / vy$7#h(?&&  s(c' tVT2> DII. /VXwvw/MNF!gfh&?""MM3h#g 6640<KK ,  ff %)F!$++}dCK"wt{{ DII. /$H)E&--33A6 $d*""$$7l'k 9fd3?KK /  fbii (''F$,C    s(c'  {{& FKK(  f0 1##y0$F$4$45;;  &fmm4OL! JJv 5 JJv// > 2r?N)rrrr%__annotations___supports_attention_backend_supports_flash_attn_supports_flex_attn_no_split_modulesrrr_can_record_outputsr3rDr'rr?r=rr`s_ "' ./'(;1E$%5Q?U]]_Z?Z?r?rc|eZdZUeed<deeddiZdeffd Z dde jdzd e jdzd e jdzd e jdzd e jdzd e dzde jdzde jdzde dzde jdzdeefdZdZxZS)BltLocalEncoderrencoder_attentionsr$r\r layer_namec t||d|_||_t j t |jDcgc]}t||c}|_ t||_ t j|j|j|jzd|_t j |j"|j|_t j |_|j(r |jnd}t |D]3}|j&j+t-|||j5|j/ycc}w)NFrr out_featuresrr$rrr)rrgradient_checkpointingrr  ModuleListrYnum_hidden_layersrlayersr rotary_embrrrhpatch_embedding_projectionr vocab_sizerXcross_attn_layerscross_attn_all_layersrr post_initrrr layers_to_addrs r=rzBltLocalEncoder.__init__s!  &+# mmEJ6KcKcEd e  3 e  -F;*,))**++f.A.AA+ ' LL):):F>|L +33 2 21 5 8P8P PRVR]R]RiRi  $(;;#D#DC! ,MD,B,B9,M-".+8#9- -)& ,.DD - E. ,222r?cF|jd}|jd}|jdjdd|jd}tj|||f|j |j }|j|d|dd}|ddd|ddf}|S) a Reduce variable length patches to single embedding per patch Note: this works with variable number of patches for different sequences in the batch It handles variable length patches by assuming that patch_lengths will be 0 for any extra patches on the *right*. Since there can be a variable number of patches this function also return the number of patches for each sequence in the batch. Any embeddings on the right that are not allocated to a patch (i.e. if the sum(patch_lengths[i]) < seq_len for any i) will be sent to a dummy patch, which is trimmed before returning. rr/r,r$amaxF)srcr2rreduce include_selfN)r7rjrkr3rEr-r.scatter_reduce)rrmax_num_patchesrdrHrreduced_embeddingss r=rQzBltLocalEncoder.patch_reducees#((+ %++B/ ''+222r=;N;Nr;RS "[[ - 8 @S@S\i\p\p 0>> ? 03CO3CQ0FG!!r? NNNNNNNNNN)rrrr(r(rrr-rr3 LongTensorrrrrrrrQrrs@r=r/r/s! !!n-=QSbc42.2-1,0.204(,266:"&)-43##d*43||d*43llT) 43  t+ 43 &&- 4343((4/43!& t 3434Z43<<$&43+,43l"r?r/c0eZdZUeed<deffd Z ddejdzdejdzdejdzdejdzdejdzd e dzd ejdzd ejdzd e e fd Z xZ S)BltLocalDecoderrc t||d|_||_d|_t j t|jDcgc]}t||c}|_ t||_ t j|j|j|j zd|_t%|j|j&|_t j |_|j,r |jnd}t|D]3}|j*j/t1|||j5|j3ycc}w)NFTr4r5rr$r7)rrr8rcross_attn_decoderr r9rYr:rr;rr<rhidden_size_globalrrhr=rrnormr?r@rrrArBs r=rzBltLocalDecoder.__init__s$  &+# "&mmEJ6KcKcEd e  3 e  -F;*,))11++f.A.AA+ ' v11v7J7JK !#4:4P4P00VW }- I  " " ) )!9RXRdRde   ! fs E%NrDrErFrrrGrHrIrc $|jd} |} |j|}|j| |jd|jjz|jj }||j s| |z} |Mtj|jd|jjdj| d}|j| |} tj| |jj|j} t!|j"D]O\} }| dk(s|jj$r!|j&| d| ||d| \}}| |z} || f| |||d| } Q|j)| }|S) Nrr$r0r/rKrOrMr)r7r=rrrhrrcr3r6r.rjrkr<rPrrrr;r@r?re)rrDrErFrrrGrHrIrrHrrNrrSrTrlogitss r=rzBltLocalDecoder.forwards#((+ % 66|D #++  **1- 0H0HH$++JaJa   #D,C,C)L8M   ]003MrrXrrrerlm_headrArs r=rzBltPatcher.__init__ s  ,DKK@mmo t{{<<= LI KK  24;; J K LLL)?)?AXAXYt{{66DKK|du|duz r td||j|}|r|t|j}|F||j nd} t j | | |jdz|j}||jd}t|j|||||} |}|j||}|jD] }|||| }|j|j|}t jj!|j#}|jdd \}}||j%|||| }n.t j&||f|j(|j }t+|| }|||fS) N:You must specify exactly one of input_ids or inputs_embedsr4rr$r0rrErrHrGr)rNr)rgr) entropiesrfr{r|r,)rmrXrrget_seq_lengthr3r6r7r.rjr r<r;ryre distributions Categoricalentropypatch_lengths_from_entropiesonesr-r)rrDrrrGrErzrHr{r|r|rpast_seen_tokens causal_maskrrNrSrgprediction_entropiesrHrfr{s r=rzBltPatcher.forwards  -t"; <YZ Z   --i8M 0*$++>O  !CRC^==?de "\\ "2]5H5H5K"KTaThThN  )33A6L(;;'))+%  & "oom\J[[ vE!-EXituM vdii 67$22>>f>MUUW&3&9&9"1&=# O  ! ==. /%# >M"JJ_-]5H5HQ^QeQeM.m=MN #]F::r?c|jd}tjddgtj|jj dj |d}|jd}|ddddf}||kD}|jd}tj||jj dj|d} tj| |} tj| | gd} tj||gd} | | j||} |jdj}| ddd|f} tj|| |zfd}tj|ddddf|dz }tj|ddddfdz |fd}||z dz}|S)z Computes patch lengths from token entropies. Depending on whether a threshold is provided, the function uses either: - Thresholding the entropy values (when `threshold` is set). rr$r,Nr0r/r1)r7r3r4longr.rjrepeatr6rk full_likerFrr8r)rrfr{r|rH init_tokensoffset patch_maskrI token_indicessentinelpadded_indices padded_mask patch_startsmax_valid_patchespatch_start_ids last_token patch_endsr{s r=rz'BltPatcher.patch_lengths_from_entropies]s__Q'  LL!Quzz):J:J K U UVW X _ _`jlm n ""1%ae$ * ""1% WY5E5EFPPQRSZZ[egij ??=':M8#rrr rrrryrA)rrrs r=rzBltForCausalLM.__init__Tsk  !113 ++f% yy!6!6!B!BFDUDU\ab  r?NrDrrrrvfull_text_row_masked_out_maskrGrElabelsrzrHlogits_to_keeprrUc |jd|||||||| | d | }|j}t| tr t | dn| }|j |dd|ddfj }d}| |j|| |jfi| }t|||j|j|jS)a cross_attention_states (`torch.FloatTensor`, *optional*): Output of the vision model, used for cross-attention. This tensor contains the processed image features that the language model will attend to. cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*): Cross-attention mask to control the interaction between text tokens and image tiles. This 4D tensor defines which image tiles each text token should attend to. For each text token (in seq_length): - 1 indicates the token **should attend** to the corresponding image tile - 0 indicates the token **should not attend** to the corresponding image tile full_text_row_masked_out_mask (`tuple[torch.Tensor, torch.Tensor]`, *optional*): A tuple containing two tensors that mask out rows in the cross-attention mechanism: - The first tensor has shape `(batch_size, 1, seq_length, 1)` and contains values of 0 or 1. A value of 0 indicates that the corresponding text token's entire row in the cross-attention matrix should be masked out (all image tokens ignored). - The second tensor has the same shape and is used internally to apply the masking during the forward pass of cross-attention layers. This mask is derived from the cross_attention_mask and is used to handle cases where a text token should not attend to any image token. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, BltForCausalLM >>> model = BltForCausalLM.from_pretrained("itazap/blt-1b-hf") >>> tokenizer = AutoTokenizer.from_pretrained("itazap/blt-1b-hf") >>> prompt = "If I had to write a haiku, it would be:" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=40, do_sample=True, temperature=0.6) >>> result = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] >>> print(result) If I had to write a haiku, it would be: "Snowflakes gently fall" - simple, yet peaceful. I love the idea of snowflakes gently falling, each one ``` ) rDrrrvrrGrErzrHN)lossrgrGrrr) rrrrsliceryr loss_functionr>r rGrr)rrDrrrrvrrGrErrzrHrroutputsr slice_indicesrgrs r=rzBltForCausalLM.forward]s~$**  )%!5*G+')     11 8B>SV8W~ot4]k mA}a,?@AGGI  %4%%ffdooPPD%#33!//))   r?) NNNNNNNNNNNr)rrrr%r(_can_compile_fullgraphbase_model_prefix_tied_weights_keysrrrr3r_rrrrrorrrr rrrs@r=rrIs "CEUVy.2.204:>8rrOrlistrcfloat32ror-rrzrrrrrrrrr/rarjrwrr__all__rr?r=rsC$ &CC)/OK5&RR0GE27   H %::<\a||),9<UX$#,,#!# #+. # #' # $' # \\#T % K ||K K K  K  K ;; K  5<< %&K \))t)X]XdXd)X ]  " <-<"`9`,., ,)0,)^n?.n?n?bp"(p"fK(K\3-3lH#HVri!rij i 'i  i X r?