
    (HJj"                     f   d dl mZ d dlmZmZmZ d dlmZ d dl	m
Z
 ddlmZ ddlmZmZmZ ddlmZmZmZmZ e G d d	e             Z G d
 de
j.                        Z G d de
j.                        Z G d de
j.                        Z G d de
j.                        Z G d de
j.                        Zy)    )	dataclass)AnyListOptionalN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)ArraysCache	CacheListKVCacheRotatingKVCachec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   ee   ed	<   eed
<   eed<   dZeed<   dZ	e
e   ed<   dZe
e   ed<   dZeed<   y)	ModelArgs
vocab_sizehidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_heads
rope_thetasliding_windowsliding_window_layersconv_windowrms_norm_epsbaichuan_m1
model_typeNnum_swa_attention_headsnum_swa_key_value_headsFtie_word_embeddings)__name__
__module____qualname__int__annotations__floatr   r   strr   r   r    r!   bool     c/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/baichuan_m1.pyr   r      sz    O9$#J#-1Xc]1-1Xc]1 %%r+   r   c            	            e Zd Zd
dedee   f fdZd
dZ	 ddej                  dej                  de
dej                  fd	Z xZS )	Attentionconfig	layer_idxc                    t         |           || _        || _        |t	        d      ||j
                  v | _        | j                  r|j                  r|j                  n|j                  | _	        | j                  r|j                  r|j                  n|j                  | _        |j                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k(  sJ | j                  dz  | _        t!        j"                  |j                  | j                  d| j                  z  | j                  z  z   d      | _        t!        j"                  | j                  | j                  z  |j                  d      | _        t!        j(                  | j                  d|j*                        | _        |j.                  | _        | j.                  dk(  sJ t1        j2                  dd| j                  d| j.                  f      | _        t1        j2                  dd| j                  d| j.                  f      | _        y )Nz1Layer index must be provided to Attention module.g         Fbias)traditionalbaser   )super__init__r/   r0   
ValueErrorr   is_swar   r   	num_headsr    r   num_kv_headsr   head_dimscalennLinearW_packo_projRoPEr   roper   mxzerosconv_kconv_vselfr/   r0   	__class__s      r,   r8   zAttention.__init__"   s   "PQQ6#?#?? {{v== **++ 	 {{v== **++ 	 "--((DNN:}}t~~-1A1AAAA]]D(
iiq4#4#44t}}DD

 iiNNT]]*F,>,>U
 GGDMMu6CTCTU	!--1$$$hh1d&7&7D<L<LMNhh1d&7&7D<L<LMNr+   c                 P   |j                   \  }}}}|j                  d|| j                  ddf      }|d d d d df   }|d d d d df   }	|$t        j                  ||d|f|j
                        }|dkD  r&t        j                  ||d d d d d df   gd      }
n|}
|
|z  ||	z  z   S )Nr   r   r2   axis)shapereshaper   rE   rF   dtypeconcatenate)rJ   uweightsstateBHLDw0w1u_prevs              r,   _custom_convolutionzAttention._custom_convolutionK   s    WW
1a//1a)9)91a"@AQ1WQ1W=HHaAq\1773Eq5^^UAaCRCiL$9BFF{QV##r+   xmaskcachereturnc                    |j                   \  }}}| j                  |      }t        j                  |||| j                  | j
                  z  z   fd      \  }}	}
|j                  ||| j                  | j
                        j                  dddd      }|	j                  ||| j                  | j
                        j                  dddd      }	|
j                  ||| j                  | j
                        j                  dddd      }
|d}|d    |d   j                  }|d   d   |d   d   }}nd}d\  }}|	}|
}| j                  |	| j                  |      }	| j                  |
| j                  |      }
| j                  ||	      }| j                  |	|	      }	|d   K|d   j                  |	|
      \  }	}
|dkD  r.|d d d d dd d d f   |d   d<   |d d d d dd d d f   |d   d<   t        ||	|
|d   | j                   |
      }|j                  dddd      j                  ||d      }| j#                  |      S )NrM   rN   r   r2   r      NN)rV   )offset)ra   r>   r`   )rP   rA   rE   splitr<   r=   rQ   r;   	transposerf   r^   rG   rH   rD   update_and_fetchr   r>   rB   )rJ   r_   r`   ra   rW   rY   rZ   projqkvrf   last_klast_vk_initv_initouts                    r,   __call__zAttention.__call__X   sA    ''1a{{1~((4!Q):):T]])J%J!KRTU1aIIaDNNDMM:DDQ1aPIIaD--t}}=GG1aQRSIIaD--t}}=GG1aQRS= E81X__F"1Xa[%(1+FFF'NFF$$Q6$B$$Q6$BIIaI'IIaI'88,,Q2DAq1u$Q23\2a$Q23\2a*q!584::D
 mmAq!Q'//1b9{{3r+   Nre   )r"   r#   r$   r   r   r%   r8   r^   rE   arrayr   rs   __classcell__rK   s   @r,   r.   r.   !   s[    'Oy 'OXc] 'OR$ @D' ' !#' 9<' 	' r+   r.   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )MLPr/   c                 H   t         |           t        j                  |j                  |j
                  d      | _        t        j                  |j                  |j
                  d      | _        t        j                  |j
                  |j                  d      | _        y NFr3   )	r7   r8   r?   r@   r   r   	gate_projup_proj	down_projrJ   r/   rK   s     r,   r8   zMLP.__init__   s{     8 8u
 yy 8 8u
 $$f&8&8u
r+   r_   rb   c                 t    | j                  t        | j                  |      | j                  |                  S rt   )r~   r   r|   r}   )rJ   r_   s     r,   rs   zMLP.__call__   s)    ~~fT^^A%6QHIIr+   )	r"   r#   r$   r   r8   rE   ru   rs   rv   rw   s   @r,   ry   ry      s-    

y 

J"(( Jrxx Jr+   ry   c            	            e Zd Zdedef fdZ	 d	dej                  dej                  dedej                  fdZ	 xZ
S )
DecoderLayerr/   r0   c                 $   t         |           t        ||      | _        t	        |      | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y )Neps)r7   r8   r.   	self_attnry   mlpr?   RMSNormr   r   input_layernormpost_attention_layernormrI   s      r,   r8   zDecoderLayer.__init__   sh    "695v;!zz&*<*<&BUBUV(*

F$7$7)
%r+   r_   r`   ra   rb   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   S rt   )r   r   r   r   )rJ   r_   r`   ra   rs        r,   rs   zDecoderLayer.__call__   sL     NN4//2D%@EHHT221561ur+   re   )r"   r#   r$   r   r%   r8   rE   ru   r   rs   rv   rw   s   @r,   r   r      sN    
y 
S 
 @D!#9<	r+   r   c                   b     e Zd Zdef fdZddej                  dedej                  fdZ xZ	S )BaichuanModelr/   c                 p   t         |           || _        t        j                  |j
                  |j                        | _        t        |j                        D cg c]  }t        ||       c}| _        t        j                  |j                  |j                        | _        |j                  | _        d | _        |j"                  r|j"                  d   | _        d | _        t'        |j"                        | _        t        |j                        D ]  }|| j(                  v r|| _         y  y c c}w )Nr   r   )r7   r8   argsr?   	Embeddingr   r   embed_tokensranger   r   layersr   r   normr   first_swa_idxr   first_global_idxset
swa_layers)rJ   r/   irK   s      r,   r8   zBaichuanModel.__init__   s    	LL):):F<N<NO8=f>V>V8WX8W1|FA.8WXJJv11v7J7JK	$33!''!'!=!=a!@D $f::;v//0ADOO#$%D!	 1 Ys   D3inputsra   rb   c                    | j                  |      }|dgt        | j                        z  }| j                  d }n|| j                     d   }| j                  d }n|| j                     d   }t        ||      }t        ||| j                        }t        t        | j                  |            D ]$  \  }\  }	}
|| j                  v r|n|} |	|||
      }& | j                  |      S )Nre   r   )window_size)r   lenr   r   r   r
   r   	enumeratezipr   r   )rJ   r   ra   r_   c_globalc_swaglobal_maskswa_maskllayercr`   s               r,   rs   zBaichuanModel.__call__   s    f%=!NS%55E  (HT223A6H%E$,,-a0E+Ax8(Et?R?RS&s4;;'>?MAzq DOO38Daq!A @ yy|r+   rt   )
r"   r#   r$   r   r8   rE   ru   r   rs   rv   rw   s   @r,   r   r      s1    y (rxx  rxx r+   r   c                        e Zd Zdef fdZdee   fdZdedefdZ	dde
j                  dede
j                  fd	Zedeej                     fd
       Z xZS )Modelr/   c                    t         |           || _        |j                  | _        t	        |      | _        |j                  | _        |j                  s2t        j                  |j                  |j                  d      | _        y y r{   )r7   r8   r/   r   r   modelr!   r?   r@   r   r   lm_headr   s     r,   r8   zModel.__init__   sk     ++"6*
#)#=#= ))99V%7%79J9JQVWDL *r+   rb   c                 0   g }t        | j                  j                        D ]q  \  }}|| j                  j                  v }t        d      }|r!t        | j                  j                        }n
t               }|j                  t        ||             s |S )Nr2   )size)max_size)r   r   r   r/   r   r   r   r   r   appendr   )rJ   cachesr   r   r:   
conv_cachekv_caches          r,   
make_cachezModel.make_cache   sz    !$**"3"34HAu$++;;;F$!,J*DKK4N4NO"9MM)J9: 5 r+   rU   c                     d|v }|srd|v rn|d   }|j                   }|j                  t        j                        }t        j                  j                  |dd      }||dz   z  j                  |      }||d<   |S )Nzlm_head.scaleszlm_head.weightrM   T)rO   keepdimsgHz>)rR   astyperE   float32linalgr   )rJ   rU   is_quantizedwrR   r   s         r,   sanitizezModel.sanitize   s~    '72 0G ;()AGGE$A99>>!"t><DdTk"**51A()G$%r+   r   ra   c                 H    | j                  ||      }| j                  |      S rt   )r   r   )rJ   r   ra   outputss       r,   rs   zModel.__call__   s!    **VU+||G$$r+   c                 .    | j                   j                  S rt   )r   r   )rJ   s    r,   r   zModel.layers   s    zz   r+   rt   )r"   r#   r$   r   r8   r   r   r   dictr   rE   ru   rs   propertyr?   Moduler   rv   rw   s   @r,   r   r      sy    Xy X
DI 
	 	 	%rxx % %rxx % !RYY ! !r+   r   )dataclassesr   typingr   r   r   mlx.corecorerE   mlx.nnr?   activationsr   r6   r	   r
   r   ra   r   r   r   r   r   r   r.   ry   r   r   r   r*   r+   r,   <module>r      s    " & &    T T C C & & &$^ 		 ^ BJ")) J"299 &+BII +\'!BII '!r+   