
    (HJjI&                        d dl mZ d dlmZmZmZ d dlmZ d dl	m
Z
 ddlmZ ddlmZmZmZmZ ddlmZmZ e G d d	e             Z G d
 de
j,                        Z G d de
j,                        Z G d de
j,                        Z G d de
j,                        Z G d de
j,                        Z G d de
j,                        Zy)    )	dataclass)AnyListOptionalN   )swiglu)BaseModelArgscreate_attention_maskcreate_ssm_maskscaled_dot_product_attention)ArraysCacheKVCachec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   dZeed<   dZ	e
e   ed<   dZe
ee      ed<   dZe
ee      ed<   d Zy)	ModelArgs
model_type
vocab_sizehidden_sizenum_hidden_layersnum_attention_headsnum_key_value_headsmax_position_embeddingsnorm_eps	conv_biasconv_L_cache	block_dimblock_ff_dimblock_multiple_ofblock_ffn_dim_multiplierblock_auto_adjust_ff_dimg    .A
rope_thetaNrope_parametersfull_attn_idxslayer_typesc                 $   | j                   "d| j                   v r| j                   d   | _        | j                  | j                  | _        | j                  3t        | j                        D cg c]  \  }}|dk(  r| c}}| _        y y c c}}w )Nr    full_attention)r!   r    r   r   r"   	enumerater#   )selfi
layer_types      \/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/lfm2.py__post_init__zModelArgs.__post_init__(   s    +@T@T0T"22<@DO##+'+'?'?D$& &/t/?/?%@#%@MAz!11 %@#D '#s   0B)__name__
__module____qualname__str__annotations__intfloatboolr    r!   r   dictr"   r   r#   r+        r*   r   r      s    OO  OON##""!J!&*OXd^**.NHT#Y'.'+K$s)$+
r6   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                    t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  |z  x| _        }|dz  | _        t        j                  ||j                        | _        t        j                  ||j                        | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  ||z  |d      | _        t        j&                  | j                  |j(                  d      | _        y )Ng      ࿩epsFbias)basetraditional)super__init__r   r   n_headsr   
n_kv_headshead_dimscalennRMSNormr   q_layernormk_layernormLinearq_projk_projv_projout_projRoPEr    rope)r'   r9   dimrC   rD   rE   	__class__s         r*   rB   zAttention.__init__6   s   !%!9!99w'+'?'??*#'#3#3w#>>t^
::hDMMB::hDMMBiiWx%7eDiiZ(%:GiiZ(%:G		'H"4cFGGMM
	r6   xmaskcachereturnc                 n   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}| j	                  |j                  ||| j                  d            j                  dddd      }| j                  |j                  ||| j                  d            j                  dddd      }|	j                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	||| j                        }
|
j                  dddd      j                  ||d      }
| j                  |
      S )Nr      r      )offset)rV   rU   rF   )shaperL   rM   rN   rI   reshaperC   	transposerJ   rD   rQ   r\   update_and_fetchr   rF   rO   )r'   rT   rU   rV   BLDquerieskeysvaluesoutputs              r*   __call__zAttention.__call__O   s    ''1a $AAAv""7??1ar#JKUUq!Q
 Q4??B GHRRq!Q
 1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6T
 !!!Q1-55aB?}}V$$r6   NN)r,   r-   r.   r   rB   mxarrayr   r   rh   __classcell__rS   s   @r*   r8   r8   5   sW    
Y 
8 $(#	%88% rxx % }	%
 
%r6   r8   c                   v     e Zd Zdedef fdZ	 	 ddej                  deej                     dee	   fdZ
 xZS )		ShortConvr9   	layer_idxc                    t         |           || _        || _        |j                  | _        |j                  | _        t        j                  |j                  |j                  | j
                  |j                  | j                        | _        t        j                  |j                  d|j                  z  | j                        | _        t        j                  |j                  |j                  | j                        | _        y )N)in_channelsout_channelskernel_sizegroupsr>   r[   r=   )rA   rB   r9   rp   r   L_cacher   r>   rG   Conv1dr   convrK   in_projrO   r'   r9   rp   rS   s      r*   rB   zShortConv.__init__q   s    
 		"((NN	II(())##
	 yy!1!11t7G7G3GdiiX		$"2"2D4D4D499Ur6   rT   rU   rV   c                    | j                  |      }t        j                  |dd      \  }}}||z  }|t        j                  |d   |d      }|#|d   St        j                  |j
                  d   | j                  dz
  | j                  j                  f|j                        }n|d   }t        j                  ||gd      }| j                  dz
  }	|j
                  d   }
|j                  _t        j                  |j                  d|
      }|d d d f   t        j                  |	      z   d   }t        j                  ||d      |d<   n|d d |	 d d d f   |d<   |j                  |
       n(t        j                   |d| j                  dz
  dfdg      }| j#                  |      }||z  }| j%                  |      S )	Nr[   rY   )axis).Nr   r   )dtype)r   r   )ry   rj   splitwherezerosr]   rv   r9   r   r}   concatenatelengthscliparangetake_along_axisadvancepadrx   rO   )r'   rT   rU   rV   BCxra   CBxstaten_keeptends	positionsconv_outys                  r*   rh   zShortConv.__call__   s    ll1o((3+1aU$y/2q1BQxXXa[$,,"2DII4I4IJ((
 a!4B\\A%F
A}}(wwu}}a3!!T']RYYv->>	J	--b)!Daa&1n-aMM!VdllQ&6%:FCDB99R=L}}Qr6   ri   r,   r-   r.   r   r1   rB   rj   rk   r   r   rh   rl   rm   s   @r*   ro   ro   p   sW    VV V0 $(#	$ 88$  rxx $  }	$ r6   ro   c                   Z     e Zd Zdededededee   f
 fdZdej                  fdZ
 xZS )	MLPrR   ff_dimmultiple_ofauto_adjust_ff_dimffn_dim_multiplierc                 2   t         |           |r/t        d|z  dz        }|t        ||z        }|||z   dz
  |z  z  }t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y )NrZ   r[   r   Fr=   )rA   rB   r1   rG   rK   w1w3w2)r'   rR   r   r   r   r   rS   s         r*   rB   zMLP.__init__   s     	Va(F!-/&89 Vk%9A%=+$MNF))Ce4))Ce4))FCe4r6   rW   c                 t    | j                  t        | j                  |      | j                  |                  S N)r   r   r   r   )r'   rT   s     r*   rh   zMLP.__call__   s(    wwvdggaj$''!*566r6   )r,   r-   r.   r1   r3   r   r2   rB   rj   rk   rh   rl   rm   s   @r*   r   r      sL    55 5 	5
 !5 %UO5&7RXX 7r6   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
Lfm2DecoderLayerr9   rp   c                    t         |           ||j                  v | _        | j                  rt	        |      | _        nt        ||      | _        t        |j                  |j                  |j                  |j                  |j                        | _        t        j                   |j"                  |j$                        | _        t        j                   |j"                  |j$                        | _        y )N)rR   r   r   r   r   r;   )rA   rB   r"   is_attention_layerr8   	self_attnro   rx   r   r   r   r   r   r   feed_forwardrG   rH   r   r   operator_normffn_normrz   s      r*   rB   zLfm2DecoderLayer.__init__   s    "+t/B/B"B""&t_DN!$	2DI$$..#<<#<<
  ZZ(8(8dmmL

4#3#3Gr6   rT   rU   rV   rW   c                     | j                   r$| j                  | j                  |      ||      }n#| j                  | j                  |      ||      }||z   }|| j	                  | j                  |            z   }|S )N)rU   rV   )r   r   r   rx   r   r   )r'   rT   rU   rV   rhouts          r*   rh   zLfm2DecoderLayer.__call__   s     ""t11!44uMA		""1%  A
 E$##DMM!$455
r6   ri   r   rm   s   @r*   r   r      sa    HY H3 H, $(#	88 rxx  }	
 
r6   r   c                   h     e Zd Zdef fdZ	 	 ddej                  deej                     fdZ xZ	S )	Lfm2Modelr9   c                 V   t         |           || _        |j                  | _        |j                  | _        t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        ||       c}| _        t        j                  |j                  |j                        | _        |j                  d   | _        d| _        t        |j                        D ]'  }||j                  v r| xj"                  dz  c_        ' y  y c c}w )N)rp   r;   r   r   )rA   rB   r9   r   r   rG   	Embeddingr   embed_tokensranger   layersrH   r   embedding_normr"   fa_idxconv_idx)r'   r9   r(   rS   s      r*   rB   zLfm2Model.__init__   s    	//!%!7!7LL$:J:JK9>t?U?U9V
9VATQ/9V
 !jj)9)9t}}M))!,t--.AD'''"	 /
s   ?D&inputsinput_embeddingsc                 \   ||}n| j                  |      }|d gt        | j                        z  }t        ||| j                           }t        ||| j                           }t        | j                  |      D ]   \  }}|j                  r|n|}	 |||	|      }" | j                  |      S )N)rV   )
r   lenr   r
   r   r   r   zipr   r   )
r'   r   rV   r   r   	attn_mask	conv_masklayercrU   s
             r*   rh   zLfm2Model.__call__  s     ' A!!&)A=FS--E)!U4;;-?@	#AuT]]';<	DKK/HE1 % 8 89iDaQ'A 0 ""1%%r6   ri   )
r,   r-   r.   r   rB   rj   rk   r   rh   rl   rm   s   @r*   r   r      s;    Y . /3	&& #288,	&r6   r   c                        e Zd Zdef fdZ	 	 d	dej                  deej                     fdZd Z	e
d        Zd Z xZS )
Modelr9   c                 r    t         |           || _        |j                  | _        t	        |      | _        y r   )rA   rB   r9   r   r   model)r'   r9   rS   s     r*   rB   zModel.__init__  s+    	//t_
r6   r   r   c                 r    | j                  |||      }| j                   j                  j                  |      S r   )r   r   	as_linear)r'   r   rV   r   r   s        r*   rh   zModel.__call__!  s2     jj(89zz&&0055r6   c                     i }|j                         D ]@  \  }}d|v r2|j                  d   |j                  d   kD  r|j                  ddd      }|||<   B |S )Nzconv.weightrY   r   r   rZ   )itemsr]   r_   )r'   weightssanitized_weightsnameparams        r*   sanitizezModel.sanitize*  sa    "==?KD%$;;r?U[[^3!OOAq!4E&+d# + ! r6   c                 .    | j                   j                  S r   )r   r   )r'   s    r*   r   zModel.layers4  s    zz   r6   c                     | j                   D cg c]$  }|j                  r
t               nt        d      & c}S c c}w )Nr   )size)r   r   r   r   )r'   ls     r*   
make_cachezModel.make_cache8  sA     [[
  --GI;A3FF 
 	
 
s   );ri   )r,   r-   r.   r   rB   rj   rk   r   rh   r   propertyr   r   rl   rm   s   @r*   r   r     sY    %Y % /3	66 #288,	6! ! !
r6   r   )dataclassesr   typingr   r   r   mlx.corecorerj   mlx.nnrG   activationsr   r?   r	   r
   r   r   rV   r   r   r   Moduler8   ro   r   r   r   r   r5   r6   r*   <module>r      s    ! & &     (   D8%		 8%v: 		 : z7")) 70%ryy %P*&		 *&Z"
BII "
r6   