
    (HJjz                     J   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZ ddlmZmZmZ e G d de             Z G d	 d
e	j"                        Z G d de	j"                        Z G d de	j"                        Z G d de	j"                        Z G d de	j"                        Zy)    )	dataclass)AnyOptionalN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   y)	ModelArgs
model_typehidden_sizenum_attention_headsnum_hidden_layers
vocab_sizeintermediate_sizeintermediate_size_mlpnum_key_value_headsrms_norm_eps
rope_thetahead_dimtie_word_embeddingsno_rope_layersuse_qk_normN)	__name__
__module____qualname__str__annotations__intfloatboollist     c/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/llama4_text.pyr   r      sS    OOMr%   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                 f   t         |           || _        |j                  | _        |j
                  | _        |j                  | _        | j                  dz  | _        t        j                  |j                  | j                  | j                  z  d      | _        t        j                  |j                  | j                  | j                  z  d      | _        t        j                  |j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  z  |j                  d      | _        || _        |r1t        j"                  | j                  d|j$                        | _        |j(                  | _        |j*                  | _        y )Ng      FbiasT)traditionalbase)super__init__r)   r   n_headsr   
n_kv_headsr   scalennLinearr   q_projk_projv_projo_projuse_ropeRoPEr   roper   r   selfr)   r:   	__class__s      r&   r0   zAttention.__init__    s<   	//22]]D(
iidllT]]:
 iidoo=E
 iidoo=E
 iiLL4==($*:*:
 !4dooVDI++ --r%   xmaskcachereturnc                    |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      }|j	                  ||| j                  d      }| j                  rVt        j                  j                  |d | j                        }t        j                  j                  |d | j                        }|j                  dddd      }|j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	| j                  r6||j                  nd}
| j                  ||
      }| j                  ||
      }||j!                  ||	      \  }}	t#        |||	|| j$                  |      }|j                  dddd      j	                  ||d      }| j'                  |      S )Nr      r      )offset)rB   r3   rA   )shaper6   r7   r8   reshaper1   r2   r   mxfastrms_normr   	transposer:   rH   r<   update_and_fetchr
   r3   r9   )r>   r@   rA   rB   BLDquerieskeysvaluesrH   outputs               r&   __call__zAttention.__call__:   s    ''1a $AAAv//!Qb9||Aq$//26gg&&wd6G6GHG77##D$0A0ABD##Aq!Q/~~aAq)1door:DDQ1aP==%*%6U\\AFiii7G99T&91D 11$?LD&-T6djjt
 !!!Q1-55aB?{{6""r%   NNr   r   r   r   r0   rK   arrayr   r   rW   __classcell__r?   s   @r&   r(   r(      sW    .Y .: $(#	#88# rxx # }	#
 
#r%   r(   c                   n     e Zd Zej                  f fd	Zdej                  dej                  fdZ xZ	S )MLPc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y NFr+   )r/   r0   r4   r5   	gate_projup_proj	down_proj)r>   dimr   
activationr?   s       r&   r0   zMLP.__init__\   sM    3(9Fyy&7eD#4cFr%   r@   rC   c                 t    | j                  t        | j                  |      | j                  |                  S N)rc   r   ra   rb   )r>   r@   s     r&   rW   zMLP.__call__b   s)    ~~fT^^A%6QHIIr%   )
r   r   r   r4   silur0   rK   rZ   rW   r[   r\   s   @r&   r^   r^   [   s.    :<'' GJ"(( Jrxx Jr%   r^   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr)   c                 N   t         |           t        ||      | _        t	        |j
                  |j                        | _        t        j                  |j
                  |j                        | _        t        j                  |j
                  |j                        | _        y )Neps)r/   r0   r(   	self_attnr^   r   r   feed_forwardr4   RMSNormr   input_layernormpost_attention_layernormr=   s      r&   r0   zTransformerBlock.__init__g   s~    "42&&

  "zz$*:*:@Q@QR(*

$"3"3)
%r%   r@   rA   rB   rC   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   S rg   )rn   rq   ro   rr   )r>   r@   rA   rB   rhs         r&   rW   zTransformerBlock.__call__u   sN     NN4//2D%@Ed;;A>?1ur%   rX   rY   r\   s   @r&   rj   rj   f   sW    
Y 
" $(#		88	 rxx 	 }		
 
	r%   rj   c                   j     e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	 xZ
S )LanguageModelr)   c                    t         |           || _        |j                  | _        t	        j
                  |j                  |j                        | _        t        |j                        D cg c]  }t        ||j                  |          c}| _        t	        j                  |j                  |j                        | _        y c c}w )N)r)   r:   rl   )r/   r0   r)   r   r4   	Embeddingr   embed_tokensranger   rj   r   layersrp   r   norm)r>   r)   ir?   s      r&   r0   zLanguageModel.__init__   s    	//LL$:J:JK 4112
2 $1D1DQ1GH2
 JJt//T5F5FG		
s   .!CinputsrB   rC   c                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rz   lenr|   r	   zipr}   )r>   r   rB   ru   rA   layercs          r&   rW   zLanguageModel.__call__   su    
 f%=FS--E$Qa1DKK/HE1aq!A 0 yy|r%   rg   rY   r\   s   @r&   rw   rw      sC    	HY 	H  $ } 
	r%   rw   c                   z     e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	e
d        Z xZS )	Modelr)   c                    t         |           || _        |j                  | _        t	        |      | _        |j                  | _        | j                  s2t        j                  |j                  |j                  d      | _        y y r`   )r/   r0   r)   r   rw   modelr   r4   r5   r   r   rV   )r>   r)   r?   s     r&   r0   zModel.__init__   sf    	//"4(
#'#;#; ''))D$4$4dooERDK (r%   r   rB   rC   c                     | j                  ||      }| j                  r-|| j                   j                  j                  j                  z  S | j                  |      S rg   )r   r   rz   weightTrV   )r>   r   rB   ru   s       r&   rW   zModel.__call__   sL    
 JJvu%##tzz..557777;;q>!r%   c                 .    | j                   j                  S rg   )r   r|   )r>   s    r&   r|   zModel.layers   s    zz   r%   rg   )r   r   r   r   r0   rK   rZ   r   r   rW   propertyr|   r[   r\   s   @r&   r   r      sW    SY S  $	"	" }	" 
		" ! !r%   r   )dataclassesr   typingr   r   mlx.corecorerK   mlx.nnr4   activationsr   r.   r   r	   r
   r   Moduler(   r^   rj   rw   r   r$   r%   r&   <module>r      s    "      T T   "9#		 9#xJ")) Jryy 6BII :!BII !r%   