
    (HJj                     ^   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ ddlmZ e G d d	e             Z G d
 dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Zy)    )	dataclass)AnyDictOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_ropec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   dZeed<   dZ	eed<   dZ
ee   ed<   dZeeeeeef   f      ed<   dZeed<   y)	ModelArgs
model_typehidden_sizeintermediate_sizemax_position_embeddingsnum_attention_headsnum_hidden_layersnum_key_value_headsrms_norm_eps
vocab_size
rope_thetaFmlp_biasattention_biasNhead_dimrope_scalingtie_word_embeddings)__name__
__module____qualname__str__annotations__intfloatr   boolr   r   r   r   r   r   r        a/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/telechat3.pyr   r      s    O  OHd ND "Hhsm";?L(4U5#:%6 678? %%r(   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	Telechat3Attentionargsc                    t         |           |j                  | _        |j                  | _        |j                  xs |j
                  |j                  z  | _        | j                  dz  | _        t        j                  |j
                  |j                  | j                  z  |j                        | _
        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j                  | j                  z  |j
                  |j                        | _        t        | j                  d|j                  |j                   |j"                        | _        y )Ng      ࿩biasF)traditionalbasescaling_configr   )super__init__r   r   r   r   scalennLinearr   q_projk_projv_projo_projr   r   r   r   ropeselfr,   	__class__s     r)   r4   zTelechat3Attention.__init__"   sb   #'#;#; #'#;#; U)9)9T=U=U)U]]D(
ii$$t}}4$$

 ii$$t}}4$$

 ii$$t}}4$$

 ii$$t}}4$$
 $MM,,$($@$@
	r(   xmaskcachereturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr      r      )offset)rB   r5   rA   )shaper8   r9   r:   reshaper   	transposer   r<   rH   update_and_fetchr   r5   r;   )r>   r@   rA   rB   BL_querieskeysvaluesoutputs              r)   __call__zTelechat3Attention.__call__G   sy    ''1a $AAAv//!Q(@(@"EOOq!Q
 ||Aq$":":B?II!QPQSTU1d&>&>CMMq!Q
 iii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r(   NNr   r    r!   r   r4   mxarrayr   r   rT   __classcell__r?   s   @r)   r+   r+   !   sX    #
Y #
P $(#	#88# rxx # }	#
 
#r(   r+   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )Telechat3MLPr,   c                    t         |           t        j                  |j                  |j
                  |j                        | _        t        j                  |j
                  |j                  |j                        | _        t        j                  |j                  |j
                  |j                        | _	        y )Nr.   )
r3   r4   r6   r7   r   r   r   	gate_proj	down_projup_projr=   s     r)   r4   zTelechat3MLP.__init__h   s    d444==
 ""D$4$44==
 yyd444==
r(   r@   rC   c                 t    | j                  t        | j                  |      | j                  |                  S N)r_   r	   r^   r`   )r>   r@   s     r)   rT   zTelechat3MLP.__call__t   s)    ~~fT^^A%6QHIIr(   )	r   r    r!   r   r4   rW   rX   rT   rY   rZ   s   @r)   r\   r\   g   s-    

Y 

J"(( Jrxx Jr(   r\   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	Telechat3DecoderLayerr,   c                 t   t         |           |j                  | _        |j                  | _        t	        |      | _        t        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        || _        y )Neps)r3   r4   r   r   r+   	self_attnr\   mlpr6   RMSNormr   input_layernormpost_attention_layernormr,   r=   s     r)   r4   zTelechat3DecoderLayer.__init__y   s    #'#;#; +++D1%!zz$*:*:@Q@QR(*

$"3"3)
% 	r(   r@   rA   rB   rC   c                     | j                  | j                  |      ||      }||z   }|| j                  | j                  |            z   }|S rb   )rh   rk   ri   rl   )r>   r@   rA   rB   rhouts          r)   rT   zTelechat3DecoderLayer.__call__   sM     NN4//2D%@E$((488;<<
r(   rU   rV   rZ   s   @r)   rd   rd   x   sW    
Y 
 $(#		88	 rxx 	 }		
 
	r(   rd   c            	            e Zd Zdef fdZ	 	 ddej                  dee   deej                     dej                  fdZ	 xZ
S )	Telechat3Modelr,   c                    t         |           || _        |j                  | _        |j                  | _        t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t        j                  |j                  |j                        | _        y c c}w )N)r,   rf   )r3   r4   r,   r   r   r6   	Embeddingr   embed_tokensrangerd   layersrj   r   norm)r>   r,   rO   r?   s      r)   r4   zTelechat3Model.__init__   s    	//!%!7!7LL$:J:JK6;D<R<R6S
6S!t,6S
 JJt//T5F5FG	
s   ?CinputsrB   input_embeddingsrC   c                     ||}n| j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )ru   lenrw   r   ziprx   )r>   ry   rB   rz   ro   rA   layercs           r)   rT   zTelechat3Model.__call__   s     ' A!!&)A=FS--E$Qa1DKK/HE1aq!A 0 yy|r(   rU   rV   rZ   s   @r)   rr   rr      sY    	HY 	H  $/3	 } #288,	
 
r(   rr   c            	            e Zd Zdef fdZ	 	 d	dej                  dee   deej                     dej                  fdZ	e
d        Z xZS )
Modelr,   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFr.   )r3   r4   r,   r   rr   modelr   r6   r7   r   r   lm_headr=   s     r)   r4   zModel.__init__   sX    	//#D)
''99T%5%5tUSDL (r(   ry   rB   rz   rC   c                     | j                  |||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S rb   )r   r,   r   ru   	as_linearr   )r>   ry   rB   rz   rp   s        r)   rT   zModel.__call__   sZ     jj(8999((**))33C8C 
 ,,s#C
r(   c                 .    | j                   j                  S rb   )r   rw   )r>   s    r)   rw   zModel.layers   s    zz   r(   rU   )r   r    r!   r   r4   rW   rX   r   r   rT   propertyrw   rY   rZ   s   @r)   r   r      sm    TY T  $/3	 } #288,	
 
 ! !r(   r   )dataclassesr   typingr   r   r   r   mlx.corecorerW   mlx.nnr6   activationsr	   r1   r
   r   r   
rope_utilsr   r   Moduler+   r\   rd   rr   r   r'   r(   r)   <module>r      s    " - -    T T ' & & &$C# C#LJ299 J"BII 2RYY B!BII !r(   