
    (HJj                     ^   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ ddlmZ e G d d	e             Z G d
 dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Zy)    )	dataclass)AnyDictOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_ropec                      e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZee   ed	<   dZ	ee   ed
<   dZ
ee   ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeeeeeef   f      ed<   dZeed<   d Zy)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizeNhead_dimmax_position_embeddingsnum_key_value_headsFattention_biasmlp_biasi'  
rope_thetarope_traditionalrope_scalingTtie_word_embeddingsc                 @    | j                   | j                  | _         y y N)r   r   selfs    ]/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/olmo2.py__post_init__zModelArgs.__post_init__!   s!    ##+'+'?'?D$ ,    )__name__
__module____qualname__str__annotations__intfloatr   r   r   r   r   boolr   r   r   r   r   r   r   r%    r&   r$   r   r      s    OO"Hhsm"-1Xc]1)-#- ND HdJ"d";?L(4U5#:%6 678? $$@r&   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                 D   t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  xs |j                  |z  x| _        }|dz  | _        t        |d      r|j                  }nd}t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  ||z  ||      | _        t#        | j                  |j$                  |j&                  |j(                  |j*                        | _        t        j.                  ||z  |j0                        | _        t        j.                  ||z  |j0                        | _        y )Ng      r   Fbias)super__init__r   r   n_headsr   
n_kv_headsr   scalehasattrr   nnLinearq_projk_projv_projo_projr   r   r   r   r   ropeRMSNormr   q_normk_norm)r#   r2   dimr8   r9   r   r   	__class__s          r$   r7   zAttention.__init__'   sX   !%!9!99w'+'?'??*#'==#OD4D4D4OOt^
4)*!00N"NiiWx%7nMiiZ(%:PiiZ(%:Pii( 2CnM#MMOO!!((
	 jj8!3T5F5FGjjh!68I8IJr&   xmaskcachereturnc                 v   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}| j	                  |      }| j                  |      }|j                  ||| j                  d      j                  dddd      }|j                  ||| j                  d      j                  dddd      }|	j                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j                  ||d      }
| j                  |
      S )Nr      r      )offset)rJ   r:   rI   )shaper>   r?   r@   rD   rE   reshaper8   	transposer9   rB   rP   update_and_fetchr   r:   rA   )r#   rH   rI   rJ   BLDquerieskeysvaluesoutputs              r$   __call__zAttention.__call__F   s    ''1a $AAAv++g&{{4  //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r&   NNr'   r(   r)   r   r7   mxarrayr   r   r\   __classcell__rG   s   @r$   r1   r1   &   sZ    KY KD $(#	#88# rxx # }	#
 
#r&   r1   c                   D     e Zd Zdef fdZdej                  fdZ xZS )MLPr2   c                 6   t         |           |j                  }|j                  }t	        |d      r|j
                  }nd}t        j                  |||      | _        t        j                  |||      | _	        t        j                  |||      | _
        y )Nr   Fr4   )r6   r7   r   r   r;   r   r<   r=   	gate_proj	down_projup_proj)r#   r2   rF   
hidden_dimr   rG   s        r$   r7   zMLP.__init__h   sx    ++
4$}}HH3
B:sByyjx@r&   rK   c                 t    | j                  t        | j                  |      | j                  |                  S r!   )rg   r	   rf   rh   )r#   rH   s     r$   r\   zMLP.__call__v   s)    ~~fT^^A%6QHIIr&   	r'   r(   r)   r   r7   r_   r`   r\   ra   rb   s   @r$   rd   rd   g   s#    AY AJRXX Jr&   rd   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr2   c                 t   t         |           |j                  | _        |j                  | _        t	        |      | _        t        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        || _        y )Neps)r6   r7   r   r   r1   	self_attnrd   mlpr<   rC   r   post_attention_layernormpost_feedforward_layernormr2   r#   r2   rG   s     r$   r7   zTransformerBlock.__init__{   s    #'#;#; ++"4t9(*

$"3"3)
% +-**$"3"3+
' 	r&   rH   rI   rJ   rK   c                     | j                  | j                  |||            }||z   }| j                  | j                  |            }||z   }|S r!   )rs   rq   rt   rr   )r#   rH   rI   rJ   rhouts          r$   r\   zTransformerBlock.__call__   sP     ))$..D%*HIE++DHHQK8!e
r&   r]   r^   rb   s   @r$   rm   rm   z   sW    Y " $(#	
88
 rxx 
 }	

 

r&   rm   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )
LlamaModelr2   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t        j                  |j                  |j                        | _        y c c}w )Nr   )r2   ro   )r6   r7   r2   r   r   r<   	Embeddingr   embed_tokensrangerm   layersrC   r   norm)r#   r2   _rG   s      r$   r7   zLlamaModel.__init__   s    	//!%!7!7"""LL$:J:JK16t7M7M1N
1NA$'1N
 JJt//T5F5FG	
s   Cinputsc                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rJ   )r~   lenr   r   zipr   )r#   r   rJ   rx   rI   layercs          r$   r\   zLlamaModel.__call__   su    
 f%=FS--E$Qa1DKK/HE1aQ'A 0 yy|r&   r!   rk   rb   s   @r$   r{   r{      s'    
HY 
H r&   r{   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )Modelr2   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFr4   )r6   r7   r2   r   r{   modelr   r<   r=   r   r   lm_headru   s     r$   r7   zModel.__init__   sX    	//%
''99T%5%5tUSDL (r&   r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r!   )r   r2   r   r~   	as_linearr   )r#   r   rJ   ry   s       r$   r\   zModel.__call__   sW    
 jj'99((**))33C8C 
 ,,s#C
r&   c                 `    |j                         D ci c]  \  }}d|vs|| c}}S c c}}w )Nzself_attn.rotary_emb.inv_freq)items)r#   weightskvs       r$   sanitizezModel.sanitize   s<     %]]_
,TQ0OWX0XAqD_
 	
 
s   **c                 .    | j                   j                  S r!   )r   r   r"   s    r$   r   zModel.layers   s    zz   r&   r!   )r'   r(   r)   r   r7   r_   r`   r\   r   propertyr   ra   rb   s   @r$   r   r      s@    TY T 


 ! !r&   r   )dataclassesr   typingr   r   r   r   mlx.corecorer_   mlx.nnr<   activationsr	   baser
   r   r   
rope_utilsr   r   Moduler1   rd   rm   r{   r   r/   r&   r$   <module>r      s    " - -    T T ' @ @ @.>#		 >#BJ")) J&ryy 8 <!BII !r&   