
    (HJj                     6   d dl Z d dlmZ d dlmZ d dlmZ ddlm	Z	m
Z
mZ e G d de	             Z G d dej                        Z G d	 d
ej                        Z G d dej                        Z G d dej                        Z G d dej                        Zy)    N)	dataclass   )BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   d	Z
eed
<   d	Zeed<   d	Zeed<   dZeed<   dZeed<   dZeed<   dZeed<   d Zy)	ModelArgsphi
model_typei   max_position_embeddingsi   
vocab_sizei 
  hidden_size    num_attention_headsnum_hidden_layersnum_key_value_headsg?partial_rotary_factori (  intermediate_sizegh㈵>layer_norm_epsg     @
rope_thetac                 @    | j                   | j                  | _         y y N)r   r   selfs    [/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/phi.py__post_init__zModelArgs.__post_init__   s!    ##+'+'?'?D$ ,    N)__name__
__module____qualname__r   str__annotations__r   intr   r   r   r   r   r   floatr   r   r   r    r   r   r	   r	      s|    J#'S'JK!!s!!#&5&"s" NE J@r   r	   c                   ,     e Zd Zdef fdZddZ xZS )PhiAttentionconfigc                 Z   t         |           |j                  | _        |j                  | _        | j                  | j                  z  | _        |j                  | _        | j                  | j                  z  | _        |j                  | _        |j                  | _	        | j
                  | j                  z  | j                  k7  r&t        d| j                   d| j                   d      t        j                  | j                  | j                  | j
                  z  d      | _        t        j                  | j                  | j                  | j
                  z  d      | _        t        j                  | j                  | j                  | j
                  z  d      | _        t        j                  | j                  | j
                  z  | j                  d      | _        t        j"                  t%        | j                  | j
                  z        d| j                        | _        y )Nz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).TbiasF)traditionalbase)super__init__r   r   	num_headshead_dimr   repeatsr   r   
ValueErrornnLinearq_projk_projv_projdenseRoPEr#   roper   r(   	__class__s     r   r/   zPhiAttention.__init__    s   !--33((DNN:#)#=#= ~~)A)AA ++%+%A%A"MMDNN*t/?/??QRVRbRbQc$T^^$4B8 
 iidnnt}}<4
 iid66FT
 iid66FT
 YYNNT]]*D,<,<4

 GG**T]]:;
	r   c                    | j                  |      | j                  |      | j                  |      }}}|j                  \  }}}	| j                  | j
                  }}
|j                  |||
d      j                  dd      }|j                  |||d      j                  dd      }|j                  |||d      j                  dd      }|P| j                  ||j                        }| j                  ||j                        }|j                  ||      \  }}n"| j                  |      }| j                  |      }t        j                  d|j                  d   z        }t        |j                  t        j                         |||||      j                  |j"                        }|j                  dd      j                  ||d      }| j%                  |      S )Nr      )offset)cachescalemask)r6   r7   r8   shaper0   r   reshapemoveaxisr;   rA   update_and_fetchmathsqrtr   astypemxfloat32dtyper9   )r   xrD   rB   querieskeysvaluesBLDn_heads
n_kv_headsrC   outputs                 r   __call__zPhiAttention.__call__D   s    $AAAv --1a"nnd.F.F //	

 (1a. 	 ||Aq*b1::1a@1j"5>>q!D iii=G99T%,,97D 11$?LD&ii(G99T?D		!gmmB//0-NN2::&
 &
 	 A&..q!R8zz&!!r   )NNr   r   r    r	   r/   rY   __classcell__r=   s   @r   r'   r'      s    "
y "
H&"r   r'   c                   D     e Zd Zdef fdZdej                  fdZ xZS )PhiMLPr(   c                     t         |           t        j                  |j                  |j
                        | _        t        j                  |j
                  |j                        | _        y r   )r.   r/   r4   r5   r   r   fc1fc2r<   s     r   r/   zPhiMLP.__init__n   sJ    99V//1I1IJ99V55v7I7IJr   returnc                 h    | j                  t        j                  | j                  |                  S r   )ra   r4   gelu_approxr`   )r   rO   s     r   rY   zPhiMLP.__call__s   s"    xxtxx{344r   )	r   r   r    r	   r/   rL   arrayrY   r[   r\   s   @r   r^   r^   m   s!    Ky K
5RXX 5r   r^   c                   *     e Zd Zdef fdZd Z xZS )PhiDecoderLayerr(   c                     t         |           t        |      | _        t	        j
                  |j                  |j                        | _        t        |      | _
        y )N)r(   eps)r.   r/   r'   	self_attnr4   	LayerNormr   r   input_layernormr^   mlpr<   s     r   r/   zPhiDecoderLayer.__init__x   sI    %V4!||F$9$9 
 &>r   c                 |    | j                  |      }| j                  |||      }| j                  |      }||z   |z   S r   )rm   rk   rn   )r   rO   rD   rB   hattn_hff_hs          r   rY   zPhiDecoderLayer.__call__   s@      #4/xx{}q  r   rZ   r\   s   @r   rg   rg   w   s    "y "!r   rg   c                   *     e Zd Zdef fdZd Z xZS )PhiModelr(   c                 L   t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        |       c}| _
        t        j                  |j
                  |j                        | _        y c c}w )Nri   )r.   r/   r4   	Embeddingr   r   embed_tokensranger   rg   layersrl   r   final_layernorm)r   r(   ir=   s      r   r/   zPhiModel.__init__   s}    LL):):F<N<NO8=f>V>V8WX8W1v.8WX!||F$9$9 
 Ys   B!c                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rw   lenry   r   ziprz   )r   rO   rB   rD   layercs         r   rY   zPhiModel.__call__   st    a =FS--E$Qa1DKK/HE1aq!A 0##A&&r   rZ   r\   s   @r   rt   rt      s    
y 
	'r   rt   c                   p     e Zd Zdef fdZ	 ddej                  dej                  fdZed        Z	 xZ
S )Modelr(   c                     t         |           |j                  | _        t        |      | _        t        j                  |j                  |j                  d      | _	        || _
        y )NTr*   )r.   r/   r   rt   modelr4   r5   r   r   lm_headargsr<   s     r   r/   zModel.__init__   sN     ++f%
yy!3!3V5F5FTR	r   rO   rb   c                 H    | j                  ||      }| j                  |      S r   )r   r   )r   rO   rB   ys       r   rY   zModel.__call__   s"    
 JJq% ||Ar   c                 .    | j                   j                  S r   )r   ry   r   s    r   ry   zModel.layers   s    zz   r   r   )r   r   r    r	   r/   rL   re   rY   propertyry   r[   r\   s   @r   r   r      sG    y  88 
	 ! !r   r   )rI   dataclassesr   mlx.corecorerL   mlx.nnr4   r-   r   r   r   r	   Moduler'   r^   rg   rt   r   r%   r   r   <module>r      s     !   T T @ @ @$K"299 K"\5RYY 5!bii ! 'ryy '*!BII !r   