
    (HJj                     >   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZmZmZ e G d de             Z G d d	e	j                        Z G d
 de	j                        Z G d de	j                        Z G d de	j                        Z G d de	j                        Zy)    )	dataclass)AnyOptionalN   )BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                   r    e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   d	Zeed
<   d Zy	)	ModelArgs
model_typen_ctxn_embdn_headn_layern_positionslayer_norm_epsilon
vocab_sizeNnum_key_value_headsc                 @    | j                   | j                  | _         y y N)r   r   selfs    \/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/gpt2.py__post_init__zModelArgs.__post_init__   s    ##+'+{{D$ ,    )	__name__
__module____qualname__str__annotations__intfloatr   r    r   r   r   r      s<    OJKKLO##3r   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                    t         |           |j                  |j                  z  dk(  sJ d       |j                  | _        |j                  | _        | j                  | j                  z  | _        | j                  dz  | _        t        j                  | j                  d| j                  z  d      | _        t        j                  | j                  | j                  d      | _	        y )Nr   z"n_embd must be divisible by n_headg         T)bias)
super__init__r   r   head_dimscalennLinearc_attnc_projr   r&   	__class__s     r   r+   zAttention.__init__   s    {{T[[(A-S/SS-kkkkt{{2]]D(
iiQ_4HiiT[[tDr   xmaskcachereturnc                 l   |j                   \  }}}| j                  |      }t        j                  |dd      \  }}	}
|j	                  ||| j
                  d      j                  dddd      }|	j	                  ||| j
                  d      j                  dddd      }	|
j	                  ||| j
                  d      j                  dddd      }
||j                  |	|
      \  }	}
t        ||	|
|| j                  |      }|j                  dddd      j	                  ||d      }| j                  |      S )Nr(   )axisr      r   )r6   r-   r5   )shaper0   mxsplitreshaper   	transposeupdate_and_fetchr	   r-   r1   )r   r4   r5   r6   BLDqkvquerieskeysvaluesoutputs               r   __call__zAttention.__call__,   s#    ''1akk!n "ab 9v //!QR8BB1aAN||Aq$++r2<<Q1aH1dkk26@@Aq!L 11$?LD&-T6djjt
 !!!Q1-55aB?{{6""r   NNr   r   r   r   r+   r=   arrayr   r   rJ   __classcell__r3   s   @r   r%   r%      sY    EY E" $(#	#88# rxx # }	#
 
#r   r%   c                   D     e Zd Zdef fdZdej                  fdZ xZS )MLPr&   c                    t         |           |j                  | _        t        j                  | j                  d| j                  z        | _        t        j                  d| j                  z  | j                        | _        y )N   )r*   r+   r   r.   r/   c_fcr1   r2   s     r   r+   zMLP.__init__H   sU    kkIIdkk1t{{?;	iiDKK=r   r7   c                 h    | j                  t        j                  | j                  |                  S r   )r1   r.   gelu_approxrT   )r   r4   s     r   rJ   zMLP.__call__O   s"    {{2>>$))A,788r   	r   r   r   r   r+   r=   rM   rJ   rN   rO   s   @r   rQ   rQ   G   s    >Y >9RXX 9r   rQ   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr&   c                    t         |           |j                  | _        |j                  | _        |j                  | _        t        |      | _        t        |      | _        t        j                  | j                  | j                        | _        t        j                  | j                  | j                        | _        y )Neps)r*   r+   r   r   r   r%   attnrQ   mlpr.   	LayerNormln_1ln_2r2   s     r   r+   zTransformerBlock.__init__T   s    kkkk"&"9"9dO	t9LLKK''
	 LL$2I2IJ	r   r4   r5   r6   r7   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S r   )r]   r`   r^   ra   )r   r4   r5   r6   rhouts          r   rJ   zTransformerBlock.__call__b   sK     IIdiilD%0EHHTYYq\"!e
r   rK   rL   rO   s   @r   rY   rY   S   sY    KY K" $(#	
88
 rxx 
 }	

 

r   rY   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )	GPT2Modelr&   c                 x   t         |           |j                  | _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        | j                  dkD  sJ t        j                  | j                  | j                        | _	        t        j                  | j                  | j                        | _
        t        | j
                        D cg c]  }t        |       c}| _        t        j                  | j                  | j                        | _        y c c}w )Nr   )r&   r[   )r*   r+   r   r   r   r   r   r.   	EmbeddingwtewperangerY   rd   r_   ln_f)r   r&   _r3   s      r   r+   zGPT2Model.__init__p   s    kk++//||"&"9"9"""<<=<< 0 0$++>7<T\\7JK7J!"-7JKLL$2I2IJ	 Ls   +D7inputsc                    |j                   \  }}| j                  |      }|d gt        | j                        z  }d}|d   |d   j                  }t        j                  |      }t        j                  |      |d   z   }|| j                  |      z  }t        ||d         }t        | j                  |      D ]  \  }	}
 |	|||
      } | j                  |      S )Nr   ).N)r6   )r<   rj   lenrd   offsetr=   rM   arangerk   r   ziprm   )r   ro   r6   rn   rC   hidden_statesrr   position_idsr5   layercs              r   rJ   zGPT2Model.__call__}   s    
 ||1(=FS[(E81X__F&!yy|fY&77,//$]E!H=DFFE*HE1!-Q?M + yy''r   r   rW   rO   s   @r   rg   rg   o   s'    KY K  ((r   rg   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )Modelr&   c                 r    t         |           || _        |j                  | _        t	        |      | _        y r   )r*   r+   r&   r   rg   modelr2   s     r   r+   zModel.__init__   s+    	//t_
r   ro   c                 t    | j                  ||      }| j                   j                  j                  |      }|S r   )r|   rj   	as_linear)r   ro   r6   re   s       r   rJ   zModel.__call__   s1    
 jj'jjnn&&s+
r   c                 
   i }t        | j                  j                        D ]  }d| d|v r|d| d= d| d|v r |d| d   j                  dd      |d| d<   d| d|v r |d| d   j                  dd      |d| d<   d| d|v r |d| d   j                  dd      |d| d<   d| d|v s|d| d   j                  dd      |d| d<    |D ]'  }|j	                  d	      s||   |d	| <    ||   ||<   ) |S )
Nzh.z
.attn.biasz.attn.c_attn.weightr   r   z.attn.c_proj.weightz.mlp.c_fc.weightz.mlp.c_proj.weightzmodel.)rl   r&   r   r@   
startswith)r   weightsnew_weightsiweights        r   sanitizezModel.sanitize   s   tyy(()AA3j!W,b:./A3)*g57>./8)Aq/ "QC234 A3)*g57>./8)Aq/ "QC234 A3&'724;+,5)Aq/ "QC/01 A3()W46=-.7)Aq/ "QC123! *& F$$X.18fVH-.&-foF#	 
 r   c                 .    | j                   j                  S r   )r|   rd   r   s    r   layerszModel.layers   s    zz||r   r   )r   r   r   r   r+   r=   rM   rJ   r   propertyr   rN   rO   s   @r   rz   rz      s>    %Y % 8  r   rz   )dataclassesr   typingr   r   mlx.corecorer=   mlx.nnr.   baser   r   r	   r   Moduler%   rQ   rY   rg   rz   r#   r   r   <module>r      s    "     T T 3 3 3 '#		 '#T	9")) 	9ryy 8((		 ((V.BII .r   