
    (HJj-                     >   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZmZmZ e G d de             Z G d d	e	j                        Z G d
 de	j                        Z G d de	j                        Z G d de	j                        Z G d de	j                        Zy)    )	dataclass)AnyOptionalN   )BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zeed<   dZ	eed<   d Z
y)	ModelArgs
model_typemax_position_embeddingshidden_sizenum_attention_headsnum_hidden_layerslayer_norm_eps
vocab_sizerotary_emb_base
rotary_pctTuse_parallel_residualNnum_key_value_headsc                 @    | j                   | j                  | _         y y N)r   r   selfs    `/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/gpt_neox.py__post_init__zModelArgs.__post_init__   s!    ##+'+'?'?D$ ,    )__name__
__module____qualname__str__annotations__intfloatr   boolr   r    r   r   r   r      sR    O  O"&4&##@r   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                 h   t         |           |j                  |j                  z  dk(  sJ d       |j                  | _        |j                  | _        | j                  | j                  z  | _        t        j                  t        | j                  |j                  z        d|j                        | _
        | j                  dz  | _        t        j                  | j                  d| j                  z  d      | _        t        j                  | j                  | j                  d      | _        y )	Nr   z4hidden_size must be divisible by num_attention_headsF)dimstraditionalbaseg         Tbias)super__init__r   r   head_dimnnRoPEr#   r   r   ropescaleLinearquery_key_valuedenser   r)   	__class__s     r   r2   zAttention.__init__#   s     t7771<	BA	B<  ++#'#;#; ((D,D,DDGGT]]T__45%%
	 ]]D(
!yya$"2"22 
 YYt//1A1AM
r   xmaskcachereturnc           	         |j                   \  }}}| j                  |      }|j                   d d | j                  d| j                  z  fz   } |j                  | }|j                  dd      D cg c]  }|j                  dddd       c}\  }	}
}|P| j                  |	|j                        }	| j                  |
|j                        }
|j                  |
|      \  }
}n"| j                  |	      }	| j                  |
      }
t        |	|
||| j                  |      }|j                  dddd      j	                  ||d      }| j                  |      S c c}w )Nr.   r      r   )offset)r?   r7   r>   )shaper9   r   r3   reshapesplit	transposer6   rD   update_and_fetchr	   r7   r:   )r   r=   r>   r?   BLDqkvnew_qkv_shapequerieskeysvaluesoutputs                r   __call__zAttention.__call__;   sL    ''1a""1%		#2$*B*BADU)VVckk=)BE))ArBR SBRQQ1a!8BR Sviii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?zz&!! !Ts   0ENNr   r   r    r   r2   mxarrayr   r   rS   __classcell__r<   s   @r   r(   r(   "   sY    NY N6 $(#	"88" rxx " }	"
 
"r   r(   c                   D     e Zd Zdef fdZdej                  fdZ xZS )MLPr)   c                    t         |           |j                  | _        t        j                  | j                  d| j                  z        | _        t        j                  d| j                  z  | j                        | _        y )N   )r1   r2   r   r4   r8   dense_h_to_4hdense_4h_to_hr;   s     r   r2   zMLP.__init__[   sd    ++YYt'7'7T=M=M9MNYYq4+;+;';T=M=MNr   r@   c                 h    | j                  t        j                  | j                  |                  S r   )r_   r4   gelu_approxr^   )r   r=   s     r   rS   zMLP.__call__b   s'    !!"..1C1CA1F"GHHr   	r   r   r    r   r2   rV   rW   rS   rX   rY   s   @r   r[   r[   Z   s#    OY OIRXX Ir   r[   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr)   c                    t         |           |j                  | _        |j                  | _        t	        |      | _        t        |      | _        |j                  | _        t        j                  | j                  | j                        | _        t        j                  | j                  | j                        | _        y )Neps)r1   r2   r   r   r(   	attentionr[   mlpr   r4   	LayerNorminput_layernormpost_attention_layernormr;   s     r   r2   zTransformerBlock.__init__h   s    ++"11"4t9%)%?%?"!||## 
 )+$"5"5)
%r   r=   r>   r?   r@   c                 R   | j                   rN|}| j                  | j                  |      ||      }| j                  | j	                  |            }||z   |z   }|S | j                  | j                  |      ||      }||z   }| j                  | j	                  |            }	||	z   }|S r   )r   rh   rk   ri   rl   )
r   r=   r>   r?   residualattnffnoutattn_output
ffn_outputs
             r   rS   zTransformerBlock.__call__x   s     %%H>>$"6"6q"94GD((488;<C*x'CJ ..)=)=a)@$NKKA$"?"?"BCJJAHr   rT   rU   rY   s   @r   rd   rd   g   sW    
Y 
& $(#	88 rxx  }	
 
r   rd   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )GPTNeoXModelr)   c                 Z   t         |           |j                  | _        |j                  | _        |j                  | _        |j
                  | _        | j                  dkD  sJ t        j                  | j                  | j                        | _        t        j                  |j                  |j                  d      | _
        t        | j                        D cg c]  }t        |       c}| _        t        j                  | j                  | j
                        | _        y c c}w )Nr   Fr/   )r)   rf   )r1   r2   r   r   r   r   r4   	Embeddingembed_inr8   	embed_outrangerd   hrj   final_layer_norm)r   r)   _r<   s      r   r2   zGPTNeoXModel.__init__   s    ++//!%!7!7"11"""T__d6F6FG4#3#3T__5Q7<T=S=S7TU7T!"-7TU "T-=-=4CVCV W Vs   D(inputsc                 0   |j                   \  }}| j                  |      }|d gt        | j                        z  }t	        ||d         }t        | j                  |      D ]  \  }} ||||      } | j                  |      }	| j                  |	      }	|	S )Nr   )r?   )rE   rx   lenr{   r   zipr|   ry   )
r   r~   r?   r}   rK   hidden_statesr>   layercrq   s
             r   rS   zGPTNeoXModel.__call__   s    
 ||1f-=FS[(E$]E!H=DFFE*HE1!-Q?M + ##M2nnS!
r   r   rb   rY   s   @r   ru   ru      s'    
XY 
X r   ru   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )Modelr)   c                 r    t         |           || _        |j                  | _        t	        |      | _        y r   )r1   r2   r)   r   ru   modelr;   s     r   r2   zModel.__init__   s,    	//!$'
r   r~   c                 *    | j                  ||      }|S r   )r   )r   r~   r?   rq   s       r   rS   zModel.__call__   s    
 jj'
r   c                    i }|j                         D ]i  \  }}g d}d}|D ]  }|j                  |      sd} n |r+|j                  d      sd| }|j                  dd      }|j                  dd      }|||<   k |S )	N)z.attention.biasz.attention.masked_biasz.attention.rotary_emb.inv_freqFTzmodel.z.gpt_neox.layers.z.h.z
.gpt_neox..)itemsendswith
startswithreplace)r   weightsnew_weightsw_keyw_valueignore_suffixesskip_weightignored_suffixs           r   sanitizezModel.sanitize   s    %mmoNE7O  K"1>>.1"&K #2
 ##H- (MM"5u=EMM,4E!(K/ .2 r   c                 .    | j                   j                  S r   )r   r{   r   s    r   layerszModel.layers   s    zz||r   r   )r   r   r    r   r2   rV   rW   rS   r   propertyr   rX   rY   s   @r   r   r      s>    (Y ( <  r   r   )dataclassesr   typingr   r   mlx.corecorerV   mlx.nnr4   r-   r   r   r	   r   Moduler(   r[   rd   ru   r   r&   r   r   <module>r      s    "     T T @ @ @$5"		 5"p
I")) 
I$ryy $N!299 !H/BII /r   