
    (HJj                     ^   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ ddlmZ e G d d	e             Z G d
 dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Zy)    )	dataclass)AnyDictOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_ropec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zee   ed<   dZ	e
ed<   dZe
ed<   dZe
ed<   dZeed<   dZe
ed<   d
Zeeeeeef   f      ed<   dZe
ed<   y
)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizenum_key_value_headshead_dimNmax_position_embeddingsFattention_biasattention_out_biasmlp_biasi'  
rope_thetarope_traditionalrope_scalingTtie_word_embeddings)__name__
__module____qualname__str__annotations__intfloatr   r   r   boolr   r   r   r   r   r   r   r         `/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/seed_oss.pyr   r      s    OOM-1Xc]1 ND $$HdJ"d";?L(4U5#:%6 678? $$r*   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                 |   t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  x| _        }|dz  | _        |j                  }|j                  }t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  ||z  ||      | _        t#        | j                  |j$                  |j&                  |j(                  |j*                        | _        y )Ng      ࿩bias)super__init__r   r   n_headsr   
n_kv_headsr   scaler   r   nnLinearq_projk_projv_projo_projr   r   r   r   r   rope)	selfr.   dimr4   r5   r   
input_biasoutput_bias	__class__s	           r+   r3   zAttention.__init__$   s   !%!9!99w'+'?'??*#'==0t^
((
--iiWx%7jIiiZ(%:LiiZ(%:Lii( 2CkJ#MMOO!!((
	r*   xmaskcachereturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr      r      )offset)rE   r6   rD   )shaper9   r:   r;   reshaper4   	transposer5   r=   rK   update_and_fetchr   r6   r<   )r>   rC   rD   rE   BLDquerieskeysvaluesoutputs              r+   __call__zAttention.__call__>   si    ''1a $AAAv //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r*   NNr!   r"   r#   r   r3   mxarrayr   r   rW   __classcell__rB   s   @r+   r-   r-   #   sW    
Y 
: $(#	#88# rxx # }	#
 
#r*   r-   c                   @     e Zd Zd fd	Zdej
                  fdZ xZS )MLPc                     t         |           t        j                  |||      | _        t        j                  |||      | _        t        j                  |||      | _        y )Nr0   )r2   r3   r7   r8   	gate_proj	down_projup_proj)r>   r?   
hidden_dimr1   rB   s       r+   r3   zMLP.__init__]   sJ    3
>:s>yyjt<r*   rF   c                 t    | j                  t        | j                  |      | j                  |                  S N)rb   r	   ra   rc   )r>   rC   s     r+   rW   zMLP.__call__c   s)    ~~fT^^A%6QHIIr*   )F)r!   r"   r#   r3   rZ   r[   rW   r\   r]   s   @r+   r_   r_   \   s    =JRXX Jr*   r_   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr.   c                    t         |           |j                  | _        |j                  | _        t	        |      | _        t        |j                  |j                  |j                        | _	        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y )Neps)r2   r3   r   r   r-   	self_attnr_   r   r   mlpr7   RMSNormr   input_layernormpost_attention_layernormr>   r.   rB   s     r+   r3   zTransformerBlock.__init__h   s    #'#;#; ++"4t'')?)?O!zz$*:*:@Q@QR(*

$"3"3)
%r*   rC   rD   rE   rF   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S rf   )rl   ro   rm   rp   )r>   rC   rD   rE   rhouts          r+   rW   zTransformerBlock.__call__s   sQ     NN4//2D%@EHHT22156!e
r*   rX   rY   r]   s   @r+   rh   rh   g   sW    	
Y 	
 $(#	
88
 rxx 
 }	

 

r*   rh   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )	SeedModelr.   c                 N   t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        |       c}| _
        t        j                  |j
                  |j                        | _        y c c}w )N)r.   rj   )r2   r3   r7   	Embeddingr   r   embed_tokensranger   rh   layersrn   r   norm)r>   r.   _rB   s      r+   r3   zSeedModel.__init__   s}    LL$:J:JK16t7M7M1N
1NA$'1N
 JJt//T5F5FG	
s   B"inputsc                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   rE   )rz   lenr|   r   zipr}   )r>   r   rE   rt   rD   layercs          r+   rW   zSeedModel.__call__   su    
 f%=FS--E$Qa1DKK/HE1aQ'A 0 yy|r*   rf   )	r!   r"   r#   r   r3   rZ   r[   rW   r\   r]   s   @r+   rw   rw      s'    HY H r*   rw   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )Modelr.   c                    t         |           || _        |j                  | _        t	        |      | _        |j                  | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFr0   )r2   r3   r.   r   rw   modelr    r7   r8   r   r   lm_headrq   s     r+   r3   zModel.__init__   se    	//t_
#'#;#; ''99T%5%5tUSDL (r*   r   c                     | j                  ||      }| j                  r-|| j                   j                  j                  j                  z  S | j                  |      S )Nr   )r   r    rz   weightTr   )r>   r   rE   rt   s       r+   rW   zModel.__call__   sN    
 JJvUJ+##tzz..557777<<?"r*   c                 B    | j                   r|j                  dd        |S )Nzlm_head.weight)r    pop)r>   weightss     r+   sanitizezModel.sanitize   s    ##KK($/r*   c                 .    | j                   j                  S rf   )r   r|   )r>   s    r+   r|   zModel.layers   s    zz   r*   rf   )r!   r"   r#   r   r3   rZ   r[   rW   r   propertyr|   r\   r]   s   @r+   r   r      s@    TY T 	#	# ! !r*   r   )dataclassesr   typingr   r   r   r   mlx.corecorerZ   mlx.nnr7   activationsr	   baser
   r   r   
rope_utilsr   r   Moduler-   r_   rh   rw   r   r)   r*   r+   <module>r      s    " - -    T T ' % % %(6#		 6#rJ")) Jryy 2		 6!BII !r*   