
    (HJj                     r   d dl mZ d dlmZmZmZmZmZ d dlm	Z
 d dlmZ ddlmZ ddlmZmZmZ ddlmZmZ ddlmZ e G d	 d
e             Z G d dej2                        Z G d dej2                        Z G d dej2                        Z G d dej2                        Z G d dej2                        Zy)    )	dataclass)AnyDictListOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)KVCacheRotatingKVCache)initialize_ropec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   dZeed<   dZ	e
ee      ed<   dZe
e   ed<   dZe
e   ed<   dZe
eeeeef   f      ed<   dZeed<   d Zy)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizemax_position_embeddingssliding_window
rope_thetaFattention_biasNlayer_typesnum_key_value_headshead_dimrope_scalingtie_word_embeddingsc                     | j                   | j                  | _         | j                  6t        | j                        D cg c]  }|dz   dz  dk(  rdnd c}| _        y y c c}w )Nr	      r   full_attentionsliding_attention)r   r   r   ranger   )selfis     ]/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/olmo3.py__post_init__zModelArgs.__post_init__"   su    ##+'+'?'?D$# t556 6A &'UaK1$4 :MM6 D $ s   A!)__name__
__module____qualname__str__annotations__intfloatr   boolr   r   r   r   r    r!   r   r   r"   r+        r*   r   r      s    OO   ND '+K$s)$+)-#-"Hhsm";?L(4U5#:%6 678? %%r5   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
Olmo3Attentionargs	layer_idxc                 *   t         |           |j                  | _        |j                  | _        || _        |j
                  xs |j                  |j                  z  | _        | j
                  dz  | _        t        j                  |j                  |j                  | j
                  z  |j                        | _        t        j                  |j                  |j                  | j
                  z  |j                        | _        t        j                  |j                  |j                  | j
                  z  |j                        | _        t        j                  |j                  | j
                  z  |j                  |j                        | _        t        j                  |j                  | j
                  z  |j                         | _        t        j                  |j                  | j
                  z  |j                         | _        |j&                  |   dk7  r2t        j(                  | j
                  d|j*                        | _        y t/        | j
                  d|j*                  |j0                  |j2                        | _        y )Ng      ࿩biasepsr%   F)traditionalbase)r?   r@   scaling_configr   )super__init__r   r   r9   r    r   scalennLinearr   q_projk_projv_projo_projRMSNormr   q_normk_normr   RoPEr   roper   r!   r   r(   r8   r9   	__class__s      r*   rC   zOlmo3Attention.__init__-   s   #'#;#; #'#;#; "U)9)9T=U=U)U]]D(
ii$$t}}4$$

 ii$$t}}4$$

 ii$$t}}4$$

 ii$$t}}4$$
 jj$$t}}4$:K:K
 jj$$t}}4$:K:K
 I&*::5tWDI'!__#00(,(D(DDIr5   xmaskcachereturnc                 n   |j                   \  }}}| j                  | j                  |            }| j                  | j	                  |            }| j                  |      }	|j                  ||| j                  d      j                  dddd      }|j                  ||| j                  d      j                  dddd      }|	j                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j                  ||d      }
| j                  |
      S )Nr      r	      )offset)rT   rD   rS   )shaperL   rG   rM   rH   rI   reshaper   	transposer   rO   rZ   update_and_fetchr   rD   rJ   )r(   rR   rS   rT   BL_querieskeysvaluesoutputs              r*   __call__zOlmo3Attention.__call__]   s    ''1a++dkk!n-{{4;;q>*Q//!Q(@(@"EOOq!Q
 ||Aq$":":B?II!QPQSTU1d&>&>CMMq!Q
 iii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r5   NNr,   r-   r.   r   r1   rC   mxarrayr   r   rf   __classcell__rQ   s   @r*   r7   r7   ,   s_    .Y .3 .f $(#	#88# rxx # }	#
 
#r5   r7   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )Olmo3MLPr8   c                 H   t         |           t        j                  |j                  |j
                  d      | _        t        j                  |j
                  |j                  d      | _        t        j                  |j                  |j
                  d      | _        y NFr;   )	rB   rC   rE   rF   r   r   	gate_proj	down_projup_projr(   r8   rQ   s     r*   rC   zOlmo3MLP.__init__   sq    4#3#3T5K5KRWX4#9#94;K;KRWXyy!1!143I3IPUVr5   rR   rU   c                 t    | j                  t        | j                  |      | j                  |                  S N)rr   r
   rq   rs   )r(   rR   s     r*   rf   zOlmo3MLP.__call__   s)    ~~fT^^A%6QHIIr5   )	r,   r-   r.   r   rC   ri   rj   rf   rk   rl   s   @r*   rn   rn      s/    WY WJ"(( Jrxx Jr5   rn   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
Olmo3DecoderLayerr8   r9   c                 x   t         |           |j                  | _        |j                  | _        t	        ||      | _        t        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        || _        y )N)r9   r=   )rB   rC   r   r   r7   	self_attnrn   mlprE   rK   r   post_attention_layernormpost_feedforward_layernormr8   rP   s      r*   rC   zOlmo3DecoderLayer.__init__   s    #'#;#; ++'	BD>(*

$"3"3)
% +-**$"3"3+
' 	r5   rR   rS   rT   rU   c                     | j                  | j                  |||            }||z   }| j                  | j                  |            }||z   }|S rv   )r|   rz   r}   r{   )r(   rR   rS   rT   rhouts          r*   rf   zOlmo3DecoderLayer.__call__   sP     ))$..D%*HIE++DHHQK8!e
r5   rg   rh   rl   s   @r*   rx   rx      s^    Y 3 " $(#	
88
 rxx 
 }	

 

r5   rx   c                   j     e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	 xZ
S )
Olmo3Modelr8   c                    t         |           |j                  | _        t        j                  |j
                  |j                        | _        t        |j                        D cg c]  }t        ||       c}| _        t        j                  |j                  |j                        | _        |j                  j!                  d      | _        |j                  j!                  d      | _        |j                  | _        y c c}w )N)r8   r9   r=   r&   r%   )rB   rC   r   rE   	Embeddingr   r   embed_tokensr'   r   rx   layersrK   r   normr   indexswa_idxga_idx)r(   r8   r)   rQ   s      r*   rC   zOlmo3Model.__init__   s    "11LL$:J:JK 4112
2 4152
 JJt//T5F5FG	''--.AB&&,,-=>++
s   'DinputsrT   rU   c                 t   | j                  |      }|d gt        | j                        z  }t        ||| j                           }t        ||| j
                     | j                        }t        | j                  || j                        D ]  \  }}}|dk(  r|n|}	 |||	|      } | j                  |      S )N)window_sizer%   )rT   )
r   lenr   r   r   r   r   zipr   r   )
r(   r   rT   r   	full_masksliding_window_masklayerc
layer_typerS   s
             r*   rf   zOlmo3Model.__call__   s    
 f%=FS--E)!U4;;-?@	3uT\\"0C0C
 %(UD<L<L$M E1j *.> >9DWDaQ'A %N yy|r5   rv   )r,   r-   r.   r   rC   ri   rj   r   r   rf   rk   rl   s   @r*   r   r      sA    ,Y ,$  $ } 
	r5   r   c                        e Zd Zdef fdZ	 d	dej                  dee   dej                  fdZ	e
d        Zd Z xZS )
Modelr8   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y rp   )rB   rC   r8   r   r   modelr"   rE   rF   r   r   lm_headrt   s     r*   rC   zModel.__init__   sX    	//%
''99T%5%5tUSDL (r5   r   rT   rU   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S rv   )r   r8   r"   r   	as_linearr   )r(   r   rT   r   s       r*   rf   zModel.__call__   sW    
 jj'99((**))33C8C 
 ,,s#C
r5   c                 .    | j                   j                  S rv   )r   r   )r(   s    r*   r   zModel.layers   s    zz   r5   c                     g }| j                   j                  D ]P  }|dk(  r|j                  t                      "|j                  t	        | j
                  j                               R |S )Nr%   )max_size)r   r   appendr   r   r8   r   )r(   cacheslts      r*   
make_cachezModel.make_cache   sT    **((B%%gi(otyy7O7OPQ	 )
 r5   rv   )r,   r-   r.   r   rC   ri   rj   r   r   rf   propertyr   r   rk   rl   s   @r*   r   r      s\    TY T  $

 }
 
	
 ! !r5   r   )dataclassesr   typingr   r   r   r   r   mlx.corecoreri   mlx.nnrE   activationsr
   r@   r   r   r   rT   r   r   
rope_utilsr   r   Moduler7   rn   rx   r   r   r4   r5   r*   <module>r      s    " 3 3    T T + '   8P#RYY P#fJryy J		 8# #L BII  r5   