
    (HJj                     x   d dl Z d dlmZ d dlmZmZ d dlmZ d dl	m
Z
 ddlmZ ddlmZmZ 	 d dlZe G d d	e             Z G d
 de
j*                        Z G d de
j*                        Z G d de
j*                        Z G d de
j*                        Zy# e$ r  ed        e j&                  d       Y w xY w)    N)	dataclass)AnyOptional   )swiglu)BaseModelArgscreate_attention_maskz2To run olmo install ai2-olmo: pip install ai2-olmoc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZeed	<   d
Ze	ed<   dZ
eed<   d
Ze	ed<   d Zy)	ModelArgs
model_typed_modeln_layersmlp_hidden_sizen_heads
vocab_sizeembedding_sizei'  
rope_thetaFrope_traditional   	mlp_ratioweight_tyingc                 |    | j                   | j                   | _         y | j                  | j                  z  | _         y N)r   r   r   selfs    \/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/olmo.py__post_init__zModelArgs.__post_init__"   s=     ##/    	 $,,. 	    N)__name__
__module____qualname__str__annotations__intr   floatr   boolr   r   r    r   r   r   r      sU    OLMLOJ"d"IsL$
r   r   c            	            e Zd Zdef fdZ	 	 d	dej                  deej                     dee   dej                  fdZ		 	 d	dej                  deej                     dee   dej                  fdZ
 xZS )
TransformerBlockargsc                    t         |           |j                  | _        |j                  }t	        j
                  ||j                  d      | _        t	        j
                  |j                  dz  |d      | _        t	        j                  |d      | _
        t	        j                  |d      | _        || j                  z  }|dz  | _        t	        j
                  |d|z  d      | _        t	        j
                  ||d      | _        t	        j                  ||j                   |j"                        | _        || _        y )NFbias   affineg         )traditionalbase)super__init__r   r   nnLinearr   ff_projff_out	LayerNormatt_normff_normscaleatt_projattn_outRoPEr   r   roper*   )r   r*   dimhead_dim	__class__s       r   r5   zTransformerBlock.__init__+   s    ||llyyd&:&:Gii 4 4 93UKS7||C6$,,&t^
		#q3wU;		#s7GG--
	 	r   xmaskcachereturnc                    |j                   \  }}}t        j                  | j                  |      dd      \  }}}	|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j
                  d      j                  dddd      }|	j	                  ||| j
                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }|| j                  z  |j                  dddd      z  }
||
|z  }
t        j                  |
j                  t        j                        d      j                  |
j                        }
|
|	z  j                  dddd      j	                  ||d      }| j                  |      S )Nr1   axisr   r.   r   )offset)shapemxsplitr>   reshaper   	transposerA   rM   update_and_fetchr=   softmaxastypefloat32dtyper?   )r   rE   rF   rG   BLDquerieskeysvaluesscoresoutputs               r   attendzTransformerBlock.attendD   s    ''1a "q)912 Fv //!Qb9CCAq!QO||Aq$,,3==aAqI1dllB7AA!Q1Miii=G99T%,,97D 11$?LD&ii(G99T?DDJJ&$..Aq!*DDdNFFMM"**5B?FFv||T6/,,Q1a8@@ArJ}}V$$r   c                    | j                  | j                  |      ||      }||z   }t        j                  | j	                  | j                  |            dd      \  }}|| j                  t        ||            z   }|S )Nr.   rJ   rK   )r`   r;   rO   rP   r8   r<   r9   r   )	r   rE   rF   rG   rhx1x2outs	            r   __call__zTransformerBlock.__call__b   sm     KKa($6E$,,t||A7DB$++fRn--
r   )NN)r   r    r!   r   r5   rO   arrayr   r   r`   rg   __classcell__rD   s   @r   r)   r)   *   s    Y 8 $(#	%88% rxx % }	%
 
%B $(#	88 rxx  }	
 
r   r)   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )Transformerr*   c                    t         |           |j                  | _        |j                  | _        t	        j
                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        | j                  s1t	        j                  |j                  |j                  d      | _        t	        j                  |j                  d      | _        y c c}w )N)r*   Fr,   r/   )r4   r5   r   r   r6   	Embeddingr   r   wteranger)   blocksr7   r9   r:   norm)r   r*   _rD   s      r   r5   zTransformer.__init__r   s     --<< 3 3T\\B<A$--<PQ<Pq'T2<PQ  ))DLL$2E2EERDKLLe<	 Rs   8C7inputsc                 ^   | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      }| j                  r| j                   j                  |      |fS | j                  |      S )Nr   )	ro   lenrq   r	   ziprr   r   	as_linearr9   )r   rt   rG   rc   rF   blockcs          r   rg   zTransformer.__call__}   s    
 HHV=FS--E$Qa1DKK/HE1aq!A 0 IIaL88%%a(%//{{1~r   r   	r   r    r!   r   r5   rO   rh   rg   ri   rj   s   @r   rl   rl   q   s%    	=Y 	= r   rl   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )	OlmoModelr*   c                 B    t         |           t        |      | _        y r   )r4   r5   rl   transformerr   r*   rD   s     r   r5   zOlmoModel.__init__   s    &t,r   rt   c                 &    | j                  ||      S r   )r   r   rt   rG   s      r   rg   zOlmoModel.__call__   s    
 ..r   r   r{   rj   s   @r   r}   r}      s%    -Y - //r   r}   c                   X     e Zd Zdef fdZ	 ddej                  fdZed        Z	 xZ
S )Modelr*   c                 r    t         |           |j                  | _        t        |      | _        || _        y r   )r4   r5   r   r}   modelr*   r   s     r   r5   zModel.__init__   s+    //t_
	r   rt   c                 &    | j                  ||      S r   )r   r   s      r   rg   zModel.__call__   s    
 zz&%((r   c                 B    | j                   j                  j                  S r   )r   r   rq   r   s    r   layerszModel.layers   s    zz%%,,,r   r   )r   r    r!   r   r5   rO   rh   rg   propertyr   ri   rj   s   @r   r   r      s9    Y  )) - -r   r   )sysdataclassesr   typingr   r   mlx.corecorerO   mlx.nnr6   activationsr   r3   r   r	   hf_olmoImportErrorprintexitr   Moduler)   rl   r}   r   r'   r   r   <module>r      s     !      6 
 
 
*Dryy DN "))  F
/		 
/-BII -e  	
>?CHHQKs   B B98B9