
    (HJj[                     Z   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZ ddlmZmZmZ ddlmZmZ e G d d	e             Z G d
 de	j(                        Z G d de	j(                        Z G d de	j(                        Z G d de	j(                        Z G d de	j(                        Zy)    )	dataclass)OptionalTupleN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)KVCacheRotatingKVCachec                       e Zd ZU eed<   dZeed<   dZeed<   dZeed<   dZ	eed	<   dZ
eed
<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   y)	ModelArgs
model_typei   hidden_size   head_dim    num_hidden_layersi 8  intermediate_sizenum_attention_heads   num_key_value_headsg     j@
rope_thetai  
vocab_sizegh㈵>layer_norm_epsg      ?logit_scaleFattention_biaslayer_norm_biassliding_window   sliding_window_patternN)__name__
__module____qualname__str__annotations__r   intr   r   r   r   r   r   floatr   r   r   r   boolr   r   r!        _/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/cohere2.pyr   r      s    OKHcs"s"!!  JJ!NE!K ND !OT!NC"#C#r+   r   c            
            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	ej                  ej                  f      dej                  fdZ
 xZS )
	Attentionargs	layer_idxc                    t         |           || _        || _        |j                  }|j
                  x| _        }|j                  x| _        }|j                  x| _	        }||z  |k7  rt        d| d| d      |dz  | _        |j                  }t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  ||z  ||      | _        t        j&                  |d|j(                        | _        |dz   |j,                  z  d	k7  | _        y )
Nz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).g      ࿩biasT)traditionalbaser   r   )super__init__r/   r0   r   r   n_headsr   
n_kv_headsr   
ValueErrorscaler   nnLinearq_projk_projv_projo_projRoPEr   roper!   use_sliding_window)	selfr/   r0   dimr8   r9   r   attetion_bias	__class__s	           r,   r7   zAttention.__init__"   s=   	"!%!9!99w'+'?'??*#'==0w3&QRUQV$WIR1  t^
++iiWx%7mLiiZ(%:OiiZ(%:Oii( 2CmLGGH$T__M	#,q=D4O4O"OST"Tr+   xmaskcachereturnc                    |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	| j                  r_|#| j                  |      }| j                  |      }n:| j                  ||j                        }| j                  ||j                        }||j                  ||	      \  }}	|j                  t        j                  k(  rt        j                  n|j                  }
t!        |j#                  |
      ||	|| j$                  |      j#                  |j                        }|j                  dddd      j	                  ||d      }| j'                  |      S )Nr      r      )offset)rK   r;   rJ   )shaper>   r?   r@   reshaper8   	transposer9   rD   rC   rQ   update_and_fetchdtypemxfloat16float32r
   astyper;   rA   )rE   rI   rJ   rK   BLDquerieskeysvalues	sdpa_typeoutputs               r,   __call__zAttention.__call__=   s    ''1a $AAAv//!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aP ""}))G,yy))GELL)Ayyelly; 11$?LD& #*--2::"=BJJ7==	-NN9%**
 &
 	 !!!Q1-55aB?{{6""r+   NNr"   r#   r$   r   r'   r7   rW   arrayr   r   rc   __classcell__rH   s   @r,   r.   r.   !   st    UY U3 U< $(59	&#88&# rxx &# bhh012	&#
 
&#r+   r.   c                   $     e Zd Z fdZd Z xZS )MLPc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y )NFr2   )r6   r7   r<   r=   	gate_projup_proj	down_proj)rE   rF   
hidden_dimrH   s      r,   r7   zMLP.__init__g   sJ    3
?yyju=:s?r+   c                 t    | j                  t        | j                  |      | j                  |                  S N)rn   r   rl   rm   )rE   rI   s     r,   rc   zMLP.__call__m   s)    ~~fT^^A%6QHIIr+   )r"   r#   r$   r7   rc   rg   rh   s   @r,   rj   rj   f   s    @Jr+   rj   c            
            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	ej                  ej                  f      dej                  fdZ
 xZS )
TransformerBlockr/   r0   c                 V   t         |           |j                  | _        |j                  | _        t        ||      | _        t        |j                  |j                        | _	        t        j                  |j                  |j                  |j                        | _        || _        y )Nepsr3   )r6   r7   r   r   r8   r.   	self_attnrj   r   mlpr<   	LayerNormr   r   input_layernormr/   )rE   r/   r0   rH   s      r,   r7   zTransformerBlock.__init__r   s    ++//"43t'')?)?@!||$"5"5D<P<P 
 	r+   rI   rJ   rK   rL   c                 |    | j                  |      }| j                  |||      }| j                  |      }||z   |z   S rq   )rz   rw   rx   )rE   rI   rJ   rK   hattn_hff_hs          r,   rc   zTransformerBlock.__call__~   sB       #4/xx{}q  r+   rd   re   rh   s   @r,   rs   rs   q   sq    
Y 
3 
 $(59	!88! rxx ! bhh012	!
 
!r+   rs   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )CohereModelr/   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ |j
                  | _        t        j                  |j                  |j                        | _
        t        |j                        D cg c]  }t        ||       c}| _        t        j                  |j                  |j                  |j                         | _        y c c}w )Nr   )r/   r0   ru   )r6   r7   r/   r   r   r   window_sizer<   	Embeddingr   embed_tokensrangers   layersry   r   r   norm)rE   r/   irH   s      r,   r7   zCohereModel.__init__   s    	//!%!7!7"""..LL$:J:JK 4112
2 $!42
 LL$"5"5D<P<P
		
s   !C9inputsc                    | j                  |      }|d gt        | j                        z  }| j                  j                  }t        |||dz
           }t        ||d   | j                        }t        t        | j                  |            D ]K  \  }\  }}	|| j                  j                  z  | j                  j                  dz
  k(  }
|
r|n|} ||||	      }M | j                  |      S )Nr   r   )r   )
r   lenr   r/   r!   r	   r   	enumeratezipr   )rE   r   rK   r|   j	full_maskswa_maskr   layerc	is_globalrJ   s               r,   rc   zCohereModel.__call__   s    
 f%=FS--EII,,)!U1q5\:	(E!H$BRBRS&s4;;'>?MAzqDII4449933a78 
 !*9xDaq!A @ yy|r+   rq   )	r"   r#   r$   r   r7   rW   rf   rc   rg   rh   s   @r,   r   r      s%    
Y 
& r+   r   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )Modelr/   c                 r    t         |           |j                  | _        t        |      | _        || _        y rq   )r6   r7   r   r   modelr/   )rE   r/   rH   s     r,   r7   zModel.__init__   s,    // &
	r+   r   c                     | j                  ||      }| j                   j                  j                  |      }|| j                   j                  j                  z  }|S rq   )r   r   	as_linearr/   r   )rE   r   rK   outs       r,   rc   zModel.__call__   sK    
 jj'jj%%//4DJJOO///
r+   c                 L   g }t        | j                  j                        D ]  }|| j                  j                  z  | j                  j                  dz
  k(  r|j	                  t                      P|j	                  t        | j                  j                  d              |S )Nr   r   )max_sizekeep)r   r/   r   r!   appendr   r   r   )rE   cachesr   s      r,   
make_cachezModel.make_cache   s    tyy223ADII4449933a78 gi(#TYY-E-EAN 4 r+   c                 .    | j                   j                  S rq   )r   r   )rE   s    r,   r   zModel.layers   s    zz   r+   rq   )r"   r#   r$   r   r7   rW   rf   rc   r   propertyr   rg   rh   s   @r,   r   r      s>    Y   ! !r+   r   )dataclassesr   typingr   r   mlx.corecorerW   mlx.nnr<   activationsr   r5   r   r	   r
   rK   r   r   r   Moduler.   rj   rs   r   r   r*   r+   r,   <module>r      s    " "    T T + $ $ $$B#		 B#JJ")) J!ryy !6)")) )X!!BII !!r+   