
    (HJj                     x    d dl Z d dlmZ d dlmZ  G d dej                        Z G d dej                        Zy)    Nc                        e Zd Ze	 	 	 ddej
                  dededefd       Zdde	fdZ
	 	 	 	 dded	edededed
e	f fdZd Zd Zd Zd Z xZS )
DoRALinearlinearrdropoutscalec                     | j                   j                  \  }}t        | t        j                        r|d| j
                  z  z  }t        |||||      }|j                  |        |S )N    )
input_dimsoutput_dimsr   r   r   )weightshape
isinstancennQuantizedLinearbitsr   
set_linear)r   r   r   r   r   r   dora_lins          [/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/tuner/dora.py	from_basezDoRALinear.from_base
   si     #)--"5"5Zfb001"++J!#
 	F#    
dequantizec                    | j                   }d|v }| j                         }|j                  \  }}t        j                  ||d      }| j
                  | j                  j                  z  }| j                  j                  }	|||	z  j                  |j                        z   }| j                  t        j                  j                  |d      z  }
|
d d d f   |z  |_        |r|j                   |_        | j#                         rC|sAt        j$                  j'                  ||j(                  |j*                  |j,                        }|S )NbiasFr      axis
group_sizer   mode)r   _dequantized_weightr   r   Linearr   lora_bTlora_aastypedtypemmxlinalgnormr   r   _is_quantizedr   from_linearr    r   r!   )selfr   r   r   r   r   r   fused_linearr$   r&   
norm_scales              r   fusezDoRALinear.fuse    s   ))+"(,,Zyy[uEdkkmm+6F?226<<@@VVbiinnV!n<<
(D1F: &L
--99!,,[[[[	 : L r   r   r   r   c                 r   t         |           | j                  t        j                  |||             t        j
                  |      | _        || _        dt        j                  |      z  }t        j                  j                  | |||f      | _        t        j                  ||f      | _        y )Nr   pr   lowhighr   r   )super__init__r   r   r#   Dropoutr   r   mathsqrtr*   randomuniformr&   zerosr$   )r/   r   r   r   r   r   r   	__class__s          r   r;   zDoRALinear.__init__:   s     	 			*kEFzzG, 
 DIIj))ii''q/ ( 

 hha%56r   c                     || _         t        j                  j                  | j	                         j                  t        j                        d      | _        y)zA
        Set the self.linear layer and recompute self.m.
        r   r   N)r   r*   r+   r,   r"   r'   float32r)   )r/   r   s     r   r   zDoRALinear.set_linearU   s<      8 8 : A A"** MTUVr   c                 P   | j                   j                  }| j                         rt        j                  || j                   j
                  | j                   j                  | j                   j                  | j                   j                  | j                   j                        }|S )zU
        Return the weight of linear layer and dequantize it if is quantized
        r   )
r   r   r-   r*   r   scalesbiasesr    r   r!   )r/   r   s     r   r"   zDoRALinear._dequantized_weight\   sx     ##]]"""";;11[[%%[[%%F r   c                 J    t        | j                  t        j                        S )N)r   r   r   r   )r/   s    r   r-   zDoRALinear._is_quantizedl   s    $++r'9'9::r   c                 v   | j                         }||j                  z  }| j                  |      | j                  z  | j                  z  }|| j
                  |z  j                  |j                        z   }|| j
                  | j                  j                  z  | j                  j                  z  z   }t        j                  t        j                  j                  |d            }| j                  |z  j                  |j                        |z  }d| j                  v r|| j                  j                  z   }|S )Nr   r   r   )r"   r%   r   r&   r$   r   r'   r(   r*   stop_gradientr+   r,   r)   r   r   )r/   xwyzoutadapteddenoms           r   __call__zDoRALinear.__call__o   s    $$&G\\!_t{{*dkk94::>))!''22 tzzDKKMM1T[[]]BB  a!@A vv~%%agg.4T[[ (((C
r                    4@F)rT   rU   rV   F)__name__
__module____qualname__staticmethodr   r#   intfloatr   boolr2   r;   r   r"   r-   rR   __classcell__rB   s   @r   r   r   	   s     			  	 *t < 77 7 	7
 7 7 76W ;r   r   c                        e Zd Z	 	 	 ddej                  dededefdZddefdZ		 	 	 dded	edededef
 fd
Z
dej                  fdZd Zd Z xZS )DoRAEmbedding	embeddingr   r   r   c                     | j                   j                  \  }}t        | t        j                        rt        d      t        |||||      }|j                  |        |S )Nz0DoRAEmbedding does not yet support quantization.)num_embeddingsdimsr   r   r   )r   r   r   r   r   
ValueErrorrb   set_embedding)rc   r   r   r   re   rf   dora_embeddings          r   r   zDoRAEmbedding.from_base   sg      )//55 i!3!34OPP&)
 	$$Y/r   r   c                    | j                   }|j                  }|j                  \  }}t        j                  ||      }| j
                  | j                  z  }| j                  }|||z  j                  |j                        z   }| j                  t        j                  j                  |d      z  }	|	d d d f   |z  |_        |S Nr   r   )rc   r   r   r   	Embeddingr   r&   r$   r'   r(   r)   r*   r+   r,   )
r/   r   rc   r   re   rf   fused_embeddingr&   r$   r1   s
             r   r2   zDoRAEmbedding.fuse   s    NN	!!%||,,~t<dkk)6F?226<<@@VVbiinnV!n<<
!+AtG!4v!=r   re   rf   c                 n   t         |           | j                  t        j                  ||             t        j
                  |      | _        || _        dt        j                  |      z  }t        j                  j                  | |||f      | _        t        j                  ||f      | _        y )Nr4   r   r6   r9   )r:   r;   rh   r   rl   r<   r   r   r=   r>   r*   r?   r@   r&   rA   r$   )r/   re   rf   r   r   r   rB   s         r   r;   zDoRAEmbedding.__init__   s     	 	2<<=>zzG, 
 DIIn--ii''!1% ( 

 hhaY/r   c                 r    || _         t        j                  j                  |j                  d      | _        y rk   )rc   r*   r+   r,   r   r)   )r/   rc   s     r   rh   zDoRAEmbedding.set_embedding   s'    "	 0 0q9r   c                 v   | j                  |      }| j                  | j                  |   z  | j                  z  }|| j	                  |      j                  |j                        z   }||z   }t        j                  t        j                  j                  |d            }| j                  |   |z  d   |z  }|S )Nr   ).N)rc   r   r&   r$   r   r'   r(   r*   rJ   r+   r,   r)   r/   rK   rM   rN   rO   rP   rQ   s          r   rR   zDoRAEmbedding.__call__   s    NN1JJQ'$++5$,,q/((11 a%  b!AB vvay5 ),s2
r   c                    | j                   j                  |      }| j                  |      | j                  j                  z  | j
                  j                  z  }|| j                  |z  j                  |j                        z   }| j                   j                  | j                  | j
                  z  | j                  z  z   }t        j                  t        j                  j                  |d            }| j                  |z  |z  }|S rk   )rc   	as_linearr   r$   r%   r&   r   r'   r(   r   r*   rJ   r+   r,   r)   rr   s          r   rt   zDoRAEmbedding.as_linear   s    NN$$Q'\\!_t{{}},=4::>))!''22 ..''4::+Ct{{*RR  a!@A vv~$
r   rS   rW   )rX   rY   rZ   r   rl   r\   r]   r   r^   r2   r;   Modulerh   rR   rt   r_   r`   s   @r   rb   rb      s     	<<  	*t & 00 0 	0
 0 04:ryy :r   rb   )	r=   mlx.corecorer*   mlx.nnr   ru   r   rb    r   r   <module>rz      s5      w wt]BII ]r   