
    Nj                     ^   d dl Z d dlmZ d dlmZ d dlmZ d dlmZ de j                  de j                  de j                  de j                  d	e
e j                  e j                  f   f
d
Zde j                  de j                  dedz  d	e j                  fdZ G d d      Z G d de      Zy)    N)
functional)RotaryEmbedding)StatefulModulecacheoffsetkvreturnc                    |j                         dkD  r5t        j                  ||j                  d      d   k(        st	        d      t        |j                  d      d   j                               }|| dd d |||j                  d   z   f<   || dd d |||j                  d   z   f<   | d d d d d ||j                  d   z   f   }|d   |d   fS )N   r   z3Linear cache offset must be identical across batch.)numeltorchallview
ValueErrorintitemshape)r   r   r   r	   offset_valuevalids         o/Users/ahmed/devFolder/Ultron/claude-voice/.venv/lib/python3.12/site-packages/pocket_tts/modules/transformer.pycomplete_kvr   	   s     ||~%))Ffkk"oa6H,H"INOOv{{2q)..01L<=E!Q|aggaj88
89<=E!Q|aggaj88
89!Q3,3334E8U1X    pos_qpos_kcontextc                     | d d d d d f   |d d d d d f   z
  }|d d d d d f   dk\  |dk\  z  }||||k  z  }|d d d f   S )Nr    )r   r   r   deltamasks        r   _build_attention_maskr"      sd     !Q*aqj 11E!T1*"uz2Duw'4=r   c                      e Zd ZdZdedefdZdededej                  dej                  d	e	e
ej                  f   f
d
Zde	e
ej                  f   ded	dfdZde	e
ej                  f   dz  dedej                  d	ej                  fdZdej                  dej                  de	e
ej                  f   dz  d	eej                  ej                  ej                  ej                  f   fdZy)_LinearKVCacheBackendT	num_headsdim_per_headc                      || _         || _        y N)r%   r&   )selfr%   r&   s      r   __init__z_LinearKVCacheBackend.__init__*   s    "(r   
batch_sizesequence_lengthdevicedtyper
   c           
          t        t        j                  |t        j                  |      t        j                  d||| j
                  | j                  ft        d      ||            S )Nr.   r-      NaNr-   r.   )r   r   )dictr   zeroslongfullr%   r&   float)r)   r+   r,   r-   r.   s        r   
init_statez _LinearKVCacheBackend.init_state.   sV     ;;zFK**JARARSe	
 	
r   state	incrementNc                     |dxx   |z  cc<   y )Nr   r   r)   r:   r;   s      r   increment_stepz$_LinearKVCacheBackend.increment_step;   s    h9$r   c                     |&t        j                  dt         j                  |      S |d   j                  d      d   S )Nr   r0   r   r   r   )r   r5   r6   r   )r)   r:   r+   r-   s       r   rope_offsetz!_LinearKVCacheBackend.rope_offset>   s:     =;;rFCCX##B'**r   r   r	   c                    ||j                  dddd      }|j                  dddd      }t        j                  |j                  d   |j                  t        j
                        }|j                  dd      j                  |j                  d   d      }t        j                  |j                  d   |j                  t        j
                        }||||fS t        |d   |d   ||      \  }}	|j                  dddd      }|	j                  dddd      }t        j                  |j                  d   |j                  t        j
                        }|j                  dd      j                  |j                  d   d      }||||d   fS )	Nr   r1   r      r3   r   r   r   )
permuter   aranger   r-   r6   r   expandr5   r   )
r)   r   r	   r:   k_attnv_attnr   r   cache_kcache_vs
             r   append_and_getz$_LinearKVCacheBackend.append_and_getE   sH    =YYq!Q*FYYq!Q*FLLaejjYEJJq"%,,V\\!_bAE[[aejjYF65&00&uW~uX1MAq!,Aq!,V\\!_V]]%**U

1b!((a"=vueHo55r   )__name__
__module____qualname__requires_stater   r*   r   r-   r.   r4   strTensorr9   r>   r@   tuplerJ   r   r   r   r$   r$   '   s'   N)# )S )

03
=B\\
RWR]R]
	c5<<	 
%Dell):$; % %PT %+#u||+,t3+AD+NSll+	+66"',,67;C<M7NQU7U6	u||U\\5<<E	F6r   r$   c            
            e Zd ZdZ	 ddededededz  f fdZded	ed
eee	j                  f   fdZddedefdZde	j                  dedz  fdZ xZS )StreamingMultiheadAttentiona  Similar to `nn.MultiheadAttention` but with support for streaming.

    Args:
        embed_dim (int): Dimension to project to.
        num_heads (int): Number of heads.
        context (int, optional): Number of time steps the attention can access to.
            Can access `context` time steps into the past.
        rope (`RotaryEmbedding`, optional): Rope embedding to use.
        device (torch.device, optional): Device on which to initialize.
        dtype (torch.dtype, optional): dtype to use.
    N	embed_dimr%   roper   c                 d   t         	|           || _        || _        || _        || _        ||z  | _        t        | j                  | j                        | _        |}|}||z  |z  }|d|z  z  }d}t        j                  |||z  d      | _        t        j                  |||z  d      | _        y )Nr1   r   F)bias)superr*   rT   rU   r%   r   r&   r$   _cache_backendnnLinearin_projout_proj)
r)   rT   r%   rU   r   out_dimnum_kvkv_dimmult	__class__s
            r   r*   z$StreamingMultiheadAttention.__init__d   s     	"	"%23DNNDDUDUVy(F21v:yyD7NG		)TI-=EJr   r+   r,   r
   c                     | j                   j                  }t        |      r" |       j                  }t        j
                  }n|j                  }|j                  }| j                  j                  ||||      S r(   )	r\   weightcallabler-   r   float32r.   rY   r9   )r)   r+   r,   rd   r-   r.   s         r   r9   z&StreamingMultiheadAttention.init_statex   s_    $$F X__FMME]]FLLE""--j/6SXYYr   r:   r;   c                 <    | j                   j                  ||       y r(   )rY   r>   r=   s      r   r>   z*StreamingMultiheadAttention.increment_step   s    **5)<r   querymodel_statec                 ~   |d n| j                  |      }| j                  |      }|j                  \  }}}| j                  }|j	                  ||d| j
                  |      }	t        j                  |	d      \  }
}}| j                  j                  |||
j                        }| j                  |
||      \  }
}|
j                  dd      }
| j                  j                  |||      \  }}}}|j	                  dd      t        j                  ||
j                  t        j                        j	                  dd      z   }t!        ||| j"                        }t%        j&                  |
|||d	      }|j                  dd      }|j                  \  }}}}|j)                  ||||z        }| j+                  |      }|S )
NrB   r1   )dim)r   r   r   r3   g        )	dropout_p)	get_stater\   r   r&   r   r%   r   unbindrY   r@   r-   rU   	transposerJ   rD   r6   r"   r   Fscaled_dot_product_attentionreshaper]   )r)   rh   ri   r:   	projectedbt_dpackedqr   r	   r@   rF   rG   r   r   r   	attn_maskxhs                         r   forwardz#StreamingMultiheadAttention.forward   s   #+1LLL'	//1a1a;,,v1-1a))55eQIyyAky21KK1(,(;(;(J(J1aQV(W%vB"U\\!AHHEJJ%W%\%\]^`b%cc)%E	**1ffiSVWKK1WW
1aIIaAE"MM!r   r(   )r   )rK   rL   rM   __doc__r   r   r*   r4   rO   r   rP   r9   r>   r}   __classcell__)rb   s   @r   rS   rS   W   s    
 \`KK),K4CKNQTXjK(
ZS 
Z3 
Z4U\\HYCZ 
Z=D =S =U\\ t r   rS   )r   torch.nnrZ   r   rp   pocket_tts.modules.roper   "pocket_tts.modules.stateful_moduler   rP   rQ   r   r   r"   r$   rS   r   r   r   <module>r      s      $ 3 =
<<
!&
27,,
CH<<

5<<%&
<< %7:Tz
\\"-6 -6`G. Gr   