NjzdZddlZddlZddlmZddlmZddlmZdZ dejdejde fd Z Gd d ejZGd d ejZGddejZGddejZGddejZGddejZy)zk Taken from https://github.com/LTH14/mar/blob/fe470ac24afbee924668d8c5c83e9fec60af3a73/models/diffloss.py N)Self) FlowLMConfigc|d|zz|zSN)xshiftscales g/Users/ahmed/devFolder/Ultron/claude-voice/.venv/lib/python3.12/site-packages/pocket_tts/modules/mlp.pymodulater s E ?U ""r alphaepsc|j|jk\sJ|j}||jddz}||j|t j |zzj|}|S)NTdimkeepdim)rdtypevartotorchrsqrt)r rrx_dtyperys r _rms_normrsl 557eiik !! !ggG "d+ +C ehhsmekk#.. /33G) __name__ __module__ __qualname__intfloatr#rTensorr, __classcell__r)s@r rrs)TCTeT 22rrc*eZdZdZdfd ZdZxZS) LayerNormzJReimplementation of LayerNorm because the default one doesn't support jvp.ct|||_|r[tjt j ||_tjt j||_ yyr+) r"r#rr$r%ronesweightzerosbias)r'channelsrelementwise_affiner)s r r#zLayerNorm.__init__*sO  ,,uzz(';Tr-r.r/__doc__r#r,r3r4s@r r6r6'sT<rr6c:eZdZdZ ddededeffd ZdZxZS)TimestepEmbedderz4Embeds scalar timesteps into vector representations. hidden_sizefrequency_embedding_size max_periodc t|tj||dtjtj||dg}|j t |tj||_||_ |dz}|jdtjtj| tjd|z|z y)NTr;freqsr)startend)r"r#r$LinearSiLUappendr SequentialmlprIregister_bufferrexpmathlogarange)r'rHrIrJblockshalfr)s r r#zTimestepEmbedder.__init__=s  II. $ G GGI IIk;T :   gk*+==&)(@%'1,  UYY 44u||!QU7VVY]]^ rc||jj|jz}tjtj |tj |gd}|jdzrJ|j|}|S)NrrrM) rNrrrcatcossinrIrU)r'targs embeddingt_embs r r,zTimestepEmbedder.forwardNsi4::==))IIuyy $@bI 11A566# r)i')r-r.r/rEr0r#r,r3r4s@r rGrG:s/>X]  := QT "rrGc(eZdZdZfdZdZxZS)ResBlockz A residual block that can optionally change the number of channels. :param channels: the number of input channels. c t|||_t|d|_t j t j||dt jt j||d|_ t j t jt j|d|zd|_ y)NrC)rTrL) r"r#r<r6in_lnr$rTrQrRrUadaLN_modulation)r'r<r)s r r#zResBlock.__init__\s   xT2 == IIht 4 GGI IIht 4  !# GGIryy1x)>q)A)G)Gr)G)R& 9h TZZ]Iy 9 HHQK8a<rrDr4s@r rhrhVs   rrhc(eZdZdZfdZdZxZS) FinalLayerz+ The final layer adopted from DiT. ct|t|dd|_t j ||d|_t jt jt j |d|zd|_ y)NFrC)r=rTrLrM) r"r#r6 norm_finalr$rQlinearrTrRrl)r'model_channels out_channelsr)s r r#zFinalLayer.__init__wsa #NuRVWii 4H " GGIryy^1C$O! rc|j|jdd\}}t|j|||}|j |}|S)NrMrr^)rlrnr rvrw)r'r cr r s r r,zFinalLayer.forwardsO,,Q/55aR5@ u T__Q' 6 KKNrrDr4s@r rtrtrs rrtc eZdZdZ d fd ZededededefdZ de jd e jd e jd e jde jf d Z xZ S)SimpleMLPAdaLNawTaken from https://arxiv.org/abs/2406.11838. The MLP for Diffusion Loss. :param in_channels: channels in the input Tensor. :param model_channels: base channel count for the model. :param out_channels: channels in the output Tensor. :param cond_channels: channels in the condition. :param num_res_blocks: number of residual blocks per downsample. ct |||_||_||_||_||_|dk7sJtjt|Dcgc] }t|c}|_ tj|||_ tj|||_g}t|D]} |jt!|tj||_t%|||_ycc}wr)r"r# in_channelsrxrynum_res_blocksnum_time_condsr$ ModuleListrangerG time_embedrQ cond_embed input_projrSrh res_blocksrt final_layer) r'rrxry cond_channelsrr_rir)s r r#zSimpleMLPAdaLN.__init__s &,(,,"""--7<^7L M7L! n -7L M ))M>B))K@ ~&A   h~6 7'-- 3%nlC NsDcfg latent_dimcond_dimreturncp|j}|j}|j}d}t||||||S)NrM)r)flowrdepthr})clsrrrconfigflow_dim flow_depthrs r from_pydantic_configz#SimpleMLPAdaLN.from_pydantic_configs=::\\  *h Sa  rr{srbr c||gj|}tjk(s Jdjdtjdk7sJtfdt jDjz }j |}||z}j D] }|||} j||S)a Apply the model to an input batch. :param c: conditioning from AR transformer. :param s: start time tensor. :param t: target time tensor. :param x: an [N x C] Tensor of inputs. :return: an [N x C] Tensor of outputs. z Expected z time conditions, got rc3NK|]}j||ywr+)r).0rr'tss r z)SimpleMLPAdaLN.forward..s)N3Ma""2a5)3Ms"%)rlenrsumrrrr) r'r{rrbr t_combinedrblockrs ` @r r,zSimpleMLPAdaLN.forwardsV OOA 2w$--- ++,,B3r7) L -""a''' N59L9L3MN NQUQdQd d  OOA  N__Ea A%1%%r)r)r-r.r/rEr# classmethodrr0rrrr2r,r3r4s@r r}r}s"D@ |  PS X\  &&"',,&38<<&DILL& &rr})rErXrtorch.nnr$typing_extensionsrpocket_tts.utils.configrr r2r1rModulerr6rGrhrtr}rrr rs  "0#  ell  2bii2 &ryy8 ryy 8(Q&RYYQ&r