
    Nj7                     >    d dl Z d dlmZ d dlmZmZ  G d de      Zy)    N)pad_sequence)
DataLoaderDatasetc                       e Zd Zdeee      dej                  ddfdZdefdZdede	ee   ej                  f   fdZ
ed	ee	eee      ef      de	ej                  ej                  f   fd
       ZddededefdZy)TextDatasettokenized_textstargetsreturnNc                 d    t        |      t        |      k7  rt        d      || _        || _        y)zA dataset of texts.

        :param tokenized_texts: The tokenized texts. Each text is a list of token ids.
        :param targets: The targets.
        :raises ValueError: If the number of targets does not match the number of texts.
        z1Number of targets does not match number of texts.N)len
ValueErrorr   r	   )selfr   r	   s      h/Users/ahmed/devFolder/Ultron/claude-voice/.venv/lib/python3.12/site-packages/model2vec/train/dataset.py__init__zTextDataset.__init__   s0     w<3//PQQ.    c                 ,    t        | j                        S )z!Return the length of the dataset.)r   r   )r   s    r   __len__zTextDataset.__len__   s    4''((r   indexc                 >    | j                   |   | j                  |   fS )zGets an item.)r   r	   )r   r   s     r   __getitem__zTextDataset.__getitem__   s!    ##E*DLL,???r   batchc                     t        |  \  }}|D cg c]  }t        j                  |       }}t        |dd      }|t        j                  |      fS c c}w )zCollate function.Tr   )batch_firstpadding_value)ziptorch
LongTensorr   stack)r   textsr	   xtensorspaddeds         r   
collate_fnzTextDataset.collate_fn   sX     ewDI&JEqu'7'7':E&Jg4qIu{{7+++ 'Ks   Ashuffle
batch_sizec                 4    t        | | j                  ||      S )z$Convert the dataset to a DataLoader.)r#   r$   r%   )r   r#   )r   r$   r%   s      r   to_dataloaderzTextDataset.to_dataloader%   s    $4??GXbccr   )    )__name__
__module____qualname__listintr   Tensorr   r   tupler   staticmethodr#   boolr   r'    r   r   r   r      s    
T#Y 
%,, 
SW 
) )@ @tCy%,,/F)G @ ,$uT$s)_c%9:; ,ellTYT`T`F`@a , ,dT ds dJ dr   r   )r   torch.nn.utils.rnnr   torch.utils.datar   r   r   r2   r   r   <module>r5      s     + 0!d' !dr   