
    (HJjC/                       d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	m
Z
mZmZmZmZ d dlmZ d dlmZ d dlmZ ddlmZ dd	lmZ dd
lmZmZmZ ddlmZmZmZ  ed      Z dZ!dddddddddddd	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZ"eedddddddddf	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZ#ddZ$ddZ%d dZ&	 	 	 	 	 	 	 	 	 	 	 	 d!dZ'd"dZ(d#dZ)d$dZ*d%dZ+d&dZ,y)'    )annotationsN)Path)AnyIterableIteratorOptionalTypeVarUnion)tqdm)
load_model   )normalize_for_wer)STTEvalSample)DEFAULT_DATASET_REPODEFAULT_REVISIONiter_seed_tts_english_samples)	WERResultaggregate_wercompute_werT)werF)dataset_revisiondataset_splitexpected_countsummary_metadatalimitlanguage
gen_kwargsskip_existingdataset_batch_sizemetricsverbosec               D   |dk  rt        d      t        |      }t        |      }|j                  dd       |dz  }t	        | t
              r| n| j                  j                  }t	        | t
              rt        |       n| }|
xs i }|rt        |      ni }g }g }d}d}|st        |dd	
      }nt        |dd	
      }	 t        ||      }|}||}n|t        ||      }t        ||dd      }t        ||      D ]  }|D ]  }|rp|j                  |v rb||j                     }|j!                  |       |j!                  t#        |             |t%        |j'                  dd      xs d      z  }|dz  }vt)        j*                         } 	 t-        ||j.                  |	||      }!t)        j*                         | z
  }"||"z  }t5        |j6                        }#t5        |!      }$t9        |#|$      }%|j!                  |%       |j                  t        |j.                        |j:                  |j6                  |!|#|$d|%j=                         |"t?        |j.                        tA        |jB                        d}|j!                  |       |jE                  tG        jH                  |d      dz           |j3                           	 |jK                          |stM        d      tO        d |D              rtM        d      tQ        |      }&tS        d |D              }'t        |      |||tU        |      |||||	d
|&||'|'r||'z  nd d}(|r|(jW                  |       tY        ||(       |(S # t0        $ r |j3                           w xY w# |jK                          w xY w)Nr   z&dataset_batch_size must be at least 1.T)parentsexist_okzpredictions.jsonlr   g        wutf-8encodingazSTT WERutt)totaldescunitwall_time_sec)	model_obj
audio_pathr   r   r"   )utt_idr1   source_path	reference
hypothesisreference_normalizedhypothesis_normalized)r/   audio_duration_secmetadataF)ensure_ascii
z#No Seed-TTS samples were evaluated.c              3  B   K   | ]  }|j                  d d         yw)r7    N)get.0records     c/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_audio/stt/eval/runner.py	<genexpr>z#run_stt_wer_eval.<locals>.<genexpr>   s      
MW6vzz1266Ws   z All model hypotheses were empty.c              3  Z   K   | ]#  }|j                  d       t        |d           % yw)r8   N)r>   floatr?   s     rB   rC   z#run_stt_wer_eval.<locals>.<genexpr>   s3      F::*+7 	f)*+s   )+)
modeldataset_namer   r   num_samplesnum_skippedr    r!   generation_kwargsr   )total_wall_time_sectotal_audio_duration_secrtf)-
ValueError_validate_metricsr   mkdir
isinstancestr	__class____name__r   _load_existing_predictionsopen_limit_iterableminr   _batchedr2   append_wer_result_from_recordrE   r>   timeperf_counter_transcribe_sampler1   	Exceptionflushr   reference_textr   r3   to_dict_wav_duration_secondsdictr9   writejsondumpscloseRuntimeErrorallr   sumlenupdate_write_summary))rF   samples
output_dirrG   r   r   r   r   r   r   r   r   r    r!   r"   output_pathpredictions_pathmodel_idr0   generate_kwargsexistingrecordswer_resultsskippedtotal_wall_timepredictions_filesample_iterabler,   progressbatchsamplerA   startedr5   	wall_timer6   r7   
wer_result
aggregatesaudio_duration_totalsummarys)                                            rB   run_stt_wer_evalr      s   $ AABB(Gz"KdT2"%88"5#.uEOO4L4LH%/s%;
5!I &BO?L)*:;RTH$&G#%KGO 0#H 0#H7!)'59=Eu%Eu95Qh(:;E V]]h%>%fmm4FNN6*&&'>v'FG#uVZZ-M-TQT'UUOqLG++-
!3"+#)#4#4!)#2 '"J !--/'9	9,'89N9N'O$(9*(E%()=?TU
"":. %mm"%f&7&7"8#)#5#5!'!6!6",,@-B !((* &/*?@Q@Q*R $V__ 5 v& &&tzz&u'MPT'TUW  X ""$[ <^ 	 @AA

MW
MM=>>{+J   X$,&7|0,   /$89M!55SWG  '(;(N} ! $**,8 	 s&   CN M/(D N /N

N NTc                    t        |      }|	rt        |	      n|dz  }t        |||||      }t        | ||d|d||ddd||||
||      S )	Nzaudio-cache)dataset_reporevisionaudio_cache_dirr   fail_on_missing_audiozseed-ttszen/standardenstandard)r   localeset)rF   ro   rp   rG   r   r   r   r   r   r   r   r    r!   r"   )r   r   r   )rF   rp   r   r   r   r   r   r   r   r   r    r!   r"   rq   audio_cache_pathro   s                   rB   run_seed_tts_evalr      s     z"K!0_kM6Q  ,!(3G !#(

 #-%     c                b   | xs dgD cg c]  }|j                          }}|st        d      t        t        |      t        t              z
        }|r8dj                  t              }dj                  |      }t        d| d| d      t        t        j                  |            S c c}w )Nr   z&At least one metric must be requested.z, zUnsupported metric(s): z. Supported: .)	lowerrN   sortedr   SUPPORTED_METRICSjoinlistrd   fromkeys)r!   metric
normalizedunsupported	supported	requesteds         rB   rO   rO      s    070BE70BD0BV&,,.0BJDABBZ3/@+AABKII/0	IIk*	29+]9+UVWXXj)** Es   B,c              #  L   K   t        |       D ]  \  }}|||k\  r y |  y wN)	enumerate)iterabler   indexitems       rB   rW   rW      s-      *t%
 +s   "$c              #  x   K   g }| D ](  }|j                  |       t        |      |k\  s#| g }* |r| y y wr   )rZ   rl   )r   
batch_sizer}   r   s       rB   rY   rY      sF     ETu:#KE	 
  s   '::c                H   t        j                  | j                        }t        |      }|||d<   d|j                  v r||d<   |j                         D ci c]  \  }}||j                  v s|| }}} | j                  t        |      fi |}	t        |	      S c c}}w )Nr   r"   )inspect	signaturegeneraterd   
parametersitemsrR   _extract_text)
r0   r1   r   r   r"   r   call_kwargskvresults
             rB   r^   r^      s     !!)"4"45Iz"K"*JI(((!(I$/$5$5$7U$7DAq1	@T@T;T1a4$7KUYJ?;?F   Vs   B/Bc                   t        | d      r#t        | j                        j                         S t	        | t              r| j                         S t	        | t
              r d| v rt        | d         j                         S t        j                  |       st        | d      rg }| D ]  }t        |d      r%|j                  t        |j                               4t	        |t
              r"d|v r|j                  t        |d                f|j                  t        |              dj                  |      j                         S t        |       j                         S )Ntext__iter__r=   )
hasattrrR   r   striprQ   rd   r   isgeneratorrZ   r   )r   partsr   s      rB   r   r     s   vv6;;%%''&#||~&$Ff$46&>"((**6"gfj&ADtV$S^,D$'FdNSf./SY'  wwu~##%%v;r   c                   | j                         si S i }t        | dd      5 }|D ]J  }|j                         st        j                  |      }|j                  d      }|s=||t        |      <   L 	 d d d        |S # 1 sw Y   |S xY w)Nrr'   r(   r2   )existsrV   r   rf   loadsr>   rR   )rr   rv   rz   linerA   r2   s         rB   rU   rU     s    ""$	)+G	g	6:J$D::<ZZ%FZZ)F'-F$ % 
7 N 
7 Ns   ?A>#A>>Bc                    t        t        | d         t        | d         t        | d         t        | d         t        | d         t        | d               S )Nsubstitutions	deletions
insertionsreference_tokenshypothesis_tokensr   )r   r   r   r   r   r   )r   intrE   )rA   s    rB   r[   r[   (  s^    &12f[)*vl+,V$678f%89:&-  r   c                    	 t        j                  t        |       d      5 }|j                         }|j	                         }|r||z  nd cd d d        S # 1 sw Y   y xY w# t
        $ r Y y w xY w)Nrb)waverV   rR   
getnframesgetframerater_   )pathwav_fileframessample_rates       rB   rc   rc   3  sb    YYs4y$'8((*F"//1K+66K'D (''  s.   A  )A
	A  AA  A   	A,+A,c           
        t        | dz  dd      5 }t        j                  ||dd       |j                  d       d d d        t        | d	z  dd      5 }|j                  d
|d    d       |d   }|j	                  d      r|d   }|j	                  d      r
| d|d    }|j                  d| d       |j                  d|d    d       |j                  d|d   dz  dd       |j                  d|d   dz  dd       |j                  d|d   dz  dd|d   dz  dd|d   dz  dd       |d   |j                  d |d   dd       d d d        y # 1 sw Y   &xY w# 1 sw Y   y xY w)!Nzsummary.jsonr&   r'   r(   F   )r:   indentr;   zsummary.txtzModel: rF   rG   r   r   @z	Dataset: z	Samples: rH   zWER micro: 	wer_microd   z.3fz%
zWER macro: 	wer_macrozSub/Del/Ins: substitution_ratez% / deletion_rateinsertion_raterM   zRTF: )rV   rf   dumpre   r>   )rq   r   summary_filedataset_labels       rB   rn   rn   =  s   	kN*C'	Bl		'<eAF4  
C 
kM)3	A\WWW%5$6b9:/;;~&#N3M;;)*,oQw7I/J.KLMY}oR89Yw}'=&>bAB[)=)CC(HLM[)=)CC(HLM*+c1#6d'#-c2$'(3.s338	
 5>%wu~c&:"=>% 
B	A	 
C	B 
B	As   +E!DE.!E+.E7) rF   Union[str, nn.Module]ro   zIterable[STTEvalSample]rp   
str | PathrG   rR   r   Optional[str]r   r   r   Optional[int]r   Optional[dict[str, Any]]r   r   r   r   r   r   r   boolr    r   r!   Optional[Iterable[str]]r"   r   returndict[str, Any])rF   r   rp   r   r   rR   r   rR   r   r   r   r   r   r   r   r   r   r   r   zOptional[str | Path]r    r   r!   r   r"   r   r   r   )r!   r   r   z	list[str])r   Iterable[T]r   r   r   zIterator[T])r   r   r   r   r   zIterator[list[T]])r0   z	nn.Moduler1   r   r   r   r   r   r"   r   r   rR   )r   r   r   rR   )rr   r   r   zdict[str, dict[str, Any]])rA   r   r   r   )r   r   r   zOptional[float])rq   r   r   r   r   None)-
__future__r   r   rf   r\   r   pathlibr   typingr   r   r   r   r	   r
   mlx.nnnnr   mlx_audio.stt.utilsr   	normalizer   schemar   seed_ttsr   r   r   r   r   r   r   r   r   r   r   rO   rW   rY   r^   r   rU   r[   rc   rn    r   rB   <module>r      s9   "      D D   * ( ! 
 7 6CL  '+#'$(15"+/'+!B B$B B
 B $B !B "B /B B B )B B B %B  !B" #BP -$"+/"&,0'+- -- - 	-
 - - )- -  - *- - %- - -`+!!! ! 	!
 ! 	!&( ?r   