
    ]iA                         S r SSKrSSKJr  SSKJr  SSKJrJr  SSK	r	SSK
Js  Jr  SSKJrJrJrJr  SSKJr  SS	KJr  SS
KJrJr  SSKJr  SSKJrJr  \R<                  " \5      r S r! " S S\"5      r#g)zMain training loop.    N)get_xp)write_and_rename)LogProgressbold   )augmentdistribstates
pretrained)apply_model)ModelEMA)evaluatenew_sdr)svd_penalty)pull_metricEMAc                 N    SR                  S U R                  5        5       5      $ )N | c              3   P   #    U  H  u  pUR                  5        S U 3v   M     g7f)=N)
capitalize).0keyvals      J/mnt/rpi/tmp/demucs-venv-sys/lib/python3.13/site-packages/demucs/solver.py	<genexpr>_summary.<locals>.<genexpr>   s%     R/hc)*!C51/s   $&)joinitems)metricss    r   _summaryr!      s    ::R'--/RRR    c                   Z    \ rS rSrS rS rS rS\S\4S jrS\S\4S jr	S	 r
SS
 jrSrg)Solver   c           
      T   X@l         Xl        X l        X0l        [        R
                  " U R                  UR                  U R                  5      U l        [        R                  " U5      U l
        [        [        U R                  R                  5       5      5      R                  U l        / / S.U l        U R                  R!                  5        Ho  n[#        UR$                  U5      nUS:X  a  U R                  OSnU(       d  M6  U H3  nU R                  U   R'                  [)        U R                  XS95        M5     Mq     [*        R,                  " [/        UR0                  R2                  UR0                  R4                  -  5      UR*                  R6                  S9/n	UR*                  R8                  (       a-  U	[*        R:                  " 5       [*        R<                  " 5       /-  n	S H_  n
[#        UR*                  U
5      nUR>                  (       d  M,  U	R'                  [#        [*        U
RA                  5       5      " S0 UD65        Ma     [B        RD                  RF                  " U	6 U l        [I        5       nURJ                  U l%        URJ                  S-  U l&        URJ                  S-  U l'        [P        RS                  S	U RL                  RU                  5       5        S U l+        S
U l,        URZ                  U l-        U RZ                  R\                  U l.        U R_                  5         g )N)batchepochr'   cpu)device)shiftsame)scaleremixcheckpoint.thzbest.thzCheckpoint will be saved to %sF )0argsloadersmodel	optimizerr
   get_quantizerquant	quantizerr	   wrapdmodelnextiter
parametersr*   emaskeysgetattremaappendr   r   Shiftintdset
samplerater+   
shift_sameflipFlipChannelsFlipSignprobar   torchnn
Sequentialr   foldercheckpoint_file	best_fileloggerdebugresolve
best_statebest_changedlinkhistory_reset)selfr2   r3   r4   r1   kinddecaysr*   decayaugmentsaugkwxps                r   __init__Solver.__init__   s2   	
"--djj$**dnnUll5)4

 5 5 789@@
 !2.	IINN$DTXXt,F$(GOT[[Fv#EIIdO**8DJJ+UV $	 % MMDII,@,@499??,R(S'+||'>'>@ A<<--/1A1A1CDDH%Cs+Bxxx1A B HR HI & xx**H5Xii!yy?:Y.5t7K7K7S7S7UV!GG	yy((r"   c                 \   0 nU R                   R                  5       US'   U R                  R                  5       US'   U R                  US'   U R                  US'   U R
                  US'   U R                  R                  5        H2  u  p4[        U5       H  u  pVUR                  5       USU SU 3'   M      M4     [        U R                  5       n[        R                  " X'5        S S S 5        U R
                  R                  nU(       ag  US-   U-  S	:X  a[  US-   U R
                  R                  :w  a>  [        U R                  S
US-    S3-  5       n[        R                  " X'5        S S S 5        U R                   (       av  [        U R"                  5       n[$        R&                  " U R                   U R
                  5      nU R                  US'   [        R                  " X'5        S S S 5        SU l        g g ! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       N:= f)Nstater4   rW   rT   r1   ema__r   r   checkpoint_z.thF)r3   
state_dictr4   rW   rT   r1   r=   r   	enumerater   rO   rK   save
save_everyepochsrN   rU   rP   r
   serialize_model)	rY   r(   packagerZ   r=   kr@   tmprk   s	            r   
_serializeSolver._serializeM   s   ::002#~~88:!\\	 $))))//+JD#D/-0^^-=$tfAaS)* * , d223sJJw$ 4 YY))
519
2a7EAIIYIY<Y!$+++eai[0L"LMQT

7( N !$..1S 00TYYG#'?? 

7( 2 !&D  43
 NM
 21s%   G:HAH:
H	
H
H+c           	      b   U R                   R                  5       (       a  [        R                  SU R                    35        [        R
                  " U R                   S5      nU R                  R                  US   5        U R                  R                  US   5        US   U R                  SS& US   U l
        U R                  R                  5        H3  u  p#[        U5       H  u  pEUR                  USU S	U 3   5        M!     M5     gU R                  R                  (       ag  [         R"                  " U R                  R                  U R                  R$                  S
9nU R                  R                  UR'                  5       5        gU R                  R(                  (       Ga  SnU R*                  R,                  nU[/        U R                  R(                  5      -  U-  n	[        R                  SU	5        [        R
                  " U	S5      nUS   U l
        U R                  R0                  (       a  U R                  R                  US   SS9  OU R                  R                  US   SS9  U R                  R2                  (       a  U R                  R                  US   5        ggg)z8Reset state of the solver, potentially using checkpoint.zLoading checkpoint model: r)   rd   r4   rW   NrT   re   rf   )namerepor/   zLoading from %sF)strict)rO   existsrQ   inforK   loadr3   load_state_dictr4   rW   rT   r=   r   ri   r1   continue_pretrainedr   	get_modelpretrained_reporh   continue_fromrN   parentstrcontinue_bestcontinue_opt)
rY   rn   rZ   r=   ro   r@   r3   rt   rootcfs
             r   rX   Solver._resetg   s   &&((KK4T5I5I4JKLjj!5!5u=GJJ&&ww'78NN**7;+?@%i0DLLO%l3DO"iioo/
'oFA''$tfAaS0A(BC . 0 YY**((YY22YY..0E JJ&&u'7'7'9:YY$$$"D;;%%DDII3344t;BKK)2.jjU+G%l3DOyy&&

**7<+@*O

**77+;E*Jyy%%..w{/CD & %r"   r    returnc                    [        US   S5      [        US   S5      S.nSU;   a  [        US   S5      US'   U R                  b  [        US   S5      US'   S	U;   a  [        US	   S5      US	'   S
U;   a  [        US
   S5      US
'   SU;   a  US   US'   SU;   a  [        US   S5      US'   SU;   a  [        US   S5      US'   U$ )z#Formatting for train/valid metrics.lossz.4freco)r   r   nsdr.3fmsz.2fgradbestbnamepenaltyhloss)formatr7   )rY   r    lossess      r   _format_trainSolver._format_train   s     76?E276?E2
 W#GFOU;F6N>>%!'$-7F4LW#GFOU;F6NW#GFOU;F6Ng%g.F7O &wy'95 AF9g$WW%5u=F7Or"   c                    0 nSU;   a  [        US   S5      US'   SU;   a  [        US   S5      US'   U R                  R                   H9  nSU 3nXA;   a  [        X   S5      X$'   SU 3nXA;   d  M)  [        X   S5      X$'   M;     U$ )zFormatting for test metrics.sdrr   r   sdr_nsdr_)r   r3   sources)rY   r    r   sourcer   s        r   _format_testSolver._format_test   s    G"75>59F5MW#GFOU;F6Njj((F/C~$W\59&"C~$W\59 ) r"   c                 T   U R                   (       a  [        R                  S5        [        U R                   5       H  u  pU R	                  US   5      n[        R                  [        SUS-    S[        U5       35      5        U R	                  US   5      n[        R                  [        SUS-    S[        U5       35      5        SU;   d  M  U R                  US   5      nU(       d  M  [        R                  [        S	US-    S[        U5       35      5        M     S
n[        [        U R                   5      U R                  R                  5       GH  nU R                  R                  5         0 n[        R                  S5        [        R                  S5        U R                  U5      US'   U R	                  US   5      n[        R                  [        SUS-    S[        U5       35      5        [        R                  S5        [        R                  S5        U R                  R                  5         [         R"                  " 5          U R                  USS9nUnSn[$        R&                  " U R                  R)                  5       5      n0 US'   XBS   S'   U R                  R*                  R,                  nU R.                  R1                  5        H  u  p[        U
5       Hx  u  pUR3                  5          U R                  USS9nS S S 5        SU	 SU 3nXBS   U'   XH   nXX   nUR5                  S5      (       a  U* nU* nX:  d  Mh  UnUR6                  nUnMz     US   R9                  U5        XbS   S'   M     S S S 5        US   W   n[;        U R<                  R                   SU 35      U/-   nUR5                  S5      (       a  [?        U5      nO[A        U5      nUUS   S'   U R                  RB                  RD                  S
:  an  [G        U R                  RB                  5      nURI                  S5        [         R"                  " 5          [K        U R                  4SS0UD6nS S S 5        WUS   S'   U R	                  US   5      n[        R                  [        SUS-    S[        U5       35      5        UU:X  d%  U R                  RL                  RN                  (       aA  [        R                  [        S5      U5        [$        R&                  " W5      U l(        SU l)        US-   U R                  R*                  RT                  -  S
:H  nXR                  R                  S-
  :H  nU(       d  U(       GaL  [        R                  S5        [        R                  S5        U R                  R*                  RV                  (       a  U RP                  nO.[$        R&                  " U R                  R)                  5       5      nU R                  R*                  RX                  =(       a    Un[$        RZ                  " U R                  U5         [         R"                  " 5          []        U US9US'   S S S 5        S S S 5        U R                  US   5      n[        R                  [        S	US-    S[        U5       35      5        U R<                  R_                  U5        [`        Rb                  S
:X  a?  U Re                  U5        [        Rg                  SU Rh                  Rk                  5       5        U(       d  GM    g    g ! , (       d  f       GNF= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN= f)Nz#Replaying metrics from previous runtrainzTrain Summary | Epoch r   r   validzValid Summary | Epoch testzTest Summary | Epoch r   zF----------------------------------------------------------------------zTraining...zCross validation...F)r   mainre   rf   r   r   zvalid.r   r   exactTzNew best valid loss %.4fzEvaluating on the test set...)compute_sdrzCheckpoint saved to %s)6rW   rQ   rx   ri   r   r   r!   r   rangelenr1   rl   r3   r   _run_one_epochevalrK   no_gradr
   
copy_staterh   r   metricr=   r   swap
startswithrd   updater   rV   maxminsvdr   dictpopr   rD   train_validrT   rU   everyr   r   
swap_stater   push_metricsr	   rankrq   rR   rO   rS   )rY   r(   r    	formattedr   bvalidr   rd   r   rZ   r=   ro   r@   rt   ab
valid_lossmets	best_lossr_   r   should_evalis_lastr   s                           r   r   Solver.train   s   <<KK=>'5NE**77+;<IKK-eai[HY<O;PQRT**77+;<IKK-eai[HY<O;PQRT  --gfo>	9KK'<UQYKs8T]K^J_%` ab 6 3t||,dii.>.>?EJJGKK!KK&#2259GG**77+;<IKK-eai[HY<O;PQRT KK!KK-.JJOO++E+?))$***?*?*AB#% +0 (iinn++"&))//"3JD"+D/ XXZ$($7$7U$7$KE (!%dV1QC016(.!J"K>>&11!"A!"A5%*F$'IIE$(E #2 G$++F305G$W-! #4 !4 !)#.Jtyy00F3%.AZLPD~~f%%I	I	'0GGV$yy}}$$q($))--(y!]]_)$**GDGBGG %.5 +**77+;<IKK-eai[HY<O;PQRT Y&$))..*D*DD!;<jI"("3"3E":$(! !19		(<(<<AKyy//!33G gH%;<99>>&& OOE"--djj.C.C.EFE"iinn00<W&&tzz59*24[*Q ) : !--gfo>	D#83xPYGZF[!\]^II""7+||q &5t7K7K7S7S7UVwI @2 (Z !H %_L ) :9s[   B.]" ]>]"3]"]4^1^?^
]]""
]1	4
^	
^^
^'	c                    U R                   nU(       a  U R                  S   OU R                  S   n[        R                  S:  a"  U(       a  UR                  R                  U5        SS/U   nUSUS-    3-   n[        U5      nUR                  (       a  [        XsR                  5      n[        [        XGU R                   R                  R                  US9n[        5       n	[        U5       GH  u  pUR                  U R                   5      nU(       a!  U R#                  U5      nUR%                  SS9nOUS S 2S	4   nUS S 2SS 24   nU(       dO  U R                   R&                  (       a4  [)        U R*                  XR                   R,                  R.                  S	S
9nOU R1                  U5      nU(       a6  [3        U R*                  S5      (       a  U R*                  R5                  X5      nUR6                  UR6                  :X  d   UR6                  UR6                  45       e[9        [;        SUR=                  5       5      5      nUR>                  R@                  S:X  a8  [B        RD                  " XSS9nURG                  U5      RG                  S	5      nUnOyUR>                  R@                  S:X  a=  [B        RH                  " XSS9nURG                  U5      nUS-  nURG                  S	5      nO"[K        SU R                   R@                   35      e[L        RN                  " URP                  5      R                  U5      nUU-  R%                  5       UR%                  5       -  nS	nU RR                  b  U RR                  RU                  5       nURV                  RX                  (       a  XRV                  RX                  U-  -  n0 nUU-  R%                  5       UR%                  5       -  US'   UUS'   U(       d|  [[        XR]                  5       5      RG                  S	5      nS	n[_        U R*                  R`                  UU5       H  u  nnnUUSU 3'   UUU-  -  nM     UUR%                  5       -  US'   U(       av  URb                  Rd                  S	:  a\  [g        URb                  5      nURi                  S5        [k        U R*                  40 UD6nUUS'   XRb                  Rd                  U-  -  nUUS'   [        U R*                  R`                  5       H  u  nnUU   USU 3'   M     U(       Ga  URm                  5         S	n/ nU R*                  Ro                  5        Ha  nURp                  c  M  UURp                  Rr                  Ru                  5       S-  -  nURw                  URp                  Rr                  5        Mc     US-  US'   UR>                  Rx                  (       aV  [L        Rz                  R|                  R                  U R*                  Ro                  5       UR>                  Rx                  5        U R                   R                  S:X  a?  U R*                  R                  5        H!  u  nnURp                  b  M  [        SU5        M#     U R                  R                  5         U R                  R                  5         U R                  S    H  nUR                  5         M     U	" U5      nU R                  U5      n UR                  " S 0 U D6  AAAAUR                  U
:X  a    OEU R                   R                  (       a	  U(       a    O!U R                   R                  S:X  d  GM    O   U(       a&  U R                  S    H  nUR                  5         M     [        R                  " WW
S-   5      $ )!Nr   r   r   ValidTrainz	 | Epoch )totalupdatesrt   )dimr   )splitoverlaptransform_target   l1none)	reductionmseg      ?zInvalid loss r   r   r   r   r   r   reco_r   unszno gradr'   rR   r(   r0   )Kr1   r2   r	   
world_sizesampler	set_epochr   max_batchesr   r   rQ   misc
num_printsr   ri   tor*   r   sumvalid_applyr   r3   r   r   r9   hasattrr   shapetupler   r   optimr   Fl1_lossmeanmse_loss
ValueErrorrK   tensorweightsr7   
model_sizer6   diffqr   detachzipr   r   r   r   r   r   backwardr<   r   datanormrA   	clip_gradrL   utilsclip_grad_norm_flagnamed_parametersprintr4   step	zero_gradr=   r   r   rR   average)!rY   r(   r   r1   data_loaderlabelrt   r   logprogaverageridxr   mixestimatedimsr   r   r   r   r   nsdrsr   r   wr_   r   ro   	grad_normgradspnr@   logss!                                    r   r   Solver._run_one_epoch#  s   yy/4dll7+$,,w:O!e))%0'"5)519+..K //0Efk&*iinn&?&?dL5%g.LCjj-G,,w/kkak(adm!!QR%.TYY22&tzz3iinn>R>R\]^;;s+-?@@**55cC>>W]]2SX^^W]]4SS2q'++-01Dzz$&yyfEyy++A.E)zz(vFyySyyy| =0@!ABBll4<<033G<G7N'')GKKM9DB~~)^^..0zz

((2--F"Wn113gkkmCF6NF4L):;@@C'*4::+=+=ug'NOFD!/3FU6(+,QX%E (O "'!6v))A-$((^y!%djj7B7$+y!((722!F6N&tzz'9'9:	6+/7vh'( ; 	..0Avv)!QVV[[%5%5%7%::	QVV[[1 1 "+c!1v::''HHNN22

--/

,,. 99>>U* $

 ; ; =166>!)Q/ !> ##%((*99W-CJJL .f%F%%f-DNN"T"hb3&yy5yy~~( /@ yy)

 *vsQw//r"   )r1   r   rU   rP   rT   rO   r*   r9   r=   rN   rW   rV   r2   r3   r4   r7   N)T)__name__
__module____qualname____firstlineno__ra   rq   rX   r   r   r   r   r   __static_attributes__r0   r"   r   r$   r$      sF    ,\&4E>T d ,D T  unr0r"   r$   )$__doc__loggingdorar   
dora.utilsr   dora.logr   r   rK   torch.nn.functionalrL   
functionalr    r   r	   r
   r   applyr   r@   r   r   r   r   r   r   r   r   	getLoggerr  rQ   r!   objectr$   r0   r"   r   <module>r     sX       ' &    2 2   '  #			8	$Sw0V w0r"   