]ie1SrSSKrSSKrSSKrSSKrSSKJr SSKJ r SSK J r SSK J r SSKJr SSKJr SSKrS S KJr S S KJrJr S S KJrJr S S\R6\\R:4S\4Sjjr"SS\ 5r "SS\ 5r!"SS\ 5r""SS5r#S S S S.Sjr$"SS5r%Sr&"SS\ 5r'g)!z Support for PyTorch lightning. You should just replace the call to `Trainer(...)` with `get_trainer(...)`. For using `dora.log.LogProgress` as a progress bar with PL, see `PLLogProgress`. N)LightningModule)Callback)ProgressBarBase)ClusterEnvironment)Trainer)from_argparse_args)distrib)get_xpis_xp)bold LogProgressTmetricsepochc0nUR5Hup4U(aURS5(aM$U(dURS5(aMCURS5(dURS5(aURSS5Sn[U[R 5(a$UR 5S:XaUR5nXBU'M U$)a Filters metrics before formatting, in particular to remove the `_step` or `_epoch` suffix. This will also convert torch tensors to float. Args: metrics: dict given by PL. epoch: if True, keep only epoch level metrics, otherwise, keep only step level metrics. _step_epoch_r r)itemsendswithrsplit isinstancetorchTensornumelitem)rroutkeyvalues K/mnt/rpi/tmp/demucs-venv-sys/lib/python3.13/site-packages/dora/lightning.py_filter_metricsr!s Cmmo  S\\'** h//  << CLL$:$:**S!$Q'C eU\\ * *u{{}/AJJLEC& Jc^\rSrSrU4SjrS\4Sjr\S\4Sj5rS\ 4Sjr S\ SS4S jr S\ 4S jr S \ SS4S jr S\ 4S jrS\ 4Sjr\S\4Sj5r\S\4Sj5r\S\ 4Sj5rSrU=r$)DoraEnvironment3c>[TU]5 [R"5Ul[R "5 gN)super__init__r get_distrib_specspecset_distrib_envself __class__s r r)DoraEnvironment.__init__4s* ,,. !r"returncgNTr.s r creates_children DoraEnvironment.creates_children9sr"cgr3r4r5s r creates_processes_externally,DoraEnvironment.creates_processes_externally<sr"c.URR$r')r+ world_sizer5s r r<DoraEnvironment.world_size@yy###r"sizeNcgr'r4)r.r?s r set_world_sizeDoraEnvironment.set_world_sizeC r"c.URR$r')r+rankr5s r global_rankDoraEnvironment.global_rankFsyy~~r"rEcgr'r4)r.rEs r set_global_rankDoraEnvironment.set_global_rankIrCr"c.URR$r')r+ local_rankr5s r rLDoraEnvironment.local_rankLr>r"c.URR$r')r+ node_rankr5s r rODoraEnvironment.node_rankOsyy"""r"cgNFr4r4r"r detectDoraEnvironment.detectRsr"c([RS$)N MAIN_ADDR)osenvironr5s r main_addressDoraEnvironment.main_addressVszz+&&r"c:[[RS5$)N MAIN_PORT)intrWrXr5s r main_portDoraEnvironment.main_portZs2::k*++r")r+)__name__ __module__ __qualname____firstlineno__r)boolr6propertyr9r]r<rArFrIrLrO staticmethodrSstrrYr^__static_attributes__ __classcell__r/s@r r$r$3s" $d$C$ 3 4 S C D $C$#3#D'c'',3,,r"r$c*\rSrSrSrSrSrSrSrg)DoraCheckpointSync_z>Make sure Dora history, and checkpoint state are in sync. c"[5Ulgr')r xpr5s r r)DoraCheckpointSync.__init__bs (r"cXUSnURRRU5 g)Ndora_link_history)rolinkupdate_history)r.trainer pl_module checkpointhistorys r on_load_checkpoint%DoraCheckpointSync.on_load_checkpointes#01  ##G,r"cURRRUS'URRUS'URRUS'U$)Nrrdora_sigdora_cfg)rorsrxsigcfgr.rurvrws r on_save_checkpoint%DoraCheckpointSync.on_save_checkpointisE*.'',,*>*> &'!% :!% :r")roN) r`rarbrc__doc__r)ryrrhr4r"r rlrl_s-r"rlc>^\rSrSrSrU4SjrSrSrSrSr U=r $)DoraHistoryLoggerpz,Save metrics to Dora using the XP link. cT>[TU]5 [5RUlgr')r(r)r rsr-s r r)DoraHistoryLogger.__init__ss HMM r"cSUlgr3 _first_validr.rurvs r on_fit_startDoraHistoryLogger.on_fit_startws  r"cSUlgrRrrs r on_train_epoch_start&DoraHistoryLogger.on_train_epoch_startzs !r"cUR(agURn[USS9nURR U5 g)NTr)rlogged_metricsr!rs push_metrics)r.rurvrs r on_epoch_endDoraHistoryLogger.on_epoch_end}s7    ((!'6 w'r")rrs) r`rarbrcrr)rrrrhrirjs@r rrps!"!"((r"rc\rSrSrSrSrg)_DummySLURMConnectorcgr'r4r5s r register_slurm_signal_handlers3_DummySLURMConnector.register_slurm_signal_handlersrCr"r4N)r`rarbrcrrhr4r"r rrs r"r) auto_resumeadd_dora_loggerno_unfinished_epochscV[5(d [S5e[Rn[ US5(aUR n[ US5(aM[ R"US/[U5-40UD6nUS URS5=(d /n[5n[[RR5UR55nUR5S:aXgS/- nXdS'URSS5=(d /n U R![#55 XS'US b [S 5eUS S:wa [S 5eXS 'UR$R&US '[)5R*US 'U(aUSR![-55 UR/S5n U(aAU c>[)5R*S- n U R15(a [3U 5n OSn XS'[S0UD6n U(a[55U lU $)aReturn a PL trainer, adding the necessary glue code to make everything works. The arguments are exactly the same as for `pytorch_lightning.trainer.Trainer`, with a few extras documented after. ..note:: You should not pass `gpus=` or `num_nodes=` arguments as those will be filled by Dora. Args: auto_resume (bool): if True, automatically resume previous checkpoints. You are still responsible for creating the `ModelCheckpoint` callback, this only handles the `resume_from_checkpoint` part. add_dora_logger (bool): if True, adds a Dora Logger to automatically forward the metrics (those logged with per_epoch=True), otherwise pushing metrics will be up to you. no_unfinished_epochs (bool): if True, deactivates SLURM signal handling by PL, which can result in half finished epoch with each interruption. It is recommended to instead dump a checkpoint every epoch and resume from that one so that training is reliable. z.This can only be called from inside a Dora XP. __wrapped__Nr.pluginsr ddp callbacksgpuszCYou cannot specify the number of GPUs, as this is provided by Dora. num_nodeszDYou cannot specify the number of nodes, as this is provided by Dora.default_root_dirresume_from_checkpointz last.ckptr4)r RuntimeErrorrr)hasattrrinspect getcallargslistpopr$minrcuda device_countr<appendrlr+rr folderrgetis_filergrslurm_connector)rrrargskwargsinitrenvrrrlastresumerus r get_trainerrs* 77KLL   D $ & & $ & &  vT ': Ef EFvjj#)rG  C uzz&&(#..*: ;D ~~!<9 ;-3I ')*#; f~!`aa kaabb6N((,,F;!'F {""#4#67#ZZ(@A-5x, <<>>YFF+1'(G"6"8 Nr"cX\rSrSr\R "\R5S5rSrg) _InterceptcXlX lgr'rr)r.rrs r r)_Intercept.__init__s   r"rN) r`rarbrc functoolswrapsrr)rhr4r"r rrs#__W%%&'r"rc b[[U40UD6n[UR0URD6$r')rrrrr)rr intercepts r trainer_from_argparse_argsrs-":t>v>I   ;)*:*: ;;r"cH^\rSrSrSrSU4SjjrS\SS4U4SjjrU4Sjr\ S\ 4S j5r SS \ R\\ R4S\S \4S jjr\ S 5rSrU4SjrU4SjrSrU4SjrU4SjrSrSrU4SjrU4SjrU4SjrSrSrSrSr U=r!$) PLLogProgressz<`dora.log.LogProgress` support for Pytorch-Lightning. r1Nc H>[TU]5 XlX lSUlgr')r(r)loggerr _pl_module)r.rrr/s r r)PLLogProgress.__init__s   8<r"stagec@>[TU]XU5 X l/Ulgr')r(setupr_replay_history)r.rurvrr/s r rPLLogProgress.setups  g%0#02r"c@>[TU]X5 SUlSUlg)NFT)r(r _in_trainrr.rurvr/s r rPLLogProgress.on_fit_starts W0 r"c8URceUR$r')rr5s r rvPLLogProgress.pl_modules***r"rrc0nUR5H*upV[U[5(dM[US5XE'M, U$)a/Default method to format metrics for displaying in the progress bar. To customize, you can define a `format_metrics()` method on your Lightning module. Args: metrics: dict of metrics given by PL. stage: "train" or "valid". epoch: if True, provided metrics are for the end of epoch summary. z.5f)rrfloatformat)r.rrrrrrs r format_metricsPLLogProgress.format_metricss=!--/JC%''!%/* r"cD[URSUR5$)Nr)getattrrvrr5s r _format_metricsPLLogProgress._format_metrics st~~'79L9LMMr"cURRS5 URRUS:XaSOS5 UR5SURRS-3-nUS:Xa[ UR 5nO*US:Xa[ UR5nO[SU35e[U5n[URU4X2S .URD6Ul [UR5 g) NzF----------------------------------------------------------------------trainz Training...z Validating...z | Epoch r validzInvalid stage )totalname)rinfo capitalizeru current_epochr]total_train_batchestotal_val_batchesrrangerrlogprogiter)r.rrrloaders r _on_epoch_startPLLogProgress._on_epoch_starts " %7*:P!i 0J0JQ0N/O$PP G 001E g ../Ew78 8u"4;;^e^RVR]R]^  T\\r"c`>URS5 SUlSUl[TU]X5$)NrTF)rrrr(rrs r r"PLLogProgress.on_train_epoch_starts0 W%!w+G??r"cD>URS5 [TU] X5$Nr)rr(on_validation_epoch_startrs r r'PLLogProgress.on_validation_epoch_start$s! W%w0DDr"cURURUR5n[USS9nUR X!SS9nUR R "S0UD6 [UR 5 g)NFrr4) get_metricsrurvr!rrupdatenext)r.rr formatteds r _on_batch_endPLLogProgress._on_batch_end(s^""4<<@!'7((u(E  (i( T\\r"cH>[TU]"U0UD6 URS5 g)Nr)r(on_train_batch_endrr.rrr/s r r PLLogProgress.on_train_batch_end/s# "D3F3 7#r"cH>[TU]"U0UD6 URS5 gr)r(on_validation_batch_endrrs r r%PLLogProgress.on_validation_batch_end3s# '88 7#r"cvUS:Xa=URRRRR S5SnOFURRRR RR S5Sn[ USS9nURXRRU5 g)NrFlogr) rufit_loop epoch_loop_resultsrval_loopr!_show_epoch_summaryr)r.rrs r _on_stage_endPLLogProgress._on_stage_end7s G ll++66??GGNuUGll++66??HHPPQVWX]^G!'7    (B(BGLr"c (URRXU45 URX1SS9nUR5nSR SUR 555nUR R[USUS-SU355 g)NTrz | c3P# UHupUR5SU3v M g7f)=N)r).0rvals r 4PLLogProgress._show_epoch_summary..Ds) 8IHCs~~ # '8Is$&z Summary | End of Epoch r ) rrrrjoinrrrr )r.rrrrrsummarys r r!PLLogProgress._show_epoch_summary@s ##U7$;<((t(D !** 8A8I   &>uqykWIVWXr"c>[TU]X5 UR(dUR(deUR(dUR S5 SUlggNrF)r( on_train_endrrrrs r on_validation_start!PLLogProgress.on_validation_startIsH W0~~!2!222    w '"DN!r"cv>[TU]X5 UR(aURS5 SUlgr)r(rrrrs r rPLLogProgress.on_epoch_endPs, W0 >>   w 'r"cF>[TU]X5 URS5 gr)r(on_validation_endrrs r r$PLLogProgress.on_validation_endVs !'5 7#r"cgr'r4r5s r disablePLLogProgress.disableZs r"cURS/5nU(aURRS5 UHnUR"U6 M g)Ndora_replay_historyzReplaying past metrics...)rrrr)r.rurvrwreplay_historysteps r ry PLLogProgress.on_load_checkpoint_sA#(=rB  KK  8 9"D  $ $d +#r"c$URUS'U$)Nr*)rrs r r PLLogProgress.on_save_checkpointfs,0,@,@ ()r")rrrrrrr)r1N)F)"r`rarbrcrr)rgrrrerrvtpDictAnyrdrrrrrrrrrrrrr$r'ryrrhrirjs@r rrs = 3s3t3 ! ? 27bggc266k&:!*."NN @ E$$MY# $ ,r"r)T)(rrrrWtypingr0pytorch_lightningrpytorch_lightning.callbacksr$pytorch_lightning.callbacks.progressr&pytorch_lightning.plugins.environmentsrpytorch_lightning.trainerr$pytorch_lightning.utilities.argparserrr ror r r r rr1rgr2rdr!r$rlrrrrrrr4r"r r;s  -0@E-C "RWWS"&&[1$*),(),X"((,  $(TXCL< IOIr"