
    ^i~9                        % S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKJ	r	  SSK
J
r
JrJrJr  SSKJr  Sr0 r\R$                  \S4   \S'    " S	 S
\
R*                  \   5      r " S S\R.                  5      r " S S\
R2                  5      r      SS\	S\S\S\S\S\S\S\R<                  \   SS4S jjr " S S5      r g)    N)Path   )corejob_environmentloggerutils)R   subprocess.Popen['bytes']
_PROCESSESc                   n  ^  \ rS rSr  SS\R
                  \\4   S\S\R                  \	   S\R                  S   SS4
U 4S	 jjjrSS
\S\4S jjr\S\4S j5       rSS\S\R                   \\4   4S jjrSS\SS4S jjrSS jrSS jrS\R*                  4S jrS\R*                  SS4S jrSrU =r$ )LocalJob   Nfolderjob_idtasksprocessr   returnc                    > [         TU ]  XU5        SU l        U R                  U l        X@l        U R                   H	  nXEl        M     g )NF)super__init___cancel_at_deletion	_sub_jobs_process)selfr   r   r   r   sjob	__class__s         Q/mnt/rpi/tmp/demucs-venv-sys/lib/python3.13/site-packages/submitit/local/local.pyr   LocalJob.__init__   s?     	/#( 59^^NND#M #    force_checkc                 2    U R                  5       S   nUS:g  $ )z#Override to avoid using the watcherjobStateRUNNING)get_info)r   r!   states      r   doneLocalJob.done.   s    
+	!!r    c                 d     U R                  5       R                  SS5      $ ! [         a     gf = f)zState of the jobr#   unknownUNKNOWN)r%   get	Exceptionr   s    r   r&   LocalJob.state3   s2    	==?&&z9== 		s   " 
//modec                     U R                   c1  SnU R                  R                  R                  5       (       a  SnSU0$ U R                   R	                  5       nUc  SnSU0$ US:  a  SnSU0$ SnSU0$ )z,Returns information about the job as a dict.zNO PROCESS AND NO RESULTFINISHEDr#   r$   r   INTERRUPTED)r   pathsresult_pickleexistspoll)r   r0   r&   r7   s       r   r%   LocalJob.get_info<   s    == .Ezz''..00"&&}}!!#<E
 E""	 AX!E E"" EE""r    checkc                 r    U R                   b*  U R                   R                  [        R                  5        g g N)r   send_signalsignalSIGINT)r   r9   s     r   cancelLocalJob.cancelL   s'    ==$MM%%fmm4 %r    c                 z    U R                   b.  U R                   R                  [        R                  5       5        gg)zBSends preemption / timeout signal to the job (for testing purpose)N)r   r<   LocalJobEnvironment_usr_sigr.   s    r   
_interruptLocalJob._interruptP   s-    ==$MM%%&9&B&B&DE %r    c                    U R                   (       a2  U R                  5       R                  S5      S:X  d  U R                  SS9  U R                  R
                  R                  5       (       a!  [        R                  U R                  S 5        g g )Nr#   r2   F)r9   )
r   r%   r,   r?   r4   r5   r6   r   popr   r.   s    r   __del__LocalJob.__del__U   sb    ##==?&&z2j@%(::##**,,NN4;;- -r    c                     [        U R                  5      nS US'   U R                  b  U R                  [        U R                  '   U$ )Nr   )dict__dict__r   r   r   )r   outs     r   __getstate__LocalJob.__getstate___   s:    4==!J==$&*mmJt{{#
r    r&   c                     U R                   R                  U5        [        R                  U R                  S 5      U l        g r;   )rL   updater   r,   r   r   )r   r&   s     r   __setstate__LocalJob.__setstate__f   s*    U#"t{{D9r    )r   r   r   ))r   N)F)force)Tr   N)__name__
__module____qualname____firstlineno__tpUnionr   strSequenceintOptionalr   boolr'   propertyr&   Dictr%   r?   rD   rH   AnyrN   rR   __static_attributes____classcell__r   s   @r   r   r      s    
 #'<@$s#$ $ {{3	$
 89$ 
$ $ " " "
 s  #S #rwwsCx/@ # 5D 5D 5F
.bff :"&& :T : :r    r   c                   8    \ rS rSrSSSSSSS.rS	\S
S4S jrSrg)rB   m   SUBMITIT_LOCAL_JOB_IDSUBMITIT_LOCAL_NTASKSSUBMITIT_LOCAL_JOB_NUM_NODESSUBMITIT_LOCAL_NODEIDSUBMITIT_LOCAL_GLOBALIDSUBMITIT_LOCAL_LOCALID)r   	num_tasks	num_nodesnodeglobal_rank
local_rank	countdownr   Nc                     U R                   n[        R                  " 5       R                  SU SU S35        [        R
                  " [        5        g )NzRequeued job z (z remaining timeouts))r   r   
get_loggerinfosysexitLOCAL_REQUEUE_RETURN_CODE)r   rt   jids      r   _requeueLocalJobEnvironment._requeuew   s?    kk  =R	{BV!WX*+r     )rV   rW   rX   rY   _envr^   r|   rd   r~   r    r   rB   rB   m   s.    ),3'0.D,# ,$ ,r    rB   c                     ^  \ rS rSrSr\r   SS\R                  \	\
4   S\S\S\R                  \	   SS4
U 4S	 jjjr\S\R                   \	   4S
 j5       rS\R$                  SS4U 4S jjrS\	S\\   4S jr\S\	4S j5       rS\4S jrS\	S\	S\	4S jr\S\R                  \\	4   S\	4S j5       rS\
S\R:                  \	   4S jrSrU =r $ )LocalExecutor}   a  Local job executor
This class is used to hold the parameters to run a job locally.
In practice, it will create a bash file in the specified directory for each job,
and pickle the task function and parameters. At completion, the job will also pickle
the output. Logs are also dumped in the same directory.

The submission file spawn several processes (one per task), with a timeout.


Parameters
----------
folder: Path/str
    folder for storing job submission/output and logs.

Note
----
- be aware that the log/output folder will be full of logs and pickled objects very fast,
  it may need cleaning.
- use update_parameters to specify custom parameters (n_gpus etc...).
Nr   max_num_timeoutmax_pickle_size_gbpythonr   c                    > [         TU ]  UUUS9  Uc$  [        R                  " [        R
                  5      OUU l        [        R                  R                  U R                  5      nUR                  SSS9  g )N)r   r   T)parentsexist_ok)r   r   shlexquoterx   
executabler   r   JobPathsget_first_id_independent_folderr   mkdir)r   r   r   r   r   indep_folderr   s         r   r   LocalExecutor.__init__   sg     	1+ 	 	

 6<^ekk#..1~~EEdkkR4$7r    c                     S1$ )z4Parameters that can be set through update_parameterssetupr~   )clss    r   _valid_parametersLocalExecutor._valid_parameters   s     yr    kwargsc                   > UR                  SS5      S:  a  [        S5      eUR                  SS5      nUR                  SS5      n[        U[        R                  5      (       d  [        SU S	35      e[        S
 U 5       5      (       d  [        SU S35      e[        U5      S:  a   U[        U5      :  a  [        U SU S35      e[        TU ]   " S0 UD6  g)a'  Update the parameters of the Executor.

Valid parameters are:
- timeout_min (float)
- gpus_per_node (int)
- visible_gpus (Sequence[int])
- tasks_per_node (int)
- nodes (int). Must be 1 if specified
- signal_delay_s (int): signal (lately: USR2) delay before timeout

Other parameters are ignored
nodesr      z0LocalExecutor can use only one node. Use nodes=1gpus_per_nodevisible_gpusr~   zProvided visible_gpus=z  is not an instance of Sequence.c              3   B   #    U  H  n[        U[        5      v   M     g 7fr;   )
isinstancer^   ).0xs     r   	<genexpr><LocalExecutor._internal_update_parameters.<locals>.<genexpr>   s     <|!:a%%|s   z( contains an element that is not an int.z gpus requested, but only z were specified visible.N)	r,   
ValueErrorr   rZ   r]   alllenr   _internal_update_parameters)r   r   gpus_requestedr   r   s       r   r   )LocalExecutor._internal_update_parameters   s     ::gq!A%OPPOQ7zz."5,445l^Ccdee<|<<<5l^Cklmm|q ^c,6G%G!""<\NJbc  	+5f5r    commandc                    U R                   R                  SS5      nU R                   R                  SS5      nU R                   R                  SS5      nUS:X  a  [        U5      OUS U n[        U R                  UUSR                  S U 5       5      U R                   R                  S	S
5      U R                   R                  SS5      U R                   R                  SS5      U R                   R                  SS5      S9n[        U R                  [        UR                  5      U[        [        U5      5      S9nU$ )Ntasks_per_noder   r   r   r   r~   ,c              3   8   #    U  H  n[        U5      v   M     g 7fr;   )r\   )r   ks     r   r   0LocalExecutor._submit_command.<locals>.<genexpr>   s     !7$Q#a&&$s   timeout_ming       @signal_delay_s   stderr_to_stdoutFr   )r   r   r   cuda_devicesr   r   r   r   )r   r   r   r   )

parametersr,   rangestart_controllerr   joinr   r\   pidlist)r   r   ntasksn_gpusr   gpusr   jobs           r   _submit_commandLocalExecutor._submit_command   s    $$%5q9$$_a8**>2> , 2uV}Wf8M";;!!7$!77++M3???../?D!__001CUK//%%gr2	
 $;;s7;;'7PTUZ[aUbPc
 
r    c           	          SR                  U R                  S[        R                  " [	        U R
                  5      5      /5      $ )N z-u -m submitit.core._submit)r   r   r   r   r\   r   r.   s    r   _submitit_command_str#LocalExecutor._submitit_command_str   s3    xx&CU[[QTUYU`U`QaEbcddr    c                 F    SnU R                   R                  SS5      nX-  $ )Nr   r   )r   r,   )r   r   r   s      r   
_num_tasksLocalExecutor._num_tasks   s'    "oo112BAF%%r    uidc                     g)N r~   )r   r   r   s      r   _make_submission_file_text(LocalExecutor._make_submission_file_text   s    r    stringc                     g)N0r~   )r   s    r   #_get_job_id_from_submission_command1LocalExecutor._get_job_id_from_submission_command   s     r    submission_file_pathc                     / $ r;   r~   )r   r   s     r   _make_submission_command&LocalExecutor._make_submission_command   s    	r    )r   )         ?N)!rV   rW   rX   rY   __doc__r   	job_classrZ   r[   r\   r   r^   floatr_   r   classmethodSetr   rc   r   r	   r   ra   r   r   r   staticmethodbytesr   Listr   rd   re   rf   s   @r   r   r   }   sN   * I
  !$'#'8d#8 8 "	8
 C 8 
8 8" "&&+  6BFF 6t 66s x{ , es e e&C &
# C C  BHHUCZ4H S  T bggcl  r    r   r   r   r   r   r   r   r   r   r   c                    [        [        R                  5      nUR                  [	        U5      U[	        [        SU-  5      5      [	        [        U5      5      SSU(       a  SOSSUU(       a  SOSS9
  [        R                  SS[	        U 5      /n	[        U5      n
U
(       a2  S	R                  [        U5      [        R                  " U	5      /-   5      n	[        R                  " XUS
9nU$ )zTStarts a job controller, which is expected to survive the end of the python session.<   r   1r   local)
rj   SUBMITIT_LOCAL_COMMANDSUBMITIT_LOCAL_TIMEOUT_SSUBMITIT_LOCAL_SIGNAL_DELAY_Srl   rk   SUBMITIT_STDERR_TO_STDOUTSUBMITIT_EXECUTORCUDA_VISIBLE_DEVICESSUBMITIT_LOCAL_WITH_SHELLz-mzsubmitit.local._localz && )shellenv)rK   osenvironrQ   r\   r^   rx   r   r`   r   r   r   
subprocessPopen)r   r   r   r   r   r   r   r   r   proc_cmd
need_shellr   s               r   r   r      s     rzz
CJJ!.1&!$Sk)9%:!;&)#n*=&>!%()9#r!)).#B   .Es6{SHeJ;;tE{ejj.B-CCDxsCGNr    c                       \ rS rSrSrS\4S jrS\R                  S\	R                  SS4S	 jrSS
 jrSS jrSS\S\	R                  \	R                   \      4S jjrSS\SS4S jjrSrg)
Controlleri  zThis controls a job:
- instantiate each of the tasks
- sends timeout signal
- stops all tasks if one of them finishes
- cleans up the tasks/closes log files when deleted
r   c                    [        [        R                  S   5      U l        [        R
                  " [        R                  S   5      U l        [        [        R                  S   5      U l        [        [        R                  S   5      U l        [        [        R                  S   5      U l
        / U l        / U l        / U l        [        [        R                  S   5      n[        U(       a  [        R                  " 5       O[        R                   " 5       5      U l        [%        U5      U l        [(        R(                  " [(        R*                  U R,                  5        g )Nrj   r   r   r   r   r   )r^   r   r   r   r   splitr   	timeout_sr   r`   r   r   stdoutsstderrsr\   getppidgetpidr   r   r   r=   SIGTERM_forward_signal)r   r   
with_shells      r   r   Controller.__init__  s    "**%<=>{{2::.F#GHRZZ(BCD!"**-L"MN $RZZ0K%L M02
/1/1"**%@AB
zrzz|ryy{C6lfnnd&:&:;r    signumargsr   Nc                 p    U R                    H  n UR                  U5        M     g ! [         a     M'  f = fr;   )r   r<   r-   )r   r   r   tasks       r   r   Controller._forward_signal-  s7    JJD  (   s   '
55c                 \   U R                   R                  SS9  [        U R                  5       Vs/ s H/  n[        R
                  " U R                   U R                  U5      PM1     nnU Vs/ s H  o3R                  R                  S5      PM     snU l	        U R                  (       a  U R                  O*U Vs/ s H  o3R                  R                  S5      PM     snU l        [        U R                  5       H  n[        [        R                  5      nUR!                  [#        U5      [#        U5      U R                  S9  U R$                  R'                  [(        R*                  " U R,                  SUU R                  U   U R                  U   SS95        M     g s  snf s  snf s  snf )NT)r   a)rn   rm   ri   Fzutf-8)r   r   stderrstdoutencoding)r   r   r   r   r   r   r   r  openr   r   r  r   rK   r   r   rQ   r\   r   appendr   r   r   )r   r   r4   pr   s        r   start_tasksController.start_tasks4  s<   4(CHCUVCUaTXXq9CUV49:Eqc*E:'+'<'<t||_dBe_dZ[88==QTCU_dBet{{#Arzz"CJJ'*1vs1veiemem   JJ  LL<<?<<?$	 $ W:Bes   6F-$F$:$F)c                    [         R                  [         R                  4 H  nU R                  U5        U[         R                  :X  d  M*  [	        S U R
                   5       5      (       d  MM  [        R                  " S5        [	        S U R
                   5       5      (       d  M  [        R                  " S5        M     / U l        U R                  U R                  -   n/ / sU l        U l	        U H  nUR                  5         M     g )Nc              3   D   #    U  H  oR                  5       S L v   M     g 7fr;   r7   r   ts     r   r   (Controller.kill_tasks.<locals>.<genexpr>N  s     +QjFFH,<j    gMbP?c              3   D   #    U  H  oR                  5       S L v   M     g 7fr;   r  r  s     r   r   r  P  s     <Avvx4'r  r   )r=   r>   SIGKILLr   anyr   timesleepr   r   close)r   sigfilesfs       r   
kill_tasksController.kill_tasksI  s    MM6>>2C  %fmm#+Qdjj+Q(Q(Q

5!<<<<JJsO 3 
t||+%'"dlAGGI r    freqc                    U R                   (       d   S5       eXR                  -  nXR                  U R                  -
  -  n[        U5       H  nU R                    Vs/ s H  oUR	                  5       PM     nn[        S U 5       5      (       a  Us  $ XC:X  a#  U R                  [        R                  5       5        [        R                  " SU-  5        M     U R                    Vs/ s H  oUR	                  5       PM     sn$ s  snf s  snf )zWaits for all tasks to finish or to time-out.

Returns
-------
Sequence[Optional[int]]:
    Exit codes of each task.
    Some tasks might still have not exited, but they will have received the "timed-out" signal.
zNothing to do!c              3   (   #    U  H  oS Lv   M
     g 7fr;   r~   r   es     r   r   "Controller.wait.<locals>.<genexpr>h  s     5*QD=*s   r   )r   r   r   r   r7   r   r   rB   rC   r  r  )r   r   timeoutalmost_timeoutstepr  
exit_codess          r   waitController.waitX  s     zz+++z'$2E2E!EF 'ND,0JJ7Jq&&(JJ75*555!!%$$%8%A%A%CDJJsTz" # #'**-*Q*-- 8 .s   D'D	max_retryc                    [        U5       Ha  n U R                  5         U R                  5       n[        S U 5       5      nU(       d   U R	                  5           g  U R	                  5         Mc     g ! U R	                  5         f = f)Nc              3   2   #    U  H  o[         :H  v   M     g 7fr;   )rz   r#  s     r   r   !Controller.run.<locals>.<genexpr>x  s     Qj#<<js   )r   r  r*  r  r  )r   r,  _r)  requeues        r   runController.runq  sn     y!A"  "!YY[
QjQQ!  ! " !s   9A22B)
r   r   r   r   r   r   r   r   r   r   rU   )   )   )rV   rW   rX   rY   r   r   r   r=   SignalsrZ   rc   r   r  r  r^   r]   r_   r*  r2  rd   r~   r    r   r   r     s|    <t <fnn RVV  *. .bkk"++c2B&C .2"S " " "r    r   )r   r   g      @r   Fr~   )!r   r   r=   r   rx   r  typingrZ   pathlibr   r   r   r   r   	core.corer	   rz   r   rb   r\   __annotations__Jobr   JobEnvironmentrB   PicklingExecutorr   r^   r   r`   r]   r   r   r~   r    r   <module>r>     s   
    
    7 7 
    9;
BGGC445 :M:txx{ M:`,/88 , rD)) rp "   	
    ;;s !Dh" h"r    