
    yf#U                         d dl Zd dlZd dlZd dlZd dlZd dlZd dlZddlm	Z	 ddl
mZ ddlmZmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZmZmZmZ ddlmZ ddlmZ  G d de      Z G d de	      Z y)    N   )FileDownloader)HttpFD   )aes_cbc_decrypt_bytesunpad_pkcs7)compat_os_name)Request)	HTTPErrorIncompleteRead)DownloadErrorRetryManagerencodeFilenametraverse_obj)HTTPHeaderDict)ProgressCalculatorc                       e Zd Zd ZeZy)HttpQuietDownloaderc                      y N )selfargskargss      </usr/lib/python3/dist-packages/yt_dlp/downloader/fragment.py	to_screenzHttpQuietDownloader.to_screen   s        N)__name__
__module____qualname__r   to_console_titler   r   r   r   r      s     !r   r   c                       e Zd ZdZd ZddZd Zd Zd Zd Z	d	 Z
dd
Zd Zd Zd Zd Zd Zd Zd Zd Zd d dddddZy)
FragmentFDa  
    A base file downloader class for fragmented media (e.g. f4m/m3u8 manifests).

    Available options:

    fragment_retries:   Number of times to retry a fragment for HTTP error
                        (DASH and hlsnative only). Default is 0 for API, but 10 for CLI
    skip_unavailable_fragments:
                        Skip unavailable fragments (DASH and hlsnative only)
    keep_fragments:     Keep downloaded fragments on disk after downloading is
                        finished
    concurrent_fragment_downloads:  The number of threads to use for native hls and dash downloads
    _no_ytdl_file:      Don't use .ytdl file

    For each incomplete fragment download yt-dlp keeps on disk a special
    bookkeeping file with download state and metadata (in future such files will
    be used for any incomplete download handled by yt-dlp). This file is
    used to properly handle resuming, check download file consistency and detect
    potential errors. The file has a .ytdl extension and represents a standard
    JSON file of the following format:

    extractor:
        Dictionary of extractor related data. TBD.

    downloader:
        Dictionary of downloader related data. May contain following data:
            current_fragment:
                Dictionary with current (being downloaded) fragment data:
                index:  0-based index of current fragment among all fragments
            fragment_count:
                Total count of fragments

    This feature is experimental and file format may change in future.
    c                 L    | j                  d       | j                  ||||      S )Nz{yt_dlp.downloader.FragmentFD.report_retry_fragment is deprecated. Use yt_dlp.downloader.FileDownloader.report_retry instead)deprecation_warningreport_retry)r   err
frag_indexcountretriess        r   report_retry_fragmentz FragmentFD.report_retry_fragment?   s.       "] 	^  eWjAAr   Nc                 J    |rd| dnd}| j                  d| d|dd       y )N ; z
[download]z Skipping fragment dz ...)r   )r   r(   r'   s      r   report_skip_fragmentzFragmentFD.report_skip_fragmentD   s1    #ajRC5(;Jq>NOr   c                 F    |j                  d      }|rt        |d |      S |S )Nhttp_headers)getr
   )r   	info_dicturlheaderss       r   _prepare_urlzFragmentFD._prepare_urlH   s&    --/.5wsD'*>3>r   c                 J    | j                  |       | j                  ||       y r   )_prepare_frag_download_start_frag_download)r   ctxr5   s      r    _prepare_and_start_frag_downloadz+FragmentFD._prepare_and_start_frag_downloadL   s     ##C(!!#y1r   c                 `    |d   duxr& |d   dk7  xr | j                   j                  d       S )NliveTtmpfilename-_no_ytdl_file)paramsr4   )r   r<   s     r   __do_ytdl_filezFragmentFD.__do_ytdl_fileP   s6    6{$&m3}+=+DmT[[__]lMmImmr   c                 X   d|vsJ | j                  | j                  |d         d      \  }}	 t        j                  |j	                               }|d   d   d   |d<   d|d   v r|d   d   |d<   |j                          y # t
        $ r d	|d<   Y !w xY w# |j                          w xY w)
Nytdl_corruptfilenamer
downloadercurrent_fragmentindexfragment_indexextra_stateT)sanitize_openytdl_filenamejsonloadsread	Exceptionclose)r   r<   stream_	ytdl_datas        r   _read_ytdl_filezFragmentFD._read_ytdl_fileS   s    S(((&&t'9'9#j/'JCP		

6;;=1I$-l$;<N$OPW$XC !	, 77%.|%<]%KM" LLN  	'"&C	' LLNs$   AB BB BB B)c                 F   | j                  | j                  |d         d      \  }}	 dd|d   ii}d|v r|d   |d<   |j                  d      |d   |d<   |j                  t	        j
                  d|i             |j                          y # |j                          w xY w)	NrG   wrJ   rK   rL   rM   fragment_countrI   )rN   rO   r4   writerP   dumpsrT   )r   r<   frag_index_streamrV   rI   s        r   _write_ytdl_filezFragmentFD._write_ytdl_file`   s    #11$2D2DS_2UWZ[1	&"S!12%J
 #,/,>
=)ww'(4/23C/D
+,##DJJj/I$JK##%##%s   AB B c                    d|d   |d   fz  }||xs |j                  d      ||j                  d      d}d}|d   j                  j                  d	d
      r | j                  | j                  |            }|x|d<   |d<   |d   j	                  ||      \  }	}
|	sy|j                  d      r|j                  d      |d<   ||d<   y
)Nz	%s-Frag%dr@   rL   r3   ctx_id)r6   r3   request_datara   r   dl
continuedlTfrag_resume_lenFfiletimefragment_filetimefragment_filename_sanitized)r4   rC   filesize_or_none	temp_namedownload)r   r<   frag_urlr5   r7   rb   fragment_filenamefragment_info_dictre   successrV   s              r   _download_fragmentzFragmentFD._download_fragmentp   s    '3}+=sCS?T*UU#Dy}}^'D(ggh'	
 t9d3"33DNNCT4UVOIXX,-4E0FY''(9;MN
!!*-'9'='=j'IC#$->)*r   c                     |j                  d      sy 	 | j                  |d   d      \  }}||d<   |j                         }|j	                          |S # t        $ r |j                  d      rY y  w xY w)Nrh   rbr?   )r4   rN   FileNotFoundErrorrR   rT   )r   r<   downfrag_sanitizedfrag_contents        r   _read_fragmentzFragmentFD._read_fragment   s~    ww45	#'#5#5c:W6XZ^#_ D.
 .<)*yy{

 ! 	wwv	s   A A1/A1c                    	 |d   j                  |       |d   j                          | j                  |      r| j                  |       | j                  j                  dd      s| j                  t        |d                |d= y # | j                  |      r| j                  |       | j                  j                  dd      s| j                  t        |d                |d= w xY w)Ndest_streamkeep_fragmentsFrh   )r\   flush_FragmentFD__do_ytdl_filer_   rC   r4   
try_remover   )r   r<   rv   s      r   _append_fragmentzFragmentFD._append_fragment   s    	3$$\2$$&""3'%%c*;;??#3U;s3P/Q RS12	 ""3'%%c*;;??#3U;s3P/Q RS12s   'B A C(c           
      6   |j                  dd      s%d|d   z  }|j                  dd      }|r|d|z  z  }nd}| j                  d	| j                   d
|        | j	                  |d          t        | j                  i | j                  dddddd      }| j                  |d         }d}| j                  |      }|dkD  rd}|j                  |dd       | j                  |      rt        j                  j                  t        | j!                  |d                     }| j                  j                  dd      }	|	rs|rq| j#                  |       |j                  d      du }
|d   dkD  xr |dk(  }|
s|rq|
rdnd}| j%                  d|z         dx|d<   }d|v r|d= | j'                  |       n7|	s|r| j#                  |       dx|d<   }| j'                  |       |d   dk(  sJ | j)                  ||      \  }}|j                  ||||d       y )Nr?   F%dtotal_fragsad_fragsr    (not including %d ad)unknown (live)[] Total fragments: rG   T)
noprogresstestsleep_intervalmax_sleep_intervalsleep_interval_subtitleswbabr@   rL   rd   rF   rL   z.ytdl file is corruptz2Inconsistent state of incomplete fragment downloadz%%s. Restarting from the beginning ...)rc   ry   r@   complete_frags_downloaded_bytes)
setdefaultr4   r   FD_NAMEreport_destinationr   ydlrC   rj   ri   updater|   ospathisfiler   rO   rX   report_warningr_   rN   )r   r<   total_frags_strr   rc   r@   	open_mode
resume_lenytdl_file_existsrd   
is_corruptis_inconsistentmessagery   s                 r   r:   z!FragmentFD._prepare_frag_download   sa   ~~fe,"S%77Owwz1-H#;h#FF.O4<<.(;O;LMNJ0  ,
kk,
"#(),
  nnS_5	 **;7
>I 	

&
 	
 s#!ww~~nT=O=OPST^P_=`.abt<J.$$S) WW^4<
"%&6"7!";"O
a3=/L  ''?'IK9::C()J%,/))#. "',,S19::C()J%%c*+,111#'#5#5k9#M [

&&/9
 	r   c                      d   }d   j                  d      d|d   d   d   dt        j                         d	<   t        |       fd
}d   j                  |       d	   S )Nr   r   ra   downloadingrL   rG   r@   )statusdownloaded_bytesrL   r[   rG   r@   startedc                 0   | d   dvry 	sj                  d      rd   d<   | j                  d      k7  ry j                  d      d<   j                  d      d<   j                  d<   | j                  d      xs d	}| j                  d
i       | d<   d   sKd   |z   d   dz   z  	z  }|_        j	                  | j                  d             j                  d<   n j	                  | j                  d             | d   dk(  r%dxx   dz  cc<   d   d<   j                          j                  xd<   d<   j                  j                  xd<   d<   j                  j                  d<   j                         y )Nr   )r   finishedr[   ra   max_progressprogress_idxelapsedtotal_bytesr   r5   rn   r?   r   rL   r   r   total_bytes_estimater   speedeta)r4   r   poptotalr   thread_reset
downloadedr   smoothr   _hook_progress)
sfrag_total_bytesestimated_sizer<   ra   r5   progressr   stater   s
      r   frag_progress_hookz;FragmentFD._start_frag_download.<locals>.frag_progress_hook   s   {"==377+;#<*-.>*?&'!aeeHo&?$'GGN$;E.!$'GGN$;E.!'//E) uu]38q&'eeK&<A"# v;:;>NN-.246AB  "0&8 9:08,-&8 9:{j(&'1,'(-.>(?$%%%'QYQdQddE$%,M(N,4NN,A,AAE'NS\#<<..E%Ly1r   rc   )r4   timer   add_progress_hook)	r   r<   r5   r   r   ra   r   r   r   s	   ```  @@@@r   r;   zFragmentFD._start_frag_download   s    :;
-(" $ *!"23)J}-
 I%j1%	2 %	2N 	D	##$679~r   c                 @   |d   j                          | j                  |      r#| j                  | j                  |d                t	        j                         |d   z
  }|d   dk7  }|r| j                  |d         }n|d   }|s(|r| j                  |d          | j                  d       y|r| j                  |d   |d          |j                  d	      }| j                  j                  d
d      rQ|rOt        j                  t              5  t        j                  |d   t	        j                         |f       d d d        | j                  |||d   d||j                  d      |j                  d      |j                  d      d|       y# 1 sw Y   UxY w)Nry   rG   r   r@   rA   r   zThe downloaded file is emptyFrg   
updatetimeTr   ra   r   r   )r   r   rG   r   r   ra   r   r   )rT   r|   r}   rO   r   ri   report_error
try_renamer4   rC   
contextlibsuppressrS   r   utimer   )r   r<   r5   r   to_filer   rf   s          r   _finish_frag_downloadz FragmentFD._finish_frag_download  s   M  "s#OOD..s:?@))+I.m$+#44S5GH"#DEM 23<=OOC.J@ww23H{{|T2x((3 GHHS_tyy{H.EFG 	 0+J ggh'GGN3GGN3	
 		 G Gs   .FFc                     d|vrd|d<   |d   s%d|d   z  }|j                  dd      }|r|d|z  z  }nd}| j                  d	| j                   d
|        | j                  |d         }|j	                  |dd       y )Nr?   Fr   r   r   r   r   r   r   r   rG   r   )r4   r   r   rj   r   )r   r<   r   r   r@   s        r   _prepare_external_frag_downloadz*FragmentFD._prepare_external_frag_downloadB  s    CK6{"S%77Owwz1-H#;h#FF.O4<<.(;O;LMNnnS_5 	

&
 	r   c                 .     i  fd fd}|S )Nc                     | vr<j                   j                  j                  |             j                         | <   |    S r   )r   urlopenr8   rR   )r6   
_key_cacher5   r   s    r   _get_keyz&FragmentFD.decrypter.<locals>._get_keyY  sD    *$"&(("2"243D3DYPS3T"U"Z"Z"\
3c?"r   c                 h   |y | j                  d      }|r|d   dk7  r|S |j                  d      xs t        j                  d| d         }|j                  d      xs  t        d      xs |d	         |d<   j                  j                  d
d      r|S t        t        ||d   |            S )Ndecrypt_infoMETHODzAES-128IVz>8xqmedia_sequenceKEY)hls_aesuriURIr   F)r4   structpackr   rC   r   r   )fragmentrv   r   ivr   r5   r   s       r   decrypt_fragmentz.FragmentFD.decrypter.<locals>.decrypt_fragment^  s    ##<<7L<#9Y#F##!!$'Z6;;vxHX?Y+ZB#/#3#3E#: $s&.|IGY/Z/q^jkp^q&r 
 {{vu-##4\<PUCVXZ[\\r   r   )r   r5   r   r   r   s   `` @@r   	decrypterzFragmentFD.decrypterV  s    
	#
	]   r   c           	          dgt        |      dk(  r  j                  |d   i S  j                  j                  dd      }dkD  r j	                         t        t        |d            } fd} G d dt        j                  j                        }t        d	k(  rd
 }nd }fd}g }	t        |      D ]V  \  }
\  }}} |t        j                  |z              }|j                  ||
| ||      ||      }|	j                  ||f       X d}|	D ]$  \  }}	 |xr  ||      }|j#                  d       & d   s|s
t!               |S # t         $ r dd<   Y 7w xY w# |j#                  d       w xY w)z
        @params (ctx1, fragments1, info_dict1), (ctx2, fragments2, info_dict2), ...
                all args must be either tuple or list
        Tr   r   concurrent_fragment_downloads).r   is_livec                 L    |d<   | |d<    j                   |||fi |dS )Nr   r   )tpeinterrupt_trigger)download_and_append_fragments)	idxr<   	fragmentsr5   r   r   kwargsr   r   s	        r   thread_funczFFragmentFD.download_and_append_fragments_multiple.<locals>.thread_func~  sJ    ".C"%C5455Y	c-3c9<Pac cr   c                       e Zd Zd Zy)?FragmentFD.download_and_append_fragments_multiple.<locals>.FTPEc                      y r   r   )r   exc_typeexc_valexc_tbs       r   __exit__zHFragmentFD.download_and_append_fragments_multiple.<locals>.FTPE.__exit__  s    r   N)r   r   r    r   r   r   r   FTPEr     s    r   r   ntc                     	 	 | j                  d      S # t        $ r  t        j                  j                  $ r Y ;w xY w)Ng?)resultKeyboardInterrupt
concurrentfuturesTimeoutErrorfutures    r   future_resultzHFragmentFD.download_and_append_fragments_multiple.<locals>.future_result  sB    !%}}S11, %--:: ! !s    &==c                 "    | j                         S r   )r   r   s    r   r   zHFragmentFD.download_and_append_fragments_multiple.<locals>.future_result  s    }}&r   c              3   2   K   | D ]  }d   s y |  y w)Nr   r   )fgfr   s     r   interrupt_trigger_iterzQFragmentFD.download_and_append_fragments_multiple.<locals>.interrupt_trigger_iter  s&      (+s   Fwait)lenr   rC   r4   _prepare_multiline_statusanyr   r   r   ThreadPoolExecutorr	   	enumeratemathceilsubmitappendr   shutdown)r   r   r   max_workersr   r   r   r   r   spinsr   r<   r   r5   r   jobr   r   r   s   ` `              @@r   &download_and_append_fragments_multiplez1FragmentFD.download_and_append_fragments_multiplep  s   
 "F4y15455tAwI&IIkkoo&EqI!**<8l4)<=>	c	:%%88 	
 T!!'	 09$ 	%,C,#y)tyy|!;<=C**[#s4J94UW`befCLL#s$	%
  	(HC(6M#$6 $'	( !#G#%%  % -',!!$- $'s$   %EE)&E,(E))E,,F c                      y)NFr   )r   s    r   <lambda>zFragmentFD.<lambda>      r   c                     | S r   r   )contentr   s     r   r  zFragmentFD.<lambda>  s    G r   )T)is_fatal	pack_funcfinish_funcr   r   c          
           j                   j                  dd      sd  fd fd}	 j                        }
t        j                   j                   j                  dd      j                  dd      z        }|dkD  rfd	}|xs t
        j                  j                  |      5 }	 |j                  ||      D ]D  \  }}}j                  ||d
        |	 |
| j                              |      r< d d d        y 	 d d d        n?|D ]:  }d   s n3	  |        |	 |
| j                              |d         }|r: y |,d   j                   |              d   j!                           j#                        S # t        $ r8  j                           j                  ddd       |j                  d        w xY w# 1 sw Y   xY w# t        $ r j                  d      rY   w xY w)Nskip_unavailable_fragmentsTc                      y)NTr   )rV   s    r   r  z:FragmentFD.download_and_append_fragments.<locals>.<lambda>  r  r   c           
      2   
d   sy | d   xd<   d d<   t        	j                  d            }| j                  d      }|rd|d   |d	   d
z
  fz  |d<    | j                  d      xs d
z
        fd}t        j                  j                  d      |      D ]@  }	 | j                  d      d<   j	                  | d   	|	j                  d            s y B y # t
        t        f$ r}||_        Y d }~_d }~wt        $ r r Y pw xY w)Nr   r(   rL   
last_errorr3   
byte_rangezbytes=%d-%dstartendr   RangerK   c                 n    r||kD  rd   j                          j                  | ||       | d<   y )Nry   r  )rT   r&   )r'   r)   r*   r<   fatalr(   r   s      r   error_callbackz[FragmentFD.download_and_append_fragments.<locals>.download_fragment.<locals>.error_callback  s=    UW_&,,.!!#ugz5I$'L!r   fragment_retriesr[   r6   rb   )	r   r4   r   rC   rp   r   r   errorr   )r   r<   r7   r  r   retryr'   r  r(   r5   r   r  r   s    `     @@r   download_fragmentzCFragmentFD.download_and_append_fragments.<locals>.download_fragment  s>   $Q'19,1GGJ-. $C$Y]]>%BCG!l3J#0Jw4GTYIZ]^I^3_#_  X\\'2FzA~GE( &dkkoo6H&I>Z 
,4LL9I,JC()22%)WimmTbFcee ">2 "%EK$  s   );C))D8DDDc                     | rj                  | | |             y |dz
        sj                  |d       y|d   j                          j                  d| d       y)Nr   zfragment not foundry   z	fragment z not found, unable to continueFT)r~   r1   rT   r   )rv   r(   r<   r  r  r   s      r   append_fragmentzAFragmentFD.download_and_append_fragments.<locals>.append_fragment  st    %%c9\:+NO  j1n-))*6JK
  M"((*!!Ij\9W"XYr   r   r   r   c                 d    j                         } | |       | | d   |j                  d      fS )Nr(   rh   )copyr4   )r   ctx_copyr<   r$  s     r   rp   zDFragmentFD.download_and_append_fragments.<locals>._download_fragment  s4    88:!(H5,!7Fc9dddr   )rh   rL   Fz;Interrupted by user. Waiting for all threads to shutdown...)is_errortbr   r   r(   r   ry   )rC   r4   r   r  r  r   r   r  mapr   rw   r   _finish_multiline_statusr   r  r\   r{   r   )r   r<   r   r5   r  r  r  r   r   r&  r   r	  rp   poolr   r(   frag_filenamer   r$  s   `` ```  `         @r   r   z(FragmentFD.download_and_append_fragments  s,   
 {{;TB%H	B		  >>)4iiKKOO;Q?#''.Z[B\\^?e
 J
**==kJ d?CxxHZ\e?f );*m

;H.8$   //?$J]J]^aJb/ceoqtu#( )   & !(+%h4,(43F3Fs3KLhWcNdfikF  ! "$$[]3$$&))#y991 ) 113%%U`ejo & qMMuM- . )  }}Y/s>   7G9AFFF),G+AGGG(+H
H
r   )NN)r   r   r    __doc__r+   r1   r8   r=   r|   rX   r_   rp   rw   r~   r:   r;   r   r   r   r  r   r   r   r   r#   r#      sv    !FB
P?2n& *	3BH:x"H( 4>B ;L3$_:r   r#   )!concurrent.futuresr   r   rP   r  r   r   r   commonr   httpr   aesr   r   compatr	   
networkingr
   networking.exceptionsr   r   utilsr   r   r   r   utils.networkingr   utils.progressr   r   r#   r   r   r   <module>r;     sR        	   "  4 #   = M M - /!& !t: t:r   