
    #HJj              	         d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlmZmZ ddlmZ ddlmZmZmZmZ 	 ddlZddlZddlZddlZddlZddlZddlZ ddl!Z ddl"Z ddl m#Z# er,dd	l$m%Z% dd
l&m'Z' ddl(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2  ejf                  e4      Z5dZ6	 dZ7	 dZ8	 dZ9dZ:dZ;dZ< e	jz                  d      Z> G d de      Z? G d d      Z@d eA       vr e@       ZB G d d      ZCd7dZDd8d ZEd9d!ZFde:e7dd"ddddf		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d:d#ZG	 	 	 	 	 	 	 	 	 	 	 	 d;d$ZHd<d%ZI G d& d'      ZJ	 	 	 	 	 	 	 	 	 	 	 	 d=d(ZK G d) d*ej                        ZM G d+ d,ej                        ZN G d- d.ej                        ZOd>d/ZP	 	 	 	 	 	 	 	 d?	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d@d1ZQd0d2 d3	 	 	 	 	 	 	 	 	 dAd4ZR	 	 	 	 	 	 	 	 	 	 	 	 dBd5ZS	 	 	 	 	 	 	 	 	 	 dCd6ZTy# e$ r dZY w xY w)DzDImplements file-like objects for reading and writing from/to AWS S3.    )annotationsN)CallableIterator)inf)TYPE_CHECKINGAny	TypedDictcastT)	constants)TracebackType)WriteableBuffer)StreamingBody)S3Client)CompletedMultipartUploadTypeDef)BufferSelf)TransportParamsi  P i   l        )s3s3ns3ai   )zs3://my_bucket/my_keyz&s3://my_key:my_secret@my_bucket/my_keyInvalidRangez'(?P<sep>[?&])versionId=(?P<value>[^&]*)c                  J    e Zd ZU ded<   ded<   ded<   ded<   ded<   ded<   y	)
_S3Uristrscheme	bucket_idkey_id
str | None	access_idaccess_secret
version_idN)__name__
__module____qualname____annotations__     W/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/smart_open/s3.pyr   r   _   s#    KNKr'   r   c                       e Zd ZdZddZddZy)RetryzJRetry policy used by the S3 transport for transient client/network errors.c                p    d| _         d| _        t        j                  j                  g| _        dg| _        y )N   
   NoSuchUpload)attemptssleep_secondsbotocore
exceptionsEndpointConnectionErrorclient_error_codesselfs    r(   __init__zRetry.__init__k   s2    "$6>6I6I6a6a5b.<-=r'   c                   t        | j                        D ]  }	  |       c S  t        j                  d       |j                   d| j                   d	}t!        |      # t        | j                        $ re}t        j                  d|| j                  |z
  dz
         t        j                  d       t        j                  | j                         Y d }~d }~wt        j                  j                  $ r}|j                  d   j                  d      }|| j                  vr t        j                  d|| j                  |z
  dz
         t        j                  d       t        j                  | j                         Y d }~pd }~ww xY w)
Nz+Caught non-fatal %s, retrying %d more times   zretryable errorErrorCodez9Caught non-fatal ClientError (%s), retrying %d more timesz0encountered too many non-fatal errors, giving upz failed after 	 attempts)ranger/   tupler2   loggercritical	exceptiontimesleepr0   r1   ClientErrorresponsegetr4   funcOSError)r6   fnattempterr
error_codemsgs         r(   _doz	Retry._doq   s@   T]]+G/t ,, 	JK	i@cl+ ) /AMMG+a/
   !23

4--..&&22 
/ \\'266v>
T%<%<<OMMG+a/
   !23

4--..
/s%   A  F8AC F8BFFNreturnNone)rI   zfunctools.partial[Any]rP   r   )r"   r#   r$   __doc__r7   rN   r&   r'   r(   r*   r*   h   s    T>r'   r*   MISSING_DEPSc                       e Zd ZdZddZddZy)_ClientWrapperax  Wraps a client to inject the appropriate keyword args into each method call.

    The keyword args are a dictionary keyed by the fully qualified method name.
    For example, S3.Client.create_multipart_upload.

    See https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#client

    This wrapper behaves identically to the client otherwise.
    c                     || _         || _        y N)clientkwargs)r6   rX   rY   s      r(   r7   z_ClientWrapper.__init__   s    r'   c                    t        | j                  |      }| j                  j                  d| i       }t	        j
                  |fi |S )Nz
S3.Client.)getattrrX   rY   rF   	functoolspartial)r6   method_namemethodrY   s       r(   __getattr__z_ClientWrapper.__getattr__   sC    k2:k]!;R@  2622r'   N)rX   r   rY   dict[str, Any]rP   rQ   )r^   r   rP   zCallable[..., Any])r"   r#   r$   rR   r7   r`   r&   r'   r(   rU   rU      s    3r'   rU   c                p   t         j                  j                  |       }|j                  t        v sJ d\  }}|j
                  |j                  z   }d|v r8|j                  dd      \  }}d|v r|j                  dd      \  }}d|vr||}}|}|j                  dd      \  }	}
d}t        j                  |
      }|ri|j                  d      }|
d|j                          }|
|j                         d }|j                  d      d	k(  r|j                  d
      rd	|dd z   }||z   }
|j                  |	|
|||dS )zLParse an ``s3://`` URI into bucket, key, credential, and version components.NN@r9   :/Nvaluesep?&)r   r   r   r   r    r!   )
smart_openutilssafe_urlsplitr   SCHEMESnetlocpathsplit_VERSION_ID_REsearchgroupstartend
startswith)uri_as_string	split_urir   r    uri
maybe_authrestmaybe_idmaybe_secretr   r   r!   matchbeforeafters                  r(   	parse_urir      s^      ..}=Iw&&&
  *I} 

Y^^
+C cz99S!,
D*%/%5%5c1%="Hl("+3\=			#q)Iv J!!&)E[[)
%++-(uyy{}%;;u$)9)9#)>%)OE% ""&  r'   c                   t              dfd}j                  d      }|3| d   s| d   r)t        j                  d       | j	                  dd       n-| d   r(| d   r# || d   | d          | j	                  dd       | d	   Aj                  d	      t        j                  d
       n| d	   d	<   | j	                  d       | fS )a`  Consolidates the parsed Uri with the additional parameters.

    This is necessary because the user can pass some of the parameters can in
    two different ways:

    1) Via the URI itself
    2) Via the transport parameters

    These are not mutually exclusive, but we have to pick one over the other
    in a sensible way in order to proceed.

    c                     	 d   }	 |d   } |j                  di |  y # t         $ r
 i x}d<   Y +w xY w# t         $ r
 i x}|d<   Y ;w xY w)Nclient_kwargs	S3.Clientr&   )KeyErrorupdate)rY   r   init_kwargstransport_paramss      r(   injectz#_consolidate_params.<locals>.inject  sy    	C,_=M	:'4K 	$V$  	C@BBM,_=	C
  	:799K-4	:s   ! 7 44A
	A
rX   Nr   r    zignoring credentials parsed from URL because they conflict with transport_params["client"]. Set transport_params["client"] to None to suppress this warning.)r   r    )aws_access_key_idaws_secret_access_keyr!   zignoring versionId parsed from URL because it conflicts with transport_params["version_id"]. Drop the URL versionId to suppress this warning.)r!   )rY   r   rP   rQ   )dictrF   r?   warningr   )rz   r   r   rX   s    `  r(   _consolidate_paramsr      s     ,-% !!(+Fs;/33G(	

 	

T
6	[	c/2!+."%o"6	
 	

T
6
<$-9NN) .1->\*

d
#   r'   c                    t        |       }t        ||      \  }}t        j                  j	                  t
        |      }t        |d   |d   |fi |S )z9Open an S3 URI using the given mode and transport params.r   r   )r   r   rk   rl   check_kwargsopen)rz   moder   
parsed_urirY   s        r(   open_urir   .  sU    3J#6zCS#T J **41ABF
;'H)=tNvNNr'   Fc           
        t         j                  dt                      |t        j                  vr!d|dt        j                  }t        |      |t        j                  k(  r|d}t        |      |t        j                  k(  rt        | ||||||	|      }nH|t        j                  k(  r%|rt        | |||	|
|      }n!t        | |||	|
      }nd|}t        |      ||_        |S )	a  Open an S3 object for reading or writing.

    Args:
        bucket_id: The name of the bucket this object resides in.
        key_id: The name of the key within the bucket.
        mode: The mode for opening the object.  Must be either "rb" or "wb".
        version_id: Version of the object, used when reading object.
            If None, will fetch the most recent version.
        buffer_size: Default: 128KB.
            The buffer size in bytes for reading. Controls memory usage. Data is
            streamed from a S3 network stream in buffer_size chunks. Forward seeks
            within the current buffer are satisfied without additional GET requests.
            Backward seeks always open a new GET request. For forward seek-intensive
            workloads, increase buffer_size to reduce GET requests at the cost of
            higher memory usage.
        min_part_size: The minimum part size for multipart uploads, in bytes.
            When the writebuffer contains this many bytes, smart_open will upload
            the bytes to S3 as a single part of a multi-part upload, freeing the
            buffer either partially or entirely.  When you close the writer, it
            will assemble the parts together.
            The value determines the upper limit for the writebuffer.  If buffer
            space is short (e.g. you are buffering to memory), then use a smaller
            value for min_part_size, or consider buffering to disk instead (see
            the writebuffer option).
            The value must be between 5MB and 5GB.  If you specify a value outside
            of this range, smart_open will adjust it for you, because otherwise the
            upload _will_ fail.
            For writing only.  Does not apply if you set multipart_upload=False.
        multipart_upload: Default: `True`.
            If set to `True`, will use multipart upload for writing to S3. If set
            to `False`, S3 upload will use the S3 Single-Part Upload API, which
            is more ideal for small file sizes.
            For writing only.
        defer_seek: Default: `False`.
            If set to `True` on a file opened for reading, GetObject will not be
            called until the first seek() or read().
            Avoids redundant API queries when seeking before reading.
        client: The S3 client to use when working with boto3.
            If you don't specify this, then smart_open will create a new client for
            you.
        client_kwargs: Additional parameters to pass to the relevant functions of
            the client. The keys are fully qualified method names,
            e.g. `S3.Client.create_multipart_upload`.
            The values are kwargs to pass to that method each time it is called.
        writebuffer: By default, this module will buffer data in memory using
            io.BytesIO when writing. Pass another binary IO instance here to use it
            instead. For example, you may pass a file object to buffer to local disk
            instead of in RAM. Use this to keep RAM usage low at the expense of
            additional disk IO. If you pass in an open file, then you are
            responsible for cleaning it up after writing completes.
        range_chunk_size: Default: `None`.
            Maximum byte range per S3 GET request when reading.
            When None (default), a single GET request is made for the entire file,
            and data is streamed from that single botocore.response.StreamingBody
            in buffer_size chunks.
            When set to a positive integer, multiple GET requests are made, each
            limited to at most this many bytes via HTTP Range headers. Each GET
            returns a new StreamingBody that is streamed in buffer_size chunks.
            Useful for reading small portions of large files without forcing
            S3-compatible systems like SeaweedFS/Ceph to load the entire file.
            Larger values mean fewer billable GET requests but higher load on S3
            servers. Smaller values mean more GET requests but less server load per
            request. Values larger than the file size result in a single GET for the
            whole file. Affects reading only. Does not affect memory usage
            (controlled by buffer_size).

    Returns:
        A file-like object for reading or writing.

    Raises:
        AssertionError: If the mode is somehow neither read nor write binary.
        NotImplementedError: If the mode is not "rb" or "wb".
        ValueError: If version_id is specified for write mode.
    z%rz
bad mode: z expected one of z$version_id must be None when writing)r!   buffer_size
defer_seekrX   r   range_chunk_size)rX   r   writebuffer	part_size)rX   r   r   zunexpected mode: )r?   debuglocalsr   BINARY_MODESNotImplementedErrorWRITE_BINARY
ValueErrorREAD_BINARYReaderMultipartWriterSinglepartWriterAssertionErrorname)r   r   r   r!   r   min_part_sizemultipart_uploadr   rX   r   r   r   rM   fileobjs                 r(   r   r   6  s	   p LLvx 9)))4("3I4J4J3MN!#&&	&&&Z-C4oy$$$!#!'-	
 
''	'%+''G '+'G "$*S!!GLNr'   c                    	 ||d}|r||d<   |r||d<   t        d | j                  d	i |      S # t        j                  j                  $ r&}t        d|d|d|d|       }||_        ||d }~ww xY w)
NBucketKey	VersionIdRangera   zunable to access bucket: z key: z
 version: z error: r&   )r
   
get_objectr1   rX   rD   rH   backend_error)rX   bucketkeyversionrange_stringparamserrorwrapped_errors           r(   _getr     s    ',23!?")F;*F7O$&7f&7&7&A&&ABB??&& ''zwjS[\a[bc
 ',#&'s   .1 A4!A//A4c                `    	 | j                   j                  d   S # t        t        f$ r Y yw xY w)zHGiven an IOError from _get, return the 'Error' dictionary from botocore.r:   N)r   rE   AttributeErrorr   )ioes    r(   _unwrap_ioerrorr     s5      ))'22H% s    --c                      e Zd ZdZ	 	 d
	 	 	 	 	 	 	 	 	 	 	 ddZedd       ZddZej                  fddZ
d
ddZdddZdd	Zy)_SeekableRawReaderzHRead an S3 object.

    This class is internal to the S3 submodule.
    Nc                t    || _         || _        || _        || _        || _        d | _        d| _        d | _        y )Nr   )_client_bucket_key_version_id_range_chunk_size_content_length	_position_body)r6   rX   r   r   r!   r   s         r(   r7   z_SeekableRawReader.__init__  s?     	%!1+/8<
r'   c                    | j                   d u S rW   )r   r5   s    r(   closedz_SeekableRawReader.closed  s    zzT!!r'   c                P    | j                   }||j                          d | _         y y rW   )r   close)r6   bodys     r(   r   z_SeekableRawReader.close  s&    zzJJLDJ r'   c                   |t         j                  vrdt         j                  }t        |      | j                          d}d}|t         j                  k(  rt        d|      }nM|t         j                  k(  rt        d|| j                  z         }n |t         j                  k(  rt        d|       }| j                  0|+|dk(  r&| j                  dd       | j                          d}nd}n||| j                  k\  s|dk(  rd}nd}|rDt        j                         | _        | j                  J | j                  | _        | j                  S | j                  ||       | j                  S )a  Seek to the specified position.

        Args:
            offset: The offset in bytes.
            whence: Where the offset is from.

        Returns:
            The position after seeking.

        Raises:
            ValueError: If whence is not one of the supported values.
        z invalid whence, expected one of Nr   ru   stopTF)r   WHENCE_CHOICESr   r   WHENCE_STARTmaxWHENCE_CURRENTr   
WHENCE_ENDr   
_open_bodyioBytesIOr   )r6   offsetwhencerM   ru   r   reached_eofs          r(   seekz_SeekableRawReader.seek  sQ    1114Y5M5M4PQCS/!
 	

Y+++6NEy///6DNN23Ey+++q6'?D '} aa0

"#ET-A-A$AdaiKKDJ''333!11DN ~~ OOE4(~~r'   c                   ||| j                   }|G| j                  ;|J || j                  z   dz
  }| j                  t        || j                  dz
        }t        j
                  j                  ||      }	 t        | j                  | j                  | j                  | j                  |      }t*        j-                  d| |d   d	          |d   d
   }|t.        j0                  j2                  k(  r@t        j
                  j5                  |d         \  }	}
}	}|
| _         || _        |d   | _        y|t.        j0                  j6                  k(  r|d   }|d   }d| _         || _        || _        |t        ||      }n||t9        d||z
        }nd}|dkD  r7t*        j-                  d| |       t;        |j=                  |            | _         yyd|}t?        |      # t        $ r}t        |      }||j                  d      t        k7  r t!        |j                  dd            }|||k\  s	|/|-||k\  r(|x| _         | _        t#        j$                         | _        n| j)                  ||dz
         Y d}~yd}~ww xY w)a  Open a connection to download the specified range of bytes.

        Store the open file handle in self._body.

        If no range is specified, start defaults to self._position.
        start and stop follow the semantics of the http range header,
        so a stop without a start will read bytes beginning at stop.

        If self._range_chunk_size is set, the S3 server is protected from open range
        headers and stop will be set such that at most self._range_chunk_size bytes
        are returned in a single GET request.

        As a side effect, set self._content_length. Set self._position
        to self._content_length if start is past end of file.
        Nr9   r;   ActualObjectSizer   r   z%s: RetryAttempts: %dResponseMetadataRetryAttemptsHTTPStatusCodeContentRangeBodyContentLengthz2%s: discarding %d bytes to reach expected positionzUnexpected status code ) r   r   r   minrk   rl   make_range_stringr   r   r   r   r   rH   r   rF   _OUT_OF_RANGEintr   r   r   r   r?   r   http
HTTPStatusPARTIAL_CONTENTparse_content_rangeOKr   lenreadr   )r6   ru   r   r   rE   r   error_responseactual_object_sizestatus_code_
resp_startlengthcontent_lengthr   expected_positionrM   s                   r(   r   z_SeekableRawReader._open_body7  s     =T\NNE <D22>$$$4111A5D##/4!5!5!9:!''99%F			  H@ 	#'(9	
 123CD$//999'1'7'7'K'KHUcLd'e$Az1f'DN#)D !&)DJDOO...%o6NF#DDN#1D DJ
  $'$>!4#3$'>D+@$A!$%! 1$H%
 "%TYY/@%A!B % ,K?;CS/!G  	,S1N%););F)C})T!$^%7%78JA%N!O "u0B'BMd&64CU;U8JJ!5ZZ\
e2Dq2HI#	s   ;7G" "	J+BI??Jc                    dk  rd }t        |      dk(  rt        nt        j                         dd fd} j                  , j
                   j                  k  rj                         k  r |       } xj
                  t        |      z  c_        j                  |       |s j                           j                  U j
                   j                  k  rj                         k  rj                         S )z9Read from the continuous connection with the remote peer.zsize must be >= -1, got c           	     b   | D ]m  }j                   rj                          j                  }|J 	 t        k(  r|j	                         c S |j	                  j                         z
        c S   dt        k(  rdn dt#        |        d}t%        |      # t        t        j                  j                  t        j                  j                  f$ rT}t        j                  d|t        k(  rdn|       j                          t        j                   |       Y d }~%d }~ww xY w)Nz?%s: caught %r while reading %d bytes, sleeping %ds before retryr   z: failed to read z bytes after r<   )r   r   r   r   r   tellConnectionResetErrorr1   r2   BotoCoreErrorurllib3	HTTPErrorr?   r   r   rB   rC   r   rH   )	r/   secondsr   rK   rM   binary_collectedr6   size
size_limits	        r(   
retry_readz+_SeekableRawReader.read.<locals>.retry_read  s   #;;OO%zz'''(!S(#yy{*99T,<,A,A,C%CDD $. F+*2CB+Nm\_`h\i[jjstC  )''55&&00 (
 NNY(C/T JJLJJw''(s   B!B<D.A	D))D.))r9               )r/   ztuple[int, ...]rP   bytes)r   r   r   r   r   r   r   r   writer   getvalue)r6   r  rM   r  binaryr  r  s   ``   @@r(   r   z_SeekableRawReader.read  s    "9,TF3CS/!)-#
::<	 	<   (!5!55$))+j8  \FNNc&k)N""6*

   (!5!55$))+j8  ((**r'   c                <    d| j                   d| j                  dS )zAReturn a short human-readable description of the seekable reader.zsmart_open.s3._SeekableReader(, )r   r   r5   s    r(   __str__z_SeekableRawReader.__str__       //?r$))aPPr'   rc   )rX   r   r   r   r   r   r!   r   r   
int | NonerP   rQ   rP   boolrO   r   r   r   r   rP   r   )ru   r  r   r  rP   rQ   r   r  r   rP   r  rP   r   )r"   r#   r$   rR   r7   propertyr   r   r   r   r   r   r   r  r&   r'   r(   r   r     s     "&'+== = 	=
 = %= 
=$ " " /8.D.D 8ti"VA+FQr'   r   c                   |i }|S|j                  di       }d|vr(t        j                  j                  ddddd	      |d<   t	        j                  di |}|sJ t        d
t        ||            | _        || _        || _	        y)zFCreate the boto3 client/bucket/key wrappers required for accessing S3.Nr   config@   Tr,   adaptivemax_attemptsr   max_pool_connectionstcp_keepaliveretriesr   r   )
rF   r1   rX   Configboto3r
   rU   r   r   r   )rwrX   r   r   r   r   s         r(   _initialize_boto3r+    s     ~#''R8;&$,OO$:$:%'tVWakEl %; %K! 2k2M6 j."GHBJBJBGr'   c                  8   e Zd ZU dZded<   ded<   ded<   ded<   deej                  d	dddf	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd
ZddZ	ddZ
dd dZdd dZd!dZdd"dZddZej                   fd#dZd$dZd%d&dZd'dZddZd(dZdd)dZdd*dZd+dZd+dZy),r   zbReads bytes from S3.

    Implements the io.BufferedIOBase interface of the standard library.
    r   r   r   r   r   r   NFc
                8   || _         || _        t        | ||||       t        | j                  ||| j                   |	      | _        d| _        t        j                  j                  |      | _
        d| _        || _        d| _        |s| j                  d       y y )N)r   r   F)r   _buffer_sizer+  r   r   _raw_reader_current_posrk   
bytebuffer
ByteBuffer_buffer_eof_line_terminator_seek_initializedr   )
r6   r   r   r!   r   line_terminatorr   rX   r   r   s
             r(   r7   zReader.__init__  s     &'$vsC-LL-
 !,,77D	 /!&IIaL r'   c                .    t         j                  d       y)zFlush and close this stream.close: calledN)r?   r   r5   s    r(   r   zReader.close9  s    _%r'   c                     y)z+Return True if the stream can be read from.Tr&   r5   s    r(   readablezReader.readable=      r'   c                   |d}|dk(  ry|dk  rN| j                         | j                  j                         z   }| j                  j                  }|J || _        |S t        | j                        |k\  r| j                  |      S | j                  r| j                         S | j                  |       | j                  |      S )z6Read up to size bytes from the object and return them.r   r   r'   )	_read_from_bufferr/  r   r   r0  r   r3  r4  _fill_buffer)r6   r  outr   s       r(   r   zReader.readA  s    <D19!8((*T-=-=-B-B-DDC!--==N!--- .DJ
 t||$))$//
 99))++$%%d++r'   c                &    | j                  |      S )zThis is the same as read().)r  )r   r6   r  s     r(   read1zReader.read1^  s    yydy##r'   c                    t        |      j                  d      }| j                  t        |            }|sy||dt        |       t        |      S )zRead up to len(b) bytes into b, and return the number of bytes read.

        Args:
            b: The buffer to read into.

        Returns:
            The number of bytes read.
        Br   N)
memoryviewr
   r   r   )r6   bmvdatas       r(   readintozReader.readintob  sI     ]$yyR!;SY4yr'   c                "   |dk7  rd}t        |      t        j                         }| j                  rt	        | j
                        dk(  s| j
                  j                  | j                        }|j                  |       | xj                  t	        |      z  c_	        |j                  | j                        r	 |j                         S | j                          | j                  st	        | j
                        dk(  s|j                         S )zCRead up to and including the next newline.  Returns the bytes read.r   z(limits other than -1 not implemented yetr   )r   r   r   r4  r   r3  readliner5  r  r0  endswithr?  r  )r6   limitrM   line	line_parts        r(   rL  zReader.readliner  s    B;<C%c**
 zz|99T\\!2a!7--d.C.CDIJJy!Y/!!$"7"78 }}  99T\\!2a!7 }}r'   c                     y)zIf False, seek(), tell() and truncate() will raise IOError.

        We offer only seek support, and no truncate support.
        Tr&   r5   s    r(   seekablezReader.seekable  s    
 r'   c                   |t         j                  k(  rt         j                  }|| j                  z  }|t         j                  k(  rq|| j                  kD  rb| j                  t	        | j
                        z   }||k  r;| j
                  j                  || j                  z
         || _        | j                  S | j                  r"|t         j                  k(  r|| j                  k(  s;| j                  j                  ||      | _        | j
                  j                          | j                  | j                  j                  k(  | _        d| _        | j                  S )zSeek to the specified position.

        Args:
            offset: The offset in bytes.
            whence: Where the offset is from.

        Returns:
            The position after seeking.
        T)r   r   r   r0  r   r3  r   r6  r/  r   emptyr   r4  )r6   r   r   
buffer_ends       r(   r   zReader.seek  s    Y---++Fd'''F Y+++9J9J0J**S->>J#!!&4+<+<"<=$*!(((%%i,,,4;L;L1L $ 0 0 5 5ff EDLL %%)9)9)I)II	!%   r'   c                    | j                   S )z,Return the current position within the file.)r0  r5   s    r(   r   zReader.tell         r'   c                "    t         j                  Unsupported.r   UnsupportedOperationrB  s     r(   truncatezReader.truncate      %%%r'   c                "    t         j                  rY  r[  r5   s    r(   detachzReader.detach  r^  r'   c                     y)zDo nothing.Nr&   r5   s    r(   	terminatezReader.terminate      r'   c                    |sJ d       |j                  | j                  | j                        }| j                  |j	                  | j                        S |S )a  Create an **independent** `boto3.s3.Object` instance.

        The returned object points to the same S3 object as this instance.
        Changes to the returned object will not affect the current instance.

        Args:
            resource: A ``boto3.resource`` instance.

        Returns:
            A ``boto3.s3.Object`` (or ``ObjectVersion`` when ``version_id`` is set).
        *resource must be a boto3.resource instance)Objectr   r   r   Version)r6   resourceobjs      r(   to_boto3zReader.to_boto3  sP     EEExoodllDII6';;t//00
r'   c                    |dk\  r|nt        | j                        }| j                  j                  |      }| xj                  t        |      z  c_        |S )z:Remove at most size bytes from our buffer and return them.r   )r   r3  r   r0  )r6   r  parts      r(   r>  zReader._read_from_buffer  sF    qytc$,,&7||  &SY&r'   c                ~   t        || j                  j                        }t        | j                        |k  r| j                  sx| j                  j                  t        d| j                              }|dk(  rt        j                  d|        d| _        t        | j                        |k  r| j                  svy y y y )Nz	io.IOBaser   z$%s: reached EOF while filling bufferT)
r   r3  _chunk_sizer   r4  fillr
   r/  r?   r   )r6   r  
bytes_reads      r(   r?  zReader._fill_buffer  s    4112$,,$&tyy**4T=M=M+NOJQCTJ 	 $,,$&tyy&y&r'   c                <    d| j                   d| j                  dS )z8Return a short human-readable description of the reader.zsmart_open.s3.Reader(r  r  r  r5   s    r(   r  zReader.__str__  s     &t||&6bQGGr'   c                    d| j                   d| j                  d| j                  d| j                  d| j                  dS )z3Return an unambiguous representation of the reader.zsmart_open.s3.Reader(bucket=, key=z, version_id=z, buffer_size=z, line_terminator=r  )r   r   r   r.  r5  r5   s    r(   __repr__zReader.__repr__  s|    -dll-=VDII=P]^b^n^n]qq  AE  AR  AR  @U  Ug  hl  h}  h}  g@  @A  B  	Br'   )r   r   r   r   r!   r   r   r   r7  r  r   r  rX   S3Client | Noner   dict[str, Any] | Noner   r  rP   rQ   rO   r  r  r  r  rP   r  )rG  r   rP   r   )rN  r  rP   r  r  rP   r   rW   r  r  rP   r   rP   zio.RawIOBaserh  r   rP   r   r  )r  r   rP   rQ   r  )r"   r#   r$   rR   r%   DEFAULT_BUFFER_SIZEr   BINARY_NEWLINEr7   r   r;  r   rC  rJ  rL  rR  r   r   r   r]  r`  rb  rj  r>  r?  r  rt  r&   r'   r(   r   r   	  s   
 IL
I "&.!*!9!9 "&/3'+  	
      - % 
J&,:$ * /8.D.D "!H!&&*!HBr'   r   c                  0   e Zd ZU dZded<   ded<   ded<   ded<   dZd	ed
<   edddf	 	 	 	 	 	 	 	 	 	 	 	 	 ddZddZddZ	e
dd       ZddZddZej                  fd dZd!d"dZd#dZd$dZd%dZddZd&dZddZd'dZ	 	 	 	 	 	 	 	 d(dZd)dZd)dZy)*r   zzWrites bytes to S3 using the multi part API.

    Implements the io.BufferedIOBase interface of the standard library.
    r   r   r   r   r   r   Nr   
_upload_idc                6   t         j                  j                  |t        t              }||k7  rt
        j                  d||       |}|| _        t        | ||||       	 t        j                  | j                  j                  ||      }t        j                  |      d   | _        |t)        j*                         | _        n|| _        d| _        d| _        g | _        y # t         j"                  j$                  $ r}	d|d|	d}
t'        |
      |	d }	~	ww xY w)Nz!adjusting part_size from %s to %sr   UploadIdthe bucket z- does not exist, or is forbidden for access (r  r   )rk   rl   clampMIN_PART_SIZEMAX_PART_SIZEr?   r   
_part_sizer+  r\   r]   r   create_multipart_uploadRETRYrN   r  r1   rX   rD   r   r   r   _buf_total_bytes_total_parts_parts)r6   r   r   r   rX   r   r   adjusted_psr]   r   rM   s              r(   r7   zMultipartWriter.__init__  s    !&&,,Y}U#NN>	;W#I#$vsC		-''44G
 $ii0<DO
 

DI#DI,. ** 	-z)VW\V__`aCS/u,	-s   A	C! !D>DDc                     y)zGNo-op flush; data is buffered until ``close`` or ``_upload_next_part``.Nr&   r5   s    r(   flushzMultipartWriter.flush   rc  r'   c                P   t         j                  d       | j                  ry| j                  j	                         r| j                          t         j                  d|        | j                  r| j                  rt        j                  | j                  j                  | j                  | j                  | j                  t        dd| j                  i            }t         j#                  |       t         j                  d|        d| _        y| j                  r| j                  j%                  | j                  | j                  | j                         | j                  j'                  | j                  | j                  d	
       t         j                  d|        d| _        y)z>Complete the multipart upload (or abort) and close the stream.r9  Nz%s: completing multipart uploadr   Parts)r   r   r  MultipartUploadz%s: completed multipart uploadr   r   r  r'   r   r   r   z-%s: wrote 0 bytes to imitate multipart upload)r?   r   r   r  r   _upload_next_partr  r  r\   r]   r   complete_multipart_uploadr   r   r
   r  r  rN   abort_multipart_upload
put_object)r6   r]   s     r(   r   zMultipartWriter.close&  s8   _%;;99>>""$6=''66||II $%FRVR]R]H^ _G IIgLL94@( ' __ LL//||II 0 
 LL##||II $ 
 LLH$Or'   c                    | j                   du S z$Return True if the stream is closed.N)r  r5   s    r(   r   zMultipartWriter.closedO  s     $&&r'   c                     y)z+Return True if the stream supports writing.Tr&   r5   s    r(   writablezMultipartWriter.writableT  r<  r'   c                     y)z<Return True; we support `tell` but not `seek` or `truncate`.Tr&   r5   s    r(   rR  zMultipartWriter.seekableX  r<  r'   c                "    t         j                  rY  r[  r6   r   r   s      r(   r   zMultipartWriter.seek\  r^  r'   c                "    t         j                  rY  r[  rB  s     r(   r]  zMultipartWriter.truncate`  r^  r'   c                    | j                   S )z#Return the current stream position.)r  r5   s    r(   r   zMultipartWriter.telld  rW  r'   c                0    d}t        j                  |      )rZ  zdetach() not supportedr[  )r6   rM   s     r(   r`  zMultipartWriter.detachk  s    &%%c**r'   c                   d}t        |      }| xj                  t        |      z  c_        |t        |      k  r|}|| j                  z   | j                  j                         z
  }| j                  j                  |||        | j                  j                         | j                  k  r|t        |      k\  sJ t        |      S | j                          |}|t        |      k  rt        |      S )am  Write the given buffer to the S3 file.

        The buffer can be bytes, bytearray, memoryview or any buffer interface
        implementation. For more information about buffers, see
        https://docs.python.org/3/c-api/buffer.html.

        There's buffering happening under the covers, so this may not actually
        do any HTTP transfer right away.
        r   )rF  r  r   r  r  r   r  r  )r6   rG  r   rH  ru   rv   s         r(   r  zMultipartWriter.writep  s     ]SW$ s2wE4??*TYY^^-==CIIOOBuSM*yy~~$//1
 c"g~%~2w""$F s2w 2wr'   c                    | j                   }|yt        j                  d|        | j                  j	                  | j
                  | j                  |       d| _         t        j                  d|        y)z'Cancel the underlying multipart upload.Nz %s: terminating multipart uploadr  z%s: terminated multipart upload)r  r?   r   r   r  r   r   )r6   	upload_ids     r(   rb  zMultipartWriter.terminate  sc    OO	7>++<<		 	, 	

 6=r'   c                `    |sJ d       |j                  | j                  | j                        S )al  Create an **independent** `boto3.s3.Object` instance.

        The returned object points to the same S3 object as this instance.
        Changes to the returned object will not affect the current instance.

        Args:
            resource: A ``boto3.resource`` instance.

        Returns:
            A ``boto3.s3.Object`` for the same bucket and key.
        re  )rf  r   r   )r6   rh  s     r(   rj  zMultipartWriter.to_boto3  s+     EEExt||TYY77r'   c           
        | j                   J | j                  dz   }t        j                  d| || j                  j                         | j                  dz         | j                  j                  d       t        j                  t        j                  | j                  j                  | j                  | j                  | j                   || j                              }| j                   j#                  |d   |d       t        j%                  d| |       | xj                  dz  c_        | j                  j                  d       | j                  j'                  d       y )	Nr9   z3%s: uploading part_num: %i, %i bytes (total %.3fGB)g      Ar   )r   r   r  
PartNumberr   ETag)r  r  z#%s: upload of part_num #%i finished)r  r  r?   infor  r   r  r   r  rN   r\   r]   r   upload_partr   r   r  appendr   r]  )r6   part_numuploads      r(   r  z!MultipartWriter._upload_next_part  s   ***$$q(AIINN	)	
 			q ((||II#YY	
 	F6N(KL:D(KQ		q		1r'   c                    | S )z+Enter the multipart writer context manager.r&   r5   s    r(   	__enter__zMultipartWriter.__enter__      r'   c                J    || j                          y| j                          y)z8Close or terminate the multipart writer on context exit.Nrb  r   r6   exc_typeexc_valexc_tbs       r(   __exit__zMultipartWriter.__exit__       NNJJLr'   c                <    d| j                   d| j                  dS )8Return a short human-readable description of the writer.zsmart_open.s3.MultipartWriter(r  r  r  r5   s    r(   r  zMultipartWriter.__str__  r  r'   c                V    d| j                   d| j                  d| j                  dS )3Return an unambiguous representation of the writer.z%smart_open.s3.MultipartWriter(bucket=rs  z, part_size=r  )r   r   r  r5   s    r(   rt  zMultipartWriter.__repr__  s0    6t||6FfTYYMYefjfufuexxyzzr'   )r   r   r   r   r   r   rX   ru  r   rv  r   io.BytesIO | NonerP   rQ   rO   r  r  rW   ry  rx  rz  rG  r   rP   r   r{  rP   r   r  ztype[BaseException] | Noner  zBaseException | Noner  zTracebackType | NonerP   rQ   r  )r"   r#   r$   rR   r%   r  DEFAULT_PART_SIZEr7   r  r   r  r   r  rR  r   r   r   r]  r   r`  r  rb  rj  r  r  r  r  rt  r&   r'   r(   r   r     s   
 IL
I!J
! +"&/3)-#/#/ #/ 	#/
  #/ -#/ '#/ 
#/JV'R ' ' /8.D.D &&!+
!F>8$#J
,
 &
 %	

 

Q{r'   r   c                  @   e Zd ZU dZded<   ded<   ded<   ded<   dZd	ed
<   	 	 	 d	 	 	 	 	 	 	 	 	 	 	 ddZed d       Zd!dZ	d!dZ
ed"d       Zd"dZd"dZd"dZej                   fd#dZd$d%dZd&dZd'dZd(d)dZd(d)dZd!dZd*dZ	 	 	 	 	 	 	 	 d+dZd,dZd,dZy)-r   a  Writes bytes to S3 using the single part API.

    Implements the io.BufferedIOBase interface of the standard library.

    This class buffers all of its input in memory until its `close` method is called. Only then will
    the data be written to S3 and the buffer is released.
    r   r   r   r   r   r   Nr  r  c                    t        | ||||       |t        j                         | _        y |j	                         sd}t        |      || _        y )Nz writebuffer needs to be seekable)r+  r   r   r  rR  r   )r6   r   r   rX   r   r   rM   s          r(   r7   zSinglepartWriter.__init__   sI     	$vsC

DI%%'4CS/!#DIr'   c                &    | j                   }|J |S )zCReturn the active write buffer, asserting the stream is still open.)r  )r6   bufs     r(   _writebufferzSinglepartWriter._writebuffer  s     ii
r'   c                     y)z,No-op flush; data is buffered until `close`.Nr&   r5   s    r(   r  zSinglepartWriter.flush  rc  r'   c                   t         j                  d       | j                  ry| j                  d       | j                  }	 | j
                  j                  | j                  | j                  |       t         j                  d|        	 |j                          y# t        j                  j                  $ r!}d| j                  d}t        |      |d}~ww xY w# |j                          w xY w)zDUpload the buffered bytes as a single-part PUT and close the stream.r9  Nr   r  z%s: direct upload finishedr  z+ does not exist, or is forbidden for access)r?   r   r   r   r  r   r  r   r   r1   rX   rD   r   r   )r6   r  erM   s       r(   r   zSinglepartWriter.close  s    _%;;		!	LL##||II $  LL5t<IIK ** 	)//Z[CS/q(	) IIKs*   2B 3C C8CCC C.c                N    | j                   du xs | j                   j                  S r  )r  r   r5   s    r(   r   zSinglepartWriter.closed6  s"     yyD 4DII$4$44r'   c                6    | j                   j                         S z
Propagate.)r  r;  r5   s    r(   r;  zSinglepartWriter.readable;        ))++r'   c                6    | j                   j                         S r  )r  r  r5   s    r(   r  zSinglepartWriter.writable?  r  r'   c                6    | j                   j                         S r  )r  rR  r5   s    r(   rR  zSinglepartWriter.seekableC  r  r'   c                :    | j                   j                  ||      S r  )r  r   r  s      r(   r   zSinglepartWriter.seekG  s      %%ff55r'   c                8    | j                   j                  |      S r  )r  r]  rB  s     r(   r]  zSinglepartWriter.truncateK  s      ))$//r'   c                6    | j                   j                         S r  )r  r   r5   s    r(   r   zSinglepartWriter.tellO  s      %%''r'   c                8    | j                   j                  |      S )a  Write the given buffer into the in-memory buffer.

        The buffer can be bytes, bytearray, memoryview or any buffer interface
        implementation. Content of the buffer will be written to S3 on close as a
        single-part upload. For more information about buffers, see
        https://docs.python.org/3/c-api/buffer.html.

        Args:
            b: The buffer to write.

        Returns:
            The number of bytes written.
        )r  r  )r6   rG  s     r(   r  zSinglepartWriter.writeS  s       &&q))r'   c                8    | j                   j                  |      S r  )r  r   rB  s     r(   r   zSinglepartWriter.readc  s      %%d++r'   c                8    | j                   j                  |      S r  )r  rC  rB  s     r(   rC  zSinglepartWriter.read1g  s      &&t,,r'   c                d    | j                   j                          t        j                  d|        y)zClose buffer and skip upload.z %s: terminated singlepart uploadN)r  r   r?   r   r5   s    r(   rb  zSinglepartWriter.terminatek  s#    !7>r'   c                    | S )z,Enter the singlepart writer context manager.r&   r5   s    r(   r  zSinglepartWriter.__enter__s  r  r'   c                J    || j                          y| j                          y)z9Close or terminate the singlepart writer on context exit.Nr  r  s       r(   r  zSinglepartWriter.__exit__w  r  r'   c                <    d| j                   d| j                  dS )r  zsmart_open.s3.SinglepartWriter(r  r  r  r5   s    r(   r  zSinglepartWriter.__str__  s     00@499-qQQr'   c                <    d| j                   d| j                  dS )r  z&smart_open.s3.SinglepartWriter(bucket=rs  r  r  r5   s    r(   rt  zSinglepartWriter.__repr__  s!    77Gvdii]Z[\\r'   )NNN)r   r   r   r   rX   ru  r   rv  r   r  rP   rQ   )rP   z
io.BytesIOrO   r  r  rW   ry  rx  r  r  rw  r  r  r  )r"   r#   r$   rR   r%   r  r7   r  r  r  r   r   r;  r  rR  r   r   r   r]  r   r  r   rC  rb  r  r  r  rt  r&   r'   r(   r   r     s    IL
I"D
" #'/3)-$$ $  	$
 -$ '$ 
$$  ;. 5 5,,, /8.D.D 60(* ,-?
,
 &
 %	

 

R]r'   r   c                     yNTr&   )r   s    r(   _accept_allr    s    r'    c	           
   #    K   |t         }| }	t        j                  t              5  |	j                  }	ddd       |	d}
t        |
      |	} d\  }}|i }t        j                  j                  di |}|i }d|vr1t        j                  j                  ||z  dz   d|dz  dd	
      |d<    |j                  di |}t        j                  j                  j                  |      }t        | |||      }t!        j"                  t$        | |||      }t'        j(                  ||      }t*        j,                  j/                  |      5 }|j1                  ||      }t3        |d      D ]L  \  }\  }}||dz  dk(  r%t4        j7                  d||t9        |      |dz         ||f |t9        |      z  }N 	 ddd       t4        j7                  d||dz         y# 1 sw Y   xY w# 1 sw Y   1xY ww)a9	  Iterate and download all S3 objects under `s3://bucket_name/prefix`.

    Args:
        bucket_name: The name of the bucket.
        prefix: Limits the iteration to keys starting with the prefix.
        accept_key: A function that accepts a key name (unicode string) and
            returns True/False, signalling whether the given key should be
            downloaded. The default behavior is to accept all keys.
        key_limit: If specified, the iterator will stop after yielding this many
            results.
        workers: The number of objects to download concurrently. The entire
            operation uses a single ThreadPoolExecutor and shared thread-safe boto3
            S3.Client. Default: 16.
        retries: The number of time to retry a failed download. Default: 3.
        max_threads_per_fileobj: The maximum number of download threads per worker.
            The maximum size of the connection pool will be
            `workers * max_threads_per_fileobj + 1`. Default: 4.
        client_kwargs: Keyword arguments to pass when creating a new session.
            For a list of available names and values, see:
            https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html#boto3.session.Session.client
        session_kwargs: Keyword arguments to pass when creating a new session.
            For a list of available names and values, see:
            https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html#boto3.session.Session

    Yields:
        Tuples of ``(key, content)``, where ``key`` is the full key name (does not
        include the bucket name) and ``content`` is the full contents of the key.

    Raises:
        ValueError: If bucket_name is None.

    Note:
        The keys are processed in parallel, using `workers` threads (default: 16),
        to speed up downloads greatly.

    Example:
        >>> # get all JSON files under "mybucket/foo/"
        >>> for key, content in iter_bucket(
        ...         bucket_name, prefix='foo/',
        ...         accept_key=lambda key: key.endswith('.json')):
        ...     print key, len(content)

        >>> # limit to 10k files, using 32 parallel workers (default is 16)
        >>> for key, content in iter_bucket(bucket_name, key_limit=10000, workers=32):
        ...     print key, len(content)
    Nzbucket_name may not be None)r   r   r  r9   Tr  r   r!  r#  )max_concurrency)bucket_nameprefix
accept_keyrX   )r  r&  rX   transfer_config)ru   i  r   z-yielding key #%i: %s, size %i (total %.1f MB)g      0Az%processed %i keys, total size %.1f MBr&   r'  )r  
contextlibsuppressr   r   r   r)  sessionSessionr1   rX   r(  r   transferTransferConfig_list_bucketr\   r]   _download_key	itertoolsislicerk   concurrencyThreadPoolExecutorimap	enumerater?   r  r   )r  r  r  	key_limitworkersr&  max_threads_per_fileobjr   session_kwargsr   rM   
total_sizekey_nor  rX   r  key_iteratordownload_keyexecutorresult_iteratorr   contents                         r(   iter_bucketr    s    r  
 F			^	, 
- ~+oKJ mm##5n5G}$"*//"8"8!(+B!BQ!F%,q[*E #9 #
h
 W^^2M2Fhh''66G^6_O	L $$'L ##L)<L				2	27	;x"--lC&/q&I"FNS'{}!CL* w,#g,&J 'J 
<$ KK/Y} 
-	,X 
<	;s5   $G7GDG7A/G+;#G7G(#G7+G40G7c                     yr  r&   )ks    r(   <lambda>r    s    $r'   )r  r  c              #     K   d }	 |r| ||d}n| |d} |j                   di |}|j                  dg       D ]  }|d   } ||      s|  |j                  dd       }|sy _w)N)r   PrefixContinuationToken)r   r  Contentsr   NextContinuationTokenr&   )list_objects_v2rF   )	r  rX   r  r  ctokenrY   rE   cr   s	            r(   r  r    s      F
  +vTZ[F +v>F)6))3F3j"-AE(C#	 . 5t< s   AA%	A%c                  t        |dz         D ]  }	 t        ||| |      }| |fc S  y # t        j                  j                  $ rE}d|j
                  v r'|j
                  d   j                  d      dk(  rY d }~ y||k(  r Y d }~xd }~ww xY w)Nr9   )rX   r  key_namer  r:   r;   404rc   )r=   _download_fileobjr1   r2   rD   rE   rF   )r  rX   r  r&  r  xcontent_bytesrK   s           r(   r  r  1  s     7Q;	-'! /	M& &&+  ,  "".. 	 #,,&3<<+@+D+DV+LPU+U!G|	s   *B/B<BBc                t    t        j                         }| j                  ||||       |j                         S )N)r   r   Fileobjr(  )r   r   download_fileobjr  )rX   r  r  r  r  s        r(   r  r  U  s<     **,C
	   <<>r'   )rx   r   rP   r   )rz   r   r   r   rP   ztuple[_S3Uri, TransportParams])rz   r   r   r   r   r   rP   io.BufferedIOBase)r   r   r   r   r   r   r!   r   r   r   r   r   r   r  r   r  rX   ru  r   rv  r   r  r   r  rP   r  )rX   r   r   r   r   r   r   r   r   r   rP   ra   )r   rH   rP   rv  )r*  z+Reader | MultipartWriter | SinglepartWriterrX   ru  r   rv  r   r   r   r   rP   rQ   )r   r   rP   r  )r  NNr
     r  NN)r  r   r  r   r  zCallable[[str], bool] | Noner  r  r  r   r&  r   r  r   r   rv  r  rv  rP   zIterator[tuple[str, bytes]])
r  r   rX   r   r  r   r  zCallable[[str], bool]rP   zIterator[str])r  r   rX   r   r  r   r&  r   r  r   rP   z,tuple[str, bytes] | tuple[None, None] | None)
rX   r   r  r   r  r   r  r   rP   r  )UrR   
__future__r   r  r\   r   r   r  loggingrerB   collections.abcr   r   mathr   typingr   r   r	   r
   r)  boto3.s3.transferboto3.sessionbotocore.clientr1   botocore.exceptionsurllib3.exceptionsr   ImportErrorrS   smart_open.bytebufferrk   smart_open.concurrencysmart_open.utilsr   typesr   	_typeshedr   botocore.responser   mypy_boto3_s3.clientr   mypy_boto3_s3.type_defsr   typing_extensionsr   r   smart_open._typingr   	getLoggerr"   r?   r  r  r  rn   r|  URI_EXAMPLESr   compilerr   r   r*   r   r  rU   r   r   r   r   r   r   r   r+  BufferedIOBaser   r   r   r  r  r  r  r  r&   r'   r(   <module>r0     s   K "    	   	  .       #)/-G.2			8	$  /   T /
   
 FGY " "R !GE3 3*HV6!rO "**!"+/%)#'CCC C 	C
 C C C C C )C #C !C CL''' 
' 	'
 ' '.LQ LQ^3 ) 	
 
 
>cBR cBL{b'' {DX]r(( X]v /3 #$+/,0CCC -C 	C
 C C !C )C *C !CT (6  	
 & 6!! ! 	!
 ! ! 2!H  	
  e1  Ls   G5 5H ?H 