o
    Gh                    @   s  d dl m Z mZmZ d dlZd dlZd dlZd dlmZ d dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlmZ d dlmZmZmZmZ d dlmZmZmZmZmZmZmZmZmZm Z m!Z! d dl"m#Z# ej$%ej$&e'Z(G dd	 d	eZ)ej*d
d Z+ej*dd Z,ej*dd Z-ej*dd Z.ej*dd Z/ej*dd Z0ej*dd Z1ej*dd Z2ej*dd Z3dZ4ej*dd Z5ej*dd  Z6ej*d!d" Z7ej*d#d$ Z8ej*d%d& Z9ej*ej:dd'd(ej:dd)d(ej:dd*d(ej:dd+ej;j<d,ej:dd-ej;j=d,ej:dd.ej;j>d,ej:d d/ej;j6d,ej:dd0d(ej:dd1d(ej:dd2d(ej:d"d3d(ej:d$d4d(ej:d&d5ej;j<d,gd6d7d8 Z?ej*d9d: Z@ej*d;d< ZAej*d=d> ZBej*d?d@ ZCdAdB ZDdCdD ZEdEdF ZFdGdH ZGdIdJ ZHej;j<dKdL ZIdMdN ZJdOdP ZKdQdR ZLdSdT ZMdUdV ZNdWdX ZOdYdZ ZPd[d\ ZQd]d^ ZRd_d` ZSdadb ZTdcdd ZUdedf ZVdgdh ZWdidj ZXdkdl ZYdmdn ZZdodp Z[dqdr Z\dsdt Z]dudv Z^dwdx Z_dydz Z`ej;jej;ad{dde`fdd|e`fd}dejbfd}d~ejbfgdd Zcdd Zddd Zeej;jej;addde`fdd|e`fd}dejffd}d~ejffgdd Zgej;jej;addde`e`fdd|e`e`fd}dejbejffd}d~ejbejffgdd Zhdd Zidd Zjdd Zkdd Zldd Zmej;j=dd Znej;j<dd Zoej;j<dd Zpej;j<dd Zqej;j>dd Zrej;j6dd Zsej;addedfdedfdedfdedfdedfdedfdedfdedfdedfdedfdedfdedfdedfgdd Ztej;addddde'gdd Zuej;j<dd Zvej;j=dd Zwdd Zxdd Zydd ZzddÄ Z{ddń Z|ddǄ Z}ddɄ Z~dd˄ Zdd̈́ Zddτ Zddф Zej;j<ddӄ Zej;j<ddՄ Zej;j<ddׄ Zej;j<ddل Zddۄ Zej;j<dd݄ Zej;j<dd߄ Zej;jej;jej;j<dd Zej;j<ej;jddej;je dddd Zdd Zdd ZdS )    )datetimetimezone	timedeltaN)urlopen)assert_file_not_found)_filesystem_uriProxyHandler_configure_s3_limited_userrunning_on_musllinux)FileTypeFileInfoFileSelector
FileSystemLocalFileSystemSubTreeFileSystem_MockFileSystemFileSystemHandlerPyFileSystemFSSpecHandler
copy_files)find_free_portc                   @   s   e Zd Zd'ddZdd Zdd Zdd	 Zd
d Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&S )(DummyHandler*   c                 C   s
   || _ d S N)_value)selfvalue r   V/var/www/html/Persson_Maskin/env/lib/python3.10/site-packages/pyarrow/tests/test_fs.py__init__/   s   
zDummyHandler.__init__c                 C   s   t |tr| j|jkS tS r   
isinstancer   r   NotImplementedr   otherr   r   r   __eq__2      
zDummyHandler.__eq__c                 C   s   t |tr| j|jkS tS r   r    r#   r   r   r   __ne__7   r&   zDummyHandler.__ne__c                 C      dS )Ndummyr   r   r   r   r   get_type_name<      zDummyHandler.get_type_namec                 C   s   |S r   r   r   pathr   r   r   normalize_path?   r,   zDummyHandler.normalize_pathc                 C   s~   g }|D ]8}d|v r| t|tj qd|v r"| t|tj qd|v r0| t|tj qd|v r;| t  qt|S )Nfiledirnotfoundbadtype)appendr   r   File	DirectoryNotFoundobjectIOError)r   pathsinfor.   r   r   r   get_file_infoB   s   zDummyHandler.get_file_infoc                 C   sZ   |j dkr|jr
g S t|j tdtjddtdtjg}|jr+|tdtjddg7 }|S )Nsomedirsomedir/file1{   )sizesomedir/subdir1somedir/subdir1/file2i  )base_dirallow_not_foundFileNotFoundErrorr   r   r5   r6   	recursive)r   selectorinfosr   r   r   get_file_info_selectorR   s   


z#DummyHandler.get_file_info_selectorc                 C   s4   |dkr|du s
J d S |dkr|du sJ d S t )NrF   Tnon-recursiveF)r9   )r   r.   rF   r   r   r   
create_dirb   s
   zDummyHandler.create_dirc                 C      |dksJ d S )N
delete_dirr   r-   r   r   r   rM   j      zDummyHandler.delete_dirc                 C   s   | dst|dksJ d S )N/delete_dir_contents)strip
ValueError)r   r.   missing_dir_okr   r   r   rP   m   s   
z DummyHandler.delete_dir_contentsc                 C   s   d S r   r   r*   r   r   r   delete_root_dir_contentsr   r,   z%DummyHandler.delete_root_dir_contentsc                 C   rL   )Ndelete_filer   r-   r   r   r   rU   u   rN   zDummyHandler.delete_filec                 C      |dksJ |dksJ d S )N	move_frommove_tor   r   srcdestr   r   r   movex      zDummyHandler.movec                 C   rV   )Ncopy_file_fromcopy_file_tor   rY   r   r   r   	copy_file|   r]   zDummyHandler.copy_filec                 C   *   d|v rt || dd}t|S )Nr2   z:input_streamutf8rE   encodepaBufferReaderr   r.   datar   r   r   open_input_stream      
zDummyHandler.open_input_streamc                 C   ra   )Nr2   z:input_filerb   rc   rg   r   r   r   open_input_file   rj   zDummyHandler.open_input_filec                 C      d|v rt |t S Nr2   rE   re   BufferOutputStreamr   r.   metadatar   r   r   open_output_stream      zDummyHandler.open_output_streamc                 C   rl   rm   rn   rp   r   r   r   open_append_stream   rs   zDummyHandler.open_append_streamN)r   )__name__
__module____qualname__r   r%   r'   r+   r/   r<   rI   rK   rM   rP   rT   rU   r\   r`   ri   rk   rr   rt   r   r   r   r   r   .   s&    
r   c                    s   t t  fdddddS )Nc                        |    S r   as_posixptempdirr   r   <lambda>       zlocalfs.<locals>.<lambda>Tfspathfnallow_move_dirallow_append_to_filedictr   requestr~   r   r}   r   localfs   s   
r   c                    s"   t ttt  fdddddS )Nc                    rx   r   ry   r{   r}   r   r   r      r   zpy_localfs.<locals>.<lambda>Tr   )r   r   r   r   r   r   r}   r   
py_localfs   s   
r   c                 C   s   t t dd dddS )Nc                 S      | S r   r   r{   r   r   r   r          zmockfs.<locals>.<lambda>Tr   )r   r   r   r   r   r   mockfs   s   r   c                 C   s   t ttt dd dddS )Nc                 S   r   r   r   r{   r   r   r   r      r   zpy_mockfs.<locals>.<lambda>Tr   )r   r   r   r   r   r   r   r   	py_mockfs   s   r   c                    s   t tdd fdddddS )NTuse_mmapc                    rx   r   ry   r{   r}   r   r   r      r   z#localfs_with_mmap.<locals>.<lambda>r   r   r   r   r}   r   localfs_with_mmap   s   
r   c                 C   s"   t tt||d dd dddS )Nr   c                 S   r   r   r   r{   r   r   r   r      r   z!subtree_localfs.<locals>.<lambda>Tr   )r   r   str)r   r~   r   r   r   r   subtree_localfs   s   r   c              
   c   s    | j jd ddlm} |d \}}d}|| d| ddtd	d
dd}z|| W n tyK } zt	d| d|  W Y d }~nd }~ww t
||jdddV  || d S )Ngcsr   GcsFileSystem
connectionpyarrow-filesystem/:httpT-   )secondstest-project-id)endpoint_overridescheme	anonymousretry_time_limit
project_idzCould not create directory in : Fr   )configpyarrowrequires
pyarrow.fsr   r   rK   OSErrorpytestskipr   __add__rM   )r   
gcs_serverr   hostportbucketr   er   r   r   gcsfs   s2   "r   c           	      c   sz    | j jd ddlm} |d \}}}}d}|||| d| dddd	}|| t||jd
d
dV  || d S )Ns3r   S3FileSystemr   r   r   r   T)
access_key
secret_keyr   r   allow_bucket_creationallow_bucket_deletionFr   )	r   r   r   r   r   rK   r   r   rM   )	r   	s3_serverr   r   r   r   r   r   r   r   r   r   s3fs   s*   
r   c                 C   s    d}t t||d |jdddS )Nzpyarrow-filesystem/prefix/r   Fr   )r   r   r   )r   r   prefixr   r   r   subtree_s3fs  s   r   a  {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:ListAllMyBuckets",
                "s3:PutObject",
                "s3:GetObject",
                "s3:ListBucket",
                "s3:PutObjectTagging",
                "s3:DeleteObject",
                "s3:GetObjectVersion"
            ],
            "Resource": [
                "arn:aws:s3:::*"
            ]
        }
    ]
}c                 c   s    | j jd ddlm} |d \}}}}| d| }d}d}	|||||||d}
|
|	 t|
|	jd	d	d
V  |
|	 d S )Nazurer   AzureFileSystemr   r   r   r   account_nameaccount_keyblob_storage_authoritydfs_storage_authorityblob_storage_schemedfs_storage_schemeTr   )	r   r   r   r   r   rK   r   r   rM   )r   azure_serverr   r   r   r   r   azurite_authorityazurite_scheme	containerr   r   r   r   azurefs-  s,   
r   c                 C   sX   | j jd t std ddlm} |\}}}||||d}t	|dd ddd	S )
NhdfsCannot locate libhdfsr   HadoopFileSystem)r   userc                 S   r   r   r   r{   r   r   r   r   W  r   zhdfs.<locals>.<lambda>Tr   )
r   r   r   re   have_libhdfsr   r   r   r   r   )r   hdfs_connectionr   r   r   r   r   r   r   r   r   J  s   

r   c                    s4   t d}|d}ttt| fdddddS )Nfsspecr0   c                    rx   r   ry   r{   r}   r   r   r   c  r   z#py_fsspec_localfs.<locals>.<lambda>Tr   )r   importorskip
filesystemr   r   r   r   r~   r   r   r   r}   r   py_fsspec_localfs]  s   



r   c                 C   sH   t jddd}|jdkrt d |d}ttt|dd d	d	d
S )Nr   z0.7.5)
minversionz0.8.5z,Bug in fsspec 0.8.5 for in-memory filesystemmemoryc                 S   r   r   r   r{   r   r   r   r   r  r   z$py_fsspec_memoryfs.<locals>.<lambda>Tr   )r   r   __version__r   r   r   r   r   r   r   r   r   py_fsspec_memoryfsi  s   



r   c           	      c   sz    t d}|d \}}}}d}|j||td| d| dd}tt|}|| t||jdd	d
V  || d S )Nr   r   r   zhttp://r   )endpoint_url)keysecretclient_kwargsFTr   )	r   r   r   r   r   r   rK   r   rM   )	r   r   r   r   r   r   r   r   r   r   r   r   py_fsspec_s3fsx  s$   

r   zLocalFileSystem())idzLocalFileSystem(use_mmap=True)z$SubTreeFileSystem(LocalFileSystem())r   )r   marksr   r   r   z_MockFileSystem()z-PyFileSystem(ProxyHandler(LocalFileSystem()))z-PyFileSystem(ProxyHandler(_MockFileSystem()))z5PyFileSystem(FSSpecHandler(fsspec.LocalFileSystem()))z8PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))z0PyFileSystem(FSSpecHandler(s3fs.S3FileSystem())))paramsc                 C   s   |  | jS r   )getfixturevalueparamr   r   r   r   filesystem_config  s   <r   c                 C      | d S Nr   r   r   r   r   r   r        r   c                 C   r   )Nr   r   r   r   r   r   r     r   r   c                 C   r   )Nr   r   r   r   r   r   r     r   r   c                 C   r   )Nr   r   r   r   r   r   r     r   r   c                 C   sz   t | jtsJ t | jtsJ | jdksJ | jt| j d ks&J | jj}|d us0J |	d t
dks;J d S )Nr   g    eA)r!   mtimer   mtime_nsintr   approx	timestamptzinfo	utcoffsetr   )	file_infor   r   r   r   check_mtime  s   
r   c                 C   s    | j d u sJ | jd u sJ d S r   )r   r   r   r   r   r   check_mtime_absent  s   r   c                 C   s"   | j d u rt|  d S t|  d S r   )r   r   r   r   r   r   r   check_mtime_or_absent  s   
r   c                 C   s   | j dkrtjdd d S d S )Npy::fsspec+('s3', 's3a')zNot working with fsspec's s3fsreason)	type_namer   xfailr   r   r   r   skip_fsspec_s3fs     
r   c                 C   s   | j dkrtj|d d S d S )Nabfsr   )r   r   r   )r   r   r   r   r   
skip_azure  r   r  c                 C   s   ddl m} t| tdd | d \}}}}|dd| d| dd}|d	 tjtjd
d |d W d    n1 s?w   Y  tjtjdd |	d W d    d S 1 s]w   Y  d S )Nr   r   test_fs_limited_user
limited123r   r   r   )r   r   r   r   zexisting-bucket/testzBucket 'new-bucket' not foundmatchz
new-bucketzWould delete bucketzexisting-bucket)
r   r   r	   _minio_limited_policyrK   r   raisesre   ArrowIOErrorrM   )r   r   r   r   _r   r   r   r   +test_s3fs_limited_permissions_create_bucket  s$   
"r  c                  C   s,  t dtj} td}|jdksJ |jdksJ |jtj	ks!J |j
d u s(J t| tdtjddd}|jdks<J |jdksCJ |jtjksKJ |j
dksRJ |jd	ksYJ t| td
tj| d}|jd
kslJ |jd
kssJ |jtjks{J |j
d u sJ |j| ksJ |jdksJ t| d S )NR]foo/barbarzfoo/baz.txtr?   g  `A)typer@   r   zbaz.txtl    t<|+. foo)r  r   l    4 |+. )r   fromtimestampr   utcr   r.   	base_namer  r   Unknownr@   r   r5   r   r   r6   r   )dtr;   r   r   r   test_file_info_constructor  s0   
r  c                   C   s6   t t t  W d    d S 1 sw   Y  d S r   )r   r  	TypeErrorr   r   r   r   r   'test_cannot_instantiate_base_filesystem7  s   "r  c                  C   s   t  } t  }t }| | sJ | |sJ tt | d W d    n1 s,w   Y  | |   kr<|ks?J  J | dksEJ ||ksKJ |t ksRJ td| td| ks^J td| td|ksjJ td| td| ksvJ d S )Nstring   /basez/other)r   r   equalsr   r  r  r   )fs0fs1fs2r   r   r   test_filesystem_equals<  s   r   c                 C   sH   t jtdd | d  W d    n1 sw   Y  | d us"J d S )Nzgot NoneTyper  )r   r  r  r  r   r   r   r   test_filesystem_equals_noneP  s   r!  c                  C   s|   t  } td| }|jdksJ |j| ksJ t|dsJ tdt  }|jdks,J |j| ks3J t|ds<J d S )Nr  z/base/zHSubTreeFileSystem(base_path=/base/, base_fs=<pyarrow._fs.LocalFileSystemz/another/base/zPSubTreeFileSystem(base_path=/another/base/, base_fs=<pyarrow._fs.LocalFileSystem)r   r   	base_pathbase_fsrepr
startswith)r   subfsr   r   r   test_subtree_filesystemW  s   
r'  c                 C   sT   | j dd dkrtjdd || }||}t|ts!J || s(J d S )N::mock"MockFileSystem is not serializabler   )	r   splitr   r   dumpsloadsr!   r   r  )r   pickle_module
serializedrestoredr   r   r   test_filesystem_picklingg  s   

r2  c                 C   s  | j dd dkrtjdd t|  |d}|d}|d}| | | | W d    n1 s5w   Y  | |}|d	 W d    n1 sOw   Y  ||	| }|
|||g\}}	}
|jtjksoJ |	jtjkswJ |
jtjksJ d S )
Nr(  r)  r*  r+  r   	a/aa/aaa/a/bbc.txt   test)r   r,  r   r   r   rK   rr   writer.  r-  r<   r  r   r6   r5   )r   r   r/  aaabbcfpr1  aaa_infobb_infoc_infor   r   r   ,test_filesystem_is_functional_after_picklingq  s$   
r?  c                  C   s,   t  } | jdks
J t } | jdksJ d S )Nlocalr*  )r   r   r   r   r   r   r   test_type_name  s   rA  c                 C   s   |  ddks	J d S )Nr  )r/   r   r   r   r   test_normalize_path  s   rB  c              	   C   sp   G dd d}dd| t  i g dd t g}|D ]}tt | | W d    n1 s0w   Y  qd S )Nc                   @   s   e Zd ZdS )z-test_non_path_like_input_raises.<locals>.PathN)ru   rv   rw   r   r   r   r   Path  s    rC     g?c                   S   r(   )NrD  r   r   r   r   r   r     r   z1test_non_path_like_input_raises.<locals>.<lambda>)tuplepathlibrC  r   r  r  rK   )r   rC  invalid_pathsr.   r   r   r   test_non_path_like_input_raises  s   rH  c                 C   s  |d}|d}|d}|d}|  | | | W d    n1 s%w   Y  | |}|d W d    n1 s?w   Y  | ||||g\}}}	}
|j|ksXJ dt|v s`J |jdksgJ | jdkru|jt	j
kstJ n|jt	jks}J d	t|v sJ |jd u sJ t| |jt|ksJ |jd
ksJ |jdksJ |jt	jksJ dt|v sJ |jdksJ | jdvrt| |	jt|ksJ |	jdksJ |	jdksJ |	jt	jksJ dt|	v sJ |	jdksJ | jdvrt|	 |
jt|ksJ |
jdksJ |
jdksJ |
jt	j
ks!J |
jd u s)J |
jd u s1J dt|
v s:J t|
 | |}|j|jksLJ |j|jksUJ d S )Nr3  r4  r5  zzzr6  r8   r   zFileType.Directoryr9  zFileType.Filer   )zpy::fsspec+memoryr   txtr  zFileType.NotFound)rK   rr   r7  r<   r.   r$  	extensionr   r  r   r7   r6   r@   r   r   r  r5   r   r   r   )r   r   r8  r9  r:  rI  r;  r<  r=  r>  zzz_info	aaa_info2r   r   r   test_get_file_info  s`   




rO  c              	   C   s  |d}|d}|d}|d}|d}|d}z|  | | | W d    n1 s.w   Y  | | W d    n1 sCw   Y  |  | | | W d    n1 s]w   Y  |  | t|ddd	}|j|ksuJ | |}	| jd
krt|	dk nt|	dksJ |	D ]D}
|
j|s|
j|s|
j|r|
j	t
jksJ n#|
jd|s|
jd|r|
j	t
jksJ ntd|
j t|
 qt|dd}| |}	| jd
krt|	dksJ nt|	dksJ W | | d S W | | d S | | w )Nzselector-dir/zselector-dir/test_file_azselector-dir/test_file_bzselector-dir/test_dir_az#selector-dir/test_dir_a/test_file_czselector-dir/test_dir_bFT)rD   rF   r   r     rO   zunexpected path rF      )rK   rr   r   rC   r<   r   lenr.   endswithr  r   r5   rstripr6   rR   r   rM   )r   r   rC   file_afile_bdir_afile_cdir_brG   rH   r;   r   r   r    test_get_file_info_with_selector  s\   








r[  c                 C   s~   t |  |d}ttj | | W d    n1 sw   Y  | | | | |d}| j|dd | | d S )Nztest-directory/zdeeply/nested/test-directory/TrQ  )r   r   r  re   r	  rM   rK   )r   r   dr   r   r   test_create_dir  s   

r]  c                 C   s   t |  |d}|d}| | | | ttj | | W d    n1 s,w   Y  ttj | | W d    d S 1 sHw   Y  d S )N
directory/directory/nested/)r   rK   rM   r   r  re   r	  r   r   r\  ndr   r   r   test_delete_dir#  s   

"rb  c                 C   s   t |  |d}|d}| | | | | | | |}|jtjks(J |d}|d}|d}| | | | | |}|d W d    n1 sSw   Y  | | | |}|jtjksjJ d S )Nr^  r_  
directory2zdirectory2/nestedzdirectory2/nested/target-file   data)	r   rK   rM   r<   r  r   r7   rr   r7  )r   r   r\  ra  dir_infofsr   r   r   $test_delete_dir_with_explicit_subdir1  s&   







rh  c                 C   s   t |  |d}|d}| | | | ttj | | W d    n1 s,w   Y  | j|dd ttj | | W d    n1 sNw   Y  | | ttj | | W d    d S 1 sow   Y  d S )Nr^  r_  T)rS   )r   rK   rP   r   r  re   r	  rM   r`  r   r   r   test_delete_dir_contentsQ  s    


"ri  c                 C   s2  | d }| d }|d}|d}| | ttj |d W d    n1 s+w   Y  ttj |d W d    n1 sFw   Y  ttj |d W d    n1 saw   Y  |jddd	 |jddd	 |jddd	 ttj || W d    d S 1 sw   Y  d S )
Nr   r   r^  r_  rJ  rO   //Taccept_root_dir)rK   r   r  re   ArrowInvalidrP   r	  rM   )r   r   r   r\  ra  r   r   r   _check_root_dir_contentsc  s(   
"rn  c                 C   s   t |  t | d S r   )rn  )r   r   r   r   r   test_delete_root_dir_contentsy  s   ro  c                 C   s^   |d}|d}|  | W d    n1 sw   Y  | || | | | | d S )Nztest-copy-source-fileztest-copy-target-file)rr   r`   rU   r   r   rg  tr   r   r   test_copy_file~  s   
rr  c                 C   s   t | d |d}|d}| | |r<| || ttj | | W d    n1 s0w   Y  | | d S ttj | || W d    d S 1 sTw   Y  d S )N.Not implemented yet in for Azure. See GH-40025zsource-dir/ztarget-dir/)r  rK   r\   r   r  re   r	  rM   )r   r   r   rg  rq  r   r   r   test_move_directory  s   

"rt  c                 C   s   t |  t| d |d}|d}| | W d    n1 s!w   Y  | || ttj | | W d    n1 sBw   Y  | | d S )Nrs  ztest-move-source-fileztest-move-target-file)	r   r  rr   r\   r   r  re   r	  rU   rp  r   r   r   test_move_file  s   
ru  c                 C   s   |d}|  | W d    n1 sw   Y  | | ttj | | W d    n1 s4w   Y  |d}| | |d}|  |}|d W d    n1 s[w   Y  | | d S )Nztest-delete-target-fileztest-delete-nestedztest-delete-nested/target-filerd  )	rr   rU   r   r  re   r	  rK   r7  rM   )r   r   r|   r\  rf  rg  r   r   r   test_delete_file  s   

rv  c                 C   r   r   r   )vr   r   r   identity  r,   rx  )compressionbuffer_size
compressor@   gzip   c           	      C   s   |d}dd }|  |}||| W d    n1 sw   Y  | |||}| }W d    n1 s:w   Y  ||ksEJ d S )Nzopen-input-streams   some data for reading
   )rr   r7  ri   read)	r   r   ry  rz  r{  r|   rh   rg  resultr   r   r   test_open_input_stream  s   
r  c                 C   s   |d}dd }|  |}|| W d    n1 sw   Y  tdd }| |}| }W d    n1 s<w   Y  ||ksGJ | |}|| | }W d    n1 s`w   Y  |||d  ksoJ d S )Nzopen-input-file	   some data   r  )rr   r7  rS  rk   r  seek)r   r   r|   rh   rg  	read_fromrf  r  r   r   r   test_open_input_file  s   


r  c                 C   sB   |d}t t | | W d    d S 1 sw   Y  d S )Nzopen-input-stream-not-found)r   r  rE   ri   )r   r   r|   r   r   r    test_open_input_stream_not_found  s   "r  )ry  rz  decompressorc                 C   s   |d}dd }|  |||}|| W d    n1 sw   Y  | |||}|t||ks7J W d    d S 1 sBw   Y  d S )Nzopen-output-streams   some data for writingr  )rr   r7  ri   r  rS  )r   r   ry  rz  r  r|   rh   rf  r   r   r   test_open_output_stream  s   "r  )ry  rz  r{  r  c                 C   s  |d}|d}|  |}	|	| W d    n1 sw   Y  |rf| j|||d}
|
d W d    n1 s<w   Y  | |}
|
 }W d    n1 sUw   Y  ||}|dksdJ d S ttj | j|||d W d    d S 1 sw   Y  d S )Nzopen-append-streams   already existing)ry  rz  s   
newly addeds   already existing
newly added)	rr   r7  rt   ri   r  r   r  re   ArrowNotImplementedError)r   r   ry  rz  r{  r  r   r|   initialrg  rf  r  r   r   r   test_open_append_stream  s,   
"r  c           	      C   s   |d}ddi}d}| j ||d}|| W d    n1 s!w   Y  | |}| |ks4J | }W d    n1 sBw   Y  | jdv sQd| jv rkt| d | D ]\}}|| | kshJ qZd S |i ksqJ d S )	Nzopen-output-stream-metadataContent-Typezx-pyarrow/testr  )rq   )r   r   r  r*  zWAzure filesystem currently only returns system metadata not user metadata. See GH-40026)	rr   r7  ri   r  rq   r   r  itemsrd   )	r   r   r|   rq   rh   rf  got_metadatakrw  r   r   r    test_open_output_stream_metadata1  s$   
r  c                   C   sD   t dd tt t dd W d    d S 1 sw   Y  d S )NFr   )xxx)r   r   r  r  r   r   r   r   test_localfs_optionsH  s   
"r  c                 C   sf  | d }t   |d W d    n1 sw   Y  t   |d W d    n1 s/w   Y  t   |jddd W d    n1 sIw   Y  t   |d W d    n1 saw   Y  t   |d W d    n1 syw   Y  t   |dd W d    n1 sw   Y  t   |dd W d    d S 1 sw   Y  d S )Nr   z/non/existent/filez/non/existent/dirFrQ  z/non/existentz/xxx)r   ri   rr   rK   rM   rU   r\   r`   )r   r   r   r   r   test_localfs_errorsP  s,   "r  c                 C   s   | d }t t}|j}|| | g\}}|j| jks#J |j	| j
ks-J t| |j	| j
ks;J t| d S r   )rF  rC  __file__parentr<   rz   r@   statst_sizer   st_mtime_nsr   )r   r   	file_pathdir_pathr   re  r   r   r   test_localfs_file_infod  s   


r  c                 C   s`   t dtj}t|}|d W d    n1 sw   Y  |dg\}|j|ks.J d S )Nr  r  )r   r  r   r  r   rr   r<   r   )r   r  r   r;   r   r   r   test_mockfs_mtime_roundtripr  s   r  c              	   C   s  ddl m} t }|dd|ddddd	}t||sJ |jdks#J |jdks*J | | ||ks6J | }t||s@J | | ||ksLJ |d
d}t||sXJ | | ||ksdJ |dddd}t||ssJ | | ||ksJ t	
t |dd W d    n1 sw   Y  t	
t |d
dd W d    n1 sw   Y  t	
t |d
dd W d    n1 sw   Y  t	
t |t d W d    d S 1 sw   Y  d S )Nr   r   abczservice_account@apachezus-west2httpslocalhost:8999r   )access_tokentarget_service_accountcredential_token_expirationdefault_bucket_locationr   r   r   Tr   authenticated-read
text/plainACLr  default_metadataaccess)r  r   )r   r  acct)r   r  )r  )r   r   r   nowr!   r  r   r.  r-  r   r  rR   )r/  r   r  r   r   r   r   test_gcs_options|  sH   

"r  c                 C   sr  ddl m}m}m}m} |ddddddd	}t||sJ |jdks$J | | ||ks0J |d
dddd}t||s?J | | ||ksKJ ||ddd}t||sZJ ||ddd}t||siJ |d
d}t||suJ | | ||ksJ ||ksJ |dd}t||sJ | | ||ksJ |dd}t||sJ | | ||ksJ |ddddd}t||sJ | | ||ksJ ||ksJ |dd}t||sJ | | ||ksJ | | || ksJ |ddd}t||sJ | | ||ksJ |dddd}t||s(J | | ||ks5J |ddd}t||sCJ | | ||ksPJ |ddd}t||s^J | | ||kskJ ||ksrJ |ddd }t||sJ | | ||ksJ |d!d"}t||sJ | | ||ksJ ||d#d"ksJ || ksJ t	
t |dd$ W d    n	1 sw   Y  t	
t |dd% W d    n	1 sw   Y  t	
t |ddd& W d    n	1 sw   Y  t	
t |ddd' W d    n	1 s"w   Y  t	
t |ddd(d) W d    n	1 s?w   Y  t	
t |dddd* W d    n	1 s\w   Y  t	
t |d(dd+ W d    n	1 sxw   Y  t	
t |d,d-gd. W d    n	1 sw   Y  t	
t || d W d    d S 1 sw   Y  d S )/Nr   )AwsDefaultS3RetryStrategyAwsStandardS3RetryStrategyr   S3RetryStrategyr  r   token	us-east-2r  r  )r   r   session_tokenregionr   r   rolesessionr   d   )role_arnsession_nameexternal_idload_frequencyrP  )max_attempts)retry_strategy)r  Tr  )background_writesr  r  r  )r  r  )allow_delayed_open)r   r   )r   r   )check_directory_existence_before_creationg      ?g      ?)request_timeoutconnect_timeout)r   force_virtual_addressingzca.pem)tls_ca_file_pathzother_ca.pem)r   )r   )r   r  )r   r  arn)r   r   r  )r   r   r   )r  r   r  r  r  )r   r  r  r   r  r!   r  r.  r-  r   r  rR   )r/  r  r  r   r  r   r  r   r   r   test_s3_options  s   




$r  c           
      C   s  ddl m} dddd}d}ddd	d}d
}||d}t||s"J ||||ks.J ||d}t||s:J ||||ksFJ ||d}t||sRJ ||||ks^J ||d}t||sjJ ||||ksvJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ks)J ||||	ks6J |||	|ksCJ ||d}||d}	||	ksTJ ||||	ksaJ |||	|ksnJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ks J ||||	ksJ |||	|ksJ ||d}| }	||	ks)J ||||	ks6J |||	|ksCJ ||d}| }	||	ksRJ ||||	ks_J |||	|kslJ ||d}| }	||	ks{J ||||	ksJ |||	|ksJ ||d}| }	||	ksJ ||||	ksJ |||	|ksJ tt |dd W d    n	1 sw   Y  tt |dddd W d    n	1 sw   Y  tt |dddd W d    n	1 sw   Y  tt |dddd W d    n	1 s.w   Y  tt	j
 |dd W d    n	1 sJw   Y  tt	j
 |ddddd W d    d S 1 skw   Y  d S )Nr   r   r   	localhosti'#  )r   r   r   zhttp://localhost:8999r  i  zhttps://localhost:8080)proxy_options)r   r  #  r  )r   r   )r   r   )r   r   zhttpsB://localhost:9000httpA)r   r   r!   r.  r-  r   r  r  KeyErrorre   rm  )
monkeypatchr/  r   proxy_opts_1_dictproxy_opts_1_strproxy_opts_2_dictproxy_opts_2_strr   r  r  r   r   r   test_s3_proxy_options
  s   



























$r  c                  C   s   ddl m}  | ddd}d}tjt|d}|d W d    n1 s%w   Y  d	t|jv r3d S | d
dd}|d d S )Nr   r   z
eu-north-1T)r  r   a  When getting information for bucket 'voltrondata-labs-datasets': AWS Error UNKNOWN \(HTTP status 301\) during HeadBucket operation: No response body. Looks like the configured region is 'eu-north-1' while the bucket is located in 'us-east-2'.|NETWORK_CONNECTIONr  voltrondata-labs-datasetsNETWORK_CONNECTIONr  )r   r   r   r  r   r<   r   r   )r   r   msgexcr   r   r   test_s3fs_wrong_region  s   r  c           	      C   s"  ddl m} |dd}t||sJ | | ||ksJ |ddd}t||s+J | | ||ks7J ||ks=J |ddd	d
ddd}t||sNJ | | ||ksZJ ||ks`J |ddd}t||smJ | | ||ksyJ ||ksJ |ddddd}t||sJ | | ||ksJ ||ksJ |ddd}t||sJ | | ||ksJ ||ksJ tjtdd |ddd W d    n1 sw   Y  tjtdd |ddd W d    n1 sw   Y  d}tjt|d |dddd W d    n	1 sw   Y  tjtdd |dddd W d    n	1 s4w   Y  tjt|d |dddd W d    n	1 sSw   Y  tt |dddd W d    n	1 spw   Y  tt |  W d    d S 1 sw   Y  d S )Nr   r   zfake-account-name)r   fakeaccountkey)r   r   zfake-accountfakeaccountzfake-blob-authorityzfake-dfs-authorityr  r   fakesastoken)r   	sas_tokenzfake-tenant-idzfake-client-idzfake-client-secret)r   	tenant_id	client_idclient_secret)r   r  zclient_id must be specifiedr  )r   r  )r   r  zInvalid Azure credential configuration: For ManagedIdentityCredential, provide only client_id. For ClientSecretCredential, provide tenant_id, client_id, and client_secret.)r   r  r  )r   r  r  )r   r  r  )r   r   r  )	r   r   r!   r.  r-  r   r  rR   r  )	r/  r   r  r  fs3fs4fs5fs6invalid_msgr   r   r   test_azurefs_options  s   
$r  c                 C   s  ddl m} t std | \}}}d}d}d}d}	|||d|||d	}
||	||d|||}||	||d
|||}||	||d
|d ||}|||}|d| d| }|||dd}|||ddd}|||dt	dd}|||ddd}|||ddddid}|
|ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ t
t |  W d    n1 sw   Y  t
t |d W d    n	1 sw   Y  |
||||||||||fD ]}||||ks#J q| \}}}||||d}|tds;J |d| d| d| }|tdsSJ d S )Nr   r   r      i   i   zIhdfs://{}:{}/?user={}&replication={}&buffer_size={}&default_block_size={}libhdfs)r   replicationrz  default_block_sizemerD  zhdfs://r   	localuser)r   
cache_path)r   kerb_ticketcache_path2
hdfs_tokenabcd)r   r  
extra_confrR  rO   z/?user=)r   r   re   r   r   r   from_uriformatrF  rC  r  r  r.  r-  r<   r   )r   r/  r   r   r   r   r  rz  r  urihdfs1hdfs2hdfs3hdfs4hdfs5hdfs6hdfs7hdfs8hdfs9hdfs10hdfs11r   r   r   r   r   test_hdfs_options  s   



r  )r  expected_klassexpected_pathzmock:rJ  zmock:foo/barr  zmock:/foo/barzmock:///foo/barzmock:///some%20path/%C3%A9u   some path/ézfile:/rO   zfile:///zfile:/foo/barz/foo/barzfile:///foo/barzfile:///some%20path/%C3%A9u   /some path/éu   /some path/%20éc                 C   s,   t | \}}t||sJ ||ksJ d S r   )r   r  r!   )r  r  r  r   r.   r   r   r   test_filesystem_from_uriU  s   r  r.   c                 C   sB   t | }t|\}} t|tsJ | |   ksJ d S r   )	rF  rC  r   r  r!   r   resolveabsoluterz   )r.   r|   r   r   r   r    test_filesystem_from_path_objectm  s   
r  c           
   	   C   s   ddl m} | d \}}}}d| d| d| d| d	}t|\}}t||s+J |dks1J || ||g\}	|	j|ksDJ |	jt	j
ksLJ d S )	Nr   r   r   s3://r   z0@mybucket/foo/bar?scheme=http&endpoint_override=z&allow_bucket_creation=Truemybucket/foo/bar)r   r   r   r  r!   rK   r<   r.   r  r   r6   )
r   r   r   r   r   r   r  r   r.   r;   r   r   r   test_filesystem_from_uri_s3x  s   
r
  c                 C   s   ddl m} | d \}}dd| d| d d }t|\}}t||s'J |d	ks-J || ||g\}|j|ks@J |jt	j
ksHJ d S )
Nr   r   r   zgs://anonymous@z/mybucket/foo/bar?scheme=http&endpoint_override=r   &z0retry_limit_seconds=5&project_id=test-project-idr	  )r   r   r   r  r!   rK   r<   r.   r  r   r6   )r   r   r   r   r  r   r.   r;   r   r   r   test_filesystem_from_uri_gcs  s   
r  c                  C   sp   t  } t| }t|tsJ |jdksJ |j| u sJ tt td  W d    d S 1 s1w   Y  d S Nz	py::dummy)r   r   r!   r   handlerr   r  r  )r  r   r   r   r   test_py_filesystem  s   
"r  c                  C   s   t d} t d}t d}t| }t| }t|}t|}||us"J ||us(J ||us.J ||ks4J ||ks:J ||ks@J |t ksGJ |t ksNJ d S )NrD  r  )r   r   r   r8   )handler1handler2handler3r  r  r  r  r   r   r   test_py_filesystem_equality  s   r  c                 C   s\   t  }t|}| |}| |}t|tsJ ||ksJ |j|ks%J |jdks,J d S r  )r   r   r-  r.  r!   r   r  r   )r/  r  r   r0  r1  r   r   r   test_py_filesystem_pickling  s   

r  c                  C   s   t  } t| }t|tsJ t| }d } | d usJ d }| d u s%J t  } t| }t| }d } | |ju s;J | d usBJ d }| d u sKJ d S r   )r   r   r!   weakrefrefr  )r  r   wrr   r   r   test_py_filesystem_lifetime  s    

r  c                  C   s  t  } t| }|dg\}|jdksJ |jtjksJ |dg\}|jdks+J |jtjks3J |dg\}|jdksAJ |jtjksIJ t	
t |dg W d    n1 s_w   Y  t	
t |dg W d    d S 1 s{w   Y  d S )Nzsome/dirz	some/filer2   r3   r  )r   r   r<   r.   r  r   r6   r5   r7   r   r  r  r9   )r  r   r;   r   r   r    test_py_filesystem_get_file_info  s"   "r  c                  C   s^  t  } t| }tdd}||}t|dksJ |d jdks"J |d jtjks,J |d j	dks5J |d jdks>J |d jtj
ksHJ |d j	d u sQJ tdd	d
}||}t|dksdJ |d jdksmJ |d jdksvJ |d jdksJ tdd}tt || W d    n1 sw   Y  tdd	d}||g ksJ d S )Nr=   )rC   r  r   r>   r?   rD  rA   T)rC   rF   rR  rB   r2   )rC   rD   )r   r   r   r<   rS  r.   r  r   r5   r@   r6   r   r  rE   )r  r   rG   rH   r   r   r   )test_py_filesystem_get_file_info_selector  s.   



r  c               	   C   s   t  } t| }|jddd |jddd tt |d W d    n1 s*w   Y  |d |d d	D ]#}tt || W d    n1 sRw   Y  |j|dd
 q;|	d |
dd |dd d S )NrF   TrQ  rJ   FfoobarrM   rP   )rJ  rO   rj  rk  rU   rW   rX   r^   r_   )r   r   rK   r   r  r9   rM   rP   rR   rU   r\   r`   )r  r   r.   r   r   r   test_py_filesystem_ops  s"   


r  c                  C   ~   t t } | d}| dksJ W d    n1 sw   Y  tt | d W d    d S 1 s8w   Y  d S )Nsomefiles   somefile:input_streamr2   )r   r   ri   r  r   r  rE   r   rf  r   r   r   test_py_open_input_stream'     
"r   c                  C   r  )Nr  s   somefile:input_filer2   )r   r   rk   r  r   r  rE   r  r   r   r   test_py_open_input_file0  r!  r"  c                  C   D   t t } | d}|d W d    d S 1 sw   Y  d S Nr  rd  )r   r   rr   r7  r  r   r   r   test_py_open_output_stream9     
"r%  c                  C   r#  r$  )r   r   rt   r7  r  r   r   r   test_py_open_append_stream@  r&  r'  c                  C   s   ddl m}  tjdpd}| dd}|j|ksJ | ddd}|td	}t|dks/J d
}|	|#}|
 }d|v sAJ |d dksIJ |d dksQJ W d    d S 1 s\w   Y  d S )Nr   r   PYARROW_TEST_S3_REGIONz	us-east-1Tr  r  )r   r  z"voltrondata-labs-datasets/nyc-taxizCvoltrondata-labs-datasets/nyc-taxi/year=2019/month=6/part-0.parquetr  zLast-Modifieds   2022-07-12T23:32:00ZETags%   "4c6a76826a695c6ac61592bc30cda3df-16")r   r   osenvirongetr  r<   r   rS  ri   rq   )r   default_regionr   entriesr   rf  mdr   r   r   test_s3_real_awsG  s$   
"r0  c                  C   s  t d\} }| jdksJ | |}d|dv sJ W d    n1 s'w   Y  t d\} }| jdks:J tjtdd t d	 W d    n1 sQw   Y  t d
\} }| jdksdJ t d\} }| |}tt |	  W d    d S 1 sw   Y  d S )Nzs3://mf-nwp-models/README.txt	eu-west-1s&   Meteo-France Atmospheric models on AWS2   z.s3://mf-nwp-models/README.txt?region=us-east-2r  zBucket '.*' not foundr  zs3://x-arrow-nonexistent-bucketz0s3://x-arrow-nonexistent-bucket?region=us-east-3z	us-east-3zMs3://x-arrow-nonexistent-bucket/T.md?region=us-east-2&allow_delayed_open=true)
r   r  r  ri   r  r   r  r9   rr   close)r   r.   rf  streamr   r   r   !test_s3_real_aws_region_selection_  s*   

"r5  c                  C   s   ddl m}  | ddksJ | ddksJ tjtdd | d	 W d    n1 s,w   Y  tjtdd | d
 W d    d S 1 sHw   Y  d S )Nr   resolve_s3_regionr  r  zmf-nwp-modelsr1  zNot a valid bucket namer  r  z	s3:bucket)r   r7  r   r  rR   r6  r   r   r   test_resolve_s3_region~  s   

"r8  c              
   C   s0  |d }|d }|d}| |}|d W d    n1 s!w   Y  | \}}}	}
d|	 d|
 d| d| d| 
}t|d	 }t|| t }||}| dksYJ W d    n1 scw   Y  t|d
 }t|||d ||}| dksJ W d    n1 sw   Y  t|d }t|}t|| ||}| dksJ W d    n1 sw   Y  t|d }t|||d ||}| dksJ W d    n1 sw   Y  t|d }t||ddd ||}| dksJ W d    d S 1 sw   Y  d S )Nr   r   r5  r6  r  r   @z?scheme=http&endpoint_override=zc_copied1.txtzc_copied2.txt)source_filesystemzc_copied3.txtzc_copied4.txt)destination_filesystemzc_copied5.txtrD  F
chunk_sizeuse_threads)rr   r7  r   r   r   ri   r  r   )s3_connectionr   r~   r   r   r.   rf  r   r   r   r   
source_urilocal_path1r   local_path2local_path3destination_urilocal_path4local_path5r   r   r   test_copy_files  sP   

$rG  c                    s|  t   | d }|   t|d }|d W d    n1 s$w   Y   t|d }|d W d    n1 sBw   Y   fdd}| d }|  tt|t| || | d	 }|  tt|t|  d
 || | d }|  tt|}tt|}t|| || | d }	|	  t||	 ||	 | d }
|
  t||
ddd ||
 d S )Nsourcefile1   test1file2   test2c                    s     t| d }| dksJ W d    n1 sw   Y    t| d }| dks3J W d    d S 1 s>w   Y  d S )NrI  rJ  rK  rL  )ri   r   r  )destination_dirrf  r   r   r   check_copied_files  s   "z5test_copy_files_directory.<locals>.check_copied_filesdestination1destination2)r:  r;  destination3destination4destination5rD  Fr<  )r   mkdirrr   r   r7  r   r   )r~   
source_dirrf  rO  destination_dir1destination_dir2destination_dir3r@  rD  destination_dir4destination_dir5r   rN  r   test_copy_files_directory  sD   

r\  c                  C      d} t tjd| g d S )Na  if 1:
        import pytest
        from pyarrow.fs import (FileSystem, S3FileSystem,
                                ensure_s3_initialized, finalize_s3)

        fs, path = FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        assert fs.region == 'eu-west-1'
        f = fs.open_input_stream(path)
        f.read(50)

        finalize_s3()

        with pytest.raises(ValueError, match="S3 .* finalized"):
            f.read(50)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            fs.open_input_stream(path)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            S3FileSystem(anonymous=True)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        -c
subprocess
check_callsys
executablecoder   r   r   test_s3_finalize  s   rf  c                  C   r]  )Na  if 1:
        import pytest
        from pyarrow.fs import resolve_s3_region, ensure_s3_initialized, finalize_s3

        resolve_s3_region('mf-nwp-models')

        finalize_s3()

        # Testing both cached and uncached accesses
        with pytest.raises(ValueError, match="S3 .* finalized"):
            resolve_s3_region('mf-nwp-models')
        with pytest.raises(ValueError, match="S3 .* finalized"):
            resolve_s3_region('voltrondata-labs-datasets')
        r^  r_  rd  r   r   r    test_s3_finalize_region_resolver  s   rg  c                  C   r]  )Na  if 1:
        import threading
        import pytest
        from pyarrow.fs import (FileSystem, S3FileSystem,
                                ensure_s3_initialized, finalize_s3)
        threads = []
        fn = lambda: FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        for i in range(4):
            thread = threading.Thread(target = fn)
            threads.append(thread)
            thread.start()

        for thread in threads:
            thread.join()

        finalize_s3()
        r^  r_  rd  r   r   r   test_concurrent_s3fs_init#  s   rh  zUatexit(ensure_s3_finalized) will be called too late with bundled aws-sdk-cpp 1.11.587r   z?Leaking S3ClientFinalizer causes segfault on musl based systemsc               
   C   sN  z	t ddg W n ty   td Y nw t } dddd|  dtjt	dg}t j
|t jd	}zdd
|  d}t }d }t |d k rzt|}|jdksUJ W d    n1 s_w   Y  W n, ty~ } z|}td W Y d }~nd }~ww t |d k sHtd|d|  W |  n|  w | dksJ d S )Nuwsgiz	--versionz"uwsgi not installed on this Pythonz-iz--httpz
127.0.0.1:z--wsgi-filezwsgi_examples.py)stdinzhttp://127.0.0.1:z/s3/rP     g?zCould not fetch r      )r`  ra  rE   r   r   r   r*  r.   joinherePopenDEVNULLtimer   statusr   sleepfail	terminatewait)r   argsprocurl
start_timeerrorrespr   r   r   r   test_uwsgi_integration<  s<   
	r}  c                  C   s   zddl m}  ddlm} W n ty   td Y nw td\}}t	t
| }||ks1J |dks7J d}td| \}}t	t
|  }||ksPJ d S )	Nr   )r   )MemoryFileSystemzfsspec not installedz$fsspec+memory://path/to/data.parquetz/path/to/data.parquetzfile:///tmp/my.filezfsspec+)fsspec.implementations.localr   fsspec.implementations.memoryr~  ImportErrorr   r   r   r  r   r   )r   r~  r   r.   expected_fsr  r
  r   r   r   test_fsspec_filesystem_from_uria  s   r  c                  C   sp   t d zddlm}  W n ty   t d Y nw td\}}tt	|  }||ks0J |dks6J d S )Nr   r   )HfFileSystemzhuggingface_hub not installedzFhf://datasets/stanfordnlp/imdb/plain_text/train-00000-of-00001.parquetzAdatasets/stanfordnlp/imdb/plain_text/train-00000-of-00001.parquet)
r   r   huggingface_hubr  r  r   r   r  r   r   )r  r   r.   r  r   r   r   $test_huggingface_filesystem_from_uriu  s   
r  )r   r   r   r}  r*  rF  urllib.requestr   r`  rb  rq  r   r  r   re   pyarrow.tests.test_ior   pyarrow.tests.utilr   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   pyarrow.utilr   r.   dirnameabspathr  rn  r   fixturer   r   r   r   r   r   r   r   r   r  r   r   r   r   r   r   markr   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r   r!  r'  r2  r?  rA  rB  rH  rO  r[  r]  rb  rh  ri  rn  ro  rr  rt  ru  rv  rx  parametrizecompressr  r  r  
decompressr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r
  r  r  r  r  r  r  r  r  r   r"  r%  r'  r0  r5  r8  rG  r\  rf  rg  	processes	threadingrh  r   skipifr}  r  r  r   r   r   r   <module>   s  4i
	
	
	
	
	
	









;





;8 

	

	

	

%
g
 

]
G



		



44

 