o
    G¾±h€©  ã                   @   sà  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	zd dl
ZW n ey7   dZY nw d dlZd dlmZmZ zd dlmZ d dlZW n	 eyY   Y nw G dd„ dƒZG dd„ deƒZG d	d
„ d
eƒZG dd„ deƒZejdd„ ƒZejdd„ ƒZejdd„ ƒZejejdddejdddgddd„ ƒZdd„ Z dd„ Z!dd„ Z"ej# $dd d!„ d"d!„ g¡d#d$„ ƒZ%d%d&„ Z&ej#jd'd(„ ƒZ'd)d*„ Z(d+d,„ Z)ej#jej#j*d-d.„ ƒƒZ+ej#jd/d0„ ƒZ,d1d2„ Z-ej# $d3ej. /¡ ej.j/d4d5g¡d6d7„ ƒZ0d8d9„ Z1ej# $d3ej. /¡ ej.j/d4d5g¡d:d;„ ƒZ2d<d=„ Z3ej#jd>d?„ ƒZ4ej#jd@dA„ ƒZ5dBdC„ Z6ej#j7dDdE„ ƒZ8dFdG„ Z9ej# $d3ej. :¡ ej.j:dHdIej.j:dHdJej.j:ej.j;j<dKej.j:dHej.j;j<dLg¡dMdN„ ƒZ=dOdP„ Z>dQdR„ Z?dSdT„ Z@dUdV„ ZAdWdX„ ZBej#jdYdZ„ ƒZCejd[d\„ ƒZDd]d^„ ZEd_d`„ ZFdadb„ ZGej#jHdcdd„ ƒZIdedf„ ZJdgdh„ ZKdidj„ ZLG dkdl„ dlejMƒZNG dmdn„ dneƒZOejdodp„ ƒZPej#jQdqdr„ ƒZRej#jQdsdt„ ƒZSej#jdudv„ ƒZTej#jdwdx„ ƒZUej#jej# $dydzd{g¡d|d}„ ƒƒZVd~d„ ZWej#jd€d„ ƒZXej#jd¦d‚dƒ„ƒZYej#jd„d…„ ƒZZej#jd†d‡„ ƒZ[ej#jdˆd‰„ ƒZ\ej#jdŠd‹„ ƒZ]ej#jdŒd„ ƒZ^ej#jdŽd„ ƒZ_ej#jdd‘„ ƒZ`d’d“„ Zad”d•„ Zbd–d—„ Zcd˜d™„ Zddšd›„ Zedœd„ ZfdždŸ„ Zgd d¡„ Zhd¢d£„ Zid¤d¥„ ZjdS )§é    )ÚUserListN)Úchanged_environÚinvoke_script)Úassert_frame_equalc                   @   s:   e Zd ZdZdd„ fdd„Zdd„ Zdd	„ Zddd„ZdS )Ú
IpcFixtureNc                   C   ó   t  ¡ S ©N©ÚioÚBytesIO© r   r   úW/var/www/html/Persson_Maskin/env/lib/python3.10/site-packages/pyarrow/tests/test_ipc.pyÚ<lambda>/   ó    zIpcFixture.<lambda>c                 C   s   || _ |  ¡ | _d S r   )Ú_sink_factoryÚget_sinkÚsink)ÚselfÚsink_factoryr   r   r   Ú__init__/   s   zIpcFixture.__init__c                 C   s   |   ¡ S r   )r   ©r   r   r   r   r   3   s   zIpcFixture.get_sinkc                 C   s
   | j  ¡ S r   )r   Úgetvaluer   r   r   r   Ú
get_source6   ó   
zIpcFixture.get_sourceé   Fc           
      C   s´   d}t  dt  ¡ fdt  ¡ fg¡}|  | j|¡}g }t|ƒD ]}t jdd„ t|ƒD ƒg d¢g|d}| |¡ q|rFt j	 
|¡}	| |	¡ n
|D ]}| |¡ qH|j| _| ¡  |S )Nr   ÚoneÚtwoc                 S   s   g | ]}t   ¡ ‘qS r   )Úrandom©Ú.0Ú_r   r   r   Ú
<listcomp>B   ó    z,IpcFixture.write_batches.<locals>.<listcomp>)ÚfooNÚbarÚbazbazÚqux©Úschema)Úpar(   Úfloat64Úutf8Ú_get_writerr   ÚrangeÚrecord_batchÚappendÚTableÚfrom_batchesÚwrite_tableÚwrite_batchÚstatsÚwrite_statsÚclose)
r   Únum_batchesÚas_tableÚnrowsr(   ÚwriterÚbatchesÚiÚbatchÚtabler   r   r   Úwrite_batches9   s(   ÿýzIpcFixture.write_batches)r   F)Ú__name__Ú
__module__Ú__qualname__r5   r   r   r   r?   r   r   r   r   r   ,   s    r   c                   @   s&   e Zd ZdZdZdd„ Zddd„ZdS )	ÚFileFormatFixtureTNc                 C   ó   t jj||| jdS ©N©Úoptions)r)   ÚipcÚnew_filerG   ©r   r   r(   r   r   r   r,   X   ó   zFileFormatFixture._get_writerFc                 C   sÀ   | j |d}t |  ¡ ¡}tj |¡}|jt|ƒksJ ‚t|ƒD ]\}}| 	|¡}||  
|¡s2J ‚|j 
|d j¡s=J ‚q t|jtjjƒsHJ ‚t| jtjjƒsRJ ‚t|jƒt| jƒks^J ‚d S )N©r8   r   )r?   r)   ÚBufferReaderr   rH   Ú	open_fileÚnum_record_batchesÚlenÚ	enumerateÚ	get_batchÚequalsr(   Ú
isinstancer4   Ú	ReadStatsr5   Ú
WriteStatsÚtuple)r   r8   r;   Úfile_contentsÚreaderr<   r=   r   r   r   Ú_check_roundtrip[   s   
z"FileFormatFixture._check_roundtrip©F)r@   rA   rB   Úis_filerG   r,   rZ   r   r   r   r   rC   S   s
    rC   c                   @   s    e Zd ZdZdZdZdd„ ZdS )ÚStreamFormatFixtureFNc                 C   rD   rE   )r)   rH   Ú
new_streamrG   rJ   r   r   r   r,   v   s
   ýzStreamFormatFixture._get_writer)r@   rA   rB   Úuse_legacy_ipc_formatrG   r\   r,   r   r   r   r   r]   n   s
    r]   c                   @   s   e Zd Zdd„ ZdS )ÚMessageFixturec                 C   ó   t  ||¡S r   ©r)   ÚRecordBatchStreamWriterrJ   r   r   r   r,   €   ó   zMessageFixture._get_writerN)r@   rA   rB   r,   r   r   r   r   r`   ~   s    r`   c                   C   ó   t ƒ S r   )r   r   r   r   r   Úipc_fixture„   ó   rf   c                   C   re   r   )rC   r   r   r   r   Úfile_fixture‰   rg   rh   c                   C   re   r   )r]   r   r   r   r   Ústream_fixtureŽ   rg   ri   zFile Format)ÚidzStream Format)Úparamsc                 C   s   |   | j¡S r   )ÚgetfixturevalueÚparam)Úrequestr   r   r   Úformat_fixture“   s   ro   c                  C   sH   d} t  tj¡ tj t | ¡¡ W d   ƒ d S 1 sw   Y  d S ©Nó    )ÚpytestÚraisesr)   ÚArrowInvalidrH   rN   rM   ©Úbufr   r   r   Útest_empty_file¡   s   "ÿrw   c                 C   ó   | j dd d S )NFrL   ©rZ   ©rh   r   r   r   Útest_file_simple_roundtrip§   ó   r{   c                 C   rx   )NTrL   ry   rz   r   r   r   Útest_file_write_table«   r|   r}   r   c                   C   r   r   r	   r   r   r   r   r   °   r   r   c                   C   r   r   )r)   ÚBufferOutputStreamr   r   r   r   r   ±   r   c                 C   sP   t | ƒ}| ¡ }t | ¡ ¡}tj |¡}| ¡ }tj 	|¡}| 
|¡s&J ‚d S r   )rC   r?   r)   rM   r   rH   rN   Úread_allr0   r1   rS   )r   Úfixturer;   rX   rY   ÚresultÚexpectedr   r   r   Útest_file_read_all¯   s   rƒ   c           	      C   s®   |   ¡  |  ¡ }tj |¡}tj t |¡¡}t |¡}| ¡ }| ¡ }| ¡ }| |¡s/J ‚| |¡s6J ‚|j	}|j
dks@J ‚|jdksGJ ‚|j	|ksNJ ‚|j	|ksUJ ‚d S ©Né   r   )r?   r   r)   rH   rN   rM   ÚRecordBatchFileReaderr   rS   r4   Únum_messagesrO   )	rh   ÚsourceÚreader1Úreader2Úreader3Úresult1Úresult2Úresult3Úst1r   r   r   Útest_open_file_from_bufferÀ   s   
r   c                 C   sT   dd„ |   ¡ D ƒ}t |  ¡ ¡}tj |¡}| ¡ }t |¡j	dd}t
||ƒ d S )Nc                 S   ó   g | ]}|  ¡ ‘qS r   ©Ú	to_pandas©r   r=   r   r   r   r!   Ù   r"   z)test_file_read_pandas.<locals>.<listcomp>T©Údrop)r?   r)   rM   r   rH   rN   Úread_pandasÚpdÚconcatÚreset_indexr   )rh   ÚframesrX   rY   r   r‚   r   r   r   Útest_file_read_pandas×   s   rœ   c                 C   sŽ   |   ¡  |  ¡ }| d¡j}t|dƒ}| |¡ W d   ƒ n1 s#w   Y  tj t	 
|¡¡ ¡ }tj t |¡¡ ¡ }| |¡sEJ ‚d S )Nú
file.arrowÚwb)r?   r   ÚjoinÚstrpathÚopenÚwriter)   rH   rN   ÚpathlibÚPathr   ÚOSFilerS   )rh   Útmpdirrˆ   ÚpathÚfÚt1Út2r   r   r   Útest_file_pathlibã   s   ÿr«   c                  C   sH   t  d¡} t tj¡ tj | ¡ W d   ƒ d S 1 sw   Y  d S rp   )r
   r   rr   rs   r)   rt   rH   Úopen_streamru   r   r   r   Útest_empty_streamñ   s   
"ÿr­   c              	   C   s¸   t  ¡ }t  t  d|¡g¡}|  d¡j}t  |d¡3}t j ||¡}t j	dg|d}t  
|g|¡}| |¡ W d  ƒ n1 sAw   Y  W d  ƒ n1 sPw   Y  td|ƒ dS )zËARROW-15783: Verify to_pandas works for interval types.

    Interval types require static structures to be enabled. This test verifies
    that they are when no other library functions are invoked.
    Únumsr   rž   ©é   é   é   ©ÚtypeNzread_record_batch.py)r)   Úmonth_day_nano_intervalr(   ÚfieldrŸ   r    r¥   rH   rI   Úarrayr.   r¢   r   )r¦   Úmdn_interval_typer(   r§   r   r:   Úinterval_arrayr=   r   r   r   Ú"test_read_year_month_nano_interval÷   s   ý€ÿrº   c              	   C   s¦   t  tj d¡t jdtjdddgddgdddœ¡}tj 	|¡}|  
| j|j¡}| |¡ W d   ƒ n1 s8w   Y  tj t |  ¡ ¡¡ ¡ }t| ¡ |ƒ d S )Nr   r#   r$   T©Ú
categoriesÚordered©r   r   )r˜   Ú	DataFrameÚnpr   ÚrandnÚCategoricalÚnanr)   ÚRecordBatchÚfrom_pandasr,   r   r(   r3   rH   r¬   rM   r   r   r   r“   )ri   Údfr=   Úwrr>   r   r   r   Ú!test_stream_categorical_roundtrip  s   
þþÿÿrÈ   c           	      C   sÄ   |   ¡  |  ¡ }tj |¡}tj t |¡¡}t |¡}| ¡ }| ¡ }| ¡ }| |¡s/J ‚| |¡s6J ‚|j	}|j
dks@J ‚|jdksGJ ‚|j	|ksNJ ‚|j	|ksUJ ‚t|ƒt| jƒks`J ‚d S r„   )r?   r   r)   rH   r¬   rM   ÚRecordBatchStreamReaderr   rS   r4   r‡   rO   rW   r5   )	ri   rˆ   r‰   rŠ   r‹   rŒ   r   rŽ   r   r   r   r   Útest_open_stream_from_buffer  s    
rÊ   rG   F©Úuse_threadsc                 C   sd   |   ¡  |  ¡ }tjj||d}| ¡  |j}|jdksJ ‚|jdks%J ‚t	|ƒt	| j
ƒks0J ‚d S ©NrF   r…   r   )r?   r   r)   rH   r¬   r   r4   r‡   rO   rW   r5   )ri   rG   rˆ   rY   Ústr   r   r   Útest_open_stream_options5  s   rÏ   c                 C   óP   |   ¡  |  ¡ }t t¡ tjj|dd W d   ƒ d S 1 s!w   Y  d S ©NTrF   )r?   r   rr   rs   Ú	TypeErrorr)   rH   r¬   )ri   rˆ   r   r   r   Ú#test_open_stream_with_wrong_optionsG  ó
   "ÿrÓ   c                 C   sN   |   ¡  |  ¡ }tjj||d}| ¡  |j}|jdksJ ‚|jdks%J ‚d S rÍ   )	r?   r   r)   rH   rN   r   r4   r‡   rO   )rh   rG   rˆ   rY   rÎ   r   r   r   Útest_open_file_optionsO  s   rÕ   c                 C   rÐ   rÑ   )r?   r   rr   rs   rÒ   r)   rH   rN   )rh   rˆ   r   r   r   Ú!test_open_file_with_wrong_options`  rÔ   rÖ   c              	   C   sÒ   t  tj d¡t jdtjdddgddgdddœ¡}tjj	|dd}tj
j	|dd}|  | j|j¡}| |¡ | |¡ W d   ƒ n1 sGw   Y  tj t |  ¡ ¡¡ ¡ }t| ¡ t j||gdd	ƒ d S )
Nr   r#   r$   Tr»   r¾   F©Úpreserve_index©Úignore_index)r˜   r¿   rÀ   r   rÁ   rÂ   rÃ   r)   r0   rÅ   rÄ   r,   r   r(   r¢   rH   r¬   rM   r   r   r   r“   r™   )ri   rÆ   r>   r=   rÇ   r   r   r   Útest_stream_write_dispatchh  s&   
þþ
þÿÿrÛ   c                 C   sú   t  dtj d¡i¡}tjj|d d… dd}tjj|dd}tj 	|||g¡}|  
| j|j¡}|j|dd W d   ƒ n1 sBw   Y  ttj |  ¡ ¡ƒ}ttt|ƒƒg d¢ks^J ‚tj 	|¡}t| ¡ t j|d d… ||d d… gd	d
ƒ d S )Nr   é   é
   Fr×   é   )Úmax_chunksize)rÝ   rÞ   r   rÝ   TrÙ   )r˜   r¿   rÀ   r   rÁ   r)   rÄ   rÅ   r0   r1   r,   r   r(   r2   ÚlistrH   r¬   r   ÚmaprP   r   r“   r™   )ri   rÆ   Úb1Úb2r>   rÇ   r;   Úresult_tabler   r   r   Útest_stream_write_table_batches}  s"   ÿÿÿÿrå   c                 C   s°   |   ¡ }t |  ¡ ¡}tj |¡}|j |d j¡sJ ‚d}t|ƒD ]\}}| || ¡s/J ‚|d7 }q"|t	|ƒks<J ‚t
 t¡ | ¡  W d   ƒ d S 1 sQw   Y  d S ©Nr   r°   )r?   r)   rM   r   rH   r¬   r(   rS   rQ   rP   rr   rs   ÚStopIterationÚread_next_batch)ri   r;   rX   rY   Útotalr<   Ú
next_batchr   r   r   Útest_stream_simple_roundtrip•  s   

"ÿrë   c                  C   s0  t  ¡ } dd„ tdƒD ƒ}tjj|gdgd}tjjdd}tjj| |j	|d}| 
|¡ W d   ƒ n1 s8w   Y  t|  ¡ ƒ}t  ¡ }tjdd	d
}tjj|d}tjj||j	|d}| 
|¡ W d   ƒ n1 sow   Y  t| ¡ ƒ}||ks€J ‚tj | ¡ ¡ }	tj |¡ ¡ }
|	|
ks–J ‚d S )Nc                 S   s   g | ]}t  d d¡‘qS )r   r²   )r   Úrandintr   r   r   r   r!   ª  s    z.test_compression_roundtrip.<locals>.<listcomp>i'  Úvalues©ÚnamesÚzstd©ÚcompressionrF   r   )Úcompression_level)r
   r   r-   r)   r0   Úfrom_arraysrH   ÚIpcWriteOptionsÚRecordBatchFileWriterr(   r2   rP   r   ÚCodecrN   r   )r   rí   r>   rG   r:   Úlen1Úsink2ÚcodecÚlen2r©   rª   r   r   r   Útest_compression_roundtrip§  s2   ÿþÿþrü   c               	   C   sò  t j ¡ } | jdu sJ ‚| jdu sJ ‚| jt jjjksJ ‚d| _| jdu s'J ‚d| _| jdu s1J ‚t jjj| _| jt jjjksAJ ‚dD ]}t	 
ttf¡ || _W d   ƒ n1 sZw   Y  qC| jd u sgJ ‚dD ]}t j |¡r‡|| _| j|ks{J ‚| ¡ | _| j|ks‡J ‚qid | _| jd u s’J ‚t	 
t¡ d| _W d   ƒ n1 s¥w   Y  | jdu s±J ‚d| _| jdu s»J ‚t j d¡rõt jjt jjjddddd} | jt jjjksÙJ ‚| jdu sàJ ‚| jdu sçJ ‚| jdksîJ ‚| jdu s÷J ‚d S d S )NFT)ÚV5é*   )Úlz4rð   r   rÿ   )Úmetadata_versionÚallow_64bitÚuse_legacy_formatrò   rÌ   )r)   rH   rõ   r  r  r   ÚMetadataVersionrý   ÚV4rr   rs   rÒ   Ú
ValueErrorrò   r÷   Úis_availableÚupperrÌ   )rG   Úvaluer   r   r   Útest_write_optionsÆ  s\   
ÿ€
€ÿûõr	  T)r  )r  )r   )r  r   c           	      C   sÖ   || _ |  ¡ }t |  ¡ ¡}tj |  ¡ ¡}|j|jksJ ‚tj |¡}|j	 
|d j	¡s/J ‚d}t|ƒD ]\}}| 
|| ¡sBJ ‚|d7 }q5|t|ƒksOJ ‚t t¡ | ¡  W d   ƒ d S 1 sdw   Y  d S ræ   )rG   r?   r)   rM   r   rH   Úread_messager   r¬   r(   rS   rQ   rP   rr   rs   rç   rè   )	ri   rG   r;   rX   ÚmessagerY   ré   r<   rê   r   r   r   Útest_stream_options_roundtrip÷  s   	

"ÿr  c                  C   s\  t j ¡ } | jdu sJ ‚| jdu sJ ‚| jt jjjksJ ‚| jg ks$J ‚d| _| jdu s.J ‚t jjj	| _| jt jjj	ks>J ‚t jjj
| _| jt jjj
ksNJ ‚d| _| jdu sXJ ‚ddg| _| jddgksfJ ‚t t¡ d | _W d   ƒ n1 syw   Y  t jjddt jjj	dgd} | jdu s“J ‚| jdu sšJ ‚| jt jjj	ks¤J ‚| jdgks¬J ‚d S )NTFr   r°   )rÌ   Úensure_native_endianÚensure_alignmentÚincluded_fields)r)   rH   ÚIpcReadOptionsrÌ   r  r  Ú	AlignmentÚAnyr  ÚDataTypeSpecificÚAt64Byterr   rs   rÒ   rF   r   r   r   Útest_read_options  s4   

ÿþr  c                 C   s  t j ¡ }t jjdgd}t jjt  g d¢¡t  g d¢¡gddgd}|  | j|j¡}| 	|¡ W d   ƒ n1 s;w   Y  |  
¡ }t jj||d}t jj||t  ¡ d	}| ¡ }| ¡ }	|jd
ksfJ ‚|	jdksmJ ‚t jjt  g d¢¡gdgd}
|	|
ks‚J ‚||ksˆJ ‚d S )Nr°   )r  ©r#   r$   Úbazr&   ©r°   r±   r²   é   ÚaÚbrî   rF   )rG   Úmemory_poolr±   )r)   rH   r  r0   rô   r·   r,   r   r(   r2   r   r¬   Úsystem_memory_poolr   Únum_columns)ri   Úoptions1Úoptions2r>   rÇ   rˆ   r‰   rŠ   rŒ   r   r‚   r   r   r   Ú!test_read_options_included_fields8  s,   
ÿþÿ
ÿr!  c                    sŒ  t  t  ¡ t  ¡ ¡‰g d¢g d¢ddgg d¢ddgg}‡fdd„|D ƒ}|d d	… }|d
 j‰d‡ ‡fdd„	}ˆ jrot t j¡ ||ƒ W d   ƒ n1 sOw   Y  t t j¡ ||ƒ W d   ƒ n1 siw   Y  n ||ƒ}|j	dkszJ ‚|j
d	ksJ ‚|jdksˆJ ‚|jd
ksJ ‚d ˆ _t jjddˆ _ˆ jr¸t t j¡ ||ƒ W d   ƒ n1 s²w   Y  n ||ƒ}|j	dksÃJ ‚|j
d	ksÊJ ‚|jdksÑJ ‚|jdksØJ ‚||ƒ}|j	d	ksãJ ‚|j
dksêJ ‚|jd
ksñJ ‚|jdksøJ ‚t jjddˆ _||dd}|j	dksJ ‚ˆ jr,|j
dksJ ‚|jd
ks"J ‚|jd
ks*J ‚d S |j
d	ks4J ‚|jdks<J ‚|jd
ksDJ ‚d S )N)r#   r#   N)r#   r$   r#   r#   r$   )r#   Nr$   Úquuxr"  c                    s*   g | ]}t jjt j|ˆ d gdgd‘qS )r³   Údictsrî   )r)   rÄ   rô   r·   )r   Úv)Útyr   r   r!   Y  s    ÿÿz)test_dictionary_delta.<locals>.<listcomp>r  r   Fc                    sj   ˆ   t ¡ ˆ¡#}|rtj | ¡}| |¡ n
| D ]}| |¡ q|jW  d   ƒ S 1 s.w   Y  d S r   )r,   r)   ÚMockOutputStreamr0   r1   r2   r3   r4   )r;   r8   r:   r>   r=   )ro   r(   r   r   r?   _  s   
ÿ$øz,test_dictionary_delta.<locals>.write_batchesr   r²   T)Úemit_dictionary_deltasr°   r±   )Úunify_dictionariesrL   r[   )r)   Ú
dictionaryÚint8r+   r(   r\   rr   rs   rt   rO   Únum_dictionary_batchesÚnum_replaced_dictionariesÚnum_dictionary_deltasr_   rH   rõ   rG   )ro   Údatar;   Úbatches_delta_onlyr?   rÎ   r   )ro   r(   r%  r   Útest_dictionary_deltaQ  sr   ü
þ

ÿ
ÿ€ÿ
ÿ€ÿr0  c               	   C   s\  t  t  dt  ¡ ¡g¡} t j t  ¡ | ¡}|jrJ ‚|jt jj	j
ks$J ‚t j t  ¡ | ¡}|jr2J ‚|jt jj	j
ks<J ‚tddƒ8 t j t  ¡ | ¡}|jsPJ ‚|jt jj	j
ksZJ ‚t j t  ¡ | ¡}|jshJ ‚|jt jj	j
ksrJ ‚W d   ƒ n1 s|w   Y  tddƒ8 t j t  ¡ | ¡}|jr•J ‚|jt jj	jksŸJ ‚t j t  ¡ | ¡}|jr­J ‚|jt jj	jks·J ‚W d   ƒ n1 sÁw   Y  tddƒX tddƒ9 t j t  ¡ | ¡}|jsàJ ‚|jt jj	jksêJ ‚t j t  ¡ | ¡}|jsøJ ‚|jt jj	jksJ ‚W d   ƒ n1 sw   Y  W d   ƒ d S W d   ƒ d S 1 s'w   Y  d S )Nr#   ÚARROW_PRE_0_15_IPC_FORMATÚ1ÚARROW_PRE_1_0_METADATA_VERSION)r)   r(   r¶   Úint32rH   r^   r~   Ú_use_legacy_formatÚ_metadata_versionr  rý   rI   r   r  )r(   r:   r   r   r   Ú!test_envvar_set_legacy_ipc_formatœ  sF   



ú

ú

 úÿ$ÿr7  c                 C   sH   |   ¡ }t |  ¡ ¡}tj |¡}| ¡ }tj |¡}| 	|¡s"J ‚d S r   )
r?   r)   rM   r   rH   r¬   r   r0   r1   rS   )ri   r;   rX   rY   r   r‚   r   r   r   Útest_stream_read_allÀ  s   r8  c                 C   sN   dd„ |   ¡ D ƒ}|  ¡ }tj |¡}| ¡ }t |¡jdd}t	||ƒ d S )Nc                 S   r‘   r   r’   r”   r   r   r   r!   Ì  r"   z+test_stream_read_pandas.<locals>.<listcomp>Tr•   )
r?   r   r)   rH   r¬   r—   r˜   r™   rš   r   )ri   r›   rX   rY   r   r‚   r   r   r   Útest_stream_read_pandasÊ  s   r9  c                 C   s2   |   ¡ }|  ¡ }t |¡}tj |¡}|t|ƒfS r   )r?   r   r)   rM   ÚMessageReaderr¬   rà   )ri   r;   rX   Ú
buf_readerrY   r   r   r   Úexample_messagesÕ  s
   
r<  c                   C   sr   t  t¡ tt ¡ ƒ W d   ƒ n1 sw   Y  t  t¡ tt ¡ ƒ W d   ƒ d S 1 s2w   Y  d S r   )rr   rs   rÒ   Úreprr)   ÚMessager:  r   r   r   r   Útest_message_ctors_no_segfaultÞ  s   ÿ"ÿr?  c                 C   sÆ   | \}}t |ƒdksJ ‚|d jdksJ ‚t|d jtjƒs J ‚t|d jtjƒs+J ‚|d jtjj	ks6J ‚|dd … D ]$}|jdksEJ ‚t|jtjƒsNJ ‚t|jtjƒsWJ ‚|jtjj	ks`J ‚q<d S )Nr…   r   r(   r°   zrecord batch)
rP   r´   rT   Úmetadatar)   ÚBufferÚbodyr   r  rý   )r<  r    ÚmessagesÚmsgr   r   r   Útest_message_readeræ  s   ürE  c           
      C   s  | \}}|d }|  ¡ }t | ¡ d ¡}tj |¡}tj |¡}tj | ¡ ¡}tj |¡}	| |¡s6J ‚| |¡s=J ‚| |¡sDJ ‚| |	¡sKJ ‚tjtj	dd tj t d¡¡ W d   ƒ n1 sgw   Y  t t
¡ tj |¡ W d   ƒ d S 1 sƒw   Y  d S )Nr   r±   zCorrupted message©Úmatchs   ab)Ú	serializer)   rM   Ú
to_pybytesrH   r
  rS   rr   rs   rt   ÚEOFError)
r<  r    rC  rD  rv   rY   ÚrestoredÚ	restored2Ú	restored3Ú	restored4r   r   r   Ú#test_message_serialize_read_messageö  s$   ÿ"ÿrO  c              	   C   s‚   | \}}|D ]8}t  ¡ }t j|dd}| |¡ W d   ƒ n1 s#w   Y  | ¡ }t j t j|dd¡}| |¡s>J ‚qd S )NÚgziprñ   )	r)   r~   Úoutput_streamÚserialize_tor   rH   r
  Úinput_streamrS   )r<  r    rC  r  Úraw_outÚcompressed_outÚcompressed_bufr   r   r   r   Ú!test_message_read_from_compressed  s   ÿÿ÷rW  c                 C   s0   | \}}t j |d ¡}| |d j¡sJ ‚d S ræ   )r)   rH   Úread_schemarS   r(   )r<  r;   rC  r(   r   r   r   Útest_message_read_schema  s   rY  c                 C   sF   | \}}t ||dd … ƒD ]\}}tj ||j¡}| |¡s J ‚qd S )Nr°   )Úzipr)   rH   Úread_record_batchr(   rS   )r<  r;   rC  r=   r  Ú
read_batchr   r   r   Útest_message_read_record_batch$  s
   þr]  c                  C   s°   t jt jdgt  ¡ dgdgd} t  ¡ }t j || j¡}| | ¡ W d   ƒ n1 s-w   Y  | 	¡ }t
jtdd t j || j¡ W d   ƒ d S 1 sQw   Y  d S )Nó   foor³   Ústrsrî   z type record batch but got schemarF  )r)   r.   r·   r+   r~   rH   r^   r(   r3   r   rr   rs   ÚIOErrorr[  )r=   Ústreamr:   rv   r   r   r   Ú.test_read_record_batch_on_stream_error_message,  s   ÿÿÿ"þrb  c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚStreamReaderServerc                 C   sT   t   t jt j¡| _| j d¡ | j d¡ | j ¡ \}}|| _d | _g | _	d | _
|S )N)ú	127.0.0.1r   r°   )ÚsocketÚAF_INETÚSOCK_STREAMÚ_sockÚbindÚlistenÚgetsocknameÚ_do_read_allÚ_schemaÚ_batchesÚ_table)r   Údo_read_allÚhostÚportr   r   r   Úinit?  s   zStreamReaderServer.initc                 C   s¤   | j  ¡ \}}z@|jdd}tj |¡}|j| _| jr!| 	¡ | _
nt|ƒD ]\}}| j |¡ q%W | ¡  | j  ¡  d S W | ¡  | j  ¡  d S | ¡  | j  ¡  w )NÚrb©Úmode)rh  ÚacceptÚmakefiler)   rH   r¬   r(   rm  rl  r   ro  rQ   rn  r/   r6   )r   Ú
connectionÚclient_addressrˆ   rY   r<   r=   r   r   r   ÚrunJ  s"   ýüÿzStreamReaderServer.runc                 C   s   | j | jr	| jfS | jfS r   )rm  rl  ro  rn  r   r   r   r   Ú
get_resultY  s   ÿzStreamReaderServer.get_resultN)r@   rA   rB   rs  r{  r|  r   r   r   r   rc  =  s    rc  c                   @   s4   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ ZdS )ÚSocketStreamFixturec                 C   s   d S r   r   r   r   r   r   r   `  s   zSocketStreamFixture.__init__c                 C   sN   t ƒ | _| j |¡}| j ¡  t tjtj¡| _| j d|f¡ |  	¡ | _
d S )Nrd  )rc  Ú_serverrs  Ústartre  rf  rg  rh  Úconnectr   r   )r   rp  rr  r   r   r   Ústart_servere  s   
z SocketStreamFixture.start_serverc                 C   sD   dd l }| j | dd¡¡ | j ¡  | j ¡  | j ¡  | j 	¡ S )Nr   ÚQ)
Ústructr   r¢   ÚpackÚflushrh  r6   r~  rŸ   r|  )r   rƒ  r   r   r   Ústop_and_get_resultm  s   



z'SocketStreamFixture.stop_and_get_resultc                 C   s   | j jddS )Nrž   ru  )rh  rx  r   r   r   r   r   u  s   zSocketStreamFixture.get_sinkc                 C   ra   r   rb   rJ   r   r   r   r,   x  rd   zSocketStreamFixture._get_writerN)r@   rA   rB   r   r  r†  r   r,   r   r   r   r   r}  ^  s    r}  c                   C   re   r   )r}  r   r   r   r   Úsocket_fixture|  rg   r‡  c                 C   sp   | j dd |  ¡ }|  ¡ \}}| |d j¡sJ ‚t|ƒt|ƒks$J ‚t|ƒD ]\}}||  |¡s5J ‚q(d S )NF©rp  r   )r  r?   r†  rS   r(   rP   rQ   )r‡  Úwriter_batchesÚreader_schemaÚreader_batchesr<   r=   r   r   r   Útest_socket_simple_roundtrip  s   ÿrŒ  c                 C   s>   | j dd |  ¡ }|  ¡ \}}tj |¡}| |¡sJ ‚d S )NTrˆ  )r  r?   r†  r)   r0   r1   rS   )r‡  r‰  r    r   r‚   r   r   r   Útest_socket_read_all  s
   r  c                  C   s`   t  ddgi¡} tj | ¡}t ¡ }t||ƒ | ¡ }tj 	|dd … ¡}| 
¡ }t| |ƒ d S )Nr#   ç      ø?é   )r˜   r¿   r)   rÄ   rÅ   r~   Ú
write_filer   rH   r¬   r—   r   )rÆ   r=   r   ÚbufferrY   Úrdfr   r   r   Útest_ipc_file_stream_has_eosš  s   
r“  c                  C   sl   t  ddgi¡} tj | ¡}t ¡ }t||ƒ | ¡ }t |¡}t	|ƒ}|d  
¡ }t  |¡}t| |ƒ d S )Nr#   rŽ  r   )r˜   r¿   r)   rÄ   rÅ   r~   r  r   rM   Ú	read_filer“   r   )rÆ   r=   r   r‘  rY   r;   r.  r’  r   r   r   Útest_ipc_zero_copy_numpy¬  s   


r•  Úipc_typera  Úfilec           
         s¢  t  ddgi¡}tj |¡}t ¡ }d}tjjtjjdœ|  }|||j	ƒ}t
|ƒD ]}|j|dt|ƒid q*| |¡ W d   ƒ n1 sGw   Y  | ¡ }| dkrqtj |¡‰ tˆ  ¡ ƒ}	W d   ƒ n1 skw   Y  n"tj |¡‰ ‡ fdd	„t
ˆ jƒD ƒ}	W d   ƒ n1 sŽw   Y  t
|ƒD ]$}|	| jjd
ks£J ‚t|	| jtjƒs®J ‚|	| jdt|ƒiks»J ‚q—|	| jjd
ksÆJ ‚|	| jd u sÏJ ‚d S )Nr#   rŽ  r±   )ra  r—  Úbatch_id)Úcustom_metadatara  c                    ó   g | ]}ˆ   |¡‘qS r   )Úget_batch_with_custom_metadata©r   r<   ©rY   r   r   r!   Ö  s    ÿz?test_batches_with_custom_metadata_roundtrip.<locals>.<listcomp>r°   )r˜   r¿   r)   rÄ   rÅ   r~   rH   r^   rI   r(   r-   r3   Ústrr   r¬   rà   Ú!iter_batches_with_custom_metadatarN   rO   r=   Únum_rowsrT   r™  ÚKeyValueMetadata)
r–  rÆ   r=   r   Úbatch_countÚfile_factoryr:   r<   r‘  Úbatch_with_metasr   r  r   Ú+test_batches_with_custom_metadata_roundtrip½  sB   ÿÿüÿ€
ÿÿÿr¥  c                  C   sÆ   t jjt  g d¢¡t  g d¢¡gddgd} t  ¡ }t j || j¡ W d   ƒ n1 s-w   Y  | ¡ }t j 	|¡}| 
¡ }W d   ƒ n1 sKw   Y  |j | j¡sYJ ‚t|ƒdksaJ ‚d S )Nr  r  r  r  rî   r   )r)   r0   rô   r·   r~   rH   r^   r(   r   r¬   r   rS   rP   )r>   r   rˆ   rY   r   r   r   r   Útest_ipc_stream_no_batchesä  s   ÿþÿ
ÿr¦  c                  C   sF   d} d}t  dtj | ¡i¡}tj |¡}tj 	|¡| | ks!J ‚d S )NrÝ   r  r#   )
r˜   r¿   rÀ   r   rÁ   r)   rÄ   rÅ   rH   Úget_record_batch_size)ÚNÚitemsizerÆ   r=   r   r   r   Útest_get_record_batch_sizeö  s
   rª  c                 C   s2   t j| |rdndd}t j||d}t|| ƒ d S )Nr±   r°   )ÚnthreadsrË   )r)   Úserialize_pandasÚdeserialize_pandasr   )rÆ   rÌ   rv   r   r   r   r   Ú"_check_serialize_pandas_round_trip   s   r®  c                  C   sD   t jg d¢dd} ddg}t jg d¢tdƒdœ| |d	}t|ƒ d S )
Nr¯   Úmy_index©Únamer#   r$   ©rŽ  gš™™™™™ù?g333333û?Úabc©r#   r$   ©ÚindexÚcolumns©r˜   ÚIndexr¿   rà   r®  ©r¶  r·  rÆ   r   r   r   Ú test_pandas_serialize_round_trip  s   þr»  c                  C   sH   t jg d¢dd} ddg}t jg d¢tdƒdœ| |d	}t|d
d d S )Nr¯   r¯  r°  r#   r$   r²  r³  r´  rµ  TrË   r¸  rº  r   r   r   Ú)test_pandas_serialize_round_trip_nthreads  s   þr¼  c                  C   sf   t jg d¢dd} t jtdƒd d}t j | |g¡}ddg}t jg d¢tdƒd	œ||d
}t|ƒ d S )Nr¯   Úlevel_1r°  Údefr#   r$   r²  r³  r´  rµ  )r˜   r¹  rà   Ú
MultiIndexrô   r¿   r®  )Úindex1Úindex2r¶  r·  rÆ   r   r   r   Ú,test_pandas_serialize_round_trip_multi_index  s   ýrÂ  c                  C   s   t  ¡ } t| ƒ d S r   )r˜   r¿   r®  )rÆ   r   r   r   Ú%test_serialize_pandas_empty_dataframe,  s   rÃ  c                  C   s:   t  ttg d¢dƒƒ¡} t | ¡}t |¡}t|| ƒ d S )Nr²  r³  )r˜   r¿   rà   rZ  r)   r¬  r­  r   )rÆ   rv   r   r   r   r   Ú3test_pandas_serialize_round_trip_not_string_columns2  s   

rÄ  c                  C   st   t jdg d¢ig d¢d} t  dg d¢i¡}tj| dd}t |¡}t||ƒ tj| dd}t |¡}t|| ƒ d S )Nr  r¯   )r¶  Fr×   T)r˜   r¿   r)   r¬  r­  r   )rÆ   r‚   rv   r   r   r   r   Ú'test_serialize_pandas_no_preserve_index:  s   


rÅ  c                  C   sr   d} t  tj | ¡dtjdddgdœ¡}tj |¡}|j	 
¡ }| 
¡ }tj |¡}tj ||¡}| |¡s7J ‚d S )Nr   r#   r$   r%   r&   r¾   )r˜   r¿   rÀ   r   rÁ   rÃ   r)   rÄ   rÅ   r(   rH  rH   rX  r[  rS   )r9   rÆ   r=   Ús_schemaÚs_batchÚrecons_schemaÚrecons_batchr   r   r   Ú#test_schema_batch_serialize_methodsH  s   
þ
rÊ  c                  C   s¢   dddœ} dddœ}t  dt  ¡ ¡}t jdt  ¡ | d}t j||g|d}| ¡ }t j |¡}| |¡s6J ‚|j	|ks=J ‚|d j	d u sFJ ‚|d	 j	| ksOJ ‚d S )
Ns   bars   field)r^  s   kinds   schemar  r  ©r@  r   r°   )
r)   r¶   r*  Ústringr(   rH  rH   rX  rS   r@  )Úfield_metadataÚschema_metadataÚf0Úf1r(   rÆ  rÈ  r   r   r   Ú'test_schema_serialization_with_metadataX  s   

rÑ  c                 C   s@   t j || j¡}| | ¡ W d   ƒ d S 1 sw   Y  d S r   )r)   rH   rI   r(   r3   )r=   r   r:   r   r   r   r  j  s   "ÿr  c                    sH   t j | ¡‰ ‡ fdd„tˆ jƒD ƒW  d   ƒ S 1 sw   Y  d S )Nc                    rš  r   )rR   rœ  r  r   r   r!   q  s    zread_file.<locals>.<listcomp>)r)   rH   rN   r-   rO   )rˆ   r   r  r   r”  o  s   $ÿr”  c                  C   s®   t  dt  ¡ fg¡} t  ¡ }t j || ¡ W d   ƒ n1 s w   Y  | ¡ }t  t  |¡¡}| 	¡ }W d   ƒ n1 s@w   Y  t
|ƒdksMJ ‚|j | ¡sUJ ‚d S )Nr¶   r   )r)   r(   Úint64r~   rH   rI   r   r†   rM   r   rP   rS   )r(   r   rv   rY   r>   r   r   r   Útest_write_empty_ipc_filet  s   ÿ
ÿrÓ  c                     sŠ  dd„ ‰ ‡ fdd„} t | ƒ ƒ}t |¡}tj ˆ ƒ |¡!}d }|ƒ d us&J ‚t|ƒ| ƒ ks/J ‚|ƒ d u s6J ‚W d   ƒ n1 s@w   Y  tt | ƒ ƒƒ}t |¡}tj ˆ ƒ |¡!}d }|ƒ d uscJ ‚t|ƒ| ƒ kslJ ‚|ƒ d u ssJ ‚W d   ƒ n1 s}w   Y  | ƒ }t 	t
¡ tj dt ¡ fg|¡}W d   ƒ n1 s¡w   Y  t 	t
¡ tj d |¡}W d   ƒ d S 1 s¾w   Y  d S )Nc                   S   s   t  dt  ¡ fg¡S )Nr¶   )r)   r(   rÒ  r   r   r   r   Úmake_schema…  rK   z0test_py_record_batch_reader.<locals>.make_schemac                     s6   ˆ ƒ } t jg d¢g| d}t jddgg| d}||gS )Nr¯   r'   r  r   )r)   r.   )r(   Úbatch1Úbatch2©rÔ  r   r   Úmake_batchesˆ  s   z1test_py_record_batch_reader.<locals>.make_batchesr¶   )r   ÚweakrefÚrefr)   ÚRecordBatchReaderr1   rà   Úiterrr   rs   rÒ   rÒ  )rØ  r;   rÇ   rY   r   r×  r   Útest_py_record_batch_reader„  sF   


ÿû

ÿû	ÿý"þrÝ  c                  C   sä  G dd„ dƒ} t jt jg d¢t  ¡ dgdgdt jt jg d¢t  ¡ dgdgdg}| |ƒ}t j |¡}t j |¡}| ¡ |ksCJ ‚t j |¡}| ¡ |ksQJ ‚t jj||d j	d	}| ¡ |ksdJ ‚t  	t  
dt  ¡ ¡g¡}t jj||d	}| ¡ | |¡ksƒJ ‚tjt jjd
d t jj|t  	t  
dt  t  ¡ ¡¡g¡d	 W d   ƒ n1 s¬w   Y  t t¡ t j |d d ¡ W d   ƒ n1 sËw   Y  t t¡ t jj||d d	 W d   ƒ d S 1 sëw   Y  d S )Nc                   @   s   e Zd Zdd„ Zddd„ZdS )zAtest_record_batch_reader_from_arrow_stream.<locals>.StreamWrapperc                 S   s
   || _ d S r   )r;   )r   r;   r   r   r   r   ´  r   zJtest_record_batch_reader_from_arrow_stream.<locals>.StreamWrapper.__init__Nc                 S   s"   t j | jd j| j¡}| |¡S )Nr   )r)   rÛ  r1   r;   r(   Ú__arrow_c_stream__)r   Úrequested_schemarY   r   r   r   rÞ  ·  s   ÿ
zTtest_record_batch_reader_from_arrow_stream.<locals>.StreamWrapper.__arrow_c_stream__r   )r@   rA   rB   r   rÞ  r   r   r   r   ÚStreamWrapper³  s    rà  r¯   r³   r  rî   ©r  r   r…   r   r'   úField 0 cannot be castrF  )r)   r.   r·   rÒ  r0   r1   rÛ  Úfrom_streamr   r(   r¶   r4  Úcastrr   rs   ÚlibÚArrowTypeErrorÚlist_rÒ   )rà  r.  Úwrapperr‚   rY   Úgood_schemar   r   r   Ú*test_record_batch_reader_from_arrow_stream±  s4   "
"þÿÿÿ"ÿrê  c                  C   s  t  t  dt  ¡ ¡g¡} t jt jg d¢t  ¡ dgdgdt jt jg d¢t  ¡ dgdgdg}t j |¡}t j | |¡}| 	| ¡ 
¡ |ksHJ ‚t  t  dt  ¡ ¡g¡}t j | |¡}| 	|¡ 
¡ | 	|¡ksiJ ‚t j | |¡}tjtdd | 	t  g ¡¡ W d   ƒ n1 sŠw   Y  t j | |¡}tjt jjdd | 	t  t  dt  t  ¡ ¡¡g¡¡ W d   ƒ n1 s¼w   Y  t  t  dt  ¡ ¡g¡} t jt d	d
d¡gt  ¡ d}t j|gdgdt j|gdgdg}t j |¡}t j | |¡}| 	| ¡ 
¡ |ksJ ‚d S )Nr  r¯   r³   rî   rá  zTarget schema's field namesrF  râ  iè  r…   é   )r)   r(   r¶   rÒ  r.   r·   r0   r1   rÛ  rä  r   r4  rr   rs   r  rå  ræ  rç  Údate32ÚdatetimeÚdate)Ú
schema_srcr.  Ú	table_srcrY   Ú
schema_dstÚarrr   r   r   Útest_record_batch_reader_castâ  s0   ""þÿ&ÿ$ró  c                  C   s\  t  t  dt  ¡ ¡g¡} t jt jg d¢t  ¡ dgdgdg}t jt jg d¢t  ¡ dgdgdg}t j |¡}t j |¡}t j | |¡}t  t  dt  	¡ ¡g¡}| 
|¡ ¡ | 
|¡ks_J ‚t j | |¡}t  t jdt  	¡ ddg¡}| 
|¡ ¡ | 
|¡ks‚J ‚t j | |¡}| 
|¡}tjt jjdd	 | ¡  W d   ƒ d S 1 s§w   Y  d S )
Nr  )r°   r±   Nr³   rî   r¯   F)ÚnullablezCan't cast arrayrF  )r)   r(   r¶   rÒ  r.   r·   r0   r1   rÛ  r4  rä  r   rr   rs   rå  rt   )rï  Údata_with_nullsÚdata_without_nullsÚtable_with_nullsÚtable_without_nullsrY   rñ  Úcasted_readerr   r   r   Ú#test_record_batch_reader_cast_nulls  s$   "ÿ"ÿ

"ÿrú  c                  C   sª   t  dg d¢i¡} dddœ}t  ¡ }t jj|| j|d}| | ¡ W d   ƒ n1 s,w   Y  | ¡ }t j |¡}|j	|ksCJ ‚W d   ƒ d S 1 sNw   Y  d S )Nr  r¯   s   tests   0.1.0)s   creators   versionrË  ©
r)   r>   r~   rH   rI   r(   r2   r   rN   r@  )ÚtblÚmetar   Úwr‘  Úrr   r   r   Ú+test_record_batch_file_writer_with_metadata$  s   
ÿ"ÿr   c                  C   sœ   t  dg d¢i¡} t  ¡ }t j || j¡}| | ¡ W d   ƒ n1 s%w   Y  | ¡ }t j |¡}|j	d u s<J ‚W d   ƒ d S 1 sGw   Y  d S )Nr  r¯   rû  )rü  r   rþ  r‘  rÿ  r   r   r   Ú1test_record_batch_file_writer_with_empty_metadata2  s   ÿ"ÿr  r[   )kÚcollectionsr   rí  r
   r£   rr   r   re  Ú	threadingrÙ  ÚnumpyrÀ   ÚImportErrorÚpyarrowr)   Úpyarrow.tests.utilr   r   Úpandas.testingr   Úpandasr˜   r   rC   r]   r`   r€   rf   rh   ri   rm   ro   rw   r{   r}   ÚmarkÚparametrizerƒ   r   rœ   r«   r­   Ú	processesrº   rÈ   rÊ   rH   r  rÏ   rÓ   rÕ   rÖ   rÛ   rå   rë   rð   rü   r	  rõ   r  r  r  r  r!  r0  r7  r8  r9  r<  r?  rE  rO  rP  rW  rY  r]  rb  ÚThreadrc  r}  r‡  ÚsocketsrŒ  r  r“  r•  r¥  r¦  rª  r®  r»  r¼  rÂ  rÃ  rÄ  rÅ  rÊ  rÑ  r  r”  rÓ  rÝ  rê  ró  rú  r   r  r   r   r   r   Ú<module>   s2  ÿÿ'


þþû

þ


þ
þ



1ÿû
"K$




!




%
	








-1%