o GhƱ@sddlZddlmZddlmZddlZddlmZddlZddl m Z ddl Z ddl Z ddl Z ddlZzddlZWn eyEdZYnwddlZddlmZmZmZiaiaiaddZd d Zd d Zd dZddZddZ dQddZ!ddZ"ddZ#ddZ$ddZ%ddZ&dd Z'd!d"Z(d#d$Z)d%d&Z*d'd(Z+dQd)d*Z,  ,dRd-d.Z-d/d0Z.dSd1d2Z/d3d4Z0 dTd6d7Z1hd8Z2d9d:Z3d;d<Z4d=d>Z5dQd?d@Z6 dQdAdBZ7dCdDZ8dEdFZ9dGdHZ:dIdJZ;dKdLZdS)UN)Sequence)futures)deepcopy) zip_longest) _pandas_api frombytesis_threading_enabledc Cstsjtitjjdtjjdtjjdtjjdtjjdtjj dtjj dtjj dtjj d tjj d tjjd tjjd tjjd tjjdtjjdtjjdtjjdtjjdtjjdtjjditS)Nemptyboolint8int16int32int64uint8uint16uint32uint64float16float32float64datetimebytesunicode)_logical_type_mapupdatepalibType_NA Type_BOOL Type_INT8 Type_INT16 Type_INT32 Type_INT64 Type_UINT8 Type_UINT16 Type_UINT32 Type_UINT64Type_HALF_FLOAT Type_FLOAT Type_DOUBLE Type_DATE32 Type_DATE64 Type_TIME32 Type_TIME64 Type_BINARYType_FIXED_SIZE_BINARY Type_STRINGr2r2V/var/www/html/Persson_Maskin/env/lib/python3.10/site-packages/pyarrow/pandas_compat.pyget_logical_type_map.sR     r4cCst}z||jWStyLt|tjjrYdSt|tjjr,dt|j dYSt|tjj r@|j durz&construct_metadata..cSsg|] }t|ts|qSr2)r=dict)r{descrr2r2r3r| )rnrFroFrz&The DataFrame has non-str index name `z@` which will be converted to string and not roundtrip correctly. stacklevellevelsnamesspandaspyarrow)libraryversion) index_columnscolumn_indexescolumnscreatorpandas_versionutf8)rbzipryappendr=r~rnrT_column_name_to_stringswarningswarn UserWarningrar_get_simple_index_descriptorjsondumpsr __version__rrencode)columns_to_convertdf column_names index_levelsindex_descriptorspreserve_indexrDcolumn_field_namesnum_serialized_index_levelsntypesdf_types index_typescolumn_metadatacolrnrorFrhindex_column_metadatanon_str_index_nameslevel descriptorrrrr2r2r3construct_metadatasp          rcCsTt|\}}t|}d|vrtjdtdd|dkr"|rJddi}|||||dS) NmixedzlThe DataFrame has column names of mixed type. They will be converted to strings and not roundtrip correctly.rrrencodingUTF-8rm)rjrZrrr)rrnrwrxrpr2r2r3r$s rcCsxt|tr|St|tr|dSt|trtttt|St|tr(td|dus6t|t r8t |r8|St|S)a!Convert a column name (or level) to either a string or a recursive collection of strings. Parameters ---------- name : str or tuple Returns ------- value : str or tuple Examples -------- >>> name = 'foo' >>> _column_name_to_strings(name) 'foo' >>> name = ('foo', 'bar') >>> _column_name_to_strings(name) "('foo', 'bar')" >>> import pandas as pd >>> name = (1, pd.Timestamp('2017-02-01 00:00:00')) >>> _column_name_to_strings(name) "('1', '2017-02-01 00:00:00')" rz%Unsupported type for MultiIndex levelN) r=rTrdecodetuplemaprrrtrrrKrsrnr2r2r3r8s     rcCs,|jdur|j|vrt|jSd|ddS)zReturn the name of an index level or a default name if `index.name` is None or is already a column name. Parameters ---------- index : pandas.Index i : int Returns ------- name : str N__index_level_d__)rnr)indexirr2r2r3_index_level_name_s rcCsPt|||}|jjstdt|j|durt|||Sg}g}|dur+t|jng}g}g}|D]-} || } t| } t | rJt d| d| | | d| | | t | q3g} g} t|D]0\} }t|| |} t|t jjr|durt|}n| || d| }| | | |qi|| }|||| | |||fS)NzDuplicate column names found: FSparse pandas data (column ) not supported.)_resolve_columns_of_interestr is_unique ValueErrorlist$_get_columns_to_convert_given_schema_get_index_level_valuesrrr is_sparsertrrT enumeraterr=pd RangeIndex_get_range_index_descriptor)rschemarrrrrrconvert_fieldsrnrrindex_column_namesr index_levelr all_namesr2r2r3_get_columns_to_convertrsR              rc Cs<g}g}g}g}g}g}|jD]} z|| } d} WnCty\zt|| } Wnttfy8td| dw|durEtd| d|durXt| tjjrXtd| dd} Ynwt | rjt d| d | | } | | | | | | | r| | | | | | q||} | |||||||fS) z Specialized version of _get_columns_to_convert in case a Schema is specified. In that case, the Schema is used as the single point of truth for the table structure (types, which columns are included, order of columns, ...). Fzname 'zF' present in the specified schema is not found in the columns or indexzd' present in the specified schema corresponds to the index, but 'preserve_index=False' was specifiedNz' is present in the schema, but it is a RangeIndex which will not be converted as a column in the Table, but saved as metadata-only not in columns. Specify 'preserve_index=True' to force it being added as a column, or remove it from the specified schemaTrr) rr<_get_index_level IndexErrorrr=rrrrrtfieldr)rrrrrrrrrrnris_indexrrr2r2r3rs^               rcCs8|}||jjvrt|rt|tdd}|j|S)z_ Get the index level of a DataFrame given 'name' (column name in an arrow Schema). r)rr_is_generated_index_nameintrbget_level_values)rrnkeyr2r2r3rs rcCs,zt||WStyt|YSwN)rrrtrTrr2r2r3 _level_names    rcCs.dt|jt|dt|dt|ddS)Nrangestartstopstep)kindrnrrr)rrnrget_rangeindex_attribute)rr2r2r3rs    rcs(ttdg}fddt|DS)Nrcsg|]}|qSr2)rr{rrr2r3r|z+_get_index_level_values..)rbrar)rnr2rr3rsrcsR|dur |dur td|dur|j}|S|dur$fdd|D}|Sj}|S)NzJSchema and columns arguments are mutually exclusive, pass only one of themcsg|] }|jvr|qSr2)rr{crr2r3r|sz0_resolve_columns_of_interest..)rrr)rrrr2rr3rsrc Cst|d||\}}}}}}} }g} | D]T} | j} t| r&tj| ddj} n       rTc" sDt||||\}}}} } } } } |dur.t|t|j}}||dkr,|dkr,t}nd}ts3d}fdddd}|dkrNfddt| | D}nNg}t|*}t| | D]\}}||j rm| ||q[| | ||q[Wdn1swYt |D]\}}t |tjr|||<qd d|D}|durg}t||D] \}}| t||qt|}t| ||| | |||d }|jrt|jnt}||||}d}t|d krz'| d d }|d kr| d d}| d d} | d d}!tt|| |!}Wn tyYnw|||fS)Ndrc s|dur d}d}n|j}|j}z tj||dd}Wn$tjtjtjfy>}z|jd|jd|j f7_|d}~ww|sR|j dkrRt d|d|j d|S) NT)rRrsafezConversion failed for column z with type rzField z( was non-nullable but pandas column had z null values) nullablerRrr ArrowInvalidArrowNotImplementedErrorArrowTypeErrorargsrnrQ null_countr)rrfield_nullablerrYe)rr2r3convert_columnbs,   z+dataframe_to_arrays..convert_columncSs$t|tjo|jjot|jjtjSr) r=rKndarrayflags contiguous issubclassrQrRinteger)arrr2r2r3_can_definitely_zero_copyws z6dataframe_to_arrays.._can_definitely_zero_copycsg|] \}}||qSr2r2)r{rf)rr2r3r|}sz'dataframe_to_arrays..cSsg|]}|jqSr2rR)r{xr2r2r3r|srrrrrrr)rrbrr cpu_countrrrThreadPoolExecutorrrsubmitrr=FuturerYrrrrhrr~r with_metadatarr)"rrrnthreadsrrrrrrrrrrnrowsncolsrarraysexecutorrrr maybe_futrDfieldsrnrpandas_metadatarhn_rowsrrrrr2)rrr3dataframe_to_arraysIs~                  rcCsd|jjtjkr ||fSt|r$|dur$|j}|j}t ||}||fS|dur.t |j}||fSr) rQrRrKrPr is_datetimetzrCunitr timestampfrom_numpy_dtype)rrQrrCrr2r2r3rs  rcCs$ddlmm}|dd}|d}d|vr%tjj||d|dd}n`d|vrZt|j \}} t ||d} t rItj j |d | d d }n<|}|rY|j|||j| d } | Sn+d |vr|d }t|dksjJ||d} || } t| ds}td| |}n|}|r|j||dS||fS)a Construct a pandas Block from the `item` dictionary coming from pyarrow's serialization or returned by arrow::python::ConvertTableToPandas. This function takes care of converting dictionary types to pandas categorical, Timestamp-with-timezones to the proper pandas Block, and conversion to pandas ExtensionBlock Parameters ---------- item : dict For basic types, this is a dictionary in the form of {'block': np.ndarray of values, 'placement': pandas block placement}. Additional keys are present for other types (dictionary, timezone, object). columns : Column names of the table being constructed, used for extension types extension_columns : dict Dictionary of {column_name: pandas_dtype} that includes all columns and corresponding dtypes that will be converted to a pandas ExtensionBlock. Returns ------- pandas Block rNblock placement dictionaryr^)rcr^r_rF)rQcopy)rklassrQpy_arrayr__from_arrow__zGThis column does not support to be converted to a pandas ExtensionArray)r)pandas.core.internalscore internalsgetrcategorical_type from_codesrK datetime_datarQmake_datetimetz is_ge_v21rrview make_blockDatetimeTZBlockrbrSrr )itemrextension_columns return_block_int block_arrrrrrrQrrn pandas_dtyper2r2r3_reconstruct_blocksF     r3cCs&trd}tj|}tj||dS)NnsrC)ris_v1rrstring_to_tzinfodatetimetz_type)rrCr2r2r3r(s r(FcsVg}g}|jj}|s3|dur3|d}|dg}|d}t||}t||||\}} t|||||ntj|j } t|g|||t |t |||} |j t j|||t} tryddlm} fdd| D} | | | | d}|Sdd lm}dd lm}fd d| D} | | g}|| |}tr|||j}|S||}|S) Nrrrr)create_dataframe_from_blockscsg|] }t|ddqS)F)r/r3r{r-rext_columns_dtypesr2r3r|+s z&table_to_dataframe..)rr) BlockManager) DataFramecsg|]}t|qSr2r:r;r<r2r3r|6s )rrr$_add_any_metadata_reconstruct_index_get_extension_dtypesrrrnum_rows'_check_data_column_metadata_consistency_deserialize_column_indexrrrtable_to_blocksrkeysis_ge_v3pandas.api.internalsr9r!r>pandasr?r) _from_mgraxes)optionstablercignore_metadata types_mapper all_columnsrrrrrrYr9blocksrr>r?rLmgrr2r<r3table_to_dataframe sV              rT> r r r r rrr:rrrrrrc Cs|d}|pg}i}tjdur|S|r)|jD]}|j}||} | dur(| ||j<q|jD]%}|j}|j|vrQt|tjrQz|} Wn t yKYq,w| ||j<q,|D]]} z| d} Wn t yi| d} Ynw| d} | |vr| t vrt | } t| tjrt| tj jr|s| |vrqTztj|j| jrWqTWn t yYnwt| dr| || <qTtr|s|jD],}|j|vrtj|jstj|jstj|jr|j|vrtj jtjd||j<q|S)a Based on the stored column pandas metadata and the extension types in the arrow schema, infer which columns should be converted to a pandas extension dtype. The 'numpy_type' field in the column metadata stores the string representation of the original pandas dtype (and, despite its name, not the 'pandas_type' field). Based on this string representation, a pandas/numpy dtype is constructed and then we can check if this dtype supports conversion from arrow. strings_to_categoricalNrornrqr )na_value)rextension_dtyperrRrnr=rBaseExtensionTypeto_pandas_dtypeNotImplementedErrorr<_pandas_supported_numpy_typesr2r StringDtyperD is_dictionaryrrSuses_string_dtype is_stringis_large_stringis_string_viewrKnan) rNcolumns_metadatarPrMrcrU ext_columnsrtypr2col_metarnrQr2r2r3rBMst                       rBcCstdd|Ds JdS)Ncss0|]}|ddurd|vp|dduVqdS)rnNror2rr2r2r3 s  z:_check_data_column_metadata_consistency..)all)rQr2r2r3rDsrDcs|rdd|Dfdd|jD}n|j}t|dkr2tjjjtttj |dd|Dd}ntjj ||r>|dd ndd }t|dkrMt ||}|S) NcSs&i|]}|dt|d|dqSrorn)r$rrr2r2r3 sz-_deserialize_column_index..csg|]}||qSr2r$rzcolumns_name_dictr2r3r|s z-_deserialize_column_index..rcSsg|]}|dqSrr2)r{ col_indexr2r2r3r|r}rrrnr) rrbrr MultiIndex from_tuplesrrast literal_evalIndex"_reconstruct_columns_from_metadata) block_tablerQrcolumns_valuesrr2rlr3rEs&     rEc Cs6dd|D}g}g}|}|D]O}t|tr't|||||\}} } | dur&qn-|ddkrK|d} tjj|d|d|d| d } t| t|krJqn td |d|| || qtj} t|d krt| j j ||d } || fSt|d kr|d } t| | j s| j | |d d} || fS| |j } || fS)NcSsi|] }|d|d|qSrirkrr2r2r3rjsz&_reconstruct_index..rrrnrrr)rrnzUnrecognized index kind: rrorr) r=rT_extract_index_levelrrrrbrrrp from_arraysrtrC) rNrrQrPfield_name_to_metadata index_arrays index_names result_tablerr index_namerrr2r2r3rAsL           rAc Csl||d}t||}|j|}|dkr|ddfS||}|j|d} d| _||j|}|| |fS)Nrn)rP) _backwards_compatible_index_namerget_field_indexrf to_pandasrn remove_column) rNr}rorzrP logical_namer~rrrr2r2r3rxs        rxcCs||kr t|r dS|S)a1Compute the name of an index column that is compatible with older versions of :mod:`pyarrow`. Parameters ---------- raw_name : str logical_name : str Returns ------- result : str Notes ----- * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager` N)r)raw_namerr2r2r3rsrcCsd}t||duS)Nz^__index_level_\d+__$)rematch)rnpatternr2r2r3r,src Cs2tstddddtjdtjtjtjtjd tS)NrHr`rT) rr8r7rrrIrfloatingr9r )_pandas_logical_type_maprrKrNrrobject_r2r2r2r3get_pandas_logical_type_map1s rcCs@t}z||WStyd|vrtjYSt|YSw)a Get the numpy dtype that corresponds to a pandas type. Parameters ---------- pandas_type : str The result of a call to pandas.lib.infer_dtype. Returns ------- dtype : np.dtype The dtype that corresponds to `pandas_type`. r)rr<rKrrQ)rppandas_logical_type_mapr2r2r3_pandas_type_to_numpy_typeDs    rc Cstj}t|ddp |g}t|ddpdg}ddt||idD}g}tdd}|D]\}} } t| } | tjkr?| |}nZ| d krht j |d d d } |j |d d| }trg|t| d }n1| dkrxtjdd|D}n!|jdkr| dkrd| vs| dvr||q+|j| kr|| }|j| kr| d kr|| }||q+t|dkr|j|||jdS|j|d |d j|jdS)a_Construct a pandas MultiIndex from `columns` and column index metadata in `column_indexes`. Parameters ---------- columns : List[pd.Index] The columns coming from a pyarrow.Table column_indexes : List[Dict[str, str]] The column index metadata deserialized from the JSON schema metadata in a :class:`~pyarrow.Table`. Returns ------- result : MultiIndex The index reconstructed using `column_indexes` metadata with levels of the correct type. Notes ----- * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager` rNrdcSs0g|]\}}||dt|j|ddfqS)rprqN)r$rTrQ)r{rrnr2r2r3r|xs  z6_reconstruct_columns_from_metadata..) fillvaluerrr7rrhr_T)utcr9cSsg|]}t|qSr2)r9Decimalrr2r2r3r|rrTr:r)rrIrro)rQrn)rrraroperator methodcallerrrKrNrrrr7 to_datetime tz_convertrHas_unitr'rtrQrastyperbrprrn) rrrrlabels levels_dtypes new_levelsencoderrr2 numpy_dtyperQrCr2r2r3ru[sH         rucCsi}i}|j}|d}dd|D}t|}t|d|}t|dD]q\}} | d} | sC| d} ||kr=|||} | durCd} || } | dkr| d d kr|| } t| jtjj s_q$| d } | sfq$| d }|r|| jj kr| }tj d |d}tj j||d}t|| j||| <||| <q$t|dkrg}g}tt|jD]$}||vr||||||q|||||j|qtjj|t|dS|S)NrcSsg|] }t|tr|qSr2)r=rT)r{idx_colr2r2r3r|rz%_add_any_metadata..rrornNonerrpr7rhr_r4r5rr)r)rrbrr$rr=rRrrrBrCrrArrayrrrnrrTablery)rNrmodified_columnsmodified_fieldsrrn_index_levels n_columnsrrfridxrrh metadata_tz converted tz_aware_typer rrr2r2r3r@s^       r@cCs$tj|}|jdj|}|S)zB Make a datetime64 Series timezone-aware for the given tz r)rrr7dt tz_localizer)seriesrCr2r2r3 make_tz_awares  rr)rNT)NNT)NFN)?rrcollections.abcr concurrentrconcurrent.futures.threadrrr9 itertoolsrrrrrnumpyrK ImportErrorrr pyarrow.librrrrrJrr4r@rOrZrjryrrrrrrrrrrrrrrr3r(rTr[rBrDrErArxrrrrrur@rr2r2r2r3s|      1 _'B>    # d E 9S !6 W A