
    XIg                     "   d dl mZ d dlmZmZmZmZmZmZm	Z	m
Z
mZ d dlZd dlZd dlmZmZmZ ddlmZ ddlmZmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlm Z m!Z! erd dl"m#Z$ ddl%m&Z&m'Z' ddl(m)Z) ddlmZ ddl*m+Z+m,Z,  G d d      Z#dgZ-y)    )reduce)	TYPE_CHECKINGAnyCallableListOptionalTupleUnioncastoverloadN)ColumnExpression
ExpressionStarExpression   )ColumnOrName   )PySparkTypeErrorPySparkValueErrorPySparkIndexErrorContributionsAcceptedError)ColumnDataFrameWriter)duckdb_to_spark_schema)Row
StructType)	DataFrameGroupedDataGrouping)SparkSession)r   )_to_column_exprcolc                   N   e Zd Zdej                  ddfdZd@dZdAdZd	eddfd
Z	d	eddfdZ
dededd fdZdededd fdZded   dededd fdZdeeeeeeef      f   dedd fdZeZdBdee   deee   ee   f   fdZeZdedee   fdZdCdZeZdDdZedee   fd       Z	 	 dEdd d eeeee   eee   f      d!ee   dd fd"Z d#edd fd$Z!dFd%Z"defd&Z#dedd fd'Z$d(efd)Z%ede&fd*       Z'e(d(eeef   defd+       Z)e(d(eeee*f   dd fd,       Z)d(eeeeee*f   deed f   fd-Z)d	edefd.Z+e(dGd0       Z,e(d1eee   ee   f   dd/fd2       Z,dGd3Z,ede-fd4       Z.d5 Z/dHd6Z0e0Z1	 dIdd d7e2dd fd8Z3dBd9eee      dd fd:Z4dDd;Z5defd<Z6dDd=Z7dDd>Z8dee   fd?Z9y)Jr   relationsessionr"   c                     || _         || _        d | _        | j                   :t        | j                   j                  | j                   j
                        | _        y y N)r&   r'   _schemar   columnstypes)selfr&   r'   s      d/var/www/html/answerous/venv/lib/python3.12/site-packages/duckdb/experimental/spark/sql/dataframe.py__init__zDataFrame.__init__%   sH     ==$1$--2G2GI\I\]DL %    returnNc                 8    | j                   j                          y r)   )r&   show)r-   kwargss     r.   r3   zDataFrame.show,   s    r0   c                 6    | j                   j                         S r)   )r&   dfr-   s    r.   toPandaszDataFrame.toPandas/   s    }}!!r0   namec                 <    | j                   j                  |d       y)a\  Creates or replaces a local temporary view with this :class:`DataFrame`.

        The lifetime of this temporary table is tied to the :class:`SparkSession`
        that was used to create this :class:`DataFrame`.

        Parameters
        ----------
        name : str
            Name of the view.

        Examples
        --------
        Create a local temporary view named 'people'.

        >>> df = spark.createDataFrame([(2, "Alice"), (5, "Bob")], schema=["age", "name"])
        >>> df.createOrReplaceTempView("people")

        Replace the local temporary view.

        >>> df2 = df.filter(df.age > 3)
        >>> df2.createOrReplaceTempView("people")
        >>> df3 = spark.sql("SELECT * FROM people")
        >>> sorted(df3.collect()) == sorted(df2.collect())
        True
        >>> spark.catalog.dropTempView("people")
        True

        TN)r&   create_viewr-   r9   s     r.   createOrReplaceTempViewz!DataFrame.createOrReplaceTempView2   s    : 	!!$-r0   c                     t         r)   )NotImplementedErrorr<   s     r.   createGlobalTempViewzDataFrame.createGlobalTempViewQ   s    !!r0   
columnNamenewNamec                 p   || j                   vrt        d|       g }| j                   j                  D ]Q  }t        |      }|j	                         |j	                         k(  r |j
                  |      }|j                  |       S  | j                   j                  | }t        || j                        S )Nz*DataFrame does not contain a column named )
r&   
ValueErrorr+   r   casefoldaliasappendselectr   r'   )r-   rA   rB   colsxr$   rels          r.   withColumnRenamedzDataFrame.withColumnRenamedT   s    T]]*I*VWW&& 	A"1%Czz|z2244cii(KK		
 #dmm""D)dll++r0   r$   c                    t        |t              s#t        ddt        |      j                  d      || j
                  v rg }| j
                  j                  D ]h  }|j                         |j                         k(  r+|j                  |j                  j                  |             O|j                  t        |             j nV| j
                  j                  D cg c]  }t        |       }}|j                  |j                  j                  |              | j
                  j                  | }t        || j                        S c c}w )N
NOT_COLUMNr$   arg_namearg_typeerror_classmessage_parameters)
isinstancer   r   type__name__r&   r+   rE   rG   exprrF   r   rH   r   r'   )r-   rA   r$   rI   rJ   rK   s         r.   
withColumnzDataFrame.withColumn`   s   #v&"(0549CUCU#V  &D]]** 5::<:#6#6#88KKz :;KK 0 34	5 261F1FGA$Q'GDGKKz23"dmm""D)dll++ Hs   Efunc).r   argsr4   c                 d     || g|i |}t        |t              sJ dt        |      z         |S )af  Returns a new :class:`DataFrame`. Concise syntax for chaining custom transformations.

        .. versionadded:: 3.0.0

        .. versionchanged:: 3.4.0
            Supports Spark Connect.

        Parameters
        ----------
        func : function
            a function that takes and returns a :class:`DataFrame`.
        *args
            Positional arguments to pass to func.

            .. versionadded:: 3.3.0
        **kwargs
            Keyword arguments to pass to func.

            .. versionadded:: 3.3.0

        Returns
        -------
        :class:`DataFrame`
            Transformed DataFrame.

        Examples
        --------
        >>> from pyspark.sql.functions import col
        >>> df = spark.createDataFrame([(1, 1.0), (2, 2.0)], ["int", "float"])
        >>> def cast_all_to_int(input_df):
        ...     return input_df.select([col(col_name).cast("int") for col_name in input_df.columns])
        ...
        >>> def sort_columns_asc(input_df):
        ...     return input_df.select(*sorted(input_df.columns))
        ...
        >>> df.transform(cast_all_to_int).transform(sort_columns_asc).show()
        +-----+---+
        |float|int|
        +-----+---+
        |    1|  1|
        |    2|  2|
        +-----+---+

        >>> def add_n(input_df, n):
        ...     return input_df.select([(col(col_name) + n).alias(col_name)
        ...                             for col_name in input_df.columns])
        >>> df.transform(add_n, 1).transform(add_n, n=10).show()
        +---+-----+
        |int|float|
        +---+-----+
        | 12| 12.0|
        | 13| 13.0|
        +---+-----+
        zCFunc returned an instance of type [%s], should have been DataFrame.)rU   r   rV   )r-   rZ   r[   r4   results        r.   	transformzDataFrame.transformt   sH    r d,T,V,&), 	
*,0L9	
, r0   rI   c                    |st        dddi      t        |      dk(  rt        |d   t              r|d   }g }|D ]  }|}t        |t              rt        |      }nXt        |t              rHt        |t              s8|dkD  r	| |dz
     }n*|dk  r| | dz
     j                         }nt        di       |j                  |        |j                  dd	      }t        |t        t        f      r |s|D cg c]  }|j                          }}ndt        |t              r1t        ||      D cg c]  \  }}|r|n|j                          }}}n#t        d
dt        |      j                  d      |D cg c]  }t!        |       }} | j"                  j$                  | }t'        || j(                        S c c}w c c}}w c c}w )a  Returns a new :class:`DataFrame` sorted by the specified column(s).

        Parameters
        ----------
        cols : str, list, or :class:`Column`, optional
             list of :class:`Column` or column names to sort by.

        Other Parameters
        ----------------
        ascending : bool or list, optional, default True
            boolean or list of boolean.
            Sort ascending vs. descending. Specify list for multiple sort orders.
            If a list is specified, the length of the list must equal the length of the `cols`.

        Returns
        -------
        :class:`DataFrame`
            Sorted DataFrame.

        Examples
        --------
        >>> from pyspark.sql.functions import desc, asc
        >>> df = spark.createDataFrame([
        ...     (2, "Alice"), (5, "Bob")], schema=["age", "name"])

        Sort the DataFrame in ascending order.

        >>> df.sort(asc("age")).show()
        +---+-----+
        |age| name|
        +---+-----+
        |  2|Alice|
        |  5|  Bob|
        +---+-----+

        Sort the DataFrame in descending order.

        >>> df.sort(df.age.desc()).show()
        +---+-----+
        |age| name|
        +---+-----+
        |  5|  Bob|
        |  2|Alice|
        +---+-----+
        >>> df.orderBy(df.age.desc()).show()
        +---+-----+
        |age| name|
        +---+-----+
        |  5|  Bob|
        |  2|Alice|
        +---+-----+
        >>> df.sort("age", ascending=False).show()
        +---+-----+
        |age| name|
        +---+-----+
        |  5|  Bob|
        |  2|Alice|
        +---+-----+

        Specify multiple columns

        >>> df = spark.createDataFrame([
        ...     (2, "Alice"), (2, "Bob"), (5, "Bob")], schema=["age", "name"])
        >>> df.orderBy(desc("age"), "name").show()
        +---+-----+
        |age| name|
        +---+-----+
        |  5|  Bob|
        |  2|Alice|
        |  2|  Bob|
        +---+-----+

        Specify multiple columns for sorting order at `ascending`.

        >>> df.orderBy(["age", "name"], ascending=[False, False]).show()
        +---+-----+
        |age| name|
        +---+-----+
        |  5|  Bob|
        |  2|  Bob|
        |  2|Alice|
        +---+-----+
        CANNOT_BE_EMPTYitemcolumnrR   r   r   
ZERO_INDEX	ascendingTNOT_BOOL_OR_LISTrO   )r   lenrU   liststrr$   intbooldescr   rG   getzipr   rV   rW   r#   r&   sortr   r'   )	r-   rI   r4   r+   c_crd   ascrK   s	            r.   rn   zDataFrame.sort   s   l #-$*H#5  t9>ja$77D 	AB!S!VAs#Jq$,?q5a!eBUqb1f**,B+$0+-  NN2!	$ JJ{D1	i$--45166855	4(<?	7<ST&#qCqQVVX-TGT".0;iIaIa#b 
 077!?1%77 dmm  '*dll++ 6T 8s   6F;/G 8Gnc                 \    || j                  d      }|r|d   S d S | j                  |      S Nr   r   )headtake)r-   rr   rss      r.   ru   zDataFrame.head8  s3    91B2a5(D(yy|r0   numc                 @    | j                  |      j                         S r)   )limitcollect)r-   rx   s     r.   rv   zDataFrame.take@  s    zz#&&((r0   c                 
   t        |t              r|j                  }n6t        |t              r|}n#t	        ddt        |      j                  d      | j                  j                  |      }t        || j                        S )a  Filters rows using the given condition.

        :func:`where` is an alias for :func:`filter`.

        Parameters
        ----------
        condition : :class:`Column` or str
            a :class:`Column` of :class:`types.BooleanType`
            or a string of SQL expressions.

        Returns
        -------
        :class:`DataFrame`
            Filtered DataFrame.

        Examples
        --------
        >>> df = spark.createDataFrame([
        ...     (2, "Alice"), (5, "Bob")], schema=["age", "name"])

        Filter by :class:`Column` instances.

        >>> df.filter(df.age > 3).show()
        +---+----+
        |age|name|
        +---+----+
        |  5| Bob|
        +---+----+
        >>> df.where(df.age == 2).show()
        +---+-----+
        |age| name|
        +---+-----+
        |  2|Alice|
        +---+-----+

        Filter by SQL expression in a string.

        >>> df.filter("age > 3").show()
        +---+----+
        |age|name|
        +---+----+
        |  5| Bob|
        +---+----+
        >>> df.where("age = 2").show()
        +---+-----+
        |age| name|
        +---+-----+
        |  2|Alice|
        +---+-----+
        NOT_COLUMN_OR_STR	conditionrO   rR   )rU   r   rX   rh   r   rV   rW   r&   filterr   r'   )r-   r~   condrK   s       r.   r   zDataFrame.filterC  sp    f i(>>D	3'D"/0;iIaIa#b  mm""4(dll++r0   c                    t        |      }t        |      dk(  r|d   }t        |t               r5|D cg c])  }t        |t              r|j                  n
t        |      + }}n(t        |t              r|j                  n
t        |      g} | j                  j                  | }t        || j                        S c c}w rt   )
rg   rf   rU   r   rX   r   r&   rH   r   r'   )r-   rI   rJ   projectionsrK   s        r.   rH   zDataFrame.select  s    Dzt9>7DdD!RVMN*Q/5Ea5HHK 
 (f5		;KD;QK #dmm""K0dll++s   .B;c                 h    | j                   j                  D cg c]  }|j                   c}S c c}w )z~Returns all column names as a list.

        Examples
        --------
        >>> df.columns
        ['age', 'name']
        )schemafieldsr9   )r-   fs     r.   r+   zDataFrame.columns  s&     !% 2 231333s   /otheronhowc                 @   |t        |t              s|g}|ft        |t              sJ |D cg c]  }t        |       }}t        |d   t              sJ d       t	        d t        t        t           |            }|(|&| j                  j                  |j                        }n]|d}|d}nt        |      }t        |t              sJ d       d } ||      }| j                  j                  |j                  ||      }t        || j                        S c c}w )a  Joins with another :class:`DataFrame`, using the given join expression.

        Parameters
        ----------
        other : :class:`DataFrame`
            Right side of the join
        on : str, list or :class:`Column`, optional
            a string for the join column name, a list of column names,
            a join expression (Column), or a list of Columns.
            If `on` is a string or a list of strings indicating the name of the join column(s),
            the column(s) must exist on both sides, and this performs an equi-join.
        how : str, optional
            default ``inner``. Must be one of: ``inner``, ``cross``, ``outer``,
            ``full``, ``fullouter``, ``full_outer``, ``left``, ``leftouter``, ``left_outer``,
            ``right``, ``rightouter``, ``right_outer``, ``semi``, ``leftsemi``, ``left_semi``,
            ``anti``, ``leftanti`` and ``left_anti``.

        Returns
        -------
        :class:`DataFrame`
            Joined DataFrame.

        Examples
        --------
        The following performs a full outer join between ``df1`` and ``df2``.

        >>> from pyspark.sql import Row
        >>> from pyspark.sql.functions import desc
        >>> df = spark.createDataFrame([(2, "Alice"), (5, "Bob")]).toDF("age", "name")
        >>> df2 = spark.createDataFrame([Row(height=80, name="Tom"), Row(height=85, name="Bob")])
        >>> df3 = spark.createDataFrame([Row(age=2, name="Alice"), Row(age=5, name="Bob")])
        >>> df4 = spark.createDataFrame([
        ...     Row(age=10, height=80, name="Alice"),
        ...     Row(age=5, height=None, name="Bob"),
        ...     Row(age=None, height=None, name="Tom"),
        ...     Row(age=None, height=None, name=None),
        ... ])

        Inner join on columns (default)

        >>> df.join(df2, 'name').select(df.name, df2.height).show()
        +----+------+
        |name|height|
        +----+------+
        | Bob|    85|
        +----+------+
        >>> df.join(df4, ['name', 'age']).select(df.name, df.age).show()
        +----+---+
        |name|age|
        +----+---+
        | Bob|  5|
        +----+---+

        Outer join for both DataFrames on the 'name' column.

        >>> df.join(df2, df.name == df2.name, 'outer').select(
        ...     df.name, df2.height).sort(desc("name")).show()
        +-----+------+
        | name|height|
        +-----+------+
        |  Bob|    85|
        |Alice|  NULL|
        | NULL|    80|
        +-----+------+
        >>> df.join(df2, 'name', 'outer').select('name', 'height').sort(desc("name")).show()
        +-----+------+
        | name|height|
        +-----+------+
        |  Tom|    80|
        |  Bob|    85|
        |Alice|  NULL|
        +-----+------+

        Outer join for both DataFrams with multiple columns.

        >>> df.join(
        ...     df3,
        ...     [df.name == df3.name, df.age == df3.age],
        ...     'outer'
        ... ).select(df.name, df3.age).show()
        +-----+---+
        | name|age|
        +-----+---+
        |Alice|  2|
        |  Bob|  5|
        +-----+---+
        r   z%on should be Column or list of Columnc                 $    | j                  |      S r)   )__and__)rJ   ys     r.   <lambda>z DataFrame.join.<locals>.<lambda>  s    QYYq\ r0   innertruezhow should be a stringc                     g g dddgddgddgdd	gd
}d }|j                         D ]  \  }}| |k(  s| |v s|} n |s| }|S )N)full	fullouter
full_outer	leftouter
left_outer
rightouterright_outerleftanti	left_antileftsemi	left_semi)r   outerleftrightantisemi)items)r   known_aliasesmapped_typerV   aliasess        r.   map_to_recognized_jointypez2DataFrame.join.<locals>.map_to_recognized_jointype  sw    @(,7*M:'5'5! #%2%8%8%: MD'd{cWn&*
 #"%K""r0   )rU   rg   r#   r   r   r   r   r&   joinrh   r   r'   )r-   r   r   r   rJ   r]   r   s          r.   r   zDataFrame.join  s   | >*R"6B>b$'''.01/!$1B1 1z 767  14Z8H"3MNB:#+]]''7F{zWc3'A)AA'#& -S1C]]''C@F..Q 2s   DrF   c                     t        |t              sJ d       t        | j                  j	                  |      | j
                        S )a  Returns a new :class:`DataFrame` with an alias set.

        Parameters
        ----------
        alias : str
            an alias name to be set for the :class:`DataFrame`.

        Returns
        -------
        :class:`DataFrame`
            Aliased DataFrame.

        Examples
        --------
        >>> from pyspark.sql.functions import col, desc
        >>> df = spark.createDataFrame(
        ...     [(14, "Tom"), (23, "Alice"), (16, "Bob")], ["age", "name"])
        >>> df_as1 = df.alias("df_as1")
        >>> df_as2 = df.alias("df_as2")
        >>> joined_df = df_as1.join(df_as2, col("df_as1.name") == col("df_as2.name"), 'inner')
        >>> joined_df.select(
        ...     "df_as1.name", "df_as2.name", "df_as2.age").sort(desc("df_as1.name")).show()
        +-----+-----+---+
        | name| name|age|
        +-----+-----+---+
        |  Tom|  Tom| 14|
        |  Bob|  Bob| 16|
        |Alice|Alice| 23|
        +-----+-----+---+
        zalias should be a string)rU   rh   r   r&   	set_aliasr'   )r-   rF   s     r.   rF   zDataFrame.alias,  s:    > %%A'AA%007FFr0   c                    t        |      dk(  rB|d   }t        |t              r|g}nVt        |t              r|j                  g}n8t        d      |D ]  }t        |t              rt        d       t        |      }|D cg c]  }|| j                  j                  v s| }}t        |      }t        | j                  j                  |      | j                        S c c}w )Nr   r   z"col should be a string or a Columnz-each col in the param list should be a string)exclude)rf   rU   rh   r   rX   	TypeErrorrg   r&   r+   r   r   rH   r'   )r-   rI   r$   r   rJ   rX   s         r.   dropzDataFrame.dropN  s    t9>q'C#s#%C(88* DEE U!#s+#$STTU 4jG%Ddmm.C.C)C1DDg.--d3T\\BB Es   C! C!c                 ,    t        | j                        S r)   )rh   r&   r7   s    r.   __repr__zDataFrame.__repr__a  s    4==!!r0   c                 d    | j                   j                  |      }t        || j                        S )a  Limits the result count to the number specified.

        Parameters
        ----------
        num : int
            Number of records to return. Will return this number of records
            or all records if the DataFrame contains less than this number of records.

        Returns
        -------
        :class:`DataFrame`
            Subset of the records

        Examples
        --------
        >>> df = spark.createDataFrame(
        ...     [(14, "Tom"), (23, "Alice"), (16, "Bob")], ["age", "name"])
        >>> df.limit(1).show()
        +---+----+
        |age|name|
        +---+----+
        | 14| Tom|
        +---+----+
        >>> df.limit(0).show()
        +---+----+
        |age|name|
        +---+----+
        +---+----+
        )r&   rz   r   r'   )r-   rx   rK   s      r.   rz   zDataFrame.limitd  s)    < mm!!#&dll++r0   ra   c                     || j                   v S )zY
        Check if the :class:`DataFrame` contains a column by the name of `item`
        )r&   r-   ra   s     r.   __contains__zDataFrame.__contains__  s     t}}$$r0   c                     | j                   S )a(  Returns the schema of this :class:`DataFrame` as a :class:`duckdb.experimental.spark.sql.types.StructType`.

        Examples
        --------
        >>> df.schema
        StructType([StructField('age', IntegerType(), True),
                    StructField('name', StringType(), True)])
        )r*   r7   s    r.   r   zDataFrame.schema  s     ||r0   c                      y r)    r   s     r.   __getitem__zDataFrame.__getitem__      r0   c                      y r)   r   r   s     r.   r   zDataFrame.__getitem__  r   r0   c                 V   t        |t              rt        |      S t        |t              r| j	                  |      S t        |t
        t        f      r | j                  | S t        |t              r"t        | j                  |   j                        S t        dt        |             )a  Returns the column as a :class:`Column`.

        Examples
        --------
        >>> df.select(df['age']).collect()
        [Row(age=2), Row(age=5)]
        >>> df[ ["name", "age"]].collect()
        [Row(name='Alice', age=2), Row(name='Bob', age=5)]
        >>> df[ df.age > 3 ].collect()
        [Row(age=5, name='Bob')]
        >>> df[df[0] > 3].collect()
        [Row(age=5, name='Bob')]
        zUnexpected item type: )rU   rh   r$   r   r   rg   tuplerH   ri   r*   r9   r   rV   r   s     r.   r   zDataFrame.__getitem__  s      dC t9f%;;t$$tUm,4;;%%c"t||D)..//4T$ZLABBr0   c                     || j                   j                  vr&t        d| j                  j                  d|d      t        |      S )zReturns the :class:`Column` denoted by ``name``.

        Examples
        --------
        >>> df.select(df.age).collect()
        [Row(age=2), Row(age=5)]
        'z' object has no attribute ')r&   r+   AttributeError	__class__rW   r$   r<   s     r.   __getattr__zDataFrame.__getattr__  sB     t}},,, 7;~~7N7NPTU  4yr0   r    c                      y r)   r   )r-   rI   s     r.   groupByzDataFrame.groupBy  r   r0   _DataFrame__colsc                      y r)   r   )r-   r   s     r.   r   zDataFrame.groupBy  r   r0   c                 |    ddl m}m} t        |      dk(  rt	        |d   t
              r|d   }n|} | || |       S )aM  Groups the :class:`DataFrame` using the specified columns,
        so we can run aggregation on them. See :class:`GroupedData`
        for all the available aggregate functions.

        :func:`groupby` is an alias for :func:`groupBy`.

        Parameters
        ----------
        cols : list, str or :class:`Column`
            columns to group by.
            Each element should be a column name (string) or an expression (:class:`Column`)
            or list of them.

        Returns
        -------
        :class:`GroupedData`
            Grouped data by given columns.

        Examples
        --------
        >>> df = spark.createDataFrame([
        ...     (2, "Alice"), (2, "Bob"), (2, "Bob"), (5, "Bob")], schema=["age", "name"])

        Empty grouping columns triggers a global aggregation.

        >>> df.groupBy().avg().show()
        +--------+
        |avg(age)|
        +--------+
        |    2.75|
        +--------+

        Group-by 'name', and specify a dictionary to calculate the summation of 'age'.

        >>> df.groupBy("name").agg({"age": "sum"}).sort("name").show()
        +-----+--------+
        | name|sum(age)|
        +-----+--------+
        |Alice|       2|
        |  Bob|       9|
        +-----+--------+

        Group-by 'name', and calculate maximum values.

        >>> df.groupBy(df.name).max().sort("name").show()
        +-----+--------+
        | name|max(age)|
        +-----+--------+
        |Alice|       2|
        |  Bob|       5|
        +-----+--------+

        Group-by 'name' and 'age', and calculate the number of rows in each group.

        >>> df.groupBy(["name", df.age]).count().sort("name", "age").show()
        +-----+---+-----+
        | name|age|count|
        +-----+---+-----+
        |Alice|  2|    1|
        |  Bob|  2|    2|
        |  Bob|  5|    1|
        +-----+---+-----+
        r   r   r   )groupr    r!   rf   rU   rg   )r-   rI   r    r!   r+   s        r.   r   zDataFrame.groupBy  s@    @ 	1t9>ja$71gGG8W-t44r0   c                     t        |       S r)   r   r7   s    r.   writezDataFrame.write  s    t$$r0   c                     t         r)   r   r7   s    r.   printSchemazDataFrame.printSchema  s    ((r0   c                 t    t        | j                  j                  |j                        | j                        S )a  Return a new :class:`DataFrame` containing union of rows in this and another
        :class:`DataFrame`.

        Parameters
        ----------
        other : :class:`DataFrame`
            Another :class:`DataFrame` that needs to be unioned

        Returns
        -------
        :class:`DataFrame`

        See Also
        --------
        DataFrame.unionAll

        Notes
        -----
        This is equivalent to `UNION ALL` in SQL. To do a SQL-style set union
        (that does deduplication of elements), use this function followed by :func:`distinct`.

        Also as standard in SQL, this function resolves columns by position (not by name).

        Examples
        --------
        >>> df1 = spark.createDataFrame([[1, 2, 3]], ["col0", "col1", "col2"])
        >>> df2 = spark.createDataFrame([[4, 5, 6]], ["col1", "col2", "col0"])
        >>> df1.union(df2).show()
        +----+----+----+
        |col0|col1|col2|
        +----+----+----+
        |   1|   2|   3|
        |   4|   5|   6|
        +----+----+----+
        >>> df1.union(df1).show()
        +----+----+----+
        |col0|col1|col2|
        +----+----+----+
        |   1|   2|   3|
        |   1|   2|   3|
        +----+----+----+
        )r   r&   unionr'   )r-   r   s     r.   r   zDataFrame.union  s)    V ,,U^^<dllKKr0   allowMissingColumnsc                     |st         t        )a  Returns a new :class:`DataFrame` containing union of rows in this and another
        :class:`DataFrame`.

        This is different from both `UNION ALL` and `UNION DISTINCT` in SQL. To do a SQL-style set
        union (that does deduplication of elements), use this function followed by :func:`distinct`.

        .. versionadded:: 2.3.0

        .. versionchanged:: 3.4.0
            Supports Spark Connect.

        Parameters
        ----------
        other : :class:`DataFrame`
            Another :class:`DataFrame` that needs to be combined.
        allowMissingColumns : bool, optional, default False
           Specify whether to allow missing columns.

           .. versionadded:: 3.1.0

        Returns
        -------
        :class:`DataFrame`
            Combined DataFrame.

        Examples
        --------
        The difference between this function and :func:`union` is that this function
        resolves columns by name (not by position):

        >>> df1 = spark.createDataFrame([[1, 2, 3]], ["col0", "col1", "col2"])
        >>> df2 = spark.createDataFrame([[4, 5, 6]], ["col1", "col2", "col0"])
        >>> df1.unionByName(df2).show()
        +----+----+----+
        |col0|col1|col2|
        +----+----+----+
        |   1|   2|   3|
        |   6|   4|   5|
        +----+----+----+

        When the parameter `allowMissingColumns` is ``True``, the set of column names
        in this and other :class:`DataFrame` can differ; missing columns will be filled with null.
        Further, the missing columns of this :class:`DataFrame` will be added at the end
        in the schema of the union result:

        >>> df1 = spark.createDataFrame([[1, 2, 3]], ["col0", "col1", "col2"])
        >>> df2 = spark.createDataFrame([[4, 5, 6]], ["col1", "col2", "col3"])
        >>> df1.unionByName(df2, allowMissingColumns=True).show()
        +----+----+----+----+
        |col0|col1|col2|col3|
        +----+----+----+----+
        |   1|   2|   3|NULL|
        |NULL|   4|   5|   6|
        +----+----+----+----+
        )r   r?   )r-   r   r   s      r.   unionByNamezDataFrame.unionByNameN  s    t #,,!!r0   subsetc                 n   |rdt        j                         j                   }dj                  |D cg c]  }d| d
 c}      }d| d| }t	        | j
                  j                  |d      | j                        }|j                  | d      j                  |      S | j                         S c c}w )a  Return a new :class:`DataFrame` with duplicate rows removed,
        optionally only considering certain columns.

        For a static batch :class:`DataFrame`, it just drops duplicate rows. For a streaming
        :class:`DataFrame`, it will keep all data across triggers as intermediate state to drop
        duplicates rows. You can use :func:`withWatermark` to limit how late the duplicate data can
        be and the system will accordingly limit the state. In addition, data older than
        watermark will be dropped to avoid any possibility of duplicates.

        :func:`drop_duplicates` is an alias for :func:`dropDuplicates`.

        Parameters
        ----------
        subset : List of column names, optional
            List of columns to use for duplicate comparison (default All columns).

        Returns
        -------
        :class:`DataFrame`
            DataFrame without duplicates.

        Examples
        --------
        >>> from pyspark.sql import Row
        >>> df = spark.createDataFrame([
        ...     Row(name='Alice', age=5, height=80),
        ...     Row(name='Alice', age=5, height=80),
        ...     Row(name='Alice', age=10, height=80)
        ... ])

        Deduplicate the same rows.

        >>> df.dropDuplicates().show()
        +-----+---+------+
        | name|age|height|
        +-----+---+------+
        |Alice|  5|    80|
        |Alice| 10|    80|
        +-----+---+------+

        Deduplicate values on 'name' and 'height' columns.

        >>> df.dropDuplicates(['name', 'height']).show()
        +-----+---+------+
        | name|age|height|
        +-----+---+------+
        |Alice|  5|    80|
        +-----+---+------+
        tmp_col_, "zOVER(PARTITION BY z) AS *z = 1)uuiduuid1hexr   r   r&   
row_numberr'   r   r   distinct)r-   r   rn_colro   
subset_strwindow_specr6   s          r.   dropDuplicateszDataFrame.dropDuplicates  s    d 

 0 012Ff#=as!H#=>J.zl%xHK4==33KEt||TB99xt_-226::}} $>s   B2c                 b    | j                   j                         }t        || j                        S )a  Returns a new :class:`DataFrame` containing the distinct rows in this :class:`DataFrame`.

        Returns
        -------
        :class:`DataFrame`
            DataFrame with distinct records.

        Examples
        --------
        >>> df = spark.createDataFrame(
        ...     [(14, "Tom"), (23, "Alice"), (23, "Alice")], ["age", "name"])

        Return the number of distinct rows in the :class:`DataFrame`

        >>> df.distinct().count()
        2
        )r&   r   r   r'   )r-   distinct_rels     r.   r   zDataFrame.distinct  s'    $ }}--/t||44r0   c                 p    | j                   j                  d      }t        |j                         d         S )a  Returns the number of rows in this :class:`DataFrame`.

        Returns
        -------
        int
            Number of rows.

        Examples
        --------
        >>> df = spark.createDataFrame(
        ...     [(14, "Tom"), (23, "Alice"), (16, "Bob")], ["age", "name"])

        Return the number of rows in the :class:`DataFrame`.

        >>> df.count()
        3
        r   r   )r&   countri   fetchone)r-   	count_rels     r.   r   zDataFrame.count  s1    $ MM'',	9%%'*++r0   c           	      @   | j                   j                  }t        |      }|t        |      k(  sJ t        ||      D cg c]  \  }}| d| d|  }}}dj	                  |      }| j                   j                  |      }t        || j                        S c c}}w )Nz::z as r   )r&   r+   rf   rm   r   projectr   r'   )r-   r,   existing_columnstypes_countexistingtarget_typecast_expressionsnew_rels           r.   _cast_typeszDataFrame._cast_types  s    ==00%jc"23333 *--=u)E
%+ j;-tH:6
 
  99%56--''(89$,,//
s   Bc                 v   | j                   j                  }t        |      }|t        |      k7  rt        d      |D cg c]  }t	        |       }}t        ||      D cg c]  \  }}|j                  |       }}} | j                   j                  | }t        || j                        S c c}w c c}}w )NzHProvided column names and number of columns in the DataFrame don't match)message)
r&   r+   rf   r   r   rm   rF   r   r   r'   )	r-   rI   r   column_countrJ   r   newr   r   s	            r.   toDFzDataFrame.toDF  s    ==004y3/00#b  :JJA,Q/JJ589I45P
$1HcHNN3
 
 ($--''5$,,// K
s    B0#B5c                     | j                   j                  }| j                   j                         }dt        fd}|D cg c]  } |||       }}|S c c}w )Nr1   c                 l    t         j                  t        t        |             }t        |      |_        |S r)   )r   __new__r   rg   
__fields__)valuesnamesrows      r.   construct_rowz(DataFrame.collect.<locals>.construct_row  s&    --T&\2C!%[CNJr0   )r&   r+   fetchallr   )r-   r+   r]   r   rJ   rowss         r.   r{   zDataFrame.collect  sU    --'''')	C 	
 4::aa):: ;s   A)r1   N)r1   PandasDataFramer)   )r~   r   r1   r   )r1   r   )NN)rI   r   r1   r   )rI   r   r1   r    )r   r   r1   r   )F):rW   
__module____qualname__duckdbDuckDBPyRelationr/   r3   r8   rh   r=   r@   rL   r   rY   r   r   r^   r
   r   rn   orderByr   ri   r   ru   firstrv   r   whererH   propertyr+   r   rF   r   r   rz   r   r   r   r   r   r	   r   r   r   r   r   r   unionAllrj   r   r   r   r   r   r   r{   r   r0   r.   r   r   $   s   ^!8!8 ^> ^".C .D .>" " "
,C 
,# 
,+ 
,,S ,v ,+ ,(>-.>7:>FI>	>@@,3U3;-?(@@A@,MP@,	@,D Ghsm uXc]DI5M/N  E) )S	 )=,~ E, 4c 4 4 EI!	L/L/ U3S	64<?@AL/ c]	L/
 
L/\ G3  G;  GDC&"# ", , ,B% % 	
 	 	 c3h F   fdE&9 : {  C#sFD%78C	v{"	#C6     eDL$s)$;<   F5P % % %)+LZ H ?D<" <"7;<"	<"@9Xd3i%8 9K 9x5*,s ,*00
c 
r0   r   ).	functoolsr   typingr   r   r   r   r   r	   r
   r   r   r   r   r   r   r   _typingr   errorsr   r   r   	exceptionr   rb   r   
readwriterr   
type_utilsr   r,   r   r   pandas.core.framer   r   r   r    r!   r'   r"   	functionsr#   r$   __all__r   r0   r.   <module>r     se    
 
 
   ? ? ! K K 2  ' . ">,% & +v vr -r0   