-
Notifications
You must be signed in to change notification settings - Fork 151
Expand file tree
/
Copy pathdataframe.py
More file actions
6451 lines (5305 loc) · 231 KB
/
Copy pathdataframe.py
File metadata and controls
6451 lines (5305 loc) · 231 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#
# Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
#
"""This module contains DataFrame docstrings that override modin's docstrings."""
from textwrap import dedent
from pandas.util._decorators import doc
from snowflake.snowpark.modin.plugin.docstrings.shared_docs import (
_doc_binary_op,
_shared_docs,
)
from .base import BasePandasDataset
_doc_binary_op_kwargs = {"returns": "BasePandasDataset", "left": "BasePandasDataset"}
_shared_doc_kwargs = {
"axes": "index, columns",
"klass": "DataFrame",
"axes_single_arg": "{0 or 'index', 1 or 'columns'}",
"axis": """axis : {0 or 'index', 1 or 'columns'}, default 0
If 0 or 'index': apply function to each column.
If 1 or 'columns': apply function to each row.""",
"inplace": """
inplace : bool, default False
Whether to modify the DataFrame rather than creating a new one.""",
"optional_by": """
by : str or list of str
Name or list of names to sort by.
- if `axis` is 0 or `'index'` then `by` may contain index
levels and/or column labels.
- if `axis` is 1 or `'columns'` then `by` may contain column
levels and/or index labels.""",
"optional_reindex": """
labels : array-like, optional
New labels / index to conform the axis specified by 'axis' to.
index : array-like, optional
New labels for the index. Preferably an Index object to avoid
duplicating data.
columns : array-like, optional
New labels for the columns. Preferably an Index object to avoid
duplicating data.
axis : int or str, optional
Axis to target. Can be either the axis name ('index', 'columns')
or number (0, 1).""",
}
class DataFrame(BasePandasDataset):
"""
Snowpark pandas representation of ``pandas.DataFrame`` with a lazily-evaluated relational dataset.
A DataFrame is considered lazy because it encapsulates the computation or query required to produce
the final dataset. The computation is not performed until the datasets need to be displayed, or I/O
methods like to_pandas, to_snowflake are called.
Internally, the underlying data are stored as Snowflake table with rows and columns.
Parameters
----------
data : DataFrame, Series, `pandas.DataFrame <https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html>`_, ndarray, Iterable or dict, optional
Dict can contain ``Series``, arrays, constants, dataclass or list-like objects.
If data is a dict, column order follows insertion-order.
index : Index or array-like, optional
Index to use for resulting frame. Will default to ``RangeIndex`` if no
indexing information part of input data and no index provided.
columns : Index or array-like, optional
Column labels to use for resulting frame. Will default to
``RangeIndex`` if no column labels are provided.
dtype : str, np.dtype, or pandas.ExtensionDtype, optional
Data type to force. Only a single dtype is allowed. If None, infer.
copy : bool, default: False
Copy data from inputs. Only affects ``pandas.DataFrame`` / 2d ndarray input.
query_compiler : BaseQueryCompiler, optional
A query compiler object to create the ``DataFrame`` from.
Notes
-----
``DataFrame`` can be created either from passed `data` or `query_compiler`. If both
parameters are provided, an assertion error will be raised. `query_compiler` can only
be specified when the `data`, `index`, and `columns` are None.
Using pandas/NumPy/Python data structures as the `data` parameter is less desirable since
importing such data structures is very inefficient.
Please use previously created Modin structures or import data using highly efficient Modin IO
tools (for example ``pd.read_csv``).
Examples
--------
Creating a Snowpark pandas DataFrame from a dictionary:
>>> d = {'col1': [1, 2], 'col2': [3, 4]}
>>> df = pd.DataFrame(data=d)
>>> df
col1 col2
0 1 3
1 2 4
Constructing DataFrame from numpy ndarray:
>>> df2 = pd.DataFrame(np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]),
... columns=['a', 'b', 'c'])
>>> df2
a b c
0 1 2 3
1 4 5 6
2 7 8 9
Constructing DataFrame from a numpy ndarray that has labeled columns:
>>> data = np.array([(1, 2, 3), (4, 5, 6), (7, 8, 9)],
... dtype=[("a", "i4"), ("b", "i4"), ("c", "i4")])
>>> df3 = pd.DataFrame(data, columns=['c', 'a'])
...
>>> df3
c a
0 3 1
1 6 4
2 9 7
Constructing DataFrame from Series/DataFrame:
>>> ser = pd.Series([1, 2, 3], index=["a", "b", "c"], name = "s")
>>> df = pd.DataFrame(data=ser, index=["a", "c"])
>>> df
s
a 1
c 3
>>> df1 = pd.DataFrame([1, 2, 3], index=["a", "b", "c"], columns=["x"])
>>> df2 = pd.DataFrame(data=df1, index=["a", "c"])
>>> df2
x
a 1
c 3
"""
def __repr__():
"""
Return a string representation for a particular ``DataFrame``.
Returns
-------
str
"""
@property
def ndim():
"""
Return the number of dimensions of the underlying data, by definition 2.
"""
def drop_duplicates():
"""
Return ``DataFrame`` with duplicate rows removed.
Considering certain columns is optional. Indexes, including time indexes are ignored.
Parameters
----------
subset : column label or sequence of labels, optional
Only consider certain columns for identifying duplicates, by default use all columns.
keep : {'first', 'last', False}, default 'first'
Determines which duplicates (if any) to keep.
'first' : Drop duplicates except for the first occurrence.
'last' : Drop duplicates except for the last occurrence.
False : Drop all duplicates.
inplace : bool, default False
Whether to modify the DataFrame rather than creating a new one.
ignore_index : bool, default False
If True, the resulting axis will be labeled 0, 1, …, n - 1.
Returns
-------
DataFrame or None
DataFrame with duplicates removed or None if inplace=True.
Examples
--------
Consider dataset containing ramen rating.
>>> df = pd.DataFrame({
... 'brand': ['Yum Yum', 'Yum Yum', 'Indomie', 'Indomie', 'Indomie'],
... 'style': ['cup', 'cup', 'cup', 'pack', 'pack'],
... 'rating': [4, 4, 3.5, 15, 5]
... })
>>> df
brand style rating
0 Yum Yum cup 4.0
1 Yum Yum cup 4.0
2 Indomie cup 3.5
3 Indomie pack 15.0
4 Indomie pack 5.0
By default, it removes duplicate rows based on all columns.
>>> df.drop_duplicates()
brand style rating
0 Yum Yum cup 4.0
2 Indomie cup 3.5
3 Indomie pack 15.0
4 Indomie pack 5.0
To remove duplicates on specific column(s), use subset.
>>> df.drop_duplicates(subset=['brand'])
brand style rating
0 Yum Yum cup 4.0
2 Indomie cup 3.5
To remove duplicates and keep last occurrences, use keep.
>>> df.drop_duplicates(subset=['brand', 'style'], keep='last')
brand style rating
1 Yum Yum cup 4.0
2 Indomie cup 3.5
4 Indomie pack 5.0
"""
def dropna():
"""
Remove missing values.
Parameters
----------
axis : {0 or 'index', 1 or 'columns'}, default 0
Determine if rows or columns which contain missing values are
removed.
* 0, or 'index' : Drop rows which contain missing values.
* 1, or 'columns' : Drop columns which contain missing value.
Only a single axis is allowed.
how : {'any', 'all'}, default 'any'
Determine if row or column is removed from DataFrame, when we have
at least one NA or all NA.
* 'any' : If any NA values are present, drop that row or column.
* 'all' : If all values are NA, drop that row or column.
thresh : int, optional
Require that many non-NA values. Cannot be combined with how.
subset : column label or sequence of labels, optional
Labels along other axis to consider, e.g. if you are dropping rows
these would be a list of columns to include.
inplace : bool, default False
Whether to modify the DataFrame rather than creating a new one.
Returns
-------
DataFrame
DataFrame with NA entries dropped from it or None if ``inplace=True``.
See Also
--------
DataFrame.isna : Indicate missing values.
DataFrame.notna : Indicate existing (non-missing) values.
DataFrame.fillna : Replace missing values.
Series.dropna : Drop missing values.
Index.dropna : Drop missing indices.
Examples
--------
>>> df = pd.DataFrame({"name": ['Alfred', 'Batman', 'Catwoman'],
... "toy": [None, 'Batmobile', 'Bullwhip'],
... "born": [pd.NaT, pd.Timestamp("1940-04-25"),
... pd.NaT]})
>>> df
name toy born
0 Alfred None NaT
1 Batman Batmobile 1940-04-25
2 Catwoman Bullwhip NaT
Drop the rows where at least one element is missing.
>>> df.dropna()
name toy born
1 Batman Batmobile 1940-04-25
Drop the rows where all elements are missing.
>>> df.dropna(how='all')
name toy born
0 Alfred None NaT
1 Batman Batmobile 1940-04-25
2 Catwoman Bullwhip NaT
Keep only the rows with at least 2 non-NA values.
>>> df.dropna(thresh=2)
name toy born
1 Batman Batmobile 1940-04-25
2 Catwoman Bullwhip NaT
Define in which columns to look for missing values.
>>> df.dropna(subset=['name', 'toy'])
name toy born
1 Batman Batmobile 1940-04-25
2 Catwoman Bullwhip NaT
Keep the DataFrame with valid entries in the same variable.
>>> df.dropna(inplace=True)
>>> df
name toy born
1 Batman Batmobile 1940-04-25
"""
@property
def dtypes():
"""
Return the dtypes in the ``DataFrame``.
This returns a Series with the data type of each column.
The result's index is the original DataFrame's columns. Columns
with mixed types are stored with the ``object`` dtype.
The returned dtype for each label is the 'largest' numpy type for the
underlying data.
For labels with integer-type data, int64 is returned.
For floating point and decimal data, float64 is returned.
For boolean data, numpy.bool is returned.
For datetime or timestamp data, datetime64[ns] is returned.
For all other data types, including string, date, binary or snowflake variants,
the dtype object is returned.
This function is lazy and does NOT trigger evaluation of the underlying
``DataFrame``.
Note that because the returned dtype(s) may be of a larger type than the underlying
data, the result of this function may differ from the dtypes of the output of the
:func:`to_pandas()` function.
Calling :func:`to_pandas()` triggers materialization into a native
pandas DataFrame. The dtypes of this materialized result are the narrowest
type(s) that can represent the underlying data (like int16, or int32).
Returns
-------
pandas.Series
Native pandas (not Snowpark pandas) Series with the dtype for each label.
Examples
--------
>>> df = pd.DataFrame({'float': [1.0],
... 'int': [1],
... 'datetime': [pd.Timestamp('20180310')],
... 'string': ['foo']})
>>> df.dtypes
float float64
int int64
datetime datetime64[ns]
string object
dtype: object
"""
def duplicated():
"""
Return boolean Series denoting duplicate rows.
Considering certain columns is optional.
Parameters
----------
subset : column label or sequence of labels, optional
Only consider certain columns for identifying duplicates, by
default use all the columns.
keep : {'first', 'last', False}, default 'first'
Determines which duplicates (if any) to mark.
- ``first`` : Mark duplicates as ``True`` except for the first occurrence.
- ``last`` : Mark duplicates as ``True`` except for the last occurrence.
- False : Mark all duplicates as ``True``.
Returns
-------
Series
Boolean series for each duplicated rows.
See Also
--------
Index.duplicated : Equivalent method on index.
Series.duplicated : Equivalent method on Series.
Series.drop_duplicates : Remove duplicate values from Series.
DataFrame.drop_duplicates : Remove duplicate values from DataFrame.
Examples
--------
Consider dataset containing ramen rating.
>>> df = pd.DataFrame({
... 'brand': ['Yum Yum', 'Yum Yum', 'Indomie', 'Indomie', 'Indomie'],
... 'style': ['cup', 'cup', 'cup', 'pack', 'pack'],
... 'rating': [4, 4, 3.5, 15, 5]
... })
>>> df
brand style rating
0 Yum Yum cup 4.0
1 Yum Yum cup 4.0
2 Indomie cup 3.5
3 Indomie pack 15.0
4 Indomie pack 5.0
By default, for each set of duplicated values, the first occurrence
is set on False and all others on True.
>>> df.duplicated()
0 False
1 True
2 False
3 False
4 False
dtype: bool
By using 'last', the last occurrence of each set of duplicated values
is set on False and all others on True.
>>> df.duplicated(keep='last')
0 True
1 False
2 False
3 False
4 False
dtype: bool
By setting ``keep`` on False, all duplicates are True.
>>> df.duplicated(keep=False)
0 True
1 True
2 False
3 False
4 False
dtype: bool
To find duplicates on specific column(s), use ``subset``.
>>> df.duplicated(subset=['brand'])
0 False
1 True
2 False
3 True
4 True
dtype: bool
"""
@property
def empty():
"""
Indicator whether the DataFrame is empty.
True if the DataFrame is entirely empty (no items), meaning any of the axes are of length 0.
Returns
-------
bool
"""
@property
def axes():
"""
Return a list representing the axes of the DataFrame.
It has the row axis labels and column axis labels as the only members.
They are returned in that order.
Examples
--------
>>> df = pd.DataFrame({'col1': [1, 2], 'col2': [3, 4]})
>>> df.axes
[Index([0, 1], dtype='int64'), Index(['col1', 'col2'], dtype='object')]
"""
@property
def shape(self):
"""
Return a tuple representing the dimensionality of the ``DataFrame``.
"""
def add_prefix():
"""
Prefix labels with string `prefix`.
For Series, the row labels are prefixed.
For DataFrame, the column labels are prefixed.
Parameters
----------
prefix : str
The string to add before each label.
Returns
-------
Series or DataFrame
New Series or DataFrame with updated labels.
See Also
--------
Series.add_suffix: Suffix row labels with string `suffix`.
DataFrame.add_suffix: Suffix column labels with string `suffix`.
Examples
--------
>>> s = pd.Series([1, 2, 3, 4])
>>> s
0 1
1 2
2 3
3 4
dtype: int64
>>> s.add_prefix('item_')
item_0 1
item_1 2
item_2 3
item_3 4
dtype: int64
>>> df = pd.DataFrame({'A': [1, 2, 3, 4], 'B': [3, 4, 5, 6]})
>>> df
A B
0 1 3
1 2 4
2 3 5
3 4 6
>>> df.add_prefix('col_')
col_A col_B
0 1 3
1 2 4
2 3 5
3 4 6
"""
def add_suffix():
"""
Suffix labels with string `suffix`.
For Series, the row labels are suffixed.
For DataFrame, the column labels are suffixed.
Parameters
----------
suffix : str
The string to add after each label.
Returns
-------
Series or DataFrame
New Series or DataFrame with updated labels.
See Also
--------
Series.add_prefix: Prefix row labels with string `prefix`.
DataFrame.add_prefix: Prefix column labels with string `prefix`.
Examples
--------
>>> s = pd.Series([1, 2, 3, 4])
>>> s
0 1
1 2
2 3
3 4
dtype: int64
>>> s.add_suffix('_item')
0_item 1
1_item 2
2_item 3
3_item 4
dtype: int64
>>> df = pd.DataFrame({'A': [1, 2, 3, 4], 'B': [3, 4, 5, 6]})
>>> df
A B
0 1 3
1 2 4
2 3 5
3 4 6
>>> df.add_suffix('_col')
A_col B_col
0 1 3
1 2 4
2 3 5
3 4 6
"""
def applymap():
# TODO SNOW-1818207 unskip tests once package resolution is fixed
"""
Apply a function to a Dataframe elementwise.
This method applies a function that accepts and returns a scalar
to every element of a DataFrame.
Parameters
----------
func : callable
Python function, returns a single value from a single value.
na_action : {None, 'ignore'}, default None
If ‘ignore’, propagate NaN values, without passing them to func.
**kwargs
Additional keyword arguments to pass as keywords arguments to
`func`.
Returns
-------
DataFrame
Transformed DataFrame.
See Also
--------
:func:`Series.apply <modin.pandas.Series.apply>` : For applying more complex functions on a Series.
:func:`DataFrame.apply <modin.pandas.DataFrame.apply>` : Apply a function row-/column-wise.
Examples
--------
>>> df = pd.DataFrame([[1, 2.12], [3.356, 4.567]])
>>> df
0 1
0 1.000 2.120
1 3.356 4.567
>>> df.applymap(lambda x: len(str(x)))
0 1
0 3 4
1 5 5
When you use the applymap function, a user-defined function (UDF) is generated and
applied to each column. However, in many cases, you can achieve the same results
more efficiently by utilizing alternative dataframe operations instead of applymap.
For example, You could square each number elementwise.
>>> df.applymap(lambda x: x**2)
0 1
0 1.000000 4.494400
1 11.262736 20.857489
But it's better to avoid applymap in that case.
>>> df ** 2
0 1
0 1.000000 4.494400
1 11.262736 20.857489
"""
_agg_examples_doc = dedent(
"""
Examples
--------
>>> df = pd.DataFrame([[1, 2, 3],
... [4, 5, 6],
... [7, 8, 9],
... [np.nan, np.nan, np.nan]],
... columns=['A', 'B', 'C'])
Aggregate these functions over the rows.
>>> df.agg(['sum', 'min'])
A B C
sum 12.0 15.0 18.0
min 1.0 2.0 3.0
Different aggregations per column.
>>> df.agg({'A' : ['sum', 'min'], 'B' : ['min', 'max']})
A B
sum 12.0 NaN
min 1.0 2.0
max NaN 8.0
Aggregate over the columns.
>>> df.agg("max", axis="columns")
0 3.0
1 6.0
2 9.0
3 NaN
dtype: float64
Different aggregations per row.
>>> df.agg({ 0: ["sum"], 1: ["min"] }, axis=1)
sum min
0 6.0 NaN
1 NaN 4.0
"""
)
@doc(
_shared_docs["aggregate"],
klass=_shared_doc_kwargs["klass"],
axis=_shared_doc_kwargs["axis"],
examples=_agg_examples_doc,
)
def aggregate():
pass
agg = aggregate
def apply():
# TODO SNOW-1818207 unskip tests once package resolution is fixed
"""
Apply a function along an axis of the DataFrame.
Objects passed to the function are Series objects whose index is
either the DataFrame's index (``axis=0``) or the DataFrame's columns
(``axis=1``). By default (``result_type=None``), the final return type
is inferred from the return type of the applied function. Otherwise,
it depends on the `result_type` argument.
Snowpark pandas currently only supports ``apply`` with ``axis=1`` and callable ``func``.
Parameters
----------
func : function
A Python function object to apply to each column or row.
axis : {0 or 'index', 1 or 'columns'}, default 0
Axis along which the function is applied:
* 0 or 'index': apply function to each column.
* 1 or 'columns': apply function to each row.
raw : bool, default False
Determines if row or column is passed as a Series or ndarray object:
* ``False`` : passes each row or column as a Series to the
function.
* ``True`` : the passed function will receive ndarray objects
instead.
result_type : {'expand', 'reduce', 'broadcast', None}, default None
These only act when ``axis=1`` (columns):
* 'expand' : list-like results will be turned into columns.
* 'reduce' : returns a Series if possible rather than expanding
list-like results. This is the opposite of 'expand'.
* 'broadcast' : results will be broadcast to the original shape
of the DataFrame, the original index and columns will be
retained.
Snowpark pandas does not yet support the ``result_type`` parameter.
args : tuple
Positional arguments to pass to `func` in addition to the
array/series.
**kwargs
Additional keyword arguments to pass as keywords arguments to
`func`.
Returns
-------
Series or DataFrame
Result of applying ``func`` along the given axis of the DataFrame.
See Also
--------
:func:`Series.apply <modin.pandas.Series.apply>` : For applying more complex functions on a Series.
:func:`DataFrame.applymap <modin.pandas.DataFrame.applymap>` : Apply a function elementwise on a whole DataFrame.
Notes
-----
1. When ``func`` has a type annotation for its return value, the result will be cast
to the corresponding dtype. When no type annotation is provided, data will be converted
to VARIANT type in Snowflake, and the result will have ``dtype=object``. In this case, the return value must
be JSON-serializable, which can be a valid input to ``json.dumps`` (e.g., ``dict`` and
``list`` objects are JSON-serializable, but ``bytes`` and ``datetime.datetime`` objects
are not). The return type hint is used only when ``func`` is a series-to-scalar function.
2. Under the hood, we use Snowflake Vectorized Python UDFs to implement apply() method with
`axis=1`. You can find type mappings from Snowflake SQL types to pandas dtypes
`here <https://docs.snowflake.com/en/developer-guide/udf/python/udf-python-batch#type-support>`_.
3. Snowflake supports two types of NULL values in variant data: `JSON NULL and SQL NULL <https://docs.snowflake.com/en/user-guide/semistructured-considerations#null-values>`_.
When no type annotation is provided and Variant data is returned, Python ``None`` is translated to
JSON NULL, and all other pandas missing values (np.nan, pd.NA, pd.NaT) are translated to SQL NULL.
4. If ``func`` is a series-to-series function that can also be used as a scalar-to-scalar function
(e.g., ``np.sqrt``, ``lambda x: x+1``), using ``df.applymap()`` to apply the function
element-wise may give better performance.
5. When ``func`` can return a series with different indices, e.g.,
``lambda x: pd.Series([1, 2], index=["a", "b"] if x.sum() > 2 else ["b", "c"])``,
the values with the same label will be merged together.
6. The index values of returned series from ``func`` must be JSON-serializable. For example,
``lambda x: pd.Series([1], index=[bytes(1)])`` will raise a SQL execption because python ``bytes``
objects are not JSON-serializable.
7. When ``func`` uses any first-party modules or third-party packages inside the function,
you need to add these dependencies via ``session.add_import()`` and ``session.add_packages()``.
8. The Snowpark pandas module cannot currently be referenced inside the definition of
``func``. If you need to call a general pandas API like ``pd.Timestamp`` inside ``func``,
please use the original ``pandas`` module (with ``import pandas``) as a workaround.
9. To create a permanent function, pass the "snowflake_udf_params" dictionary argument to
``apply``. See examples below for details.
Examples
--------
>>> df = pd.DataFrame([[2, 0], [3, 7], [4, 9]], columns=['A', 'B'])
>>> df
A B
0 2 0
1 3 7
2 4 9
Using a reducing function on ``axis=1``:
>>> df.apply(np.sum, axis=1)
0 2
1 10
2 13
dtype: int64
Returning a list-like object will result in a Series:
>>> df.apply(lambda x: [1, 2], axis=1)
0 [1, 2]
1 [1, 2]
2 [1, 2]
dtype: object
To work with 3rd party packages, add them to the current session:
>>> import scipy.stats
>>> pd.session.custom_package_usage_config['enabled'] = True
>>> pd.session.add_packages(['numpy', scipy])
>>> df.apply(lambda x: np.dot(x * scipy.stats.norm.cdf(0), x * scipy.stats.norm.cdf(0)), axis=1)
0 1.00
1 14.50
2 24.25
dtype: float64
To generate a permanent UDTF, pass a dictionary as the `snowflake_udf_params` argument to `apply`.
The following example generates a permanent UDTF named "permanent_double":
>>> session.sql("CREATE STAGE sample_upload_stage").collect() # doctest: +SKIP
>>> def double(x: int) -> int: # doctest: +SKIP
... return x * 2
...
>>> df.apply(double, snowflake_udf_params={"name": "permanent_double", "stage_location": "@sample_upload_stage"}) # doctest: +SKIP
A B
0 4 0
1 6 14
2 8 18
You may also pass "replace" and "if_not_exists" in the dictionary to overwrite or re-use existing UDTFs.
With the "replace" flag:
>>> df.apply(double, snowflake_udf_params={ # doctest: +SKIP
... "name": "permanent_double",
... "stage_location": "@sample_upload_stage",
... "replace": True,
... })
With the "if_not_exists" flag:
>>> df.apply(double, snowflake_udf_params={ # doctest: +SKIP
... "name": "permanent_double",
... "stage_location": "@sample_upload_stage",
... "if_not_exists": True,
... })
Note that Snowpark pandas may still attempt to upload a new UDTF even when "if_not_exists"
is passed; the generated SQL will just contain a `CREATE FUNCTION IF NOT EXISTS` query
instead. Subsequent calls to `apply` within the same session may skip this query.
Passing the `immutable` keyword creates an immutable UDTF, which assumes that the
UDTF will return the same result for the same inputs.
>>> df.apply(double, snowflake_udf_params={ # doctest: +SKIP
... "name": "permanent_double",
... "stage_location": "@sample_upload_stage",
... "replace": True,
... "immutable": True,
... })
"""
def assign():
"""
Assign new columns to a ``DataFrame``.
Returns a new object with all original columns in addition to new ones. Existing
columns that are re-assigned will be overwritten.
Parameters
----------
**kwargs: dict of {str: callable or Series}
The column names are the keywords. If the values are callable, they are computed
on the DataFrame and assigned to the new columns. The callable must not change input
DataFrame (though Snowpark pandas doesn't check it). If the values are not callable,
(e.g. a Series, scalar, or array), they are simply assigned.
Returns
-------
DataFrame
A new DataFrame with the new columns in addition to all the existing columns.
Notes
-----
- Assigning multiple columns within the same assign is possible. Later items in `**kwargs`
may refer to newly created or modified columns in `df`; items are computed and assigned into `df` in order.
- If an array that of the wrong length is passed in to assign, Snowpark pandas will either truncate the array, if it is too long,
or broadcast the last element of the array until the array is the correct length if it is too short. This differs from native pandas,
which will error out with a ValueError if the length of the array does not match the length of `df`.
This is done to preserve Snowpark pandas' lazy evaluation paradigm.
Examples
--------
>>> df = pd.DataFrame({'temp_c': [17.0, 25.0]},
... index=['Portland', 'Berkeley'])
>>> df
temp_c
Portland 17.0
Berkeley 25.0
>>> df.assign(temp_f=lambda x: x.temp_c * 9 / 5 + 32)
temp_c temp_f
Portland 17.0 62.6
Berkeley 25.0 77.0
>>> df.assign(temp_f=df['temp_c'] * 9 / 5 + 32)
temp_c temp_f
Portland 17.0 62.6
Berkeley 25.0 77.0
>>> df.assign(temp_f=lambda x: x['temp_c'] * 9 / 5 + 32,
... temp_k=lambda x: (x['temp_f'] + 459.67) * 5 / 9)
temp_c temp_f temp_k
Portland 17.0 62.6 290.15
Berkeley 25.0 77.0 298.15
>>> df = pd.DataFrame({'col1': [17.0, 25.0, 22.0]})
>>> df
col1
0 17.0
1 25.0
2 22.0
>>> df.assign(new_col=[10, 11])
col1 new_col
0 17.0 10
1 25.0 11
2 22.0 11
>>> df.assign(new_col=[10, 11, 12, 13, 14])
col1 new_col
0 17.0 10
1 25.0 11
2 22.0 12
"""
def groupby():
"""
Group DataFrame using a mapper or by a Series of columns.
Args:
by: mapping, function, label, Snowpark pandas Series or a list of such. Used to determine the groups for the groupby.
If by is a function, it’s called on each value of the object’s index. If a dict or Snowpark pandas Series is
passed, the Series or dict VALUES will be used to determine the groups (the Series’ values are first aligned;
see .align() method). If a list or ndarray of length equal to the selected axis is passed (see the groupby
user guide), the values are used as-is to determine the groups. A label or list of labels may be passed
to group by the columns in self. Notice that a tuple is interpreted as a (single) key.
axis: {0 or ‘index’, 1 or ‘columns’}, default 0
Split along rows (0) or columns (1). For Series this parameter is unused and defaults to 0.
level: int, level name, or sequence of such, default None
If the axis is a MultiIndex (hierarchical), group by a particular level or levels. Do not specify both by and level.
as_index: bool, default True
For aggregated output, return object with group labels as the index. Only relevant for DataFrame input.
as_index=False is effectively “SQL-style” grouped output.
sort: bool, default True
Sort group keys. Groupby preserves the order of rows within each group. Note that in pandas,
better performance can be achieved by turning sort off, this is not going to be true with
SnowparkPandas. When sort=False, the performance will be no better than sort=True.
group_keys: bool, default True
When calling apply and the by argument produces a like-indexed (i.e. a transform) result, add group