+ jg0*^RIHt^RIHtHtHt^RIHtHtH t ^RI H t ^RI H t Ht^RIHt.ROt] !R R R 7tR Rlt] !R4!RR]] ,44t] !R4!RR ]44t] !R4!RR]] ,44tR#)) defaultdict)CallableIteratorSized)AnyNoReturnTypeVar)functional_datapipe) DataChunk IterDataPipe)_check_unpickable_fnBatcherIterDataPipeGrouperIterDataPipeUnBatcherIterDataPipe_T_coT) covariantc0V^8dQhR\R\/#)namereturn)strr)formats"i/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torch/utils/data/datapipes/iter/grouping.py __annotate__rsGGcGhGc,\R\ RV 24h)zmodule z has no attribute )AttributeError__name__)rs&r __getattr__rs 78*,>tfE FFrbatchczaa]tRt^toRtR]3V3RlV3RllltV3RlRltV3RlRltV3R lt R t Vt V;t #) ra Creates mini-batches of data (functional name: ``batch``). An outer dimension will be added as ``batch_size`` if ``drop_last`` is set to ``True``, or ``length % batch_size`` for the last batch if ``drop_last`` is set to ``False``. Args: datapipe: Iterable DataPipe being batched batch_size: The size of each batch drop_last: Option to drop the last batch if it's not full wrapper_class: wrapper to apply onto each batch (type ``List``) before yielding, defaults to ``DataChunk`` Example: >>> # xdoctest: +SKIP >>> from torchdata.datapipes.iter import IterableWrapper >>> dp = IterableWrapper(range(10)) >>> dp = dp.batch(batch_size=3, drop_last=True) >>> list(dp) [[0, 1, 2], [3, 4, 5], [6, 7, 8]] Fc F<V^8dQhRS[RS[RS[RS[S[,RR/#)rdatapipe batch_size drop_last wrapper_classrN)r intbooltyper )r __classdict__s"rr BatcherIterDataPipe.__annotate__5sA + + + + + I +  +rcx<V^8:d \R4h\SV` 4WnW nW0nW@nR#)rz+Batch size is required to be larger than 0!N)AssertionErrorsuper__init__r#r$r%r&)selfr#r$r%r& __class__s&&&&&rr/BatcherIterDataPipe.__init__5s8 ? !NO O  $"*rc0<V^8dQhRS[S[,/#rr)rr )rr*s"rrr+Ds 0 0(9- 0rc#.".pVPFEpVPV4\V4VP8XgK0VP V4x.pKG \V4^8d*VP 'gVP V4xR#R#R#5irN)r#appendlenr$r&r%)r0r xs& r__iter__BatcherIterDataPipe.__iter__DszA LLO5zT__,((//  u:>>>>((//" s:B:B<Bc <V^8dQhRS[/#r4)r')rr*s"rrr+OsYYYrcp\VP\4'dwVP'd'\ VP4VP ,#\ VP4VP ,^, VP ,#\ \V4P R24h)z# instance doesn't have valid length) isinstancer#rr%r8r$ TypeErrorr)rr0s&r__len__BatcherIterDataPipe.__len__Osx dmmU + +~~~4==)T__<<DMM*T__<V^8dQh/S[;R&S[;R&S[;R&#)rr#r$r%)r r'r()rr*s"rrr+s*012O34O5r)r$r#r%r&) r __module__ __qualname____firstlineno____doc__r r/r:rB__annotate_func____static_attributes____classdictcell__ __classcell__)r1r*s@@rrrs:< )2 + + 0 0YYmrunbatchcFa]tRt^ZtoRtRV3RlRlltRtRtRtVt R#) ra Undoes batching of data (functional name: ``unbatch``). In other words, it flattens the data up to the specified level within a batched DataPipe. Args: datapipe: Iterable DataPipe being un-batched unbatch_level: Defaults to ``1`` (only flattening the top level). If set to ``2``, it will flatten the top two levels, and ``-1`` will flatten the entire DataPipe. Example: >>> # xdoctest: +SKIP >>> from torchdata.datapipes.iter import IterableWrapper >>> source_dp = IterableWrapper([[[0, 1], [2]], [[3, 4], [5]], [[6]]]) >>> dp1 = source_dp.unbatch() >>> list(dp1) [[0, 1], [2], [3, 4], [5], [6]] >>> dp2 = source_dp.unbatch(unbatch_level=2) >>> list(dp2) [0, 1, 2, 3, 4, 5, 6] c*<V^8dQhRS[RS[RR/#)rr# unbatch_levelrN)r r')rr*s"rr"UnBatcherIterDataPipe.__annotate__rs"+++c+$+rcWnW nR#Nr#rP)r0r#rPs&&&rr/UnBatcherIterDataPipe.__init__rs  *rc#"VPF'pVPWPR7RjxL K) R#L 5i)rPN)r#_diverP)r0elements& rr:UnBatcherIterDataPipe.__iter__vs0}}Gzz'9K9KzL L L% Ls />< >c#"VR8d \R4hVR8XdI\V\\34'd'VFpVP VRR7RjxL K R#VxR#V^8XdVxR#\V\\34'd-VF$pVP W2^, R7RjxL K& R#\ RVP R24hL~L%5i)r>z unbatch_level must be -1 or >= 0rWNzunbatch_level z" exceeds the depth of the DataPipe) ValueErrorr?listr rX IndexErrorrP)r0rYrPitems&&& rrXUnBatcherIterDataPipe._divezs 2 ?@ @ B 'D)#455#D#zz$bzAAA$ a M'D)#455#D#zz$a>OzPPP$!$T%7%7$88Z[BQs%ACCAC.C/$CCrTN)r>) rrErFrGrHr/r:rXrJrKr*s@rrrZs$,++Mrgroupbyc a]tRt^toRtRRRRRRRRR R/V3R lR lltR tR tV3RlRltRt Rt V3RlRlt Rt Vt R#)ra Groups data from IterDataPipe by keys from ``group_key_fn``, yielding a ``DataChunk`` with batch size up to ``group_size``. (functional name: ``groupby``). The samples are read sequentially from the source ``datapipe``, and a batch of samples belonging to the same group will be yielded as soon as the size of the batch reaches ``group_size``. When the buffer is full, the DataPipe will yield the largest batch with the same key, provided that its size is larger than ``guaranteed_group_size``. If its size is smaller, it will be dropped if ``drop_remaining=True``. After iterating through the entirety of source ``datapipe``, everything not dropped due to the buffer capacity will be yielded from the buffer, even if the group sizes are smaller than ``guaranteed_group_size``. Args: datapipe: Iterable datapipe to be grouped group_key_fn: Function used to generate group key from the data of the source datapipe keep_key: Option to yield the matching key along with the items in a tuple, resulting in `(key, [items])` otherwise returning [items] buffer_size: The size of buffer for ungrouped data group_size: The max size of each group, a batch is yielded as soon as it reaches this size guaranteed_group_size: The guaranteed minimum group size to be yielded in case the buffer is full drop_remaining: Specifies if the group smaller than ``guaranteed_group_size`` will be dropped from buffer when the buffer is full Example: >>> import os >>> # xdoctest: +SKIP >>> from torchdata.datapipes.iter import IterableWrapper >>> def group_fn(file): ... return os.path.basename(file).split(".")[0] >>> source_dp = IterableWrapper( ... ["a.png", "b.png", "a.json", "b.json", "a.jpg", "c.json"] ... ) >>> dp0 = source_dp.groupby(group_key_fn=group_fn) >>> list(dp0) [['a.png', 'a.json', 'a.jpg'], ['b.png', 'b.json'], ['c.json']] >>> # A group is yielded as soon as its size equals to `group_size` >>> dp1 = source_dp.groupby(group_key_fn=group_fn, group_size=2) >>> list(dp1) [['a.png', 'a.json'], ['b.png', 'b.json'], ['a.jpg'], ['c.json']] >>> # Scenario where `buffer` is full, and group 'a' needs to be yielded since its size > `guaranteed_group_size` >>> dp2 = source_dp.groupby( ... group_key_fn=group_fn, ... buffer_size=3, ... group_size=3, ... guaranteed_group_size=2, ... ) >>> list(dp2) [['a.png', 'a.json'], ['b.png', 'b.json'], ['a.jpg'], ['c.json']] keep_keyF buffer_sizei' group_sizeNguaranteed_group_sizedrop_remainingc<V^8dQhRS[S[,RS[S[.S[3,RS[RS[RS[R,RS[R,RS[R R/#) rr# group_key_fnrerfrgNrhrir)r rrrr(r')rr*s"rr GrouperIterDataPipe.__annotate__ss"'"'u%"'w|,"'  "'  "'$J"' #Tz"'"' "'rcf\V4WnW nW0nW@n\ \ 4Vn^VnWPn RVn Ve'Ve#^Tu;8dV8:gM\R4hWPn Ve'Ve^Tu;8dV8:gM\R4hW`n Wpn \VnR#)rNz)group_size must be > 0 and <= buffer_sizezNguaranteed_group_size must be > 0 and <= group_size and group_size must be set)r r#rkremax_buffer_sizerr^buffer_elementscurr_buffer_sizergrhr-rir r&)r0r#rkrerfrgrhris&&&$$$$$rr/GrouperIterDataPipe.__init__s \* ( *7B47H !$%)"  !k&= 1k1$%PQQ)3 & ,!!.C*Qz*Q$d*? &,&rcRp^pRpVPFDp\VPV,4V8gK&\VPV,4pTpKF VPeIW P8d9VP'g'\ R\ VPV,44hVPeW P8dVPV,pV;P V,unVPVV#)NzFailed to group items)ror8rhri RuntimeErrorrrp)r0 biggest_key biggest_sizeresult_to_yieldfindkeys& r_remove_biggest_key'GrouperIterDataPipe._remove_biggest_keys  ++G4''01L@"4#7#7#@A % ,  & & 2999''''T-A-A+-N)O   & & .999"22;?O -   -rc#"VPEFcpVPV4pVPV,PV4V;P^, unVP eVP \ VPV,48Xd|VPVPV,4pVP'dW#3MTxV;P\ VPV,4,unVPVVPVP8XgEK"VP4pVfEK9VPV4pVP'dW#3MTxEKf \VPP44FhpVPVPPV44pV;P\ V4,unVP'dW#3MTxKj R#5i)r>N)r#rkror7rprgr8r&rernrxtuplekeyspop)r0r9keyresultrvs& rr:GrouperIterDataPipe.__iter__sA##A&C   % , ,Q /  ! !Q & !*t#$$S)C0*.););Drs#55))EGH  4(GW=Y,y1=Y=Y@Y1L1 1hYu%,y1u% u%r