o i@sUddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z m Z mZmZmZddlmZmZddlmZddlmZddlmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*dd l+m,Z,ddl-Z-dd l-m.Z.m/Z/m0Z0ddl1m2m3m4Z5ddl6m7m8Z9ddl:Z;ddlZ?dd l@mAZAdd lBmCZCdd lDmEZEddlFmGZGddlHmIZIddlJmKZKmLZLmMZMmNZNmOZOddlPmQZQddlRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZddl[m\Z\ddl]m^Z^ddl_m`Z`maZambZbddlcmdZdddlemfZfmgZgddlhmiZimjZjmkZkmlZlmmZmddlgmnZnmoZompZpmqZqmrZrddlsmtZtddlumvZvmwZwmxZxmyZyddlzm{Z{ddl|m}Z}m~Z~ddl8mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZdd lmZmZmZe rdd!lmZdd"lRmZdd#l[mZdd$lmZdd%lmZdd&lmZdd'l8mZneZd(ed)<z ddlZejZd*ZWneydZd+ZYnwe'd,Ze!d-Ze!d.Ze!d/Ze"ee.fZd(ed0<e"eee.fZd(ed1<e"e;jje;jjfZd(ed2<eeZeje jd3d4Ze;jjZeed5d6Zeed7d8Z e"ed9eed9fd:d;eee"eeed9fd9d:d;ffZd(ed<<ddAdBZejd*dCGdDdEdEZÐddIdJZĐddNdOZŐddSdTZƐddUdVZǐdd\d]Zgd^Zgd_Z dddddeZːddgdhZ dddidjZedddmdnZedddpdnZ *dddsdnZΐddwdxZϐdd{d|ZАdd~dZѐdddZҐdddZӐdddZԐdddZՐdddZ֐dddZGdd;d;Zed+dCGdddZeGddde؃ZڐdddZeGdddeڃZeGddde܃ZededededededdZded< *ddddZeGdddeڃZde/dfddd„Zeee.ee.gefZd(ed<GddńdeZGddDŽdeZGddɄdeZeGdd˄deڃZeGdd̈́deZeGddτdeڃZdddфZdddӄZ * +  + dddd܄ZdddބZdddZeGddde؃ZeGdddeZeGdddeZeGdddeZeGdddeZeGdddeZeGdddeZeGdddeZGdddeZeGddde؃ZeGdddeZeGdddeZdddZdddZGdddZeGdddeZGdddeZGdddeZGdddeZGdd d eZGd d d eZeGd d d eZGdddeZed+dCGdddeejZed+dCGdddeeكZGdddeZGdddeZGdddeZ eGddde؃Z eGddde؃Z ed+dCGdddeZ Gd d!d!eZ Gd"d#d#e Ze"eeeeee"eeeeffZGd$d%d%ZGd&d'd'eZGd(d)d)eZGd*d+d+e ZGd,d-d-e ZGd.d/d/e Zdd3d4Zed+dCGd5d6d6eZGd7d8d8eZGd9d:d:eZed+dCGd;d<d<eZed+dCGd=d>d>eZGd?d@d@eZGdAdBdBeZGdCdDdDeZ GdEdFdFeZ!GdGdHdHe!Z"GdIdJdJe!Z#GdKdLdLeZ$GdMdNdNeZ%GdOdPdPeZ&GdQdRdReZ'GdSdTdTeZ(GdUdVdVe(Z)GdWdXdXeZ*GdYdZdZeZ+Gd[d\d\eZ,Gd]d^d^eZ-Gd_d`d`eZ.GdadbdbeZ/GdcddddeZ0ed+dCGdedfdfZ1GdgdhdheZ2ed+dCGdidjdje2Z3Gdkdldle2Z4eGdmdndneZ5GdodpdpeZ6ejGdqdrdre؃Z7Gdsd9d9e7Z8Gdtdudue7Z9ed+dCGdvdwdwe؃Z:ddydzZ;ed+dCGd{d|d|eZed+dCGdddeZ?Gddde2Z@Gddde؃ZAeGdddeAZBeGdddeAZCGddde2ZDGdddeDZEGdddeDZFGdddeDZGdddZHdddZIdS() annotationsN) Container GeneratorIterableIteratorSequence)AbstractContextManager nullcontext)Enum)partial) AnyCallablecastClassVarLiteralOptionaloverload SupportsFloat SupportsInt TYPE_CHECKINGTypeVarUnion) assert_neverNeveroverride ParamSpecSelf TypeAliasTypeIs)patch)ExprIntegerSymbol)identity)GraphModuleSerializer)can_auto_functionalize)metricsget_free_symbols)compute_required_storage_lengthis_boolean_dtypeis_float_dtypemake_channels_last_strides_for StrideType)get_schema_info)&_remove_effect_token_unbacked_bindingscompute_unbacked_bindings free_symbolsfree_unbacked_symbolsrebind_unbackedresolve_unbacked_bindingsShapeEnvSymTypes)Node OrderedSet)CleanDivFloorDivModularIndexing)SymT)config dependencies)BackendFeature CodegenSymbolget_scheduling_for_deviceindex_prevent_reorderingKernel)Depextract_free_symbols#extract_input_node_reduction_rangesextract_read_writes var_builder)LoopBody) OpCounterCSE OpCountResult ReductionType StoreMode) benchmarker)DeviceProperties ReductionHint)argsort argsort_sym cache_on_selfceildivconvert_shape_to_inductorconvert_shape_to_symintdeveloper_warningdo_bench_using_profilingdtype_from_sizeget_dtype_sizeget_kernel_metadataGPU_ALIGN_BYTES ir_dataclass is_dynamicis_gpu sympy_dotsympy_index_symbolsympy_index_symbol_with_prefix sympy_product sympy_substensor_is_aligned)opsOpsValueV)FakeScriptObject) SympyBoolean)Argument) CUDATemplate)PythonWrapperCodegen) GraphLowering)IndentedBufferrrnTF_P_T_U_V_IntLike_NumLike _OpOverloadsz prefixTORCH_AUTOTUNE_WARMUPTORCH_AUTOTUNE_REPd TensorBoxr"IRNode _NodeOrNodesxobjectreturnboolcCst|ttfSN) isinstanceintr!rrH/mnt/sdb/aimis/docanh/lib/python3.10/site-packages/torch/_inductor/ir.py _is_staticr)frozenc@s>eZdZUded<ded<ded<ded<d ed <d ed <d S)GraphPartitionSignatureOrderedSet[sympy.Symbol] symbol_inputsz5dict[str, Union[IRNode, sympy.Expr, TorchBindObject]] input_nodes list[IRNode] output_nodeszdict[str, bool]input_deallocationrskip_cudagraph list[str]constant_namesN__name__ __module__ __qualname____annotations__rrrrrs  r node_or_nodesOptional[_NodeOrNodes]Nonecsdfdd |dS)Nnodesrrrc s|durdSt|ttfr|D]}|qdSt|tr*|D]}|q!dSt|ttttt j j j t tttf sEJdt|ddS)NzFound zE, which is not a supported top level IR node. See [Note: Inductor IR])rlisttupledictvalues ExpandView DynamicScalar AssertScalarrsympylogicboolalgBooleanr rEffectfulKernelShapeAsConstantBuffertype)rnode_check_tensorboxrrrs2    z%validate_ir.._check_tensorbox)rrrrr)rrrr validate_irs rnamestrCallable[..., OpsValue]cs(tts Jtdfdd }|S) Nargsrkwargsrricstt|i|Sr)getattrrhrrrrrfnzops_wrapper..fn)rrrrrri)rrr)rrrrr ops_wrappersrorder Sequence[int]&Callable[[Sequence[_T]], Sequence[_T]]cs(tt|tt|dfdd }|S)Nindex Sequence[_T]rc0ttks JfddttDS)Ncg|]}|qSrr.0i)r inv_orderrr z4inverse_reorder..reindex..lenrangerrrrreindexz inverse_reorder..reindexrrrr)rziprrrrrrrinverse_reordersrcsdfdd }|S)Nrrrcr)Ncrrrr)rrrrr(rz1same_reorder..reindex..rrrrrr&rzsame_reorder..reindexrrrrrr same_reorder%srreindex1&Callable[[Sequence[_U]], Sequence[_V]]reindex2&Callable[[Sequence[_T]], Sequence[_U]]&Callable[[Sequence[_T]], Sequence[_V]]csdfdd }|S)Nrrr Sequence[_V]c |Srrrrrrrr1 z fuse_reindexing..reindex)rrrrr)rrrrrrfuse_reindexing-sr)rr>)rrrr>seq(Sequence[Union[int, torch.SymInt, Expr]] shape_envOptional[ShapeEnv]cCs4|dus tdd|Drt|}|St||}|S)z1 Convert strides to fill order (argsort) Ncss |] }t|ttjfVqdSr)rrrr!rsrrr Asz!get_fill_order..)allrSrT)rr sorted_idxrrrget_fill_order;s  rSequence[Union[int, Integer]]cs0ddt|Dfddtt|D}|S)z Convert stride order to fill order For channel last format, stride order = [3, 0, 2, 1] and fill order = [1, 3, 2, 0] cSi|]\}}||qSrrridxposrrr Pz+stride_order2fill_order..cg|]}|qSrrrlookuprrrQz+stride_order2fill_order..) enumeraterr)r fill_orderrrrstride_order2fill_orderIsrcCs>t||}ddtt|D}t|D]\}}|||<q|S)z) Convert strides to stride order cSg|]}dqSrrr_rrrr\z$get_stride_order..)rrrr)rrroutrelemrrrget_stride_orderUs  r Literal[None] guard_shapecCdSrrrrrrrir_node_to_tensorbr  torch.TensorcCrrrr rrrr fr Optional[IRNode]Optional[torch.Tensor]cs|durdS|stjjjntfdd|D}t|r,fdd|jD}nt |}| }| }t |}t |}tjjjtj||||d}Wd|S1s_wY|S)Ncg|]}|qSrrrshape_fnrrrurz%ir_node_to_tensor..crrrrrrrrxr)sizestridedtypedevice)rjgraphsizevars size_hintr#get_sizeis_storage_and_layout get_layoutrFlexibleLayoutcontiguous_strides get_dtype get_devicerXrsuppress_guardstorch empty_stridedzero_)rrrrrrtrrrr js.   valueOptional[Sequence[_T]] Optional[Sequence[Optional[_T]]]cCst|tr |s dgS|Sr)rrr%rrrmay_convert_to_optionalsr)2Union[IRNode, OutputSpec, torch.device, None, str] Optional[str]cCsbt|ts |dur |St|tjr|jSt|ttfr!t|St d|dt|j ddS)Nzget_device_type(: )) rrr!rrr OutputSpecget_device_typerrrrrrrr/s   r/&Union[IRNode, torch.device, None, str]cCstt|}|dvrtt|ddkrdSdS|dus"t|}dur$dSddlm}t|ts5Jt|t||S)N)cpucuda_backendtritonTFr>)TritonScheduling) r/rr?rCcodegen.tritonr5rr issubclass)rrdevice_schedulingr5rrr is_tritons  r9cCs t|dkS)Nr1)r/rrrris_cpurr:Union[Buffer, TensorBox] alignmentrcsttrduststrdStfddttdD}t j j ddkpFt j j ddk}|oJ|S)NFc3s.|]}tjj|dkVqdS)rN)rjrrsize_hint_or_throw get_striderr<rrrrs  z2is_aligned_realized_tensor_hint..r>) rrmaybe_get_strider2r>rrrrrjrrr=)rr<aligned_stridesaligned_last_dimrr?ris_aligned_realized_tensor_hints   rDstrides1Sequence[_IntLike]strides2shapecCst|t|krt|t|ksJt|||D]'\}}}tjj|dr&qtjj||s?tjj|tjj|ks?dSqdS)zP Returns true if the strides are equal, ignoring dimensions of size 1 . r>FT)rrrjrrstatically_known_leqstatically_known_equals symbolic_hint)rErGrHdims1s2rrrsignificant_strides_equals$rOtensorstrides"Sequence[Union[int, torch.SymInt]]cCst|s|Stddt||Dr|St|||s"|St|\}}g|j}t|D]\}}t j j |drE||||<q3t |j|j|j||j|j}tt||dS)a Tries to match the strides of the tensor to those in the meta_strides. Strides of insignificant dimensions - size 0 or 1 - will be updated. If there are real stride differences (NHWC vs NCHW), or the tensor is not realized, then the input will be returned cs$|] \}}tjj||VqdSrrjrrrJrrMrNrrrr  z2try_match_insignificant_strides..r>datalayout)rrrr>rOras_storage_and_layoutrrrjrrrI FixedLayoutrrroffset is_pinnedrReinterpretView)rPrQstorage old_layout new_striderr new_layoutrrrtry_match_insignificant_stridess.     rcgmtorch.fx.GraphModulecCsD|jjddd}ddt|jD|jd<ddlm}||dS)Noutput)oprcSsg|]\}}|qSrr)rrrrrrr sz.gm_original_output_strides..user_visible_output_idxs)record_original_output_strides)r find_nodesrrmetatorch._inductor.compile_fxri)rd output_noderirrrgm_original_output_strides s   rninputsSequence[IRNode] list[Expr]cCs@t}|D]}|t|ddO}|t|ddO}qt|S)NF unbacked_only)r9r(rr>r)rosym_varsinprrrget_symbolic_inputss rvc@seZdZUdZeZded<ejddZ ded<ejddZ ded <ejddZ d ed <e e jdddZe dddZdddZdddZddd Zdd!d"Zdd#d$Zdd&d'Zdd(d)Zddd-d.Z *ddd2d3Zdd5d6Zdd8d9Zdd;d<Zdd>d?ZddAdBZddDdEZddFdGZ ddIdJZ!ddLdMZ"e#ddOdPZ$ddRdSZ%ddTdUZ&ddWdXZ'ddd\d]Z(dd_d`Z)ddbdcZ*ddddeZ+ddgdhZ,ddjdkZ-ddmdnZ.ddodpZ/ddqdrZ0ddsdtZ1ddudvZ2dddydzZ3dd}d~Z4dddZ5dddZ6dddZ7 ddddZ8dddZ9dddZ: ddddZ;dddZdddZ? ddddZ@dddZAdddZBdddZCdddZDdddZEdddZFdddZGdddZHeIrie#dddZJdYSdYS)rzBase class for all intermediate representation (IR) nodes in TorchInductor. Note: This is an abstract base class. Most methods raise NotImplementedError and must be overridden by concrete subclasses. zClassVar[OrderedSet[Any]]_current_originsF)initOrderedSet[Any]originsOptional[list[str]] tracebackOptional[torch.fx.Node] origin_nodeOrderedSet[Node]rGenerator[None, None, None]ccs.tj}||Bt_z dVW|t_dS|t_wr)rrw)rzoldrrrcurrent_origins-s  zIRNode.current_originsrrcCst|tttttfSr)rComputedBuffer InputsKernel InputBufferr^TemplateBuffer)rrrris_realized_node7szIRNode.is_realized_nodeattrrr%r rcCst|||dSr)r __setattr__)selfrr%rrr_post_init_setattrDszIRNode._post_init_setattrcCs@t|j}|d||dtjrtnd|dddS)Nrzr|r~)r9rwrr?debug_ir_tracebackr| format_stack)rrzrrr __post_init__Js  zIRNode.__post_init__OrderedSet[str]cCtdd|DS)Ncs|]}|jVqdSrrrdeprrrrSz(IRNode.get_read_names..r9 get_readsrrrrget_read_namesRzIRNode.get_read_namescC|jSr)r|rrrr get_tracebackUzIRNode.get_tracebackcCrrr~rrrrget_origin_nodeXrzIRNode.get_origin_nodeOptional[Operation]cCrrrrrrrget_defining_op[r zIRNode.get_defining_opcCst}|j}t|tr|}|jrt|g}|D]:}t|dr*|jr*||jqt j j j di |jg}t|ts>q|D]}t j j j |d}|rR||q@q|S)N stack_trace postToPre)r9rzr ExternKernelrr~hasattrraddr! _inductordebug _inductor_post_to_pre_grad_nodesgetrr#_inductor_pre_grad_node_stack_trace)r stack_tracesrzr~rpre_grad_nodes node_namerrrrget_stack_traces^s4       zIRNode.get_stack_tracesTshorten Sequence[str]cCsdt|dd}|rt|dkr|ddd}|s!|gSg}|D]}|d||d7}|d q'|g|S) Nzorigins=rz@=z...zstack_traces = { })rrrappendsplit)rrrzstack_trace_strrrrr common_repr}s    zIRNode.common_reprlinesSequence[object] multilinecCsbt|t||}ttt|}|r&td|}t|jd|dSt|jd|dS)Nz, z( z )(r-)rrmaprindentjoinrr)rrrr new_linesrrr str_helpers zIRNode.str_helper torch.dtypecCrrrrrrrrrzIRNode.get_dtypeOptional[torch.dtype]cC z|WStyYdSwr)rNotImplementedErrorrrrrmaybe_get_dtype   zIRNode.maybe_get_dtypeLayoutcCtdt|d)Nz#get_layout() is not implemented by !rrrrrrrrzIRNode.get_layoutOptional[Layout]cCrr)rrrrrrmaybe_get_layoutrzIRNode.maybe_get_layoutr.cC|Sr)rrrrrget_output_speczIRNode.get_output_specOptional[OutputSpec]cCrr)rrrrrrmaybe_get_output_specrzIRNode.maybe_get_output_speccCst|tS)z4True for single tensor output (excludes MultiOutput))rrrrrrrhas_tensor_outputszIRNode.has_tensor_outputSequence[Expr]cCr)Nz!get_size() is not implemented by rrrrrrrrzIRNode.get_sizeOptional[Sequence[_IntLike]]cCrr)rrrrrrmaybe_get_sizerzIRNode.maybe_get_size.Union[_IntLike, sympy.Rel, Sequence[_IntLike]]cCrrrrrrrrHz IRNode.shaper cC t|Sr)rerrrrr get_numelrzIRNode.get_numelcCtjjt|dSNrrjrrstatically_known_truerEqrrrrris_zero_elementszIRNode.is_zero_elementsr+cCtdt|)a) If the IRNode refers to data which has not been materialized (e.g., it is a Pointwise/Reduction that could potentially have more compute fused into it), realize the IRNode into physical memory, ending the possibility of fusing into it, but allowing, e.g., multiple users to access the data without having to recompute. Check StorageBox.realize for a particularly notable implementation. TODO(ezyang): I think, in principle, every IRNode should have an implementation of this, and most of the time no-op is OK, but you really do have to audit each IRNode for this, so for now, raise an error if it's not implemented. Note that some code in graph.py will catch this thrown error and suppress it with a warning. zrealize NYI on rrrrrrealizeszIRNode.realizeNwriterOptional[IndentedBuffer]cCr)Nzcodegen_reference NYI on rrrrrrcodegen_referencezIRNode.codegen_referenceOptional[torch.device]cCrrrrrrrrr zIRNode.get_device torch.devicecCs|}|dus J|Srrrrrrrget_device_or_errors zIRNode.get_device_or_errorcCdSNFrrrrrhas_exceeded_max_readsr zIRNode.has_exceeded_max_reads$Callable[[Sequence[Expr]], OpsValue]cCtt|jrrrrrrrr make_loaderrzIRNode.make_loader Callable[[Sequence[Expr]], Expr]cCrrrrrrr make_indexerrzIRNode.make_indexerrFcCrrrrrrrr>rzIRNode.get_stridecCrr)r>rrrrrrArzIRNode.maybe_get_stridecCrrrrrrrget_namerzIRNode.get_namecCrr)rrrrrrmaybe_get_namerzIRNode.maybe_get_namecCs(z |tjjvWStyYdSwr)rrjr graph_inputsrrrrris_input_buffers  zIRNode.is_input_buffer threshold Optional[int]cCrrrrrrrrhas_large_inner_fnr zIRNode.has_large_inner_fnusersrcCrrrrrrrr mark_reuse r zIRNode.mark_reusecCrrrrrrr realize_hint r zIRNode.realize_hintcCrrrrrrr unwrap_viewrzIRNode.unwrap_viewcCrrrrrrr freeze_layoutrzIRNode.freeze_layoutrr allow_paddingcCrrrrrrrrrfreeze_layout_with_stride_orderz&IRNode.freeze_layout_with_stride_ordercCrrrrrrrrfreeze_layout_with_fill_orderrz$IRNode.freeze_layout_with_fill_orderrcCrrrrrrrrfreeze_layout_with_same_orderrz$IRNode.freeze_layout_with_same_order exact_stridescCrrrrrrrrr freeze_layout_with_exact_strides!r z'IRNode.freeze_layout_with_exact_stridesdependencies.ReadWritescCrrrrrrrget_read_writes&rzIRNode.get_read_writesOrderedSet[Dep]cC |jSrrreadsrrrrr) zIRNode.get_readscCrr)rrrrrr num_reads,rzIRNode.num_readsrvcCrrrrrrrget_storage_numel/rzIRNode.get_storage_numelrsrcCrrrrrsrrrget_free_symbol_uses2r zIRNode.get_free_symbol_usescCrrrrrrrget_reduction_type7rzIRNode.get_reduction_typecCrrrrrrrget_reduction_size:rzIRNode.get_reduction_sizecCrrrrrrr is_extern=r zIRNode.is_externcCrrrrrrris_no_op@r zIRNode.is_no_oprcCrrrrrrrconstant_to_deviceCrzIRNode.constant_to_devicecCrrrrrrrget_mutation_namesFrzIRNode.get_mutation_namescCrrrrrrrget_operation_nameIrzIRNode.get_operation_namecCrrrrrrrget_inputs_that_alias_outputLrz#IRNode.get_inputs_that_alias_outputcCrrrrrrrrQr z IRNode.dtype)rzrrrrrrr)rrr%r rrrrrr)rr{rr}rrT)rrrr)TT)rrrrrrrrrr)rrrr)rrrr.)rrrrrr)rr)rrrr rr+rrrrrrrrrrrrrrrFrrrrrrrrrrrrFrrrrrrrrrrrrFrrrrFrrrrrrrrrrrrvrsrrrrrrrrr)Krrr__doc__r9rwr dataclassesfieldrzr|r~ staticmethod contextlibcontextmanagerrrrrrrrrrrrrrrrrrrrrpropertyrHrrrrrrrrrr>rArrrrrrrrr r rrrrrrrrrr r!r"r#r$r%rrrrrrrs                                                   c@seZdZd0ddZd1ddZd2d d Zd3d d Zd4ddZd5ddZd5ddZ d6ddZ d7ddZ d8ddZ d9d d!Z d:d#d$Zd;d&d'Z (dd-d.Zd/S)? OperationrrcCs d|_dSroperation_namerrrrrWrzOperation.__post_init__rcCtrrrrrrrZr zOperation.get_devicer}cCt|dsJ|jSNr~)rr~rrrrr]zOperation.get_origin_noderycCrU)Nrz)rrzrrrr get_originsarWzOperation.get_originsrcCs|jdusJ|jSrrQrrrrr$erWzOperation.get_operation_namercCrrrrrrrr ir zOperation.is_externcCrrrrrrrr!lr zOperation.is_no_oprcCrSrrTrrrrror zOperation.get_read_writesrcCs ||vSr)r)rrrrr is_user_ofrrzOperation.is_user_ofrcCr)Ncsrrrrrrrrvrz+Operation.get_read_names..rrrrrrurzOperation.get_read_namesrcCrrrrrrrrxrzOperation.get_reads list[Buffer]cCrSrrTrrrr get_outputs{r zOperation.get_outputsrcCtSrr8rrrrget_unbacked_symbol_defs~rz"Operation.get_unbacked_symbol_defsFrscCr\)a When unbacked_only=True: Returns the unbacked symbols which are required to be in scope in order to successfully perform codegen for this buffer. For example, a buffer that corresponds to an extern kernel call that takes i0 as an argument would return {i0} here. This is used to generate necessary dependencies that ensure we actually bind i0 in codegen before you try to use it. Note that this is NOT transitive; in particular, if this buffer takes in as input another buffer with dynamic shape (e.g., (i0,)), we will not report it here, because you will already have a dependency on that buffer, which will eventually have a dependency on i0 if necessary. When unbacked_only=False: Similar to `unbacked_only=True` but including all free symbols instead of only free unbacked symbols. r8rrrrrszOperation.get_free_symbol_usesrcCr)z Gets extra global memory size needed by this buffer. Some algorithms (e.g. group gemm) may require extra global memory in the generated code. rrrrrrget_workspace_sizeszOperation.get_workspace_sizeNr'r4r))rryr9r/rB)rrrrr(rCrrZrrr=rFrD)rrrrrrrXr$r r!rrYrrr[r]rr^rrrrrPUs"              rPcs*eZdZUded<ded<ded<ded< dPdQddZdRddZdSfdd ZdTddZeZdUddZ dVddZ dWd!d"Z dWd#d$Z e dXd)d*ZeejfdYd-d.ZedZd0d1Zd[d3d4ZedTd5d6Zd\d]d:d;ZdPd^d=d>Zd_d@dAZd`dCdDZdadFdGZdWdHdIZdbdKdLZdcdNdOZZS)dLoopsrrrrCallable[..., Any]inner_fnrFrangesFrsrrrcs,tjgfdd|jD|RS)Nc3|]}t|VqdSrr'rerrrrrz-Loops.get_free_symbol_uses..)r9unionrdinner_fn_free_symbolsrrrrrrs zLoops.get_free_symbol_usesnamesrrcsFdjjdtjgfdd|DdjgS)N'cs g|] }|dt|qS=)rrrrrrrs z!Loops._to_str.. origin_node=)rrrrr inner_fn_strr~)rrkrrr_to_strs z Loops._to_strrcstdSr)superrr __class__rrrrzLoops.__post_init__cC |dS)Nrdrrrrrr__str__rz Loops.__str__rcCrrrrrrrrrzLoops.get_devicer}cCrrrrrrrrrzLoops.get_origin_nodercCrrrwrrrrrrzLoops.get_sizecCrrrwrrrrget_pointwise_sizerzLoops.get_pointwise_sizerr r'Union[TensorBox, ShapeAsConstantBuffer]cOsN|dd}|dd}||i|}|d||d|p |jt|S)Nr~r|)poprr|rcreate)clsrrr~tbrrrrr~s    z Loops.createrzr=csfddt|DS)Ncs*g|]\}}|dkrtjjnt|qSr>)rSZerord)rnrryrrrz Loops._index..)r)rdrzrryr_indexs z Loops._indexrMc Cstt}t|2ttdd|j|| WdWdS1s0wYWddS1s@wYdSNallow_indexingT) rLrj MockHandlerset_ops_handlerrrrrc inner_fn_argsgetvalue)r opcounterrrrinner_fn_opcounts  RzLoops.inner_fn_opcountSequence[Sequence[_IntLike]]cCs||jfSr)rrdrrrrrrzLoops.inner_fn_argscCstjj|jg|RSr)rjKernelFormatterHandler ir_to_stringrcrrrrrrqs zLoops.inner_fn_strNrrcCs&|durd}t|tj}|j|kSr)maxr?realize_opcount_thresholdrnum_opsrrrrrs zLoops.has_large_inner_fnOrderedSet[Symbol]cCs||j}t|j||dSNrr)rrdrGrc)rrsrrrrrjs zLoops.inner_fn_free_symbolsrcCsvttdd*|r t|||jWdSt||jWdS1s4wYdSr) rrrrrIrrrrrrrrrs$zLoops.get_readsrcCt|jSr)r9r read_buffersrrrrrrzLoops.get_read_namesrcCrr)rrrrrrrrrzLoops.num_readscCr)Nz+get_reduction_size() is not implemented by rrrrrrrzLoops.get_reduction_sizer+cCr)Nz+get_reduction_type() is not implemented by rrrrrrrrzLoops.get_reduction_typercCr)Nz+constant_to_device() is not implemented by rrrrrrr"rzLoops.constant_to_devicer=rF)rkrrrr'r9r4r)r0)rr rr rr|)rdrFrzr=rr)rrMrrrr:rsrrrrCr(rDr2rG) rrrrrrrrry__repr__rrrr{ classmethodr~rLr=INDEXrrUrrrqrrjrrrrrr" __classcell__rrrtrras@                   rarUnion[Expr, Sequence[Expr]]rrricCs"|jr ttd|Std|S)Nnanr)is_floating_pointrhconstantfloat)rrrrr nop_loader_fn%s rc@s>eZdZdddZdddZdd d ZdddZdddZdS) PointwiserrcCs|r tt|jdS|jSNr)rr rrrcrrrrr.szPointwise.make_loaderSequence[sympy.Expr]cCsgSrrrrrrr5r zPointwise.get_reduction_sizer+cCrrrrrrrr8r zPointwise.get_reduction_type output_nameindexer!Callable[[Sequence[Expr]], Never]varsrrcCs"|}t|p d||||SNunnamed)rrhstorerrrrloaderrrr store_output;szPointwise.store_outputrrrcCs.|}ttd||}t||j||jdSFMove this to a given device. Requires that all reads are to constants.override_devicerrrcrd)rrrConstantBufferrrrdrrrrrrr"DszPointwise.constant_to_deviceNr6)rrr2)rr+rrrrrrrG)rrrrrrrr"rrrrr,s      rc@s6eZdZUded<dZded<dd d ZdddZdS)Scatterroutput_indexerNrO scatter_moderrrrcCs6|}ttd||}t||j||j|j|jdS)rr)rrrcrdrr) rrrrrrrdrrrrrrr"UszScatter.constant_to_devicerr+rrrrr cCs6|}|dur d}tj|||||||jdS)Nr)mode)rrhrrrrrrrrbs zScatter.store_outputrG)rr+rrrrrr )rrrrrr"rrrrrrPs    r logical_ormaximumminimummulr bitwise_xor)anyrminprodsumxor_sumz"dict[str, Callable[..., OpsValue]]REDUCTION_COMBINE_FNreduction_typearg_break_ties_leftCallable[..., object]csRtvrtSdvrdfdd }|Sd kr"dd d }|Std )Nargmaxargminatuple[object, object]brtuple[OpsValue, OpsValue]c s|\}}|\}}dkrt||}nt||}t||}trCt||}t||} t|t|| }t|t|| }rKt||nt||} t|t|| }t|||t|||fS)Nr) rhltgteqr+ner logical_andwhere) rra_valuea_indexb_valueb_indexmaskequala_isnanb_isnantierrrrrargmax_combine_fns&       z3get_reduction_combine_fn..argmax_combine_fnwelford_combine#tuple[OpsValue, OpsValue, OpsValue]c SsR|\}}}|\}}}||}||} || } ||| |||||| | fSrr) rra_meana_m2a_weightb_meanb_m2b_weightdelta new_weight w2_over_wrrrwelford_combine_fns   z4get_reduction_combine_fn..welford_combine_fnzunknown reduction_type=)rrrrrr)rrrrrr)rr)rrrrrrrrget_reduction_combine_fn}s rcsTeZdZUded<ded<ded<ded<dkd d ZeZdldmfdd ZdnddZdoddZdpd d!Z dqd#d$Z drd&d'Z dldmd(d)Z dsd-d.Z e /dtdud:d;Zedvd>d?Zeejd/fdwdBdCZedxdFdGZedxdHdIZedydMdNZedzdPdQZe /dtd{dWdXZed|d`daZed}dedfZe /dtd~dgdhZeddidjZZS) ReductionrFreduction_rangesrNrr src_dtyperRreduction_hintrrcCrv)N)rdrrrxrrrrryrzReduction.__str__Frsrrcs(ttjfdd|jDBS)Nc3rerr'rfrrrrrrhz1Reduction.get_free_symbol_uses..)rsrr9rirrrtrrrrszReduction.get_free_symbol_usesrcCrr)rrrrrrrzReduction.get_reduction_sizer+cCrr)rrrrrrrzReduction.get_reduction_typerrrrreduction_varsSequence[Symbol]rc Cs8t|j|j|j|||}t|pd|||dSr)rh reductionrrrrcstore_reduction)rrrrrr%rrrrs zReduction.store_reductionrcCt|jt|jSr)rrdrrrrr index_lengthrzReduction.index_lengthSequence[Sequence[Expr]]cCs$||j}||jtj}||fSr)rrdrr=R0_INDEX)rrrindexrrrrs zReduction.inner_fn_argscCs.||j}||jtj}t|j|||dSr)rrdrr=rrGrc)rrsrrrrrrjs  zReduction.inner_fn_free_symbolsrrrc Cs>|}ttd||}t||j||j|j|j|j t j dS)rrrrrcrdrrrr) rrrrrrrdrrrrRDEFAULTrrrrr"szReduction.constant_to_deviceN dst_dtypercCallable[_P, OpsValue]rd%Union[ReductionType, Literal['scan']]reduction_numelr input_noder tuple[ReductionHint, _IntLike]c " Cstjj|} tjjt|} |dkp#tj|tj o#|dvo#tj } t | r,t | s1t j dfSt |} | j} d}| rRtjtjj|dd}tjtjj|dd}ndd d }|}| dkr|| | }|dkrkt j|fS|durt|trttddt|\}}Wdn1swY|dur|durtjjt||}| |krtd|||||t jdfSt j|fS| |ks| | ddkrt j dfSt||||||dkr|nd|t j d}ddd}||\}}|r||\}}t|dkrt j dfSt |!|"\\}}}d}d}|D].}tjj#||}tjj$||t%|&} t'dd| D}!|!r;|d7}q|d7}q||krNt j|| | fSt j(|| | fS)Nscanrr> T)inner_reductionFreduction_numel_hintr numel_hintrcSrNr>r)r r rrrinner_reduction_splits)z4Reduction.num_splits..inner_reduction_splitsrzUse previous IRNode's range and reduction_ranges instead of split. current ranges: %s, current reduction ranges: %s, current split: %d, new ranges: %s, new reduction ranges: %sr@rrrrrtuple[Sequence[Expr], bool]c s|}|dus Jtdt|||d|d}|}|jdus%Jdd|jD}g}d}t|jdddD]7t fd d |Drq| j j t jjvrqt jjj }t|jd d}|t|jd d|krqd }q:||fS) NrrrrrYrXcSs&g|]}t|trt|tjs|qSr)rr rNumberrrrrrrrs zBReduction.num_splits..get_read_indices..FcSrrrrrrryz@Reduction.num_splits..get_read_indices..keyc3s|] }|jjvVqdSr)rr1rmdrrrzzAReduction.num_splits..get_read_indices..rT)rrrrrr range_varssortedrrrrrrjrname_to_bufferrrY decide_layout) rrcb read_writesrindiceschangedbuforiginal_striderrrget_read_indicesas:   z.Reduction.num_splits..get_read_indicesrcss|]}|dkVqdSr>Nrrrrrrz'Reduction.num_splits..)r rr rrr)rrrr))rjrrrKre has_featurerAREDUCE_TO_SINGLE_ELEMENTr?split_reductionsrrRrrQr~multi_processor_count functoolsr choicesreduction_split_factorINNERrrrrrrHlogrrrr@index_vars_squeezerrsimplify_with_ranges stride_hintsrkeysrOUTER)"rrrrcrdrrrrr r  should_splitpropsnum_smmin_elements_per_threadr outer_reduction_splitsr new_rangesnew_reduction_rangesextracted_numel_hintrr$r r!rrranges1 num_outer num_innerrjrQouterrrr num_splitss            #        zReduction.num_splits.fn..cSsg|]}t|qSr)rrrrrrrrz>Reduction._unroll_reduction_fn..fn..)r+reduce itertoolsproductr) combine_fnrrErrrs  z*Reduction._unroll_reduction_fn..fnrrrrcs*dd|D}||t|tjfS)NcSg|]}t|qSrrexpandrrrrrrzDReduction._unroll_reduction_fn..value_fn..)rh index_exprr!int64rr) flatten_indexrcrrrEsz0Reduction._unroll_reduction_fn..value_fncs |dSr rr)rrrrs z0Reduction._unroll_reduction_fn..N)rrFrr )rrFrrFrr)rjrr guard_int_seqr_fixed_indexerrr)rcrrrr)rJrRrrcrrEr_unroll_reduction_fns   zReduction._unroll_reduction_fnrbr|c  s$tjjtdkrDdfdd } | d| d| d| dd vs0Jd dfdd } tj||| t|dSdkredvrTdfdd } ndfdd } tj|| |dSt t rtjj t j krt|dkst|jrtj||||dS||||| \} }dfdd }||}|tjkr| }|dkr| dusJttddt| \}}Wdn1swY|dusJ|dusJ||||||| S|dkr||||||| Stt||||dS)NrvalrrUnion[bool, float, int]csTtjkr t|Sjrt|tsJt|t|St|ts&Jt|t|Sr) r!rrrrrrrrrVrrrpy_cnsts z!Reduction.create..py_cnstr>)rrrrz* not supported for zero-dimension tensors!rrricstSrrhrr)rrrtypes_to_initsrrconst_fnz"Reduction.create..const_fnrrKcs tdSrr[rrYrrrrzReduction.create..fncddD}||S)NcSg|]}tjjqSrrrrrrrrr rz0Reduction.create..fn..rrreduction_index)rcrrrr  rcs$tr|S|dkrt|tjS|Sr )rrr? min_num_split)r)rrr_maybe_increase_split2s  z/Reduction.create.._maybe_increase_splitr@rTr)rVrrrW)rrrri)rrrr)rjrrsimplifyrer3rr~rrr!r=r?unroll_reductions_thresholdrarrUrBrRrrrrrH!create_multilayer_existing_rangescreate_multilayerrr)rrrrrcrdrrrrrZr]rhintrrfr:r;r)rrcrrrr\rr~s            zReduction.creater#Union[_NumLike, Sequence[_NumLike]]cCs|dvrt|r tdSt|rdSt|jS|dvr0t|r$tdSt|r*dSt|jSt|r6dnd}t|r>dnd}|||||||f|||ftd|fd |S) N)rrz-infF)rrinfTrr>)rrrrwelford_reduceronline_softmax_reduce)r+rr*r!iinforr)rrzeroonerrrdefault_accumulatorrs0   zReduction.default_accumulatorcCs|dkrdSt||S)Nrnr)rrsrrrrr default_values zReduction.default_valuerrvr cCsP|dkr|S|dkr|dkr|tjkrtjS|dkr&|dkr&|tjkr&tjS|S)Nr@i)rRr4 OUTER_TINY)rr rrrr_multilayer_second_step_hints z&Reduction._multilayer_second_step_hintrcCs|durdStjj||sdS|zt|Wn ty&YdSw|}t |ddD]\}}tjj|drC|Sq3dS)z If we are reducing over the full tensor, and it is non-dense in the last dimension, reindex so we reduce over the dense dimension. initially just handle complete reduction case Nr@r>) rjrrrJrrrZrr>r)rrrrQrrrrr$check_for_split_dense_dim_reindexings$   z.Reduction.check_for_split_dense_dim_reindexingrr block_sizedefaultrc sT||}t|g|tjjt|d d fdd } | S) Nrrrrcrricsl|\}|^}||dfdd }r3t}tt|t|}t||S|S)NrricsgSrrr)r r new_indexrrrbodyr^zCReduction._multilayer_wrap_loader..wrapper_fn..body)rri)r[rhrrOmasked)rrcreduction_blockr~ index_dtyperr{r|r need_maskrr)r r}r wrapper_fns    z5Reduction._multilayer_wrap_loader..wrapper_fn)rrrcrrri) rzViewdynamic_reshape_indexerrjrrrrr) rrrrrr{r|r dense_indexrrrr_multilayer_wrap_loaders z!Reduction._multilayer_wrap_loader4Callable[[Sequence[Expr], Sequence[Expr]], OpsValue]original_rangesoriginal_reduction_rangesr:Sequence[Integer]r;@Callable[[Sequence[sympy.Expr], Sequence[sympy.Expr]], OpsValue]csNtddDsJdt|t|t|d fd d }|S) Ncss|]}|dkVqdSr%rrrrrrr&zDReduction._multilayer_wrap_loader_existing_ranges..z8Only enabled for numel_hint == 1, found original_ranges= merged_indexrnew_reduction_indexrrics:|dt}|td}|t|t|Sr)rr)rr original_idxr}rrrrrrs zEReduction._multilayer_wrap_loader_existing_ranges..wrapper_fn)rrrrrri)rrrr)rrrrr:r;rrrr'_multilayer_wrap_loader_existing_rangess  z1Reduction._multilayer_wrap_loader_existing_rangesrrq list[Integer]c  s|tjtjfvr |ntj} t|| ||||| | } | | d fdd }tj j t |}| | || } ||dt|ksDJtt|||||t|d| || d S) a Break a large reduction up into multiple smaller reductions recursively rrFrcrricsg||Srrrbintermediate_loaderrrintermediate_fn:sz;Reduction.create_multilayer_helper..intermediate_fnNr)rrFrcrFrri)r!float16bfloat16rrr~rrrjrrrreryrr)rrrrrrrr:r;rrrintermediate_dtype intermediaterr rrrcreate_multilayer_helpersD z"Reduction.create_multilayer_helperc  Csdt|} t| |d|} |||} |||| || | | }|||||||g||| g||| S)rr>)rer;rurr)rrrrrcrdrrrrrrr{r|rrrrrjRs2   zReduction.create_multilayerc Cs8||||||} ||||| ||g|||| d| S)rr@)rr) rrrrrcrrr:r;rrrrrrris( z+Reduction.create_multilayer_existing_rangesr9r=rr0r2) rr+rrrrrrrrrDrrrGr)rrrrrrrcrrdrFrrFrrrr rr rr) rcrCrrFrrrrrrD)rrrrrrrcrbrdrrrrrNrrRrr rr|rrrrrrl)rrvr rrrRrrR)rrvrr rr)rrrrFrrvrrvr{rvr|rlrr rr) rrrrrrr:rr;rrr)rrrrrrrrbrrrrr:rqr;rrrNrrvrrRrr|)rrrrrrrcrbrdrrrrrNrrvrrRrr rr|)rrrrrrrcrbrrrrr:rr;rrrNrrRrr|)rrrrryrrrrrrrrjr"rLrBrUrrRrr~rsruryrzrrrrjrirrrrtrrs\          # + "     *  ? -rrrOptional[Sequence[int]]r\r rcsdfdd }|S) 1A closure containing math to read a given elementrrrrcsddur t|tksJt|tksJ}t|D]\}}}|dkr/|||}q |Sr )rr)rresultrstszr\rrrrrs z_fixed_indexer..indexerN)rrrrr)rrr\rrrrrTs rT INNER_FN_TYcs2eZdZUded<dfdd Zd ddZZS)!MultiOutputReductionr output_indexrrrr inner_fns)Union[INNER_FN_TY, Sequence[INNER_FN_TY]]rdrrrrNrrrRc sXtrftdkrd} nd fdd } tj||| |||||d | |_dS) Nr>rrr reduction_idxrtuple[OpsValue, ...]cstfddDS)Nc3s|]}|VqdSrrrrrrrrrrhz@MultiOutputReduction.__init__..loader..)rrrrrrsz-MultiOutputReduction.__init__..loaderr)rrrrrr)callablerrs__init__r) rrrrrdrrrrrrrtrrrs     zMultiOutputReduction.__init__rr+rrrrrrrr c CsXt|j|j|j|||}t|ttfsJt |||j }t |p&d|||Sr) rhrrrrrcrrrrrr)rrrrrrr%rrrrs  z$MultiOutputReduction.store_reduction)rrrrrrrdrrrrrNrrrrRrr) rr+rrrrrrrr )rrrrrrrrrrtrrs %rc@s"eZdZeejdfdddZdS)OnlineSoftmaxReductionNrrrrrrcrbrdrr num_outputrrrRrr r1Sequence[Union[TensorBox, ShapeAsConstantBuffer]]c s<tfddt|D} | D]} | q| S)z> Create the reduction disregarding splitting. c 3s.|]}ttd| VqdS)roN)rr~rr output_idxrrrcrdrrrrrr s" z0OnlineSoftmaxReduction.create..)rrr) rrrrrcrdrrrrresultsr$rrrr~s  zOnlineSoftmaxReduction.create)rrrrrrrcrbrdrrrrrrrRrr rr)rrrrrRrr~rrrrrs  rc@s<eZdZeejfdddZedddZedddZ dS)WelfordReductionrrrrrSequence[Callable[..., Any]]rdrrrrNrrRrrc s6dvsJtjjt}dfdd } |dkr-| d} | d} | d} | | | fS|d krWdfd d dkrLd| d| d fStfddDStjd|d\} }tj kro| |d kr| |Sfddt dD}|D]}| q|S)N)rnrrVrrr|cs&dfdd }tj|tdS) Nrrrrics tSrr[r)rrVrrrc/sz8WelfordReduction.create..const..inner_fnrrrrrirr~r)rVrc)rrrdrXrconst.sz&WelfordReduction.create..constrr>rrcs&dfdd }tj|tdS) Nrrrricr_)NcSr`rrarrrrrHrzKWelfordReduction.create..copy..inner_fn..r)rrc)rrrrrcGrdz7WelfordReduction.create..copy..inner_fnrrr)rrc)rrrdrrrcopyDsz%WelfordReduction.create..copyrnc3|]}|VqdSrrr)rrrrUr&z*WelfordReduction.create..)rrcs*g|]}tt| qSr)rr~rr)rrrrdrrrrrrs z+WelfordReduction.create..r)rVrrr|)rrrr|) rjrrrgrerrrBrRrrjrr)rrrrrdrrrrrmeanm2weightrkrrr$r)rrrrrdrrrrr~sT     zWelfordReduction.createrrlcCr)N)rrrrrtrrrrur zWelfordReduction.default_valuerrvc  s$ttjjtd } | r9|dkr9dfd d } j||dt| dd t| d d f|d|dSt d t |t fdd|Dg|g||} | D]} | qadddtjjt|} | |}t |t fdd| D|gd|S)rrrrrrr%rrrics t|Srr[)rrr%rrrr z4WelfordReduction.create_multilayer..constantr(r>)rrrrdrrrrc 3s&|]}j|ddVqdS)r)r|N)r)rr)r{rrrrrrrs  z5WelfordReduction.create_multilayer..rrcrrcSs|g||Srr)rrcrrrrintermediate_loader_fnszBWelfordReduction.create_multilayer..intermediate_loader_fnc3s |] }t|dVqdS)rN)r rr)rrrrs  N)rrrrr%rrri)rrrcrrrrri)rerjrrrrrrjr r;rr~rrrry)rrrrrdrrrrrr intermediatesrr r)r{rrrrrrrrjsb        z"WelfordReduction.create_multilayerN)rrrrrrrdrrrrrNrrRrrr)rrrrrrrdrrrrrNrrvrrRrr) rrrrrRrr~rLrurjrrrrrs x rcseZdZUded<ded<ded<ded<ded <d ed <d ed <ded<dIdJfdd ZdKfdd ZdLd#d$ZdMd%d&ZdNd'd(ZdNd)d*Z dNd+d,Z dOd-d.Z dPd0d1Z dIdJd2d3Z eejfd4d5dQd=d>ZedRdGdHZZS)SScanr scan_rangesr=Callable[[tuple[Any, ...], tuple[Any, ...]], tuple[Any, ...]]rJzFCallable[[Sequence[_IntLike], Sequence[_IntLike]], Sequence[_IntLike]]rrRrrrtuple[torch.dtype, ...]dtypestuple[Callable[..., Any], ...]rFrsrrrcDttjfdd|jDBtjfdd|jDBS)Nc3rerr'rfrrrrr rhz,Scan.get_free_symbol_uses..c3rerr'rfrrrrr rh)rsrr9rirrrrtrrrr s zScan.get_free_symbol_usesrc0t|jt|jt|jksJtdSr)rrdrrrsrrrtrrr "zScan.__post_init__rr+r%Callable[[Sequence[_IntLike]], Never]rr scan_varsrr csR|||tfdd|jD}t|j|j|}t|p d|||jS)Nc3|]}|VqdSrrrrcrrrr$ r&z'Scan.store_reduction..r) rrrrhrrrJrr)rrrrrrrrrrr s zScan.store_reductioncCr)Ncustomrrrrrr* zScan.get_reduction_typecCrr)rrrrrr. rzScan.get_reduction_sizecCrrrrrrrr1 rz Scan.get_sizecCrrrwrrrrr{4 rzScan.get_pointwise_sizecCrr)rrdrrrrrr7 rzScan.index_lengthrcC.||j}||jtj}|||}|fSr)rrdrr=rrrrrrrrrr:   zScan.inner_fn_argscC8||j}||jtj}|||}t|j||dSr)rrdrr=rrrGrcrrsrrrrrrrj@   zScan.inner_fn_free_symbolsT)can_fallback_to_atenrr+tuple[Callable[[Sequence[Expr]], Any], ...]axisrr;Sequence[Optional[Union[TensorBox, ShapeAsConstantBuffer]]]c sg d dd g tjtjs$dgtStdkr9tjtjs9dgtStjj} | t } ttksNJ| t | drg fddt tDS|jdd | d\} t | dkrtjjduptotdkotdk} | s|rdgtSd} nt d fd d  f ddt tD}|D]}|q|S)Nr>c&g|]}tj||dqSrrr~rrrrrrrrrg zScan.create..r)rrrcrpointwise_rangesrrJ scan_numelz3.3.0rr scan_indexrrqcHt|tks Jt|tksJg|d||dSrr)rr)rrrrrr  zScan.create..reindexcsBg|]}t d|| |d qS)) rrrrcrrrdrrJrrrr)rr~r) rJrrrrrrrr scan_typerrrr s* )rrrrrrq)rjrr'rASCANrTUPLE_REDUCTIONrrgrerrLerrBrr!versionhip has_tritontriton_version SplitScanr)rrrrrrrJrrrrrrBsupports_splitrrr) rrJrrrrrrrrrrrr~F sV         z Scan.createrrrcrrrr rc s*d fdd } tj|||| ||d|d S) Nrrrrrics$g|d||dSrrrrrcrrr s$z#Scan.num_splits..wrapper_fnr)rrrrcrdrrr)rrrrrri)rrB) rrrrcrrrrJrrrrrrB s zScan.num_splitsr=rr') rr+rrrrrrrr r2r0rDr)rrrrrrrrrrrJrrrRrrrr rr)rrrrrcrrrrrrrrJrrr rr)rrrrrrrrrrr{rrrjrrRrr~rBrrrrtrrs4           arc@s eZdZdS)rN)rrrrrrrr srcseZdZUded<ded<ded<ded<ded <d ed <d ed <ded<ded<d;d<fdd Zd=fdd Zd>d"d#Zd?d$d%Zd@d&d'Zd@d(d)Z d@d*d+Z dAd,d-Z dBd/d0Z d;d.c3rerr'rfrrrrr rh)rsrr9rirrrrtrrrr s zSort.get_free_symbol_usesrcrr)rrdrrrsrrrtrrr rzSort.__post_init__rr+rrrrrr csV|||tfdd|jD}t|j||j|j}t|p"d|||j S)Nc3rrrrrrrr r&z'Sort.store_reduction..r) rrrrhsortrrrrr)rrrrrrrrrrr s zSort.store_reductioncCr)Nrrrrrrr r zSort.get_reduction_typecCrr)rrrrrr rzSort.get_reduction_sizecCrrrrrrrr rz Sort.get_sizecCrrrwrrrrr{ rzSort.get_pointwise_sizecCrr)rrdrrrrrr rzSort.index_lengthrcCrr)rrdrr=rrrrrrr rzSort.inner_fn_argscCrr)rrdrr=rrrGrcrrrrrj rzSort.inner_fn_free_symbolsrr'tuple[Callable[[list[Expr]], Any], ...]rrrc  s*g d dd g tjtjs$dgtStjj} | t } d} t j j o=| t | | } | sGdgtSttksQJ| t | drj fddttDSd  fd d  f d dttD}|D]}|q|S)Nr>rvcrrrrrrrr8 rzSort.create..rr sort_indexrrqcrrr)rr)rrrrrrB rzSort.create..reindexcsDg|]}ttd|| | d qS)) rrrrcrrrdrrrrrrr)rr~rr) rrrrrrrrrrrrrrG s,)rrrrrrq)rjrr'rASORTrrrgrer?r4persistent_reductionsrrrrr)rrrrrrrrrrr sort_numel max_rblockis_persistent_kernelrrr) rrrrrrrrrrrrrr~ s0     z Sort.creater=rr') rr+rrrrrrrr r2r0rDr)rrrrrrrrrrrrrrrrRrr rr)rrrrrrrrrrr{rrrjrrRrr~rrrrtrr s.          rcCs(z t|ddWdStyYdSw)NFfreezeT)rZrrrrrrc s   rcCs@zt|dd\}}|r||WStyYdSwNFr)rZshould_pad_strides pad_strides is_contiguousr)r_bufferrYrrr is_contiguous_storage_and_layoutk s  r rwant_contiguous stride_order'Optional[Sequence[Union[int, Integer]]]rrtuple[StorageBox, Layout]c Cst|trt|j|||||dSt|tr)t|j|||||d\}}||jfSt|trc|r[|r?||s>Jn|durK|j ||dn|durW|j ||dn| t||fSt|t rvt|j|d\}}||j fSt)z Try to simplify x into a StorageBox and a Layout. allow_padding only affect how we apply stride_order. When allow_padding is True, we have the freedom to add padding when applying the stride_order. rr r rrNrr)rrrZrX StorageBoxrBufferrr r rrr^rYr) rrr r rrrrYbufferrrrrZw sR      rZcCs2zt|dd\}}||WStyYdSwr)rZis_stride_orderedr)rr r rYrrr"is_stride_order_storage_and_layout s   rrcCsrt|ttfr t|jSt|tr*|j}tjj |j t |j t }t|jp)|St|tr7|tjjvSdSr)rrr is_unalignedrXr^rYrjrrstatically_known_multiple_ofr\r\rr^rrunaligned_buffers)rrYhas_unaligned_layoutrrrr s   rc@seZdZUded<dEdFdd ZdGd d ZdHddZdIddZedJddZ dKddZ dLddZ dMddZ dNd d!Z dOd#d$ZdPd(d)ZdQd*d+ZdRd-d.ZdSd/d0ZdTd2d3ZdQd4d5ZdQd6d7ZdUd9d:ZdVdd?ZdXdBdCZdDS)YBaseViewrrXFrsrrrcC |j|SrrXrrrrrr rzBaseView.get_free_symbol_uses*Callable[[Sequence[Expr]], Sequence[Expr]]cCstd|)Nzmake_reindexer NYI on rTrrrrmake_reindexer rzBaseView.make_reindexerrc&|j|dfdd }|S)Nrrrr crrrrinnerrrrr rz&BaseView.make_indexer..indexer)rrrr )rXrrrrrr!rr  zBaseView.make_indexerrcr )Nrrrricrrrrr!rrr rz$BaseView.make_loader..loaderr)rXrrrrrr!rr r$zBaseView.make_loaderrcC |jSr)rXrrrrrr  zBaseView.dtypercCr&rrXrrrrrr rzBaseView.get_layoutrcCr&rrXrrrrrr rzBaseView.get_devicer}cCrrrrrrrr r zBaseView.get_origin_nodercCr&rrXrrrrrr rzBaseView.get_namercCrrrrrrrr{ rzBaseView.get_pointwise_sizerrrcCrrrXrrrrrr rzBaseView.mark_reusecCr&rrXrrrrrr rzBaseView.has_exceeded_max_readsr+cCr&rrXrrrrrr rzBaseView.realizecCs|jdSrrXrrrrrr rzBaseView.realize_hintrvcCr&rrXrrrrrr rzBaseView.get_storage_numelcCr&rrXr rrrrr  rzBaseView.is_externcCs$t|jts Jt|j|jSr)rrXrris_module_bufferrrrrr1 s zBaseView.is_module_bufferrcCr&rrXrrrrrr rzBaseView.get_read_namesrcCsFttddt||jWdS1swYdSr)rrrrIrrrrrrrr s$zBaseView.get_readscCs"|}t|tr|j}t|ts|Sr)rrrX)rrrrrr s   zBaseView.unwrap_viewrrcCs2|}ttd||}t||||dSr)rrrrrrrrrrrr"$ szBaseView.constant_to_deviceNr=rrrr7r6r,r-r4r)r9r0r;r/r2r'rEr(rCr<rG)rrrrrrrrrOrrrrrr{rrrrrr r1rrrr"rrrrr s0                     rc@sFeZdZUded<eddd Zedd d Zdd dZdddZ dS)rrrrrnew_sizerFrcCstjj}dd|D}|}dgt|t|t|}t|t|ks(Jtt|D]:}||dkrE||dus>J||||<q.||dusTtjj||rUq.|j||||dddkshJdq.|S)zReplace `-1` with correct sizescSrLrrMrrrrr8 rz.ExpandView._normalize_size..Nr@rfallbackzKBroadcast failed in ExpandView({x.get_size()}, {new_size}) on dimension {i}) rjrrrrrris_size_one_or_falser)rr4rold_sizerrrr_normalize_size4 s"  zExpandView._normalize_sizerc Cs|||}t|rWt|\}}t|t|j}|dksJtjjg|}t|j |jD]\}}| t j j |s<|ntjjq-t|j|jt|||j|j} t|| dSt||dS)NrrW)rXr)r9rrZrrrrrrrrrjrrr7r[rrrr\r]r^r) rrr4r_r`skiprarrrbrrrr~O s,      zExpandView.createcCrrrrrrrrj rzExpandView.get_sizercs6|}|jt|tdfdd }|S)NrrrcsRt|d}t|tksJttD]}|dkr&tjj||<q|Sr )rrrrrr)rractualr:rrrt s  z*ExpandView.make_reindexer..reindexrrrr)rrXr)rtargetrrr;rrm s   zExpandView.make_reindexerN)rrr4rFrrF)rrr4rFrrr0r3) rrrrrLr9rr~rrrrrrr0 s    rc@sFeZdZUded<eddd Zedd d ZdddZdddZdS) PermuteViewrqdimsrrrrrcs||}t|ttt|ksJt|r>t|\}tjjfdd|Dfdd|Dj j }t ||dSt ||dS)Ncg|]}j|qSrrrr`rrr rz&PermuteView.create..crArrrrBrrr rrW)rXr@) _map_neg_dimsr9rrrrZr[rrr\r]r^r?)rrr@r_rbrrBrr~ s    zPermuteView.create list[int]csfddDS)Ncs$g|]}|dkr |nt|qSrr)rrLr@rrr s$z-PermuteView._map_neg_dims..r)rr@rrFrrD zPermuteView._map_neg_dimsrcsDt||jttt|jksJ|jfdd|jDS)Ncrrrrrrrr rz(PermuteView.get_size..)r9rDr@rrrXrrrrrr s   zPermuteView.get_sizercs`ddt|jDfddtt|jDtttt|jks'Jd fdd }|S) NcSrrr)rrr@rrrr rz.PermuteView.make_reindexer..crrrrinvrrr rz.PermuteView.make_reindexer..rrrcsfddDS)Ncrrrrrrrr rz?PermuteView.make_reindexer..reindex..rrrHrrr z+PermuteView.make_reindexer..reindexr=)rr@rrr9)rrrrHrr s zPermuteView.make_reindexerN)rrr@rrr)r@rrrEr0r3) rrrrrr~rDrrrrrrr? s    r?c@s8eZdZeddddd Zedd dZdddZdS) SqueezeViewNrLrrrLrrc sFt|rwt|\}}g}g}dur*ttsJtdkr(t|jks*Jtt|j|j D]0\}\}} durL|dkrK| || | q3|kr[| || | q3|dkscJdq3t |j |j |||j|j} t|| dSdurt|dd|DS|dksJt|fddt|DS)Nrr>zexpected squeezed size to be 1rWcSg|]}|dkr|qSrrrrrrr rz&SqueezeView.create..csg|] \}}|kr|qSrrrrrrLrrr )rrZrrrrrrrrrr[rrr\r]r^rr~r) rrrLr_r`r4rarrrrbrrLrr~ s<      "zSqueezeView.createrr9tuple[list[int], Callable[[Sequence[Expr]], tuple[Expr]]]cs@dd|D}ddt|Dt|d fdd }||fS) NcSrMrrrrrrr rz(SqueezeView.squeezer..cSsg|] \}}|dkr|qSrrrNrrrr rOrrr tuple[Expr]csTt|tksJ|dtjjg}t|D]\}}|||<qt|S)N )rrrrrr)rr}rrlengthnot_onerrr s " z%SqueezeView.squeezer..reindex)rrrrQ)rr)rr4rrrSrsqueezer s zSqueezeView.squeezerrXr rcCstd)Nzuse SqueezeView.create())AssertionError)rrXrrrr rzSqueezeView.__init__)rrrLrrr)rrrrP)rXr rr)rrrrr~rLrVrrrrrrK s & rKc@sXeZdZUded<ded<dddZdd d Zdd d ZeZedddZ dddZ dS) GenericViewrrrrrcCrr)rrrrrr zGenericView.make_reindexerrcCsBddtt|jD}t||}ddtt|d|S)NcSg|]}ttj|qSr)rdr=rrrrrrr  z+GenericView.reindex_str..zlambda , r,)rrrrrrrr)r index_old index_newrrr reindex_str s  zGenericView.reindex_strcCs$||jd|jd|gS)Nsize=zreindex=)rrXrr`rrrrry szGenericView.__str__rrr4rcCs||t||dS)NrXrr)r)rrr4rrrrr~ szGenericView.createcCrrrrrrrr rzGenericView.get_sizeNr3r9)rrr4rrrrrr0) rrrrrr`ryrrr~rrrrrrX s     rXc@sZeZdZedddZedd d ZedddZe ddddZe dd ddZ dS)!rrr rrcCs<t|}t|}tjjjj}|t|dr||}|Sr)rrNrjrrr evaluate_exprLt)rrrcrrrhandle_negative_index s   zView.handle_negative_indexrrr4rc s t|ts Jt||||\}tjj|r|Sd}t t dks1t t |dkr3d}d|vrGd fdd }||t ||d St |sM|rv|rXt |sXt |}t|dd \}}t|j|j|t||j|j}t||d S||}||t ||d S)NFrTrr rtuple[int, ...]cstdgtSr)rrrr8rr fake_reindex3 rz!View.create..fake_reindexrb)r rW)rr rrf)rrrresolve_negative_sizerrjrrstatically_known_list_equalsrr2rr rrequire_contiguousrZr[rrrrr\r]r^r) rrr4unbacked_symbols_in_sizesrhr_r`rbrrrgrr~! s4     z View.creater8tuple[list[Expr], list[Expr]]cCsdd|D}dd|D}t|}tt|D]}||dkr3tjj||<tt|t|||<nqtj j t|t|||fS)NcSg|] }tjj|qSrrjrrrgrFrrrrS z.View.resolve_negative_size..cSrnrrorFrrrrT rpr@) rrrrrOner:rerjrr check_equals)r8r4rrrrriO s  zView.resolve_negative_sizeNrF dense_dimrrc CsZz ||||}W|Sttfy,t|g}|||}|||}t||}Y|Swr)_dynamic_reshape_indexerrW IndexErrorrer)rr8r4rsrflatrrrrrr` s    zView.dynamic_reshape_indexerrcstjjj}ddtt|Dtt|}t|}|duo,|t|dko,t|dk}|r?|dus5J||}| |g|r |r |}|\} } |dkrd t j j | | | fn| dkrn| |n|| ||kr | tjj | |n|| ||kr|| ||kr|\} } | | | } | | } || ||ks | tjj | |nL|| ||krt j j} |} t| | || |} || ||kr|} t| | || |} ||}|| ||kstjj | |nt|r |sG|r$|}tjj |d t j j |s|r8|\} } tjj | d|s'|durSt|dkrS}||ntt|ksbJd fdd }|S) zG Perform a reshape entirely by modifying indexing math cSrZr)rdr=VIEWrrrrr} r\z1View._dynamic_reshape_indexer..Nr>rrrcsHt|tksJt|tftt|tfddDS)Nc3rer)rfrF replacementsrrr rhzAView._dynamic_reshape_indexer..reindex..)rrrrrr view_exprrxrr s$z.View._dynamic_reshape_indexer..reindexr=)rjrrrrrrrr}rrrrrrrqr<rWreverseinsert)r8r4rsr stack_new stack_oldreordering_dense_dimold_dimsize_oldvarsize_newvar2 size_new2divisormodulus dense_exprrrrzrrtq s                zView._dynamic_reshape_indexer)rr rr rr )rrr4rrr)r8rr4rrrmr)r8rFr4rFrsrrr)r8rr4rrsrrr) rrrrLrerr~rirrtrrrrr s  - rcseZdZUdZded<d3fdd Zd4d d ZeZd4d d Zd5ddZ d6ddZ e d7ddZ d8ddZ d8ddZd9ddZd:dd Zd;d!d"Zd3d#d$Z %dd?d.d/Zd@d1d2ZZS)Ar^z*Pretend our storage has a different layoutrrYrrcs2tt|jtrt|d|jdSdS)NrX)rsrrrXrrrrrrtrrr s  zReinterpretView.__post_init__rcC||j|jgSr)rrXrYrrrrry s zReinterpretView.__str__cCr&rr*rrrrr rzReinterpretView.get_namercC|jjSr)rYrrrrrr rzReinterpretView.get_devicer}cCrrrrrrrr r zReinterpretView.get_origin_nodercCrr)rYrrrrrr rzReinterpretView.dtypercC t|jjSr)rrYrrrrrr rzReinterpretView.get_sizecCrr)rrYrrrrrr> rzReinterpretView.get_stridercdfdd }|S)NrrrricsFj}t||}jjjjkr!t|jjjS|Sr)rYrrhloadrrrXto_dtype_bitcast)rr tmp_loaderrrrr s z+ReinterpretView.make_loader..loaderrrrrirr%rrrr szReinterpretView.make_loaderrcCr&r)rYrrrrrr rzReinterpretView.make_indexercCrrrYrrrrr rzReinterpretView.get_layoutcCrrrrrrrr r zReinterpretView.freeze_layoutFrsrrcCs*t|jj|t|jj|Bt|jj|BSr)r(rYrrr\rrrrr s   z$ReinterpretView.get_free_symbol_usesNrrcCs@tjjj|j|jj|jj|jj|dur|j ntjjj |jj dSr) rjr wrapper_codecodegen_reinterpret_viewrXrYrrr\ writelinerrrrrr sz!ReinterpretView.codegen_referencercCrr rrrrrr r zReinterpretView.num_readsr'r9r4r)r,r0r6r7r-r=rFrr3rD)rrrrIrrryrrrrrOrrr>rrrrrrrrrrrtrr^ s*             r^c@sXeZdZUdZded<edd d Zdd d ZeZe dddZ dddZ dddZ dS) DtypeViewz(Pretend our storage has a different typer target_dtyperr new_dtyperrcCsHt|rt|\}}t|j||j|j|j|j}t||dSt ||dS)NrW)rXr) rrZr[rrrr\r]r^r)rrrr_r`rbrrrr~) s   zDtypeView.creatercCrr)rrXrrrrrry8 rzDtypeView.__str__cCrr)rrrrrr= szDtypeView.dtypercCr&rrXrrrrrrA rzDtypeView.get_sizercsjdfdd }|S)Nrrrricst|jjjSr)rhrrrXrrr"rrrrG sz%DtypeView.make_loader..loaderrrXrr%rrrrD s zDtypeView.make_loaderN)rrrrrrr9r,r0r6) rrrrIrrr~ryrrOrrrrrrrr# s     rc@s.eZdZedd d Ze ddddZdS) SliceViewrrrLrstartendrtuple[int, int]cstjj||tdd||fDrtjtjnjj dfd d dfdd }||dd}|||}||fS)zz Normalize start and end such that both are in the range [0, x.get_size()[dim]] and start <= end. cs|]}t|VqdSr)r2rFrrrrY r&z0SliceView.normalize_start_end..rr lowerrupperrcs<||r|n||}||r|}|S||}|Sr)statically_known_geqrI)rrr clamped_lower clamped_full)max_funcmin_funcrrrclamp` s z,SliceView.normalize_start_end..clamprVUnion[int, None]r|Union[Expr, int]cs$|dur|S|}|||Sr)re)rVrrr|)rrdim_sizerr clamp_wrapk s  z1SliceView.normalize_start_end..clamp_wraprN)rr rrrrrr ) rVrrrrrr|rrr) rjrrrrrMinMax evaluate_min evaluate_max)rrrLrrrr)rrrrrrrnormalize_start_endN s   zSliceView.normalize_start_endr>Tsteprrc stttsdksJzdkr"|dkr"dkr"|WSWn ty,Ynwt||r?|||\}t|d<t |r}t |\}}t|j } | | <t |j |j| |j|j |j} t|| dSd fdd } t|| d S) Nrlr>rWrrrcsDt|tksJd|dt|}||<|S)Nz wrong ndim rR)rrrrLr4rrrrr s$z!SliceView.create..reindexrbr=)rrNrr TypeErrorrrrr;rrZrr[rrr\r]r^r) rrrLrrrrr_r`rarbrrrrr~x s8       zSliceView.createN) rrrLrrrrrrr)r>T)rrrLrrrrrrrrrrr)rrrrrr~rrrrrM s )rc@sFeZdZUded<ded<dddZdd d Zdd dZdddZdS) BaseConstantrrrrrrcCrNrrrrrrr r zBaseConstant.get_sizercCrrrzrrrrr rzBaseConstant.get_devicer}cCrrrrrrrr r zBaseConstant.get_origin_nodercCr\rr8rrrrr rzBaseConstant.get_readsNr0r4r)rC)rrrrrrrrrrrrr s    rc@sDeZdZUded<ded<ded<dd d Zdd d ZdddZdS)Constantr r%rrrrrrcr)NrrrrictjjSr)rhrr%rrrrrr r^z$Constant.make_loader..loaderrrr%rrrr zConstant.make_loaderr+cCrrrrrrrr r zConstant.realizercCt|j|j|dS)N)r%rr)rr%rrrrrr" rzConstant.constant_to_deviceNr6r2rG)rrrrrrr"rrrrr s   rc@s:eZdZUded<ded<ded<dd d Zdd d ZdS)IndexingConstantr rrrrrrrcr)Nrrrricrr)rhrOrrrrrrr r^z,IndexingConstant.make_loader..loaderrrr%rrrr rzIndexingConstant.make_loaderrcCr)N)rrr)rrrrrrrr" rz#IndexingConstant.constant_to_deviceNr6rG)rrrrrr"rrrrr s  rcCspd}d}ttt||D](\}}|dkrq tjj||s)tjj||s)dS|td|9}||9}q dSNr>FT) reversedrrrjrrrJrr)rrHexpected_strideexpected_stride_maxryrrris_contiguous_strides_for_shape s rcCs tj|jSr)r?padding_alignment_bytesitemsizerrrrget_align_for_dtype rrc@s2eZdZdZdddZdddZ ddd dZdS)r.zxAbstract base for Layout, MultiOutputLayout, NoneLayout. Represents the memory layout of the output of an Operation.rrcCrrrrrrrrrzOutputSpec.get_devicercCrrrrrrr storage_sizerzOutputSpec.storage_sizeFrsrrcCrrrrrrrrr zOutputSpec.get_free_symbol_usesNr4rDr=rF)rrrrIrrrrrrrr. s   r.c@seZdZdZdeddfdCddZdDddZeZdEddZdFddZ dGddZ e dHd"d#Z dGd$d%Z dId(d)ZdGd*d+Ze dJd-d.ZdKd/d0ZdGd1d2ZdLd4d5ZdMd7d8ZdNd;d<ZdOd=d>Z dPdQdAdBZdS)Rrzo Layout base class Carries tensor meta-information including offset and whether it is pinned. NrFrrrrrrrOptional[Sequence[Expr]]r\r r]rrrcCs|dur t|}||_||_t|t|ks!Jd|d|tdd|Ds,J||_||_||_||_ |j rC|jj dksEJdSdS)Nra , stride=css|] }t|ttfVqdSr)rr rrrrrr#z"Layout.__init__..r1) rrrrrrrrr\r]r)rrrrrr\r]rrrrs $zLayout.__init__rc Csd}|jdkr d|j}|jjdurdnd|jj}d}|jr'd|j}t|jd|jj|d|jd|jd |j||d S) Nrrz , offset=:z , is_pinned=z('z', z, size=rr-) r\rrr]rrrrr)rr\device_index_str is_pinned_strrrrry+s"   "zLayout.__str__cCrrrzrrrrr;rzLayout.get_devicer cCsPtjtjt|jt|j|j|j|j dWdS1s!wYdS)N)rr pin_memory) rj fake_moder!r"rXrrrrr]rrrr get_example>s$zLayout.get_examplecCst|j|jSr)rrrrrrrr HrzLayout.is_contiguousrHrFrQcCsVt|}|dvs|ddkrdSt|t||D]\}}}|dkr(||kr(dSqdS)N)rr>FT)rrr,)rHrQndimleftrightrrrris_channels_last_contiguousKs z"Layout.is_channels_last_contiguouscCsJt|jtttt|j|jD]\}}}|dkr"||kr"dSqdSr)rrrrrrr)rrrrrrr is_transposedYszLayout.is_transposedrrcstjtks JddtjD}fdd|D}fdd|Dddd }|d gt}ttD] }||||<q=ttd D]'}||||d k}t|tsrtjj j ||||d kd d }|rwdSqPd S)NcSs*g|]\}}tjjj|dddkr|qS)rr5r>)rjrrr)rrrLrrrrgs z,Layout.is_stride_ordered..crArrCrrrrrmrcrrrrrrrrnrarrrrcst|fdd|DS)Ncg|]}|qSrr)relement sorted_arrrrrrrzDLayout.is_stride_ordered..sorted_indices..)r)rrrrsorted_indicespsz0Layout.is_stride_ordered..sorted_indicesr@r>T)size_obliviousF)rrrr) rrrrrrrrjr _shape_envrc)rr non_1_indicesrrstride_orderedrexprr)rrrrcs*  zLayout.is_stride_orderedcCs:dgtttdt|jd}t|g|}||SNrr>)rrrrrrr rrris_channels_last_stride_ordereds" z&Layout.is_channels_last_stride_ordered in_stridescs|t|}t|dkr |Stjst||r|St}t|dr)|j ddr)|Sttj dr3tj j nddfd d rKt fd d|DrK|St|}t|}ddtt|D}d||d<d}t|ddddD]?\} } || d} || || } t| ttjfo| tjko| |dkpt| tjotj} | || <| rt| |||| <d}qq|s|Stjd7_|S)z The padding does not change stride order but makes sure all strides larger than the threshold are multiple of align. rrkdislike_paddingFrNrsympy.Expr | intrrcs4durdSt|tjsdStfdd|jDS)NFc3s|]}|VqdSr)is_unbacked_symintrrrrrrhzILayout._pad_strides..contains_unbacked_symints..)rrr rr1rrrrcontains_unbacked_symintss  z6Layout._pad_strides..contains_unbacked_symintsc3rrrr)rrrrr&z&Layout._pad_strides..cSrrrrrrrrrz'Layout._pad_strides..r>)rT)rrrr)rrr?pad_channels_lastrrrjget_current_noderrkrrrrrrrrrrrr!padding_stride_thresholdr pad_dynamic_shapesrVr&num_comprehensive_padding)rrraligncurrent_fx_noder r new_stridespaddedrankrprev_idxrrequire_paddingr)rrr _pad_stridessP      zLayout._pad_stridescCs>t|ts Jt||jdusJ||j|j|j|_dSr)rrrrrrrrrrrrszLayout.pad_stridescCstjot|tSr)r?comprehensive_paddingrrrrrrrr^zLayout.should_pad_stridesr[cCs<t|tr|S|r|t|j|j|j|j|j|j Sr) rr[rrrrrrr\r]rrrras_fixeds zLayout.as_fixedrcCs(tjsJdt|jd|S)Nzconvert z to FixedLayout first)rrrrrrrrrrrs zLayout.make_indexerotherrcCsRt|to(|j|jko(|j|jko(|j|jko(|j|jko(|j|jko(|j|jkSr)rrrrrrr\r])rrrrr__eq__s       z Layout.__eq__cCt|j|j|jSr)r)rrr\rrrrrrzLayout.storage_sizersrcCs$t|j|t|j|Bt|j|BSr)r(rrr\rrrrrs   zLayout.get_free_symbol_uses)rrrrrrrrr\r r]rrrr9r5)rr r/)rHrFrQrFrr)rrrr)rrrrrrrrr')rr[r7)rrrrr1r=rF)rrrrIr!rryrrrr rLrrrrrrrrrrrrrrrrr s4       #  D     rc@seZdZdZdddZdS)r[z A Tensor layout we cannot changerrcCr)r)rTrrr\rrrrr rGzFixedLayout.make_indexerNr7)rrrrIrrrrrr[sr[cseZdZdZdZed/ddZed0d d Zed1d dZed2ddZ ed3ddZ d4d5ddZ d4d6ddZ d7dd Z d8d!d"Z # d9d:fd-d. ZZS);rz-A Tensor layout that we are allowed to changeFsizesrrrqcCsNt|dkrgStjjg}t|ddD] }|||dqtt|S)Nrr>r@)rrrrqrrr)rreversed_stridesrrrrrs   z!FlexibleLayout.contiguous_stridesrcCs\ttt|t|ksJ||ftjj}dgt|}|D] }|||<|||}q|S)z Create a stride based on the order the dimensions should be filled in. In this format, channels last would be: [1, 3, 2, 0] N)r9rrrrrq)rr next_striderQrrrr fill_ordereds$zFlexibleLayout.fill_orderedrcCs0ttt|t|ksJt|}t||S)z Create a stride based on the sorted order of a permuted range. In this format, channels last would be: [3, 0, 2, 1] )r9rrrrr)rrrrrrr.s zFlexibleLayout.stride_ordered memory_formattorch.memory_formatcCsP|tjkr t|tS|tjkrt|tS|tjkr t|St d|t )aq Create a stride based on a memory format. Memory format is translasted into a stride order, so channels_last is the same as: FlexibleLayout.stride_ordered(sizes, [3, 0, 2, 1]) This interface does not support memory_format `torch.preserve_format` which should be used to deduce a format from another source z>stride_ordered_for_memory_format, unsuppored memory_format: %s) r! channels_lastrrNHWC_STRIDE_ORDERchannels_last_3dNHWDC_STRIDE_ORDERcontiguous_formatrr/rr)rrrrr stride_ordered_for_memory_format:s      z/FlexibleLayout.stride_ordered_for_memory_formatrrFcCsDt|t|ks Jdd|D}ttt||jd}t||S)z Create a stride that has the same stride order as given stride For example, if given stride is [1000, 1, 100, 10], the fill order should be [1, 3, 2, 0] cSrnr)rjrrr=rFrrrr`rpz/FlexibleLayout.same_ordered..r)rrr __getitem__rr)rrrrrr same_orderedUs  zFlexibleLayout.same_orderedrrr[cCsH||j|}|r|r|||j|j}t|j|j|j||j|jSr) rrrrrr[rr\r])rrrrarrras_stride_orderds zFlexibleLayout.as_stride_orderrcCs>|}|r|r|||j|j}t|j|j|j||j|jSr)rrrrr[rr\r])rrrrarrras_exact_stridests zFlexibleLayout.as_exact_stridescCD||j|}|r|||j|j}t|j|j|j||j|jSr) rrrrrr[rr\r])rrrarrr as_fill_orderzFlexibleLayout.as_fill_ordercCr r) rrrrrr[rr\r])rrrarrr as_same_orderr zFlexibleLayout.as_same_orderNrrrrrr rr]rcs6|r t||}nt|}tj|||||ddS)Nr])rrrrsr)rrrrr r]rQrtrrrs zFlexibleLayout.__init__)rrrrq)rrrrrrq)rrrrrr)rrrrrr)rrrrFrrr=)rrrrrr[)rrFrrrr[)rrrr[)rrFrr[r) rrrrrrr rr]rrr)rrrrIrrLrrrrrr r r rrrrrrtrrs,        rcsDeZdZdZdfdd Zdd d Zdd d Z ddddZZS)NonOwningLayoutz,Is a view into the storage of another tensorviewUnion[BaseView, TensorBox]rrcs,|}t|j|j|j|j||_dSr)rrsrrrrrr)rrrYrtrrrs zNonOwningLayout.__init__rcC |Sr)rrrrrrrrzNonOwningLayout.make_indexerrcCs4|jj}|dkr dSddlm}tjj||S)NrTr>) ALIGNMENT) rrr\utilsrrjrrr)rr\rrrrmaybe_guard_aligneds  z#NonOwningLayout.maybe_guard_alignedFrsrcCsVt|jtsJ|jj}t|tsJt||j}t|ts%Jt||j|Sr) rrr^rXrrrrYr)rrsbox input_bufferrrrrs  z$NonOwningLayout.get_free_symbol_uses)rrrrr7r/r=rF) rrrrIrrrrrrrrtrrs  rc@eZdZdZdS)CommBufferTypesymm_memN)rrrSYMM_MEMrrrrrrcs4eZdZUdZded<ded<d fdd ZZS) CommBufferLayoutax A layout that signifies the buffer is a comm buffer. In terms of striding, the layout is identical to `FixedLayout`. Buffers with this layout do not participate in in-place reuse - it can be neither the source nor the target for in-place reuse. For detailed motivation and usage of this layout, see NOTE [lowering-time collective optimization]. rcomm_buffer_typer group_namerYrcsVt|ts td|d|}tj|j|j|j|j |j |j d||_ ||_ dS)NzJA `CommBufferLayout` can only be initialized with a `FlexibleLayout` (got z).rrrrr\r])rrrWrrsrrrrrr\r]rr )rrYrr fixedrtrrrs"  zCommBufferLayout.__init__)rYrrrr r)rrrrIrrrrrrtrrs  rc@sdeZdZUded<ejdddZded<ejdddZded <dd d ZdddZ dddZ dS) NoneLayoutrrcCdgSrrrrrrrrzNoneLayout.default_factoryrErcCr$rrrrrrrrrrrcCrrrrrrrr r zNoneLayout.storage_sizer.cC|Srrrrrrr r zNoneLayout.as_fixedcCrrrzrrrrrrzNoneLayout.get_deviceNrDr.r4) rrrrrJrKrrrrrrrrrr#s    r#cseZdZd#fdd Zed$dd Zejd%d d Zd&ddZd'ddZd(ddZ e d)d*ddZ d+ddZ d,d!d"Z ZS)-MutationLayoutSHOULDREMOVEr>rrrcs@t|||d||_|}tj |dSr) rsrrrrr> get_bufferrrjrmark_buffer_mutated)rr>rrtrrrs z#MutationLayoutSHOULDREMOVE.__init__rcCrr) real_layoutrrrrrr r'z!MutationLayoutSHOULDREMOVE.strider%rcCrrr)rr%rrrr$rr cCrr)r+rrrrrr(rz'MutationLayoutSHOULDREMOVE.storage_sizercs2dfdd |j}t|tsJt||S)Nr>r rcsBt|tr |jSt|tr|St|tr|jS|Sr)rr(r>rr MutableBoxrX)r> unwrap_viewsrrr.,s      z;MutationLayoutSHOULDREMOVE.get_buffer..unwrap_views)r>r rr )r>rrr)rrrr-rr)+s z%MutationLayoutSHOULDREMOVE.get_bufferrcCs|j}t|ts J|Sr)r)rYrr)rrYrrrr+9s z&MutationLayoutSHOULDREMOVE.real_layoutFsrcdst unsafe_aliasrc Cs|tj|t|tr|j}||sAt j | | | ddt||Dd}t|ttfs>J|j}|t|dsNJ|t|jjts]Jt|jjt||j_|jS)NcS g|] \}}tjj||qSrrjrrcheck_equals_and_simplifyrrrrrrrXz;MutationLayoutSHOULDREMOVE.realize_into..rrX)rrjrr*rrrrXrrr~rrrrrrr,rrYrrr()rr/r0r1rrrr realize_into>s*   z'MutationLayoutSHOULDREMOVE.realize_intorcCr'rrrrrrrfr z#MutationLayoutSHOULDREMOVE.as_fixedrcCr&r)r>rrrrrrirz'MutationLayoutSHOULDREMOVE.make_indexer)r>rrrr0)r%rrrr1)rrr-r=)r/rr0rr1rrr)rrr7)rrrrrOrsetterrr)r+rr7rrrrrrtrr(s      'r(csdeZdZUded<ded<d]fdd Zd^d d Zd_d dZd`ddZdaddZdbddZ e dcddZ ddddZ dedd Z dfd"d#Zdgd%d&Zdhd'd(Zdid*d+Zdjd-d.Zd]d/d0Z 1dkdld5d6Zdmd7d8Zdnd:d;Z 1dkdod=d>Zdjd?d@ZdpdBdCZdqdrdGdHZd]dIdJZdsdLdMZdsdNdOZdtdQdRZ 1dkdudUdVZdvdWdXZ dwdYdZZ!djd[d\Z"Z#S)xrr+rr.rYrrcst|dddSrV)rsrrrrtrrrws zBuffer.__post_init__rcCrr)rrrrrrr{rzBuffer.make_indexerrcCs|jsJ||jSrrrrrrr~rWzBuffer.get_name!Union[torch.Tensor, sympy.Symbol]cCs&t|jtr |jStt|jjr)rrYrrrrrrrrrrs  zBuffer.get_examplercCrr)rrrrrrrrzBuffer.get_devicercCrrrrrrrrr zBuffer.get_defining_oprcCrr)rrrrrrrr'z Buffer.dtypercCg|jSr)rrrrrrrrzBuffer.get_sizerqcCr:r)rrrrrrr>rzBuffer.get_strider cCrr)rr\rrrr get_offsetrzBuffer.get_offsetrcCs"t|jtr |jStt|jjr)rrYrrrrrrrrrs zBuffer.get_layoutcCrrrrrrrrrzBuffer.get_output_specrcCrr)rrrrrrrzBuffer.get_storage_numelrcCrr)rr]rrrr get_is_pinnedrzBuffer.get_is_pinnedcCs0t|jtrt|jts|j|_dSdSdSr)rrYrrrrrrrrs zBuffer.freeze_layoutFrrrcC0t|jts Jt|j|jj||d|_dSNr)rrYrrr r rrrr sz&Buffer.freeze_layout_with_stride_ordercC,t|jts Jt|j|j||_dSr)rrYrrr r rrrr z$Buffer.freeze_layout_with_fill_orderrcCr?r)rrYrrrrrrrrr@z$Buffer.freeze_layout_with_same_orderrcCr=r>)rrYrrr rrrrrs z'Buffer.freeze_layout_with_exact_stridescCrrrrrrrrrzBuffer.is_zero_elementsrcs*r ttdSdfdd }|S) Nrrrrrics}tjp d||Sr)rrhrrrrrrrrsz"Buffer.make_loader..loaderr)rr rrr%rrrrszBuffer.make_loaderNrrcCrrrrrrrrrzBuffer.codegen_referencecCrrrrrrrrr zBuffer.decide_layoutrcCt|jtr |jjgSdSr)rrYrrrrrrrr% z#Buffer.get_inputs_that_alias_outputcCrCr)rrYr(r>rrrrrr#rDzBuffer.get_mutation_namesrcCst|gSr)r9rrrrrrrzBuffer.get_read_namesrsrcCr\rr8rrrrrrYzBuffer.get_free_symbol_usescCr\rr8rrrrr]rzBuffer.get_unbacked_symbol_defscCrrrrrrrrr zBuffer.realizecCrrrrrrrshould_allocaterzBuffer.should_allocater'r7r9)rr9r4r*r,r0)rrqr1r-r.rDr/r=r>r?)rrrr)rrrrrrr6rr3rHr(rFr`r2)$rrrrrrrrrrrOrrr>r;rrrr<rr r rrrrrrr%r#rrr]rrErrrrtrrmsJ                         rc@s0eZdZd ddZd ddZejZdd d Zd S)OperationBufferrrZcCs|gSrrrrrrr[rzOperationBuffer.get_outputsrPcCr'rrrrrrrr zOperationBuffer.get_defining_oprcCst|t|dSr)rrrPrrrrrs zOperationBuffer.__post_init__Nr_rrPr')rrrr[rrPr$rrrrrrFs   rFc@eZdZdddZdS)rrrcCrr rrrrrrr zInputBuffer.num_readsNrD)rrrrrrrrrrc@r) DonatedBufferaY Represents a donated buffer which is a saved tensor that is not alias to any fwd inputs, fwd user outputs, and bwd outputs. We generally cannot inplace reuse the input tensor memory during backward since it might be used in another function. However, donated buffer can be inplace reused during backward to save memory. N)rrrrIrrrrrJ rrJc@s.eZdZUdZded<d ddZdd d ZdS)rNrrrrcr)Nrrrrics,}ttjj||Sr) rrrhrrjr constant_namerrrArrrrs z*ConstantBuffer.make_loader..loaderrrr%rrrrszConstantBuffer.make_loaderrrrcCsttj|||jdSN)rrY)rrjrrKrrYrrrrr""sz!ConstantBuffer.constant_to_devicer6rG)rrrrrrr"rrrrrs   rc@sDeZdZdddZ ddd d ZddddZdddZdddZd S)NoneAsConstantBufferrrcCr\rr8rrrrr*rzNoneAsConstantBuffer.get_readsFrsrrcCr\rr8rrrrr-rYz)NoneAsConstantBuffer.get_free_symbol_usesNrrrcCs tjjjSr)rjrrnone_strrrrrr2rz&NoneAsConstantBuffer.codegen_referencer.cCs tddSNrz)r#rrrrr5rz$NoneAsConstantBuffer.get_output_speccCrrrrrrrr8r z&NoneAsConstantBuffer.has_tensor_outputrCr=rFrr3r.r/)rrrrrrrrrrrrrM(s   rMc@s:eZdZUded< dddd ZddddZdddZd S)rr rFrsrrrcC t|j|Sr)r(rrrrrr@rz*ShapeAsConstantBuffer.get_free_symbol_usesNrrrcCstjj|jSr)rjrrcodegen_sizevarrrrrrrEr^z'ShapeAsConstantBuffer.codegen_referencecCrrrrrrrrHr z'ShapeAsConstantBuffer.has_tensor_outputr=rFrr3r/)rrrrrrrrrrrr<s  rcseZdZUdZded<dZded<eejdQd d Z dRd d Z dSddZ dTddZ dUddZ dVddZ dWdXddZdYfd d! ZdZd"d#Zd[d%d&Zd\d(d)Zd]d+d,Zed^d.d/Z 0 0d_d`d6d7Ze 0dadbdAdBZdcdDdEZdRdFdGZdZdHdIZdZdJdKZdddOdPZZS)erzb Represents a buffer that is computed during kernel execution rather than being an input. rarXFzClassVar[bool]_force_realizerIterator[None]ccs*tj}z dt_dVW|t_dS|t_wNT)rrR) old_valuerrr force_realizeUs zComputedBuffer.force_realizer+cCs(|jdur|jSt|jdr|jjSdS)z Returns self.name if it exists, otherwise returns the name of the data node if that exists. If neither exist, returns None. Nr)rrrXrrrrget_computed_buffer_name_s  z'ComputedBuffer.get_computed_buffer_namercCr&rrXrrrrrrjrzComputedBuffer.num_readsrcCr&rrXrrrrrrmrzComputedBuffer.get_readsrcCr&rr2rrrrrprzComputedBuffer.get_read_namesrcCst|jttttfstjtttdSt t dd,|j r7t ||j|jWdSt ||jWdS1sKwYdS)Nrwrites index_exprsrT)rrXrrrrr@ ReadWritesr9rrrrrIget_store_functionr{rrrrrrrss& $zComputedBuffer.get_read_writesrsrrcCsD|j||j|B}|r t|tr |||O}|Sr)rYrrXhas_store_functionrr^rKr)rrsrrrrrs  z#ComputedBuffer.get_free_symbol_usesrcs<|s|jtjjvr|dkr|js|jSt Sr) rrrjrmutated_buffersrrRrXrrsrrtrrrs   zComputedBuffer.make_loadercCst|jttttfSr)rrXrrrrrrrrr_rz!ComputedBuffer.has_store_functionCallable[..., None]cCs`|}t|jtttfrt|jj |j |St|jt s'Jt |jt|jj |j |Sr)rrrrrXrrrr rrrrrr#rrrr^s z!ComputedBuffer.get_store_functionOptional[list[int]]cst|jtrYt|j|j\\}}|j }t dd|Ds&Jfdd|D}|rYt|jt t frA|j |n|fdd|D}ddlm}|||SdS) al If our layout is still flexible, try to determine the stride order based on stride orders of reads. TODO(jansel): A better algorithm here would look at downstream consumers of this value and try to do global graph-level layout optimization. This is also something just begging to be autotuned. css"|] }t|tjtjfVqdSr)rr@StarDep MemoryDeprrrrr  z0ComputedBuffer.get_fill_order..cs.g|]}t|tjrt|jddDqS)cSsi|] }|dkr|tjjqSrra)rvrrrrz..)rr@rdrfrr)rrrrs  z1ComputedBuffer.get_fill_order..csg|] }tjj|qSrrjrrr2rr)r rrrsr>pick_loop_orderN)rrYrr@r0rXr{rrrrrrr schedulerrkr)r index_varsrrstride_lengthsrkr)r rrrs*     zComputedBuffer.get_fill_orderrcCs6t|jtr|}|r||dS|dSdSr)rrYrrr rr rrrrs  zComputedBuffer.decide_layoutMtuple[tuple[list[Expr], list[Expr]], LoopBody, tuple[list[Expr], list[Expr]]]c Cstj|j|jdd\}}ttd|t | | r$|n|dd|g|R}Wdn1s:wYg}g}g}g}| D]+\}} ||dvrd|rYJ| || | qK||dvslJ| || | qK||f|||ffS)Nqryrr>r)r@r0rXr{rrrrrrKr^ritemsr) rr var_rangesr~rm reduce_vars index_size reduce_sizerfrrrrget_default_sizes_bodys2       z%ComputedBuffer.get_default_sizes_bodyNextra_indexing_constraints*Optional[tuple[dict[Any, Any], list[Any]]]recompute_sizes_body_funcOptional[Callable[..., Any]]8tuple[tuple[list[Expr], list[Expr]], Optional[LoopBody]]cs\\}}}\}}|r|||f|||f\\}}}\}}g|j|durut|tr4t|dks6J|\}} t|tsEJt|t| tsPJt| t dd| Ds[J|j } | |kshJ| |ffdd| D} | 7g| t j tjs|dfdd } ||} tt ptj } | || || \}}}| || || \}}}tj||dd\\}}}t|||||g|||}||f|fS)an This is a main place where we do loop transformations in a backend-agnostic way. Here we: 1) Remove any 1 dimensions 2) Fuse contiguous dimensions together 3) Reorder dimensions based on stride orders Optional argument extra_indexing_constraints can be used to append additional indexing expressions to existing ones derived from buffer's body. This can be useful to fuse scheduler nodes with compatible ranges, e.g. (s0*s1*...,) and (s0, s1, s2, ...) on CPU by preventing indexing simplifications and obtaining index/reduce ranges for the scheduler node compatible with other nodes. Optional argument recompute_sizes_body_func can be used to recompute sizes and body on the default body. This can be useful to append additional loop transformations. Nrcs|]}t|tVqdSr)rr )rfrrrr7rhz6ComputedBuffer.simplify_and_reorder..csg|]}|vr|qSrrrf)index_formulasrrr?z7ComputedBuffer.simplify_and_reorder..x_varsSequence[sympy.Symbol] support_varsrrsimplify_loopsrrdtuple[list[int], Callable[[Sequence[int]], Sequence[int]], Callable[[Sequence[int]], Sequence[int]]]c s\|||\}}}||}|r'tjj||t||\}}}t||}n|}|||fSr)_apply_loop_reorderingrjrr_simplify_loopsrDr) rrrrreindex0rr_prunerr~ memory_addrsrrrsimplify_and_reorderHs     zAComputedBuffer.simplify_and_reorder..simplify_and_reorderpry) rrrrrrrrrr)rvindexing_exprsrrrrrrrrrrget_write_exprsrjrr'rAPREFER_STORE_LOOP_ORDERextendget_read_exprsrar/r?loop_ordering_after_fusionr@index_vars_no_squeezerK)rrwryrtrur~rmrsextra_indexing_rangesextra_indexing_exprexpected_var_rangesrrshould_merge_loops iter_ranges iter_reindexr reduce_rangesreduce_reindex iter_varsrrrrrr sx       z#ComputedBuffer.simplify_and_reorderrmrrrrrlist[sympy.Expr] priority_idxrc sddlm}|dur g}z*fdd|D}t|t|kr)t|dtks+Jtt|||}Wn tyVtjrLt dt t |tt t}Ynwfdd|Dt |t|fS) zU Shuffle the order of loops around to hopefully improve performance. r>rjNcsg|] }tjj|qSrrhri)rmrrrrz9ComputedBuffer._apply_loop_reordering..rz%Did not simplify complex index: %s %scrrrr)rrrrr)rlrkrrr Exceptionr?rr/warningrrrrr)rmrrrrrkrQrr)rmrrrrs,     z%ComputedBuffer._apply_loop_reorderingrcCr&rrXrrrrrrrz!ComputedBuffer.get_reduction_sizecCr&rrXrrrrrrrz!ComputedBuffer.get_reduction_typecCr&r)rXrrrrrr!rzComputedBuffer.is_no_opcCrrTrrrrrrEr zComputedBuffer.should_allocaterrrcCr)rrXr"rrrrr" z!ComputedBuffer.constant_to_device)rrSr2rDrCr(rBr=rFr6r/)rra)rrbr')rroNN)rwrxryrzrr{r) rmrrrrrrrrrbrrr0rG)rrrrIrrRrLrMrNrVrWrrrrrrr_r^rrrUrvrrrrr!rEr"rrrrtrrLs@          ' " | '   rcsfeZdZdZd#fd d Zd$d dZd%d&ddZd'ddZd(ddZd)ddZ  d*d+d!d"Z Z S),rzt Represents a Triton (in the future other type) of template operator that we can fuse an epilogue onto. rYr.rorpmake_kernel_renderrzrrcs@tjd|dt||_||_tj||_ tj |dSrL) rsrrunwrap_storagerorrjrregister_bufferrregister_operation)rrYrorrtrrrs  zTemplateBuffer.__init__rcCs |jddS)NT normalize)rIrrrrrrzTemplateBuffer.get_read_writesFrrc s||d fdd }tj||d|d }|jD]:ttt fs/Jt tj t s.dummyrrcs$t|dksJt|Sr)rrhrrrQ)rrurrrs)rrrrrr )rrrr@rIrrorr^rrrYrr)rrrdepsr)rrurrrIs"      z"TemplateBuffer.extract_read_writesrcCstjjSr)rrrqrrrrrrz!TemplateBuffer.get_reduction_sizer+cCrrrrrrrrr z!TemplateBuffer.get_reduction_typecCrrTrrrrrrEr zTemplateBuffer.should_allocateNrwrxry.)rsrroutputsr!rh higher_orderflex_attentionflex_attention_backwardrjr current_noder>rrorrrr9r subgraph_inps subgraph_outs)rrYrorrr allowed_setrrtrrrs(  "  zTritonTemplateBuffer.__init__Frsrrcst|}|jr |jng}|jr|jng}|D]%}t|tjr)|t||qt|t r7|||q|dus=Jq|D]}t|t rP|||q@|dusVJq@|Sr) rsrrrrrr updater(r)rrsresrrrurrtrrr3s    z)TritonTemplateBuffer.get_free_symbol_usesrZcCrrrrrrrr[Jrz TritonTemplateBuffer.get_outputsrcCrr)rrrrrget_allowed_prologue_inpsMrz.TritonTemplateBuffer.get_allowed_prologue_inpsrcCsd|jd}|S)NzTritonTemplateBuffer(layout=r-r)rrrrrryPszTritonTemplateBuffer.__str__r) rYrrorprrrrrrrrr=rFr_r(r9) rrrrrr[rryrrrrtrrs-  rcsreZdZdZd%fd d Zd&ddZd'ddZd(ddZd'ddZd'ddZ d)ddZ d*d!d"Z d'd#d$Z Z S)+ ChoiceCallera. Represents a possible choice used in autotune_process.py. During autotuning, self.benchmark() is first called to get benchmark result, and if this choice is selected, self.output_node() is called to get the output_node. Children classes: TritonTemplateCaller, CUDATemplateCaller. rrrrZrYr descriptionrrcs&t||_||_||_||_dSr)rsrrrYrr)rrrrYrrtrrras  zChoiceCaller.__init__rr rr rcsL|ttd}tjrtfddfi|Stjd|ifi|S)N)warmuprepcsSrrralgorrrrwsz(ChoiceCaller.benchmark..r) to_callableautotune_warmup autotune_repr?/profile_bandwidth_with_do_bench_using_profilingrZrP benchmark)rrrbenchmark_configsrrrrpszChoiceCaller.benchmarkcCrSrrTrrrr call_namezr zChoiceCaller.call_namerbcCrSrrTrrrrr}r zChoiceCaller.to_callablecCr)z Hash key for the underlying kernel. By default, we assume there are no runtime params, so kernel hash key defaults to choice caller's hash key. )hash_keyrrrrkernel_hash_keyszChoiceCaller.kernel_hash_keycCrSrrTrrrrrr zChoiceCaller.hash_keyr|cCrSrrTrrrrrmr zChoiceCaller.output_node.)rsr_choice_timings_fn_choice_timingsoriginal_inputsr_output_plannable_make_kernel_renders)rrYrorrrrtrrrs zMultiTemplateBuffer.__init__rcCr)z^ Are all possible choices TritonTemplates or Extern Kernels with out variants )rrrrroutput_plannablesz$MultiTemplateBuffer.output_plannableN hint_overriderdict[ChoiceCaller, float]cCs$||jvr |||j|<|j|Sr)rr)rrrrrchoice_timingss  z"MultiTemplateBuffer.choice_timingscallerrrSccsZt|tjjjsJt||j|jksJ|j}||_z dVW||_dS||_wr) rr!rrTritonTemplateCallerrrYrr)rrrenderrrrswap_as_triton_callers  z)MultiTemplateBuffer.swap_as_triton_callercCsRt|tjjjsJt|||jjksJ| |jj ks"J| |_ dSr) rr!rrrrrrYrr>rrr)rrrrrfinalize_as_triton_callers z-MultiTemplateBuffer.finalize_as_triton_callertuple[ChoiceCaller, float]cCs&|j|d}t||jd}|||fS)N)rr)rrr)rrtimings min_choicerrrget_min_choices  z"MultiTemplateBuffer.get_min_choicecallers-dict[Optional[int], TritonTemplateCallerBase]cCs0|D] \}}||j|<q|jd|_dS)z;Finalize with multiple callers for different hint overridesN)rqrrr)rrrrrrrfinalize_as_triton_callerssz.MultiTemplateBuffer.finalize_as_triton_callers) rYrrorprrrrrrrrr/r)rrrr)rrrrS)rrrr)rrrr)rrrr)rrrrIrrOrrrMrNrrrrrrrrtrrs     rcs2eZdZdfdd ZdddZdddZZS)CUDATemplateBufferrYrrorprCallable[_P, _T]workspace_sizertemplaternsupports_epilogue_fusionrrrcs&t|||||_||_||_dSr)rsrrrr)rrYrorrrrrtrrrs  zCUDATemplateBuffer.__init__cCs|jdur|jSdSr)rrrrrr^rz%CUDATemplateBuffer.get_workspace_sizecCs$|D] }t|ddqdSr)r[rhrr)rrfrrremulate_store_fns z#CUDATemplateBuffer.emulate_store_fn)rYrrorprrrrrrnrrrrrDr')rrrrr^rrrrrtrrs rcs,eZdZdfd d Zdfdd ZZS)CppTemplateBufferrYrrorprrrrnrr rrcs&t|||||_||_d|_dSr)rsrrrr)rrYrorrrrtrrr s zCppTemplateBuffer.__init__cspt|jtr3t|jtsJt|j|jd}t|ts#Jt||j}t|ts1Jt||St Sr) rrYMultiOutputLayoutrrrrrrsr)r first_outputrYrtrrrs   zCppTemplateBuffer.get_layout) rYrrorprrrrnrr rrr-)rrrrrrrrrtrr s rcs0eZdZdZ ddfdd ZdddZZS)CuteDSLTemplateBufferz Buffer for CuteDSL (CUTLASS Python DSL) template kernels. Similar to other template buffers but specialized for CuteDSL operations. NrYrrorprrrr rrrrcst||||_|_g_|dur?tjdts'Jtjdjd jfdd|D7_dSdS)Nrcrrrrrrrr<rz2CuteDSLTemplateBuffer.__init__..) rsrrrrrrorrr)rrYrorrrrtrrr,s"zCuteDSLTemplateBuffer.__init__rZcCrrrrrrrr[Arz!CuteDSLTemplateBuffer.get_outputsr) rYrrorprrrr rrrrr_)rrrrIrr[rrrrtrr&s  rr)Sequence[Union[IRNode, Sequence[IRNode]]]TypeIs[Sequence[IRNode]]cCstdd|DS)Ncsr|rrrr[rrrrHrhz#is_node_sequence..)r)rrrris_node_sequenceErJr c@sreZdZUded<d!ddZd"d d Zd#d dZed$ddZe d%ddZ d&ddZ d'ddZ d(d)ddZ d S)*rrrorrrrcCs |j|}t|ts J|Srrorrr)rrinputrrr input_nameOs zInputsKernel.input_namercsttj}tj|jD]#}t|tr |fdd|Dq t|tr&q | | q ttjfdd| D}tj ||tdS)Nc3|] }|VqdSrrBrFrcrrrYrz/InputsKernel.get_read_writes..c3rrrBrrrrr`  rZ) r9r@rFrcrorrrrrrr[r])rrr r[rrrrTs    zInputsKernel.get_read_writesrcCrrrrrrrrjrzInputsKernel.get_readsrrcCs~t|tr|j}t|tr|j}t|trt|tst|}t|tr)||St|t r0|St|t tfs=Jt ||Sr) rrrXrrr^r realize_inputunwrap_storage_for_inputTorchBindObjectrrrrrrrrms      z%InputsKernel.unwrap_storage_for_input%list[Union[IRNode, Sequence[IRNode]]]cCs@g}|D]}t|trdd|D}nt|}||q|S)NcSrLr)rrrrrrrrz/InputsKernel.unwrap_storage..)rrrrr)ro inputs_newrrrrrs   zInputsKernel.unwrap_storagercCrrTrrrrrr r zInputsKernel.is_externcCrr rrrrrrr zInputsKernel.num_readsFrsrcCsNttj}|jD]}t|tr|||O}q |D] }|||O}qq |Sr)r9rr"rorrr)rrsrru inner_inprrrrs   z!InputsKernel.get_free_symbol_usesN)rrrrrBrCrrrr)rorrrr/rDr=rF)rrrrr rrrrrLrr rrrrrrrKs       rc@s eZdZd ddZd ddZdS) NopKernelrrcCrrTrrrrrr!r zNopKernel.is_no_oprcCr\rr8rrrrrrzNopKernel.get_readsNr/rC)rrrr!rrrrrrs rc@sVeZdZdZeddd Ze ddddZ ddddZed ddZd!ddZ d S)" ConcatKernelzn There isn't actually a real kernel for concat, we just change the storage for the upstream data. rorprLrrrc Cs|d}|d}t|d}dg}||g}d|kr)t|ks,JJtdt|D]Z}||} |||t| t|ksLJ|||ksVJ|||ks`Jtt|D]} | |krw|| | | || <qftjj || | | || <qf|||q3t |} t jrt| ||dj} tt|D]!}||} t| r| } t| trt| j| jrt|} nqtdd|D}tjjjd}t|tsJt||durtdd|Drt|} tdd|D}|dusJtdt|||| |d gd }t |}g}t!|D]{\}}t|t"t#fs.Jt||$|t%j&||||||dd }t|t'sLJt|t|j(tsZJt|j(|j(|t|j)t"rm|j)*}n|j)}t|t r|+r|}durt,|jrt-|s||.qt|dkrtj/|t0j1rtj2|tj3||_4|5|j(|_(tj6||S) z6 Create the concat kernel from inputs rr>csrr)rrFrrrrr&z&ConcatKernel.create..FcssB|]}d|jvo|jdjtjdp|jdjtjdVqdS)rVrN)rkr r!rrrargrrrrs  css"|] }t|o |jVqdSr)rrr]rFrrrrs N)rrrrr]rrYro)r)7rrrrrrrrjrrr4rrr?rrrrrrrr[rrrr,rrrrrrrrrr,r7rr~rrorXrrrar`r$r'rAFOREACHregister_operation_listrrrr)rrorLrrr4 offsets_start offsets_endr input_sizer@ output_striderrYany_input_is_storage_and_layout fx_node_argsr] concat_kernelkernelop_namesrurinput_unwrappeddevrrrr~s               zConcatKernel.createNr/rr0r rcCst|tr ||j|St|ttfsJt|t|jtrPt|jjt r*|jj s,dS|dur2dSt | t | ks@dSt ddt| | DSt|jdoct|jjtoct|jt S)NFTcsrSrrTrUrrrr6rVz=ConcatKernel.can_realize_into_without_copy..rY)rrcan_realize_into_without_copyrXrrrrrYr[rrr>rrrrExternKernelAlloc)rr/r0rrrr,s*      z*ConcatKernel.can_realize_into_without_copyFrsrcCs t||Sr)rrrrrrrArz!ConcatKernel.get_free_symbol_usesc Cst|tst|rt|\}}t||d}t|ts Jt|t|tr,||j|St|trL| t |jds=J| ||rLt ||j_ |jStj|||ddt||Dd}|||S)NrWrYcSr2rr3r5rrrr`r6z-ConcatKernel.realize_into..r)rr^rrZrrr7rXrrrr,rrYrr~rrrrr)rr/r0r_rYpwrrrr7Fs,        zConcatKernel.realize_intocCrrTrrrrrrEgr zConcatKernel.should_allocate)rorprLrrrr)r/rr0r rrr=rF)r/rr0rrrr/) rrrrIrr~r,rr7rErrrrrs n$   rcseZdZUdZdZded<ejedZ ded<dZ d ed <dZ d ed <dZ d ed <eje dZded<dZded<dZded<ejedZded<dZded<ejedZded<eje dZded<       ddfd$d% Zdd'd(Zdd*d+Zdd,d-Zdd.d/Zdd2d3Zdd4d5Zddd6d7Zdd8d9Zdd;d<Zedd@dAZ e!ddGdHZ"e!ddJdKZ#e!ddLdMZ$e!ddNdOZ%e!   PdddWdXZ&e! PdddZd[Z'e! Pddd]d^Z(e!dd_d`Z)e!ddadbZ*e!ddcddZ+e!ddedfZ,ddgdhZ-ddidjZ.dddndoZ/ddpdqZ0ddsdtZ1dddvdwZ2ddxdyZ3ddzd{Z4dd|d}Z5dd~dZ6dddZ7dddZ8 PddddZ9dddZ:e:Z;Z.cSsg|]}iqSrrrrrrrrcSsi|] }|j|j|jdqS))rru)rrErurFrrrrrz=ExternKernel.collect_arg_kwarg_properties..cSg|]}|jr|jqSrrDrrFrrrr cSsg|]}|jr|qSr)rDrFrrrrs )rr8r!_ops OpOverload_schema argumentsrrror9r:r6 schema_kwargsrrrrrBs*    z)ExternKernel.collect_arg_kwarg_propertiescCs$t|jtr||dSdSr)rrYrapply_constraintrrrrrrs  zExternKernel.decide_layoutwrapperrocCs$t||\}}|r||dSdSr)r] make_comment)rrO origin_str_detailed_origin_strrrrcodegen_commentszExternKernel.codegen_commentcCrSrrTrrOrrrcodegenr zExternKernel.codegencCs||_tjjrt|jtjjsdS|j}|jdurB|j dkr;|j dkr+|j ddn|j dd}d|d|_dS|jj|_dSdS)Natenr|.rrz at::_ops::z::call)r4rjr cpp_wrapperrr8r!rIrJ namespace _overloadnamerrreplacerKr)rr4r(opnamerrrr@s      z ExternKernel.set_cpp_kernel_namecCsd||_|dur dS|j}|durdSt|tjjr"d|j|_dS|jddd|j|_dS)Nztorch.ops.higher_order.._ops..ops.rW) r3r8rr!rIHigherOrderOperatorrrr[)rr3r(rrrrAsz#ExternKernel.set_python_kernel_namercCsddlm}|}r|jntjj}tjjr!|jdusJ|jStjj rDt tjj |s4Jttjj |j dus;Jtjj |j |S|jdusKJ|jS)Nr>) CppWrapperCpu)codegen.cpp_wrapper_cpur`rrrjr device_type fx_wrapperr3rXrrr4get_c_shim_func_name)rr`drrrrget_kernel_names zExternKernel.get_kernel_namerrr|cCs:tj||||||d}||S)N)rrrcrdr~r|) rr~rrrrrrr)rr.rrr copy_inputszExternKernel.copy_inputr(rxrr ituple[Any, list[Any], list[Any], Callable[[Any, Any], Any], Optional[dict[sympy.Symbol, pytree.KeyPath]]]cs||d}t|\}gg}g}|D].}t|to"t|t dr.||qt|tr=tjj j j |dd}||qdfd d } fd d |D}|D] } t | rbt | d dqVg} |D]n} t| ts| tjjvr| tjj| qgt| ts| tjjvr| tjj| qgt| tr| | qgt| tjjjr| jj} | jjdkr| dusJ| tjj| qg| t| d dqg| | |\} }|| i|}d}tjj }r*tjj !d}t"}tjj#tj$j%j&kr |d}t'tj}|t(|tj|Wdn 1swYt)|||}t|t*t+fs5|gn|}|D]'}t|tj,r_|j-r_d}tjjj !dd}r[|d|}|tj_.q9|||| |fS)Nrr@)rknew_tensor_argsrnew_non_tensor_argsrtuple[list[_T], dict[str, _T]]csdg}t|}t|}D]}|r|t|q |t|q t|}|dg|difS)Nrr)iterrnextpytreetree_unflattenr)rirjr it_tensorsit_non_tensors is_tensorr) args_spec is_arg_tensorrrunflatten_argsAs z3ExternKernel.process_kernel..unflatten_argscrrrrFrrrrOrz/ExternKernel.process_kernel..Trr2)rrVr>zEsparsity not handled. Please file issue for sparse inference weights.rz Found from : )rirrjrrrk)/rn tree_flattenrrrGeneratorStater rjrrrcreate_symintnoderrZrr constantstorchbind_constantsr get_valuer!rirrrrr2default_generators clone_stater rrrkrr r>_higher_order_opseffects with_effectsr/r3r0rrTensor is_sparsedisable_cudagraphs_reason)rr(rr binded_args args_flat tensor_argsnon_tensor_argsrrur example_args device_indexnew_args new_kwargsexample_outputr<r node_meta_valctxexample_out_lir$msgrr)rsrrtrprocess_kernel$s           zExternKernel.process_kernelr^c Cspt|ts Jt|t|tr|S|}tj|}|dus$J| }|dur]d|j vr]t|tt t fr]t|j tr]|j djtjdsS|j djtjdr]|t|n|tj|dd\}}|d}||}tjj||}tjj||} tjj||} t|| | } || krt d| | |t!t|j"t#|$|%|| | dd d S) z In order to pass this to an extern kernel we need a ReinterpretView not a View. This allows us to avoid some unneeded copies. NrVrrryrz@convert_to_reinterpret_view failed: stride=%s offset=%s index=%sFr!rW)&rrrr^rrjrr)rrrkrr,rYrr r!rrrr,rrr@r0rrr1 stride_vars offset_varrbr/rrrXr[rr) rr x_unwrap_viewr"x_unwrap_view_fx_node index_argsrrrrrQr\expectedrrrconvert_to_reinterpret_viewsj         z(ExternKernel.convert_to_reinterpret_viewcCs|durtSt|ttjjjtfrt|dSt|t r-t j t j|j||dSt|tr4|St|tr?||jSt|trPt||j|dSt|tro|t|roz||WStynYnwt|trz||St|ttfr|S| |S)Nr)rrrW)!rMrr rrrrrrrrjradd_tensor_constantr!rPr%rrrrrrXr^rrrrrrrr NonTensorObjrgrrrrrs<            zExternKernel.realize_inputcCsDt|rt|dkr|S|D] }|dkr|Sq||Sr)rrr>rg)rrrrrrrequire_stride1s  zExternKernel.require_stride1Frrrrrrc s|dus dus JdvrsStrttrS|rGt|o-tj }tdd|r@t t j j jn||dStddd|dStttfr}|re|srr}tjr}dur{tSSt}trt|}trtdt|tr|r||srt|jrSttr|r|sLjrtjrɈSttrtjtrtjtst}rt|drt|jtsz!|j_|r|j ||dWSr|j!|dWSWn t"yYnwd} } durOt j j fd d t#t$D} | D] } t%j&j'(| d d qA|)tdd||d|rkt|siJS| r| durxduszJt%j&j'*| tSS) N)rr>TF)rr r rrzHthe MutationLayoutSHOULDREMOVE's real layout shouldn't be FlexibleLayoutrXrcs4g|]}|dr|dr|qS)rr)rJrrrrrrrrrsz0ExternKernel.require_strides..rr>)+rrrrrrr2rrZrrjrrsize_hints_or_throwr[rrrOrrcr(r+rWrrrXrr^rrr-rrequire_stride_orderrequire_exact_stridesrrrr!rloweringslice_rgrN) rrrrruse_current_stride_ordermutation_layoutr+r expanded_dims orig_sizerLrrrrequire_stridess                 zExternKernel.require_stridesrFcC|j|||dS)N)rrr)rrrrrrrrsz"ExternKernel.require_exact_stridesrcCr)N)rrr)rrrrrrrrsz!ExternKernel.require_stride_ordercC ||tSr)rrrrrrrequire_channels_lastrz"ExternKernel.require_channels_lastcCrr)rrrrrrrequire_channels_last_3drz%ExternKernel.require_channels_last_3dcCs,ddd}||r |S||t|S)Nrrrrc Ss@z|}Wn ttfyYdSw|tjjvotjj|jSr)rAttributeErrorrrjrr{ is_mkldnn)rrrrris_mkldnn_tensors  z9ExternKernel.require_contiguous..is_mkldnn_tensorrrrrrrrr)rrrrrrrks  zExternKernel.require_contiguouscCs||t|Srrrrrrrequire_contiguous_stridessz'ExternKernel.require_contiguous_stridescCrrrrrrrrNr zExternKernel.apply_constraintcCst|ts Jt|t|tst|}|jsJdt|}t|j}||krStd|j||t ||D]}|j|d}| ||vrJ||n|j|dq7|S)Nz/ExternKernel.arg_properties should not be emptyzv%s has %d unprovided positional arguments. Will check if they are in the keyword arguments or will use default values.rru) rrrrr9rr/rr8rr)rrrn_args n_pos_argsrarg_namerrrfill_non_provided_argss(   z#ExternKernel.fill_non_provided_argsrkr{rc Cstjjrog}d}|r"|jr"t|jt|ksJddd|jD}t|jD]E\}}|durF|dus5J|||}|rC|dnd}nt|j|}|jr_|t|jkr_|j|dnd}| tjj ||q'|Sdd|jDS)NzDnames passed to codegen_const_args does not match self.constant_argscSsi|]}|d|qSr)rrrrrrr\z3ExternKernel.codegen_const_args..rcSrnrrjrrval_to_arg_strrrrrrr)rpz3ExternKernel.codegen_const_args..) rjrrXr9rr/rrrorrr) rrkrname_to_arg_propertiesrrproptype_rrrrcodegen_const_args s2  zExternKernel.codegen_const_argscCstjjr|jdur|g|j|j|j}d}n|j}d}g}t|D]4\}}tjjrN|j r6|t |j ks:Jd|j | d}| tjj ||q$| tjj |q$|rb|||S)NFTz-Invalid access to ExternKernel.arg_propertiesr)rjrrXr8rror/rrr9rrrrrrr)rroneed_codegen_constant_argsrrrrrrr codegen_args+s&zExternKernel.codegen_argsrcKsT||vr ||S||jvr|j|S|j|}dur#|dSt|d)zGiven an argument name, queries for values in (in order): 1. any provided kwargs for this function. 2. the class self.kwargs member. 3. any available default arguments in self.allarg_properties.Nruz not in self.allarg_properties)rrr:rW)rrrrrrrget_kwargs_valueEs    zExternKernel.get_kwargs_valueskip_outcCstjjrO|jdurt|jdkrgSg}|jD]5}|r |dkr q||}t|t r0| |q|j dus7J|j |i d}| tjj ||q|Sdd|jD}|S)NrrrcSs(g|]\}}|dtjj|qSrmr)rkrfrrrrfsz/ExternKernel.codegen_kwargs..)rjrrXr8rrMr6rrr rr:rrrrrq)rrrrrfrrrrcodegen_kwargsRs$     zExternKernel.codegen_kwargscCsT|jdur&|jj}t|dd}|dd}|ddd}|d|}|Sd}|S) Nrunknown_namespacer]r^rWr>r unknown_op)rCr>rr[rsplit)rr> op_namespaceop_namerrr get_op_namels   zExternKernel.get_op_namec Cstjr=tjjs?t|dkrdStjj|}tjj| }| }| d| d|d|d|d dSdSdS)Nrzassert_size_stride(r]r-) r? size_assertsrjrrXrerrcodegen_shape_tupler>rrr)rrOrrrrrrcodegen_size_assertsws z!ExternKernel.codegen_size_assertsc Csttjr6tjjs8|}|tjjv}|}|r(|d|dt d|ddS|d|d|ddSdSdS)Nzassert_alignment(r]r-z # buffer z (op: z) is assumed to be not aligned) r?alignment_assertsrjrrXrrrrr^)rrOralignedrrrrcodegen_alignment_assertss z&ExternKernel.codegen_alignment_assertscCs@tjjrtjjr dS||}|d|d|ddS)zc Track outputs of fallback operators if config.test_configs.track_memory_lifecycle Nz track_tensor(z, 'z')) r? test_configstrack_memory_lifecyclerjrrX"write_memory_track_allocation_oncerr)rrOrrrrcodegen_memory_trackings z$ExternKernel.codegen_memory_tracking'tuple[list[Sequence[Expr]], list[Expr]]cCs|}|}|gg|fS)zD get output sizes and strides, for template_codegen )rr>)r_size_striderrrget_group_strides zExternKernel.get_group_stridetuple[Expr, Sequence[Expr]]c stjj|}|}fdd|D}ddtt|Dttt||jdd}ddt |DfddttD}fd d|D| }|}tjj ||g\}}} t d \} t t|fd d|D} tt|| }|t|fS) zC Manually get canonicalization of the output index crr)rrF)rrrrrz-ExternKernel.canonicalize..cSsg|] }td|qS)re)rcrrrrrrpT)rr|cSrrrrrrrrrz-ExternKernel.canonicalize..crrrrrrrrrcrrrr)rmrrrrccrrrrF)add_varrrrr)rjrrrr>rrrrrrrrJrrrfrrNr) rrrQ index_orderrrr new_sizesrrr replacementr)rrmrrr canonicalizes$   zExternKernel.canonicalizerscCsP|rtnt}t||}|jD]}|||O}q|jD]}|||O}q|Sr)maybe_free_unbacked_symbolsmaybe_free_symbolsrrr/rr)rrsmaybe_get_symbolsrrrrrrs   z!ExternKernel.get_free_symbol_usescsPtdd}d|g}|fddtD7}|dj|S)Nr3zpython_kernel_name=cs$g|]}|jdt|jqSrm)rr)rrKrrrrsz(ExternKernel.__str__..rp)rrJfieldsrr~r)r kernel_namerrrrrys   zExternKernel.__str__rNNNNrN)rr+rYr.rorr/rrr>r2r1r3r+r4r+r6r5r8r7rrr_r`r'rOrorrrr4r+rr)r3r+rrr9)rrrr|)r(rxrr rr rrh)rrrr^r)NNF) rrrrrrrrrrr=)rrrrFrrrr)rrrrrrrr)rrrr0rr)rkr{rrrr)rrrr rr )rrrr)rr)rrrF)=rrrrIr/rrJrKrrr2r3r4rr6r8r9r:r;r<r=rr[r]rBrrSrUr@rArfrLrgrrrrrrrrrrrkrrNrrrrrrrrrrrrryrrrrrtrrks            %       y E   $       $ !      rcsBeZdZdddZ       dd fdd Zd!ddZZS)"ExternKernelOutrOrorrcC||dSr)generate_extern_kernel_outrTrrrrUrzExternKernelOut.codegenrNrYrrorpr/rrr>r2r1r3r+r4r6r8r7c s`||} t| tsJt| td|| ||pid|||| tj||_ tj |dSr) rrrrrsrrjrrrr) rrYror/rr2r3r4r6r8unwrapped_inputsrtrrrs  zExternKernelOut.__init__rcCrrTrrrrrrEr zExternKernelOut.should_allocaterr)rYrrorpr/rrr>r2r1r3r+r4r+r6rr8r7rrr/)rrrrUrrErrrrtrrs rceZdZd fdd ZZS) RandomSeedscountrrrrrcsFttj}tjt|tj|gdg|j|j|ggddtj j ddS)Nrzaten.randint.low_outzat::_ops::randint_low_out::call)rYror/r3r4r8) r!rprPrsrr[rrrVrandintlow_out)rrrlimitsrtrrr s  zRandomSeeds.__init__)rrrrrrrrrrrrrrtrrrcsJeZdZdddZ      dd fdd Zd!ddZd"ddZZS)#r-rOrorrcCrr)generate_extern_kernel_allocrTrrrrUrzExternKernelAlloc.codegenrNrYr.rorpr/rrr>r3r+r4r6r8r7c sp||} tdd| DsJtd|ttt| ||pid|||| g|_tj ||_ tj |dS)Ncsr|rr rrrrr,rhz-ExternKernelAlloc.__init__..) rrrsrrrrrrjrrrr) rrYror/rr3r4r6r8rrtrrr s"  zExternKernelAlloc.__init__rcCrrrrrrrrE@r z!ExternKernelAlloc.should_allocatecCrSrrTrrrrrNCr z"ExternKernelAlloc.apply_constraintr)rNNNrN)rYr.rorpr/rrr>r3r+r4r+r6rr8r7rrr/r')rrrrUrrErNrrrrtrr-s  r-csJeZdZdZdfd d Zdd d ZdddZdddZdddZZ S)rzP An output buffer that represents the mutation of a pre-existing buffer rYr. mutated_noder mutating_noderPrrcsDtjd|d|}tj||g|_||_tj||_ dSrL) rsrrrjrr*mutation_namesrrr)rrYrrmutated_node_namertrrrLs  zMutationOutput.__init__cCrr)rrrrrrVrzMutationOutput.get_defining_oprcCrr)rrrrrr#Yrz!MutationOutput.get_mutation_namesrcCrrrrrrrrE\r zMutationOutput.should_allocaterpcCs |}dddd|DDS)NcSg|]}|dur|qSrrrrrrras z7MutationOutput.get_mutation_buffers..cs|] }tj|VqdSr)rjrtry_get_bufferrorrrrcrz6MutationOutput.get_mutation_buffers..)r#)rrrrrget_mutation_buffers_s z#MutationOutput.get_mutation_buffers)rYr.rrrrPrrrGrHr/rrp) rrrrIrrr#rErrrrrtrrGs  rcs`eZdZUdZiZded<edd d Zedd d Zdfdd Z dddZ dddZ Z S) TMADescriptorad An IR node representing a generic host-side TMA descriptor in the Triton API Mostly useful for user-defined Triton kernels relying on host-side TMA; but can, in principle, be used for Inductor's Triton templates, too. See TMADescriptorExperimental and TMADescriptorStable for the two implementations (the old API and the new API) zdict[Any, TMADescriptor]_CACHErPrtma_metatuple[str, tuple[Any, ...]]rcCsTt|dksJ|ddkrt|g|dRS|ddks Jt|g|dRS)Nrr experimentalr>r)rTMADescriptorExperimentalTMADescriptorStable)rrPrrrr _create_implvs  zTMADescriptor._create_implcCs2t||f}||jvr||||j|<|j|Sr)idrr)rrPrrrrrr~s   zTMADescriptor.createrorr/rcsVtdtt||dttt|t|d||_ t j ||_ t j |dS)NrW)rsrrr^rrrrrrPrjrrrr)rrPror/rtrrrs zTMADescriptor.__init__rOrocCrr)generate_tma_descriptorrTrrrrUrzTMADescriptor.codegencCrr)rPrrrr get_tensorrzTMADescriptor.get_tensor)rPrrrrr)rPrrorr/rrrrr<) rrrrIrrrrr~rrUr rrrrtrrhs     rcs&eZdZdZ ddfd d ZZS)rz the new host-side TMA Descriptor API: (the ones obtained via create_{1d,2d}_tma_descriptor calls). See also TMADescriptorStable for the new API. NrPrr@list[Union[int, torch.SymInt]] block_dims element_sizerrrcst|dvsJt|t|ksJ|dur|j}||_||_||_t|j|_|g}g|j|j|j}tj|||ddS)N)r>rrPror/) rrrr@r r rrsr)rrPr@r r ror/rtrrrs*   z"TMADescriptorExperimental.__init__r) rPrr@r r r r rrrrrrrIrrrrrtrrs rcs"eZdZdZdfdd ZZS) rz the new host-side TMA descriptor API (the ones obtained via TensorDescriptor.from_tensor). See also TMADescriptorExperimental for the old API. rPr block_shaper cs||_tj||g|ddS)Nr)rrsr)rrPrrtrrrs  zTMADescriptorStable.__init__)rPrrr rrrrtrrsrcs(eZdZdfd d ZdddZZS)SubgraphBufferrYrrrZrdreexample_inputs list[Any] subgraph_namerc s,td||||_||_tj||_tj|tj |j|||_ t |j s,Jt |j }|D]}||j j|j<|j j|jq3dd|D|_ddlmm}t|j 0|jdddd|j j|jWdn1swwYWddSWddS1swYdS)NcSsg|]}|jqSrr)rsym_varrrrrsz+SubgraphBuffer.__init__..rFATEN) max_autotunemax_autotune_gemmmax_autotune_gemm_backends)rsrrdrrjrrrr make_subgraphsubgraphr rorvrgraph_input_namesr sym_inputstorch._inductor.configrr?set_graph_handlerrrun) rrYrrdrrrsym_inpinductor_configrtrrrs0  "zSubgraphBuffer.__init__rOrorrcCsRGddd}t|jsJdd|jD}|||jg|j||jgdS)Nc@rH)z,SubgraphBuffer.codegen..CodegenGraphrrpcSs||_|j|_dSr)rr)rrrrrr s z5SubgraphBuffer.codegen..CodegenGraph.__init__N)rrp)rrrrrrrr CodegenGraphrIr#cSg|]}|qSrrrr$rrrrrz*SubgraphBuffer.codegen..)r ro'codegen_subgraph_with_flattened_outputsrrr)rrOr# outer_inputsrrrrUs zSubgraphBuffer.codegen) rYrrrZrdrerrrrrrrrrrUrrrrtrrs$rcsfeZdZd!ddZed"dd Z d#d$fdd Zd%ddZd&fdd Zd'ddZ d(dd Z Z S))UserDefinedTritonKernelr(tuple[Kernel, Any, list[str], list[str]]csddlm}ddlm}||jg}g}g}t|retdr0|fddj Dn tds7J|j tdrRj D] }| j j|qEn td sYJ|jj}j |||fS) Nr) Autotuner)kernel_side_table restore_idxc3s|] }jj|VqdSr)r arg_namesrr(rrr$rzBUserDefinedTritonKernel.get_kernel_and_metadata.. restore_value reset_idx reset_to_zero)triton.runtime.autotunerr,*torch._higher_order_ops.triton_kernel_wrapr- get_kernel kernel_idxrrrr.r1r2rrr/r3configs)rr,r-r8restore_value_argsreset_to_zero_argsrrr0rget_kernel_and_metadatas,           z/UserDefinedTritonKernel.get_kernel_and_metadatarOrorc sddlm}\}}}||j||j\}}}fddjD} tdr1tds7Jtt fddj D} g} g} g} g}t | tt d |D]w\}}|| vre|reqY| |||t|tr| || |qYt|ttttjfr| || t|qY|| vr| d | tqY|d ur |r| d | tqY| |qYtd t|d |||j|| | || |djj d d S)YOverrides the parent member. See https://github.com/pytorch/pytorch/issues/151692r)triton_version_uses_attrs_dictcsi|]}||qSrrrrrrrrQr\z3UserDefinedTritonKernel.codegen..r/ constexprsc3s|]}j|VqdSr)r/rr0rrrWrhz2UserDefinedTritonKernel.codegen..rr@NzUnsupported arg type: r,T) arg_typesraw_argsraw_keys triton_metar4roriginal_fxnode_name)!torch._inductor.utilsr=r;!define_user_defined_triton_kernelrgridr6rrr9r@rHchainrqrrepeatrrrrrrrrrr r}rrSgenerate_kernel_callrrCr)rrOr=r8r9r:new_namerDextra_launch_args named_argsconstexpr_namesrrAraw_keys_filteredraw_args_filteredrrr)r(rrrU6s              zUserDefinedTritonKernel.codegenFrsrrcst|t|j|BSr)rsrr(rHrrtrrrs z,UserDefinedTritonKernel.get_free_symbol_usescCr\rr8rrrrr]rz0UserDefinedTritonKernel.get_unbacked_symbol_defsr7rrHr tma_descriptor_metadatar0 kernel_argscszg}i}g}D]0\}} t| tr1t| } ||vr't| ||} || | ||<q || | ||<q t |dksCJ|d _ t|t sUJt |tdtj d|t|||_|_\} } } } t| dszJfdd| jD_ddlm}t | dkr| djni}fdd|| i||D_fddjD_tjdS) Nrrzr/csg|]}|vr|qSrrrrSrrrrz4UserDefinedTritonKernel.__init__..)identify_mutated_tensorscrrrrrrTrrrscs g|] }ttjd|qSr)rr#rrrrrrs) rqrrrrrrr~rrrrrrrsrr#rr7rHr;rr/r6r5rUr mutable_argsr=rjrr)rr7rHrRrSrorr/rrfr$r(r8rrUautotuned_kwargsrt)rSrrrsP          z UserDefinedTritonKernel.__init__rZcCs t|jSr)rr=rrrrr[rz#UserDefinedTritonKernel.get_outputsrcCrrrzrrrrrrz"UserDefinedTritonKernel.get_device)rr+rr=rFr`) r7rrHr rRr0rSr0rrr_r4) rrrr;rrUrr]rr[rrrrrtrr*s  Z  =r*csJeZdZdZdddZdd d Zdd d ZdddZdfdd ZZ S)InplaceBernoulliFallbackE This needs to be a custom class to handle mutation properly rOrorrc Cstdd|jDs Jdd|jD\}tjjr5||d|ddtt |j d|j dS||d|ddtt |j d|j dS)Ncsr|rr r&rrrrrhz3InplaceBernoulliFallback.codegen..css|] }tt|VqdSr)rrrr&rrrrrrr]z, NULL)r-) rrorjrrXrrfrrreprr/ending)rrOrrrrrUs,,z InplaceBernoulliFallback.codegenrcCrrrrrrrrEr z(InplaceBernoulliFallback.should_allocatercC |dgSrr rrrrr#rz+InplaceBernoulliFallback.get_mutation_namesrcCr\rr8rrrrr]rz1InplaceBernoulliFallback.get_unbacked_symbol_defsr8rxrrr/r csVtjdt|d||g||dtj|tj ||_ tj |dS)Nrzr8) rsrr#rrrjrr*rrrr)rr8rr/rtrrrs  z!InplaceBernoulliFallback.__init__rr/rHr`)r8rxrrr/r rr rrrrIrUrEr#r]rrrrrtrrY    rYcs\eZdZdZd ddZd!d d Zd"d d Zd#ddZd$fdd Ze d%d&ddZ Z S)'InplaceCopyFallbackrZrOrorrcCs |\}}}||||dSr)rcodegen_device_copy)rrOr0r/ non_blockingrrrrUszInplaceCopyFallback.codegenrcCrrrrrrrrEr z#InplaceCopyFallback.should_allocatercCr]rr^rrrrr#rz&InplaceCopyFallback.get_mutation_namesrcCr\rr8rrrrr]rz,InplaceCopyFallback.get_unbacked_symbol_defsrYr.rorpr/rcsJtjd|||dddtj|dtj||_tj|dS)Nz aten.copy_aoti_torch_copy_)r3r4r) rsrrjrr*rrrr)rrYror/rtrrrszInplaceCopyFallback.__init__Fr0rr/rdcs6fdd||fD}|f}tt|d||}|S)Ncrrrvr&rwrrr5rz.InplaceCopyFallback.create..rz)rbr#r)rr0r/rdror/rrrwrr~1s zInplaceCopyFallback.createrr/rHr`)rYr.rorpr/rrrr=)r0rr/rrdrrrb) rrrrIrUrEr#r]rrr~rrrrtrrb s    rbc@sBeZdZdZdddZdd d Zdd d ZdddZdddZdS)MutatingFirstArgExternKernelrZrOrorrcCsXt|jsJgdd|jDtt|j}||dd|d|jdS)Ncs|]}|VqdSrr%r&rrrrGr&z7MutatingFirstArgExternKernel.codegen..rr]r-) r rorr[r/rrfrr\)rrOargrefsrrrrUDs z$MutatingFirstArgExternKernel.codegenrcCrrrrrrrrENr z,MutatingFirstArgExternKernel.should_allocatercCr]rr^rrrrr#Qrz/MutatingFirstArgExternKernel.get_mutation_namesrcCr\rr8rrrrr]Trz5MutatingFirstArgExternKernel.get_unbacked_symbol_defscCrrTrrrrrhas_side_effectsWr z-MutatingFirstArgExternKernel.has_side_effectsNrr/rHr`) rrrrIrUrEr#r]rirrrrrf?s   rfcr) ResizeStorageBytesvariablerr4rrrcst|ts Jdtjdt|d||g|fdtj | tj ||_ tj |d|_d|_t|tttfsFJt|tjj|j dS)NzTODO: dynamic shapesrz)r/z"inductor_ops.resize_storage_bytes_z&torch::inductor::resize_storage_bytes_)rrrsrr#rrrjrr*rrrrr3r4rrrrnever_reuse_buffersrrX)rrkr4rtrrr\s   zResizeStorageBytes.__init__)rkrr4rrrrrrrtrrj[rrjcs(eZdZd fdd Zd d d ZZS) SetSourceTensorKernel self_tensorrstorage_tensorrrcs|tj|||gdtjjjjdt |t t t fs$Jt |tjj|jtjj|tjj||}tt|d||tt|d||g|_dS)Nz!torch.ops.aten.set_.source_Tensor)r3r8rz)rrsrrr!rhrVset_ source_TensorrrrrrrjrrlrrXrrrr#r=)rrnrorrtrrrns"  zSetSourceTensorKernel.__init__rcCs|d|dgSrr^rrrrr%rz2SetSourceTensorKernel.get_inputs_that_alias_output)rnrrorrrrH)rrrrr%rrrrtrrmmsrmcsReZdZdZd!ddZd"d d Zd#d d Zd$ddZdddd%fdd ZZ S)&ScatterFallbackz This needs to be a custom class to handle mutation properly. This class handles both aten.scatter_ and aten.scatter_reduce_. It also handle the case `src` being a scalar properly. rOrorrc Cs|jd}tjjrddd}||vr||}t|jsJ|jr,dd|jD\}}}ndd|jD\}}|jd}||||jd ||g|j |j |j|| dS) NrGrr)rmultiplycsrgrr%r&rrrrr&z*ScatterFallback.codegen..csrgrr%r&rrrrr&r>r) rrjrrXr ro src_is_tensorr/generate_scatter_fallbackr4r3r)rrOrGget_operator_enumrrr/rrrrUs&   zScatterFallback.codegenrcCrrrrrrrrEr zScatterFallback.should_allocatercCs"|jd}t|ts J|gSrr )rrurrrr#s  z"ScatterFallback.get_mutation_namesrcCr\rr8rrrrr]rz(ScatterFallback.get_unbacked_symbol_defsNTrG include_selfr8rxrrrLrrr/rGr+rxc st|t_jrfdd|||fD}|f} nfdd||fD}||f} tjdt|d|| ||dt|ddg|dt j | t j _t j dS) Ncrrrvr&rrrrrz,ScatterFallback.__init__..crrrvr&rrrrrrzrwrGrx)r3r6r8)rrrtrsrr#rrrrjrr*rrrr) rr8rrLrr/rGrxtensorsr/rtrrrs&   zScatterFallback.__init__rr/rr`)r8rxrrrLrrrr/rrGr+rxrrrr`rrrtrrrs     rrcsJeZdZdZdddZdd d Zdd d Zd ddZd!fdd ZZ S)"IndexPutFallbackzQ This needs to be a custom class to handle mutation and indices properly rOrorrc Cst|jsJdd|jD^}}}g}t|}t|jD]\}}|j|dur0|t|q|tjj j q|j | |||g| RdS)Ncsrgrr%r&rrrrr&z+IndexPutFallback.codegen..)r rorlrr rrmrjrrrNgenerate_index_put_fallbackrfr) rrOrr valid_indicesr iter_valid_indicesrrrrrrUs  zIndexPutFallback.codegenrcCrrrrrrrrEr z IndexPutFallback.should_allocatercCr]rr^rrrrr#rz#IndexPutFallback.get_mutation_namesrcCr\rr8rrrrr]rz)IndexPutFallback.get_unbacked_symbol_defsr8torch._ops.OpOverloadrrr rrr accumulater c s|_dd|D}fdd||g|D}d}tjdt|d||fd||dtj dtj _ tj dS) NcSrrrrrrrrrz-IndexPutFallback.__init__..crrrvrFrrrrraoti_torch_index_put_outrzzaten.index_put_)r3r4r8r) r rsrr#rrrjrr*r rrr) rr8rr rrr|ryr4rtrrrs   zIndexPutFallback.__init__rr/rHr`) r8r~rrr rrrrr rrr`rrrtrrzrarzc@s$eZdZeddd Zdd dZdS) DeviceCopyrrrrrdrrc Cs|stdd|Drtjjs||Stj || }|dus(Jtj |t d|f}t |}d}|rD|}t|joO|jdkoO|}|jdko[t|jo[|}|rgt|rgd|_tt|||||d||g|S)Ncss|] }|tjjvVqdSr)rjrr{rrrrrrz$DeviceCopy.create..zDeviceCopy in input programr1Tr)r rrr? aot_inductoruse_runtime_constant_foldingr"rjradd_device_inforrYrrkrr>rarrrr]rr[rr) rrrrdx_devicer/ris_destination_pinnedis_source_pinnedrrrr~ sD        zDeviceCopy.createrOrorcCs\|}t|dks J|jr||d|j|ddS||d||ddS)Nrrr>)rrr2rcr)rrOrrrrrU6szDeviceCopy.codegenN)rrrrrdrrrr)rrrrr~rUrrrrr s )rcsXeZdZdZdddZdddZd fdd Zd!ddZ d"d#ddZd$ddZ Z S)%DynamicSelectStorageOffseta The result of computing a dynamic selection index is determined as follows: when the index in the select operation is unbacked, the actual index calculation is ambiguous for negative indices (index + size) versus non-negative indices (just index). To resolve this, we allocate an unbacked SymInt to represent the storage offset and decompose the select operation into a call to as_strided, computing the storage offset at runtime with this node. rrcCr\rr8rrrrrJrz$DynamicSelectStorageOffset.get_readsrcCrrrrrrrrEMr z*DynamicSelectStorageOffset.should_allocateunbacked_offset_symbol sympy.Symbolr base_offsetUnion[sympy.Symbol, int]base_dim_striderrcs>tdttddg||_||_||_||_||_ dSNr1rz) rsrr#r!rrrrrr)rrrrrrrtrrrPs  z#DynamicSelectStorageOffset.__init__rcC t|jgSr)r9rrrrrr]arz3DynamicSelectStorageOffset.get_unbacked_symbol_defsFrscCrPr)r(rrrrrrdrz/DynamicSelectStorageOffset.get_free_symbol_usesrOrocCrr)codegen_dynamic_select_indexrTrrrrUirz"DynamicSelectStorageOffset.codegenrCr/) rrrrrrrrrrrrr`r=rFr) rrrrIrrErr]rrUrrrrtrrAs    rcsJeZdZdZdddZdddZdfdd ZdddZdddZZ S)rz; The result of a call to aten._local_scalar_dense. rrcCr\rr8rrrrrrrzDynamicScalar.get_readsrcCrrrrrrrrEur zDynamicScalar.should_allocatesymrkeypathpytree.KeyPathrXrrcs<|tdttdd||g||_||_dSr) rrsrr#r!rrrr)rrrrXrtrrrxs  zDynamicScalar.__init__rcCrr)r9rrrrrr]rz&DynamicScalar.get_unbacked_symbol_defsrOrocCrr)codegen_dynamic_scalarrTrrrrUrzDynamicScalar.codegenrCr/)rrrrrXrrrr`r) rrrrIrrErr]rUrrrrtrrms   rcsXeZdZdZdddZdddZdfdd ZdddZ ddddZd ddZ Z S)!rz5 The result of a call to aten._assert_scalar rrcCr\rr8rrrrrrzAssertScalar.get_readsrcCrrrrrrrrEr zAssertScalar.should_allocatescalarrlrrrcs,tdttddg||_||_dSr)rsrr#r!rrr)rrrrtrrrs zAssertScalar.__init__cCrrTrrrrrrir zAssertScalar.has_side_effectsFrsrcCrPr)r(rrrrrrrz!AssertScalar.get_free_symbol_usesrOroc CstjsdStt|jdd}tjjrdStjjr;d|d}tjj j |j dd}| d|d|j d|d dStjj j|j dd}| d |d | d t|j d| |d dS)NFrrzstd::to_string(r-)rgzif (!(z()) { throw std::runtime_error("Expected z but received " + z); }zif not (z):z raise RuntimeError(z = None)r?scalar_assertsrmrlrrjrrcrXrcodegen_cpp_sizevarrrrcodegen_python_sizevarr[r)rrOsymbol symbol_strsizevarrrrrUs& zAssertScalar.codegenrCr/)rrlrrrrr=rFr) rrrrIrrErrirrUrrrrtrrs    rc@seZdZUded<ded<dS)ExternKernelNoderrzexport_schema.NoderNrrrrrrs  rcseZdZdZ d>ddd?fddZd@fdd ZdAddZdBddZdCd!d"Ze dDd&d'Z dEd)d*Z dFd,d-Z dFd.d/Z d0d1ZedAd2d3Ze dGd7d8ZedHd:d;ZdIfdr<,Optional[dict[sympy.Symbol, pytree.KeyPath]]rrc s`tj|t|t||dd_|pi_t|tjjtjj fs.Jd|dt |d|_ |_ |dur:in|_ jdusDJtjjg_g_tj tjj r[dSdj vrddSj j}tjjj r|j|ddS|jrt|std| jj\} }dfdd } tjj || |D] \} } | | | qdS)Nr_F#Fails to create FallbackKernel for r, not supported_c10d_functionalrz'NYI: Can't generate FallbackKernel for infotorch._C.Argumentrr rrcstjtjrt|ttfsJt|tjr#t|ttfr#J|dur)dSjdur0dSdfdd }t jrO|durK|D]}||qDdSdStjsWJ||dS)Nr$rrrcsLj|jdusJjjr$jtt|d|dSdSrO) alias_namesrr alias_infois_writer=rr#rr$)rrrr add_alias+szPFallbackKernel.__init__..handle_aliasing_and_mutation..add_alias)r$rrr) rrr!ListTyperr library_utilsis_tensor_like_typeris_tensorlist_like_type)rrroptional_tensor_argr)rrhandle_aliasing_and_mutations"     z=FallbackKernel.__init__..handle_aliasing_and_mutation)rrrr rr)!rsrruse_runtime_dispatchr<rr!rIrJr_rr8rurr3rjr warn_fallbackrrrrK_libraryrmutates_and_returns_first_argrr is_mutabler%rror/ zip_schema) rrYr(rrrurr<schemarrrrrtrrrsJ   zFallbackKernel.__init__rcsHt}|jtjjjur"|jD]}t|t r!| t | }q|Sr)rsrr8r!_prims rng_primsgraphsafe_run_with_rng_stater/rry with_readr@rcr)rrrrtrrr>s    zFallbackKernel.get_read_writesrOrocCs|||jt|ddSNr<)(codegen_unbacked_symbol_defs_for_outputsrrrrTrrrcodegen_unbacked_symbol_defsJsz+FallbackKernel.codegen_unbacked_symbol_defsContainer[sympy.Symbol]cCs:t|dd}rttjjj|}|dusJ|StSr)rr4rjrrrr3r9rr<resolvedrrrr]Os  z'FallbackKernel.get_unbacked_symbol_defsrcstjGdddt|jsJfdd|jD}|||j\}}tjjrDt |j t j j rD|||}ddt|j jj|D}ndd|D}|j||S)Nc@ eZdZUded<dddZdS) z)FallbackKernel.codegen_args..Shimr refrrcSrr)rrrrrr^rz2FallbackKernel.codegen_args..Shim.__repr__Nr9)rrrrrrrrrShimZ rcsg|]}|qSrr%rFrrrrbrz/FallbackKernel.codegen_args..cSs"g|] \}}tjj||jqSr)rjrrrrE)rparamrrrrrfscSrnrrrFrrrrkrp)rJ dataclassr rorur/rjrrXrr8r!rIrJrrrKrLrr)rrrrrrrrYs  zFallbackKernel.codegen_args Optional[Sequence[torch.Tensor]]rr cCs|r dd|Dnd}|r|sJdd|D}|dSt|tjr%|jSt|ttfr`tdd|D}dd|D}t|dkrF|dS|D]}t|tjsRJt|j r[|SqH|dSdS) NcSsg|] }t|ts|qSr)rrr&rrrrvrpz.FallbackKernel.find_device..cSsg|] }|r|qSrrrrrrr|rOrcss|] }td|VqdSr)r find_devicerFrrrrrz-FallbackKernel.find_device..cSsg|]}|r|qSrr)rrrrrrrr>) rr!rrrrr9rrar)rrnon_torch_bind_tensor_argsdevices device_setrrrrrqs0   zFallbackKernel.find_devicercCs"t|jtjjr dSt|jSr)rr8r!rIr_r.rrrrrriszFallbackKernel.has_side_effectsrcCsrt|jtjjtjjfsJd|jdt|jdt|jtjjs6d|jvr6|jjj r6t |jr6gS|j S)Nrr,rr) rr8r!rIrJr_rrrKrr%rrrrrr%s   z+FallbackKernel.get_inputs_that_alias_outputcCst|jdks J|jSr )rrrrrrr#sz!FallbackKernel.get_mutation_namesc sftdjttsJtjj \} |}fddj D}j}t j js=g||Stdg}|||}dd d t|tjjjrb||d |dj}n|jj}t|dkrjrrjnj}|d j}||g} n fddt|jD} jdusJttjj || idd} t j!"| g||S)a ProxyExecutor Design Note We export the ExternFallbackNodes (for custom ops) into a serialized file and run it with a host side proxy executor to address the ABI problem This is currently only implemented for fbcode. Eventually, we will also make this work for OSS. Detailed design doc can be found at https://docs.google.com/document/d/1wC4DOZFaYym2t1Esz0X5yxlLI3RDnSiyRbUus3bkJ64/edit?usp=sharing z4Extern kernel node added for node %s with target %s.csg|] }j|fiqSrr>rVrrrrrrz.N return_type6Union[torch.TensorType, torch.ListType, torch.JitType]rfUnion[IRNode, Sequence[IRNode]]rexport_schema.ArgumentcSsnt|tjtjfrE|}t|ttfrt|dksJ|d}t|tjr8t|ts+Jtj j tj | ddS|dus>Jtj j ddSt|tj rjt|tjrjt|ts^Jt|tj j dd|Dd St|tjrt|tjr|durtj j tjj ddd St|tsJtj j tjj tj | ddd St|tjrtj j |d Std t|) Nr>rr) as_tensorT)as_nonecSsg|] }tj|dqS)r) export_schemaTensorArgumentr)rrrrrrszZFallbackKernel.export_extern_kernel_node..handle_single_output..) as_tensors)as_optional_tensor)as_intzUnsupported return type )rr! TensorTypeNoneTyperrrrrrmr~rrrgetElementTyperr OptionalTypeOptionalTensorArgumentIntType RuntimeError)rrfrrrrhandle_single_outputsR     zFFallbackKernel.export_extern_kernel_node..handle_single_outputrr>csg|] \}}|j|qSr)rE)r return_schemarf)rrrrs)r>rormetadata)rr)rrrfrrr)#r/rrr8rrrruror/rr6rjraot_moder$serialize_inputsr!r torchbind CallTorchBindrreturnsrKrrr=rErrrr7rextern_kernel_nodesr) rrordered_kwargsr> serializernamed_argumentsrrroutput_argumentsrr)rrrrexport_extern_kernel_nodesN      5     z(FallbackKernel.export_extern_kernel_nodecsj}|dus J|jdkr5t|tjjsJt|tjj r4ddl m }t ||vr4t d|d_n|jdkrHt|tjjsGJt|n tjj rS|tjjv_tjj rt|tjjrjsdfd d jj\}t|fddjD}tfddt||jjD_|jrƈ}jdusJjdusJ| !jfddj|j"rj"nj#n|$tj%t&r'|(|)|*|dS)r<NrVr)inductor_fallback_opszG%s is missing a c-shim implementation, using proxy executor as fallbackT _quantizedr$ torch.JitTyperrcs$t|tjr |St|tjSr)rr!rr NumberTyper is_numberrrrMs   z)FallbackKernel.codegen..is_numberc3s"|] }j|fiVqdSrr>r?rrrrZrez)FallbackKernel.codegen..c3s(|]\}}t|to|jVqdSr)rcomplexrE)rrfrrrrr_s  csgSr)rrrrrrrmrz(FallbackKernel.codegen..)r$rrr)+r8rYrr!rIrJrrjrrXtorchgen.aoti.fallback_opsrrr/rrr?rcustom_ops_to_c_shimsruror/rHrIr6rrrKrLrSrr3,generate_fallback_kernel_with_runtime_lookuprrr=generate_fallback_kernelrYrrrrr)rrOr(rr args_iter exported_argsr)rrrrrU&sl                zFallbackKernel.codegenrfr r[cCsJd}z|}Wn tyYnwt|j|jt|t||dS)NFr)r]rr[rrrWrr)rfr]rrrtensor_to_layout|s    zFallbackKernel.tensor_to_layoutrc sPtjf}||vrttdtjj}nt}|j|g|Ri|\}}}} } Wdn1s3wYt dd|D ||} | sVt |t j jjrVt d} |durht| d|||| | dn| snJdt| d|||| | ddfd d |g} t | ttfr| _| St | trt| _| S| g_| S)z9Create an instance of FallbackKernel from an _OpOverloadsNcsrr)rrrrrrr&z(FallbackKernel.create..r1rzrz"Not sure where to find device inforfr r list[tuple[Any, int]]rcstttfrtfddttDSttr,fddDSttj rNt  }t j sDsDtsLtjj|j|SttrUSttjr_jjSdusmJdtddS)Nc3s,|]}|t|fgVqdSrrrgenerate_outputr rfrrrs  zAFallbackKernel.create..generate_output..cs*i|]\}}||t|fgqSrr)rrrVrrrrrzBFallbackKernel.create..generate_output..zFallbackKernel output type z is not supported)rrrrrrrrqr!r MultiOutputrr? assume_unaligned_fallback_outputrgrjrrrrrSymIntrr)rfr r"rrhas_unaligned_inputpacked)r rfrrs<      z.FallbackKernel.create..generate_output)rfr r rrr )rV*_fused_moving_avg_obs_fq_helper_functionalrrrjrrr rrrrr!rrrrr#rrrrr) rr(rrfake_incorrect_kernelscontextrrrrur<rrrrrr~s^     " zFallbackKernel.createcs tSr)rsrNrrtrrrNrzFallbackKernel.apply_constraintrrYr.r(rxrrprrrurbrr>r<rrrrBr)rrr)rrrrrr r/rH)rfr rr[)r(rxrr rr rrr')rrrrIrrrr]rrLrrir%r#rrrUrrr~rNrrrrtrrs.  j      y U  ]rcs<eZdZdZdddZdddZd d dfddZZS) ComplexViewz9View a complex number as two dtyped numbers or vice versarrcCrrrrrrrrEr zComplexView.should_allocatercCr]rr^rrrrr%rz(ComplexView.get_inputs_that_alias_outputNrrYr.r(rxrrprrrurbr<rrcstj||||||ddS)Nr)rsr)rrYr(rrrur<rtrrrs  zComplexView.__init__r/rH)rYr.r(rxrrprrrurbr<rrr)rrrrIrEr%rrrrrtrr s    r c@seZdZdZd ddZdS) MemoryCheckKernelz Custom kernel for memory checking that generates direct function calls TODO - the custom op was erroring with str inputs. should be able to custom op directly. rOrorrcCsl||j\}}}t|}t|}|r&|dd|d|d|d}n d|d|d}||dS)z.Override codegen to write direct function callzV# note: dont currently distinguish between buffers returned and dealloc'd in last stepzcheck_memory_step(allocated=z, freed=z, is_final_step=r-N)rr/r[r)rrO alive_list dead_list is_final_step alive_repr dead_reprcallrrrrUs zMemoryCheckKernel.codegenNr)rrrrIrUrrrrr  sr c@r) rrrrrcCrrrzrrrrr*rzMultiOutputLayout.get_deviceNr4)rrrrrrrrrr&rrcsNeZdZdddZ ddfdd Z ddddZdddZd ddZZS)!rrOrorrcCs,|||js||||dSdSr)codegen_multi_output!skip_size_stride_alignment_checksrrrTrrrrU/s  zMultiOutput.codegenFrYr.r rr list[tuple[Any, ...]]rrcs>td||gdtj||_tj|||_||_dSr) rsrrjrrrrr r)rrYr r rrtrrr5s   zMultiOutput.__init__rsrcCs&|jd}t|tsJ|||Sr)rorrr)rrsrrrrrBs  z MultiOutput.get_free_symbol_usescCst|jdkot|jdtS)Nr>r)rrorrrrrrrEIszMultiOutput.should_allocatercCsdd|jDS)NcSs.g|]}t|trt|dkr|qSr)rrrr%rrrurrrrOsz.)rorrrrr%Nsz(MultiOutput.get_inputs_that_alias_outputrr=) rYr.r rr rrrrrrFr/rH) rrrrUrrrEr%rrrrtrr.s   rc@seZdZUdZded<dvddZdwd d Zdxd d ZdyddZdzddZ d{ddZ d|d}ddZ d~dd Z dd!d"Z dd#d$Zdvd%d&Zdd'd(Z )ddd-d.Zdd/d0Zdd2d3Z )ddd5d6Zdd8d9Zdd;d<Zdd=d>Zdd@dAZddCdDZddFdGZdvdHdIZdvdJdKZddNdOZddQdRZd{dSdTZddUdVZ ddWdXZ! )ddd[d\Z"dd^d_Z#ddadbZ$d|ddedfZ%e&ddhdiZ'ddkdlZ(ddmdnZ)ddodpZ*e&ddrdsZ+d{dtduZ,e,Z-dS)r,zC TensorBox / StorageBox allow in-place mutation of Tensors rrXrrcCr&rr,rrrrrarz!MutableBox.has_exceeded_max_readsrcCr&rr)rrrrrdrzMutableBox.get_devicercCr&rrrrrrrgrzMutableBox.make_loaderrcCr&r)rXrrrrrrjrzMutableBox.make_indexerrFcCr&r)rXr>rrrrr>mrzMutableBox.get_stridercCr&rr*rrrrrprzMutableBox.get_nameNrrcCrr)rXrrrrrrsrzMutableBox.has_large_inner_fnrrrcCrrr+rrrrrvrzMutableBox.mark_reusecCr&rr.rrrrryrzMutableBox.realize_hintcCr&r)rXrrrrrr|rzMutableBox.unwrap_viewcCr&r)rXrrrrrrrzMutableBox.is_input_buffercCr&r)rXrrrrrrrzMutableBox.freeze_layoutFrrrcC|j||Sr)rXr r rrrr r z*MutableBox.freeze_layout_with_stride_ordercCrr)rXr r rrrr rz(MutableBox.freeze_layout_with_fill_orderrcCrr)rXrrrrrrrz(MutableBox.freeze_layout_with_same_orderrcCrr)rXrrrrrrr z+MutableBox.freeze_layout_with_exact_stridesrcCr&r)rXrrrrrrrzMutableBox.get_read_writesrcCr&rrYrrrrrrzMutableBox.get_readscCr&rrXrrrrrrzMutableBox.num_readsrvcCr&rr/rrrrrrzMutableBox.get_storage_numelr+cCr&rrrrrrrrzMutableBox.get_reduction_typercCr&rrrrrrrrzMutableBox.get_reduction_sizecCr&rr0rrrrr rzMutableBox.is_externcCr&r)rXr!rrrrr!rzMutableBox.is_no_oprrcCrrrrrrrr"rzMutableBox.constant_to_devicercCr&r)rXr#rrrrr#rzMutableBox.get_mutation_namescCr&r)rXr$rrrrr$rzMutableBox.get_operation_namecCr&r)rXr%rrrrr%rz'MutableBox.get_inputs_that_alias_outputcCr&rr-rrrrrrzMutableBox.realizersrcCrrrrrrrrrzMutableBox.get_free_symbol_usesrcCr&rr2rrrrrrzMutableBox.get_read_namesrcCr&r)rXrrrrrrrzMutableBox.get_defining_oprrcCrr)rXrrrrrrrzMutableBox.codegen_referencer.cCr&rrXrrrrrrYs zMutableBox.layoutrcCr&rr(rrrrrrzMutableBox.get_layoutcCr&rrrrrrrrzMutableBox.get_output_speccCr&rrrrrrrrzMutableBox.get_sizercCrr)rXrrrrrrrzMutableBox.dtypecCsnt|jtrt|jdt|jjd}d}|jj}n t|jd}|j}d}|tt||g}d|S)Nrz))r-r)rrXr,rrrrr)rline0endlr"rrrrrys    zMutableBox.__str__r/r4r6r7r8r9rr:r;r'r<r=r>r?r@rArBrCrDrEr2r0rGrHrFr(r*r3r.r-r,).rrrrIrrrrrr>rrrrrrrr r rrrrrrrrr r!r"r#r$r%rrrrrrOrYrrrrryrrrrrr,Ys`                                     r,c@seZdZedddZdS) rrXrrr|cCst|tr|Stt|Sr)rrrr)rXrrrr~s  zTensorBox.createN)rXrrr|)rrrrLr~rrrrrsc@sjeZdZdZdddZdddZdd d Zdd d ZdddZdddZ d ddZ d!ddZ d"ddZ dS)#rz7 StorageBox allow in-place mutation of Tensors rrcCs&t|jttfr|jtjjvSdSr)rrXrr^rrjrrrrrrrszStorageBox.is_input_buffercCst|jto|jtjjvSr)rrXrrrjrr{rrrrr1 s zStorageBox.is_module_bufferr+cCst|jr |jSt|jttttfsJt |j|j }|j }|j }|dus1Jt dt||j|jdd|jd|_tj|j|j_tj|j|j|j_||j_||j_|jjS)NF)rrrr]r)rrrXrrrrrrrrrrrrrrrjrrrrrzr~r|)rr~r|rrrrr s2        zStorageBox.realizercCs4t|jttfr|jjdkr|dSdSdS)zL Called on buffers we expect to be forced to realize later. r>N)rrXrrrnontrivial_read_countrrrrrr% s  zStorageBox.realize_hintrrcCstdd|D|kS)Ncsrr)rjrget_dep_size_hintrrrrr1 rzBStorageBox.has_accumulated_enough_reads_by_size..)rrrrrr$has_accumulated_enough_reads_by_size/ sz/StorageBox.has_accumulated_enough_reads_by_sizecCs8t|jto|tjkp|ptjduo|tjSr) rrXrrr?realize_acc_reads_thresholdr realize_acc_reads_size_thresholdrrrrrr4 s  z!StorageBox.has_exceeded_max_readsrcsh|dkr2t|jttfr2t|jr'|jddg}tfdd|Dr'dS|tj kp1| SdS)zj A heuristic to decide if we should realize a tensor that is used multiple times. r>expsigmoidc3s|]}|jvVqdSr)used_opsrFopcountrrrJ rhz5StorageBox.should_realize_on_reuse..TF) rrXrrr:rrrr?realize_reads_thresholdr)rr heavy_opsrr#rshould_realize_on_reuse@ s  z"StorageBox.should_realize_on_reusecCs||r |dSdSr)r'rrrrrrR s  zStorageBox.mark_reusecCr&rrXrrrrrV rzStorageBox.num_readsNr/r2r')rrrr)rrrrr;rD) rrrrIrr1rrrrr'rrrrrrrs      rc@s*eZdZUded<ded<dZded<dS)Subgraphrrre graph_moduleNzOptional[GraphLowering]r)rrrrrrrrrr(Z s r(bufferscCs,dd|D}ttdd|Dt|kS)NcS"g|] }t|tr |n|qSrrr^rrrrrrrb z(_has_aliased_buffers..csrr)rr-rrrrg r&z'_has_aliased_buffers..)rr9)r*rrr_has_aliased_buffersa sr/cs`eZdZUdZdZded<dZded<dZded<dfdd Ze dddZ dddZ Z S)InvokeSubgraphz. Ir node for the invoke_subgraph HOP. NOptional[Subgraph]rOptional[Sequence[IRNode]]operandsrr(rprYrrrcs6tjd||d||_tj||_tj|dSr?)rsrrrjrrrr)rrr3rYrtrrrt szInvokeSubgraph.__init__rElist[Union[ShapeAsConstantBuffer, NoneAsConstantBuffer, MultiOutput]]csddlm}tjj}d}|jd}r|ddd}n|jdd}dd|D}fd d|D}g}t|D]\} } t | t t frK| | q:| || || q:|}|jdurtjj |j||jd |_t|j|jj|Wdn1swY|jj} d} |D] } t | t s| } nq| dusJt||t| d d dfdd fddt| D} | _| S)zFor each operand, get a realized input, force it to have the same strides as the subgraph inputs, then use an InvokeSubgraphr>)constrain_to_fake_tensorNeager_input_valsrrcSg|]}|jdqSrXrkrFrrrr rz)InvokeSubgraph.create..crrrvrFrwrrr rrdrrrz)rr3rYrfrindrr?Union[ShapeAsConstantBuffer, NoneAsConstantBuffer, MultiOutput]c sdt|ttfr |S|}|dusJtt||||| j | j dt |fgddS)Nr!T)r) rrrMrrr[rrr>rr\r]r)rfr:r)invoke_subgraphrr create_output s" z,InvokeSubgraph.create..create_outputcsg|] \}}||qSrr)rrrf)r=rrr rp)rfrr:rrr;)rr5rjrrrkrrrrrryrrr)rrr  graph_outputsrr0rr)rrr3r5r fake_operandsr6 fx_operands new_operandsroperandrroutsr)rr=r<rr~ sT      zInvokeSubgraph.createrOrocCrr)codegen_invoke_subgraphrTrrrrU rzInvokeSubgraph.codegen)rr(r3rprYrrr)rr(r3rrr4r) rrrrIrrr3rrrr~rUrrrrtrr0j s     Yr0cseZdZUdZded<dZded<dZded<dZded<dZd ed <d(fdd Z e d)ddZ e d*dd Z d+d#d$Zd,d&d'ZZS)- ConditionalNr  predicater2r3r1 true_subgraphfalse_subgraphOptional[Sequence[MultiOutput]]rrrpr(rYrr<rrrc sj||_||_||_||_t|g|\}}tjd|||d|dur&||_tj ||_ tj |dSN)rrYror/) rFr3rGrH_split_by_sym_typersrr<rjrrrr) rrFr3rGrHrYr<sym_argsrrtrrr s zConditional.__init__rUnion[int, torch.SymInt]Union[int, sympy.Expr]cCst|tr|S|jjSr)rrrr)rrrr _maybe_expr!s zConditional._maybe_exprrtrue_fnfalse_fn-list[Union[TensorBox, ShapeAsConstantBuffer]]c sX|}fdd|D}tjjjd}t|ts Jt|tdd|Ds+Jdd|D}||fD]/}|jduretjj |j ||j d|_t |j|jj |Wdn1s`wYq6|jdusmJ|jdustJ|jj}|jj} d |fd | ffD]\} } t|rtd | d | qt|t| ksJ|| ftt|| D]5\} \} }| |ksJ| | |f| |ksJ| | |f| j|jksJ| | |fqtd d|g|Dttjjjtjjjdd}dusJdt||||td|dfddtt|tjjjdD} | _ | S)zNCreate a Sequence of IRNodes from a conditional statement (see .lowering.cond)crrrvrFrwrrr!rz&Conditional.create..r@csr|r)rr7r[rrrr!rhz%Conditional.create..cSsg|] }tt|jdqSrX)rr7rkrFrrrr!rONr9rPrQzVOutput aliasing is currently not supported in compiled torch.cond. The outputs of the z% subgraph of torch.cond are aliased: css"|] }t|ts|VqdSr)rrr)rorrrr7!s r<zcannot determine devicerz)rFr3rGrHrYr<c sbg|]-\}\}}tt|dd|Ddd|D|j|jdt|fgqS)cSrLrrErOrrrrrrO!rz1Conditional.create...cSrLrrTrUrrrrP!s r!) rr[rrrrr\r]r)rrrf merged_output) conditionalrrrrJ!s"  rV)!rrjrrrrrrrrr)rrr r>r/rWrrrrrrr\rmr4rrrkrrErr)rrFrPrQr3r@r?r true_outputs false_outputsrrrt_of_or<r)rrWrrr~!sv   $ zConditional.createrOrocCs*|||||jt|didSr)codegen_conditionalrrrrrTrrrrUc!s zConditional.codegenrcC>t|dd}rttjjj|}|dusJt|StSrrr4rjrrrr9r3rrrrr]i!   z$Conditional.get_unbacked_symbol_defs)rFrr3rprGr(rHr(rYrr<rrr)rrMrrN) rFrrPr(rQr(r3rRrrprr`)rrrrFrr3rGrHrrrLrOrr~rUr]rrrrtrrE s        ZrErr-tuple[list[ShapeAsConstantBuffer], list[Any]]cCs<g}g}|D]}t|tr||jq||q||fSr)rrrr)r non_sym_argsrLrrrrrKt!s  rKcseZdZUdZdZded<dZded<dZded<dZded<dZ d ed <d%fdd Z e d&ddZ e d'ddZd(d d!Zd)d#d$ZZS)* WhileLoopzSThe IR node for while_loop and while_loop_stack_output. It supports input mutation.Nr2carried_inputsadditional_inputsr1 cond_subgraph body_subgraphrIrrpr(rYrr<r stack_outputrrrc sr||_||_||_||_tg||\}} tjd|| |d|dur'||_||_t j ||_ t j |dSrJ)rcrdrerfrKrsrr<rgrjrrrr) rrcrdrerfrYr<rgrLrrtrrr!s$  zWhileLoop.__init__cCst|s|Sddlm}dd|D}t}g}tt||D] \}\}}t||vr3|||q|t|||q|S)Nr>)clonecSr+rr,r-rrrr!r.z3WhileLoop._clone_aliased_inputs..) r/rrhr9rrrrr)rcrhunwrapped_buffers seen_buffersrroriginal_inputunwrapped_bufferrrr_clone_aliased_inputs!s    zWhileLoop._clone_aliased_inputscond_fnbody_fnrc& sddlm}d*d d }tjjjd }tjjjd } || } d d| D} dd|D} dd| D} fdd|D}t|}||| }fdd|D}||| }||||fD]S}|jdurt| t smJt | tjj |j | |j d|_t|j'|jj| ||urt|jjt| ksJ||jj| |j_Wdn1swYq[|jr|jsJ|jj}|jj}t|rtd|t|dksJ||d}t|ts|tjksJ|t|dksJ|tdksJdd}|dus Jt|t|ksJ||ftt||D]?\}\}}d+dd}||||||||ksOJ||||f||ks_J|||fq!|dushJttjjjtjjj !dd}t||||t"|d||d }|jdurt|jj#tj$j%sJ||jj#| d!}t&|}fd"d|D}t'|}g}g|_(g|_)|rt|dksJd#ttjjj d$D]/\} }!t*t+|!j,|!j-d%d|!.Dd&d|!/Dd'|t0| fg}"|j(1|"|1|"qnVt|D]Q\} }!| |vr/| t|ksJd(t2|}#|j)1t3|#j4|#||1|#qt*t+|!|!|!|!|!5j6d)|t0| fg}"|j(1|"|1|"qt||D]\}$}%|$7tjj8vrrtjj9:|%7q\|S),zcreate the while_loop IR node. stack_output controls whether it stack each iterations' output, which is necessary for training. r)check_input_alias_and_mutation tensor_boxesrp fake_tensors,list[Union[int, torch.SymInt, torch.Tensor]]rrcSsbt|t|ks Jg}t||D]\}}t|tjr)|tj||ddq||q|S)NFr) rrrr!rrrrr)rqrrretrfkrrr_require_exact_strides!s   z0WhileLoop.create.._require_exact_stridesr@cSr7rXr8rFrrrr!rz$WhileLoop.create..cSr7rXr8rFrrrr!rcSr7rXr8rFrrrr!rcrrrvrFrwrrr!rcrrrvrFrwrrr!rNr9zOutput aliasing is currently not supported in compiled torch.while_loop. The outputs of the body_fn subgraph of torch.while_loop are aliased: r>z9torch.while_loop is assumed to have at least one operand. lhs_exprs Sequence[Union[int, sympy.Expr]] rhs_exprsrcSs<t|t|ks Jt||D] \}}tjj||qdSr)rrrjrrrr)rxrzlhsrhsrrr_guard_list_equals3"sz,WhileLoop.create.._guard_list_equalsr<rz)rcrdrerfrYr<rgrcrrr)rr) all_inputsrrr]"rz-NYI: while_loop_stack_output input mutations.rVcSrLrrTrUrrrrn"rcSrLrrT)rrrrrro"r)rrrrzonly carries can be mutated.)rrrrr\)rqrprrrsrr)rxryrzryrr);torch._higher_order_ops.utilsrprjrrrrbrmrrrrr)rrr rr>r/rWrrr!rrrrrr>r4rrrkrrmodulefx GraphModuler9rlrr=rr[rrrrrrrmrrYrr\rrrlr)&rrnrorcrdrgrprvfx_carried_inputsfx_additional_inputs fx_all_inputsfake_all_inputsfake_carried_inputsfake_additional_inputscarried_inputs_additional_inputs_r cond_outputs body_outputsrrrrgbor}r< while_loop mutated_idxsmutated_idx_setrmutated_inputs_iter all_outputsrrf multi_out mutated_inputrurr)r~rrr~!s           "$      zWhileLoop.createrOrocCs.|||j|||jt|didSr)codegen_while_looprgrrrrrTrrrrU"szWhileLoop.codegenrcCr]rr^rrrrr]"r_z"WhileLoop.get_unbacked_symbol_defs)rcrprdrprer(rfr(rYrr<rrgrrr)rcrprrp) rnr(ror(rcrprdrprgrrrrr`)rrrrIrcrrdrerfrrrLrmrr~rUr]rrrrtrrb!s      "   Nrbcs@eZdZ ddddfddZdfdd ZdddZZS)rNrrYr.r(rxrrprrrurbrr>r<rrrc s~tj|||||d|dddlm}dd|D} ||g|| R|} | dus+J| |_tjj| d|_ |tjj| <dS)Nrr<r)get_effect_keycSs g|] }t|tr |jn|qSr)rrr%rrrrr"sz,EffectfulKernel.__init__..) rsrtorch._higher_order_ops.effectsr effect_typerjr effectful_opsrprev_effect_buffer) rrYr(rrrurr<runcovered_argsrrtrrr"s$   zEffectfulKernel.__init__rcs0t}|jdur|jt|j|Sr)rsrrrrr@rcr)rrrtrrr"s  zEffectfulKernel.get_read_writesrcCrrTrrrrrri"r z EffectfulKernel.has_side_effectsrr rBr/)rrrrrrirrrrtrr"s   rc@seZdZ d d ddZdS) rFrsrrrcCr\rr8rrrrr"rYz!NonTensorObj.get_free_symbol_usesNr=rF)rrrrrrrrr"src@sReZdZUded<ded<dddZddd d Zdd dZdddZdddZdS)rrr+Union[FakeScriptObject, torch.ScriptObject]r%rcCrrrrrrrr"rzTorchBindObject.get_nameNrrcCrrrrrrrr"rz!TorchBindObject.codegen_referencecCrrr(rrrrr}"rzTorchBindObject.get_valuetorch.ScriptObjectcCst|jtjr |jS|jjSr)rr%r! ScriptObjectreal_objrrrr get_real_obj"szTorchBindObject.get_real_objrcCsN|}t|ds Jt|}t|d}dd|D}ttj |dS)N__obj_flatten__rcSs(g|]}t|tjr||qSr)rr!rr numelrFrrrr"s  z1TorchBindObject.get_buf_bytes..) rrrrrnrxr+rGoperatorr)rreal_script_obj flat_dict flat_elems flat_sizesrrr get_buf_bytes"s zTorchBindObject.get_buf_bytesr9rr3)rr)rrrD) rrrrrrr}rrrrrrr"s     rc@s4eZdZUded<ded<d ddZddd d ZdS)ryrrrrrcCrrrrrrrr#rzGeneratorState.get_nameNrrcCrrrrrrrr #rz GeneratorState.codegen_referencer9rr3)rrrrrrrrrrry#s  ryc@sHeZdZdddZdddZddd d ZedddZedddZdS)_CollectiveKernelrrcCrrrrrrrrE#r z!_CollectiveKernel.should_allocatecCrrTrrrrrri#r z"_CollectiveKernel.has_side_effectsNr4r+rcCsRt|jtjjus Jd|j}|dur||_n|jj|_dd|jjD|_ dS)Nz,Setting cpp kernel needs a valid op_overloadcSrFrrGrFrrrr!#rHz9_CollectiveKernel.set_cpp_kernel_name..) rr8r!rIrJr4rKrrLr6)rr4r(rrrr@#s  z%_CollectiveKernel.set_cpp_kernel_namer(rxroUnion[IRNode, list[IRNode]]rr rc stjj|j||g|Ri|\}}}}} Wdn1s!wY| r1J|d| |D]} | q3|d|td||||t|} j fdd| Dj dd| Dd|vrj t td|dj |ddSdS)NrRrrzcrrrrrrrrrJ#rgz4_CollectiveKernel.create_inplace..cSr$rrBrrrrrN#rr)rjrrrrrr#rn tree_leavesr=rrrrr) rr(rorr_example_outputrrrur< tensor_arginpsrrrcreate_inplace+#s>    z _CollectiveKernel.create_inplace!Union[TensorBox, list[TensorBox]]+Union[list[MultiOutput], _CollectiveKernel]cs@tjjj||g|Ri|\}}}}} Wdn1s!wY| r1J|d| |D]} | q3t|tr||} | dusKJt| d||||fddt |D_ t j |D]\} } t j stt| s|tjj| jqij S|||||t j st|stjjjg_ S)Nr]rzcs(g|]\}}t|t|fgqSr)rrr)rrrPrrrrr#sz9_CollectiveKernel.create_out_of_place..)rjrrrrrrrrrrrr?rrgrrrr)rr(rorrrrrrur<rrr"rPrrrcreate_out_of_placel#sZ      z%_CollectiveKernel.create_out_of_placer/rr) r(rxrorrr rr rr) r(rxrorrr rr rr) rrrrErir@rrrrrrrr#s    @rc2eZdZ ddddfddZdddZZS)_AllReduce_KernelNrrYr.r(rxrrprrrurbrr>r<rrrc (tj|||||d|d|ddS)Nr+aoti_torch_cpu__c10d_functional_all_reduce_rsrr@rrYr(rrrurr<rtrrr#  z_AllReduce_Kernel.__init__rOrocC2|d||t|jtr||dSdSNz+torch/csrc/inductor/aoti_torch/c/shim_cpu.hinclude_extra_headerrrrYrrrTrrrrU#   z_AllReduce_Kernel.codegenrr rr)rrrtrr#  rcr)_AllReduceKernelNrrYr.r(rxrrprrrurbrr>r<rrrc r)Nr*aoti_torch_cpu__c10d_functional_all_reducerrrtrrr#rz_AllReduceKernel.__init__rOrocCrrrrTrrrrU#rz_AllReduceKernel.codegenrr rr)rrrtrr#rrcsXeZdZ d"ddd#fddZd$ddZd%ddZed&ddZd'fd d! ZZ S)( _WaitKernelNrrYr.r(rxrrprrrurbrr>r<rrrc r)Nr+aoti_torch_cpu__c10d_functional_wait_tensorrrrtrrr#rz_WaitKernel.__init__rOrocCrrrrTrrrrU#rz_WaitKernel.codegencCs|jd}t|ts Jt|tr$|jd}t|ts!Jt||gSt|trB|jd}t|tr@|jd\}}|j|gSgSgSr)rorrrrrr )rrurcollrrrrrget_volatile_reads$s       z_WaitKernel.get_volatile_readsrurc Cstjj|||\}}}}}Wdn1swY|r*J|d||t|d||||}|jtt|d||dS)NrRrz) rjrrrr#rr=rr) rr(rurrrrur<rrrr create_wait$s(   z_WaitKernel.create_waitrcs6t}|}|D] }|jt|q |Sr)rsrrrrr@rcr)rrvolatile_readsvrrtrrr2$s z_WaitKernel.get_read_writesrr rr)r(rxrurrrrB) rrrrrUrrrrrrrrtrr#s    rrrcCdt|ttfr t|St|ttfr%ttj}|D]}|t |O}q|St|t j r/t|StSr) rr6r r2rrr9rr"rr!rrrr$rrrr>$  rcCrr) rr6r r1rrr9rr"rr!rrrrrrN$rr)rrrr)rrrr)rrrr)rrrr)rrrrrrr)rrrrrr)rrrrr+)rrrrrr)rrrrrr )rr rrrr)r%r&rr')rr*rr+)rr0rr)rr;r<rrr)rErFrGrFrHrFrr)rPrrQrRrr)rdrerr)rorprrq)rrrrrri)rrrrrrrr)rrrrr\r rrr)TFNFN)rrrrr rr rrrrrrr)rrr rrrr&)rrFrHrFrr)rrrr)rrrr)r*rprr)rrrr`)rrrr(J __future__rrMrJr+rHloggingrostextwrapr|collections.abcrrrrrrr enumr r typingr r rrrrrrrrrrtyping_extensionsrrrrrrr unittest.mockrrr r!r"torch._export.serde.schema_exportserderrtorch._library.utilsrrrtorch._loggingr!torch.fxtorch.utils._pytree_pytreerntorch._dynamo.utilsr#torch._export.serde.serializer$*torch._higher_order_ops.auto_functionalizer%torch._inductorr&rFr(torch._prims_commonr)r*r+r,r-torch._subclasses.fake_tensorr.%torch.fx.experimental.symbolic_shapesr/r0r1r2r3r4r5r6 torch.fx.noder7torch.utils._ordered_setr9torch.utils._sympy.functionsr:r;r<torch.utils._sympy.symbolr=rr?r@codegen.commonrArBrCrDrErFrGrHrIrJ loop_bodyrK ops_handlerrLrMrNrOruntime.benchmarkingrP runtime.hintsrQrRrSrTrUrVrWrXrYrZr[r\r]r^r_r`rarbrcrdrerfrg virtualizedrhrirj"torch._library.fake_class_registryrkrlrmcodegen.cuda.cuda_templaterncodegen.wrapperrorrprqrrr4 __version__rr ImportErrorrrrsrtrurrvrrwrIrJr_rx getLoggerrr/rrVgetenvrrrrrrrrrrrrrrrrrrr r)r/r9r:rDrOrcrnrvrrPrarrrrrrrTrrrrrrrrr rZrrrrr?rKrXrr^rrrrrrrr.rr[rrrrr#r(rrFrrJrrMrrrrrrPrimitiveInfoTyperrrrrrr rrrrrrr-rrrrrr*rYrbrfrjrmrrrzrrrrrrr r rrr,rrr(r/r0rErKrbrrrryrrrrrrrrrrsv  8$       (    \        *    "           '  :K ## ?r :&_I   < `Q0=$7U)b  z #(Y sH*O=ZTD|%+!@(4K-2N76,>%+b s)/! X