@@ -116,11 +116,14 @@ function inline_cost_clamp(x::Int)
116
116
return convert (InlineCostType, x)
117
117
end
118
118
119
+ const SRC_FLAG_DECLARED_INLINE = 0x1
120
+ const SRC_FLAG_DECLARED_NOINLINE = 0x2
121
+
119
122
is_declared_inline (@nospecialize src:: MaybeCompressed ) =
120
- ccall (:jl_ir_flag_inlining , UInt8, (Any,), src) == 1
123
+ ccall (:jl_ir_flag_inlining , UInt8, (Any,), src) == SRC_FLAG_DECLARED_INLINE
121
124
122
125
is_declared_noinline (@nospecialize src:: MaybeCompressed ) =
123
- ccall (:jl_ir_flag_inlining , UInt8, (Any,), src) == 2
126
+ ccall (:jl_ir_flag_inlining , UInt8, (Any,), src) == SRC_FLAG_DECLARED_NOINLINE
124
127
125
128
# ####################
126
129
# OptimizationState #
@@ -144,19 +147,50 @@ struct InliningState{Interp<:AbstractInterpreter}
144
147
edges:: Vector{Any}
145
148
world:: UInt
146
149
interp:: Interp
150
+ opt_cache:: IdDict{MethodInstance,CodeInstance}
147
151
end
148
- function InliningState (sv:: InferenceState , interp:: AbstractInterpreter )
149
- return InliningState (sv. edges, frame_world (sv), interp)
152
+ function InliningState (sv:: InferenceState , interp:: AbstractInterpreter ,
153
+ opt_cache:: IdDict{MethodInstance,CodeInstance} = IdDict {MethodInstance,CodeInstance} ())
154
+ return InliningState (sv. edges, frame_world (sv), interp, opt_cache)
150
155
end
151
- function InliningState (interp:: AbstractInterpreter )
152
- return InliningState (Any[], get_inference_world (interp), interp)
156
+ function InliningState (interp:: AbstractInterpreter ,
157
+ opt_cache:: IdDict{MethodInstance,CodeInstance} = IdDict {MethodInstance,CodeInstance} ())
158
+ return InliningState (Any[], get_inference_world (interp), interp, opt_cache)
159
+ end
160
+
161
+ struct OptimizerCache{CodeCache}
162
+ wvc:: WorldView{CodeCache}
163
+ owner
164
+ opt_cache:: IdDict{MethodInstance,CodeInstance}
165
+ function OptimizerCache (
166
+ wvc:: WorldView{CodeCache} ,
167
+ @nospecialize (owner),
168
+ opt_cache:: IdDict{MethodInstance,CodeInstance} ) where CodeCache
169
+ new {CodeCache} (wvc, owner, opt_cache)
170
+ end
171
+ end
172
+ function get ((; wvc, owner, opt_cache):: OptimizerCache , mi:: MethodInstance , default)
173
+ if haskey (opt_cache, mi)
174
+ codeinst = opt_cache[mi]
175
+ @assert codeinst. min_world ≤ wvc. worlds. min_world &&
176
+ wvc. worlds. max_world ≤ codeinst. max_world &&
177
+ codeinst. owner === owner
178
+ @assert isdefined (codeinst, :inferred ) && codeinst. inferred === nothing
179
+ return codeinst
180
+ end
181
+ return get (wvc, mi, default)
153
182
end
154
183
155
184
# get `code_cache(::AbstractInterpreter)` from `state::InliningState`
156
- code_cache (state:: InliningState ) = WorldView (code_cache (state. interp), state. world)
185
+ function code_cache (state:: InliningState )
186
+ cache = WorldView (code_cache (state. interp), state. world)
187
+ owner = cache_owner (state. interp)
188
+ return OptimizerCache (cache, owner, state. opt_cache)
189
+ end
157
190
158
191
mutable struct OptimizationResult
159
192
ir:: IRCode
193
+ inline_flag:: UInt8
160
194
simplified:: Bool # indicates whether the IR was processed with `cfg_simplify!`
161
195
end
162
196
168
202
mutable struct OptimizationState{Interp<: AbstractInterpreter }
169
203
linfo:: MethodInstance
170
204
src:: CodeInfo
171
- result :: Union{Nothing, OptimizationResult}
205
+ optresult :: Union{Nothing, OptimizationResult}
172
206
stmt_info:: Vector{CallInfo}
173
207
mod:: Module
174
208
sptypes:: Vector{VarState}
@@ -179,13 +213,15 @@ mutable struct OptimizationState{Interp<:AbstractInterpreter}
179
213
bb_vartables:: Vector{Union{Nothing,VarTable}}
180
214
insert_coverage:: Bool
181
215
end
182
- function OptimizationState (sv:: InferenceState , interp:: AbstractInterpreter )
183
- inlining = InliningState (sv, interp)
216
+ function OptimizationState (sv:: InferenceState , interp:: AbstractInterpreter ,
217
+ opt_cache:: IdDict{MethodInstance,CodeInstance} = IdDict {MethodInstance,CodeInstance} ())
218
+ inlining = InliningState (sv, interp, opt_cache)
184
219
return OptimizationState (sv. linfo, sv. src, nothing , sv. stmt_info, sv. mod,
185
220
sv. sptypes, sv. slottypes, inlining, sv. cfg,
186
221
sv. unreachable, sv. bb_vartables, sv. insert_coverage)
187
222
end
188
- function OptimizationState (mi:: MethodInstance , src:: CodeInfo , interp:: AbstractInterpreter )
223
+ function OptimizationState (mi:: MethodInstance , src:: CodeInfo , interp:: AbstractInterpreter ,
224
+ opt_cache:: IdDict{MethodInstance,CodeInstance} = IdDict {MethodInstance,CodeInstance} ())
189
225
# prepare src for running optimization passes if it isn't already
190
226
nssavalues = src. ssavaluetypes
191
227
if nssavalues isa Int
@@ -205,7 +241,7 @@ function OptimizationState(mi::MethodInstance, src::CodeInfo, interp::AbstractIn
205
241
mod = isa (def, Method) ? def. module : def
206
242
# Allow using the global MI cache, but don't track edges.
207
243
# This method is mostly used for unit testing the optimizer
208
- inlining = InliningState (interp)
244
+ inlining = InliningState (interp, opt_cache )
209
245
cfg = compute_basic_blocks (src. code)
210
246
unreachable = BitSet ()
211
247
bb_vartables = Union{VarTable,Nothing}[]
@@ -236,13 +272,29 @@ include("ssair/EscapeAnalysis.jl")
236
272
include (" ssair/passes.jl" )
237
273
include (" ssair/irinterp.jl" )
238
274
275
+ function ir_to_codeinf! (opt:: OptimizationState , frame:: InferenceState , edges:: SimpleVector )
276
+ ir_to_codeinf! (opt, edges, compute_inlining_cost (frame. interp, frame. result, opt. optresult))
277
+ end
278
+
279
+ function ir_to_codeinf! (opt:: OptimizationState , edges:: SimpleVector , inlining_cost:: InlineCostType )
280
+ src = ir_to_codeinf! (opt, edges)
281
+ src. inlining_cost = inlining_cost
282
+ src
283
+ end
284
+
285
+ function ir_to_codeinf! (opt:: OptimizationState , edges:: SimpleVector )
286
+ src = ir_to_codeinf! (opt)
287
+ src. edges = edges
288
+ src
289
+ end
290
+
239
291
function ir_to_codeinf! (opt:: OptimizationState )
240
- (; linfo, src, result ) = opt
241
- if result === nothing
292
+ (; linfo, src, optresult ) = opt
293
+ if optresult === nothing
242
294
return src
243
295
end
244
- src = ir_to_codeinf! (src, result . ir)
245
- opt. result = nothing
296
+ src = ir_to_codeinf! (src, optresult . ir)
297
+ opt. optresult = nothing
246
298
opt. src = src
247
299
maybe_validate_code (linfo, src, " optimized" )
248
300
return src
@@ -485,63 +537,12 @@ end
485
537
abstract_eval_ssavalue (s:: SSAValue , src:: Union{IRCode,IncrementalCompact} ) = types (src)[s]
486
538
487
539
"""
488
- finish(interp::AbstractInterpreter, opt::OptimizationState,
489
- ir::IRCode, caller::InferenceResult)
540
+ finishopt!(interp::AbstractInterpreter, opt::OptimizationState, ir::IRCode)
490
541
491
- Post-process information derived by Julia-level optimizations for later use.
492
- In particular, this function determines the inlineability of the optimized code.
542
+ Called at the end of optimization to store the resulting IR back into the OptimizationState.
493
543
"""
494
- function finish (interp:: AbstractInterpreter , opt:: OptimizationState ,
495
- ir:: IRCode , caller:: InferenceResult )
496
- (; src, linfo) = opt
497
- (; def, specTypes) = linfo
498
-
499
- force_noinline = is_declared_noinline (src)
500
-
501
- # compute inlining and other related optimizations
502
- result = caller. result
503
- @assert ! (result isa LimitedAccuracy)
504
- result = widenslotwrapper (result)
505
-
506
- opt. result = OptimizationResult (ir, false )
507
-
508
- # determine and cache inlineability
509
- if ! force_noinline
510
- sig = unwrap_unionall (specTypes)
511
- if ! (isa (sig, DataType) && sig. name === Tuple. name)
512
- force_noinline = true
513
- end
514
- if ! is_declared_inline (src) && result === Bottom
515
- force_noinline = true
516
- end
517
- end
518
- if force_noinline
519
- set_inlineable! (src, false )
520
- elseif isa (def, Method)
521
- if is_declared_inline (src) && isdispatchtuple (specTypes)
522
- # obey @inline declaration if a dispatch barrier would not help
523
- set_inlineable! (src, true )
524
- else
525
- # compute the cost (size) of inlining this code
526
- params = OptimizationParams (interp)
527
- cost_threshold = default = params. inline_cost_threshold
528
- if ⊑ (optimizer_lattice (interp), result, Tuple) && ! isconcretetype (widenconst (result))
529
- cost_threshold += params. inline_tupleret_bonus
530
- end
531
- # if the method is declared as `@inline`, increase the cost threshold 20x
532
- if is_declared_inline (src)
533
- cost_threshold += 19 * default
534
- end
535
- # a few functions get special treatment
536
- if def. module === _topmod (def. module)
537
- name = def. name
538
- if name === :iterate || name === :unsafe_convert || name === :cconvert
539
- cost_threshold += 4 * default
540
- end
541
- end
542
- src. inlining_cost = inline_cost (ir, params, cost_threshold)
543
- end
544
- end
544
+ function finishopt! (interp:: AbstractInterpreter , opt:: OptimizationState , ir:: IRCode )
545
+ opt. optresult = OptimizationResult (ir, ccall (:jl_ir_flag_inlining , UInt8, (Any,), opt. src), false )
545
546
return nothing
546
547
end
547
548
@@ -1015,7 +1016,8 @@ end
1015
1016
function optimize (interp:: AbstractInterpreter , opt:: OptimizationState , caller:: InferenceResult )
1016
1017
@zone " CC: OPTIMIZER" ir = run_passes_ipo_safe (opt. src, opt)
1017
1018
ipo_dataflow_analysis! (interp, opt, ir, caller)
1018
- return finish (interp, opt, ir, caller)
1019
+ finishopt! (interp, opt, ir)
1020
+ return nothing
1019
1021
end
1020
1022
1021
1023
const ALL_PASS_NAMES = String[]
@@ -1466,7 +1468,7 @@ function statement_or_branch_cost(@nospecialize(stmt), line::Int, src::Union{Cod
1466
1468
return thiscost
1467
1469
end
1468
1470
1469
- function inline_cost (ir:: IRCode , params:: OptimizationParams , cost_threshold:: Int )
1471
+ function inline_cost_model (ir:: IRCode , params:: OptimizationParams , cost_threshold:: Int )
1470
1472
bodycost = 0
1471
1473
for i = 1 : length (ir. stmts)
1472
1474
stmt = ir[SSAValue (i)][:stmt ]
0 commit comments