diff --git a/src/cmd/compile/internal/base/debug.go b/src/cmd/compile/internal/base/debug.go index 5a40b1bb2a9414e5d6aafb50a60ed59f5812d56a..4a92364332951f484b1f5573d543e1fd1adfce23 100644 --- a/src/cmd/compile/internal/base/debug.go +++ b/src/cmd/compile/internal/base/debug.go @@ -73,7 +73,7 @@ type DebugFlags struct { PGODevirtualize int `help:"enable profile-guided devirtualization; 0 to disable, 1 to enable interface devirtualization, 2 to enable function devirtualization" concurrent:"ok"` CFGODebug int `help:"debug continuous feature guided optimizations"` CFGOHash string `help:"hash value for debugging continuous feature guided optimizations" concurrent:"ok"` - CFGOInline int `help:"enable continuous feature guided inlining" concurrent:"ok"` + CFGOInline int `help:"enable continuous feature guided inlining (0 => off, 1 => default, 2 => multi-level mode)" concurrent:"ok"` CFGOInlineCDFThreshold string `help:"cumulative threshold percentage for determining call sites as hot candidates for inlining" concurrent:"ok"` CFGOInlineBudget int `help:"inline budget for hot functions" concurrent:"ok"` CFGODevirtualize int `help:"enable continuous feature guided devirtualization; 0 to disable, 1 to enable interface devirtualization, 2 to enable function devirtualization" concurrent:"ok"` diff --git a/src/cmd/compile/internal/inline/inl.go b/src/cmd/compile/internal/inline/inl.go index 2c518243e8b203346763a7de2cc3d86b922e4b07..96156da5fec2f428b4c8b390d82efe64c9a34379 100644 --- a/src/cmd/compile/internal/inline/inl.go +++ b/src/cmd/compile/internal/inline/inl.go @@ -945,7 +945,20 @@ func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller, closureCal } lineOffset := pgoir.NodeLineOffset(n, caller) - csi := pgoir.CallSiteInfo{LineOffset: lineOffset, Caller: caller} + parentCaller := caller + baseOffset := lineOffset + if base.ENABLE_CFGO && base.Debug.CFGOInline == 2 { + parentCallerName := ir.LinkFuncName(caller) + parentInlIdx := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex() + if parentInlIdx >= 0 { + parentCallerName = base.Ctxt.InlTree.InlinedFunction(parentInlIdx).Name + } + if f, err := pgoir.LookupFunc(parentCallerName); err == nil { + parentCaller = f + lineOffset = pgoir.NodeLineOffset(n, parentCaller) + } + } + csi := pgoir.CallSiteInfo{LineOffset: lineOffset, Caller: parentCaller} _, hot := candHotEdgeMap[csi] if metric <= maxCost { @@ -961,6 +974,12 @@ func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller, closureCal return false, maxCost, metric, false } + if base.ENABLE_CFGO && base.Debug.CFGODebug > 0 && base.Debug.CFGOInline == 2 && baseOffset != lineOffset { + fmt.Printf("changed offset from %v to %v; callee = %v, parent caller = %v (start line: %v), root caller = %v\n", + baseOffset, lineOffset, ir.LinkFuncName(callee), ir.LinkFuncName(parentCaller), + int(base.Ctxt.InnermostPos(parentCaller.Pos()).RelLine()), ir.LinkFuncName(caller)) + } + // Hot if bigCaller { @@ -970,6 +989,16 @@ func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller, closureCal return false, maxCost, metric, false } + if base.ENABLE_CFGO && base.Debug.CFGOInline == 2 { + if caller.Pragma&ir.Nosplit != 0 && parentCaller != caller { + if base.Debug.CFGODebug > 0 { + fmt.Printf("%v: prevent inlining of hot call %s (cost %d) in function %s: root function is nosplit. Parent caller = %v\n", ir.Line(n), + ir.PkgFuncName(callee), callee.Inl.Cost, ir.PkgFuncName(caller), ir.LinkFuncName(parentCaller)) + } + return false, maxCost, metric, false + } + } + if metric > inlineHotMaxBudget { return false, inlineHotMaxBudget, metric, false }