Skip to content

JIT: Generalize entry-block parameter register rewriting to all blocks #133690

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jakobbotsch merged 4 commits into from
Sep 21, 2026
Merged

JIT: Generalize entry-block parameter register rewriting to all blocks #133690

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c107322
JIT: Generalize entry-block parameter register rewriting to all blocks
jakobbotsch Sep 11, 2026
8794b0d
Improve throughput
jakobbotsch Sep 17, 2026
77ac27f
Feedback, remove an added defensive check
jakobbotsch Sep 18, 2026
949afb4
Run jit-format
jakobbotsch Sep 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/coreclr/jit/asyncanalysis.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ void DefaultValueAnalysis::ComputePerBlockMutatedVars()
// Transfer function: mutatedOut[B] = mutatedIn[B] | mutated[B]
// Merge: mutatedIn[B] = union of mutatedOut[pred] for all preds
//
// At entry, only parameters and OSR locals are considered mutated.
// At entry, parameters, parameter register targets, and OSR locals are considered mutated.
//
void DefaultValueAnalysis::ComputeInterBlockDefaultValues()
{
Expand All @@ -271,13 +271,13 @@ void DefaultValueAnalysis::ComputeInterBlockDefaultValues()
VarSetOps::AssignNoCopy(m_compiler, m_mutatedVarsIn[i], VarSetOps::MakeEmpty(m_compiler));
}

// Parameters and OSR locals are considered mutated at method entry.
// Parameters, parameter register targets, and OSR locals are non-default at method entry.
for (unsigned i = 0; i < m_compiler->lvaTrackedCount; i++)
{
unsigned lclNum = m_compiler->lvaTrackedToVarNum[i];
LclVarDsc* varDsc = m_compiler->lvaGetDesc(lclNum);

if (varDsc->lvIsParam || varDsc->lvIsOSRLocal)
if (varDsc->lvIsParam || varDsc->lvIsParamRegTarget || varDsc->lvIsOSRLocal)
{
VarSetOps::AddElemD(m_compiler, m_mutatedVarsIn[m_compiler->fgFirstBB->bbNum], varDsc->lvVarIndex);
}
Expand Down
339 changes: 5 additions & 334 deletions src/coreclr/jit/lower.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9061,8 +9061,11 @@ PhaseStatus Lowering::DoPhase()
//
void Lowering::MapParameterRegisterLocals()
{
m_compiler->m_paramRegLocalMappings =
new (m_compiler, CMK_ABI) ArrayStack<ParameterRegisterLocalMapping>(m_compiler->getAllocator(CMK_ABI));
if (m_compiler->m_paramRegLocalMappings == nullptr)
{
m_compiler->m_paramRegLocalMappings =
new (m_compiler, CMK_ABI) ArrayStack<ParameterRegisterLocalMapping>(m_compiler->getAllocator(CMK_ABI));
}

// Create initial mappings for promotions.
for (unsigned lclNum = 0; lclNum < m_compiler->info.compArgsCount; lclNum++)
Expand Down Expand Up @@ -9117,8 +9120,6 @@ void Lowering::MapParameterRegisterLocals()
}
}

FindInducedParameterRegisterLocals();

#ifdef DEBUG
if (m_compiler->verbose)
{
Expand All @@ -9132,336 +9133,6 @@ void Lowering::MapParameterRegisterLocals()
#endif
}

//------------------------------------------------------------------------
// Lowering::FindInducedParameterRegisterLocals:
// Find locals that would be profitable to map from parameter registers,
// based on IR in the initialization block.
//
void Lowering::FindInducedParameterRegisterLocals()
{
#ifdef TARGET_ARM
// On arm32 the profiler hook does not preserve arg registers, so
// parameters are prespilled and cannot stay enregistered.
if (m_compiler->compIsProfilerHookNeeded())
{
JITDUMP("Skipping FindInducedParameterRegisterLocals on arm32 with profiler hook\n");
return;
}
#endif

// Check if we possibly have any parameters we can induce new register
// locals from.
bool anyCandidates = false;
for (unsigned lclNum = 0; lclNum < m_compiler->info.compArgsCount; lclNum++)
{
LclVarDsc* lcl = m_compiler->lvaGetDesc(lclNum);
if (lcl->lvPromoted || !lcl->lvDoNotEnregister)
{
continue;
}

const ABIPassingInformation& abiInfo = m_compiler->lvaGetParameterABIInfo(lclNum);
if (!abiInfo.HasAnyRegisterSegment())
{
continue;
}

anyCandidates = true;
break;
}

if (!anyCandidates)
{
return;
}

bool hasRegisterKill = false;
LocalSet storedToLocals(m_compiler->getAllocator(CMK_ABI));
// Now look for optimization opportunities in the first block: places where
// we read fields out of struct parameters that can be mapped cleanly. This
// is frequently created by physical promotion.
for (GenTree* node : LIR::AsRange(m_compiler->fgFirstBB))
{
hasRegisterKill |= node->IsCall();

auto visitDefs = [&](GenTreeLclVarCommon* lcl) {
storedToLocals.Emplace(lcl->GetLclNum(), true);
return GenTree::VisitResult::Continue;
};

node->VisitLocalDefNodes(m_compiler, visitDefs);

if (node->OperIs(GT_LCL_ADDR))
{
// Model these as stored to, since we cannot reason about them in the same way.
storedToLocals.Emplace(node->AsLclVarCommon()->GetLclNum(), true);
continue;
}

if (!node->OperIs(GT_LCL_FLD))
{
continue;
}

GenTreeLclFld* fld = node->AsLclFld();
if (fld->GetLclNum() >= m_compiler->info.compArgsCount)
{
continue;
}

LclVarDsc* paramDsc = m_compiler->lvaGetDesc(fld);
if (paramDsc->lvPromoted)
{
// These are complicated to reason about since they may be
// defined/used through their fields, so just skip them.
continue;
}

if (fld->TypeIs(TYP_STRUCT))
{
continue;
}

if (storedToLocals.Lookup(fld->GetLclNum()))
{
// LCL_FLD does not necessarily take the value of the parameter
// anymore.
continue;
}

const ABIPassingInformation& dataAbiInfo = m_compiler->lvaGetParameterABIInfo(fld->GetLclNum());
const ABIPassingSegment* regSegment = nullptr;
for (const ABIPassingSegment& segment : dataAbiInfo.Segments())
{
if (!segment.IsPassedInRegister())
{
continue;
}

assert(fld->GetLclOffs() <= m_compiler->lvaLclExactSize(fld->GetLclNum()));
unsigned structAccessedSize =
min(genTypeSize(fld), m_compiler->lvaLclExactSize(fld->GetLclNum()) - fld->GetLclOffs());
if ((fld->GetLclOffs() < segment.Offset) ||
(fld->GetLclOffs() + structAccessedSize > segment.Offset + segment.Size))
{
continue;
}

// TODO-CQ: Float -> !float extractions are not supported
// TODO-CQ: Float -> float extractions with non-zero offset is not supported
if (genIsValidFloatReg(segment.GetRegister()) &&
(!varTypeUsesFloatReg(fld) || (fld->GetLclOffs() != segment.Offset)))
{
continue;
}

// Found a register segment this field is contained in
regSegment = &segment;
break;
}

if (regSegment == nullptr)
{
continue;
}

JITDUMP("LCL_FLD use [%06u] of unenregisterable parameter is contained in ", Compiler::dspTreeID(fld));
DBEXEC(VERBOSE, regSegment->Dump());
JITDUMP("\n");

// Now see if we want to introduce a new local for this value, or if we
// can reuse one because this is the source of a store (frequently
// created by physical promotion).
LIR::Use use;
if (!LIR::AsRange(m_compiler->fgFirstBB).TryGetUse(fld, &use))
{
JITDUMP(" ..but no use was found\n");
continue;
}

const ParameterRegisterLocalMapping* existingMapping =
m_compiler->FindParameterRegisterLocalMappingByRegister(regSegment->GetRegister());

unsigned remappedLclNum = BAD_VAR_NUM;
if (existingMapping == nullptr)
{
remappedLclNum = m_compiler->lvaGrabTemp(false DEBUGARG(
m_compiler->printfAlloc("V%02u.%s", fld->GetLclNum(), getRegName(regSegment->GetRegister()))));

// We always use the full width for integer registers even if the
// width is shorter, because various places in the JIT will type
// accesses larger to generate smaller code.

#ifdef TARGET_WASM
var_types fullWidthType = genActualType(regSegment->GetRegisterType());
#else
var_types fullWidthType = TYP_I_IMPL;
#endif
var_types registerType =
genIsValidIntReg(regSegment->GetRegister()) ? fullWidthType : regSegment->GetRegisterType();
if ((registerType == TYP_I_IMPL) && varTypeIsGC(fld))
{
registerType = fld->TypeGet();
}

LclVarDsc* varDsc = m_compiler->lvaGetDesc(remappedLclNum);
varDsc->lvType = genActualType(registerType);
JITDUMP("Created new local V%02u for the mapping\n", remappedLclNum);

m_compiler->m_paramRegLocalMappings->Emplace(regSegment, remappedLclNum, 0);
varDsc->lvIsParamRegTarget = true;

JITDUMP("New mapping: ");
DBEXEC(VERBOSE, regSegment->Dump());
JITDUMP(" -> V%02u\n", remappedLclNum);
}
else
{
remappedLclNum = existingMapping->LclNum;
}

GenTree* value = m_compiler->gtNewLclVarNode(remappedLclNum);

#ifdef TARGET_WASM
if (varTypeIsSIMD(value) && !varTypeIsSIMD(fld))
{
// Unlike native targets, wasm cannot reinterpret a v128 local access as a scalar.
const unsigned laneOffset = fld->GetLclOffs() - regSegment->Offset;
const unsigned scalarSize = genTypeSize(fld);
assert((laneOffset % scalarSize) == 0);

const unsigned laneIndex = laneOffset / scalarSize;
value = m_compiler->gtNewSimdGetElementNode(fld->TypeGet(), value,
m_compiler->gtNewIconNode(static_cast<ssize_t>(laneIndex)),
fld->TypeGet(), genTypeSize(value));
}
else if (varTypeUsesFloatReg(value))
#else
if (varTypeUsesFloatReg(value))
#endif // TARGET_WASM
{
assert(fld->GetLclOffs() == regSegment->Offset);

value->gtType = fld->TypeGet();

#ifdef FEATURE_SIMD
// SIMD12s should be widened. We cannot do that with
// WidenSIMD12IfNecessary as it does not expect to see SIMD12
// accesses of SIMD16 locals here.
if (value->TypeIs(TYP_SIMD12))
{
value->gtType = TYP_SIMD16;
}
#endif
}
else
{
var_types registerType = value->TypeGet();

if (fld->GetLclOffs() > regSegment->Offset)
{
assert(value->TypeIs(TYP_INT, TYP_LONG));
GenTree* shiftAmount = m_compiler->gtNewIconNode((fld->GetLclOffs() - regSegment->Offset) * 8, TYP_INT);
value = m_compiler->gtNewOperNode(varTypeIsSmall(fld) && varTypeIsSigned(fld) ? GT_RSH : GT_RSZ,
value->TypeGet(), value, shiftAmount);
}

// Insert explicit normalization for small types (the LCL_FLD we
// are replacing comes with this normalization). This is only required
// if we didn't get the normalization via a right shift.
if (varTypeIsSmall(fld) && (regSegment->Offset + genTypeSize(fld) != genTypeSize(registerType)))
{
value = m_compiler->gtNewCastNode(TYP_INT, value, false, fld->TypeGet());
}

// If the node is still too large then get it to the right size
if (genTypeSize(value) != genTypeSize(genActualType((fld))))
{
assert(genTypeSize(value) == 8);
assert(genTypeSize(genActualType(fld)) == 4);

if (value->OperIsScalarLocal())
{
// We can use lower bits directly
value->gtType = TYP_INT;
}
else
{
value = m_compiler->gtNewCastNode(TYP_INT, value, false, TYP_INT);
}
}

// Finally insert a bitcast if necessary
if (value->TypeGet() != genActualType(fld))
{
value = m_compiler->gtNewBitCastNode(genActualType(fld), value);
}
}

// Now replace the LCL_FLD.
LIR::AsRange(m_compiler->fgFirstBB).InsertAfter(fld, LIR::SeqTree(m_compiler, value));
use.ReplaceWith(value);
JITDUMP("New user tree range:\n");
DISPTREERANGE(LIR::AsRange(m_compiler->fgFirstBB), use.User());

fld->gtBashToNOP();
}
}

//------------------------------------------------------------------------
// Lowering::TryReuseLocalForParameterAccess:
// Try to figure out if a LCL_FLD that corresponds to a parameter register is
// being stored directly to a LCL_VAR, and in that case whether it would be
// profitable to reuse that local as the parameter register.
//
// Parameters:
// use - The use of the LCL_FLD
// storedToLocals - Map of locals that have had potential definitions to them
// up until the use
//
// Returns:
// The local number to reuse, or BAD_VAR_NUM to create a new local instead.
//
unsigned Lowering::TryReuseLocalForParameterAccess(const LIR::Use& use, const LocalSet& storedToLocals)
{
GenTree* useNode = use.User();

if (!useNode->OperIs(GT_STORE_LCL_VAR))
{
return BAD_VAR_NUM;
}

LclVarDsc* destLclDsc = m_compiler->lvaGetDesc(useNode->AsLclVarCommon());

if (destLclDsc->lvIsParam || destLclDsc->lvIsParamRegTarget)
{
return BAD_VAR_NUM;
}

if (destLclDsc->lvIsStructField)
{
return BAD_VAR_NUM;
}

if (destLclDsc->TypeIs(TYP_STRUCT))
{
return BAD_VAR_NUM;
}

if (destLclDsc->lvDoNotEnregister)
{
return BAD_VAR_NUM;
}

if (storedToLocals.Lookup(useNode->AsLclVarCommon()->GetLclNum()))
{
// Destination may change value before this access
return BAD_VAR_NUM;
}

return useNode->AsLclVarCommon()->GetLclNum();
}

#ifdef DEBUG

//------------------------------------------------------------------------
Expand Down
6 changes: 1 addition & 5 deletions src/coreclr/jit/lower.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,11 +137,7 @@ class Lowering final : public Phase
static bool CheckBlock(Compiler* compiler, BasicBlock* block);
#endif // DEBUG

typedef JitHashTable<unsigned, JitSmallPrimitiveKeyFuncs<unsigned>, bool> LocalSet;

void MapParameterRegisterLocals();
void FindInducedParameterRegisterLocals();
unsigned TryReuseLocalForParameterAccess(const LIR::Use& use, const LocalSet& storedToLocals);
void MapParameterRegisterLocals();

void LowerBlock(BasicBlock* block);
void AfterLowerBlocks();
Expand Down
Loading
Loading