mirror of
https://github.com/capstone-engine/llvm-capstone.git
synced 2024-11-23 13:50:11 +00:00
[mlir][Affine][NFC] Wrap dialect in "affine" namespace
This cleanup aligns the affine dialect with all the other dialects. Differential Revision: https://reviews.llvm.org/D148687
This commit is contained in:
parent
fbc63e5658
commit
4c48f016ef
@ -27,7 +27,7 @@
|
|||||||
namespace fir::support {
|
namespace fir::support {
|
||||||
|
|
||||||
#define FLANG_NONCODEGEN_DIALECT_LIST \
|
#define FLANG_NONCODEGEN_DIALECT_LIST \
|
||||||
mlir::AffineDialect, FIROpsDialect, hlfir::hlfirDialect, \
|
mlir::affine::AffineDialect, FIROpsDialect, hlfir::hlfirDialect, \
|
||||||
mlir::acc::OpenACCDialect, mlir::omp::OpenMPDialect, \
|
mlir::acc::OpenACCDialect, mlir::omp::OpenMPDialect, \
|
||||||
mlir::scf::SCFDialect, mlir::arith::ArithDialect, \
|
mlir::scf::SCFDialect, mlir::arith::ArithDialect, \
|
||||||
mlir::cf::ControlFlowDialect, mlir::func::FuncDialect, \
|
mlir::cf::ControlFlowDialect, mlir::func::FuncDialect, \
|
||||||
@ -63,25 +63,25 @@ inline void loadDialects(mlir::MLIRContext &context) {
|
|||||||
inline void registerMLIRPassesForFortranTools() {
|
inline void registerMLIRPassesForFortranTools() {
|
||||||
mlir::registerCanonicalizerPass();
|
mlir::registerCanonicalizerPass();
|
||||||
mlir::registerCSEPass();
|
mlir::registerCSEPass();
|
||||||
mlir::registerAffineLoopFusionPass();
|
mlir::affine::registerAffineLoopFusionPass();
|
||||||
mlir::registerLoopInvariantCodeMotionPass();
|
mlir::registerLoopInvariantCodeMotionPass();
|
||||||
mlir::registerLoopCoalescingPass();
|
mlir::affine::registerLoopCoalescingPass();
|
||||||
mlir::registerStripDebugInfoPass();
|
mlir::registerStripDebugInfoPass();
|
||||||
mlir::registerPrintOpStatsPass();
|
mlir::registerPrintOpStatsPass();
|
||||||
mlir::registerInlinerPass();
|
mlir::registerInlinerPass();
|
||||||
mlir::registerSCCPPass();
|
mlir::registerSCCPPass();
|
||||||
mlir::registerAffineScalarReplacementPass();
|
mlir::affine::registerAffineScalarReplacementPass();
|
||||||
mlir::registerSymbolDCEPass();
|
mlir::registerSymbolDCEPass();
|
||||||
mlir::registerLocationSnapshotPass();
|
mlir::registerLocationSnapshotPass();
|
||||||
mlir::registerAffinePipelineDataTransferPass();
|
mlir::affine::registerAffinePipelineDataTransferPass();
|
||||||
|
|
||||||
mlir::registerAffineVectorizePass();
|
mlir::affine::registerAffineVectorizePass();
|
||||||
mlir::registerAffineLoopUnrollPass();
|
mlir::affine::registerAffineLoopUnrollPass();
|
||||||
mlir::registerAffineLoopUnrollAndJamPass();
|
mlir::affine::registerAffineLoopUnrollAndJamPass();
|
||||||
mlir::registerSimplifyAffineStructuresPass();
|
mlir::affine::registerSimplifyAffineStructuresPass();
|
||||||
mlir::registerAffineLoopInvariantCodeMotionPass();
|
mlir::affine::registerAffineLoopInvariantCodeMotionPass();
|
||||||
mlir::registerAffineLoopTilingPass();
|
mlir::affine::registerAffineLoopTilingPass();
|
||||||
mlir::registerAffineDataCopyGenerationPass();
|
mlir::affine::registerAffineDataCopyGenerationPass();
|
||||||
|
|
||||||
mlir::registerConvertAffineToStandardPass();
|
mlir::registerConvertAffineToStandardPass();
|
||||||
}
|
}
|
||||||
|
@ -64,7 +64,8 @@ def AffineDialectPromotion : Pass<"promote-to-affine", "::mlir::func::FuncOp"> {
|
|||||||
}];
|
}];
|
||||||
let constructor = "::fir::createPromoteToAffinePass()";
|
let constructor = "::fir::createPromoteToAffinePass()";
|
||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"fir::FIROpsDialect", "mlir::func::FuncDialect", "mlir::AffineDialect"
|
"fir::FIROpsDialect", "mlir::func::FuncDialect",
|
||||||
|
"mlir::affine::AffineDialect"
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -78,7 +79,8 @@ def AffineDialectDemotion : Pass<"demote-affine", "::mlir::func::FuncOp"> {
|
|||||||
}];
|
}];
|
||||||
let constructor = "::fir::createAffineDemotionPass()";
|
let constructor = "::fir::createAffineDemotionPass()";
|
||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"fir::FIROpsDialect", "mlir::func::FuncDialect", "mlir::AffineDialect"
|
"fir::FIROpsDialect", "mlir::func::FuncDialect",
|
||||||
|
"mlir::affine::AffineDialect"
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -46,16 +46,17 @@ using namespace mlir;
|
|||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
class AffineLoadConversion : public OpConversionPattern<mlir::AffineLoadOp> {
|
class AffineLoadConversion
|
||||||
|
: public OpConversionPattern<mlir::affine::AffineLoadOp> {
|
||||||
public:
|
public:
|
||||||
using OpConversionPattern<mlir::AffineLoadOp>::OpConversionPattern;
|
using OpConversionPattern<mlir::affine::AffineLoadOp>::OpConversionPattern;
|
||||||
|
|
||||||
LogicalResult
|
LogicalResult
|
||||||
matchAndRewrite(mlir::AffineLoadOp op, OpAdaptor adaptor,
|
matchAndRewrite(mlir::affine::AffineLoadOp op, OpAdaptor adaptor,
|
||||||
ConversionPatternRewriter &rewriter) const override {
|
ConversionPatternRewriter &rewriter) const override {
|
||||||
SmallVector<Value> indices(adaptor.getIndices());
|
SmallVector<Value> indices(adaptor.getIndices());
|
||||||
auto maybeExpandedMap =
|
auto maybeExpandedMap = affine::expandAffineMap(rewriter, op.getLoc(),
|
||||||
expandAffineMap(rewriter, op.getLoc(), op.getAffineMap(), indices);
|
op.getAffineMap(), indices);
|
||||||
if (!maybeExpandedMap)
|
if (!maybeExpandedMap)
|
||||||
return failure();
|
return failure();
|
||||||
|
|
||||||
@ -68,16 +69,17 @@ public:
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
class AffineStoreConversion : public OpConversionPattern<mlir::AffineStoreOp> {
|
class AffineStoreConversion
|
||||||
|
: public OpConversionPattern<mlir::affine::AffineStoreOp> {
|
||||||
public:
|
public:
|
||||||
using OpConversionPattern<mlir::AffineStoreOp>::OpConversionPattern;
|
using OpConversionPattern<mlir::affine::AffineStoreOp>::OpConversionPattern;
|
||||||
|
|
||||||
LogicalResult
|
LogicalResult
|
||||||
matchAndRewrite(mlir::AffineStoreOp op, OpAdaptor adaptor,
|
matchAndRewrite(mlir::affine::AffineStoreOp op, OpAdaptor adaptor,
|
||||||
ConversionPatternRewriter &rewriter) const override {
|
ConversionPatternRewriter &rewriter) const override {
|
||||||
SmallVector<Value> indices(op.getIndices());
|
SmallVector<Value> indices(op.getIndices());
|
||||||
auto maybeExpandedMap =
|
auto maybeExpandedMap = affine::expandAffineMap(rewriter, op.getLoc(),
|
||||||
expandAffineMap(rewriter, op.getLoc(), op.getAffineMap(), indices);
|
op.getAffineMap(), indices);
|
||||||
if (!maybeExpandedMap)
|
if (!maybeExpandedMap)
|
||||||
return failure();
|
return failure();
|
||||||
|
|
||||||
|
@ -227,7 +227,7 @@ private:
|
|||||||
if (auto blockArg = value.dyn_cast<mlir::BlockArgument>()) {
|
if (auto blockArg = value.dyn_cast<mlir::BlockArgument>()) {
|
||||||
affineArgs.push_back(value);
|
affineArgs.push_back(value);
|
||||||
if (isa<fir::DoLoopOp>(blockArg.getOwner()->getParentOp()) ||
|
if (isa<fir::DoLoopOp>(blockArg.getOwner()->getParentOp()) ||
|
||||||
isa<mlir::AffineForOp>(blockArg.getOwner()->getParentOp()))
|
isa<mlir::affine::AffineForOp>(blockArg.getOwner()->getParentOp()))
|
||||||
return {mlir::getAffineDimExpr(dimCount++, value.getContext())};
|
return {mlir::getAffineDimExpr(dimCount++, value.getContext())};
|
||||||
return {mlir::getAffineSymbolExpr(symCount++, value.getContext())};
|
return {mlir::getAffineSymbolExpr(symCount++, value.getContext())};
|
||||||
}
|
}
|
||||||
@ -397,7 +397,7 @@ static void populateIndexArgs(fir::ArrayCoorOp acoOp,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Returns affine.apply and fir.convert from array_coor and gendims
|
/// Returns affine.apply and fir.convert from array_coor and gendims
|
||||||
static std::pair<mlir::AffineApplyOp, fir::ConvertOp>
|
static std::pair<affine::AffineApplyOp, fir::ConvertOp>
|
||||||
createAffineOps(mlir::Value arrayRef, mlir::PatternRewriter &rewriter) {
|
createAffineOps(mlir::Value arrayRef, mlir::PatternRewriter &rewriter) {
|
||||||
auto acoOp = arrayRef.getDefiningOp<ArrayCoorOp>();
|
auto acoOp = arrayRef.getDefiningOp<ArrayCoorOp>();
|
||||||
auto affineMap =
|
auto affineMap =
|
||||||
@ -407,8 +407,8 @@ createAffineOps(mlir::Value arrayRef, mlir::PatternRewriter &rewriter) {
|
|||||||
|
|
||||||
populateIndexArgs(acoOp, indexArgs, rewriter);
|
populateIndexArgs(acoOp, indexArgs, rewriter);
|
||||||
|
|
||||||
auto affineApply = rewriter.create<mlir::AffineApplyOp>(acoOp.getLoc(),
|
auto affineApply = rewriter.create<affine::AffineApplyOp>(
|
||||||
affineMap, indexArgs);
|
acoOp.getLoc(), affineMap, indexArgs);
|
||||||
auto arrayElementType = coordinateArrayElement(acoOp);
|
auto arrayElementType = coordinateArrayElement(acoOp);
|
||||||
auto newType =
|
auto newType =
|
||||||
mlir::MemRefType::get({mlir::ShapedType::kDynamic}, arrayElementType);
|
mlir::MemRefType::get({mlir::ShapedType::kDynamic}, arrayElementType);
|
||||||
@ -420,7 +420,7 @@ createAffineOps(mlir::Value arrayRef, mlir::PatternRewriter &rewriter) {
|
|||||||
static void rewriteLoad(fir::LoadOp loadOp, mlir::PatternRewriter &rewriter) {
|
static void rewriteLoad(fir::LoadOp loadOp, mlir::PatternRewriter &rewriter) {
|
||||||
rewriter.setInsertionPoint(loadOp);
|
rewriter.setInsertionPoint(loadOp);
|
||||||
auto affineOps = createAffineOps(loadOp.getMemref(), rewriter);
|
auto affineOps = createAffineOps(loadOp.getMemref(), rewriter);
|
||||||
rewriter.replaceOpWithNewOp<mlir::AffineLoadOp>(
|
rewriter.replaceOpWithNewOp<affine::AffineLoadOp>(
|
||||||
loadOp, affineOps.second.getResult(), affineOps.first.getResult());
|
loadOp, affineOps.second.getResult(), affineOps.first.getResult());
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -428,9 +428,9 @@ static void rewriteStore(fir::StoreOp storeOp,
|
|||||||
mlir::PatternRewriter &rewriter) {
|
mlir::PatternRewriter &rewriter) {
|
||||||
rewriter.setInsertionPoint(storeOp);
|
rewriter.setInsertionPoint(storeOp);
|
||||||
auto affineOps = createAffineOps(storeOp.getMemref(), rewriter);
|
auto affineOps = createAffineOps(storeOp.getMemref(), rewriter);
|
||||||
rewriter.replaceOpWithNewOp<mlir::AffineStoreOp>(storeOp, storeOp.getValue(),
|
rewriter.replaceOpWithNewOp<affine::AffineStoreOp>(
|
||||||
affineOps.second.getResult(),
|
storeOp, storeOp.getValue(), affineOps.second.getResult(),
|
||||||
affineOps.first.getResult());
|
affineOps.first.getResult());
|
||||||
}
|
}
|
||||||
|
|
||||||
static void rewriteMemoryOps(Block *block, mlir::PatternRewriter &rewriter) {
|
static void rewriteMemoryOps(Block *block, mlir::PatternRewriter &rewriter) {
|
||||||
@ -483,7 +483,7 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
std::pair<mlir::AffineForOp, mlir::Value>
|
std::pair<affine::AffineForOp, mlir::Value>
|
||||||
createAffineFor(fir::DoLoopOp op, mlir::PatternRewriter &rewriter) const {
|
createAffineFor(fir::DoLoopOp op, mlir::PatternRewriter &rewriter) const {
|
||||||
if (auto constantStep = constantIntegerLike(op.getStep()))
|
if (auto constantStep = constantIntegerLike(op.getStep()))
|
||||||
if (*constantStep > 0)
|
if (*constantStep > 0)
|
||||||
@ -492,10 +492,10 @@ private:
|
|||||||
}
|
}
|
||||||
|
|
||||||
// when step for the loop is positive compile time constant
|
// when step for the loop is positive compile time constant
|
||||||
std::pair<mlir::AffineForOp, mlir::Value>
|
std::pair<affine::AffineForOp, mlir::Value>
|
||||||
positiveConstantStep(fir::DoLoopOp op, int64_t step,
|
positiveConstantStep(fir::DoLoopOp op, int64_t step,
|
||||||
mlir::PatternRewriter &rewriter) const {
|
mlir::PatternRewriter &rewriter) const {
|
||||||
auto affineFor = rewriter.create<mlir::AffineForOp>(
|
auto affineFor = rewriter.create<affine::AffineForOp>(
|
||||||
op.getLoc(), ValueRange(op.getLowerBound()),
|
op.getLoc(), ValueRange(op.getLowerBound()),
|
||||||
mlir::AffineMap::get(0, 1,
|
mlir::AffineMap::get(0, 1,
|
||||||
mlir::getAffineSymbolExpr(0, op.getContext())),
|
mlir::getAffineSymbolExpr(0, op.getContext())),
|
||||||
@ -506,14 +506,14 @@ private:
|
|||||||
return std::make_pair(affineFor, affineFor.getInductionVar());
|
return std::make_pair(affineFor, affineFor.getInductionVar());
|
||||||
}
|
}
|
||||||
|
|
||||||
std::pair<mlir::AffineForOp, mlir::Value>
|
std::pair<affine::AffineForOp, mlir::Value>
|
||||||
genericBounds(fir::DoLoopOp op, mlir::PatternRewriter &rewriter) const {
|
genericBounds(fir::DoLoopOp op, mlir::PatternRewriter &rewriter) const {
|
||||||
auto lowerBound = mlir::getAffineSymbolExpr(0, op.getContext());
|
auto lowerBound = mlir::getAffineSymbolExpr(0, op.getContext());
|
||||||
auto upperBound = mlir::getAffineSymbolExpr(1, op.getContext());
|
auto upperBound = mlir::getAffineSymbolExpr(1, op.getContext());
|
||||||
auto step = mlir::getAffineSymbolExpr(2, op.getContext());
|
auto step = mlir::getAffineSymbolExpr(2, op.getContext());
|
||||||
mlir::AffineMap upperBoundMap = mlir::AffineMap::get(
|
mlir::AffineMap upperBoundMap = mlir::AffineMap::get(
|
||||||
0, 3, (upperBound - lowerBound + step).floorDiv(step));
|
0, 3, (upperBound - lowerBound + step).floorDiv(step));
|
||||||
auto genericUpperBound = rewriter.create<mlir::AffineApplyOp>(
|
auto genericUpperBound = rewriter.create<affine::AffineApplyOp>(
|
||||||
op.getLoc(), upperBoundMap,
|
op.getLoc(), upperBoundMap,
|
||||||
ValueRange({op.getLowerBound(), op.getUpperBound(), op.getStep()}));
|
ValueRange({op.getLowerBound(), op.getUpperBound(), op.getStep()}));
|
||||||
auto actualIndexMap = mlir::AffineMap::get(
|
auto actualIndexMap = mlir::AffineMap::get(
|
||||||
@ -521,7 +521,7 @@ private:
|
|||||||
(lowerBound + mlir::getAffineDimExpr(0, op.getContext())) *
|
(lowerBound + mlir::getAffineDimExpr(0, op.getContext())) *
|
||||||
mlir::getAffineSymbolExpr(1, op.getContext()));
|
mlir::getAffineSymbolExpr(1, op.getContext()));
|
||||||
|
|
||||||
auto affineFor = rewriter.create<mlir::AffineForOp>(
|
auto affineFor = rewriter.create<affine::AffineForOp>(
|
||||||
op.getLoc(), ValueRange(),
|
op.getLoc(), ValueRange(),
|
||||||
AffineMap::getConstantMap(0, op.getContext()),
|
AffineMap::getConstantMap(0, op.getContext()),
|
||||||
genericUpperBound.getResult(),
|
genericUpperBound.getResult(),
|
||||||
@ -529,7 +529,7 @@ private:
|
|||||||
1 + mlir::getAffineSymbolExpr(0, op.getContext())),
|
1 + mlir::getAffineSymbolExpr(0, op.getContext())),
|
||||||
1);
|
1);
|
||||||
rewriter.setInsertionPointToStart(affineFor.getBody());
|
rewriter.setInsertionPointToStart(affineFor.getBody());
|
||||||
auto actualIndex = rewriter.create<mlir::AffineApplyOp>(
|
auto actualIndex = rewriter.create<affine::AffineApplyOp>(
|
||||||
op.getLoc(), actualIndexMap,
|
op.getLoc(), actualIndexMap,
|
||||||
ValueRange(
|
ValueRange(
|
||||||
{affineFor.getInductionVar(), op.getLowerBound(), op.getStep()}));
|
{affineFor.getInductionVar(), op.getLowerBound(), op.getStep()}));
|
||||||
@ -558,7 +558,7 @@ public:
|
|||||||
<< "AffineIfConversion: couldn't calculate affine condition\n";);
|
<< "AffineIfConversion: couldn't calculate affine condition\n";);
|
||||||
return failure();
|
return failure();
|
||||||
}
|
}
|
||||||
auto affineIf = rewriter.create<mlir::AffineIfOp>(
|
auto affineIf = rewriter.create<affine::AffineIfOp>(
|
||||||
op.getLoc(), affineCondition.getIntegerSet(),
|
op.getLoc(), affineCondition.getIntegerSet(),
|
||||||
affineCondition.getAffineArgs(), !op.getElseRegion().empty());
|
affineCondition.getAffineArgs(), !op.getElseRegion().empty());
|
||||||
rewriter.startRootUpdate(affineIf);
|
rewriter.startRootUpdate(affineIf);
|
||||||
@ -596,7 +596,7 @@ public:
|
|||||||
patterns.insert<AffineIfConversion>(context, functionAnalysis);
|
patterns.insert<AffineIfConversion>(context, functionAnalysis);
|
||||||
patterns.insert<AffineLoopConversion>(context, functionAnalysis);
|
patterns.insert<AffineLoopConversion>(context, functionAnalysis);
|
||||||
mlir::ConversionTarget target = *context;
|
mlir::ConversionTarget target = *context;
|
||||||
target.addLegalDialect<mlir::AffineDialect, FIROpsDialect,
|
target.addLegalDialect<mlir::affine::AffineDialect, FIROpsDialect,
|
||||||
mlir::scf::SCFDialect, mlir::arith::ArithDialect,
|
mlir::scf::SCFDialect, mlir::arith::ArithDialect,
|
||||||
mlir::func::FuncDialect>();
|
mlir::func::FuncDialect>();
|
||||||
target.addDynamicallyLegalOp<IfOp>([&functionAnalysis](fir::IfOp op) {
|
target.addDynamicallyLegalOp<IfOp>([&functionAnalysis](fir::IfOp op) {
|
||||||
|
@ -110,7 +110,7 @@ public:
|
|||||||
mlir::RewritePatternSet patterns(context);
|
mlir::RewritePatternSet patterns(context);
|
||||||
patterns.insert<CharacterConvertConversion>(context);
|
patterns.insert<CharacterConvertConversion>(context);
|
||||||
mlir::ConversionTarget target(*context);
|
mlir::ConversionTarget target(*context);
|
||||||
target.addLegalDialect<mlir::AffineDialect, fir::FIROpsDialect,
|
target.addLegalDialect<mlir::affine::AffineDialect, fir::FIROpsDialect,
|
||||||
mlir::arith::ArithDialect,
|
mlir::arith::ArithDialect,
|
||||||
mlir::func::FuncDialect>();
|
mlir::func::FuncDialect>();
|
||||||
|
|
||||||
|
@ -316,8 +316,9 @@ public:
|
|||||||
patterns.insert<CfgLoopConv, CfgIfConv, CfgIterWhileConv>(
|
patterns.insert<CfgLoopConv, CfgIfConv, CfgIterWhileConv>(
|
||||||
context, forceLoopToExecuteOnce);
|
context, forceLoopToExecuteOnce);
|
||||||
mlir::ConversionTarget target(*context);
|
mlir::ConversionTarget target(*context);
|
||||||
target.addLegalDialect<mlir::AffineDialect, mlir::cf::ControlFlowDialect,
|
target.addLegalDialect<mlir::affine::AffineDialect,
|
||||||
FIROpsDialect, mlir::func::FuncDialect>();
|
mlir::cf::ControlFlowDialect, FIROpsDialect,
|
||||||
|
mlir::func::FuncDialect>();
|
||||||
|
|
||||||
// apply the patterns
|
// apply the patterns
|
||||||
target.addIllegalOp<ResultOp, DoLoopOp, IfOp, IterWhileOp>();
|
target.addIllegalOp<ResultOp, DoLoopOp, IfOp, IterWhileOp>();
|
||||||
|
@ -238,8 +238,9 @@ public:
|
|||||||
patterns.insert<SelectTypeConv>(context, moduleMutex);
|
patterns.insert<SelectTypeConv>(context, moduleMutex);
|
||||||
patterns.insert<DispatchOpConv>(context, bindingTables);
|
patterns.insert<DispatchOpConv>(context, bindingTables);
|
||||||
mlir::ConversionTarget target(*context);
|
mlir::ConversionTarget target(*context);
|
||||||
target.addLegalDialect<mlir::AffineDialect, mlir::cf::ControlFlowDialect,
|
target.addLegalDialect<mlir::affine::AffineDialect,
|
||||||
FIROpsDialect, mlir::func::FuncDialect>();
|
mlir::cf::ControlFlowDialect, FIROpsDialect,
|
||||||
|
mlir::func::FuncDialect>();
|
||||||
|
|
||||||
// apply the patterns
|
// apply the patterns
|
||||||
target.addIllegalOp<SelectTypeOp>();
|
target.addIllegalOp<SelectTypeOp>();
|
||||||
|
@ -98,7 +98,8 @@ struct MyTarget : public ConversionTarget {
|
|||||||
|
|
||||||
/// Mark all operations within Affine dialect have dynamic legality
|
/// Mark all operations within Affine dialect have dynamic legality
|
||||||
/// constraints.
|
/// constraints.
|
||||||
addDynamicallyLegalDialect<AffineDialect>([](Operation *op) { ... });
|
addDynamicallyLegalDialect<affine::AffineDialect>(
|
||||||
|
[](Operation *op) { ... });
|
||||||
|
|
||||||
/// Mark `func.return` as dynamically legal, but provide a specific legality
|
/// Mark `func.return` as dynamically legal, but provide a specific legality
|
||||||
/// callback.
|
/// callback.
|
||||||
|
@ -64,7 +64,7 @@ void ToyToAffineLoweringPass::runOnOperation() {
|
|||||||
// We define the specific operations, or dialects, that are legal targets for
|
// We define the specific operations, or dialects, that are legal targets for
|
||||||
// this lowering. In our case, we are lowering to a combination of the
|
// this lowering. In our case, we are lowering to a combination of the
|
||||||
// `Affine`, `Arith`, `Func`, and `MemRef` dialects.
|
// `Affine`, `Arith`, `Func`, and `MemRef` dialects.
|
||||||
target.addLegalDialect<AffineDialect, arith::ArithDialect,
|
target.addLegalDialect<affine::AffineDialect, arith::ArithDialect,
|
||||||
func::FuncDialect, memref::MemRefDialect>();
|
func::FuncDialect, memref::MemRefDialect>();
|
||||||
|
|
||||||
// We also define the Toy dialect as Illegal so that the conversion will fail
|
// We also define the Toy dialect as Illegal so that the conversion will fail
|
||||||
|
@ -75,14 +75,15 @@ static void lowerOpToLoops(Operation *op, ValueRange operands,
|
|||||||
// loop induction variables.
|
// loop induction variables.
|
||||||
SmallVector<int64_t, 4> lowerBounds(tensorType.getRank(), /*Value=*/0);
|
SmallVector<int64_t, 4> lowerBounds(tensorType.getRank(), /*Value=*/0);
|
||||||
SmallVector<int64_t, 4> steps(tensorType.getRank(), /*Value=*/1);
|
SmallVector<int64_t, 4> steps(tensorType.getRank(), /*Value=*/1);
|
||||||
buildAffineLoopNest(
|
affine::buildAffineLoopNest(
|
||||||
rewriter, loc, lowerBounds, tensorType.getShape(), steps,
|
rewriter, loc, lowerBounds, tensorType.getShape(), steps,
|
||||||
[&](OpBuilder &nestedBuilder, Location loc, ValueRange ivs) {
|
[&](OpBuilder &nestedBuilder, Location loc, ValueRange ivs) {
|
||||||
// Call the processing function with the rewriter, the memref operands,
|
// Call the processing function with the rewriter, the memref operands,
|
||||||
// and the loop induction variables. This function will return the value
|
// and the loop induction variables. This function will return the value
|
||||||
// to store at the current index.
|
// to store at the current index.
|
||||||
Value valueToStore = processIteration(nestedBuilder, operands, ivs);
|
Value valueToStore = processIteration(nestedBuilder, operands, ivs);
|
||||||
nestedBuilder.create<AffineStoreOp>(loc, valueToStore, alloc, ivs);
|
nestedBuilder.create<affine::AffineStoreOp>(loc, valueToStore, alloc,
|
||||||
|
ivs);
|
||||||
});
|
});
|
||||||
|
|
||||||
// Replace this operation with the generated alloc.
|
// Replace this operation with the generated alloc.
|
||||||
@ -113,9 +114,9 @@ struct BinaryOpLowering : public ConversionPattern {
|
|||||||
|
|
||||||
// Generate loads for the element of 'lhs' and 'rhs' at the
|
// Generate loads for the element of 'lhs' and 'rhs' at the
|
||||||
// inner loop.
|
// inner loop.
|
||||||
auto loadedLhs = builder.create<AffineLoadOp>(
|
auto loadedLhs = builder.create<affine::AffineLoadOp>(
|
||||||
loc, binaryAdaptor.getLhs(), loopIvs);
|
loc, binaryAdaptor.getLhs(), loopIvs);
|
||||||
auto loadedRhs = builder.create<AffineLoadOp>(
|
auto loadedRhs = builder.create<affine::AffineLoadOp>(
|
||||||
loc, binaryAdaptor.getRhs(), loopIvs);
|
loc, binaryAdaptor.getRhs(), loopIvs);
|
||||||
|
|
||||||
// Create the binary operation performed on the loaded
|
// Create the binary operation performed on the loaded
|
||||||
@ -174,7 +175,7 @@ struct ConstantOpLowering : public OpRewritePattern<toy::ConstantOp> {
|
|||||||
// The last dimension is the base case of the recursion, at this point
|
// The last dimension is the base case of the recursion, at this point
|
||||||
// we store the element at the given index.
|
// we store the element at the given index.
|
||||||
if (dimension == valueShape.size()) {
|
if (dimension == valueShape.size()) {
|
||||||
rewriter.create<AffineStoreOp>(
|
rewriter.create<affine::AffineStoreOp>(
|
||||||
loc, rewriter.create<arith::ConstantOp>(loc, *valueIt++), alloc,
|
loc, rewriter.create<arith::ConstantOp>(loc, *valueIt++), alloc,
|
||||||
llvm::ArrayRef(indices));
|
llvm::ArrayRef(indices));
|
||||||
return;
|
return;
|
||||||
@ -291,8 +292,8 @@ struct TransposeOpLowering : public ConversionPattern {
|
|||||||
// Transpose the elements by generating a load from the
|
// Transpose the elements by generating a load from the
|
||||||
// reverse indices.
|
// reverse indices.
|
||||||
SmallVector<Value, 2> reverseIvs(llvm::reverse(loopIvs));
|
SmallVector<Value, 2> reverseIvs(llvm::reverse(loopIvs));
|
||||||
return builder.create<AffineLoadOp>(loc, input,
|
return builder.create<affine::AffineLoadOp>(loc, input,
|
||||||
reverseIvs);
|
reverseIvs);
|
||||||
});
|
});
|
||||||
return success();
|
return success();
|
||||||
}
|
}
|
||||||
@ -313,7 +314,8 @@ struct ToyToAffineLoweringPass
|
|||||||
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ToyToAffineLoweringPass)
|
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ToyToAffineLoweringPass)
|
||||||
|
|
||||||
void getDependentDialects(DialectRegistry ®istry) const override {
|
void getDependentDialects(DialectRegistry ®istry) const override {
|
||||||
registry.insert<AffineDialect, func::FuncDialect, memref::MemRefDialect>();
|
registry.insert<affine::AffineDialect, func::FuncDialect,
|
||||||
|
memref::MemRefDialect>();
|
||||||
}
|
}
|
||||||
void runOnOperation() final;
|
void runOnOperation() final;
|
||||||
};
|
};
|
||||||
@ -327,8 +329,9 @@ void ToyToAffineLoweringPass::runOnOperation() {
|
|||||||
// We define the specific operations, or dialects, that are legal targets for
|
// We define the specific operations, or dialects, that are legal targets for
|
||||||
// this lowering. In our case, we are lowering to a combination of the
|
// this lowering. In our case, we are lowering to a combination of the
|
||||||
// `Affine`, `Arith`, `Func`, and `MemRef` dialects.
|
// `Affine`, `Arith`, `Func`, and `MemRef` dialects.
|
||||||
target.addLegalDialect<AffineDialect, BuiltinDialect, arith::ArithDialect,
|
target.addLegalDialect<affine::AffineDialect, BuiltinDialect,
|
||||||
func::FuncDialect, memref::MemRefDialect>();
|
arith::ArithDialect, func::FuncDialect,
|
||||||
|
memref::MemRefDialect>();
|
||||||
|
|
||||||
// We also define the Toy dialect as Illegal so that the conversion will fail
|
// We also define the Toy dialect as Illegal so that the conversion will fail
|
||||||
// if any of these operations are *not* converted. Given that we actually want
|
// if any of these operations are *not* converted. Given that we actually want
|
||||||
|
@ -148,8 +148,8 @@ int dumpMLIR() {
|
|||||||
|
|
||||||
// Add optimizations if enabled.
|
// Add optimizations if enabled.
|
||||||
if (enableOpt) {
|
if (enableOpt) {
|
||||||
optPM.addPass(mlir::createLoopFusionPass());
|
optPM.addPass(mlir::affine::createLoopFusionPass());
|
||||||
optPM.addPass(mlir::createAffineScalarReplacementPass());
|
optPM.addPass(mlir::affine::createAffineScalarReplacementPass());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -75,14 +75,15 @@ static void lowerOpToLoops(Operation *op, ValueRange operands,
|
|||||||
// loop induction variables.
|
// loop induction variables.
|
||||||
SmallVector<int64_t, 4> lowerBounds(tensorType.getRank(), /*Value=*/0);
|
SmallVector<int64_t, 4> lowerBounds(tensorType.getRank(), /*Value=*/0);
|
||||||
SmallVector<int64_t, 4> steps(tensorType.getRank(), /*Value=*/1);
|
SmallVector<int64_t, 4> steps(tensorType.getRank(), /*Value=*/1);
|
||||||
buildAffineLoopNest(
|
affine::buildAffineLoopNest(
|
||||||
rewriter, loc, lowerBounds, tensorType.getShape(), steps,
|
rewriter, loc, lowerBounds, tensorType.getShape(), steps,
|
||||||
[&](OpBuilder &nestedBuilder, Location loc, ValueRange ivs) {
|
[&](OpBuilder &nestedBuilder, Location loc, ValueRange ivs) {
|
||||||
// Call the processing function with the rewriter, the memref operands,
|
// Call the processing function with the rewriter, the memref operands,
|
||||||
// and the loop induction variables. This function will return the value
|
// and the loop induction variables. This function will return the value
|
||||||
// to store at the current index.
|
// to store at the current index.
|
||||||
Value valueToStore = processIteration(nestedBuilder, operands, ivs);
|
Value valueToStore = processIteration(nestedBuilder, operands, ivs);
|
||||||
nestedBuilder.create<AffineStoreOp>(loc, valueToStore, alloc, ivs);
|
nestedBuilder.create<affine::AffineStoreOp>(loc, valueToStore, alloc,
|
||||||
|
ivs);
|
||||||
});
|
});
|
||||||
|
|
||||||
// Replace this operation with the generated alloc.
|
// Replace this operation with the generated alloc.
|
||||||
@ -113,9 +114,9 @@ struct BinaryOpLowering : public ConversionPattern {
|
|||||||
|
|
||||||
// Generate loads for the element of 'lhs' and 'rhs' at the
|
// Generate loads for the element of 'lhs' and 'rhs' at the
|
||||||
// inner loop.
|
// inner loop.
|
||||||
auto loadedLhs = builder.create<AffineLoadOp>(
|
auto loadedLhs = builder.create<affine::AffineLoadOp>(
|
||||||
loc, binaryAdaptor.getLhs(), loopIvs);
|
loc, binaryAdaptor.getLhs(), loopIvs);
|
||||||
auto loadedRhs = builder.create<AffineLoadOp>(
|
auto loadedRhs = builder.create<affine::AffineLoadOp>(
|
||||||
loc, binaryAdaptor.getRhs(), loopIvs);
|
loc, binaryAdaptor.getRhs(), loopIvs);
|
||||||
|
|
||||||
// Create the binary operation performed on the loaded
|
// Create the binary operation performed on the loaded
|
||||||
@ -174,7 +175,7 @@ struct ConstantOpLowering : public OpRewritePattern<toy::ConstantOp> {
|
|||||||
// The last dimension is the base case of the recursion, at this point
|
// The last dimension is the base case of the recursion, at this point
|
||||||
// we store the element at the given index.
|
// we store the element at the given index.
|
||||||
if (dimension == valueShape.size()) {
|
if (dimension == valueShape.size()) {
|
||||||
rewriter.create<AffineStoreOp>(
|
rewriter.create<affine::AffineStoreOp>(
|
||||||
loc, rewriter.create<arith::ConstantOp>(loc, *valueIt++), alloc,
|
loc, rewriter.create<arith::ConstantOp>(loc, *valueIt++), alloc,
|
||||||
llvm::ArrayRef(indices));
|
llvm::ArrayRef(indices));
|
||||||
return;
|
return;
|
||||||
@ -291,8 +292,8 @@ struct TransposeOpLowering : public ConversionPattern {
|
|||||||
// Transpose the elements by generating a load from the
|
// Transpose the elements by generating a load from the
|
||||||
// reverse indices.
|
// reverse indices.
|
||||||
SmallVector<Value, 2> reverseIvs(llvm::reverse(loopIvs));
|
SmallVector<Value, 2> reverseIvs(llvm::reverse(loopIvs));
|
||||||
return builder.create<AffineLoadOp>(loc, input,
|
return builder.create<affine::AffineLoadOp>(loc, input,
|
||||||
reverseIvs);
|
reverseIvs);
|
||||||
});
|
});
|
||||||
return success();
|
return success();
|
||||||
}
|
}
|
||||||
@ -313,7 +314,8 @@ struct ToyToAffineLoweringPass
|
|||||||
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ToyToAffineLoweringPass)
|
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ToyToAffineLoweringPass)
|
||||||
|
|
||||||
void getDependentDialects(DialectRegistry ®istry) const override {
|
void getDependentDialects(DialectRegistry ®istry) const override {
|
||||||
registry.insert<AffineDialect, func::FuncDialect, memref::MemRefDialect>();
|
registry.insert<affine::AffineDialect, func::FuncDialect,
|
||||||
|
memref::MemRefDialect>();
|
||||||
}
|
}
|
||||||
void runOnOperation() final;
|
void runOnOperation() final;
|
||||||
};
|
};
|
||||||
@ -327,8 +329,9 @@ void ToyToAffineLoweringPass::runOnOperation() {
|
|||||||
// We define the specific operations, or dialects, that are legal targets for
|
// We define the specific operations, or dialects, that are legal targets for
|
||||||
// this lowering. In our case, we are lowering to a combination of the
|
// this lowering. In our case, we are lowering to a combination of the
|
||||||
// `Affine`, `Arith`, `Func`, and `MemRef` dialects.
|
// `Affine`, `Arith`, `Func`, and `MemRef` dialects.
|
||||||
target.addLegalDialect<AffineDialect, BuiltinDialect, arith::ArithDialect,
|
target.addLegalDialect<affine::AffineDialect, BuiltinDialect,
|
||||||
func::FuncDialect, memref::MemRefDialect>();
|
arith::ArithDialect, func::FuncDialect,
|
||||||
|
memref::MemRefDialect>();
|
||||||
|
|
||||||
// We also define the Toy dialect as Illegal so that the conversion will fail
|
// We also define the Toy dialect as Illegal so that the conversion will fail
|
||||||
// if any of these operations are *not* converted. Given that we actually want
|
// if any of these operations are *not* converted. Given that we actually want
|
||||||
|
@ -166,8 +166,8 @@ int loadAndProcessMLIR(mlir::MLIRContext &context,
|
|||||||
|
|
||||||
// Add optimizations if enabled.
|
// Add optimizations if enabled.
|
||||||
if (enableOpt) {
|
if (enableOpt) {
|
||||||
optPM.addPass(mlir::createLoopFusionPass());
|
optPM.addPass(mlir::affine::createLoopFusionPass());
|
||||||
optPM.addPass(mlir::createAffineScalarReplacementPass());
|
optPM.addPass(mlir::affine::createAffineScalarReplacementPass());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -75,14 +75,15 @@ static void lowerOpToLoops(Operation *op, ValueRange operands,
|
|||||||
// loop induction variables.
|
// loop induction variables.
|
||||||
SmallVector<int64_t, 4> lowerBounds(tensorType.getRank(), /*Value=*/0);
|
SmallVector<int64_t, 4> lowerBounds(tensorType.getRank(), /*Value=*/0);
|
||||||
SmallVector<int64_t, 4> steps(tensorType.getRank(), /*Value=*/1);
|
SmallVector<int64_t, 4> steps(tensorType.getRank(), /*Value=*/1);
|
||||||
buildAffineLoopNest(
|
affine::buildAffineLoopNest(
|
||||||
rewriter, loc, lowerBounds, tensorType.getShape(), steps,
|
rewriter, loc, lowerBounds, tensorType.getShape(), steps,
|
||||||
[&](OpBuilder &nestedBuilder, Location loc, ValueRange ivs) {
|
[&](OpBuilder &nestedBuilder, Location loc, ValueRange ivs) {
|
||||||
// Call the processing function with the rewriter, the memref operands,
|
// Call the processing function with the rewriter, the memref operands,
|
||||||
// and the loop induction variables. This function will return the value
|
// and the loop induction variables. This function will return the value
|
||||||
// to store at the current index.
|
// to store at the current index.
|
||||||
Value valueToStore = processIteration(nestedBuilder, operands, ivs);
|
Value valueToStore = processIteration(nestedBuilder, operands, ivs);
|
||||||
nestedBuilder.create<AffineStoreOp>(loc, valueToStore, alloc, ivs);
|
nestedBuilder.create<affine::AffineStoreOp>(loc, valueToStore, alloc,
|
||||||
|
ivs);
|
||||||
});
|
});
|
||||||
|
|
||||||
// Replace this operation with the generated alloc.
|
// Replace this operation with the generated alloc.
|
||||||
@ -113,9 +114,9 @@ struct BinaryOpLowering : public ConversionPattern {
|
|||||||
|
|
||||||
// Generate loads for the element of 'lhs' and 'rhs' at the
|
// Generate loads for the element of 'lhs' and 'rhs' at the
|
||||||
// inner loop.
|
// inner loop.
|
||||||
auto loadedLhs = builder.create<AffineLoadOp>(
|
auto loadedLhs = builder.create<affine::AffineLoadOp>(
|
||||||
loc, binaryAdaptor.getLhs(), loopIvs);
|
loc, binaryAdaptor.getLhs(), loopIvs);
|
||||||
auto loadedRhs = builder.create<AffineLoadOp>(
|
auto loadedRhs = builder.create<affine::AffineLoadOp>(
|
||||||
loc, binaryAdaptor.getRhs(), loopIvs);
|
loc, binaryAdaptor.getRhs(), loopIvs);
|
||||||
|
|
||||||
// Create the binary operation performed on the loaded
|
// Create the binary operation performed on the loaded
|
||||||
@ -174,7 +175,7 @@ struct ConstantOpLowering : public OpRewritePattern<toy::ConstantOp> {
|
|||||||
// The last dimension is the base case of the recursion, at this point
|
// The last dimension is the base case of the recursion, at this point
|
||||||
// we store the element at the given index.
|
// we store the element at the given index.
|
||||||
if (dimension == valueShape.size()) {
|
if (dimension == valueShape.size()) {
|
||||||
rewriter.create<AffineStoreOp>(
|
rewriter.create<affine::AffineStoreOp>(
|
||||||
loc, rewriter.create<arith::ConstantOp>(loc, *valueIt++), alloc,
|
loc, rewriter.create<arith::ConstantOp>(loc, *valueIt++), alloc,
|
||||||
llvm::ArrayRef(indices));
|
llvm::ArrayRef(indices));
|
||||||
return;
|
return;
|
||||||
@ -291,8 +292,8 @@ struct TransposeOpLowering : public ConversionPattern {
|
|||||||
// Transpose the elements by generating a load from the
|
// Transpose the elements by generating a load from the
|
||||||
// reverse indices.
|
// reverse indices.
|
||||||
SmallVector<Value, 2> reverseIvs(llvm::reverse(loopIvs));
|
SmallVector<Value, 2> reverseIvs(llvm::reverse(loopIvs));
|
||||||
return builder.create<AffineLoadOp>(loc, input,
|
return builder.create<affine::AffineLoadOp>(loc, input,
|
||||||
reverseIvs);
|
reverseIvs);
|
||||||
});
|
});
|
||||||
return success();
|
return success();
|
||||||
}
|
}
|
||||||
@ -313,7 +314,8 @@ struct ToyToAffineLoweringPass
|
|||||||
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ToyToAffineLoweringPass)
|
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ToyToAffineLoweringPass)
|
||||||
|
|
||||||
void getDependentDialects(DialectRegistry ®istry) const override {
|
void getDependentDialects(DialectRegistry ®istry) const override {
|
||||||
registry.insert<AffineDialect, func::FuncDialect, memref::MemRefDialect>();
|
registry.insert<affine::AffineDialect, func::FuncDialect,
|
||||||
|
memref::MemRefDialect>();
|
||||||
}
|
}
|
||||||
void runOnOperation() final;
|
void runOnOperation() final;
|
||||||
};
|
};
|
||||||
@ -327,8 +329,9 @@ void ToyToAffineLoweringPass::runOnOperation() {
|
|||||||
// We define the specific operations, or dialects, that are legal targets for
|
// We define the specific operations, or dialects, that are legal targets for
|
||||||
// this lowering. In our case, we are lowering to a combination of the
|
// this lowering. In our case, we are lowering to a combination of the
|
||||||
// `Affine`, `Arith`, `Func`, and `MemRef` dialects.
|
// `Affine`, `Arith`, `Func`, and `MemRef` dialects.
|
||||||
target.addLegalDialect<AffineDialect, BuiltinDialect, arith::ArithDialect,
|
target.addLegalDialect<affine::AffineDialect, BuiltinDialect,
|
||||||
func::FuncDialect, memref::MemRefDialect>();
|
arith::ArithDialect, func::FuncDialect,
|
||||||
|
memref::MemRefDialect>();
|
||||||
|
|
||||||
// We also define the Toy dialect as Illegal so that the conversion will fail
|
// We also define the Toy dialect as Illegal so that the conversion will fail
|
||||||
// if any of these operations are *not* converted. Given that we actually want
|
// if any of these operations are *not* converted. Given that we actually want
|
||||||
|
@ -167,8 +167,8 @@ int loadAndProcessMLIR(mlir::MLIRContext &context,
|
|||||||
|
|
||||||
// Add optimizations if enabled.
|
// Add optimizations if enabled.
|
||||||
if (enableOpt) {
|
if (enableOpt) {
|
||||||
optPM.addPass(mlir::createLoopFusionPass());
|
optPM.addPass(mlir::affine::createLoopFusionPass());
|
||||||
optPM.addPass(mlir::createAffineScalarReplacementPass());
|
optPM.addPass(mlir::affine::createAffineScalarReplacementPass());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -12,16 +12,18 @@
|
|||||||
#include "mlir/Support/LLVM.h"
|
#include "mlir/Support/LLVM.h"
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
class AffineForOp;
|
|
||||||
class Location;
|
class Location;
|
||||||
struct LogicalResult;
|
struct LogicalResult;
|
||||||
class OpBuilder;
|
class OpBuilder;
|
||||||
class Pass;
|
class Pass;
|
||||||
class RewritePattern;
|
class RewritePattern;
|
||||||
|
class RewritePatternSet;
|
||||||
class Value;
|
class Value;
|
||||||
class ValueRange;
|
class ValueRange;
|
||||||
|
|
||||||
class RewritePatternSet;
|
namespace affine {
|
||||||
|
class AffineForOp;
|
||||||
|
} // namespace affine
|
||||||
|
|
||||||
#define GEN_PASS_DECL_CONVERTAFFINETOSTANDARD
|
#define GEN_PASS_DECL_CONVERTAFFINETOSTANDARD
|
||||||
#include "mlir/Conversion/Passes.h.inc"
|
#include "mlir/Conversion/Passes.h.inc"
|
||||||
@ -37,11 +39,11 @@ void populateAffineToVectorConversionPatterns(RewritePatternSet &patterns);
|
|||||||
|
|
||||||
/// Emit code that computes the lower bound of the given affine loop using
|
/// Emit code that computes the lower bound of the given affine loop using
|
||||||
/// standard arithmetic operations.
|
/// standard arithmetic operations.
|
||||||
Value lowerAffineLowerBound(AffineForOp op, OpBuilder &builder);
|
Value lowerAffineLowerBound(affine::AffineForOp op, OpBuilder &builder);
|
||||||
|
|
||||||
/// Emit code that computes the upper bound of the given affine loop using
|
/// Emit code that computes the upper bound of the given affine loop using
|
||||||
/// standard arithmetic operations.
|
/// standard arithmetic operations.
|
||||||
Value lowerAffineUpperBound(AffineForOp op, OpBuilder &builder);
|
Value lowerAffineUpperBound(affine::AffineForOp op, OpBuilder &builder);
|
||||||
|
|
||||||
/// Lowers affine control flow operations (ForStmt, IfStmt and AffineApplyOp)
|
/// Lowers affine control flow operations (ForStmt, IfStmt and AffineApplyOp)
|
||||||
/// to equivalent lower-level constructs (flow of basic blocks and arithmetic
|
/// to equivalent lower-level constructs (flow of basic blocks and arithmetic
|
||||||
|
@ -856,7 +856,7 @@ def ConvertAffineForToGPU
|
|||||||
def ConvertParallelLoopToGpu : Pass<"convert-parallel-loops-to-gpu"> {
|
def ConvertParallelLoopToGpu : Pass<"convert-parallel-loops-to-gpu"> {
|
||||||
let summary = "Convert mapped scf.parallel ops to gpu launch operations";
|
let summary = "Convert mapped scf.parallel ops to gpu launch operations";
|
||||||
let constructor = "mlir::createParallelLoopToGpuPass()";
|
let constructor = "mlir::createParallelLoopToGpuPass()";
|
||||||
let dependentDialects = ["AffineDialect", "gpu::GPUDialect"];
|
let dependentDialects = ["affine::AffineDialect", "gpu::GPUDialect"];
|
||||||
}
|
}
|
||||||
|
|
||||||
//===----------------------------------------------------------------------===//
|
//===----------------------------------------------------------------------===//
|
||||||
@ -1033,7 +1033,7 @@ def ConvertVectorToGPU : Pass<"convert-vector-to-gpu"> {
|
|||||||
"dialect";
|
"dialect";
|
||||||
let constructor = "mlir::createConvertVectorToGPUPass()";
|
let constructor = "mlir::createConvertVectorToGPUPass()";
|
||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"memref::MemRefDialect", "gpu::GPUDialect", "AffineDialect",
|
"memref::MemRefDialect", "gpu::GPUDialect", "affine::AffineDialect",
|
||||||
"vector::VectorDialect", "nvgpu::NVGPUDialect"
|
"vector::VectorDialect", "nvgpu::NVGPUDialect"
|
||||||
];
|
];
|
||||||
|
|
||||||
@ -1052,7 +1052,7 @@ def ConvertVectorToSCF : Pass<"convert-vector-to-scf"> {
|
|||||||
"dialect";
|
"dialect";
|
||||||
let constructor = "mlir::createConvertVectorToSCFPass()";
|
let constructor = "mlir::createConvertVectorToSCFPass()";
|
||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"AffineDialect",
|
"affine::AffineDialect",
|
||||||
"memref::MemRefDialect",
|
"memref::MemRefDialect",
|
||||||
"scf::SCFDialect",
|
"scf::SCFDialect",
|
||||||
"tensor::TensorDialect"
|
"tensor::TensorDialect"
|
||||||
|
@ -11,7 +11,6 @@
|
|||||||
#include "mlir/Support/LLVM.h"
|
#include "mlir/Support/LLVM.h"
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
class AffineForOp;
|
|
||||||
class ConversionTarget;
|
class ConversionTarget;
|
||||||
struct LogicalResult;
|
struct LogicalResult;
|
||||||
class MLIRContext;
|
class MLIRContext;
|
||||||
@ -19,6 +18,10 @@ class Value;
|
|||||||
class Operation;
|
class Operation;
|
||||||
class RewritePatternSet;
|
class RewritePatternSet;
|
||||||
|
|
||||||
|
namespace affine {
|
||||||
|
class AffineForOp;
|
||||||
|
} // namespace affine
|
||||||
|
|
||||||
namespace scf {
|
namespace scf {
|
||||||
class ForOp;
|
class ForOp;
|
||||||
} // namespace scf
|
} // namespace scf
|
||||||
@ -37,7 +40,7 @@ class ForOp;
|
|||||||
// TODO: Consider removing this in favor of affine.for -> affine.parallel
|
// TODO: Consider removing this in favor of affine.for -> affine.parallel
|
||||||
// detection followed by an affine.parallel -> scf.parallel -> gpu.launch
|
// detection followed by an affine.parallel -> scf.parallel -> gpu.launch
|
||||||
// conversion
|
// conversion
|
||||||
LogicalResult convertAffineLoopNestToGPULaunch(AffineForOp forOp,
|
LogicalResult convertAffineLoopNestToGPULaunch(affine::AffineForOp forOp,
|
||||||
unsigned numBlockDims,
|
unsigned numBlockDims,
|
||||||
unsigned numThreadDims);
|
unsigned numThreadDims);
|
||||||
|
|
||||||
|
@ -21,13 +21,14 @@
|
|||||||
#include <optional>
|
#include <optional>
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
class Operation;
|
||||||
|
|
||||||
|
namespace affine {
|
||||||
class AffineApplyOp;
|
class AffineApplyOp;
|
||||||
class AffineForOp;
|
class AffineForOp;
|
||||||
class AffineValueMap;
|
class AffineValueMap;
|
||||||
class FlatAffineRelation;
|
class FlatAffineRelation;
|
||||||
class FlatAffineValueConstraints;
|
class FlatAffineValueConstraints;
|
||||||
class Operation;
|
|
||||||
|
|
||||||
/// A description of a (parallelizable) reduction in an affine loop.
|
/// A description of a (parallelizable) reduction in an affine loop.
|
||||||
struct LoopReduction {
|
struct LoopReduction {
|
||||||
@ -191,6 +192,7 @@ void getDependenceComponents(
|
|||||||
AffineForOp forOp, unsigned maxLoopDepth,
|
AffineForOp forOp, unsigned maxLoopDepth,
|
||||||
std::vector<SmallVector<DependenceComponent, 2>> *depCompsVec);
|
std::vector<SmallVector<DependenceComponent, 2>> *depCompsVec);
|
||||||
|
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#endif // MLIR_DIALECT_AFFINE_ANALYSIS_AFFINEANALYSIS_H
|
#endif // MLIR_DIALECT_AFFINE_ANALYSIS_AFFINEANALYSIS_H
|
||||||
|
@ -22,23 +22,24 @@
|
|||||||
#include <optional>
|
#include <optional>
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
|
||||||
class AffineCondition;
|
|
||||||
class AffineForOp;
|
|
||||||
class AffineIfOp;
|
|
||||||
class AffineParallelOp;
|
|
||||||
class AffineMap;
|
class AffineMap;
|
||||||
class AffineValueMap;
|
|
||||||
class IntegerSet;
|
class IntegerSet;
|
||||||
class MLIRContext;
|
|
||||||
class Value;
|
|
||||||
class MemRefType;
|
class MemRefType;
|
||||||
|
class MLIRContext;
|
||||||
struct MutableAffineMap;
|
struct MutableAffineMap;
|
||||||
|
class Value;
|
||||||
|
|
||||||
namespace presburger {
|
namespace presburger {
|
||||||
class MultiAffineFunction;
|
class MultiAffineFunction;
|
||||||
} // namespace presburger
|
} // namespace presburger
|
||||||
|
|
||||||
|
namespace affine {
|
||||||
|
class AffineCondition;
|
||||||
|
class AffineForOp;
|
||||||
|
class AffineIfOp;
|
||||||
|
class AffineParallelOp;
|
||||||
|
class AffineValueMap;
|
||||||
|
|
||||||
/// FlatAffineValueConstraints is an extension of FlatLinearValueConstraints
|
/// FlatAffineValueConstraints is an extension of FlatLinearValueConstraints
|
||||||
/// with helper functions for Affine dialect ops.
|
/// with helper functions for Affine dialect ops.
|
||||||
class FlatAffineValueConstraints : public FlatLinearValueConstraints {
|
class FlatAffineValueConstraints : public FlatLinearValueConstraints {
|
||||||
@ -254,6 +255,7 @@ LogicalResult getRelationFromMap(AffineMap &map, FlatAffineRelation &rel);
|
|||||||
LogicalResult getRelationFromMap(const AffineValueMap &map,
|
LogicalResult getRelationFromMap(const AffineValueMap &map,
|
||||||
FlatAffineRelation &rel);
|
FlatAffineRelation &rel);
|
||||||
|
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#endif // MLIR_DIALECT_AFFINE_ANALYSIS_AFFINESTRUCTURES_H
|
#endif // MLIR_DIALECT_AFFINE_ANALYSIS_AFFINESTRUCTURES_H
|
||||||
|
@ -18,16 +18,17 @@
|
|||||||
#include <optional>
|
#include <optional>
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
|
||||||
class AffineExpr;
|
class AffineExpr;
|
||||||
class AffineForOp;
|
|
||||||
class AffineMap;
|
class AffineMap;
|
||||||
class BlockArgument;
|
class BlockArgument;
|
||||||
class MemRefType;
|
class MemRefType;
|
||||||
class NestedPattern;
|
|
||||||
class Operation;
|
class Operation;
|
||||||
class Value;
|
class Value;
|
||||||
|
|
||||||
|
namespace affine {
|
||||||
|
class AffineForOp;
|
||||||
|
class NestedPattern;
|
||||||
|
|
||||||
/// Returns the trip count of the loop as an affine map with its corresponding
|
/// Returns the trip count of the loop as an affine map with its corresponding
|
||||||
/// operands if the latter is expressible as an affine expression, and nullptr
|
/// operands if the latter is expressible as an affine expression, and nullptr
|
||||||
/// otherwise. This method always succeeds as long as the lower bound is not a
|
/// otherwise. This method always succeeds as long as the lower bound is not a
|
||||||
@ -83,6 +84,7 @@ bool isVectorizableLoopBody(AffineForOp loop, int *memRefDim,
|
|||||||
// the support.
|
// the support.
|
||||||
bool isOpwiseShiftValid(AffineForOp forOp, ArrayRef<uint64_t> shifts);
|
bool isOpwiseShiftValid(AffineForOp forOp, ArrayRef<uint64_t> shifts);
|
||||||
|
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#endif // MLIR_DIALECT_AFFINE_ANALYSIS_LOOPANALYSIS_H
|
#endif // MLIR_DIALECT_AFFINE_ANALYSIS_LOOPANALYSIS_H
|
||||||
|
@ -14,10 +14,11 @@
|
|||||||
#include "llvm/Support/Allocator.h"
|
#include "llvm/Support/Allocator.h"
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
|
||||||
class NestedPattern;
|
|
||||||
class Operation;
|
class Operation;
|
||||||
|
|
||||||
|
namespace affine {
|
||||||
|
class NestedPattern;
|
||||||
|
|
||||||
/// An NestedPattern captures nested patterns in the IR.
|
/// An NestedPattern captures nested patterns in the IR.
|
||||||
/// It is used in conjunction with a scoped NestedPatternContext which is an
|
/// It is used in conjunction with a scoped NestedPatternContext which is an
|
||||||
/// llvm::BumpPtrAllocator that handles memory allocations efficiently and
|
/// llvm::BumpPtrAllocator that handles memory allocations efficiently and
|
||||||
@ -191,6 +192,7 @@ bool isReductionLoop(Operation &op);
|
|||||||
bool isLoadOrStore(Operation &op);
|
bool isLoadOrStore(Operation &op);
|
||||||
|
|
||||||
} // namespace matcher
|
} // namespace matcher
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#endif // MLIR_DIALECT_AFFINE_ANALYSIS_NESTEDMATCHER_H
|
#endif // MLIR_DIALECT_AFFINE_ANALYSIS_NESTEDMATCHER_H
|
||||||
|
@ -22,15 +22,16 @@
|
|||||||
#include <optional>
|
#include <optional>
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
|
||||||
class AffineForOp;
|
|
||||||
class AffineValueMap;
|
|
||||||
class Block;
|
class Block;
|
||||||
class Location;
|
class Location;
|
||||||
struct MemRefAccess;
|
|
||||||
class Operation;
|
class Operation;
|
||||||
class Value;
|
class Value;
|
||||||
|
|
||||||
|
namespace affine {
|
||||||
|
class AffineForOp;
|
||||||
|
class AffineValueMap;
|
||||||
|
struct MemRefAccess;
|
||||||
|
|
||||||
// LoopNestStateCollector walks loop nests and collects load and store
|
// LoopNestStateCollector walks loop nests and collects load and store
|
||||||
// operations, and whether or not a region holding op other than ForOp and IfOp
|
// operations, and whether or not a region holding op other than ForOp and IfOp
|
||||||
// was encountered in the loop nest.
|
// was encountered in the loop nest.
|
||||||
@ -576,6 +577,7 @@ FailureOr<AffineValueMap>
|
|||||||
simplifyConstrainedMinMaxOp(Operation *op,
|
simplifyConstrainedMinMaxOp(Operation *op,
|
||||||
FlatAffineValueConstraints constraints);
|
FlatAffineValueConstraints constraints);
|
||||||
|
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#endif // MLIR_DIALECT_AFFINE_ANALYSIS_UTILS_H
|
#endif // MLIR_DIALECT_AFFINE_ANALYSIS_UTILS_H
|
||||||
|
@ -17,8 +17,6 @@
|
|||||||
#include "mlir/IR/BuiltinTypes.h"
|
#include "mlir/IR/BuiltinTypes.h"
|
||||||
#include "mlir/IR/OpDefinition.h"
|
#include "mlir/IR/OpDefinition.h"
|
||||||
|
|
||||||
namespace mlir {
|
|
||||||
#include "mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.h.inc"
|
#include "mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.h.inc"
|
||||||
} // namespace mlir
|
|
||||||
|
|
||||||
#endif // MLIR_DIALECT_AFFINE_IR_AFFINEMEMORYOPINTERFACES_H
|
#endif // MLIR_DIALECT_AFFINE_IR_AFFINEMEMORYOPINTERFACES_H
|
||||||
|
@ -20,62 +20,59 @@ def AffineReadOpInterface : OpInterface<"AffineReadOpInterface"> {
|
|||||||
Interface to query characteristics of read-like ops with affine
|
Interface to query characteristics of read-like ops with affine
|
||||||
restrictions.
|
restrictions.
|
||||||
}];
|
}];
|
||||||
|
let cppNamespace = "::mlir::affine";
|
||||||
|
|
||||||
let methods = [
|
let methods = [
|
||||||
InterfaceMethod<
|
InterfaceMethod<
|
||||||
/*desc=*/"Returns the memref operand to read from.",
|
/*desc=*/"Returns the memref operand to read from.",
|
||||||
/*retTy=*/"Value",
|
/*retTy=*/"::mlir::Value",
|
||||||
/*methodName=*/"getMemRef",
|
/*methodName=*/"getMemRef",
|
||||||
/*args=*/(ins),
|
/*args=*/(ins),
|
||||||
/*methodBody*/[{}],
|
/*methodBody*/[{}],
|
||||||
/*defaultImplementation=*/ [{
|
/*defaultImplementation=*/ [{
|
||||||
ConcreteOp op = cast<ConcreteOp>(this->getOperation());
|
return $_op.getOperand($_op.getMemRefOperandIndex());
|
||||||
return op.getOperand(op.getMemRefOperandIndex());
|
|
||||||
}]
|
}]
|
||||||
>,
|
>,
|
||||||
InterfaceMethod<
|
InterfaceMethod<
|
||||||
/*desc=*/"Returns the type of the memref operand.",
|
/*desc=*/"Returns the type of the memref operand.",
|
||||||
/*retTy=*/"MemRefType",
|
/*retTy=*/"::mlir::MemRefType",
|
||||||
/*methodName=*/"getMemRefType",
|
/*methodName=*/"getMemRefType",
|
||||||
/*args=*/(ins),
|
/*args=*/(ins),
|
||||||
/*methodBody=*/[{}],
|
/*methodBody=*/[{}],
|
||||||
/*defaultImplementation=*/[{
|
/*defaultImplementation=*/[{
|
||||||
ConcreteOp op = cast<ConcreteOp>(this->getOperation());
|
return $_op.getMemRef().getType().template cast<::mlir::MemRefType>();
|
||||||
return op.getMemRef().getType().template cast<MemRefType>();
|
|
||||||
}]
|
}]
|
||||||
>,
|
>,
|
||||||
InterfaceMethod<
|
InterfaceMethod<
|
||||||
/*desc=*/"Returns affine map operands.",
|
/*desc=*/"Returns affine map operands.",
|
||||||
/*retTy=*/"Operation::operand_range",
|
/*retTy=*/"::mlir::Operation::operand_range",
|
||||||
/*methodName=*/"getMapOperands",
|
/*methodName=*/"getMapOperands",
|
||||||
/*args=*/(ins),
|
/*args=*/(ins),
|
||||||
/*methodBody=*/[{}],
|
/*methodBody=*/[{}],
|
||||||
/*defaultImplementation=*/[{
|
/*defaultImplementation=*/[{
|
||||||
ConcreteOp op = cast<ConcreteOp>(this->getOperation());
|
return llvm::drop_begin($_op.getOperands(), 1);
|
||||||
return llvm::drop_begin(op.getOperands(), 1);
|
|
||||||
}]
|
}]
|
||||||
>,
|
>,
|
||||||
InterfaceMethod<
|
InterfaceMethod<
|
||||||
/*desc=*/[{
|
/*desc=*/[{
|
||||||
Returns the affine map used to index the memref for this operation.
|
Returns the affine map used to index the memref for this operation.
|
||||||
}],
|
}],
|
||||||
/*retTy=*/"AffineMap",
|
/*retTy=*/"::mlir::AffineMap",
|
||||||
/*methodName=*/"getAffineMap",
|
/*methodName=*/"getAffineMap",
|
||||||
/*args=*/(ins),
|
/*args=*/(ins),
|
||||||
/*methodBody=*/[{}],
|
/*methodBody=*/[{}],
|
||||||
/*defaultImplementation=*/[{
|
/*defaultImplementation=*/[{
|
||||||
ConcreteOp op = cast<ConcreteOp>(this->getOperation());
|
return $_op.getAffineMapAttr().getValue();
|
||||||
return op.getAffineMapAttr().getValue();
|
|
||||||
}]
|
}]
|
||||||
>,
|
>,
|
||||||
InterfaceMethod<
|
InterfaceMethod<
|
||||||
/*desc=*/"Returns the value read by this operation.",
|
/*desc=*/"Returns the value read by this operation.",
|
||||||
/*retTy=*/"Value",
|
/*retTy=*/"::mlir::Value",
|
||||||
/*methodName=*/"getValue",
|
/*methodName=*/"getValue",
|
||||||
/*args=*/(ins),
|
/*args=*/(ins),
|
||||||
/*methodBody=*/[{}],
|
/*methodBody=*/[{}],
|
||||||
/*defaultImplementation=*/[{
|
/*defaultImplementation=*/[{
|
||||||
return cast<ConcreteOp>(this->getOperation());
|
return $_op;
|
||||||
}]
|
}]
|
||||||
>,
|
>,
|
||||||
];
|
];
|
||||||
@ -86,63 +83,59 @@ def AffineWriteOpInterface : OpInterface<"AffineWriteOpInterface"> {
|
|||||||
Interface to query characteristics of write-like ops with affine
|
Interface to query characteristics of write-like ops with affine
|
||||||
restrictions.
|
restrictions.
|
||||||
}];
|
}];
|
||||||
|
let cppNamespace = "::mlir::affine";
|
||||||
|
|
||||||
let methods = [
|
let methods = [
|
||||||
InterfaceMethod<
|
InterfaceMethod<
|
||||||
/*desc=*/"Returns the memref operand to write to.",
|
/*desc=*/"Returns the memref operand to write to.",
|
||||||
/*retTy=*/"Value",
|
/*retTy=*/"::mlir::Value",
|
||||||
/*methodName=*/"getMemRef",
|
/*methodName=*/"getMemRef",
|
||||||
/*args=*/(ins),
|
/*args=*/(ins),
|
||||||
/*methodBody=*/[{}],
|
/*methodBody=*/[{}],
|
||||||
/*defaultImplementation=*/[{
|
/*defaultImplementation=*/[{
|
||||||
ConcreteOp op = cast<ConcreteOp>(this->getOperation());
|
return $_op.getOperand($_op.getMemRefOperandIndex());
|
||||||
return op.getOperand(op.getMemRefOperandIndex());
|
|
||||||
}]
|
}]
|
||||||
>,
|
>,
|
||||||
InterfaceMethod<
|
InterfaceMethod<
|
||||||
/*desc=*/"Returns the type of the memref operand.",
|
/*desc=*/"Returns the type of the memref operand.",
|
||||||
/*retTy=*/"MemRefType",
|
/*retTy=*/"::mlir::MemRefType",
|
||||||
/*methodName=*/"getMemRefType",
|
/*methodName=*/"getMemRefType",
|
||||||
/*args=*/(ins),
|
/*args=*/(ins),
|
||||||
/*methodBody=*/[{}],
|
/*methodBody=*/[{}],
|
||||||
/*defaultImplementation=*/[{
|
/*defaultImplementation=*/[{
|
||||||
ConcreteOp op = cast<ConcreteOp>(this->getOperation());
|
return $_op.getMemRef().getType().template cast<::mlir::MemRefType>();
|
||||||
return op.getMemRef().getType().template cast<MemRefType>();
|
|
||||||
}]
|
}]
|
||||||
>,
|
>,
|
||||||
InterfaceMethod<
|
InterfaceMethod<
|
||||||
/*desc=*/"Returns affine map operands.",
|
/*desc=*/"Returns affine map operands.",
|
||||||
/*retTy=*/"Operation::operand_range",
|
/*retTy=*/"::mlir::Operation::operand_range",
|
||||||
/*methodName=*/"getMapOperands",
|
/*methodName=*/"getMapOperands",
|
||||||
/*args=*/(ins),
|
/*args=*/(ins),
|
||||||
/*methodBody=*/[{}],
|
/*methodBody=*/[{}],
|
||||||
/*defaultImplementation=*/[{
|
/*defaultImplementation=*/[{
|
||||||
ConcreteOp op = cast<ConcreteOp>(this->getOperation());
|
return llvm::drop_begin($_op.getOperands(), 2);
|
||||||
return llvm::drop_begin(op.getOperands(), 2);
|
|
||||||
}]
|
}]
|
||||||
>,
|
>,
|
||||||
InterfaceMethod<
|
InterfaceMethod<
|
||||||
/*desc=*/[{
|
/*desc=*/[{
|
||||||
Returns the affine map used to index the memref for this operation.
|
Returns the affine map used to index the memref for this operation.
|
||||||
}],
|
}],
|
||||||
/*retTy=*/"AffineMap",
|
/*retTy=*/"::mlir::AffineMap",
|
||||||
/*methodName=*/"getAffineMap",
|
/*methodName=*/"getAffineMap",
|
||||||
/*args=*/(ins),
|
/*args=*/(ins),
|
||||||
/*methodName=*/[{}],
|
/*methodName=*/[{}],
|
||||||
/*defaultImplementation=*/[{
|
/*defaultImplementation=*/[{
|
||||||
ConcreteOp op = cast<ConcreteOp>(this->getOperation());
|
return $_op.getAffineMapAttr().getValue();
|
||||||
return op.getAffineMapAttr().getValue();
|
|
||||||
}]
|
}]
|
||||||
>,
|
>,
|
||||||
InterfaceMethod<
|
InterfaceMethod<
|
||||||
/*desc=*/"Returns the value to store.",
|
/*desc=*/"Returns the value to store.",
|
||||||
/*retTy=*/"Value",
|
/*retTy=*/"::mlir::Value",
|
||||||
/*methodName=*/"getValueToStore",
|
/*methodName=*/"getValueToStore",
|
||||||
/*args=*/(ins),
|
/*args=*/(ins),
|
||||||
/*methodBody=*/[{}],
|
/*methodBody=*/[{}],
|
||||||
/*defaultImplementation=*/[{
|
/*defaultImplementation=*/[{
|
||||||
ConcreteOp op = cast<ConcreteOp>(this->getOperation());
|
return $_op.getOperand($_op.getStoredValOperandIndex());
|
||||||
return op.getOperand(op.getStoredValOperandIndex());
|
|
||||||
}]
|
}]
|
||||||
>,
|
>,
|
||||||
];
|
];
|
||||||
@ -155,20 +148,21 @@ def AffineMapAccessInterface : OpInterface<"AffineMapAccessInterface"> {
|
|||||||
memref operand. The memref argument given to this interface much match
|
memref operand. The memref argument given to this interface much match
|
||||||
one of those memref operands.
|
one of those memref operands.
|
||||||
}];
|
}];
|
||||||
|
let cppNamespace = "::mlir::affine";
|
||||||
|
|
||||||
let methods = [
|
let methods = [
|
||||||
InterfaceMethod<
|
InterfaceMethod<
|
||||||
/*desc=*/"Returns the AffineMapAttr associated with 'memref'.",
|
/*desc=*/"Returns the AffineMapAttr associated with 'memref'.",
|
||||||
/*retTy=*/"NamedAttribute",
|
/*retTy=*/"::mlir::NamedAttribute",
|
||||||
/*methodName=*/"getAffineMapAttrForMemRef",
|
/*methodName=*/"getAffineMapAttrForMemRef",
|
||||||
/*args=*/(ins "Value":$memref),
|
/*args=*/(ins "::mlir::Value":$memref),
|
||||||
/*methodBody=*/[{}],
|
/*methodBody=*/[{}],
|
||||||
/*defaultImplementation=*/[{
|
/*defaultImplementation=*/[{
|
||||||
ConcreteOp op = cast<ConcreteOp>(this->getOperation());
|
assert(memref == $_op.getMemRef() &&
|
||||||
assert(memref == op.getMemRef() &&
|
|
||||||
"Expected memref argument to match memref operand");
|
"Expected memref argument to match memref operand");
|
||||||
return {StringAttr::get(op.getContext(), op.getMapAttrStrName()),
|
return {::mlir::StringAttr::get(
|
||||||
op.getAffineMapAttr()};
|
$_op.getContext(), $_op.getMapAttrStrName()),
|
||||||
|
$_op.getAffineMapAttr()};
|
||||||
}]
|
}]
|
||||||
>,
|
>,
|
||||||
];
|
];
|
||||||
|
@ -22,13 +22,12 @@
|
|||||||
#include "mlir/Interfaces/LoopLikeInterface.h"
|
#include "mlir/Interfaces/LoopLikeInterface.h"
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
|
|
||||||
class AffineApplyOp;
|
class AffineApplyOp;
|
||||||
class AffineBound;
|
class AffineBound;
|
||||||
class AffineValueMap;
|
class AffineValueMap;
|
||||||
|
|
||||||
/// TODO: These should be renamed if they are on the mlir namespace.
|
|
||||||
/// Ideally, they should go in a mlir::affine:: namespace.
|
|
||||||
|
|
||||||
/// A utility function to check if a value is defined at the top level of an
|
/// A utility function to check if a value is defined at the top level of an
|
||||||
/// op with trait `AffineScope` or is a region argument for such an op. A value
|
/// op with trait `AffineScope` or is a region argument for such an op. A value
|
||||||
/// of index type defined at the top level is always a valid symbol for all its
|
/// of index type defined at the top level is always a valid symbol for all its
|
||||||
@ -438,13 +437,18 @@ SmallVector<Value, 4> applyMapToValues(OpBuilder &b, Location loc,
|
|||||||
/// argument.
|
/// argument.
|
||||||
void fullyComposeAffineMapAndOperands(AffineMap *map,
|
void fullyComposeAffineMapAndOperands(AffineMap *map,
|
||||||
SmallVectorImpl<Value> *operands);
|
SmallVectorImpl<Value> *operands);
|
||||||
|
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#include "mlir/Dialect/Affine/IR/AffineOpsDialect.h.inc"
|
#include "mlir/Dialect/Affine/IR/AffineOpsDialect.h.inc"
|
||||||
|
|
||||||
#define GET_OP_CLASSES
|
#define GET_OP_CLASSES
|
||||||
#include "mlir/Dialect/Affine/IR/AffineOps.h.inc"
|
#include "mlir/Dialect/Affine/IR/AffineOps.h.inc"
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
|
|
||||||
/// Returns true if the provided value is the induction variable of an
|
/// Returns true if the provided value is the induction variable of an
|
||||||
/// AffineForOp.
|
/// AffineForOp.
|
||||||
bool isAffineForInductionVar(Value val);
|
bool isAffineForInductionVar(Value val);
|
||||||
@ -537,6 +541,7 @@ private:
|
|||||||
friend class AffineForOp;
|
friend class AffineForOp;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
@ -21,7 +21,7 @@ include "mlir/Interfaces/SideEffectInterfaces.td"
|
|||||||
|
|
||||||
def Affine_Dialect : Dialect {
|
def Affine_Dialect : Dialect {
|
||||||
let name = "affine";
|
let name = "affine";
|
||||||
let cppNamespace = "mlir";
|
let cppNamespace = "::mlir::affine";
|
||||||
let hasConstantMaterializer = 1;
|
let hasConstantMaterializer = 1;
|
||||||
let dependentDialects = ["arith::ArithDialect"];
|
let dependentDialects = ["arith::ArithDialect"];
|
||||||
}
|
}
|
||||||
|
@ -18,6 +18,7 @@
|
|||||||
#include "mlir/IR/Value.h"
|
#include "mlir/IR/Value.h"
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
|
|
||||||
/// An AffineValueMap is an affine map plus its ML value operands and
|
/// An AffineValueMap is an affine map plus its ML value operands and
|
||||||
/// results for analysis purposes. The structure is still a tree form that is
|
/// results for analysis purposes. The structure is still a tree form that is
|
||||||
@ -89,6 +90,7 @@ private:
|
|||||||
SmallVector<Value, 4> results;
|
SmallVector<Value, 4> results;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#endif // MLIR_DIALECT_AFFINE_IR_AFFINEVALUEMAP_H
|
#endif // MLIR_DIALECT_AFFINE_IR_AFFINEVALUEMAP_H
|
||||||
|
@ -21,9 +21,11 @@
|
|||||||
#include "llvm/ADT/SmallVector.h"
|
#include "llvm/ADT/SmallVector.h"
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
class Operation;
|
||||||
|
|
||||||
|
namespace affine {
|
||||||
class AffineForOp;
|
class AffineForOp;
|
||||||
struct ComputationSliceState;
|
struct ComputationSliceState;
|
||||||
class Operation;
|
|
||||||
|
|
||||||
// TODO: Extend this module to include utility functions for querying fusion
|
// TODO: Extend this module to include utility functions for querying fusion
|
||||||
// cost/storage reduction, and for performing the loop fusion transformation.
|
// cost/storage reduction, and for performing the loop fusion transformation.
|
||||||
@ -165,6 +167,8 @@ bool getFusionComputeCost(AffineForOp srcForOp, LoopNestStats &srcStats,
|
|||||||
void gatherProducerConsumerMemrefs(ArrayRef<Operation *> srcOps,
|
void gatherProducerConsumerMemrefs(ArrayRef<Operation *> srcOps,
|
||||||
ArrayRef<Operation *> dstOps,
|
ArrayRef<Operation *> dstOps,
|
||||||
DenseSet<Value> &producerConsumerMemrefs);
|
DenseSet<Value> &producerConsumerMemrefs);
|
||||||
|
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#endif // MLIR_DIALECT_AFFINE_LOOPFUSIONUTILS_H
|
#endif // MLIR_DIALECT_AFFINE_LOOPFUSIONUTILS_H
|
||||||
|
@ -22,10 +22,8 @@
|
|||||||
#include <optional>
|
#include <optional>
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
class AffineForOp;
|
|
||||||
class AffineMap;
|
class AffineMap;
|
||||||
class LoopLikeOpInterface;
|
class LoopLikeOpInterface;
|
||||||
struct MemRefRegion;
|
|
||||||
class OpBuilder;
|
class OpBuilder;
|
||||||
class Value;
|
class Value;
|
||||||
class ValueRange;
|
class ValueRange;
|
||||||
@ -39,6 +37,10 @@ class ForOp;
|
|||||||
class ParallelOp;
|
class ParallelOp;
|
||||||
} // namespace scf
|
} // namespace scf
|
||||||
|
|
||||||
|
namespace affine {
|
||||||
|
class AffineForOp;
|
||||||
|
struct MemRefRegion;
|
||||||
|
|
||||||
/// Unrolls this for operation completely if the trip count is known to be
|
/// Unrolls this for operation completely if the trip count is known to be
|
||||||
/// constant. Returns failure otherwise.
|
/// constant. Returns failure otherwise.
|
||||||
LogicalResult loopUnrollFull(AffineForOp forOp);
|
LogicalResult loopUnrollFull(AffineForOp forOp);
|
||||||
@ -345,6 +347,7 @@ LogicalResult coalescePerfectlyNestedLoops(LoopOpTy op) {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#endif // MLIR_DIALECT_AFFINE_LOOPUTILS_H
|
#endif // MLIR_DIALECT_AFFINE_LOOPUTILS_H
|
||||||
|
@ -23,6 +23,7 @@ namespace func {
|
|||||||
class FuncOp;
|
class FuncOp;
|
||||||
} // namespace func
|
} // namespace func
|
||||||
|
|
||||||
|
namespace affine {
|
||||||
class AffineForOp;
|
class AffineForOp;
|
||||||
|
|
||||||
/// Fusion mode to attempt. The default mode `Greedy` does both
|
/// Fusion mode to attempt. The default mode `Greedy` does both
|
||||||
@ -123,6 +124,7 @@ std::unique_ptr<Pass> createAffineExpandIndexOpsPass();
|
|||||||
#define GEN_PASS_REGISTRATION
|
#define GEN_PASS_REGISTRATION
|
||||||
#include "mlir/Dialect/Affine/Passes.h.inc"
|
#include "mlir/Dialect/Affine/Passes.h.inc"
|
||||||
|
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#endif // MLIR_DIALECT_AFFINE_PASSES_H
|
#endif // MLIR_DIALECT_AFFINE_PASSES_H
|
||||||
|
@ -17,7 +17,7 @@ include "mlir/Pass/PassBase.td"
|
|||||||
|
|
||||||
def AffineDataCopyGeneration : Pass<"affine-data-copy-generate", "func::FuncOp"> {
|
def AffineDataCopyGeneration : Pass<"affine-data-copy-generate", "func::FuncOp"> {
|
||||||
let summary = "Generate explicit copying for affine memory operations";
|
let summary = "Generate explicit copying for affine memory operations";
|
||||||
let constructor = "mlir::createAffineDataCopyGenerationPass()";
|
let constructor = "mlir::affine::createAffineDataCopyGenerationPass()";
|
||||||
let dependentDialects = ["memref::MemRefDialect"];
|
let dependentDialects = ["memref::MemRefDialect"];
|
||||||
let options = [
|
let options = [
|
||||||
Option<"fastMemoryCapacity", "fast-mem-capacity", "uint64_t",
|
Option<"fastMemoryCapacity", "fast-mem-capacity", "uint64_t",
|
||||||
@ -152,7 +152,7 @@ def AffineLoopFusion : Pass<"affine-loop-fusion"> {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
}];
|
}];
|
||||||
let constructor = "mlir::createLoopFusionPass()";
|
let constructor = "mlir::affine::createLoopFusionPass()";
|
||||||
let options = [
|
let options = [
|
||||||
Option<"computeToleranceThreshold", "fusion-compute-tolerance", "double",
|
Option<"computeToleranceThreshold", "fusion-compute-tolerance", "double",
|
||||||
/*default=*/"0.30f", "Fractional increase in additional computation "
|
/*default=*/"0.30f", "Fractional increase in additional computation "
|
||||||
@ -166,12 +166,12 @@ def AffineLoopFusion : Pass<"affine-loop-fusion"> {
|
|||||||
Option<"maximalFusion", "fusion-maximal", "bool", /*default=*/"false",
|
Option<"maximalFusion", "fusion-maximal", "bool", /*default=*/"false",
|
||||||
"Enables maximal loop fusion">,
|
"Enables maximal loop fusion">,
|
||||||
Option<"affineFusionMode", "mode", "enum FusionMode",
|
Option<"affineFusionMode", "mode", "enum FusionMode",
|
||||||
"mlir::FusionMode::Greedy", "fusion mode to attempt",
|
"mlir::affine::FusionMode::Greedy", "fusion mode to attempt",
|
||||||
"llvm::cl::values(clEnumValN(mlir::FusionMode::Greedy,"
|
"llvm::cl::values(clEnumValN(mlir::affine::FusionMode::Greedy,"
|
||||||
" \"greedy\", \"Perform greedy (both producer-consumer and sibling) fusion\"), "
|
" \"greedy\", \"Perform greedy (both producer-consumer and sibling) fusion\"), "
|
||||||
"clEnumValN( mlir::FusionMode::ProducerConsumer, "
|
"clEnumValN( mlir::affine::FusionMode::ProducerConsumer, "
|
||||||
"\"producer\", \"Perform only producer-consumer fusion\"), "
|
"\"producer\", \"Perform only producer-consumer fusion\"), "
|
||||||
"clEnumValN( mlir::FusionMode::Sibling, "
|
"clEnumValN( mlir::affine::FusionMode::Sibling, "
|
||||||
"\"sibling\", \"Perform only sibling fusion\"))">,
|
"\"sibling\", \"Perform only sibling fusion\"))">,
|
||||||
];
|
];
|
||||||
let dependentDialects = ["memref::MemRefDialect"];
|
let dependentDialects = ["memref::MemRefDialect"];
|
||||||
@ -180,12 +180,12 @@ def AffineLoopFusion : Pass<"affine-loop-fusion"> {
|
|||||||
def AffineLoopInvariantCodeMotion
|
def AffineLoopInvariantCodeMotion
|
||||||
: Pass<"affine-loop-invariant-code-motion", "func::FuncOp"> {
|
: Pass<"affine-loop-invariant-code-motion", "func::FuncOp"> {
|
||||||
let summary = "Hoist loop invariant instructions outside of affine loops";
|
let summary = "Hoist loop invariant instructions outside of affine loops";
|
||||||
let constructor = "mlir::createAffineLoopInvariantCodeMotionPass()";
|
let constructor = "mlir::affine::createAffineLoopInvariantCodeMotionPass()";
|
||||||
}
|
}
|
||||||
|
|
||||||
def AffineLoopTiling : Pass<"affine-loop-tile", "func::FuncOp"> {
|
def AffineLoopTiling : Pass<"affine-loop-tile", "func::FuncOp"> {
|
||||||
let summary = "Tile affine loop nests";
|
let summary = "Tile affine loop nests";
|
||||||
let constructor = "mlir::createLoopTilingPass()";
|
let constructor = "mlir::affine::createLoopTilingPass()";
|
||||||
let options = [
|
let options = [
|
||||||
Option<"cacheSizeInKiB", "cache-size", "uint64_t", /*default=*/"512",
|
Option<"cacheSizeInKiB", "cache-size", "uint64_t", /*default=*/"512",
|
||||||
"Set size of cache to tile for in KiB (default: 512)">,
|
"Set size of cache to tile for in KiB (default: 512)">,
|
||||||
@ -201,7 +201,7 @@ def AffineLoopTiling : Pass<"affine-loop-tile", "func::FuncOp"> {
|
|||||||
|
|
||||||
def AffineLoopUnroll : Pass<"affine-loop-unroll", "func::FuncOp"> {
|
def AffineLoopUnroll : Pass<"affine-loop-unroll", "func::FuncOp"> {
|
||||||
let summary = "Unroll affine loops";
|
let summary = "Unroll affine loops";
|
||||||
let constructor = "mlir::createLoopUnrollPass()";
|
let constructor = "mlir::affine::createLoopUnrollPass()";
|
||||||
let options = [
|
let options = [
|
||||||
Option<"unrollFactor", "unroll-factor", "unsigned", /*default=*/"4",
|
Option<"unrollFactor", "unroll-factor", "unsigned", /*default=*/"4",
|
||||||
"Use this unroll factor for all loops being unrolled">,
|
"Use this unroll factor for all loops being unrolled">,
|
||||||
@ -221,7 +221,7 @@ def AffineLoopUnroll : Pass<"affine-loop-unroll", "func::FuncOp"> {
|
|||||||
|
|
||||||
def AffineLoopUnrollAndJam : Pass<"affine-loop-unroll-jam", "func::FuncOp"> {
|
def AffineLoopUnrollAndJam : Pass<"affine-loop-unroll-jam", "func::FuncOp"> {
|
||||||
let summary = "Unroll and jam affine loops";
|
let summary = "Unroll and jam affine loops";
|
||||||
let constructor = "mlir::createLoopUnrollAndJamPass()";
|
let constructor = "mlir::affine::createLoopUnrollAndJamPass()";
|
||||||
let options = [
|
let options = [
|
||||||
Option<"unrollJamFactor", "unroll-jam-factor", "unsigned",
|
Option<"unrollJamFactor", "unroll-jam-factor", "unsigned",
|
||||||
/*default=*/"4",
|
/*default=*/"4",
|
||||||
@ -295,7 +295,7 @@ def AffinePipelineDataTransfer
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
}];
|
}];
|
||||||
let constructor = "mlir::createPipelineDataTransferPass()";
|
let constructor = "mlir::affine::createPipelineDataTransferPass()";
|
||||||
}
|
}
|
||||||
|
|
||||||
def AffineScalarReplacement : Pass<"affine-scalrep", "func::FuncOp"> {
|
def AffineScalarReplacement : Pass<"affine-scalrep", "func::FuncOp"> {
|
||||||
@ -341,7 +341,7 @@ def AffineScalarReplacement : Pass<"affine-scalrep", "func::FuncOp"> {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
}];
|
}];
|
||||||
let constructor = "mlir::createAffineScalarReplacementPass()";
|
let constructor = "mlir::affine::createAffineScalarReplacementPass()";
|
||||||
}
|
}
|
||||||
|
|
||||||
def AffineVectorize : Pass<"affine-super-vectorize", "func::FuncOp"> {
|
def AffineVectorize : Pass<"affine-super-vectorize", "func::FuncOp"> {
|
||||||
@ -369,7 +369,7 @@ def AffineVectorize : Pass<"affine-super-vectorize", "func::FuncOp"> {
|
|||||||
|
|
||||||
def AffineParallelize : Pass<"affine-parallelize", "func::FuncOp"> {
|
def AffineParallelize : Pass<"affine-parallelize", "func::FuncOp"> {
|
||||||
let summary = "Convert affine.for ops into 1-D affine.parallel";
|
let summary = "Convert affine.for ops into 1-D affine.parallel";
|
||||||
let constructor = "mlir::createAffineParallelizePass()";
|
let constructor = "mlir::affine::createAffineParallelizePass()";
|
||||||
let options = [
|
let options = [
|
||||||
Option<"maxNested", "max-nested", "unsigned", /*default=*/"-1u",
|
Option<"maxNested", "max-nested", "unsigned", /*default=*/"-1u",
|
||||||
"Maximum number of nested parallel loops to produce. "
|
"Maximum number of nested parallel loops to produce. "
|
||||||
@ -382,7 +382,7 @@ def AffineParallelize : Pass<"affine-parallelize", "func::FuncOp"> {
|
|||||||
|
|
||||||
def AffineLoopNormalize : Pass<"affine-loop-normalize", "func::FuncOp"> {
|
def AffineLoopNormalize : Pass<"affine-loop-normalize", "func::FuncOp"> {
|
||||||
let summary = "Apply normalization transformations to affine loop-like ops";
|
let summary = "Apply normalization transformations to affine loop-like ops";
|
||||||
let constructor = "mlir::createAffineLoopNormalizePass()";
|
let constructor = "mlir::affine::createAffineLoopNormalizePass()";
|
||||||
let options = [
|
let options = [
|
||||||
Option<"promoteSingleIter", "promote-single-iter", "bool",
|
Option<"promoteSingleIter", "promote-single-iter", "bool",
|
||||||
/*default=*/"true", "Promote single iteration loops">,
|
/*default=*/"true", "Promote single iteration loops">,
|
||||||
@ -392,19 +392,19 @@ def AffineLoopNormalize : Pass<"affine-loop-normalize", "func::FuncOp"> {
|
|||||||
def LoopCoalescing : Pass<"affine-loop-coalescing", "func::FuncOp"> {
|
def LoopCoalescing : Pass<"affine-loop-coalescing", "func::FuncOp"> {
|
||||||
let summary = "Coalesce nested loops with independent bounds into a single "
|
let summary = "Coalesce nested loops with independent bounds into a single "
|
||||||
"loop";
|
"loop";
|
||||||
let constructor = "mlir::createLoopCoalescingPass()";
|
let constructor = "mlir::affine::createLoopCoalescingPass()";
|
||||||
let dependentDialects = ["arith::ArithDialect"];
|
let dependentDialects = ["arith::ArithDialect"];
|
||||||
}
|
}
|
||||||
|
|
||||||
def SimplifyAffineStructures : Pass<"affine-simplify-structures", "func::FuncOp"> {
|
def SimplifyAffineStructures : Pass<"affine-simplify-structures", "func::FuncOp"> {
|
||||||
let summary = "Simplify affine expressions in maps/sets and normalize "
|
let summary = "Simplify affine expressions in maps/sets and normalize "
|
||||||
"memrefs";
|
"memrefs";
|
||||||
let constructor = "mlir::createSimplifyAffineStructuresPass()";
|
let constructor = "mlir::affine::createSimplifyAffineStructuresPass()";
|
||||||
}
|
}
|
||||||
|
|
||||||
def AffineExpandIndexOps : Pass<"affine-expand-index-ops"> {
|
def AffineExpandIndexOps : Pass<"affine-expand-index-ops"> {
|
||||||
let summary = "Lower affine operations operating on indices into more fundamental operations";
|
let summary = "Lower affine operations operating on indices into more fundamental operations";
|
||||||
let constructor = "mlir::createAffineExpandIndexOpsPass()";
|
let constructor = "mlir::affine::createAffineExpandIndexOpsPass()";
|
||||||
}
|
}
|
||||||
|
|
||||||
#endif // MLIR_DIALECT_AFFINE_PASSES
|
#endif // MLIR_DIALECT_AFFINE_PASSES
|
||||||
|
@ -15,12 +15,11 @@
|
|||||||
#include "mlir/IR/OpImplementation.h"
|
#include "mlir/IR/OpImplementation.h"
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
class AffineForOp;
|
|
||||||
namespace func {
|
namespace func {
|
||||||
class FuncOp;
|
class FuncOp;
|
||||||
} // namespace func
|
} // namespace func
|
||||||
namespace affine {
|
namespace affine {
|
||||||
class ForOp;
|
class AffineForOp;
|
||||||
} // namespace affine
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
|
@ -18,7 +18,6 @@
|
|||||||
#include "mlir/Support/LogicalResult.h"
|
#include "mlir/Support/LogicalResult.h"
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
class AffineApplyOp;
|
|
||||||
class Location;
|
class Location;
|
||||||
class OpBuilder;
|
class OpBuilder;
|
||||||
class OpFoldResult;
|
class OpFoldResult;
|
||||||
@ -30,6 +29,9 @@ namespace presburger {
|
|||||||
enum class BoundType;
|
enum class BoundType;
|
||||||
} // namespace presburger
|
} // namespace presburger
|
||||||
|
|
||||||
|
namespace affine {
|
||||||
|
class AffineApplyOp;
|
||||||
|
|
||||||
/// Populate patterns that expand affine index operations into more fundamental
|
/// Populate patterns that expand affine index operations into more fundamental
|
||||||
/// operations (not necessarily restricted to Affine dialect).
|
/// operations (not necessarily restricted to Affine dialect).
|
||||||
void populateAffineExpandIndexOpsPatterns(RewritePatternSet &patterns);
|
void populateAffineExpandIndexOpsPatterns(RewritePatternSet &patterns);
|
||||||
@ -83,6 +85,7 @@ FailureOr<OpFoldResult> reifyShapedValueDimBound(
|
|||||||
ValueBoundsConstraintSet::StopConditionFn stopCondition = nullptr,
|
ValueBoundsConstraintSet::StopConditionFn stopCondition = nullptr,
|
||||||
bool closedUB = false);
|
bool closedUB = false);
|
||||||
|
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#endif // MLIR_DIALECT_AFFINE_TRANSFORMS_TRANSFORMS_H
|
#endif // MLIR_DIALECT_AFFINE_TRANSFORMS_TRANSFORMS_H
|
||||||
|
@ -18,10 +18,6 @@
|
|||||||
#include <optional>
|
#include <optional>
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
|
||||||
class AffineForOp;
|
|
||||||
class AffineIfOp;
|
|
||||||
class AffineParallelOp;
|
|
||||||
class DominanceInfo;
|
class DominanceInfo;
|
||||||
class Operation;
|
class Operation;
|
||||||
class PostDominanceInfo;
|
class PostDominanceInfo;
|
||||||
@ -36,6 +32,11 @@ class AllocOp;
|
|||||||
|
|
||||||
struct LogicalResult;
|
struct LogicalResult;
|
||||||
|
|
||||||
|
namespace affine {
|
||||||
|
class AffineForOp;
|
||||||
|
class AffineIfOp;
|
||||||
|
class AffineParallelOp;
|
||||||
|
|
||||||
using ReductionLoopMap = DenseMap<Operation *, SmallVector<LoopReduction, 2>>;
|
using ReductionLoopMap = DenseMap<Operation *, SmallVector<LoopReduction, 2>>;
|
||||||
|
|
||||||
/// Replaces a parallel affine.for op with a 1-d affine.parallel op. `forOp`'s
|
/// Replaces a parallel affine.for op with a 1-d affine.parallel op. `forOp`'s
|
||||||
@ -384,6 +385,7 @@ private:
|
|||||||
Location loc;
|
Location loc;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#endif // MLIR_DIALECT_AFFINE_UTILS_H
|
#endif // MLIR_DIALECT_AFFINE_UTILS_H
|
||||||
|
@ -16,6 +16,8 @@
|
|||||||
namespace mlir {
|
namespace mlir {
|
||||||
class RewriterBase;
|
class RewriterBase;
|
||||||
|
|
||||||
|
namespace affine {
|
||||||
|
|
||||||
/// Fills the `combinedOffsets`, `combinedSizes` and `combinedStrides` to use
|
/// Fills the `combinedOffsets`, `combinedSizes` and `combinedStrides` to use
|
||||||
/// when combining a producer slice **into** a consumer slice.
|
/// when combining a producer slice **into** a consumer slice.
|
||||||
///
|
///
|
||||||
@ -99,6 +101,7 @@ void resolveSizesIntoOpWithSizes(
|
|||||||
const llvm::SmallBitVector &rankReducedSourceDims,
|
const llvm::SmallBitVector &rankReducedSourceDims,
|
||||||
SmallVectorImpl<OpFoldResult> &resolvedSizes);
|
SmallVectorImpl<OpFoldResult> &resolvedSizes);
|
||||||
|
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#endif // MLIR_DIALECT_AFFINE_VIEWLIKEINTERFACEUTILS_H
|
#endif // MLIR_DIALECT_AFFINE_VIEWLIKEINTERFACEUTILS_H
|
||||||
|
@ -26,7 +26,7 @@ def Bufferization_Dialect : Dialect {
|
|||||||
deallocation](/docs/BufferDeallocationInternals/).
|
deallocation](/docs/BufferDeallocationInternals/).
|
||||||
}];
|
}];
|
||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"AffineDialect", "memref::MemRefDialect", "tensor::TensorDialect"
|
"affine::AffineDialect", "memref::MemRefDialect", "tensor::TensorDialect"
|
||||||
];
|
];
|
||||||
|
|
||||||
let extraClassDeclaration = [{
|
let extraClassDeclaration = [{
|
||||||
|
@ -37,7 +37,7 @@ def Linalg_Dialect : Dialect {
|
|||||||
let cppNamespace = "::mlir::linalg";
|
let cppNamespace = "::mlir::linalg";
|
||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"arith::ArithDialect",
|
"arith::ArithDialect",
|
||||||
"AffineDialect",
|
"affine::AffineDialect",
|
||||||
"math::MathDialect",
|
"math::MathDialect",
|
||||||
"memref::MemRefDialect",
|
"memref::MemRefDialect",
|
||||||
"tensor::TensorDialect",
|
"tensor::TensorDialect",
|
||||||
|
@ -37,7 +37,7 @@ def LinalgFoldUnitExtentDims : Pass<"linalg-fold-unit-extent-dims", ""> {
|
|||||||
"Generate rank-reducing slices instead of reassociative reshapes">
|
"Generate rank-reducing slices instead of reassociative reshapes">
|
||||||
];
|
];
|
||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"linalg::LinalgDialect", "AffineDialect", "memref::MemRefDialect"
|
"linalg::LinalgDialect", "affine::AffineDialect", "memref::MemRefDialect"
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -45,7 +45,7 @@ def LinalgElementwiseOpFusion : Pass<"linalg-fuse-elementwise-ops"> {
|
|||||||
let summary = "Fuse elementwise operations on tensors";
|
let summary = "Fuse elementwise operations on tensors";
|
||||||
let constructor = "mlir::createLinalgElementwiseOpFusionPass()";
|
let constructor = "mlir::createLinalgElementwiseOpFusionPass()";
|
||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"AffineDialect", "linalg::LinalgDialect", "memref::MemRefDialect"
|
"affine::AffineDialect", "linalg::LinalgDialect", "memref::MemRefDialect"
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -68,7 +68,7 @@ def LinalgLowerToAffineLoops : Pass<"convert-linalg-to-affine-loops", "func::Fun
|
|||||||
"loops";
|
"loops";
|
||||||
let constructor = "mlir::createConvertLinalgToAffineLoopsPass()";
|
let constructor = "mlir::createConvertLinalgToAffineLoopsPass()";
|
||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"AffineDialect", "linalg::LinalgDialect", "memref::MemRefDialect"];
|
"affine::AffineDialect", "linalg::LinalgDialect", "memref::MemRefDialect"];
|
||||||
}
|
}
|
||||||
|
|
||||||
def LinalgLowerToLoops : Pass<"convert-linalg-to-loops", "func::FuncOp"> {
|
def LinalgLowerToLoops : Pass<"convert-linalg-to-loops", "func::FuncOp"> {
|
||||||
@ -77,7 +77,7 @@ def LinalgLowerToLoops : Pass<"convert-linalg-to-loops", "func::FuncOp"> {
|
|||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"linalg::LinalgDialect",
|
"linalg::LinalgDialect",
|
||||||
"scf::SCFDialect",
|
"scf::SCFDialect",
|
||||||
"AffineDialect"
|
"affine::AffineDialect"
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -87,7 +87,7 @@ def LinalgLowerToParallelLoops
|
|||||||
"loops";
|
"loops";
|
||||||
let constructor = "mlir::createConvertLinalgToParallelLoopsPass()";
|
let constructor = "mlir::createConvertLinalgToParallelLoopsPass()";
|
||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"AffineDialect",
|
"affine::AffineDialect",
|
||||||
"linalg::LinalgDialect",
|
"linalg::LinalgDialect",
|
||||||
"memref::MemRefDialect",
|
"memref::MemRefDialect",
|
||||||
"scf::SCFDialect"
|
"scf::SCFDialect"
|
||||||
@ -98,7 +98,7 @@ def LinalgBufferize : Pass<"linalg-bufferize", "func::FuncOp"> {
|
|||||||
let summary = "Bufferize the linalg dialect";
|
let summary = "Bufferize the linalg dialect";
|
||||||
let constructor = "mlir::createLinalgBufferizePass()";
|
let constructor = "mlir::createLinalgBufferizePass()";
|
||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"AffineDialect",
|
"affine::AffineDialect",
|
||||||
"bufferization::BufferizationDialect",
|
"bufferization::BufferizationDialect",
|
||||||
"linalg::LinalgDialect",
|
"linalg::LinalgDialect",
|
||||||
"memref::MemRefDialect",
|
"memref::MemRefDialect",
|
||||||
|
@ -18,10 +18,13 @@
|
|||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
class AffineExpr;
|
class AffineExpr;
|
||||||
class AffineForOp;
|
|
||||||
class AffineMap;
|
class AffineMap;
|
||||||
class PatternRewriter;
|
class PatternRewriter;
|
||||||
|
|
||||||
|
namespace affine {
|
||||||
|
class AffineForOp;
|
||||||
|
} // namespace affine
|
||||||
|
|
||||||
namespace tensor {
|
namespace tensor {
|
||||||
class ExtractSliceOp;
|
class ExtractSliceOp;
|
||||||
} // namespace tensor
|
} // namespace tensor
|
||||||
|
@ -24,7 +24,7 @@ def FoldMemRefAliasOps : Pass<"fold-memref-alias-ops"> {
|
|||||||
}];
|
}];
|
||||||
let constructor = "mlir::memref::createFoldMemRefAliasOpsPass()";
|
let constructor = "mlir::memref::createFoldMemRefAliasOpsPass()";
|
||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"AffineDialect", "memref::MemRefDialect", "vector::VectorDialect"
|
"affine::AffineDialect", "memref::MemRefDialect", "vector::VectorDialect"
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -156,7 +156,7 @@ def NormalizeMemRefs : Pass<"normalize-memrefs", "ModuleOp"> {
|
|||||||
```
|
```
|
||||||
}];
|
}];
|
||||||
let constructor = "mlir::memref::createNormalizeMemRefsPass()";
|
let constructor = "mlir::memref::createNormalizeMemRefsPass()";
|
||||||
let dependentDialects = ["AffineDialect"];
|
let dependentDialects = ["affine::AffineDialect"];
|
||||||
}
|
}
|
||||||
|
|
||||||
def ResolveRankedShapeTypeResultDims :
|
def ResolveRankedShapeTypeResultDims :
|
||||||
@ -184,7 +184,7 @@ def ResolveShapedTypeResultDims : Pass<"resolve-shaped-type-result-dims"> {
|
|||||||
}];
|
}];
|
||||||
let constructor = "mlir::memref::createResolveShapedTypeResultDimsPass()";
|
let constructor = "mlir::memref::createResolveShapedTypeResultDimsPass()";
|
||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"AffineDialect", "memref::MemRefDialect", "tensor::TensorDialect"
|
"affine::AffineDialect", "memref::MemRefDialect", "tensor::TensorDialect"
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -199,7 +199,7 @@ def ExpandStridedMetadata : Pass<"expand-strided-metadata"> {
|
|||||||
}];
|
}];
|
||||||
let constructor = "mlir::memref::createExpandStridedMetadataPass()";
|
let constructor = "mlir::memref::createExpandStridedMetadataPass()";
|
||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"AffineDialect", "memref::MemRefDialect"
|
"affine::AffineDialect", "memref::MemRefDialect"
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
#endif // MLIR_DIALECT_MEMREF_TRANSFORMS_PASSES
|
#endif // MLIR_DIALECT_MEMREF_TRANSFORMS_PASSES
|
||||||
|
@ -24,7 +24,7 @@ def SCFForLoopCanonicalization
|
|||||||
: Pass<"scf-for-loop-canonicalization"> {
|
: Pass<"scf-for-loop-canonicalization"> {
|
||||||
let summary = "Canonicalize operations within scf.for loop bodies";
|
let summary = "Canonicalize operations within scf.for loop bodies";
|
||||||
let constructor = "mlir::createSCFForLoopCanonicalizationPass()";
|
let constructor = "mlir::createSCFForLoopCanonicalizationPass()";
|
||||||
let dependentDialects = ["AffineDialect", "tensor::TensorDialect",
|
let dependentDialects = ["affine::AffineDialect", "tensor::TensorDialect",
|
||||||
"memref::MemRefDialect"];
|
"memref::MemRefDialect"];
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -37,7 +37,7 @@ def SCFForLoopPeeling : Pass<"scf-for-loop-peeling"> {
|
|||||||
"Do not peel loops inside of the last, partial iteration of another "
|
"Do not peel loops inside of the last, partial iteration of another "
|
||||||
"already peeled loop.">
|
"already peeled loop.">
|
||||||
];
|
];
|
||||||
let dependentDialects = ["AffineDialect"];
|
let dependentDialects = ["affine::AffineDialect"];
|
||||||
}
|
}
|
||||||
|
|
||||||
def SCFForLoopSpecialization : Pass<"scf-for-loop-specialization"> {
|
def SCFForLoopSpecialization : Pass<"scf-for-loop-specialization"> {
|
||||||
@ -109,7 +109,7 @@ def SCFParallelLoopTiling : Pass<"scf-parallel-loop-tiling"> {
|
|||||||
"Perform tiling with fixed upper bound with inbound check "
|
"Perform tiling with fixed upper bound with inbound check "
|
||||||
"inside the internal loops">
|
"inside the internal loops">
|
||||||
];
|
];
|
||||||
let dependentDialects = ["AffineDialect"];
|
let dependentDialects = ["affine::AffineDialect"];
|
||||||
}
|
}
|
||||||
|
|
||||||
def SCFForLoopRangeFolding : Pass<"scf-for-loop-range-folding"> {
|
def SCFForLoopRangeFolding : Pass<"scf-for-loop-range-folding"> {
|
||||||
|
@ -18,9 +18,7 @@
|
|||||||
#include "mlir/Support/LogicalResult.h"
|
#include "mlir/Support/LogicalResult.h"
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
class AffineApplyOp;
|
|
||||||
class AffineMap;
|
class AffineMap;
|
||||||
class FlatAffineValueConstraints;
|
|
||||||
struct LogicalResult;
|
struct LogicalResult;
|
||||||
class Operation;
|
class Operation;
|
||||||
class OpFoldResult;
|
class OpFoldResult;
|
||||||
@ -28,6 +26,10 @@ class RewriterBase;
|
|||||||
class Value;
|
class Value;
|
||||||
class ValueRange;
|
class ValueRange;
|
||||||
|
|
||||||
|
namespace affine {
|
||||||
|
class FlatAffineValueConstraints;
|
||||||
|
} // namespace affine
|
||||||
|
|
||||||
namespace scf {
|
namespace scf {
|
||||||
class IfOp;
|
class IfOp;
|
||||||
|
|
||||||
@ -45,7 +47,7 @@ LogicalResult matchForLikeLoop(Value iv, OpFoldResult &lb, OpFoldResult &ub,
|
|||||||
|
|
||||||
/// Populate the given constraint set with induction variable constraints of a
|
/// Populate the given constraint set with induction variable constraints of a
|
||||||
/// "for" loop with the given range and step.
|
/// "for" loop with the given range and step.
|
||||||
LogicalResult addLoopRangeConstraints(FlatAffineValueConstraints &cstr,
|
LogicalResult addLoopRangeConstraints(affine::FlatAffineValueConstraints &cstr,
|
||||||
Value iv, OpFoldResult lb,
|
Value iv, OpFoldResult lb,
|
||||||
OpFoldResult ub, OpFoldResult step);
|
OpFoldResult ub, OpFoldResult step);
|
||||||
|
|
||||||
|
@ -70,7 +70,7 @@ def SparsificationPass : Pass<"sparsification", "ModuleOp"> {
|
|||||||
}];
|
}];
|
||||||
let constructor = "mlir::createSparsificationPass()";
|
let constructor = "mlir::createSparsificationPass()";
|
||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"AffineDialect",
|
"affine::AffineDialect",
|
||||||
"arith::ArithDialect",
|
"arith::ArithDialect",
|
||||||
"bufferization::BufferizationDialect",
|
"bufferization::BufferizationDialect",
|
||||||
"LLVM::LLVMDialect",
|
"LLVM::LLVMDialect",
|
||||||
|
@ -48,7 +48,7 @@ def Tensor_Dialect : Dialect {
|
|||||||
let hasCanonicalizer = 1;
|
let hasCanonicalizer = 1;
|
||||||
let hasConstantMaterializer = 1;
|
let hasConstantMaterializer = 1;
|
||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"AffineDialect",
|
"affine::AffineDialect",
|
||||||
"arith::ArithDialect",
|
"arith::ArithDialect",
|
||||||
"complex::ComplexDialect",
|
"complex::ComplexDialect",
|
||||||
];
|
];
|
||||||
|
@ -23,7 +23,7 @@ def FoldTensorSubsetOps : Pass<"fold-tensor-subset-ops"> {
|
|||||||
}];
|
}];
|
||||||
let constructor = "mlir::tensor::createFoldTensorSubsetOpsPass()";
|
let constructor = "mlir::tensor::createFoldTensorSubsetOpsPass()";
|
||||||
let dependentDialects = [
|
let dependentDialects = [
|
||||||
"AffineDialect", "tensor::TensorDialect", "vector::VectorDialect"
|
"affine::AffineDialect", "tensor::TensorDialect", "vector::VectorDialect"
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -18,8 +18,6 @@
|
|||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
|
||||||
// Forward declarations.
|
// Forward declarations.
|
||||||
class AffineApplyOp;
|
|
||||||
class AffineForOp;
|
|
||||||
class AffineMap;
|
class AffineMap;
|
||||||
class Block;
|
class Block;
|
||||||
class Location;
|
class Location;
|
||||||
@ -30,6 +28,11 @@ class Value;
|
|||||||
class VectorType;
|
class VectorType;
|
||||||
class VectorTransferOpInterface;
|
class VectorTransferOpInterface;
|
||||||
|
|
||||||
|
namespace affine {
|
||||||
|
class AffineApplyOp;
|
||||||
|
class AffineForOp;
|
||||||
|
} // namespace affine
|
||||||
|
|
||||||
namespace vector {
|
namespace vector {
|
||||||
/// Helper function that creates a memref::DimOp or tensor::DimOp depending on
|
/// Helper function that creates a memref::DimOp or tensor::DimOp depending on
|
||||||
/// the type of `source`.
|
/// the type of `source`.
|
||||||
|
@ -86,7 +86,7 @@ namespace mlir {
|
|||||||
inline void registerAllDialects(DialectRegistry ®istry) {
|
inline void registerAllDialects(DialectRegistry ®istry) {
|
||||||
// clang-format off
|
// clang-format off
|
||||||
registry.insert<acc::OpenACCDialect,
|
registry.insert<acc::OpenACCDialect,
|
||||||
AffineDialect,
|
affine::AffineDialect,
|
||||||
arith::ArithDialect,
|
arith::ArithDialect,
|
||||||
amdgpu::AMDGPUDialect,
|
amdgpu::AMDGPUDialect,
|
||||||
amx::AMXDialect,
|
amx::AMXDialect,
|
||||||
|
@ -55,7 +55,7 @@ inline void registerAllPasses() {
|
|||||||
registerConversionPasses();
|
registerConversionPasses();
|
||||||
|
|
||||||
// Dialect passes
|
// Dialect passes
|
||||||
registerAffinePasses();
|
affine::registerAffinePasses();
|
||||||
registerAsyncPasses();
|
registerAsyncPasses();
|
||||||
arith::registerArithPasses();
|
arith::registerArithPasses();
|
||||||
bufferization::registerBufferizationPasses();
|
bufferization::registerBufferizationPasses();
|
||||||
|
@ -30,6 +30,7 @@ namespace mlir {
|
|||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
using namespace mlir::vector;
|
using namespace mlir::vector;
|
||||||
|
|
||||||
/// Given a range of values, emit the code that reduces them with "min" or "max"
|
/// Given a range of values, emit the code that reduces them with "min" or "max"
|
||||||
|
@ -141,8 +141,9 @@ struct ConvertLinalgToStandardPass
|
|||||||
void ConvertLinalgToStandardPass::runOnOperation() {
|
void ConvertLinalgToStandardPass::runOnOperation() {
|
||||||
auto module = getOperation();
|
auto module = getOperation();
|
||||||
ConversionTarget target(getContext());
|
ConversionTarget target(getContext());
|
||||||
target.addLegalDialect<AffineDialect, arith::ArithDialect, func::FuncDialect,
|
target.addLegalDialect<affine::AffineDialect, arith::ArithDialect,
|
||||||
memref::MemRefDialect, scf::SCFDialect>();
|
func::FuncDialect, memref::MemRefDialect,
|
||||||
|
scf::SCFDialect>();
|
||||||
target.addLegalOp<ModuleOp, func::FuncOp, func::ReturnOp>();
|
target.addLegalOp<ModuleOp, func::FuncOp, func::ReturnOp>();
|
||||||
RewritePatternSet patterns(&getContext());
|
RewritePatternSet patterns(&getContext());
|
||||||
populateLinalgToStandardConversionPatterns(patterns);
|
populateLinalgToStandardConversionPatterns(patterns);
|
||||||
|
@ -36,6 +36,7 @@
|
|||||||
#define DEBUG_TYPE "loops-to-gpu"
|
#define DEBUG_TYPE "loops-to-gpu"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
using namespace mlir::scf;
|
using namespace mlir::scf;
|
||||||
|
|
||||||
// Name of internal attribute to mark visited operations during conversion.
|
// Name of internal attribute to mark visited operations during conversion.
|
||||||
|
@ -42,7 +42,7 @@ struct ForLoopMapper : public impl::ConvertAffineForToGPUBase<ForLoopMapper> {
|
|||||||
void runOnOperation() override {
|
void runOnOperation() override {
|
||||||
for (Operation &op : llvm::make_early_inc_range(
|
for (Operation &op : llvm::make_early_inc_range(
|
||||||
getOperation().getFunctionBody().getOps())) {
|
getOperation().getFunctionBody().getOps())) {
|
||||||
if (auto forOp = dyn_cast<AffineForOp>(&op)) {
|
if (auto forOp = dyn_cast<affine::AffineForOp>(&op)) {
|
||||||
if (failed(convertAffineLoopNestToGPULaunch(forOp, numBlockDims,
|
if (failed(convertAffineLoopNestToGPULaunch(forOp, numBlockDims,
|
||||||
numThreadDims)))
|
numThreadDims)))
|
||||||
signalPassFailure();
|
signalPassFailure();
|
||||||
|
@ -66,7 +66,7 @@ static void getXferIndices(RewriterBase &rewriter, TransferOpType xferOp,
|
|||||||
SmallVector<Value, 3> dims(dimValues.begin(), dimValues.end());
|
SmallVector<Value, 3> dims(dimValues.begin(), dimValues.end());
|
||||||
dims.push_back(prevIdx);
|
dims.push_back(prevIdx);
|
||||||
AffineExpr d0 = rewriter.getAffineDimExpr(offsetMap.getNumDims());
|
AffineExpr d0 = rewriter.getAffineDimExpr(offsetMap.getNumDims());
|
||||||
indices[dim.getPosition()] = makeComposedAffineApply(
|
indices[dim.getPosition()] = affine::makeComposedAffineApply(
|
||||||
rewriter, loc, d0 + offsetMap.getResult(offsetsIdx++), dims);
|
rewriter, loc, d0 + offsetMap.getResult(offsetsIdx++), dims);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
@ -102,7 +102,8 @@ static void getXferIndices(OpBuilder &b, OpTy xferOp, Value iv,
|
|||||||
AffineExpr d0, d1;
|
AffineExpr d0, d1;
|
||||||
bindDims(xferOp.getContext(), d0, d1);
|
bindDims(xferOp.getContext(), d0, d1);
|
||||||
Value offset = adaptor.getIndices()[*dim];
|
Value offset = adaptor.getIndices()[*dim];
|
||||||
indices[*dim] = makeComposedAffineApply(b, loc, d0 + d1, {offset, iv});
|
indices[*dim] =
|
||||||
|
affine::makeComposedAffineApply(b, loc, d0 + d1, {offset, iv});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -178,7 +179,8 @@ static Value generateInBoundsCheck(
|
|||||||
AffineExpr d0, d1;
|
AffineExpr d0, d1;
|
||||||
bindDims(xferOp.getContext(), d0, d1);
|
bindDims(xferOp.getContext(), d0, d1);
|
||||||
Value base = xferOp.getIndices()[*dim];
|
Value base = xferOp.getIndices()[*dim];
|
||||||
Value memrefIdx = makeComposedAffineApply(b, loc, d0 + d1, {base, iv});
|
Value memrefIdx =
|
||||||
|
affine::makeComposedAffineApply(b, loc, d0 + d1, {base, iv});
|
||||||
cond = lb.create<arith::CmpIOp>(arith::CmpIPredicate::sgt, memrefDim,
|
cond = lb.create<arith::CmpIOp>(arith::CmpIPredicate::sgt, memrefDim,
|
||||||
memrefIdx);
|
memrefIdx);
|
||||||
}
|
}
|
||||||
@ -1111,7 +1113,8 @@ get1dMemrefIndices(OpBuilder &b, OpTy xferOp, Value iv,
|
|||||||
AffineExpr d0, d1;
|
AffineExpr d0, d1;
|
||||||
bindDims(xferOp.getContext(), d0, d1);
|
bindDims(xferOp.getContext(), d0, d1);
|
||||||
Value offset = memrefIndices[dim];
|
Value offset = memrefIndices[dim];
|
||||||
memrefIndices[dim] = makeComposedAffineApply(b, loc, d0 + d1, {offset, iv});
|
memrefIndices[dim] =
|
||||||
|
affine::makeComposedAffineApply(b, loc, d0 + d1, {offset, iv});
|
||||||
return dim;
|
return dim;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -31,6 +31,7 @@
|
|||||||
#define DEBUG_TYPE "affine-analysis"
|
#define DEBUG_TYPE "affine-analysis"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace affine;
|
||||||
using namespace presburger;
|
using namespace presburger;
|
||||||
|
|
||||||
/// Get the value that is being reduced by `pos`-th reduction in the loop if
|
/// Get the value that is being reduced by `pos`-th reduction in the loop if
|
||||||
@ -78,7 +79,7 @@ static Value getSupportedReduction(AffineForOp forOp, unsigned pos,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Populate `supportedReductions` with descriptors of the supported reductions.
|
/// Populate `supportedReductions` with descriptors of the supported reductions.
|
||||||
void mlir::getSupportedReductions(
|
void mlir::affine::getSupportedReductions(
|
||||||
AffineForOp forOp, SmallVectorImpl<LoopReduction> &supportedReductions) {
|
AffineForOp forOp, SmallVectorImpl<LoopReduction> &supportedReductions) {
|
||||||
unsigned numIterArgs = forOp.getNumIterOperands();
|
unsigned numIterArgs = forOp.getNumIterOperands();
|
||||||
if (numIterArgs == 0)
|
if (numIterArgs == 0)
|
||||||
@ -94,8 +95,8 @@ void mlir::getSupportedReductions(
|
|||||||
/// Returns true if `forOp' is a parallel loop. If `parallelReductions` is
|
/// Returns true if `forOp' is a parallel loop. If `parallelReductions` is
|
||||||
/// provided, populates it with descriptors of the parallelizable reductions and
|
/// provided, populates it with descriptors of the parallelizable reductions and
|
||||||
/// treats them as not preventing parallelization.
|
/// treats them as not preventing parallelization.
|
||||||
bool mlir::isLoopParallel(AffineForOp forOp,
|
bool mlir::affine::isLoopParallel(
|
||||||
SmallVectorImpl<LoopReduction> *parallelReductions) {
|
AffineForOp forOp, SmallVectorImpl<LoopReduction> *parallelReductions) {
|
||||||
unsigned numIterArgs = forOp.getNumIterOperands();
|
unsigned numIterArgs = forOp.getNumIterOperands();
|
||||||
|
|
||||||
// Loop is not parallel if it has SSA loop-carried dependences and reduction
|
// Loop is not parallel if it has SSA loop-carried dependences and reduction
|
||||||
@ -132,7 +133,7 @@ static bool isLocallyDefined(Value v, Operation *enclosingOp) {
|
|||||||
return viewOp && isLocallyDefined(viewOp.getViewSource(), enclosingOp);
|
return viewOp && isLocallyDefined(viewOp.getViewSource(), enclosingOp);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool mlir::isLoopMemoryParallel(AffineForOp forOp) {
|
bool mlir::affine::isLoopMemoryParallel(AffineForOp forOp) {
|
||||||
// Any memref-typed iteration arguments are treated as serializing.
|
// Any memref-typed iteration arguments are treated as serializing.
|
||||||
if (llvm::any_of(forOp.getResultTypes(),
|
if (llvm::any_of(forOp.getResultTypes(),
|
||||||
[](Type type) { return type.isa<BaseMemRefType>(); }))
|
[](Type type) { return type.isa<BaseMemRefType>(); }))
|
||||||
@ -186,7 +187,7 @@ bool mlir::isLoopMemoryParallel(AffineForOp forOp) {
|
|||||||
/// and ending at operands which are not defined by AffineApplyOps.
|
/// and ending at operands which are not defined by AffineApplyOps.
|
||||||
// TODO: Add a method to AffineApplyOp which forward substitutes the
|
// TODO: Add a method to AffineApplyOp which forward substitutes the
|
||||||
// AffineApplyOp into any user AffineApplyOps.
|
// AffineApplyOp into any user AffineApplyOps.
|
||||||
void mlir::getReachableAffineApplyOps(
|
void mlir::affine::getReachableAffineApplyOps(
|
||||||
ArrayRef<Value> operands, SmallVectorImpl<Operation *> &affineApplyOps) {
|
ArrayRef<Value> operands, SmallVectorImpl<Operation *> &affineApplyOps) {
|
||||||
struct State {
|
struct State {
|
||||||
// The ssa value for this node in the DFS traversal.
|
// The ssa value for this node in the DFS traversal.
|
||||||
@ -236,8 +237,8 @@ void mlir::getReachableAffineApplyOps(
|
|||||||
// FlatAffineValueConstraints. (For eg., by using iv - lb % step = 0 and/or by
|
// FlatAffineValueConstraints. (For eg., by using iv - lb % step = 0 and/or by
|
||||||
// introducing a method in FlatAffineValueConstraints
|
// introducing a method in FlatAffineValueConstraints
|
||||||
// setExprStride(ArrayRef<int64_t> expr, int64_t stride)
|
// setExprStride(ArrayRef<int64_t> expr, int64_t stride)
|
||||||
LogicalResult mlir::getIndexSet(MutableArrayRef<Operation *> ops,
|
LogicalResult mlir::affine::getIndexSet(MutableArrayRef<Operation *> ops,
|
||||||
FlatAffineValueConstraints *domain) {
|
FlatAffineValueConstraints *domain) {
|
||||||
SmallVector<Value, 4> indices;
|
SmallVector<Value, 4> indices;
|
||||||
SmallVector<Operation *, 8> loopOps;
|
SmallVector<Operation *, 8> loopOps;
|
||||||
size_t numDims = 0;
|
size_t numDims = 0;
|
||||||
@ -594,7 +595,7 @@ void MemRefAccess::getAccessMap(AffineValueMap *accessMap) const {
|
|||||||
//
|
//
|
||||||
//
|
//
|
||||||
// TODO: Support AffineExprs mod/floordiv/ceildiv.
|
// TODO: Support AffineExprs mod/floordiv/ceildiv.
|
||||||
DependenceResult mlir::checkMemrefAccessDependence(
|
DependenceResult mlir::affine::checkMemrefAccessDependence(
|
||||||
const MemRefAccess &srcAccess, const MemRefAccess &dstAccess,
|
const MemRefAccess &srcAccess, const MemRefAccess &dstAccess,
|
||||||
unsigned loopDepth, FlatAffineValueConstraints *dependenceConstraints,
|
unsigned loopDepth, FlatAffineValueConstraints *dependenceConstraints,
|
||||||
SmallVector<DependenceComponent, 2> *dependenceComponents, bool allowRAR) {
|
SmallVector<DependenceComponent, 2> *dependenceComponents, bool allowRAR) {
|
||||||
@ -671,7 +672,7 @@ DependenceResult mlir::checkMemrefAccessDependence(
|
|||||||
|
|
||||||
/// Gathers dependence components for dependences between all ops in loop nest
|
/// Gathers dependence components for dependences between all ops in loop nest
|
||||||
/// rooted at 'forOp' at loop depths in range [1, maxLoopDepth].
|
/// rooted at 'forOp' at loop depths in range [1, maxLoopDepth].
|
||||||
void mlir::getDependenceComponents(
|
void mlir::affine::getDependenceComponents(
|
||||||
AffineForOp forOp, unsigned maxLoopDepth,
|
AffineForOp forOp, unsigned maxLoopDepth,
|
||||||
std::vector<SmallVector<DependenceComponent, 2>> *depCompsVec) {
|
std::vector<SmallVector<DependenceComponent, 2>> *depCompsVec) {
|
||||||
// Collect all load and store ops in loop nest rooted at 'forOp'.
|
// Collect all load and store ops in loop nest rooted at 'forOp'.
|
||||||
|
@ -31,6 +31,7 @@
|
|||||||
#define DEBUG_TYPE "affine-structures"
|
#define DEBUG_TYPE "affine-structures"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace affine;
|
||||||
using namespace presburger;
|
using namespace presburger;
|
||||||
|
|
||||||
|
|
||||||
@ -489,8 +490,8 @@ void FlatAffineRelation::removeVarRange(VarKind kind, unsigned varStart,
|
|||||||
numRangeDims -= intersectRangeLHS - intersectRangeRHS;
|
numRangeDims -= intersectRangeLHS - intersectRangeRHS;
|
||||||
}
|
}
|
||||||
|
|
||||||
LogicalResult mlir::getRelationFromMap(AffineMap &map,
|
LogicalResult mlir::affine::getRelationFromMap(AffineMap &map,
|
||||||
FlatAffineRelation &rel) {
|
FlatAffineRelation &rel) {
|
||||||
// Get flattened affine expressions.
|
// Get flattened affine expressions.
|
||||||
std::vector<SmallVector<int64_t, 8>> flatExprs;
|
std::vector<SmallVector<int64_t, 8>> flatExprs;
|
||||||
FlatAffineValueConstraints localVarCst;
|
FlatAffineValueConstraints localVarCst;
|
||||||
@ -525,8 +526,8 @@ LogicalResult mlir::getRelationFromMap(AffineMap &map,
|
|||||||
return success();
|
return success();
|
||||||
}
|
}
|
||||||
|
|
||||||
LogicalResult mlir::getRelationFromMap(const AffineValueMap &map,
|
LogicalResult mlir::affine::getRelationFromMap(const AffineValueMap &map,
|
||||||
FlatAffineRelation &rel) {
|
FlatAffineRelation &rel) {
|
||||||
|
|
||||||
AffineMap affineMap = map.getAffineMap();
|
AffineMap affineMap = map.getAffineMap();
|
||||||
if (failed(getRelationFromMap(affineMap, rel)))
|
if (failed(getRelationFromMap(affineMap, rel)))
|
||||||
|
@ -28,13 +28,14 @@
|
|||||||
#include <type_traits>
|
#include <type_traits>
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
/// Returns the trip count of the loop as an affine expression if the latter is
|
/// Returns the trip count of the loop as an affine expression if the latter is
|
||||||
/// expressible as an affine expression, and nullptr otherwise. The trip count
|
/// expressible as an affine expression, and nullptr otherwise. The trip count
|
||||||
/// expression is simplified before returning. This method only utilizes map
|
/// expression is simplified before returning. This method only utilizes map
|
||||||
/// composition to construct lower and upper bounds before computing the trip
|
/// composition to construct lower and upper bounds before computing the trip
|
||||||
/// count expressions.
|
/// count expressions.
|
||||||
void mlir::getTripCountMapAndOperands(
|
void mlir::affine::getTripCountMapAndOperands(
|
||||||
AffineForOp forOp, AffineMap *tripCountMap,
|
AffineForOp forOp, AffineMap *tripCountMap,
|
||||||
SmallVectorImpl<Value> *tripCountOperands) {
|
SmallVectorImpl<Value> *tripCountOperands) {
|
||||||
MLIRContext *context = forOp.getContext();
|
MLIRContext *context = forOp.getContext();
|
||||||
@ -83,7 +84,7 @@ void mlir::getTripCountMapAndOperands(
|
|||||||
/// otherwise. This method uses affine expression analysis (in turn using
|
/// otherwise. This method uses affine expression analysis (in turn using
|
||||||
/// getTripCount) and is able to determine constant trip count in non-trivial
|
/// getTripCount) and is able to determine constant trip count in non-trivial
|
||||||
/// cases.
|
/// cases.
|
||||||
std::optional<uint64_t> mlir::getConstantTripCount(AffineForOp forOp) {
|
std::optional<uint64_t> mlir::affine::getConstantTripCount(AffineForOp forOp) {
|
||||||
SmallVector<Value, 4> operands;
|
SmallVector<Value, 4> operands;
|
||||||
AffineMap map;
|
AffineMap map;
|
||||||
getTripCountMapAndOperands(forOp, &map, &operands);
|
getTripCountMapAndOperands(forOp, &map, &operands);
|
||||||
@ -109,7 +110,7 @@ std::optional<uint64_t> mlir::getConstantTripCount(AffineForOp forOp) {
|
|||||||
/// Returns the greatest known integral divisor of the trip count. Affine
|
/// Returns the greatest known integral divisor of the trip count. Affine
|
||||||
/// expression analysis is used (indirectly through getTripCount), and
|
/// expression analysis is used (indirectly through getTripCount), and
|
||||||
/// this method is thus able to determine non-trivial divisors.
|
/// this method is thus able to determine non-trivial divisors.
|
||||||
uint64_t mlir::getLargestDivisorOfTripCount(AffineForOp forOp) {
|
uint64_t mlir::affine::getLargestDivisorOfTripCount(AffineForOp forOp) {
|
||||||
SmallVector<Value, 4> operands;
|
SmallVector<Value, 4> operands;
|
||||||
AffineMap map;
|
AffineMap map;
|
||||||
getTripCountMapAndOperands(forOp, &map, &operands);
|
getTripCountMapAndOperands(forOp, &map, &operands);
|
||||||
@ -183,7 +184,8 @@ static bool isAccessIndexInvariant(Value iv, Value index) {
|
|||||||
return !composeOp.getAffineValueMap().isFunctionOf(0, iv);
|
return !composeOp.getAffineValueMap().isFunctionOf(0, iv);
|
||||||
}
|
}
|
||||||
|
|
||||||
DenseSet<Value> mlir::getInvariantAccesses(Value iv, ArrayRef<Value> indices) {
|
DenseSet<Value> mlir::affine::getInvariantAccesses(Value iv,
|
||||||
|
ArrayRef<Value> indices) {
|
||||||
DenseSet<Value> res;
|
DenseSet<Value> res;
|
||||||
for (auto val : indices) {
|
for (auto val : indices) {
|
||||||
if (isAccessIndexInvariant(iv, val)) {
|
if (isAccessIndexInvariant(iv, val)) {
|
||||||
@ -335,8 +337,8 @@ isVectorizableLoopBodyWithOpCond(AffineForOp loop,
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool mlir::isVectorizableLoopBody(AffineForOp loop, int *memRefDim,
|
bool mlir::affine::isVectorizableLoopBody(
|
||||||
NestedPattern &vectorTransferMatcher) {
|
AffineForOp loop, int *memRefDim, NestedPattern &vectorTransferMatcher) {
|
||||||
*memRefDim = -1;
|
*memRefDim = -1;
|
||||||
VectorizableOpFun fun([memRefDim](AffineForOp loop, Operation &op) {
|
VectorizableOpFun fun([memRefDim](AffineForOp loop, Operation &op) {
|
||||||
auto load = dyn_cast<AffineLoadOp>(op);
|
auto load = dyn_cast<AffineLoadOp>(op);
|
||||||
@ -358,8 +360,8 @@ bool mlir::isVectorizableLoopBody(AffineForOp loop, int *memRefDim,
|
|||||||
return isVectorizableLoopBodyWithOpCond(loop, fun, vectorTransferMatcher);
|
return isVectorizableLoopBodyWithOpCond(loop, fun, vectorTransferMatcher);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool mlir::isVectorizableLoopBody(AffineForOp loop,
|
bool mlir::affine::isVectorizableLoopBody(
|
||||||
NestedPattern &vectorTransferMatcher) {
|
AffineForOp loop, NestedPattern &vectorTransferMatcher) {
|
||||||
return isVectorizableLoopBodyWithOpCond(loop, nullptr, vectorTransferMatcher);
|
return isVectorizableLoopBodyWithOpCond(loop, nullptr, vectorTransferMatcher);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -368,7 +370,8 @@ bool mlir::isVectorizableLoopBody(AffineForOp loop,
|
|||||||
/// 'def' and all its uses have the same shift factor.
|
/// 'def' and all its uses have the same shift factor.
|
||||||
// TODO: extend this to check for memory-based dependence violation when we have
|
// TODO: extend this to check for memory-based dependence violation when we have
|
||||||
// the support.
|
// the support.
|
||||||
bool mlir::isOpwiseShiftValid(AffineForOp forOp, ArrayRef<uint64_t> shifts) {
|
bool mlir::affine::isOpwiseShiftValid(AffineForOp forOp,
|
||||||
|
ArrayRef<uint64_t> shifts) {
|
||||||
auto *forBody = forOp.getBody();
|
auto *forBody = forOp.getBody();
|
||||||
assert(shifts.size() == forBody->getOperations().size());
|
assert(shifts.size() == forBody->getOperations().size());
|
||||||
|
|
||||||
|
@ -17,6 +17,7 @@
|
|||||||
#include "llvm/Support/raw_ostream.h"
|
#include "llvm/Support/raw_ostream.h"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
llvm::BumpPtrAllocator *&NestedMatch::allocator() {
|
llvm::BumpPtrAllocator *&NestedMatch::allocator() {
|
||||||
thread_local llvm::BumpPtrAllocator *allocator = nullptr;
|
thread_local llvm::BumpPtrAllocator *allocator = nullptr;
|
||||||
@ -130,6 +131,7 @@ static bool isAffineForOp(Operation &op) { return isa<AffineForOp>(op); }
|
|||||||
static bool isAffineIfOp(Operation &op) { return isa<AffineIfOp>(op); }
|
static bool isAffineIfOp(Operation &op) { return isa<AffineIfOp>(op); }
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
namespace matcher {
|
namespace matcher {
|
||||||
|
|
||||||
NestedPattern Op(FilterFunctionType filter) {
|
NestedPattern Op(FilterFunctionType filter) {
|
||||||
@ -176,4 +178,5 @@ bool isLoadOrStore(Operation &op) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
} // namespace matcher
|
} // namespace matcher
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
@ -28,6 +28,7 @@
|
|||||||
#define DEBUG_TYPE "analysis-utils"
|
#define DEBUG_TYPE "analysis-utils"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace affine;
|
||||||
using namespace presburger;
|
using namespace presburger;
|
||||||
|
|
||||||
using llvm::SmallDenseMap;
|
using llvm::SmallDenseMap;
|
||||||
@ -501,7 +502,8 @@ void MemRefDependenceGraph::print(raw_ostream &os) const {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void mlir::getAffineForIVs(Operation &op, SmallVectorImpl<AffineForOp> *loops) {
|
void mlir::affine::getAffineForIVs(Operation &op,
|
||||||
|
SmallVectorImpl<AffineForOp> *loops) {
|
||||||
auto *currOp = op.getParentOp();
|
auto *currOp = op.getParentOp();
|
||||||
AffineForOp currAffineForOp;
|
AffineForOp currAffineForOp;
|
||||||
// Traverse up the hierarchy collecting all 'affine.for' operation while
|
// Traverse up the hierarchy collecting all 'affine.for' operation while
|
||||||
@ -514,8 +516,8 @@ void mlir::getAffineForIVs(Operation &op, SmallVectorImpl<AffineForOp> *loops) {
|
|||||||
std::reverse(loops->begin(), loops->end());
|
std::reverse(loops->begin(), loops->end());
|
||||||
}
|
}
|
||||||
|
|
||||||
void mlir::getEnclosingAffineOps(Operation &op,
|
void mlir::affine::getEnclosingAffineOps(Operation &op,
|
||||||
SmallVectorImpl<Operation *> *ops) {
|
SmallVectorImpl<Operation *> *ops) {
|
||||||
ops->clear();
|
ops->clear();
|
||||||
Operation *currOp = op.getParentOp();
|
Operation *currOp = op.getParentOp();
|
||||||
|
|
||||||
@ -1063,7 +1065,7 @@ LogicalResult MemRefRegion::compute(Operation *op, unsigned loopDepth,
|
|||||||
}
|
}
|
||||||
|
|
||||||
std::optional<int64_t>
|
std::optional<int64_t>
|
||||||
mlir::getMemRefIntOrFloatEltSizeInBytes(MemRefType memRefType) {
|
mlir::affine::getMemRefIntOrFloatEltSizeInBytes(MemRefType memRefType) {
|
||||||
auto elementType = memRefType.getElementType();
|
auto elementType = memRefType.getElementType();
|
||||||
|
|
||||||
unsigned sizeInBits;
|
unsigned sizeInBits;
|
||||||
@ -1113,7 +1115,7 @@ std::optional<int64_t> MemRefRegion::getRegionSize() {
|
|||||||
/// into account size of the vector as well.
|
/// into account size of the vector as well.
|
||||||
// TODO: improve/complete this when we have target data.
|
// TODO: improve/complete this when we have target data.
|
||||||
std::optional<uint64_t>
|
std::optional<uint64_t>
|
||||||
mlir::getIntOrFloatMemRefSizeInBytes(MemRefType memRefType) {
|
mlir::affine::getIntOrFloatMemRefSizeInBytes(MemRefType memRefType) {
|
||||||
if (!memRefType.hasStaticShape())
|
if (!memRefType.hasStaticShape())
|
||||||
return std::nullopt;
|
return std::nullopt;
|
||||||
auto elementType = memRefType.getElementType();
|
auto elementType = memRefType.getElementType();
|
||||||
@ -1130,8 +1132,8 @@ mlir::getIntOrFloatMemRefSizeInBytes(MemRefType memRefType) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
template <typename LoadOrStoreOp>
|
template <typename LoadOrStoreOp>
|
||||||
LogicalResult mlir::boundCheckLoadOrStoreOp(LoadOrStoreOp loadOrStoreOp,
|
LogicalResult mlir::affine::boundCheckLoadOrStoreOp(LoadOrStoreOp loadOrStoreOp,
|
||||||
bool emitError) {
|
bool emitError) {
|
||||||
static_assert(llvm::is_one_of<LoadOrStoreOp, AffineReadOpInterface,
|
static_assert(llvm::is_one_of<LoadOrStoreOp, AffineReadOpInterface,
|
||||||
AffineWriteOpInterface>::value,
|
AffineWriteOpInterface>::value,
|
||||||
"argument should be either a AffineReadOpInterface or a "
|
"argument should be either a AffineReadOpInterface or a "
|
||||||
@ -1186,9 +1188,11 @@ LogicalResult mlir::boundCheckLoadOrStoreOp(LoadOrStoreOp loadOrStoreOp,
|
|||||||
|
|
||||||
// Explicitly instantiate the template so that the compiler knows we need them!
|
// Explicitly instantiate the template so that the compiler knows we need them!
|
||||||
template LogicalResult
|
template LogicalResult
|
||||||
mlir::boundCheckLoadOrStoreOp(AffineReadOpInterface loadOp, bool emitError);
|
mlir::affine::boundCheckLoadOrStoreOp(AffineReadOpInterface loadOp,
|
||||||
|
bool emitError);
|
||||||
template LogicalResult
|
template LogicalResult
|
||||||
mlir::boundCheckLoadOrStoreOp(AffineWriteOpInterface storeOp, bool emitError);
|
mlir::affine::boundCheckLoadOrStoreOp(AffineWriteOpInterface storeOp,
|
||||||
|
bool emitError);
|
||||||
|
|
||||||
// Returns in 'positions' the Block positions of 'op' in each ancestor
|
// Returns in 'positions' the Block positions of 'op' in each ancestor
|
||||||
// Block from the Block containing operation, stopping at 'limitBlock'.
|
// Block from the Block containing operation, stopping at 'limitBlock'.
|
||||||
@ -1250,7 +1254,7 @@ static LogicalResult addMissingLoopIVBounds(SmallPtrSet<Value, 8> &ivs,
|
|||||||
|
|
||||||
/// Returns the innermost common loop depth for the set of operations in 'ops'.
|
/// Returns the innermost common loop depth for the set of operations in 'ops'.
|
||||||
// TODO: Move this to LoopUtils.
|
// TODO: Move this to LoopUtils.
|
||||||
unsigned mlir::getInnermostCommonLoopDepth(
|
unsigned mlir::affine::getInnermostCommonLoopDepth(
|
||||||
ArrayRef<Operation *> ops, SmallVectorImpl<AffineForOp> *surroundingLoops) {
|
ArrayRef<Operation *> ops, SmallVectorImpl<AffineForOp> *surroundingLoops) {
|
||||||
unsigned numOps = ops.size();
|
unsigned numOps = ops.size();
|
||||||
assert(numOps > 0 && "Expected at least one operation");
|
assert(numOps > 0 && "Expected at least one operation");
|
||||||
@ -1282,10 +1286,10 @@ unsigned mlir::getInnermostCommonLoopDepth(
|
|||||||
/// then verifies if it is valid. Returns 'SliceComputationResult::Success' if
|
/// then verifies if it is valid. Returns 'SliceComputationResult::Success' if
|
||||||
/// union was computed correctly, an appropriate failure otherwise.
|
/// union was computed correctly, an appropriate failure otherwise.
|
||||||
SliceComputationResult
|
SliceComputationResult
|
||||||
mlir::computeSliceUnion(ArrayRef<Operation *> opsA, ArrayRef<Operation *> opsB,
|
mlir::affine::computeSliceUnion(ArrayRef<Operation *> opsA,
|
||||||
unsigned loopDepth, unsigned numCommonLoops,
|
ArrayRef<Operation *> opsB, unsigned loopDepth,
|
||||||
bool isBackwardSlice,
|
unsigned numCommonLoops, bool isBackwardSlice,
|
||||||
ComputationSliceState *sliceUnion) {
|
ComputationSliceState *sliceUnion) {
|
||||||
// Compute the union of slice bounds between all pairs in 'opsA' and
|
// Compute the union of slice bounds between all pairs in 'opsA' and
|
||||||
// 'opsB' in 'sliceUnionCst'.
|
// 'opsB' in 'sliceUnionCst'.
|
||||||
FlatAffineValueConstraints sliceUnionCst;
|
FlatAffineValueConstraints sliceUnionCst;
|
||||||
@ -1322,8 +1326,9 @@ mlir::computeSliceUnion(ArrayRef<Operation *> opsA, ArrayRef<Operation *> opsB,
|
|||||||
|
|
||||||
// Compute slice bounds for 'srcAccess' and 'dstAccess'.
|
// Compute slice bounds for 'srcAccess' and 'dstAccess'.
|
||||||
ComputationSliceState tmpSliceState;
|
ComputationSliceState tmpSliceState;
|
||||||
mlir::getComputationSliceState(i, j, &dependenceConstraints, loopDepth,
|
mlir::affine::getComputationSliceState(i, j, &dependenceConstraints,
|
||||||
isBackwardSlice, &tmpSliceState);
|
loopDepth, isBackwardSlice,
|
||||||
|
&tmpSliceState);
|
||||||
|
|
||||||
if (sliceUnionCst.getNumDimAndSymbolVars() == 0) {
|
if (sliceUnionCst.getNumDimAndSymbolVars() == 0) {
|
||||||
// Initialize 'sliceUnionCst' with the bounds computed in previous step.
|
// Initialize 'sliceUnionCst' with the bounds computed in previous step.
|
||||||
@ -1465,7 +1470,7 @@ static std::optional<uint64_t> getConstDifference(AffineMap lbMap,
|
|||||||
// nest surrounding represented by slice loop bounds in 'slice'. Returns true
|
// nest surrounding represented by slice loop bounds in 'slice'. Returns true
|
||||||
// on success, false otherwise (if a non-constant trip count was encountered).
|
// on success, false otherwise (if a non-constant trip count was encountered).
|
||||||
// TODO: Make this work with non-unit step loops.
|
// TODO: Make this work with non-unit step loops.
|
||||||
bool mlir::buildSliceTripCountMap(
|
bool mlir::affine::buildSliceTripCountMap(
|
||||||
const ComputationSliceState &slice,
|
const ComputationSliceState &slice,
|
||||||
llvm::SmallDenseMap<Operation *, uint64_t, 8> *tripCountMap) {
|
llvm::SmallDenseMap<Operation *, uint64_t, 8> *tripCountMap) {
|
||||||
unsigned numSrcLoopIVs = slice.ivs.size();
|
unsigned numSrcLoopIVs = slice.ivs.size();
|
||||||
@ -1503,7 +1508,7 @@ bool mlir::buildSliceTripCountMap(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Return the number of iterations in the given slice.
|
// Return the number of iterations in the given slice.
|
||||||
uint64_t mlir::getSliceIterationCount(
|
uint64_t mlir::affine::getSliceIterationCount(
|
||||||
const llvm::SmallDenseMap<Operation *, uint64_t, 8> &sliceTripCountMap) {
|
const llvm::SmallDenseMap<Operation *, uint64_t, 8> &sliceTripCountMap) {
|
||||||
uint64_t iterCount = 1;
|
uint64_t iterCount = 1;
|
||||||
for (const auto &count : sliceTripCountMap) {
|
for (const auto &count : sliceTripCountMap) {
|
||||||
@ -1517,7 +1522,7 @@ const char *const kSliceFusionBarrierAttrName = "slice_fusion_barrier";
|
|||||||
// 'dependenceConstraints' at depth greater than 'loopDepth', and computes slice
|
// 'dependenceConstraints' at depth greater than 'loopDepth', and computes slice
|
||||||
// bounds in 'sliceState' which represent the one loop nest's IVs in terms of
|
// bounds in 'sliceState' which represent the one loop nest's IVs in terms of
|
||||||
// the other loop nest's IVs, symbols and constants (using 'isBackwardsSlice').
|
// the other loop nest's IVs, symbols and constants (using 'isBackwardsSlice').
|
||||||
void mlir::getComputationSliceState(
|
void mlir::affine::getComputationSliceState(
|
||||||
Operation *depSourceOp, Operation *depSinkOp,
|
Operation *depSourceOp, Operation *depSinkOp,
|
||||||
FlatAffineValueConstraints *dependenceConstraints, unsigned loopDepth,
|
FlatAffineValueConstraints *dependenceConstraints, unsigned loopDepth,
|
||||||
bool isBackwardSlice, ComputationSliceState *sliceState) {
|
bool isBackwardSlice, ComputationSliceState *sliceState) {
|
||||||
@ -1631,10 +1636,9 @@ void mlir::getComputationSliceState(
|
|||||||
// entire destination index set. Subtract out the dependent destination
|
// entire destination index set. Subtract out the dependent destination
|
||||||
// iterations from destination index set and check for emptiness --- this is one
|
// iterations from destination index set and check for emptiness --- this is one
|
||||||
// solution.
|
// solution.
|
||||||
AffineForOp
|
AffineForOp mlir::affine::insertBackwardComputationSlice(
|
||||||
mlir::insertBackwardComputationSlice(Operation *srcOpInst, Operation *dstOpInst,
|
Operation *srcOpInst, Operation *dstOpInst, unsigned dstLoopDepth,
|
||||||
unsigned dstLoopDepth,
|
ComputationSliceState *sliceState) {
|
||||||
ComputationSliceState *sliceState) {
|
|
||||||
// Get loop nest surrounding src operation.
|
// Get loop nest surrounding src operation.
|
||||||
SmallVector<AffineForOp, 4> srcLoopIVs;
|
SmallVector<AffineForOp, 4> srcLoopIVs;
|
||||||
getAffineForIVs(*srcOpInst, &srcLoopIVs);
|
getAffineForIVs(*srcOpInst, &srcLoopIVs);
|
||||||
@ -1713,7 +1717,7 @@ bool MemRefAccess::isStore() const {
|
|||||||
|
|
||||||
/// Returns the nesting depth of this statement, i.e., the number of loops
|
/// Returns the nesting depth of this statement, i.e., the number of loops
|
||||||
/// surrounding this statement.
|
/// surrounding this statement.
|
||||||
unsigned mlir::getNestingDepth(Operation *op) {
|
unsigned mlir::affine::getNestingDepth(Operation *op) {
|
||||||
Operation *currOp = op;
|
Operation *currOp = op;
|
||||||
unsigned depth = 0;
|
unsigned depth = 0;
|
||||||
while ((currOp = currOp->getParentOp())) {
|
while ((currOp = currOp->getParentOp())) {
|
||||||
@ -1741,7 +1745,7 @@ bool MemRefAccess::operator==(const MemRefAccess &rhs) const {
|
|||||||
[](AffineExpr e) { return e == 0; });
|
[](AffineExpr e) { return e == 0; });
|
||||||
}
|
}
|
||||||
|
|
||||||
void mlir::getAffineIVs(Operation &op, SmallVectorImpl<Value> &ivs) {
|
void mlir::affine::getAffineIVs(Operation &op, SmallVectorImpl<Value> &ivs) {
|
||||||
auto *currOp = op.getParentOp();
|
auto *currOp = op.getParentOp();
|
||||||
AffineForOp currAffineForOp;
|
AffineForOp currAffineForOp;
|
||||||
// Traverse up the hierarchy collecting all 'affine.for' and affine.parallel
|
// Traverse up the hierarchy collecting all 'affine.for' and affine.parallel
|
||||||
@ -1758,7 +1762,8 @@ void mlir::getAffineIVs(Operation &op, SmallVectorImpl<Value> &ivs) {
|
|||||||
|
|
||||||
/// Returns the number of surrounding loops common to 'loopsA' and 'loopsB',
|
/// Returns the number of surrounding loops common to 'loopsA' and 'loopsB',
|
||||||
/// where each lists loops from outer-most to inner-most in loop nest.
|
/// where each lists loops from outer-most to inner-most in loop nest.
|
||||||
unsigned mlir::getNumCommonSurroundingLoops(Operation &a, Operation &b) {
|
unsigned mlir::affine::getNumCommonSurroundingLoops(Operation &a,
|
||||||
|
Operation &b) {
|
||||||
SmallVector<Value, 4> loopsA, loopsB;
|
SmallVector<Value, 4> loopsA, loopsB;
|
||||||
getAffineIVs(a, loopsA);
|
getAffineIVs(a, loopsA);
|
||||||
getAffineIVs(b, loopsB);
|
getAffineIVs(b, loopsB);
|
||||||
@ -1817,8 +1822,8 @@ static std::optional<int64_t> getMemoryFootprintBytes(Block &block,
|
|||||||
return totalSizeInBytes;
|
return totalSizeInBytes;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::optional<int64_t> mlir::getMemoryFootprintBytes(AffineForOp forOp,
|
std::optional<int64_t> mlir::affine::getMemoryFootprintBytes(AffineForOp forOp,
|
||||||
int memorySpace) {
|
int memorySpace) {
|
||||||
auto *forInst = forOp.getOperation();
|
auto *forInst = forOp.getOperation();
|
||||||
return ::getMemoryFootprintBytes(
|
return ::getMemoryFootprintBytes(
|
||||||
*forInst->getBlock(), Block::iterator(forInst),
|
*forInst->getBlock(), Block::iterator(forInst),
|
||||||
@ -1826,7 +1831,7 @@ std::optional<int64_t> mlir::getMemoryFootprintBytes(AffineForOp forOp,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Returns whether a loop is parallel and contains a reduction loop.
|
/// Returns whether a loop is parallel and contains a reduction loop.
|
||||||
bool mlir::isLoopParallelAndContainsReduction(AffineForOp forOp) {
|
bool mlir::affine::isLoopParallelAndContainsReduction(AffineForOp forOp) {
|
||||||
SmallVector<LoopReduction> reductions;
|
SmallVector<LoopReduction> reductions;
|
||||||
if (!isLoopParallel(forOp, &reductions))
|
if (!isLoopParallel(forOp, &reductions))
|
||||||
return false;
|
return false;
|
||||||
@ -1835,8 +1840,8 @@ bool mlir::isLoopParallelAndContainsReduction(AffineForOp forOp) {
|
|||||||
|
|
||||||
/// Returns in 'sequentialLoops' all sequential loops in loop nest rooted
|
/// Returns in 'sequentialLoops' all sequential loops in loop nest rooted
|
||||||
/// at 'forOp'.
|
/// at 'forOp'.
|
||||||
void mlir::getSequentialLoops(AffineForOp forOp,
|
void mlir::affine::getSequentialLoops(
|
||||||
llvm::SmallDenseSet<Value, 8> *sequentialLoops) {
|
AffineForOp forOp, llvm::SmallDenseSet<Value, 8> *sequentialLoops) {
|
||||||
forOp->walk([&](Operation *op) {
|
forOp->walk([&](Operation *op) {
|
||||||
if (auto innerFor = dyn_cast<AffineForOp>(op))
|
if (auto innerFor = dyn_cast<AffineForOp>(op))
|
||||||
if (!isLoopParallel(innerFor))
|
if (!isLoopParallel(innerFor))
|
||||||
@ -1844,7 +1849,7 @@ void mlir::getSequentialLoops(AffineForOp forOp,
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
IntegerSet mlir::simplifyIntegerSet(IntegerSet set) {
|
IntegerSet mlir::affine::simplifyIntegerSet(IntegerSet set) {
|
||||||
FlatAffineValueConstraints fac(set);
|
FlatAffineValueConstraints fac(set);
|
||||||
if (fac.isEmpty())
|
if (fac.isEmpty())
|
||||||
return IntegerSet::getEmptySet(set.getNumDims(), set.getNumSymbols(),
|
return IntegerSet::getEmptySet(set.getNumDims(), set.getNumSymbols(),
|
||||||
@ -1930,9 +1935,8 @@ static AffineMap addConstToResults(AffineMap map, int64_t val) {
|
|||||||
// ... | 0 | 0 | -1 | ... | ... | = 0
|
// ... | 0 | 0 | -1 | ... | ... | = 0
|
||||||
// 0 | 0 | 1 | -1 | 0 | -1 | >= 0
|
// 0 | 0 | 1 | -1 | 0 | -1 | >= 0
|
||||||
//
|
//
|
||||||
FailureOr<AffineValueMap>
|
FailureOr<AffineValueMap> mlir::affine::simplifyConstrainedMinMaxOp(
|
||||||
mlir::simplifyConstrainedMinMaxOp(Operation *op,
|
Operation *op, FlatAffineValueConstraints constraints) {
|
||||||
FlatAffineValueConstraints constraints) {
|
|
||||||
bool isMin = isa<AffineMinOp>(op);
|
bool isMin = isa<AffineMinOp>(op);
|
||||||
assert((isMin || isa<AffineMaxOp>(op)) && "expect AffineMin/MaxOp");
|
assert((isMin || isa<AffineMaxOp>(op)) && "expect AffineMin/MaxOp");
|
||||||
MLIRContext *ctx = op->getContext();
|
MLIRContext *ctx = op->getContext();
|
||||||
@ -2032,6 +2036,6 @@ mlir::simplifyConstrainedMinMaxOp(Operation *op,
|
|||||||
newMap.getNumDims(), newMap.getNumSymbols());
|
newMap.getNumDims(), newMap.getNumSymbols());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
mlir::canonicalizeMapAndOperands(&newMap, &newOperands);
|
affine::canonicalizeMapAndOperands(&newMap, &newOperands);
|
||||||
return AffineValueMap(newMap, newOperands);
|
return AffineValueMap(newMap, newOperands);
|
||||||
}
|
}
|
||||||
|
@ -9,6 +9,7 @@
|
|||||||
#include "mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.h"
|
#include "mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.h"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
//===----------------------------------------------------------------------===//
|
//===----------------------------------------------------------------------===//
|
||||||
// Affine Memory Op Interfaces
|
// Affine Memory Op Interfaces
|
||||||
|
@ -26,6 +26,7 @@
|
|||||||
#include <optional>
|
#include <optional>
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
#define DEBUG_TYPE "affine-ops"
|
#define DEBUG_TYPE "affine-ops"
|
||||||
|
|
||||||
@ -35,7 +36,7 @@ using namespace mlir;
|
|||||||
/// `region` or is an argument of `region`. A value of index type defined at the
|
/// `region` or is an argument of `region`. A value of index type defined at the
|
||||||
/// top level of a `AffineScope` region is always a valid symbol for all
|
/// top level of a `AffineScope` region is always a valid symbol for all
|
||||||
/// uses in that region.
|
/// uses in that region.
|
||||||
bool mlir::isTopLevelValue(Value value, Region *region) {
|
bool mlir::affine::isTopLevelValue(Value value, Region *region) {
|
||||||
if (auto arg = value.dyn_cast<BlockArgument>())
|
if (auto arg = value.dyn_cast<BlockArgument>())
|
||||||
return arg.getParentRegion() == region;
|
return arg.getParentRegion() == region;
|
||||||
return value.getDefiningOp()->getParentRegion() == region;
|
return value.getDefiningOp()->getParentRegion() == region;
|
||||||
@ -231,7 +232,7 @@ Operation *AffineDialect::materializeConstant(OpBuilder &builder,
|
|||||||
/// op with trait `AffineScope`. If the value is defined in an unlinked region,
|
/// op with trait `AffineScope`. If the value is defined in an unlinked region,
|
||||||
/// conservatively assume it is not top-level. A value of index type defined at
|
/// conservatively assume it is not top-level. A value of index type defined at
|
||||||
/// the top level is always a valid symbol.
|
/// the top level is always a valid symbol.
|
||||||
bool mlir::isTopLevelValue(Value value) {
|
bool mlir::affine::isTopLevelValue(Value value) {
|
||||||
if (auto arg = value.dyn_cast<BlockArgument>()) {
|
if (auto arg = value.dyn_cast<BlockArgument>()) {
|
||||||
// The block owning the argument may be unlinked, e.g. when the surrounding
|
// The block owning the argument may be unlinked, e.g. when the surrounding
|
||||||
// region has not yet been attached to an Op, at which point the parent Op
|
// region has not yet been attached to an Op, at which point the parent Op
|
||||||
@ -246,7 +247,7 @@ bool mlir::isTopLevelValue(Value value) {
|
|||||||
|
|
||||||
/// Returns the closest region enclosing `op` that is held by an operation with
|
/// Returns the closest region enclosing `op` that is held by an operation with
|
||||||
/// trait `AffineScope`; `nullptr` if there is no such region.
|
/// trait `AffineScope`; `nullptr` if there is no such region.
|
||||||
Region *mlir::getAffineScope(Operation *op) {
|
Region *mlir::affine::getAffineScope(Operation *op) {
|
||||||
auto *curOp = op;
|
auto *curOp = op;
|
||||||
while (auto *parentOp = curOp->getParentOp()) {
|
while (auto *parentOp = curOp->getParentOp()) {
|
||||||
if (parentOp->hasTrait<OpTrait::AffineScope>())
|
if (parentOp->hasTrait<OpTrait::AffineScope>())
|
||||||
@ -261,7 +262,7 @@ Region *mlir::getAffineScope(Operation *op) {
|
|||||||
// *) It is valid as a symbol.
|
// *) It is valid as a symbol.
|
||||||
// *) It is an induction variable.
|
// *) It is an induction variable.
|
||||||
// *) It is the result of affine apply operation with dimension id arguments.
|
// *) It is the result of affine apply operation with dimension id arguments.
|
||||||
bool mlir::isValidDim(Value value) {
|
bool mlir::affine::isValidDim(Value value) {
|
||||||
// The value must be an index type.
|
// The value must be an index type.
|
||||||
if (!value.getType().isIndex())
|
if (!value.getType().isIndex())
|
||||||
return false;
|
return false;
|
||||||
@ -281,7 +282,7 @@ bool mlir::isValidDim(Value value) {
|
|||||||
// *) It is valid as a symbol.
|
// *) It is valid as a symbol.
|
||||||
// *) It is an induction variable.
|
// *) It is an induction variable.
|
||||||
// *) It is the result of an affine apply operation with dimension id operands.
|
// *) It is the result of an affine apply operation with dimension id operands.
|
||||||
bool mlir::isValidDim(Value value, Region *region) {
|
bool mlir::affine::isValidDim(Value value, Region *region) {
|
||||||
// The value must be an index type.
|
// The value must be an index type.
|
||||||
if (!value.getType().isIndex())
|
if (!value.getType().isIndex())
|
||||||
return false;
|
return false;
|
||||||
@ -358,7 +359,7 @@ static bool isDimOpValidSymbol(ShapedDimOpInterface dimOp, Region *region) {
|
|||||||
// *) It is the result of an affine.apply operation with symbol operands.
|
// *) It is the result of an affine.apply operation with symbol operands.
|
||||||
// *) It is a result of the dim op on a memref whose corresponding size is a
|
// *) It is a result of the dim op on a memref whose corresponding size is a
|
||||||
// valid symbol.
|
// valid symbol.
|
||||||
bool mlir::isValidSymbol(Value value) {
|
bool mlir::affine::isValidSymbol(Value value) {
|
||||||
if (!value)
|
if (!value)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
@ -387,7 +388,7 @@ bool mlir::isValidSymbol(Value value) {
|
|||||||
/// If `region` is null, conservatively assume the symbol definition scope does
|
/// If `region` is null, conservatively assume the symbol definition scope does
|
||||||
/// not exist and only accept the values that would be symbols regardless of
|
/// not exist and only accept the values that would be symbols regardless of
|
||||||
/// the surrounding region structure, i.e. the first three cases above.
|
/// the surrounding region structure, i.e. the first three cases above.
|
||||||
bool mlir::isValidSymbol(Value value, Region *region) {
|
bool mlir::affine::isValidSymbol(Value value, Region *region) {
|
||||||
// The value must be an index type.
|
// The value must be an index type.
|
||||||
if (!value.getType().isIndex())
|
if (!value.getType().isIndex())
|
||||||
return false;
|
return false;
|
||||||
@ -447,9 +448,8 @@ static void printDimAndSymbolList(Operation::operand_iterator begin,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Parses dimension and symbol list and returns true if parsing failed.
|
/// Parses dimension and symbol list and returns true if parsing failed.
|
||||||
ParseResult mlir::parseDimAndSymbolList(OpAsmParser &parser,
|
ParseResult mlir::affine::parseDimAndSymbolList(
|
||||||
SmallVectorImpl<Value> &operands,
|
OpAsmParser &parser, SmallVectorImpl<Value> &operands, unsigned &numDims) {
|
||||||
unsigned &numDims) {
|
|
||||||
SmallVector<OpAsmParser::UnresolvedOperand, 8> opInfos;
|
SmallVector<OpAsmParser::UnresolvedOperand, 8> opInfos;
|
||||||
if (parser.parseOperandList(opInfos, OpAsmParser::Delimiter::Paren))
|
if (parser.parseOperandList(opInfos, OpAsmParser::Delimiter::Paren))
|
||||||
return failure();
|
return failure();
|
||||||
@ -541,7 +541,7 @@ LogicalResult AffineApplyOp::verify() {
|
|||||||
// its operands are valid dimension ids.
|
// its operands are valid dimension ids.
|
||||||
bool AffineApplyOp::isValidDim() {
|
bool AffineApplyOp::isValidDim() {
|
||||||
return llvm::all_of(getOperands(),
|
return llvm::all_of(getOperands(),
|
||||||
[](Value op) { return mlir::isValidDim(op); });
|
[](Value op) { return affine::isValidDim(op); });
|
||||||
}
|
}
|
||||||
|
|
||||||
// The result of the affine apply operation can be used as a dimension id if all
|
// The result of the affine apply operation can be used as a dimension id if all
|
||||||
@ -556,14 +556,14 @@ bool AffineApplyOp::isValidDim(Region *region) {
|
|||||||
// operands are symbols.
|
// operands are symbols.
|
||||||
bool AffineApplyOp::isValidSymbol() {
|
bool AffineApplyOp::isValidSymbol() {
|
||||||
return llvm::all_of(getOperands(),
|
return llvm::all_of(getOperands(),
|
||||||
[](Value op) { return mlir::isValidSymbol(op); });
|
[](Value op) { return affine::isValidSymbol(op); });
|
||||||
}
|
}
|
||||||
|
|
||||||
// The result of the affine apply operation can be used as a symbol in `region`
|
// The result of the affine apply operation can be used as a symbol in `region`
|
||||||
// if all its operands are symbols in `region`.
|
// if all its operands are symbols in `region`.
|
||||||
bool AffineApplyOp::isValidSymbol(Region *region) {
|
bool AffineApplyOp::isValidSymbol(Region *region) {
|
||||||
return llvm::all_of(getOperands(), [&](Value operand) {
|
return llvm::all_of(getOperands(), [&](Value operand) {
|
||||||
return mlir::isValidSymbol(operand, region);
|
return affine::isValidSymbol(operand, region);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1071,8 +1071,8 @@ static void composeAffineMapAndOperands(AffineMap *map,
|
|||||||
*map = simplifyAffineMap(*map);
|
*map = simplifyAffineMap(*map);
|
||||||
}
|
}
|
||||||
|
|
||||||
void mlir::fullyComposeAffineMapAndOperands(AffineMap *map,
|
void mlir::affine::fullyComposeAffineMapAndOperands(
|
||||||
SmallVectorImpl<Value> *operands) {
|
AffineMap *map, SmallVectorImpl<Value> *operands) {
|
||||||
while (llvm::any_of(*operands, [](Value v) {
|
while (llvm::any_of(*operands, [](Value v) {
|
||||||
return isa_and_nonnull<AffineApplyOp>(v.getDefiningOp());
|
return isa_and_nonnull<AffineApplyOp>(v.getDefiningOp());
|
||||||
})) {
|
})) {
|
||||||
@ -1168,9 +1168,9 @@ createOrFold(OpBuilder &b, Location loc, ValueRange operands,
|
|||||||
return op->getResult(0);
|
return op->getResult(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
AffineApplyOp mlir::makeComposedAffineApply(OpBuilder &b, Location loc,
|
AffineApplyOp mlir::affine::makeComposedAffineApply(OpBuilder &b, Location loc,
|
||||||
AffineMap map,
|
AffineMap map,
|
||||||
ValueRange operands) {
|
ValueRange operands) {
|
||||||
AffineMap normalizedMap = map;
|
AffineMap normalizedMap = map;
|
||||||
SmallVector<Value, 8> normalizedOperands(operands.begin(), operands.end());
|
SmallVector<Value, 8> normalizedOperands(operands.begin(), operands.end());
|
||||||
composeAffineMapAndOperands(&normalizedMap, &normalizedOperands);
|
composeAffineMapAndOperands(&normalizedMap, &normalizedOperands);
|
||||||
@ -1178,8 +1178,9 @@ AffineApplyOp mlir::makeComposedAffineApply(OpBuilder &b, Location loc,
|
|||||||
return b.create<AffineApplyOp>(loc, normalizedMap, normalizedOperands);
|
return b.create<AffineApplyOp>(loc, normalizedMap, normalizedOperands);
|
||||||
}
|
}
|
||||||
|
|
||||||
AffineApplyOp mlir::makeComposedAffineApply(OpBuilder &b, Location loc,
|
AffineApplyOp mlir::affine::makeComposedAffineApply(OpBuilder &b, Location loc,
|
||||||
AffineExpr e, ValueRange values) {
|
AffineExpr e,
|
||||||
|
ValueRange values) {
|
||||||
return makeComposedAffineApply(
|
return makeComposedAffineApply(
|
||||||
b, loc, AffineMap::inferFromExprList(ArrayRef<AffineExpr>{e}).front(),
|
b, loc, AffineMap::inferFromExprList(ArrayRef<AffineExpr>{e}).front(),
|
||||||
values);
|
values);
|
||||||
@ -1216,8 +1217,9 @@ static void composeMultiResultAffineMap(AffineMap &map,
|
|||||||
}
|
}
|
||||||
|
|
||||||
OpFoldResult
|
OpFoldResult
|
||||||
mlir::makeComposedFoldedAffineApply(OpBuilder &b, Location loc, AffineMap map,
|
mlir::affine::makeComposedFoldedAffineApply(OpBuilder &b, Location loc,
|
||||||
ArrayRef<OpFoldResult> operands) {
|
AffineMap map,
|
||||||
|
ArrayRef<OpFoldResult> operands) {
|
||||||
assert(map.getNumResults() == 1 && "building affine.apply with !=1 result");
|
assert(map.getNumResults() == 1 && "building affine.apply with !=1 result");
|
||||||
|
|
||||||
SmallVector<Operation *> constants;
|
SmallVector<Operation *> constants;
|
||||||
@ -1234,14 +1236,16 @@ mlir::makeComposedFoldedAffineApply(OpBuilder &b, Location loc, AffineMap map,
|
|||||||
}
|
}
|
||||||
|
|
||||||
OpFoldResult
|
OpFoldResult
|
||||||
mlir::makeComposedFoldedAffineApply(OpBuilder &b, Location loc, AffineExpr expr,
|
mlir::affine::makeComposedFoldedAffineApply(OpBuilder &b, Location loc,
|
||||||
ArrayRef<OpFoldResult> operands) {
|
AffineExpr expr,
|
||||||
|
ArrayRef<OpFoldResult> operands) {
|
||||||
return makeComposedFoldedAffineApply(
|
return makeComposedFoldedAffineApply(
|
||||||
b, loc, AffineMap::inferFromExprList(ArrayRef<AffineExpr>{expr}).front(),
|
b, loc, AffineMap::inferFromExprList(ArrayRef<AffineExpr>{expr}).front(),
|
||||||
operands);
|
operands);
|
||||||
}
|
}
|
||||||
|
|
||||||
SmallVector<OpFoldResult> mlir::makeComposedFoldedMultiResultAffineApply(
|
SmallVector<OpFoldResult>
|
||||||
|
mlir::affine::makeComposedFoldedMultiResultAffineApply(
|
||||||
OpBuilder &b, Location loc, AffineMap map,
|
OpBuilder &b, Location loc, AffineMap map,
|
||||||
ArrayRef<OpFoldResult> operands) {
|
ArrayRef<OpFoldResult> operands) {
|
||||||
return llvm::to_vector(llvm::map_range(
|
return llvm::to_vector(llvm::map_range(
|
||||||
@ -1251,8 +1255,8 @@ SmallVector<OpFoldResult> mlir::makeComposedFoldedMultiResultAffineApply(
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
Value mlir::makeComposedAffineMin(OpBuilder &b, Location loc, AffineMap map,
|
Value mlir::affine::makeComposedAffineMin(OpBuilder &b, Location loc,
|
||||||
ValueRange operands) {
|
AffineMap map, ValueRange operands) {
|
||||||
SmallVector<Value> allOperands = llvm::to_vector(operands);
|
SmallVector<Value> allOperands = llvm::to_vector(operands);
|
||||||
composeMultiResultAffineMap(map, allOperands);
|
composeMultiResultAffineMap(map, allOperands);
|
||||||
return b.createOrFold<AffineMinOp>(loc, b.getIndexType(), map, allOperands);
|
return b.createOrFold<AffineMinOp>(loc, b.getIndexType(), map, allOperands);
|
||||||
@ -1277,14 +1281,16 @@ static OpFoldResult makeComposedFoldedMinMax(OpBuilder &b, Location loc,
|
|||||||
}
|
}
|
||||||
|
|
||||||
OpFoldResult
|
OpFoldResult
|
||||||
mlir::makeComposedFoldedAffineMin(OpBuilder &b, Location loc, AffineMap map,
|
mlir::affine::makeComposedFoldedAffineMin(OpBuilder &b, Location loc,
|
||||||
ArrayRef<OpFoldResult> operands) {
|
AffineMap map,
|
||||||
|
ArrayRef<OpFoldResult> operands) {
|
||||||
return makeComposedFoldedMinMax<AffineMinOp>(b, loc, map, operands);
|
return makeComposedFoldedMinMax<AffineMinOp>(b, loc, map, operands);
|
||||||
}
|
}
|
||||||
|
|
||||||
OpFoldResult
|
OpFoldResult
|
||||||
mlir::makeComposedFoldedAffineMax(OpBuilder &b, Location loc, AffineMap map,
|
mlir::affine::makeComposedFoldedAffineMax(OpBuilder &b, Location loc,
|
||||||
ArrayRef<OpFoldResult> operands) {
|
AffineMap map,
|
||||||
|
ArrayRef<OpFoldResult> operands) {
|
||||||
return makeComposedFoldedMinMax<AffineMaxOp>(b, loc, map, operands);
|
return makeComposedFoldedMinMax<AffineMaxOp>(b, loc, map, operands);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1299,8 +1305,9 @@ static Value createFoldedComposedAffineApply(OpBuilder &b, Location loc,
|
|||||||
return b.createOrFold<AffineApplyOp>(loc, map, operands);
|
return b.createOrFold<AffineApplyOp>(loc, map, operands);
|
||||||
}
|
}
|
||||||
|
|
||||||
SmallVector<Value, 4> mlir::applyMapToValues(OpBuilder &b, Location loc,
|
SmallVector<Value, 4> mlir::affine::applyMapToValues(OpBuilder &b, Location loc,
|
||||||
AffineMap map, ValueRange values) {
|
AffineMap map,
|
||||||
|
ValueRange values) {
|
||||||
SmallVector<Value, 4> res;
|
SmallVector<Value, 4> res;
|
||||||
res.reserve(map.getNumResults());
|
res.reserve(map.getNumResults());
|
||||||
unsigned numDims = map.getNumDims(), numSym = map.getNumSymbols();
|
unsigned numDims = map.getNumDims(), numSym = map.getNumSymbols();
|
||||||
@ -1436,13 +1443,13 @@ static void canonicalizeMapOrSetAndOperands(MapOrSet *mapOrSet,
|
|||||||
*operands = resultOperands;
|
*operands = resultOperands;
|
||||||
}
|
}
|
||||||
|
|
||||||
void mlir::canonicalizeMapAndOperands(AffineMap *map,
|
void mlir::affine::canonicalizeMapAndOperands(
|
||||||
SmallVectorImpl<Value> *operands) {
|
AffineMap *map, SmallVectorImpl<Value> *operands) {
|
||||||
canonicalizeMapOrSetAndOperands<AffineMap>(map, operands);
|
canonicalizeMapOrSetAndOperands<AffineMap>(map, operands);
|
||||||
}
|
}
|
||||||
|
|
||||||
void mlir::canonicalizeSetAndOperands(IntegerSet *set,
|
void mlir::affine::canonicalizeSetAndOperands(
|
||||||
SmallVectorImpl<Value> *operands) {
|
IntegerSet *set, SmallVectorImpl<Value> *operands) {
|
||||||
canonicalizeMapOrSetAndOperands<IntegerSet>(set, operands);
|
canonicalizeMapOrSetAndOperands<IntegerSet>(set, operands);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2518,19 +2525,19 @@ Speculation::Speculatability AffineForOp::getSpeculatability() {
|
|||||||
|
|
||||||
/// Returns true if the provided value is the induction variable of a
|
/// Returns true if the provided value is the induction variable of a
|
||||||
/// AffineForOp.
|
/// AffineForOp.
|
||||||
bool mlir::isAffineForInductionVar(Value val) {
|
bool mlir::affine::isAffineForInductionVar(Value val) {
|
||||||
return getForInductionVarOwner(val) != AffineForOp();
|
return getForInductionVarOwner(val) != AffineForOp();
|
||||||
}
|
}
|
||||||
|
|
||||||
bool mlir::isAffineParallelInductionVar(Value val) {
|
bool mlir::affine::isAffineParallelInductionVar(Value val) {
|
||||||
return getAffineParallelInductionVarOwner(val) != nullptr;
|
return getAffineParallelInductionVarOwner(val) != nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool mlir::isAffineInductionVar(Value val) {
|
bool mlir::affine::isAffineInductionVar(Value val) {
|
||||||
return isAffineForInductionVar(val) || isAffineParallelInductionVar(val);
|
return isAffineForInductionVar(val) || isAffineParallelInductionVar(val);
|
||||||
}
|
}
|
||||||
|
|
||||||
AffineForOp mlir::getForInductionVarOwner(Value val) {
|
AffineForOp mlir::affine::getForInductionVarOwner(Value val) {
|
||||||
auto ivArg = val.dyn_cast<BlockArgument>();
|
auto ivArg = val.dyn_cast<BlockArgument>();
|
||||||
if (!ivArg || !ivArg.getOwner())
|
if (!ivArg || !ivArg.getOwner())
|
||||||
return AffineForOp();
|
return AffineForOp();
|
||||||
@ -2541,7 +2548,7 @@ AffineForOp mlir::getForInductionVarOwner(Value val) {
|
|||||||
return AffineForOp();
|
return AffineForOp();
|
||||||
}
|
}
|
||||||
|
|
||||||
AffineParallelOp mlir::getAffineParallelInductionVarOwner(Value val) {
|
AffineParallelOp mlir::affine::getAffineParallelInductionVarOwner(Value val) {
|
||||||
auto ivArg = val.dyn_cast<BlockArgument>();
|
auto ivArg = val.dyn_cast<BlockArgument>();
|
||||||
if (!ivArg || !ivArg.getOwner())
|
if (!ivArg || !ivArg.getOwner())
|
||||||
return nullptr;
|
return nullptr;
|
||||||
@ -2554,15 +2561,15 @@ AffineParallelOp mlir::getAffineParallelInductionVarOwner(Value val) {
|
|||||||
|
|
||||||
/// Extracts the induction variables from a list of AffineForOps and returns
|
/// Extracts the induction variables from a list of AffineForOps and returns
|
||||||
/// them.
|
/// them.
|
||||||
void mlir::extractForInductionVars(ArrayRef<AffineForOp> forInsts,
|
void mlir::affine::extractForInductionVars(ArrayRef<AffineForOp> forInsts,
|
||||||
SmallVectorImpl<Value> *ivs) {
|
SmallVectorImpl<Value> *ivs) {
|
||||||
ivs->reserve(forInsts.size());
|
ivs->reserve(forInsts.size());
|
||||||
for (auto forInst : forInsts)
|
for (auto forInst : forInsts)
|
||||||
ivs->push_back(forInst.getInductionVar());
|
ivs->push_back(forInst.getInductionVar());
|
||||||
}
|
}
|
||||||
|
|
||||||
void mlir::extractInductionVars(ArrayRef<mlir::Operation *> affineOps,
|
void mlir::affine::extractInductionVars(ArrayRef<mlir::Operation *> affineOps,
|
||||||
SmallVectorImpl<mlir::Value> &ivs) {
|
SmallVectorImpl<mlir::Value> &ivs) {
|
||||||
ivs.reserve(affineOps.size());
|
ivs.reserve(affineOps.size());
|
||||||
for (Operation *op : affineOps) {
|
for (Operation *op : affineOps) {
|
||||||
// Add constraints from forOp's bounds.
|
// Add constraints from forOp's bounds.
|
||||||
@ -2640,7 +2647,7 @@ buildAffineLoopFromValues(OpBuilder &builder, Location loc, Value lb, Value ub,
|
|||||||
/*iterArgs=*/std::nullopt, bodyBuilderFn);
|
/*iterArgs=*/std::nullopt, bodyBuilderFn);
|
||||||
}
|
}
|
||||||
|
|
||||||
void mlir::buildAffineLoopNest(
|
void mlir::affine::buildAffineLoopNest(
|
||||||
OpBuilder &builder, Location loc, ArrayRef<int64_t> lbs,
|
OpBuilder &builder, Location loc, ArrayRef<int64_t> lbs,
|
||||||
ArrayRef<int64_t> ubs, ArrayRef<int64_t> steps,
|
ArrayRef<int64_t> ubs, ArrayRef<int64_t> steps,
|
||||||
function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuilderFn) {
|
function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuilderFn) {
|
||||||
@ -2648,7 +2655,7 @@ void mlir::buildAffineLoopNest(
|
|||||||
buildAffineLoopFromConstants);
|
buildAffineLoopFromConstants);
|
||||||
}
|
}
|
||||||
|
|
||||||
void mlir::buildAffineLoopNest(
|
void mlir::affine::buildAffineLoopNest(
|
||||||
OpBuilder &builder, Location loc, ValueRange lbs, ValueRange ubs,
|
OpBuilder &builder, Location loc, ValueRange lbs, ValueRange ubs,
|
||||||
ArrayRef<int64_t> steps,
|
ArrayRef<int64_t> steps,
|
||||||
function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuilderFn) {
|
function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuilderFn) {
|
||||||
@ -2656,11 +2663,12 @@ void mlir::buildAffineLoopNest(
|
|||||||
buildAffineLoopFromValues);
|
buildAffineLoopFromValues);
|
||||||
}
|
}
|
||||||
|
|
||||||
AffineForOp mlir::replaceForOpWithNewYields(OpBuilder &b, AffineForOp loop,
|
AffineForOp mlir::affine::replaceForOpWithNewYields(OpBuilder &b,
|
||||||
ValueRange newIterOperands,
|
AffineForOp loop,
|
||||||
ValueRange newYieldedValues,
|
ValueRange newIterOperands,
|
||||||
ValueRange newIterArgs,
|
ValueRange newYieldedValues,
|
||||||
bool replaceLoopResults) {
|
ValueRange newIterArgs,
|
||||||
|
bool replaceLoopResults) {
|
||||||
assert(newIterOperands.size() == newYieldedValues.size() &&
|
assert(newIterOperands.size() == newYieldedValues.size() &&
|
||||||
"newIterOperands must be of the same size as newYieldedValues");
|
"newIterOperands must be of the same size as newYieldedValues");
|
||||||
// Create a new loop before the existing one, with the extra operands.
|
// Create a new loop before the existing one, with the extra operands.
|
||||||
|
@ -10,6 +10,7 @@
|
|||||||
#include "mlir/Dialect/Affine/IR/AffineOps.h"
|
#include "mlir/Dialect/Affine/IR/AffineOps.h"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
AffineValueMap::AffineValueMap(AffineMap map, ValueRange operands,
|
AffineValueMap::AffineValueMap(AffineMap map, ValueRange operands,
|
||||||
ValueRange results)
|
ValueRange results)
|
||||||
|
@ -12,6 +12,7 @@
|
|||||||
#include "mlir/Interfaces/ValueBoundsOpInterface.h"
|
#include "mlir/Interfaces/ValueBoundsOpInterface.h"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
namespace {
|
namespace {
|
||||||
|
@ -19,6 +19,7 @@
|
|||||||
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
|
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
using namespace mlir::transform;
|
using namespace mlir::transform;
|
||||||
|
|
||||||
//===----------------------------------------------------------------------===//
|
//===----------------------------------------------------------------------===//
|
||||||
|
@ -35,13 +35,16 @@
|
|||||||
#include <optional>
|
#include <optional>
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
#define GEN_PASS_DEF_AFFINEDATACOPYGENERATION
|
#define GEN_PASS_DEF_AFFINEDATACOPYGENERATION
|
||||||
#include "mlir/Dialect/Affine/Passes.h.inc"
|
#include "mlir/Dialect/Affine/Passes.h.inc"
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#define DEBUG_TYPE "affine-data-copy-generate"
|
#define DEBUG_TYPE "affine-data-copy-generate"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
@ -57,7 +60,8 @@ namespace {
|
|||||||
// TODO: We currently can't generate copies correctly when stores
|
// TODO: We currently can't generate copies correctly when stores
|
||||||
// are strided. Check for strided stores.
|
// are strided. Check for strided stores.
|
||||||
struct AffineDataCopyGeneration
|
struct AffineDataCopyGeneration
|
||||||
: public impl::AffineDataCopyGenerationBase<AffineDataCopyGeneration> {
|
: public affine::impl::AffineDataCopyGenerationBase<
|
||||||
|
AffineDataCopyGeneration> {
|
||||||
AffineDataCopyGeneration() = default;
|
AffineDataCopyGeneration() = default;
|
||||||
explicit AffineDataCopyGeneration(unsigned slowMemorySpace,
|
explicit AffineDataCopyGeneration(unsigned slowMemorySpace,
|
||||||
unsigned fastMemorySpace,
|
unsigned fastMemorySpace,
|
||||||
@ -85,17 +89,15 @@ struct AffineDataCopyGeneration
|
|||||||
/// by the latter. Only load op's handled for now.
|
/// by the latter. Only load op's handled for now.
|
||||||
/// TODO: extend this to store op's.
|
/// TODO: extend this to store op's.
|
||||||
std::unique_ptr<OperationPass<func::FuncOp>>
|
std::unique_ptr<OperationPass<func::FuncOp>>
|
||||||
mlir::createAffineDataCopyGenerationPass(unsigned slowMemorySpace,
|
mlir::affine::createAffineDataCopyGenerationPass(
|
||||||
unsigned fastMemorySpace,
|
unsigned slowMemorySpace, unsigned fastMemorySpace, unsigned tagMemorySpace,
|
||||||
unsigned tagMemorySpace,
|
int minDmaTransferSize, uint64_t fastMemCapacityBytes) {
|
||||||
int minDmaTransferSize,
|
|
||||||
uint64_t fastMemCapacityBytes) {
|
|
||||||
return std::make_unique<AffineDataCopyGeneration>(
|
return std::make_unique<AffineDataCopyGeneration>(
|
||||||
slowMemorySpace, fastMemorySpace, tagMemorySpace, minDmaTransferSize,
|
slowMemorySpace, fastMemorySpace, tagMemorySpace, minDmaTransferSize,
|
||||||
fastMemCapacityBytes);
|
fastMemCapacityBytes);
|
||||||
}
|
}
|
||||||
std::unique_ptr<OperationPass<func::FuncOp>>
|
std::unique_ptr<OperationPass<func::FuncOp>>
|
||||||
mlir::createAffineDataCopyGenerationPass() {
|
mlir::affine::createAffineDataCopyGenerationPass() {
|
||||||
return std::make_unique<AffineDataCopyGeneration>();
|
return std::make_unique<AffineDataCopyGeneration>();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -18,11 +18,14 @@
|
|||||||
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
|
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
#define GEN_PASS_DEF_AFFINEEXPANDINDEXOPS
|
#define GEN_PASS_DEF_AFFINEEXPANDINDEXOPS
|
||||||
#include "mlir/Dialect/Affine/Passes.h.inc"
|
#include "mlir/Dialect/Affine/Passes.h.inc"
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
/// Lowers `affine.delinearize_index` into a sequence of division and remainder
|
/// Lowers `affine.delinearize_index` into a sequence of division and remainder
|
||||||
@ -43,7 +46,7 @@ struct LowerDelinearizeIndexOps
|
|||||||
};
|
};
|
||||||
|
|
||||||
class ExpandAffineIndexOpsPass
|
class ExpandAffineIndexOpsPass
|
||||||
: public impl::AffineExpandIndexOpsBase<ExpandAffineIndexOpsPass> {
|
: public affine::impl::AffineExpandIndexOpsBase<ExpandAffineIndexOpsPass> {
|
||||||
public:
|
public:
|
||||||
ExpandAffineIndexOpsPass() = default;
|
ExpandAffineIndexOpsPass() = default;
|
||||||
|
|
||||||
@ -59,10 +62,11 @@ public:
|
|||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
void mlir::populateAffineExpandIndexOpsPatterns(RewritePatternSet &patterns) {
|
void mlir::affine::populateAffineExpandIndexOpsPatterns(
|
||||||
|
RewritePatternSet &patterns) {
|
||||||
patterns.insert<LowerDelinearizeIndexOps>(patterns.getContext());
|
patterns.insert<LowerDelinearizeIndexOps>(patterns.getContext());
|
||||||
}
|
}
|
||||||
|
|
||||||
std::unique_ptr<Pass> mlir::createAffineExpandIndexOpsPass() {
|
std::unique_ptr<Pass> mlir::affine::createAffineExpandIndexOpsPass() {
|
||||||
return std::make_unique<ExpandAffineIndexOpsPass>();
|
return std::make_unique<ExpandAffineIndexOpsPass>();
|
||||||
}
|
}
|
||||||
|
@ -34,13 +34,16 @@
|
|||||||
#include "llvm/Support/raw_ostream.h"
|
#include "llvm/Support/raw_ostream.h"
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
#define GEN_PASS_DEF_AFFINELOOPINVARIANTCODEMOTION
|
#define GEN_PASS_DEF_AFFINELOOPINVARIANTCODEMOTION
|
||||||
#include "mlir/Dialect/Affine/Passes.h.inc"
|
#include "mlir/Dialect/Affine/Passes.h.inc"
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#define DEBUG_TYPE "licm"
|
#define DEBUG_TYPE "licm"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
@ -50,7 +53,8 @@ namespace {
|
|||||||
/// TODO: This code should be removed once the new LICM pass can handle its
|
/// TODO: This code should be removed once the new LICM pass can handle its
|
||||||
/// uses.
|
/// uses.
|
||||||
struct LoopInvariantCodeMotion
|
struct LoopInvariantCodeMotion
|
||||||
: public impl::AffineLoopInvariantCodeMotionBase<LoopInvariantCodeMotion> {
|
: public affine::impl::AffineLoopInvariantCodeMotionBase<
|
||||||
|
LoopInvariantCodeMotion> {
|
||||||
void runOnOperation() override;
|
void runOnOperation() override;
|
||||||
void runOnAffineForOp(AffineForOp forOp);
|
void runOnAffineForOp(AffineForOp forOp);
|
||||||
};
|
};
|
||||||
@ -71,9 +75,9 @@ areAllOpsInTheBlockListInvariant(Region &blockList, Value indVar,
|
|||||||
SmallPtrSetImpl<Operation *> &opsToHoist);
|
SmallPtrSetImpl<Operation *> &opsToHoist);
|
||||||
|
|
||||||
// Returns true if the individual op is loop invariant.
|
// Returns true if the individual op is loop invariant.
|
||||||
bool isOpLoopInvariant(Operation &op, Value indVar, ValueRange iterArgs,
|
static bool isOpLoopInvariant(Operation &op, Value indVar, ValueRange iterArgs,
|
||||||
SmallPtrSetImpl<Operation *> &opsWithUsers,
|
SmallPtrSetImpl<Operation *> &opsWithUsers,
|
||||||
SmallPtrSetImpl<Operation *> &opsToHoist) {
|
SmallPtrSetImpl<Operation *> &opsToHoist) {
|
||||||
LLVM_DEBUG(llvm::dbgs() << "iterating on op: " << op;);
|
LLVM_DEBUG(llvm::dbgs() << "iterating on op: " << op;);
|
||||||
|
|
||||||
if (auto ifOp = dyn_cast<AffineIfOp>(op)) {
|
if (auto ifOp = dyn_cast<AffineIfOp>(op)) {
|
||||||
@ -167,10 +171,11 @@ bool isOpLoopInvariant(Operation &op, Value indVar, ValueRange iterArgs,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Checks if all ops in a region (i.e. list of blocks) are loop invariant.
|
// Checks if all ops in a region (i.e. list of blocks) are loop invariant.
|
||||||
bool areAllOpsInTheBlockListInvariant(
|
static bool
|
||||||
Region &blockList, Value indVar, ValueRange iterArgs,
|
areAllOpsInTheBlockListInvariant(Region &blockList, Value indVar,
|
||||||
SmallPtrSetImpl<Operation *> &opsWithUsers,
|
ValueRange iterArgs,
|
||||||
SmallPtrSetImpl<Operation *> &opsToHoist) {
|
SmallPtrSetImpl<Operation *> &opsWithUsers,
|
||||||
|
SmallPtrSetImpl<Operation *> &opsToHoist) {
|
||||||
|
|
||||||
for (auto &b : blockList) {
|
for (auto &b : blockList) {
|
||||||
for (auto &op : b) {
|
for (auto &op : b) {
|
||||||
@ -183,10 +188,10 @@ bool areAllOpsInTheBlockListInvariant(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Returns true if the affine.if op can be hoisted.
|
// Returns true if the affine.if op can be hoisted.
|
||||||
bool checkInvarianceOfNestedIfOps(AffineIfOp ifOp, Value indVar,
|
static bool
|
||||||
ValueRange iterArgs,
|
checkInvarianceOfNestedIfOps(AffineIfOp ifOp, Value indVar, ValueRange iterArgs,
|
||||||
SmallPtrSetImpl<Operation *> &opsWithUsers,
|
SmallPtrSetImpl<Operation *> &opsWithUsers,
|
||||||
SmallPtrSetImpl<Operation *> &opsToHoist) {
|
SmallPtrSetImpl<Operation *> &opsToHoist) {
|
||||||
if (!areAllOpsInTheBlockListInvariant(ifOp.getThenRegion(), indVar, iterArgs,
|
if (!areAllOpsInTheBlockListInvariant(ifOp.getThenRegion(), indVar, iterArgs,
|
||||||
opsWithUsers, opsToHoist))
|
opsWithUsers, opsToHoist))
|
||||||
return false;
|
return false;
|
||||||
@ -243,6 +248,6 @@ void LoopInvariantCodeMotion::runOnOperation() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
std::unique_ptr<OperationPass<func::FuncOp>>
|
std::unique_ptr<OperationPass<func::FuncOp>>
|
||||||
mlir::createAffineLoopInvariantCodeMotionPass() {
|
mlir::affine::createAffineLoopInvariantCodeMotionPass() {
|
||||||
return std::make_unique<LoopInvariantCodeMotion>();
|
return std::make_unique<LoopInvariantCodeMotion>();
|
||||||
}
|
}
|
||||||
|
@ -17,11 +17,14 @@
|
|||||||
#include "mlir/Dialect/Func/IR/FuncOps.h"
|
#include "mlir/Dialect/Func/IR/FuncOps.h"
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
#define GEN_PASS_DEF_AFFINELOOPNORMALIZE
|
#define GEN_PASS_DEF_AFFINELOOPNORMALIZE
|
||||||
#include "mlir/Dialect/Affine/Passes.h.inc"
|
#include "mlir/Dialect/Affine/Passes.h.inc"
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
@ -29,7 +32,7 @@ namespace {
|
|||||||
/// As currently implemented, this pass cannot fail, but it might skip over ops
|
/// As currently implemented, this pass cannot fail, but it might skip over ops
|
||||||
/// that are already in a normalized form.
|
/// that are already in a normalized form.
|
||||||
struct AffineLoopNormalizePass
|
struct AffineLoopNormalizePass
|
||||||
: public impl::AffineLoopNormalizeBase<AffineLoopNormalizePass> {
|
: public affine::impl::AffineLoopNormalizeBase<AffineLoopNormalizePass> {
|
||||||
explicit AffineLoopNormalizePass(bool promoteSingleIter) {
|
explicit AffineLoopNormalizePass(bool promoteSingleIter) {
|
||||||
this->promoteSingleIter = promoteSingleIter;
|
this->promoteSingleIter = promoteSingleIter;
|
||||||
}
|
}
|
||||||
@ -47,6 +50,6 @@ struct AffineLoopNormalizePass
|
|||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
std::unique_ptr<OperationPass<func::FuncOp>>
|
std::unique_ptr<OperationPass<func::FuncOp>>
|
||||||
mlir::createAffineLoopNormalizePass(bool promoteSingleIter) {
|
mlir::affine::createAffineLoopNormalizePass(bool promoteSingleIter) {
|
||||||
return std::make_unique<AffineLoopNormalizePass>(promoteSingleIter);
|
return std::make_unique<AffineLoopNormalizePass>(promoteSingleIter);
|
||||||
}
|
}
|
||||||
|
@ -27,18 +27,21 @@
|
|||||||
#include <deque>
|
#include <deque>
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
#define GEN_PASS_DEF_AFFINEPARALLELIZE
|
#define GEN_PASS_DEF_AFFINEPARALLELIZE
|
||||||
#include "mlir/Dialect/Affine/Passes.h.inc"
|
#include "mlir/Dialect/Affine/Passes.h.inc"
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#define DEBUG_TYPE "affine-parallel"
|
#define DEBUG_TYPE "affine-parallel"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
/// Convert all parallel affine.for op into 1-D affine.parallel op.
|
/// Convert all parallel affine.for op into 1-D affine.parallel op.
|
||||||
struct AffineParallelize
|
struct AffineParallelize
|
||||||
: public impl::AffineParallelizeBase<AffineParallelize> {
|
: public affine::impl::AffineParallelizeBase<AffineParallelize> {
|
||||||
void runOnOperation() override;
|
void runOnOperation() override;
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -89,6 +92,6 @@ void AffineParallelize::runOnOperation() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
std::unique_ptr<OperationPass<func::FuncOp>>
|
std::unique_ptr<OperationPass<func::FuncOp>>
|
||||||
mlir::createAffineParallelizePass() {
|
mlir::affine::createAffineParallelizePass() {
|
||||||
return std::make_unique<AffineParallelize>();
|
return std::make_unique<AffineParallelize>();
|
||||||
}
|
}
|
||||||
|
@ -23,24 +23,28 @@
|
|||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
#define GEN_PASS_DEF_AFFINESCALARREPLACEMENT
|
#define GEN_PASS_DEF_AFFINESCALARREPLACEMENT
|
||||||
#include "mlir/Dialect/Affine/Passes.h.inc"
|
#include "mlir/Dialect/Affine/Passes.h.inc"
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#define DEBUG_TYPE "affine-scalrep"
|
#define DEBUG_TYPE "affine-scalrep"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
struct AffineScalarReplacement
|
struct AffineScalarReplacement
|
||||||
: public impl::AffineScalarReplacementBase<AffineScalarReplacement> {
|
: public affine::impl::AffineScalarReplacementBase<
|
||||||
|
AffineScalarReplacement> {
|
||||||
void runOnOperation() override;
|
void runOnOperation() override;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
std::unique_ptr<OperationPass<func::FuncOp>>
|
std::unique_ptr<OperationPass<func::FuncOp>>
|
||||||
mlir::createAffineScalarReplacementPass() {
|
mlir::affine::createAffineScalarReplacementPass() {
|
||||||
return std::make_unique<AffineScalarReplacement>();
|
return std::make_unique<AffineScalarReplacement>();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -18,6 +18,7 @@
|
|||||||
#include "llvm/Support/Debug.h"
|
#include "llvm/Support/Debug.h"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
#define DEBUG_TYPE "decompose-affine-ops"
|
#define DEBUG_TYPE "decompose-affine-ops"
|
||||||
#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE "]: ")
|
#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE "]: ")
|
||||||
@ -38,8 +39,8 @@ static int64_t numEnclosingInvariantLoops(OpOperand &operand) {
|
|||||||
return count;
|
return count;
|
||||||
}
|
}
|
||||||
|
|
||||||
void mlir::reorderOperandsByHoistability(RewriterBase &rewriter,
|
void mlir::affine::reorderOperandsByHoistability(RewriterBase &rewriter,
|
||||||
AffineApplyOp op) {
|
AffineApplyOp op) {
|
||||||
SmallVector<int64_t> numInvariant = llvm::to_vector(
|
SmallVector<int64_t> numInvariant = llvm::to_vector(
|
||||||
llvm::map_range(op->getOpOperands(), [&](OpOperand &operand) {
|
llvm::map_range(op->getOpOperands(), [&](OpOperand &operand) {
|
||||||
return numEnclosingInvariantLoops(operand);
|
return numEnclosingInvariantLoops(operand);
|
||||||
@ -92,8 +93,8 @@ static AffineApplyOp createSubApply(RewriterBase &rewriter,
|
|||||||
rhsOperands);
|
rhsOperands);
|
||||||
}
|
}
|
||||||
|
|
||||||
FailureOr<AffineApplyOp> mlir::decompose(RewriterBase &rewriter,
|
FailureOr<AffineApplyOp> mlir::affine::decompose(RewriterBase &rewriter,
|
||||||
AffineApplyOp op) {
|
AffineApplyOp op) {
|
||||||
// 1. Preconditions: only handle dimensionless AffineApplyOp maps with a
|
// 1. Preconditions: only handle dimensionless AffineApplyOp maps with a
|
||||||
// top-level binary expression that we can reassociate (i.e. add or mul).
|
// top-level binary expression that we can reassociate (i.e. add or mul).
|
||||||
AffineMap m = op.getAffineMap();
|
AffineMap m = op.getAffineMap();
|
||||||
|
@ -19,18 +19,21 @@
|
|||||||
#include "llvm/Support/Debug.h"
|
#include "llvm/Support/Debug.h"
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
#define GEN_PASS_DEF_LOOPCOALESCING
|
#define GEN_PASS_DEF_LOOPCOALESCING
|
||||||
#include "mlir/Dialect/Affine/Passes.h.inc"
|
#include "mlir/Dialect/Affine/Passes.h.inc"
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#define PASS_NAME "loop-coalescing"
|
#define PASS_NAME "loop-coalescing"
|
||||||
#define DEBUG_TYPE PASS_NAME
|
#define DEBUG_TYPE PASS_NAME
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
struct LoopCoalescingPass
|
struct LoopCoalescingPass
|
||||||
: public impl::LoopCoalescingBase<LoopCoalescingPass> {
|
: public affine::impl::LoopCoalescingBase<LoopCoalescingPass> {
|
||||||
|
|
||||||
void runOnOperation() override {
|
void runOnOperation() override {
|
||||||
func::FuncOp func = getOperation();
|
func::FuncOp func = getOperation();
|
||||||
@ -45,6 +48,7 @@ struct LoopCoalescingPass
|
|||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
std::unique_ptr<OperationPass<func::FuncOp>> mlir::createLoopCoalescingPass() {
|
std::unique_ptr<OperationPass<func::FuncOp>>
|
||||||
|
mlir::affine::createLoopCoalescingPass() {
|
||||||
return std::make_unique<LoopCoalescingPass>();
|
return std::make_unique<LoopCoalescingPass>();
|
||||||
}
|
}
|
||||||
|
@ -36,13 +36,16 @@
|
|||||||
#include <sstream>
|
#include <sstream>
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
#define GEN_PASS_DEF_AFFINELOOPFUSION
|
#define GEN_PASS_DEF_AFFINELOOPFUSION
|
||||||
#include "mlir/Dialect/Affine/Passes.h.inc"
|
#include "mlir/Dialect/Affine/Passes.h.inc"
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#define DEBUG_TYPE "affine-loop-fusion"
|
#define DEBUG_TYPE "affine-loop-fusion"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
/// Loop fusion pass. This pass currently supports a greedy fusion policy,
|
/// Loop fusion pass. This pass currently supports a greedy fusion policy,
|
||||||
@ -54,7 +57,7 @@ namespace {
|
|||||||
// TODO: Extend this pass to check for fusion preventing dependences,
|
// TODO: Extend this pass to check for fusion preventing dependences,
|
||||||
// and add support for more general loop fusion algorithms.
|
// and add support for more general loop fusion algorithms.
|
||||||
|
|
||||||
struct LoopFusion : public impl::AffineLoopFusionBase<LoopFusion> {
|
struct LoopFusion : public affine::impl::AffineLoopFusionBase<LoopFusion> {
|
||||||
LoopFusion() = default;
|
LoopFusion() = default;
|
||||||
LoopFusion(unsigned fastMemorySpace, uint64_t localBufSizeThresholdBytes,
|
LoopFusion(unsigned fastMemorySpace, uint64_t localBufSizeThresholdBytes,
|
||||||
bool maximalFusion, enum FusionMode affineFusionMode) {
|
bool maximalFusion, enum FusionMode affineFusionMode) {
|
||||||
@ -1039,7 +1042,7 @@ public:
|
|||||||
depthSliceUnions.resize(dstLoopDepthTest);
|
depthSliceUnions.resize(dstLoopDepthTest);
|
||||||
FusionStrategy strategy(FusionStrategy::ProducerConsumer);
|
FusionStrategy strategy(FusionStrategy::ProducerConsumer);
|
||||||
for (unsigned i = 1; i <= dstLoopDepthTest; ++i) {
|
for (unsigned i = 1; i <= dstLoopDepthTest; ++i) {
|
||||||
FusionResult result = mlir::canFuseLoops(
|
FusionResult result = affine::canFuseLoops(
|
||||||
srcAffineForOp, dstAffineForOp,
|
srcAffineForOp, dstAffineForOp,
|
||||||
/*dstLoopDepth=*/i, &depthSliceUnions[i - 1], strategy);
|
/*dstLoopDepth=*/i, &depthSliceUnions[i - 1], strategy);
|
||||||
|
|
||||||
@ -1259,7 +1262,7 @@ public:
|
|||||||
unsigned maxLegalFusionDepth = 0;
|
unsigned maxLegalFusionDepth = 0;
|
||||||
FusionStrategy strategy(memref);
|
FusionStrategy strategy(memref);
|
||||||
for (unsigned i = 1; i <= dstLoopDepthTest; ++i) {
|
for (unsigned i = 1; i <= dstLoopDepthTest; ++i) {
|
||||||
FusionResult result = mlir::canFuseLoops(
|
FusionResult result = affine::canFuseLoops(
|
||||||
sibAffineForOp, dstAffineForOp,
|
sibAffineForOp, dstAffineForOp,
|
||||||
/*dstLoopDepth=*/i, &depthSliceUnions[i - 1], strategy);
|
/*dstLoopDepth=*/i, &depthSliceUnions[i - 1], strategy);
|
||||||
|
|
||||||
@ -1291,9 +1294,9 @@ public:
|
|||||||
// further inside `fuseLoops`.
|
// further inside `fuseLoops`.
|
||||||
bool isInnermostInsertion = (bestDstLoopDepth == dstLoopDepthTest);
|
bool isInnermostInsertion = (bestDstLoopDepth == dstLoopDepthTest);
|
||||||
// Fuse computation slice of 'sibLoopNest' into 'dstLoopNest'.
|
// Fuse computation slice of 'sibLoopNest' into 'dstLoopNest'.
|
||||||
mlir::fuseLoops(sibAffineForOp, dstAffineForOp,
|
affine::fuseLoops(sibAffineForOp, dstAffineForOp,
|
||||||
depthSliceUnions[bestDstLoopDepth - 1],
|
depthSliceUnions[bestDstLoopDepth - 1],
|
||||||
isInnermostInsertion);
|
isInnermostInsertion);
|
||||||
|
|
||||||
auto dstForInst = cast<AffineForOp>(dstNode->op);
|
auto dstForInst = cast<AffineForOp>(dstNode->op);
|
||||||
// Update operation position of fused loop nest (if needed).
|
// Update operation position of fused loop nest (if needed).
|
||||||
@ -1501,10 +1504,9 @@ void LoopFusion::runOnOperation() {
|
|||||||
runOnBlock(&block);
|
runOnBlock(&block);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::unique_ptr<Pass>
|
std::unique_ptr<Pass> mlir::affine::createLoopFusionPass(
|
||||||
mlir::createLoopFusionPass(unsigned fastMemorySpace,
|
unsigned fastMemorySpace, uint64_t localBufSizeThreshold,
|
||||||
uint64_t localBufSizeThreshold, bool maximalFusion,
|
bool maximalFusion, enum FusionMode affineFusionMode) {
|
||||||
enum FusionMode affineFusionMode) {
|
|
||||||
return std::make_unique<LoopFusion>(fastMemorySpace, localBufSizeThreshold,
|
return std::make_unique<LoopFusion>(fastMemorySpace, localBufSizeThreshold,
|
||||||
maximalFusion, affineFusionMode);
|
maximalFusion, affineFusionMode);
|
||||||
}
|
}
|
||||||
|
@ -28,18 +28,21 @@
|
|||||||
#include <optional>
|
#include <optional>
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
#define GEN_PASS_DEF_AFFINELOOPTILING
|
#define GEN_PASS_DEF_AFFINELOOPTILING
|
||||||
#include "mlir/Dialect/Affine/Passes.h.inc"
|
#include "mlir/Dialect/Affine/Passes.h.inc"
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
#define DEBUG_TYPE "affine-loop-tile"
|
#define DEBUG_TYPE "affine-loop-tile"
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
/// A pass to perform loop tiling on all suitable loop nests of a Function.
|
/// A pass to perform loop tiling on all suitable loop nests of a Function.
|
||||||
struct LoopTiling : public impl::AffineLoopTilingBase<LoopTiling> {
|
struct LoopTiling : public affine::impl::AffineLoopTilingBase<LoopTiling> {
|
||||||
LoopTiling() = default;
|
LoopTiling() = default;
|
||||||
explicit LoopTiling(uint64_t cacheSizeBytes, bool avoidMaxMinBounds = true)
|
explicit LoopTiling(uint64_t cacheSizeBytes, bool avoidMaxMinBounds = true)
|
||||||
: avoidMaxMinBounds(avoidMaxMinBounds) {
|
: avoidMaxMinBounds(avoidMaxMinBounds) {
|
||||||
@ -62,10 +65,11 @@ struct LoopTiling : public impl::AffineLoopTilingBase<LoopTiling> {
|
|||||||
/// Creates a pass to perform loop tiling on all suitable loop nests of a
|
/// Creates a pass to perform loop tiling on all suitable loop nests of a
|
||||||
/// Function.
|
/// Function.
|
||||||
std::unique_ptr<OperationPass<func::FuncOp>>
|
std::unique_ptr<OperationPass<func::FuncOp>>
|
||||||
mlir::createLoopTilingPass(uint64_t cacheSizeBytes) {
|
mlir::affine::createLoopTilingPass(uint64_t cacheSizeBytes) {
|
||||||
return std::make_unique<LoopTiling>(cacheSizeBytes);
|
return std::make_unique<LoopTiling>(cacheSizeBytes);
|
||||||
}
|
}
|
||||||
std::unique_ptr<OperationPass<func::FuncOp>> mlir::createLoopTilingPass() {
|
std::unique_ptr<OperationPass<func::FuncOp>>
|
||||||
|
mlir::affine::createLoopTilingPass() {
|
||||||
return std::make_unique<LoopTiling>();
|
return std::make_unique<LoopTiling>();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -97,7 +101,7 @@ static void adjustToDivisorsOfTripCounts(ArrayRef<AffineForOp> band,
|
|||||||
/// hyper-rectangles, which are scheduled in the lexicographically increasing
|
/// hyper-rectangles, which are scheduled in the lexicographically increasing
|
||||||
/// order on the vector of loop indices. This function will return failure when
|
/// order on the vector of loop indices. This function will return failure when
|
||||||
/// any dependence component is negative along any of `origLoops`.
|
/// any dependence component is negative along any of `origLoops`.
|
||||||
static bool checkTilingLegality(MutableArrayRef<mlir::AffineForOp> origLoops) {
|
static bool checkTilingLegality(MutableArrayRef<AffineForOp> origLoops) {
|
||||||
assert(!origLoops.empty() && "no original loops provided");
|
assert(!origLoops.empty() && "no original loops provided");
|
||||||
|
|
||||||
// We first find out all dependences we intend to check.
|
// We first find out all dependences we intend to check.
|
||||||
|
@ -25,13 +25,16 @@
|
|||||||
#include <optional>
|
#include <optional>
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
#define GEN_PASS_DEF_AFFINELOOPUNROLL
|
#define GEN_PASS_DEF_AFFINELOOPUNROLL
|
||||||
#include "mlir/Dialect/Affine/Passes.h.inc"
|
#include "mlir/Dialect/Affine/Passes.h.inc"
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#define DEBUG_TYPE "affine-loop-unroll"
|
#define DEBUG_TYPE "affine-loop-unroll"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
@ -42,7 +45,7 @@ namespace {
|
|||||||
/// full unroll threshold was specified, in which case, fully unrolls all loops
|
/// full unroll threshold was specified, in which case, fully unrolls all loops
|
||||||
/// with trip count less than the specified threshold. The latter is for testing
|
/// with trip count less than the specified threshold. The latter is for testing
|
||||||
/// purposes, especially for testing outer loop unrolling.
|
/// purposes, especially for testing outer loop unrolling.
|
||||||
struct LoopUnroll : public impl::AffineLoopUnrollBase<LoopUnroll> {
|
struct LoopUnroll : public affine::impl::AffineLoopUnrollBase<LoopUnroll> {
|
||||||
// Callback to obtain unroll factors; if this has a callable target, takes
|
// Callback to obtain unroll factors; if this has a callable target, takes
|
||||||
// precedence over command-line argument or passed argument.
|
// precedence over command-line argument or passed argument.
|
||||||
const std::function<unsigned(AffineForOp)> getUnrollFactor;
|
const std::function<unsigned(AffineForOp)> getUnrollFactor;
|
||||||
@ -142,7 +145,7 @@ LogicalResult LoopUnroll::runOnAffineForOp(AffineForOp forOp) {
|
|||||||
cleanUpUnroll);
|
cleanUpUnroll);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::unique_ptr<OperationPass<func::FuncOp>> mlir::createLoopUnrollPass(
|
std::unique_ptr<OperationPass<func::FuncOp>> mlir::affine::createLoopUnrollPass(
|
||||||
int unrollFactor, bool unrollUpToFactor, bool unrollFull,
|
int unrollFactor, bool unrollUpToFactor, bool unrollFull,
|
||||||
const std::function<unsigned(AffineForOp)> &getUnrollFactor) {
|
const std::function<unsigned(AffineForOp)> &getUnrollFactor) {
|
||||||
return std::make_unique<LoopUnroll>(
|
return std::make_unique<LoopUnroll>(
|
||||||
|
@ -49,19 +49,22 @@
|
|||||||
#include <optional>
|
#include <optional>
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
#define GEN_PASS_DEF_AFFINELOOPUNROLLANDJAM
|
#define GEN_PASS_DEF_AFFINELOOPUNROLLANDJAM
|
||||||
#include "mlir/Dialect/Affine/Passes.h.inc"
|
#include "mlir/Dialect/Affine/Passes.h.inc"
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#define DEBUG_TYPE "affine-loop-unroll-jam"
|
#define DEBUG_TYPE "affine-loop-unroll-jam"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
/// Loop unroll jam pass. Currently, this just unroll jams the first
|
/// Loop unroll jam pass. Currently, this just unroll jams the first
|
||||||
/// outer loop in a Function.
|
/// outer loop in a Function.
|
||||||
struct LoopUnrollAndJam
|
struct LoopUnrollAndJam
|
||||||
: public impl::AffineLoopUnrollAndJamBase<LoopUnrollAndJam> {
|
: public affine::impl::AffineLoopUnrollAndJamBase<LoopUnrollAndJam> {
|
||||||
explicit LoopUnrollAndJam(
|
explicit LoopUnrollAndJam(
|
||||||
std::optional<unsigned> unrollJamFactor = std::nullopt) {
|
std::optional<unsigned> unrollJamFactor = std::nullopt) {
|
||||||
if (unrollJamFactor)
|
if (unrollJamFactor)
|
||||||
@ -73,7 +76,7 @@ struct LoopUnrollAndJam
|
|||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
std::unique_ptr<OperationPass<func::FuncOp>>
|
std::unique_ptr<OperationPass<func::FuncOp>>
|
||||||
mlir::createLoopUnrollAndJamPass(int unrollJamFactor) {
|
mlir::affine::createLoopUnrollAndJamPass(int unrollJamFactor) {
|
||||||
return std::make_unique<LoopUnrollAndJam>(
|
return std::make_unique<LoopUnrollAndJam>(
|
||||||
unrollJamFactor == -1 ? std::nullopt
|
unrollJamFactor == -1 ? std::nullopt
|
||||||
: std::optional<unsigned>(unrollJamFactor));
|
: std::optional<unsigned>(unrollJamFactor));
|
||||||
|
@ -27,17 +27,21 @@
|
|||||||
#include "llvm/Support/Debug.h"
|
#include "llvm/Support/Debug.h"
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
#define GEN_PASS_DEF_AFFINEPIPELINEDATATRANSFER
|
#define GEN_PASS_DEF_AFFINEPIPELINEDATATRANSFER
|
||||||
#include "mlir/Dialect/Affine/Passes.h.inc"
|
#include "mlir/Dialect/Affine/Passes.h.inc"
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#define DEBUG_TYPE "affine-pipeline-data-transfer"
|
#define DEBUG_TYPE "affine-pipeline-data-transfer"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
struct PipelineDataTransfer
|
struct PipelineDataTransfer
|
||||||
: public impl::AffinePipelineDataTransferBase<PipelineDataTransfer> {
|
: public affine::impl::AffinePipelineDataTransferBase<
|
||||||
|
PipelineDataTransfer> {
|
||||||
void runOnOperation() override;
|
void runOnOperation() override;
|
||||||
void runOnAffineForOp(AffineForOp forOp);
|
void runOnAffineForOp(AffineForOp forOp);
|
||||||
|
|
||||||
@ -49,7 +53,7 @@ struct PipelineDataTransfer
|
|||||||
/// Creates a pass to pipeline explicit movement of data across levels of the
|
/// Creates a pass to pipeline explicit movement of data across levels of the
|
||||||
/// memory hierarchy.
|
/// memory hierarchy.
|
||||||
std::unique_ptr<OperationPass<func::FuncOp>>
|
std::unique_ptr<OperationPass<func::FuncOp>>
|
||||||
mlir::createPipelineDataTransferPass() {
|
mlir::affine::createPipelineDataTransferPass() {
|
||||||
return std::make_unique<PipelineDataTransfer>();
|
return std::make_unique<PipelineDataTransfer>();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -328,7 +332,7 @@ void PipelineDataTransfer::runOnAffineForOp(AffineForOp forOp) {
|
|||||||
instShiftMap[dmaStartOp] = 0;
|
instShiftMap[dmaStartOp] = 0;
|
||||||
// Set shifts for DMA start op's affine operand computation slices to 0.
|
// Set shifts for DMA start op's affine operand computation slices to 0.
|
||||||
SmallVector<AffineApplyOp, 4> sliceOps;
|
SmallVector<AffineApplyOp, 4> sliceOps;
|
||||||
mlir::createAffineComputationSlice(dmaStartOp, &sliceOps);
|
affine::createAffineComputationSlice(dmaStartOp, &sliceOps);
|
||||||
if (!sliceOps.empty()) {
|
if (!sliceOps.empty()) {
|
||||||
for (auto sliceOp : sliceOps) {
|
for (auto sliceOp : sliceOps) {
|
||||||
instShiftMap[sliceOp.getOperation()] = 0;
|
instShiftMap[sliceOp.getOperation()] = 0;
|
||||||
|
@ -14,6 +14,7 @@
|
|||||||
#include "mlir/Interfaces/ValueBoundsOpInterface.h"
|
#include "mlir/Interfaces/ValueBoundsOpInterface.h"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
static FailureOr<OpFoldResult>
|
static FailureOr<OpFoldResult>
|
||||||
reifyValueBound(OpBuilder &b, Location loc, presburger::BoundType type,
|
reifyValueBound(OpBuilder &b, Location loc, presburger::BoundType type,
|
||||||
@ -54,7 +55,7 @@ reifyValueBound(OpBuilder &b, Location loc, presburger::BoundType type,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Simplify and return bound.
|
// Simplify and return bound.
|
||||||
mlir::canonicalizeMapAndOperands(&boundMap, &operands);
|
affine::canonicalizeMapAndOperands(&boundMap, &operands);
|
||||||
// Check for special cases where no affine.apply op is needed.
|
// Check for special cases where no affine.apply op is needed.
|
||||||
if (boundMap.isSingleConstant()) {
|
if (boundMap.isSingleConstant()) {
|
||||||
// Bound is a constant: return an IntegerAttr.
|
// Bound is a constant: return an IntegerAttr.
|
||||||
@ -69,10 +70,10 @@ reifyValueBound(OpBuilder &b, Location loc, presburger::BoundType type,
|
|||||||
operands[expr.getPosition() + boundMap.getNumDims()]);
|
operands[expr.getPosition() + boundMap.getNumDims()]);
|
||||||
// General case: build affine.apply op.
|
// General case: build affine.apply op.
|
||||||
return static_cast<OpFoldResult>(
|
return static_cast<OpFoldResult>(
|
||||||
b.create<AffineApplyOp>(loc, boundMap, operands).getResult());
|
b.create<affine::AffineApplyOp>(loc, boundMap, operands).getResult());
|
||||||
}
|
}
|
||||||
|
|
||||||
FailureOr<OpFoldResult> mlir::reifyShapedValueDimBound(
|
FailureOr<OpFoldResult> mlir::affine::reifyShapedValueDimBound(
|
||||||
OpBuilder &b, Location loc, presburger::BoundType type, Value value,
|
OpBuilder &b, Location loc, presburger::BoundType type, Value value,
|
||||||
int64_t dim, ValueBoundsConstraintSet::StopConditionFn stopCondition,
|
int64_t dim, ValueBoundsConstraintSet::StopConditionFn stopCondition,
|
||||||
bool closedUB) {
|
bool closedUB) {
|
||||||
@ -89,7 +90,7 @@ FailureOr<OpFoldResult> mlir::reifyShapedValueDimBound(
|
|||||||
closedUB);
|
closedUB);
|
||||||
}
|
}
|
||||||
|
|
||||||
FailureOr<OpFoldResult> mlir::reifyIndexValueBound(
|
FailureOr<OpFoldResult> mlir::affine::reifyIndexValueBound(
|
||||||
OpBuilder &b, Location loc, presburger::BoundType type, Value value,
|
OpBuilder &b, Location loc, presburger::BoundType type, Value value,
|
||||||
ValueBoundsConstraintSet::StopConditionFn stopCondition, bool closedUB) {
|
ValueBoundsConstraintSet::StopConditionFn stopCondition, bool closedUB) {
|
||||||
auto reifyToOperands = [&](Value v, std::optional<int64_t> d) {
|
auto reifyToOperands = [&](Value v, std::optional<int64_t> d) {
|
||||||
|
@ -20,13 +20,16 @@
|
|||||||
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
|
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
#define GEN_PASS_DEF_SIMPLIFYAFFINESTRUCTURES
|
#define GEN_PASS_DEF_SIMPLIFYAFFINESTRUCTURES
|
||||||
#include "mlir/Dialect/Affine/Passes.h.inc"
|
#include "mlir/Dialect/Affine/Passes.h.inc"
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
#define DEBUG_TYPE "simplify-affine-structure"
|
#define DEBUG_TYPE "simplify-affine-structure"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
@ -35,7 +38,8 @@ namespace {
|
|||||||
/// all memrefs with non-trivial layout maps are converted to ones with trivial
|
/// all memrefs with non-trivial layout maps are converted to ones with trivial
|
||||||
/// identity layout ones.
|
/// identity layout ones.
|
||||||
struct SimplifyAffineStructures
|
struct SimplifyAffineStructures
|
||||||
: public impl::SimplifyAffineStructuresBase<SimplifyAffineStructures> {
|
: public affine::impl::SimplifyAffineStructuresBase<
|
||||||
|
SimplifyAffineStructures> {
|
||||||
void runOnOperation() override;
|
void runOnOperation() override;
|
||||||
|
|
||||||
/// Utility to simplify an affine attribute and update its entry in the parent
|
/// Utility to simplify an affine attribute and update its entry in the parent
|
||||||
@ -78,7 +82,7 @@ struct SimplifyAffineStructures
|
|||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
std::unique_ptr<OperationPass<func::FuncOp>>
|
std::unique_ptr<OperationPass<func::FuncOp>>
|
||||||
mlir::createSimplifyAffineStructuresPass() {
|
mlir::affine::createSimplifyAffineStructuresPass() {
|
||||||
return std::make_unique<SimplifyAffineStructures>();
|
return std::make_unique<SimplifyAffineStructures>();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -31,11 +31,14 @@
|
|||||||
#include <optional>
|
#include <optional>
|
||||||
|
|
||||||
namespace mlir {
|
namespace mlir {
|
||||||
|
namespace affine {
|
||||||
#define GEN_PASS_DEF_AFFINEVECTORIZE
|
#define GEN_PASS_DEF_AFFINEVECTORIZE
|
||||||
#include "mlir/Dialect/Affine/Passes.h.inc"
|
#include "mlir/Dialect/Affine/Passes.h.inc"
|
||||||
|
} // namespace affine
|
||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace affine;
|
||||||
using namespace vector;
|
using namespace vector;
|
||||||
|
|
||||||
///
|
///
|
||||||
@ -585,7 +588,7 @@ isVectorizableLoopPtrFactory(const DenseSet<Operation *> ¶llelLoops,
|
|||||||
static std::optional<NestedPattern>
|
static std::optional<NestedPattern>
|
||||||
makePattern(const DenseSet<Operation *> ¶llelLoops, int vectorRank,
|
makePattern(const DenseSet<Operation *> ¶llelLoops, int vectorRank,
|
||||||
ArrayRef<int64_t> fastestVaryingPattern) {
|
ArrayRef<int64_t> fastestVaryingPattern) {
|
||||||
using matcher::For;
|
using affine::matcher::For;
|
||||||
int64_t d0 = fastestVaryingPattern.empty() ? -1 : fastestVaryingPattern[0];
|
int64_t d0 = fastestVaryingPattern.empty() ? -1 : fastestVaryingPattern[0];
|
||||||
int64_t d1 = fastestVaryingPattern.size() < 2 ? -1 : fastestVaryingPattern[1];
|
int64_t d1 = fastestVaryingPattern.size() < 2 ? -1 : fastestVaryingPattern[1];
|
||||||
int64_t d2 = fastestVaryingPattern.size() < 3 ? -1 : fastestVaryingPattern[2];
|
int64_t d2 = fastestVaryingPattern.size() < 3 ? -1 : fastestVaryingPattern[2];
|
||||||
@ -606,7 +609,7 @@ makePattern(const DenseSet<Operation *> ¶llelLoops, int vectorRank,
|
|||||||
}
|
}
|
||||||
|
|
||||||
static NestedPattern &vectorTransferPattern() {
|
static NestedPattern &vectorTransferPattern() {
|
||||||
static auto pattern = matcher::Op([](Operation &op) {
|
static auto pattern = affine::matcher::Op([](Operation &op) {
|
||||||
return isa<vector::TransferReadOp, vector::TransferWriteOp>(op);
|
return isa<vector::TransferReadOp, vector::TransferWriteOp>(op);
|
||||||
});
|
});
|
||||||
return pattern;
|
return pattern;
|
||||||
@ -616,7 +619,7 @@ namespace {
|
|||||||
|
|
||||||
/// Base state for the vectorize pass.
|
/// Base state for the vectorize pass.
|
||||||
/// Command line arguments are preempted by non-empty pass arguments.
|
/// Command line arguments are preempted by non-empty pass arguments.
|
||||||
struct Vectorize : public impl::AffineVectorizeBase<Vectorize> {
|
struct Vectorize : public affine::impl::AffineVectorizeBase<Vectorize> {
|
||||||
using Base::Base;
|
using Base::Base;
|
||||||
|
|
||||||
void runOnOperation() override;
|
void runOnOperation() override;
|
||||||
@ -1796,7 +1799,6 @@ verifyLoopNesting(const std::vector<SmallVector<AffineForOp, 2>> &loops) {
|
|||||||
return success();
|
return success();
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace mlir {
|
|
||||||
|
|
||||||
/// External utility to vectorize affine loops in 'loops' using the n-D
|
/// External utility to vectorize affine loops in 'loops' using the n-D
|
||||||
/// vectorization factors in 'vectorSizes'. By default, each vectorization
|
/// vectorization factors in 'vectorSizes'. By default, each vectorization
|
||||||
@ -1806,10 +1808,10 @@ namespace mlir {
|
|||||||
/// If `reductionLoops` is not empty, the given reduction loops may be
|
/// If `reductionLoops` is not empty, the given reduction loops may be
|
||||||
/// vectorized along the reduction dimension.
|
/// vectorized along the reduction dimension.
|
||||||
/// TODO: Vectorizing reductions is supported only for 1-D vectorization.
|
/// TODO: Vectorizing reductions is supported only for 1-D vectorization.
|
||||||
void vectorizeAffineLoops(Operation *parentOp, DenseSet<Operation *> &loops,
|
void mlir::affine::vectorizeAffineLoops(
|
||||||
ArrayRef<int64_t> vectorSizes,
|
Operation *parentOp, DenseSet<Operation *> &loops,
|
||||||
ArrayRef<int64_t> fastestVaryingPattern,
|
ArrayRef<int64_t> vectorSizes, ArrayRef<int64_t> fastestVaryingPattern,
|
||||||
const ReductionLoopMap &reductionLoops) {
|
const ReductionLoopMap &reductionLoops) {
|
||||||
// Thread-safe RAII local context, BumpPtrAllocator freed on exit.
|
// Thread-safe RAII local context, BumpPtrAllocator freed on exit.
|
||||||
NestedPatternContext mlContext;
|
NestedPatternContext mlContext;
|
||||||
vectorizeLoops(parentOp, loops, vectorSizes, fastestVaryingPattern,
|
vectorizeLoops(parentOp, loops, vectorSizes, fastestVaryingPattern,
|
||||||
@ -1851,14 +1853,12 @@ void vectorizeAffineLoops(Operation *parentOp, DenseSet<Operation *> &loops,
|
|||||||
/// loops = {{%i2}}, to vectorize only the first innermost loop;
|
/// loops = {{%i2}}, to vectorize only the first innermost loop;
|
||||||
/// loops = {{%i3}}, to vectorize only the second innermost loop;
|
/// loops = {{%i3}}, to vectorize only the second innermost loop;
|
||||||
/// loops = {{%i1}}, to vectorize only the middle loop.
|
/// loops = {{%i1}}, to vectorize only the middle loop.
|
||||||
LogicalResult
|
LogicalResult mlir::affine::vectorizeAffineLoopNest(
|
||||||
vectorizeAffineLoopNest(std::vector<SmallVector<AffineForOp, 2>> &loops,
|
std::vector<SmallVector<AffineForOp, 2>> &loops,
|
||||||
const VectorizationStrategy &strategy) {
|
const VectorizationStrategy &strategy) {
|
||||||
// Thread-safe RAII local context, BumpPtrAllocator freed on exit.
|
// Thread-safe RAII local context, BumpPtrAllocator freed on exit.
|
||||||
NestedPatternContext mlContext;
|
NestedPatternContext mlContext;
|
||||||
if (failed(verifyLoopNesting(loops)))
|
if (failed(verifyLoopNesting(loops)))
|
||||||
return failure();
|
return failure();
|
||||||
return vectorizeLoopNest(loops, strategy);
|
return vectorizeLoopNest(loops, strategy);
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace mlir
|
|
||||||
|
@ -26,6 +26,7 @@
|
|||||||
#define DEBUG_TYPE "loop-fusion-utils"
|
#define DEBUG_TYPE "loop-fusion-utils"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
|
|
||||||
// Gathers all load and store memref accesses in 'opA' into 'values', where
|
// Gathers all load and store memref accesses in 'opA' into 'values', where
|
||||||
// 'values[memref] == true' for each store operation.
|
// 'values[memref] == true' for each store operation.
|
||||||
@ -245,10 +246,11 @@ static unsigned getMaxLoopDepth(ArrayRef<Operation *> srcOps,
|
|||||||
// TODO: This pass performs some computation that is the same for all the depths
|
// TODO: This pass performs some computation that is the same for all the depths
|
||||||
// (e.g., getMaxLoopDepth). Implement a version of this utility that processes
|
// (e.g., getMaxLoopDepth). Implement a version of this utility that processes
|
||||||
// all the depths at once or only the legal maximal depth for maximal fusion.
|
// all the depths at once or only the legal maximal depth for maximal fusion.
|
||||||
FusionResult mlir::canFuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
|
FusionResult mlir::affine::canFuseLoops(AffineForOp srcForOp,
|
||||||
unsigned dstLoopDepth,
|
AffineForOp dstForOp,
|
||||||
ComputationSliceState *srcSlice,
|
unsigned dstLoopDepth,
|
||||||
FusionStrategy fusionStrategy) {
|
ComputationSliceState *srcSlice,
|
||||||
|
FusionStrategy fusionStrategy) {
|
||||||
// Return 'failure' if 'dstLoopDepth == 0'.
|
// Return 'failure' if 'dstLoopDepth == 0'.
|
||||||
if (dstLoopDepth == 0) {
|
if (dstLoopDepth == 0) {
|
||||||
LLVM_DEBUG(llvm::dbgs() << "Cannot fuse loop nests at depth 0\n");
|
LLVM_DEBUG(llvm::dbgs() << "Cannot fuse loop nests at depth 0\n");
|
||||||
@ -303,7 +305,7 @@ FusionResult mlir::canFuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
|
|||||||
|
|
||||||
// Calculate the number of common loops surrounding 'srcForOp' and 'dstForOp'.
|
// Calculate the number of common loops surrounding 'srcForOp' and 'dstForOp'.
|
||||||
unsigned numCommonLoops =
|
unsigned numCommonLoops =
|
||||||
mlir::getNumCommonSurroundingLoops(*srcForOp, *dstForOp);
|
affine::getNumCommonSurroundingLoops(*srcForOp, *dstForOp);
|
||||||
|
|
||||||
// Filter out ops in 'opsA' to compute the slice union based on the
|
// Filter out ops in 'opsA' to compute the slice union based on the
|
||||||
// assumptions made by the fusion strategy.
|
// assumptions made by the fusion strategy.
|
||||||
@ -335,9 +337,9 @@ FusionResult mlir::canFuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
|
|||||||
|
|
||||||
// Compute union of computation slices computed between all pairs of ops
|
// Compute union of computation slices computed between all pairs of ops
|
||||||
// from 'forOpA' and 'forOpB'.
|
// from 'forOpA' and 'forOpB'.
|
||||||
SliceComputationResult sliceComputationResult =
|
SliceComputationResult sliceComputationResult = affine::computeSliceUnion(
|
||||||
mlir::computeSliceUnion(strategyOpsA, opsB, dstLoopDepth, numCommonLoops,
|
strategyOpsA, opsB, dstLoopDepth, numCommonLoops,
|
||||||
isSrcForOpBeforeDstForOp, srcSlice);
|
isSrcForOpBeforeDstForOp, srcSlice);
|
||||||
if (sliceComputationResult.value == SliceComputationResult::GenericFailure) {
|
if (sliceComputationResult.value == SliceComputationResult::GenericFailure) {
|
||||||
LLVM_DEBUG(llvm::dbgs() << "computeSliceUnion failed\n");
|
LLVM_DEBUG(llvm::dbgs() << "computeSliceUnion failed\n");
|
||||||
return FusionResult::FailPrecondition;
|
return FusionResult::FailPrecondition;
|
||||||
@ -353,8 +355,8 @@ FusionResult mlir::canFuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
|
|||||||
|
|
||||||
/// Patch the loop body of a forOp that is a single iteration reduction loop
|
/// Patch the loop body of a forOp that is a single iteration reduction loop
|
||||||
/// into its containing block.
|
/// into its containing block.
|
||||||
LogicalResult promoteSingleIterReductionLoop(AffineForOp forOp,
|
static LogicalResult promoteSingleIterReductionLoop(AffineForOp forOp,
|
||||||
bool siblingFusionUser) {
|
bool siblingFusionUser) {
|
||||||
// Check if the reduction loop is a single iteration loop.
|
// Check if the reduction loop is a single iteration loop.
|
||||||
std::optional<uint64_t> tripCount = getConstantTripCount(forOp);
|
std::optional<uint64_t> tripCount = getConstantTripCount(forOp);
|
||||||
if (!tripCount || *tripCount != 1)
|
if (!tripCount || *tripCount != 1)
|
||||||
@ -416,9 +418,9 @@ LogicalResult promoteSingleIterReductionLoop(AffineForOp forOp,
|
|||||||
|
|
||||||
/// Fuses 'srcForOp' into 'dstForOp' with destination loop block insertion point
|
/// Fuses 'srcForOp' into 'dstForOp' with destination loop block insertion point
|
||||||
/// and source slice loop bounds specified in 'srcSlice'.
|
/// and source slice loop bounds specified in 'srcSlice'.
|
||||||
void mlir::fuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
|
void mlir::affine::fuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
|
||||||
const ComputationSliceState &srcSlice,
|
const ComputationSliceState &srcSlice,
|
||||||
bool isInnermostSiblingInsertion) {
|
bool isInnermostSiblingInsertion) {
|
||||||
// Clone 'srcForOp' into 'dstForOp' at 'srcSlice->insertPoint'.
|
// Clone 'srcForOp' into 'dstForOp' at 'srcSlice->insertPoint'.
|
||||||
OpBuilder b(srcSlice.insertPoint->getBlock(), srcSlice.insertPoint);
|
OpBuilder b(srcSlice.insertPoint->getBlock(), srcSlice.insertPoint);
|
||||||
IRMapping mapper;
|
IRMapping mapper;
|
||||||
@ -465,7 +467,8 @@ void mlir::fuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
|
|||||||
/// Collect loop nest statistics (eg. loop trip count and operation count)
|
/// Collect loop nest statistics (eg. loop trip count and operation count)
|
||||||
/// in 'stats' for loop nest rooted at 'forOp'. Returns true on success,
|
/// in 'stats' for loop nest rooted at 'forOp'. Returns true on success,
|
||||||
/// returns false otherwise.
|
/// returns false otherwise.
|
||||||
bool mlir::getLoopNestStats(AffineForOp forOpRoot, LoopNestStats *stats) {
|
bool mlir::affine::getLoopNestStats(AffineForOp forOpRoot,
|
||||||
|
LoopNestStats *stats) {
|
||||||
auto walkResult = forOpRoot.walk([&](AffineForOp forOp) {
|
auto walkResult = forOpRoot.walk([&](AffineForOp forOp) {
|
||||||
auto *childForOp = forOp.getOperation();
|
auto *childForOp = forOp.getOperation();
|
||||||
auto *parentForOp = forOp->getParentOp();
|
auto *parentForOp = forOp->getParentOp();
|
||||||
@ -553,7 +556,7 @@ static int64_t getComputeCostHelper(
|
|||||||
/// Currently, the total cost is computed by counting the total operation
|
/// Currently, the total cost is computed by counting the total operation
|
||||||
/// instance count (i.e. total number of operations in the loop body * loop
|
/// instance count (i.e. total number of operations in the loop body * loop
|
||||||
/// trip count) for the entire loop nest.
|
/// trip count) for the entire loop nest.
|
||||||
int64_t mlir::getComputeCost(AffineForOp forOp, LoopNestStats &stats) {
|
int64_t mlir::affine::getComputeCost(AffineForOp forOp, LoopNestStats &stats) {
|
||||||
return getComputeCostHelper(forOp, stats,
|
return getComputeCostHelper(forOp, stats,
|
||||||
/*tripCountOverrideMap=*/nullptr,
|
/*tripCountOverrideMap=*/nullptr,
|
||||||
/*computeCostMap=*/nullptr);
|
/*computeCostMap=*/nullptr);
|
||||||
@ -564,10 +567,12 @@ int64_t mlir::getComputeCost(AffineForOp forOp, LoopNestStats &stats) {
|
|||||||
/// the total cost is computed by counting the total operation instance count
|
/// the total cost is computed by counting the total operation instance count
|
||||||
/// (i.e. total number of operations in the loop body * loop trip count) for
|
/// (i.e. total number of operations in the loop body * loop trip count) for
|
||||||
/// the entire loop nest.
|
/// the entire loop nest.
|
||||||
bool mlir::getFusionComputeCost(AffineForOp srcForOp, LoopNestStats &srcStats,
|
bool mlir::affine::getFusionComputeCost(AffineForOp srcForOp,
|
||||||
AffineForOp dstForOp, LoopNestStats &dstStats,
|
LoopNestStats &srcStats,
|
||||||
const ComputationSliceState &slice,
|
AffineForOp dstForOp,
|
||||||
int64_t *computeCost) {
|
LoopNestStats &dstStats,
|
||||||
|
const ComputationSliceState &slice,
|
||||||
|
int64_t *computeCost) {
|
||||||
llvm::SmallDenseMap<Operation *, uint64_t, 8> sliceTripCountMap;
|
llvm::SmallDenseMap<Operation *, uint64_t, 8> sliceTripCountMap;
|
||||||
DenseMap<Operation *, int64_t> computeCostMap;
|
DenseMap<Operation *, int64_t> computeCostMap;
|
||||||
|
|
||||||
@ -634,7 +639,7 @@ bool mlir::getFusionComputeCost(AffineForOp srcForOp, LoopNestStats &srcStats,
|
|||||||
/// Returns in 'producerConsumerMemrefs' the memrefs involved in a
|
/// Returns in 'producerConsumerMemrefs' the memrefs involved in a
|
||||||
/// producer-consumer dependence between write ops in 'srcOps' and read ops in
|
/// producer-consumer dependence between write ops in 'srcOps' and read ops in
|
||||||
/// 'dstOps'.
|
/// 'dstOps'.
|
||||||
void mlir::gatherProducerConsumerMemrefs(
|
void mlir::affine::gatherProducerConsumerMemrefs(
|
||||||
ArrayRef<Operation *> srcOps, ArrayRef<Operation *> dstOps,
|
ArrayRef<Operation *> srcOps, ArrayRef<Operation *> dstOps,
|
||||||
DenseSet<Value> &producerConsumerMemrefs) {
|
DenseSet<Value> &producerConsumerMemrefs) {
|
||||||
// Gather memrefs from stores in 'srcOps'.
|
// Gather memrefs from stores in 'srcOps'.
|
||||||
|
@ -34,6 +34,7 @@
|
|||||||
#define DEBUG_TYPE "loop-utils"
|
#define DEBUG_TYPE "loop-utils"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace affine;
|
||||||
using namespace presburger;
|
using namespace presburger;
|
||||||
using llvm::SmallMapVector;
|
using llvm::SmallMapVector;
|
||||||
|
|
||||||
@ -128,7 +129,7 @@ static void replaceIterArgsAndYieldResults(AffineForOp forOp) {
|
|||||||
/// Promotes the loop body of a forOp to its containing block if the forOp
|
/// Promotes the loop body of a forOp to its containing block if the forOp
|
||||||
/// was known to have a single iteration.
|
/// was known to have a single iteration.
|
||||||
// TODO: extend this for arbitrary affine bounds.
|
// TODO: extend this for arbitrary affine bounds.
|
||||||
LogicalResult mlir::promoteIfSingleIteration(AffineForOp forOp) {
|
LogicalResult mlir::affine::promoteIfSingleIteration(AffineForOp forOp) {
|
||||||
std::optional<uint64_t> tripCount = getConstantTripCount(forOp);
|
std::optional<uint64_t> tripCount = getConstantTripCount(forOp);
|
||||||
if (!tripCount || *tripCount != 1)
|
if (!tripCount || *tripCount != 1)
|
||||||
return failure();
|
return failure();
|
||||||
@ -232,9 +233,9 @@ static AffineForOp generateShiftedLoop(
|
|||||||
// asserts preservation of SSA dominance. A check for that as well as that for
|
// asserts preservation of SSA dominance. A check for that as well as that for
|
||||||
// memory-based dependence preservation check rests with the users of this
|
// memory-based dependence preservation check rests with the users of this
|
||||||
// method.
|
// method.
|
||||||
LogicalResult mlir::affineForOpBodySkew(AffineForOp forOp,
|
LogicalResult mlir::affine::affineForOpBodySkew(AffineForOp forOp,
|
||||||
ArrayRef<uint64_t> shifts,
|
ArrayRef<uint64_t> shifts,
|
||||||
bool unrollPrologueEpilogue) {
|
bool unrollPrologueEpilogue) {
|
||||||
assert(forOp.getBody()->getOperations().size() == shifts.size() &&
|
assert(forOp.getBody()->getOperations().size() == shifts.size() &&
|
||||||
"too few/many shifts");
|
"too few/many shifts");
|
||||||
if (forOp.getBody()->begin() == std::prev(forOp.getBody()->end()))
|
if (forOp.getBody()->begin() == std::prev(forOp.getBody()->end()))
|
||||||
@ -363,7 +364,8 @@ LogicalResult mlir::affineForOpBodySkew(AffineForOp forOp,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Checks whether a loop nest is hyper-rectangular or not.
|
/// Checks whether a loop nest is hyper-rectangular or not.
|
||||||
LogicalResult checkIfHyperRectangular(MutableArrayRef<AffineForOp> input) {
|
static LogicalResult
|
||||||
|
checkIfHyperRectangular(MutableArrayRef<AffineForOp> input) {
|
||||||
FlatAffineValueConstraints cst;
|
FlatAffineValueConstraints cst;
|
||||||
SmallVector<Operation *, 8> ops(input.begin(), input.end());
|
SmallVector<Operation *, 8> ops(input.begin(), input.end());
|
||||||
// 0-d or 1-d is trivially hyper-rectangular.
|
// 0-d or 1-d is trivially hyper-rectangular.
|
||||||
@ -384,8 +386,8 @@ LogicalResult checkIfHyperRectangular(MutableArrayRef<AffineForOp> input) {
|
|||||||
/// Check if the input nest is supported for tiling and whether tiling would be
|
/// Check if the input nest is supported for tiling and whether tiling would be
|
||||||
/// legal or not.
|
/// legal or not.
|
||||||
template <typename t>
|
template <typename t>
|
||||||
LogicalResult performPreTilingChecks(MutableArrayRef<AffineForOp> input,
|
static LogicalResult performPreTilingChecks(MutableArrayRef<AffineForOp> input,
|
||||||
ArrayRef<t> tileSizes) {
|
ArrayRef<t> tileSizes) {
|
||||||
assert(input.size() == tileSizes.size() && "Too few/many tile sizes");
|
assert(input.size() == tileSizes.size() && "Too few/many tile sizes");
|
||||||
|
|
||||||
if (llvm::any_of(input,
|
if (llvm::any_of(input,
|
||||||
@ -418,15 +420,15 @@ static void moveLoopBodyImpl(AffineForOp src, AffineForOp dest,
|
|||||||
|
|
||||||
/// Move the loop body of AffineForOp 'src' from 'src' to the start of dest
|
/// Move the loop body of AffineForOp 'src' from 'src' to the start of dest
|
||||||
/// body.
|
/// body.
|
||||||
void moveLoopBody(AffineForOp src, AffineForOp dest) {
|
static void moveLoopBody(AffineForOp src, AffineForOp dest) {
|
||||||
moveLoopBodyImpl(src, dest, dest.getBody()->begin());
|
moveLoopBodyImpl(src, dest, dest.getBody()->begin());
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Constructs tiled loop nest, without setting the loop bounds and move the
|
/// Constructs tiled loop nest, without setting the loop bounds and move the
|
||||||
/// body of the original loop nest to the tiled loop nest.
|
/// body of the original loop nest to the tiled loop nest.
|
||||||
void constructTiledLoopNest(MutableArrayRef<AffineForOp> origLoops,
|
static void constructTiledLoopNest(MutableArrayRef<AffineForOp> origLoops,
|
||||||
AffineForOp rootAffineForOp, unsigned width,
|
AffineForOp rootAffineForOp, unsigned width,
|
||||||
MutableArrayRef<AffineForOp> tiledLoops) {
|
MutableArrayRef<AffineForOp> tiledLoops) {
|
||||||
Location loc = rootAffineForOp.getLoc();
|
Location loc = rootAffineForOp.getLoc();
|
||||||
|
|
||||||
// The outermost among the loops as we add more..
|
// The outermost among the loops as we add more..
|
||||||
@ -773,9 +775,9 @@ constructTiledIndexSetHyperRect(MutableArrayRef<AffineForOp> origLoops,
|
|||||||
}
|
}
|
||||||
|
|
||||||
LogicalResult
|
LogicalResult
|
||||||
mlir::tilePerfectlyNested(MutableArrayRef<AffineForOp> input,
|
mlir::affine::tilePerfectlyNested(MutableArrayRef<AffineForOp> input,
|
||||||
ArrayRef<unsigned> tileSizes,
|
ArrayRef<unsigned> tileSizes,
|
||||||
SmallVectorImpl<AffineForOp> *tiledNest) {
|
SmallVectorImpl<AffineForOp> *tiledNest) {
|
||||||
if (input.empty())
|
if (input.empty())
|
||||||
return success();
|
return success();
|
||||||
|
|
||||||
@ -816,10 +818,9 @@ mlir::tilePerfectlyNested(MutableArrayRef<AffineForOp> input,
|
|||||||
/// loops and intra-tile loops, using SSA values as tiling parameters. A band
|
/// loops and intra-tile loops, using SSA values as tiling parameters. A band
|
||||||
/// is a contiguous set of loops.
|
/// is a contiguous set of loops.
|
||||||
// TODO: handle non hyper-rectangular spaces.
|
// TODO: handle non hyper-rectangular spaces.
|
||||||
LogicalResult
|
LogicalResult mlir::affine::tilePerfectlyNestedParametric(
|
||||||
mlir::tilePerfectlyNestedParametric(MutableArrayRef<AffineForOp> input,
|
MutableArrayRef<AffineForOp> input, ArrayRef<Value> tileSizes,
|
||||||
ArrayRef<Value> tileSizes,
|
SmallVectorImpl<AffineForOp> *tiledNest) {
|
||||||
SmallVectorImpl<AffineForOp> *tiledNest) {
|
|
||||||
if (input.empty())
|
if (input.empty())
|
||||||
return success();
|
return success();
|
||||||
|
|
||||||
@ -859,8 +860,8 @@ mlir::tilePerfectlyNestedParametric(MutableArrayRef<AffineForOp> input,
|
|||||||
/// (the first op being another AffineFor, and the second op - a terminator).
|
/// (the first op being another AffineFor, and the second op - a terminator).
|
||||||
/// A loop is perfectly nested iff: the first op in the loop's body is another
|
/// A loop is perfectly nested iff: the first op in the loop's body is another
|
||||||
/// AffineForOp, and the second op is a terminator).
|
/// AffineForOp, and the second op is a terminator).
|
||||||
void mlir::getPerfectlyNestedLoops(SmallVectorImpl<AffineForOp> &nestedLoops,
|
void mlir::affine::getPerfectlyNestedLoops(
|
||||||
AffineForOp root) {
|
SmallVectorImpl<AffineForOp> &nestedLoops, AffineForOp root) {
|
||||||
for (unsigned i = 0; i < std::numeric_limits<unsigned>::max(); ++i) {
|
for (unsigned i = 0; i < std::numeric_limits<unsigned>::max(); ++i) {
|
||||||
nestedLoops.push_back(root);
|
nestedLoops.push_back(root);
|
||||||
Block &body = root.getRegion().front();
|
Block &body = root.getRegion().front();
|
||||||
@ -876,8 +877,8 @@ void mlir::getPerfectlyNestedLoops(SmallVectorImpl<AffineForOp> &nestedLoops,
|
|||||||
/// Identify valid and profitable bands of loops to tile. This is currently just
|
/// Identify valid and profitable bands of loops to tile. This is currently just
|
||||||
/// a temporary placeholder to test the mechanics of tiled code generation.
|
/// a temporary placeholder to test the mechanics of tiled code generation.
|
||||||
/// Returns all maximal outermost perfect loop nests to tile.
|
/// Returns all maximal outermost perfect loop nests to tile.
|
||||||
void mlir::getTileableBands(func::FuncOp f,
|
void mlir::affine::getTileableBands(
|
||||||
std::vector<SmallVector<AffineForOp, 6>> *bands) {
|
func::FuncOp f, std::vector<SmallVector<AffineForOp, 6>> *bands) {
|
||||||
// Get maximal perfect nest of 'affine.for' insts starting from root
|
// Get maximal perfect nest of 'affine.for' insts starting from root
|
||||||
// (inclusive).
|
// (inclusive).
|
||||||
for (AffineForOp forOp : f.getOps<AffineForOp>()) {
|
for (AffineForOp forOp : f.getOps<AffineForOp>()) {
|
||||||
@ -888,7 +889,7 @@ void mlir::getTileableBands(func::FuncOp f,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Unrolls this loop completely.
|
/// Unrolls this loop completely.
|
||||||
LogicalResult mlir::loopUnrollFull(AffineForOp forOp) {
|
LogicalResult mlir::affine::loopUnrollFull(AffineForOp forOp) {
|
||||||
std::optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
|
std::optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
|
||||||
if (mayBeConstantTripCount.has_value()) {
|
if (mayBeConstantTripCount.has_value()) {
|
||||||
uint64_t tripCount = *mayBeConstantTripCount;
|
uint64_t tripCount = *mayBeConstantTripCount;
|
||||||
@ -903,8 +904,8 @@ LogicalResult mlir::loopUnrollFull(AffineForOp forOp) {
|
|||||||
|
|
||||||
/// Unrolls this loop by the specified factor or by the trip count (if constant)
|
/// Unrolls this loop by the specified factor or by the trip count (if constant)
|
||||||
/// whichever is lower.
|
/// whichever is lower.
|
||||||
LogicalResult mlir::loopUnrollUpToFactor(AffineForOp forOp,
|
LogicalResult mlir::affine::loopUnrollUpToFactor(AffineForOp forOp,
|
||||||
uint64_t unrollFactor) {
|
uint64_t unrollFactor) {
|
||||||
std::optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
|
std::optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
|
||||||
if (mayBeConstantTripCount.has_value() &&
|
if (mayBeConstantTripCount.has_value() &&
|
||||||
*mayBeConstantTripCount < unrollFactor)
|
*mayBeConstantTripCount < unrollFactor)
|
||||||
@ -1011,7 +1012,7 @@ static LogicalResult generateCleanupLoopForUnroll(AffineForOp forOp,
|
|||||||
|
|
||||||
/// Unrolls this loop by the specified factor. Returns success if the loop
|
/// Unrolls this loop by the specified factor. Returns success if the loop
|
||||||
/// is successfully unrolled.
|
/// is successfully unrolled.
|
||||||
LogicalResult mlir::loopUnrollByFactor(
|
LogicalResult mlir::affine::loopUnrollByFactor(
|
||||||
AffineForOp forOp, uint64_t unrollFactor,
|
AffineForOp forOp, uint64_t unrollFactor,
|
||||||
function_ref<void(unsigned, Operation *, OpBuilder)> annotateFn,
|
function_ref<void(unsigned, Operation *, OpBuilder)> annotateFn,
|
||||||
bool cleanUpUnroll) {
|
bool cleanUpUnroll) {
|
||||||
@ -1078,8 +1079,8 @@ LogicalResult mlir::loopUnrollByFactor(
|
|||||||
return success();
|
return success();
|
||||||
}
|
}
|
||||||
|
|
||||||
LogicalResult mlir::loopUnrollJamUpToFactor(AffineForOp forOp,
|
LogicalResult mlir::affine::loopUnrollJamUpToFactor(AffineForOp forOp,
|
||||||
uint64_t unrollJamFactor) {
|
uint64_t unrollJamFactor) {
|
||||||
std::optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
|
std::optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
|
||||||
if (mayBeConstantTripCount.has_value() &&
|
if (mayBeConstantTripCount.has_value() &&
|
||||||
*mayBeConstantTripCount < unrollJamFactor)
|
*mayBeConstantTripCount < unrollJamFactor)
|
||||||
@ -1129,8 +1130,8 @@ struct JamBlockGatherer {
|
|||||||
};
|
};
|
||||||
|
|
||||||
/// Unrolls and jams this loop by the specified factor.
|
/// Unrolls and jams this loop by the specified factor.
|
||||||
LogicalResult mlir::loopUnrollJamByFactor(AffineForOp forOp,
|
LogicalResult mlir::affine::loopUnrollJamByFactor(AffineForOp forOp,
|
||||||
uint64_t unrollJamFactor) {
|
uint64_t unrollJamFactor) {
|
||||||
assert(unrollJamFactor > 0 && "unroll jam factor should be positive");
|
assert(unrollJamFactor > 0 && "unroll jam factor should be positive");
|
||||||
|
|
||||||
std::optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
|
std::optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
|
||||||
@ -1212,7 +1213,7 @@ LogicalResult mlir::loopUnrollJamByFactor(AffineForOp forOp,
|
|||||||
}
|
}
|
||||||
// Create a new loop with additional iterOperands, iter_args and yield
|
// Create a new loop with additional iterOperands, iter_args and yield
|
||||||
// operands. This new loop will take the loop body of the original loop.
|
// operands. This new loop will take the loop body of the original loop.
|
||||||
AffineForOp newForOp = mlir::replaceForOpWithNewYields(
|
AffineForOp newForOp = affine::replaceForOpWithNewYields(
|
||||||
builder, oldForOp, dupIterOperands, dupYieldOperands, dupIterArgs);
|
builder, oldForOp, dupIterOperands, dupYieldOperands, dupIterArgs);
|
||||||
newLoopsWithIterArgs.push_back(newForOp);
|
newLoopsWithIterArgs.push_back(newForOp);
|
||||||
// `forOp` has been replaced with a new loop.
|
// `forOp` has been replaced with a new loop.
|
||||||
@ -1323,7 +1324,7 @@ LogicalResult mlir::loopUnrollJamByFactor(AffineForOp forOp,
|
|||||||
|
|
||||||
/// Performs loop interchange on 'forOpA' and 'forOpB', where 'forOpB' is
|
/// Performs loop interchange on 'forOpA' and 'forOpB', where 'forOpB' is
|
||||||
/// nested within 'forOpA' as the only non-terminator operation in its block.
|
/// nested within 'forOpA' as the only non-terminator operation in its block.
|
||||||
void mlir::interchangeLoops(AffineForOp forOpA, AffineForOp forOpB) {
|
void mlir::affine::interchangeLoops(AffineForOp forOpA, AffineForOp forOpB) {
|
||||||
assert(&*forOpA.getBody()->begin() == forOpB.getOperation());
|
assert(&*forOpA.getBody()->begin() == forOpB.getOperation());
|
||||||
auto &forOpABody = forOpA.getBody()->getOperations();
|
auto &forOpABody = forOpA.getBody()->getOperations();
|
||||||
auto &forOpBBody = forOpB.getBody()->getOperations();
|
auto &forOpBBody = forOpB.getBody()->getOperations();
|
||||||
@ -1380,8 +1381,8 @@ static bool checkLoopInterchangeDependences(
|
|||||||
|
|
||||||
/// Checks if the loop interchange permutation 'loopPermMap' of the perfectly
|
/// Checks if the loop interchange permutation 'loopPermMap' of the perfectly
|
||||||
/// nested sequence of loops in 'loops' would violate dependences.
|
/// nested sequence of loops in 'loops' would violate dependences.
|
||||||
bool mlir::isValidLoopInterchangePermutation(ArrayRef<AffineForOp> loops,
|
bool mlir::affine::isValidLoopInterchangePermutation(
|
||||||
ArrayRef<unsigned> loopPermMap) {
|
ArrayRef<AffineForOp> loops, ArrayRef<unsigned> loopPermMap) {
|
||||||
// Gather dependence components for dependences between all ops in loop nest
|
// Gather dependence components for dependences between all ops in loop nest
|
||||||
// rooted at 'loops[0]', at loop depths in range [1, maxLoopDepth].
|
// rooted at 'loops[0]', at loop depths in range [1, maxLoopDepth].
|
||||||
assert(loopPermMap.size() == loops.size());
|
assert(loopPermMap.size() == loops.size());
|
||||||
@ -1394,7 +1395,7 @@ bool mlir::isValidLoopInterchangePermutation(ArrayRef<AffineForOp> loops,
|
|||||||
/// Returns true if `loops` is a perfectly nested loop nest, where loops appear
|
/// Returns true if `loops` is a perfectly nested loop nest, where loops appear
|
||||||
/// in it from outermost to innermost.
|
/// in it from outermost to innermost.
|
||||||
bool LLVM_ATTRIBUTE_UNUSED
|
bool LLVM_ATTRIBUTE_UNUSED
|
||||||
mlir::isPerfectlyNested(ArrayRef<AffineForOp> loops) {
|
mlir::affine::isPerfectlyNested(ArrayRef<AffineForOp> loops) {
|
||||||
assert(!loops.empty() && "no loops provided");
|
assert(!loops.empty() && "no loops provided");
|
||||||
|
|
||||||
// We already know that the block can't be empty.
|
// We already know that the block can't be empty.
|
||||||
@ -1416,8 +1417,8 @@ mlir::isPerfectlyNested(ArrayRef<AffineForOp> loops) {
|
|||||||
|
|
||||||
// input[i] should move from position i -> permMap[i]. Returns the position in
|
// input[i] should move from position i -> permMap[i]. Returns the position in
|
||||||
// `input` that becomes the new outermost loop.
|
// `input` that becomes the new outermost loop.
|
||||||
unsigned mlir::permuteLoops(MutableArrayRef<AffineForOp> input,
|
unsigned mlir::affine::permuteLoops(MutableArrayRef<AffineForOp> input,
|
||||||
ArrayRef<unsigned> permMap) {
|
ArrayRef<unsigned> permMap) {
|
||||||
assert(input.size() == permMap.size() && "invalid permutation map size");
|
assert(input.size() == permMap.size() && "invalid permutation map size");
|
||||||
// Check whether the permutation spec is valid. This is a small vector - we'll
|
// Check whether the permutation spec is valid. This is a small vector - we'll
|
||||||
// just sort and check if it's iota.
|
// just sort and check if it's iota.
|
||||||
@ -1486,7 +1487,7 @@ unsigned mlir::permuteLoops(MutableArrayRef<AffineForOp> input,
|
|||||||
// Sinks all sequential loops to the innermost levels (while preserving
|
// Sinks all sequential loops to the innermost levels (while preserving
|
||||||
// relative order among them) and moves all parallel loops to the
|
// relative order among them) and moves all parallel loops to the
|
||||||
// outermost (while again preserving relative order among them).
|
// outermost (while again preserving relative order among them).
|
||||||
AffineForOp mlir::sinkSequentialLoops(AffineForOp forOp) {
|
AffineForOp mlir::affine::sinkSequentialLoops(AffineForOp forOp) {
|
||||||
SmallVector<AffineForOp, 4> loops;
|
SmallVector<AffineForOp, 4> loops;
|
||||||
getPerfectlyNestedLoops(loops, forOp);
|
getPerfectlyNestedLoops(loops, forOp);
|
||||||
if (loops.size() < 2)
|
if (loops.size() < 2)
|
||||||
@ -1621,8 +1622,8 @@ static AffineForOp stripmineSink(AffineForOp forOp, SizeType factor,
|
|||||||
}
|
}
|
||||||
|
|
||||||
SmallVector<SmallVector<AffineForOp, 8>, 8>
|
SmallVector<SmallVector<AffineForOp, 8>, 8>
|
||||||
mlir::tile(ArrayRef<AffineForOp> forOps, ArrayRef<uint64_t> sizes,
|
mlir::affine::tile(ArrayRef<AffineForOp> forOps, ArrayRef<uint64_t> sizes,
|
||||||
ArrayRef<AffineForOp> targets) {
|
ArrayRef<AffineForOp> targets) {
|
||||||
SmallVector<SmallVector<AffineForOp, 8>, 8> res;
|
SmallVector<SmallVector<AffineForOp, 8>, 8> res;
|
||||||
SmallVector<AffineForOp, 8> currentTargets(targets.begin(), targets.end());
|
SmallVector<AffineForOp, 8> currentTargets(targets.begin(), targets.end());
|
||||||
for (auto it : llvm::zip(forOps, sizes)) {
|
for (auto it : llvm::zip(forOps, sizes)) {
|
||||||
@ -1633,9 +1634,9 @@ mlir::tile(ArrayRef<AffineForOp> forOps, ArrayRef<uint64_t> sizes,
|
|||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
SmallVector<AffineForOp, 8> mlir::tile(ArrayRef<AffineForOp> forOps,
|
SmallVector<AffineForOp, 8> mlir::affine::tile(ArrayRef<AffineForOp> forOps,
|
||||||
ArrayRef<uint64_t> sizes,
|
ArrayRef<uint64_t> sizes,
|
||||||
AffineForOp target) {
|
AffineForOp target) {
|
||||||
SmallVector<AffineForOp, 8> res;
|
SmallVector<AffineForOp, 8> res;
|
||||||
for (auto loops : tile(forOps, sizes, ArrayRef<AffineForOp>(target))) {
|
for (auto loops : tile(forOps, sizes, ArrayRef<AffineForOp>(target))) {
|
||||||
assert(loops.size() == 1);
|
assert(loops.size() == 1);
|
||||||
@ -1644,7 +1645,7 @@ SmallVector<AffineForOp, 8> mlir::tile(ArrayRef<AffineForOp> forOps,
|
|||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
LogicalResult mlir::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
|
LogicalResult mlir::affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
|
||||||
if (loops.size() < 2)
|
if (loops.size() < 2)
|
||||||
return success();
|
return success();
|
||||||
|
|
||||||
@ -1758,8 +1759,9 @@ LogicalResult mlir::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
|
|||||||
return success();
|
return success();
|
||||||
}
|
}
|
||||||
|
|
||||||
void mlir::mapLoopToProcessorIds(scf::ForOp forOp, ArrayRef<Value> processorId,
|
void mlir::affine::mapLoopToProcessorIds(scf::ForOp forOp,
|
||||||
ArrayRef<Value> numProcessors) {
|
ArrayRef<Value> processorId,
|
||||||
|
ArrayRef<Value> numProcessors) {
|
||||||
assert(processorId.size() == numProcessors.size());
|
assert(processorId.size() == numProcessors.size());
|
||||||
if (processorId.empty())
|
if (processorId.empty())
|
||||||
return;
|
return;
|
||||||
@ -2300,11 +2302,11 @@ static bool getFullMemRefAsRegion(Operation *op, unsigned numParamLoopIVs,
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
LogicalResult mlir::affineDataCopyGenerate(Block::iterator begin,
|
LogicalResult
|
||||||
Block::iterator end,
|
mlir::affine::affineDataCopyGenerate(Block::iterator begin, Block::iterator end,
|
||||||
const AffineCopyOptions ©Options,
|
const AffineCopyOptions ©Options,
|
||||||
std::optional<Value> filterMemRef,
|
std::optional<Value> filterMemRef,
|
||||||
DenseSet<Operation *> ©Nests) {
|
DenseSet<Operation *> ©Nests) {
|
||||||
if (begin == end)
|
if (begin == end)
|
||||||
return success();
|
return success();
|
||||||
|
|
||||||
@ -2490,16 +2492,15 @@ LogicalResult mlir::affineDataCopyGenerate(Block::iterator begin,
|
|||||||
|
|
||||||
// A convenience version of affineDataCopyGenerate for all ops in the body of
|
// A convenience version of affineDataCopyGenerate for all ops in the body of
|
||||||
// an AffineForOp.
|
// an AffineForOp.
|
||||||
LogicalResult mlir::affineDataCopyGenerate(AffineForOp forOp,
|
LogicalResult mlir::affine::affineDataCopyGenerate(
|
||||||
const AffineCopyOptions ©Options,
|
AffineForOp forOp, const AffineCopyOptions ©Options,
|
||||||
std::optional<Value> filterMemRef,
|
std::optional<Value> filterMemRef, DenseSet<Operation *> ©Nests) {
|
||||||
DenseSet<Operation *> ©Nests) {
|
|
||||||
return affineDataCopyGenerate(forOp.getBody()->begin(),
|
return affineDataCopyGenerate(forOp.getBody()->begin(),
|
||||||
std::prev(forOp.getBody()->end()), copyOptions,
|
std::prev(forOp.getBody()->end()), copyOptions,
|
||||||
filterMemRef, copyNests);
|
filterMemRef, copyNests);
|
||||||
}
|
}
|
||||||
|
|
||||||
LogicalResult mlir::generateCopyForMemRegion(
|
LogicalResult mlir::affine::generateCopyForMemRegion(
|
||||||
const MemRefRegion &memrefRegion, Operation *analyzedOp,
|
const MemRefRegion &memrefRegion, Operation *analyzedOp,
|
||||||
const AffineCopyOptions ©Options, CopyGenerateResult &result) {
|
const AffineCopyOptions ©Options, CopyGenerateResult &result) {
|
||||||
Block *block = analyzedOp->getBlock();
|
Block *block = analyzedOp->getBlock();
|
||||||
@ -2543,8 +2544,8 @@ gatherLoopsInBlock(Block *block, unsigned currLoopDepth,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Gathers all AffineForOps in 'func.func' grouped by loop depth.
|
/// Gathers all AffineForOps in 'func.func' grouped by loop depth.
|
||||||
void mlir::gatherLoops(func::FuncOp func,
|
void mlir::affine::gatherLoops(
|
||||||
std::vector<SmallVector<AffineForOp, 2>> &depthToLoops) {
|
func::FuncOp func, std::vector<SmallVector<AffineForOp, 2>> &depthToLoops) {
|
||||||
for (auto &block : func)
|
for (auto &block : func)
|
||||||
gatherLoopsInBlock(&block, /*currLoopDepth=*/0, depthToLoops);
|
gatherLoopsInBlock(&block, /*currLoopDepth=*/0, depthToLoops);
|
||||||
|
|
||||||
@ -2559,7 +2560,7 @@ void mlir::gatherLoops(func::FuncOp func,
|
|||||||
// affine.applys, fold to constant if all result dimensions of the map are
|
// affine.applys, fold to constant if all result dimensions of the map are
|
||||||
// constant (canonicalizeMapAndOperands below already does this for single
|
// constant (canonicalizeMapAndOperands below already does this for single
|
||||||
// result bound maps), and use simplifyMap to perform algebraic simplification.
|
// result bound maps), and use simplifyMap to perform algebraic simplification.
|
||||||
AffineForOp mlir::createCanonicalizedAffineForOp(
|
AffineForOp mlir::affine::createCanonicalizedAffineForOp(
|
||||||
OpBuilder b, Location loc, ValueRange lbOperands, AffineMap lbMap,
|
OpBuilder b, Location loc, ValueRange lbOperands, AffineMap lbMap,
|
||||||
ValueRange ubOperands, AffineMap ubMap, int64_t step) {
|
ValueRange ubOperands, AffineMap ubMap, int64_t step) {
|
||||||
SmallVector<Value, 4> lowerOperands(lbOperands);
|
SmallVector<Value, 4> lowerOperands(lbOperands);
|
||||||
@ -2716,8 +2717,8 @@ createFullTiles(MutableArrayRef<AffineForOp> inputNest,
|
|||||||
}
|
}
|
||||||
|
|
||||||
LogicalResult
|
LogicalResult
|
||||||
mlir::separateFullTiles(MutableArrayRef<AffineForOp> inputNest,
|
mlir::affine::separateFullTiles(MutableArrayRef<AffineForOp> inputNest,
|
||||||
SmallVectorImpl<AffineForOp> *fullTileNest) {
|
SmallVectorImpl<AffineForOp> *fullTileNest) {
|
||||||
if (inputNest.empty())
|
if (inputNest.empty())
|
||||||
return success();
|
return success();
|
||||||
|
|
||||||
|
@ -30,6 +30,7 @@
|
|||||||
#define DEBUG_TYPE "affine-utils"
|
#define DEBUG_TYPE "affine-utils"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace affine;
|
||||||
using namespace presburger;
|
using namespace presburger;
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
@ -209,17 +210,18 @@ private:
|
|||||||
|
|
||||||
/// Create a sequence of operations that implement the `expr` applied to the
|
/// Create a sequence of operations that implement the `expr` applied to the
|
||||||
/// given dimension and symbol values.
|
/// given dimension and symbol values.
|
||||||
mlir::Value mlir::expandAffineExpr(OpBuilder &builder, Location loc,
|
mlir::Value mlir::affine::expandAffineExpr(OpBuilder &builder, Location loc,
|
||||||
AffineExpr expr, ValueRange dimValues,
|
AffineExpr expr,
|
||||||
ValueRange symbolValues) {
|
ValueRange dimValues,
|
||||||
|
ValueRange symbolValues) {
|
||||||
return AffineApplyExpander(builder, dimValues, symbolValues, loc).visit(expr);
|
return AffineApplyExpander(builder, dimValues, symbolValues, loc).visit(expr);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Create a sequence of operations that implement the `affineMap` applied to
|
/// Create a sequence of operations that implement the `affineMap` applied to
|
||||||
/// the given `operands` (as it it were an AffineApplyOp).
|
/// the given `operands` (as it it were an AffineApplyOp).
|
||||||
std::optional<SmallVector<Value, 8>>
|
std::optional<SmallVector<Value, 8>>
|
||||||
mlir::expandAffineMap(OpBuilder &builder, Location loc, AffineMap affineMap,
|
mlir::affine::expandAffineMap(OpBuilder &builder, Location loc,
|
||||||
ValueRange operands) {
|
AffineMap affineMap, ValueRange operands) {
|
||||||
auto numDims = affineMap.getNumDims();
|
auto numDims = affineMap.getNumDims();
|
||||||
auto expanded = llvm::to_vector<8>(
|
auto expanded = llvm::to_vector<8>(
|
||||||
llvm::map_range(affineMap.getResults(),
|
llvm::map_range(affineMap.getResults(),
|
||||||
@ -341,8 +343,8 @@ static AffineIfOp hoistAffineIfOp(AffineIfOp ifOp, Operation *hoistOverOp) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
LogicalResult
|
LogicalResult
|
||||||
mlir::affineParallelize(AffineForOp forOp,
|
mlir::affine::affineParallelize(AffineForOp forOp,
|
||||||
ArrayRef<LoopReduction> parallelReductions) {
|
ArrayRef<LoopReduction> parallelReductions) {
|
||||||
// Fail early if there are iter arguments that are not reductions.
|
// Fail early if there are iter arguments that are not reductions.
|
||||||
unsigned numReductions = parallelReductions.size();
|
unsigned numReductions = parallelReductions.size();
|
||||||
if (numReductions != forOp.getNumIterOperands())
|
if (numReductions != forOp.getNumIterOperands())
|
||||||
@ -400,7 +402,7 @@ mlir::affineParallelize(AffineForOp forOp,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Returns success if any hoisting happened.
|
// Returns success if any hoisting happened.
|
||||||
LogicalResult mlir::hoistAffineIfOp(AffineIfOp ifOp, bool *folded) {
|
LogicalResult mlir::affine::hoistAffineIfOp(AffineIfOp ifOp, bool *folded) {
|
||||||
// Bail out early if the ifOp returns a result. TODO: Consider how to
|
// Bail out early if the ifOp returns a result. TODO: Consider how to
|
||||||
// properly support this case.
|
// properly support this case.
|
||||||
if (ifOp.getNumResults() != 0)
|
if (ifOp.getNumResults() != 0)
|
||||||
@ -454,8 +456,9 @@ LogicalResult mlir::hoistAffineIfOp(AffineIfOp ifOp, bool *folded) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Return the min expr after replacing the given dim.
|
// Return the min expr after replacing the given dim.
|
||||||
AffineExpr mlir::substWithMin(AffineExpr e, AffineExpr dim, AffineExpr min,
|
AffineExpr mlir::affine::substWithMin(AffineExpr e, AffineExpr dim,
|
||||||
AffineExpr max, bool positivePath) {
|
AffineExpr min, AffineExpr max,
|
||||||
|
bool positivePath) {
|
||||||
if (e == dim)
|
if (e == dim)
|
||||||
return positivePath ? min : max;
|
return positivePath ? min : max;
|
||||||
if (auto bin = e.dyn_cast<AffineBinaryOpExpr>()) {
|
if (auto bin = e.dyn_cast<AffineBinaryOpExpr>()) {
|
||||||
@ -480,7 +483,7 @@ AffineExpr mlir::substWithMin(AffineExpr e, AffineExpr dim, AffineExpr min,
|
|||||||
return e;
|
return e;
|
||||||
}
|
}
|
||||||
|
|
||||||
void mlir::normalizeAffineParallel(AffineParallelOp op) {
|
void mlir::affine::normalizeAffineParallel(AffineParallelOp op) {
|
||||||
// Loops with min/max in bounds are not normalized at the moment.
|
// Loops with min/max in bounds are not normalized at the moment.
|
||||||
if (op.hasMinMaxBounds())
|
if (op.hasMinMaxBounds())
|
||||||
return;
|
return;
|
||||||
@ -544,7 +547,8 @@ void mlir::normalizeAffineParallel(AffineParallelOp op) {
|
|||||||
ubExprs, op.getContext());
|
ubExprs, op.getContext());
|
||||||
op.setUpperBounds(ranges.getOperands(), newUpperMap);
|
op.setUpperBounds(ranges.getOperands(), newUpperMap);
|
||||||
}
|
}
|
||||||
LogicalResult mlir::normalizeAffineFor(AffineForOp op, bool promoteSingleIter) {
|
LogicalResult mlir::affine::normalizeAffineFor(AffineForOp op,
|
||||||
|
bool promoteSingleIter) {
|
||||||
if (promoteSingleIter && succeeded(promoteIfSingleIteration(op)))
|
if (promoteSingleIter && succeeded(promoteIfSingleIteration(op)))
|
||||||
return success();
|
return success();
|
||||||
|
|
||||||
@ -701,7 +705,7 @@ static bool mayHaveEffect(Operation *srcMemOp, Operation *destMemOp,
|
|||||||
}
|
}
|
||||||
|
|
||||||
template <typename EffectType, typename T>
|
template <typename EffectType, typename T>
|
||||||
bool mlir::hasNoInterveningEffect(Operation *start, T memOp) {
|
bool mlir::affine::hasNoInterveningEffect(Operation *start, T memOp) {
|
||||||
auto isLocallyAllocated = [](Value memref) {
|
auto isLocallyAllocated = [](Value memref) {
|
||||||
auto *defOp = memref.getDefiningOp();
|
auto *defOp = memref.getDefiningOp();
|
||||||
return defOp && hasSingleEffect<MemoryEffects::Allocate>(defOp, memref);
|
return defOp && hasSingleEffect<MemoryEffects::Allocate>(defOp, memref);
|
||||||
@ -894,7 +898,7 @@ static LogicalResult forwardStoreToLoad(
|
|||||||
|
|
||||||
// 4. Ensure there is no intermediate operation which could replace the
|
// 4. Ensure there is no intermediate operation which could replace the
|
||||||
// value in memory.
|
// value in memory.
|
||||||
if (!mlir::hasNoInterveningEffect<MemoryEffects::Write>(storeOp, loadOp))
|
if (!affine::hasNoInterveningEffect<MemoryEffects::Write>(storeOp, loadOp))
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
// We now have a candidate for forwarding.
|
// We now have a candidate for forwarding.
|
||||||
@ -921,9 +925,10 @@ static LogicalResult forwardStoreToLoad(
|
|||||||
return success();
|
return success();
|
||||||
}
|
}
|
||||||
|
|
||||||
template bool mlir::hasNoInterveningEffect<mlir::MemoryEffects::Read,
|
template bool
|
||||||
mlir::AffineReadOpInterface>(
|
mlir::affine::hasNoInterveningEffect<mlir::MemoryEffects::Read,
|
||||||
mlir::Operation *, mlir::AffineReadOpInterface);
|
affine::AffineReadOpInterface>(
|
||||||
|
mlir::Operation *, affine::AffineReadOpInterface);
|
||||||
|
|
||||||
// This attempts to find stores which have no impact on the final result.
|
// This attempts to find stores which have no impact on the final result.
|
||||||
// A writing op writeA will be eliminated if there exists an op writeB if
|
// A writing op writeA will be eliminated if there exists an op writeB if
|
||||||
@ -961,7 +966,7 @@ static void findUnusedStore(AffineWriteOpInterface writeA,
|
|||||||
|
|
||||||
// There cannot be an operation which reads from memory between
|
// There cannot be an operation which reads from memory between
|
||||||
// the two writes.
|
// the two writes.
|
||||||
if (!mlir::hasNoInterveningEffect<MemoryEffects::Read>(writeA, writeB))
|
if (!affine::hasNoInterveningEffect<MemoryEffects::Read>(writeA, writeB))
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
opsToErase.push_back(writeA);
|
opsToErase.push_back(writeA);
|
||||||
@ -997,7 +1002,7 @@ static void loadCSE(AffineReadOpInterface loadA,
|
|||||||
continue;
|
continue;
|
||||||
|
|
||||||
// 3. There is no write between loadA and loadB.
|
// 3. There is no write between loadA and loadB.
|
||||||
if (!mlir::hasNoInterveningEffect<MemoryEffects::Write>(
|
if (!affine::hasNoInterveningEffect<MemoryEffects::Write>(
|
||||||
loadB.getOperation(), loadA))
|
loadB.getOperation(), loadA))
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
@ -1055,8 +1060,8 @@ static void loadCSE(AffineReadOpInterface loadA,
|
|||||||
// currently only eliminates the stores only if no other loads/uses (other
|
// currently only eliminates the stores only if no other loads/uses (other
|
||||||
// than dealloc) remain.
|
// than dealloc) remain.
|
||||||
//
|
//
|
||||||
void mlir::affineScalarReplace(func::FuncOp f, DominanceInfo &domInfo,
|
void mlir::affine::affineScalarReplace(func::FuncOp f, DominanceInfo &domInfo,
|
||||||
PostDominanceInfo &postDomInfo) {
|
PostDominanceInfo &postDomInfo) {
|
||||||
// Load op's whose results were replaced by those forwarded from stores.
|
// Load op's whose results were replaced by those forwarded from stores.
|
||||||
SmallVector<Operation *, 8> opsToErase;
|
SmallVector<Operation *, 8> opsToErase;
|
||||||
|
|
||||||
@ -1109,13 +1114,11 @@ void mlir::affineScalarReplace(func::FuncOp f, DominanceInfo &domInfo,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Perform the replacement in `op`.
|
// Perform the replacement in `op`.
|
||||||
LogicalResult mlir::replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
|
LogicalResult mlir::affine::replaceAllMemRefUsesWith(
|
||||||
Operation *op,
|
Value oldMemRef, Value newMemRef, Operation *op,
|
||||||
ArrayRef<Value> extraIndices,
|
ArrayRef<Value> extraIndices, AffineMap indexRemap,
|
||||||
AffineMap indexRemap,
|
ArrayRef<Value> extraOperands, ArrayRef<Value> symbolOperands,
|
||||||
ArrayRef<Value> extraOperands,
|
bool allowNonDereferencingOps) {
|
||||||
ArrayRef<Value> symbolOperands,
|
|
||||||
bool allowNonDereferencingOps) {
|
|
||||||
unsigned newMemRefRank = newMemRef.getType().cast<MemRefType>().getRank();
|
unsigned newMemRefRank = newMemRef.getType().cast<MemRefType>().getRank();
|
||||||
(void)newMemRefRank; // unused in opt mode
|
(void)newMemRefRank; // unused in opt mode
|
||||||
unsigned oldMemRefRank = oldMemRef.getType().cast<MemRefType>().getRank();
|
unsigned oldMemRefRank = oldMemRef.getType().cast<MemRefType>().getRank();
|
||||||
@ -1285,7 +1288,7 @@ LogicalResult mlir::replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
|
|||||||
return success();
|
return success();
|
||||||
}
|
}
|
||||||
|
|
||||||
LogicalResult mlir::replaceAllMemRefUsesWith(
|
LogicalResult mlir::affine::replaceAllMemRefUsesWith(
|
||||||
Value oldMemRef, Value newMemRef, ArrayRef<Value> extraIndices,
|
Value oldMemRef, Value newMemRef, ArrayRef<Value> extraIndices,
|
||||||
AffineMap indexRemap, ArrayRef<Value> extraOperands,
|
AffineMap indexRemap, ArrayRef<Value> extraOperands,
|
||||||
ArrayRef<Value> symbolOperands, Operation *domOpFilter,
|
ArrayRef<Value> symbolOperands, Operation *domOpFilter,
|
||||||
@ -1401,7 +1404,7 @@ LogicalResult mlir::replaceAllMemRefUsesWith(
|
|||||||
/// all the affine.apply op's supplying operands to this opInst did not have any
|
/// all the affine.apply op's supplying operands to this opInst did not have any
|
||||||
/// uses besides this opInst; otherwise returns the list of affine.apply
|
/// uses besides this opInst; otherwise returns the list of affine.apply
|
||||||
/// operations created in output argument `sliceOps`.
|
/// operations created in output argument `sliceOps`.
|
||||||
void mlir::createAffineComputationSlice(
|
void mlir::affine::createAffineComputationSlice(
|
||||||
Operation *opInst, SmallVectorImpl<AffineApplyOp> *sliceOps) {
|
Operation *opInst, SmallVectorImpl<AffineApplyOp> *sliceOps) {
|
||||||
// Collect all operands that are results of affine apply ops.
|
// Collect all operands that are results of affine apply ops.
|
||||||
SmallVector<Value, 4> subOperands;
|
SmallVector<Value, 4> subOperands;
|
||||||
@ -1709,7 +1712,7 @@ static void createNewDynamicSizes(MemRefType oldMemRefType,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TODO: Currently works for static memrefs with a single layout map.
|
// TODO: Currently works for static memrefs with a single layout map.
|
||||||
LogicalResult mlir::normalizeMemRef(memref::AllocOp *allocOp) {
|
LogicalResult mlir::affine::normalizeMemRef(memref::AllocOp *allocOp) {
|
||||||
MemRefType memrefType = allocOp->getType();
|
MemRefType memrefType = allocOp->getType();
|
||||||
OpBuilder b(*allocOp);
|
OpBuilder b(*allocOp);
|
||||||
|
|
||||||
@ -1767,8 +1770,8 @@ LogicalResult mlir::normalizeMemRef(memref::AllocOp *allocOp) {
|
|||||||
return success();
|
return success();
|
||||||
}
|
}
|
||||||
|
|
||||||
MemRefType mlir::normalizeMemRefType(MemRefType memrefType,
|
MemRefType mlir::affine::normalizeMemRefType(MemRefType memrefType,
|
||||||
unsigned numSymbolicOperands) {
|
unsigned numSymbolicOperands) {
|
||||||
unsigned rank = memrefType.getRank();
|
unsigned rank = memrefType.getRank();
|
||||||
if (rank == 0)
|
if (rank == 0)
|
||||||
return memrefType;
|
return memrefType;
|
||||||
@ -1848,13 +1851,15 @@ MemRefType mlir::normalizeMemRefType(MemRefType memrefType,
|
|||||||
return newMemRefType;
|
return newMemRefType;
|
||||||
}
|
}
|
||||||
|
|
||||||
DivModValue mlir::getDivMod(OpBuilder &b, Location loc, Value lhs, Value rhs) {
|
DivModValue mlir::affine::getDivMod(OpBuilder &b, Location loc, Value lhs,
|
||||||
|
Value rhs) {
|
||||||
DivModValue result;
|
DivModValue result;
|
||||||
AffineExpr d0, d1;
|
AffineExpr d0, d1;
|
||||||
bindDims(b.getContext(), d0, d1);
|
bindDims(b.getContext(), d0, d1);
|
||||||
result.quotient =
|
result.quotient =
|
||||||
makeComposedAffineApply(b, loc, d0.floorDiv(d1), {lhs, rhs});
|
affine::makeComposedAffineApply(b, loc, d0.floorDiv(d1), {lhs, rhs});
|
||||||
result.remainder = makeComposedAffineApply(b, loc, d0 % d1, {lhs, rhs});
|
result.remainder =
|
||||||
|
affine::makeComposedAffineApply(b, loc, d0 % d1, {lhs, rhs});
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1871,9 +1876,9 @@ static FailureOr<OpFoldResult> getIndexProduct(OpBuilder &b, Location loc,
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
FailureOr<SmallVector<Value>> mlir::delinearizeIndex(OpBuilder &b, Location loc,
|
FailureOr<SmallVector<Value>>
|
||||||
Value linearIndex,
|
mlir::affine::delinearizeIndex(OpBuilder &b, Location loc, Value linearIndex,
|
||||||
ArrayRef<Value> basis) {
|
ArrayRef<Value> basis) {
|
||||||
unsigned numDims = basis.size();
|
unsigned numDims = basis.size();
|
||||||
|
|
||||||
SmallVector<Value> divisors;
|
SmallVector<Value> divisors;
|
||||||
|
@ -12,8 +12,9 @@
|
|||||||
#include "mlir/IR/PatternMatch.h"
|
#include "mlir/IR/PatternMatch.h"
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace affine;
|
||||||
|
|
||||||
LogicalResult mlir::mergeOffsetsSizesAndStrides(
|
LogicalResult mlir::affine::mergeOffsetsSizesAndStrides(
|
||||||
OpBuilder &builder, Location loc, ArrayRef<OpFoldResult> producerOffsets,
|
OpBuilder &builder, Location loc, ArrayRef<OpFoldResult> producerOffsets,
|
||||||
ArrayRef<OpFoldResult> producerSizes,
|
ArrayRef<OpFoldResult> producerSizes,
|
||||||
ArrayRef<OpFoldResult> producerStrides,
|
ArrayRef<OpFoldResult> producerStrides,
|
||||||
@ -58,7 +59,7 @@ LogicalResult mlir::mergeOffsetsSizesAndStrides(
|
|||||||
return success();
|
return success();
|
||||||
}
|
}
|
||||||
|
|
||||||
LogicalResult mlir::mergeOffsetsSizesAndStrides(
|
LogicalResult mlir::affine::mergeOffsetsSizesAndStrides(
|
||||||
OpBuilder &builder, Location loc, OffsetSizeAndStrideOpInterface producer,
|
OpBuilder &builder, Location loc, OffsetSizeAndStrideOpInterface producer,
|
||||||
OffsetSizeAndStrideOpInterface consumer,
|
OffsetSizeAndStrideOpInterface consumer,
|
||||||
const llvm::SmallBitVector &droppedProducerDims,
|
const llvm::SmallBitVector &droppedProducerDims,
|
||||||
@ -77,7 +78,7 @@ LogicalResult mlir::mergeOffsetsSizesAndStrides(
|
|||||||
combinedOffsets, combinedSizes, combinedStrides);
|
combinedOffsets, combinedSizes, combinedStrides);
|
||||||
}
|
}
|
||||||
|
|
||||||
void mlir::resolveIndicesIntoOpWithOffsetsAndStrides(
|
void mlir::affine::resolveIndicesIntoOpWithOffsetsAndStrides(
|
||||||
RewriterBase &rewriter, Location loc,
|
RewriterBase &rewriter, Location loc,
|
||||||
ArrayRef<OpFoldResult> mixedSourceOffsets,
|
ArrayRef<OpFoldResult> mixedSourceOffsets,
|
||||||
ArrayRef<OpFoldResult> mixedSourceStrides,
|
ArrayRef<OpFoldResult> mixedSourceStrides,
|
||||||
@ -109,7 +110,7 @@ void mlir::resolveIndicesIntoOpWithOffsetsAndStrides(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void mlir::resolveSizesIntoOpWithSizes(
|
void mlir::affine::resolveSizesIntoOpWithSizes(
|
||||||
ArrayRef<OpFoldResult> sourceSizes, ArrayRef<OpFoldResult> destSizes,
|
ArrayRef<OpFoldResult> sourceSizes, ArrayRef<OpFoldResult> destSizes,
|
||||||
const llvm::SmallBitVector &rankReducedSourceDims,
|
const llvm::SmallBitVector &rankReducedSourceDims,
|
||||||
SmallVectorImpl<OpFoldResult> &resolvedSizes) {
|
SmallVectorImpl<OpFoldResult> &resolvedSizes) {
|
||||||
|
@ -61,7 +61,7 @@ static Value buildLinearThreadId(RewriterBase &rewriter, Location loc,
|
|||||||
rewriter.create<ThreadIdOp>(loc, indexType, Dimension::z).getResult()};
|
rewriter.create<ThreadIdOp>(loc, indexType, Dimension::z).getResult()};
|
||||||
threadsAndWorkGroups.push_back(blockDimsOfr[0]);
|
threadsAndWorkGroups.push_back(blockDimsOfr[0]);
|
||||||
threadsAndWorkGroups.push_back(blockDimsOfr[1]);
|
threadsAndWorkGroups.push_back(blockDimsOfr[1]);
|
||||||
OpFoldResult ofr = makeComposedFoldedAffineApply(
|
OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
|
||||||
rewriter, loc, tx + ty * BDX + tz * BDX * BDY, threadsAndWorkGroups);
|
rewriter, loc, tx + ty * BDX + tz * BDX * BDY, threadsAndWorkGroups);
|
||||||
return getValueOrCreateConstantIndexOp(rewriter, loc, ofr);
|
return getValueOrCreateConstantIndexOp(rewriter, loc, ofr);
|
||||||
}
|
}
|
||||||
@ -137,7 +137,7 @@ struct GpuWarpIdBuilder : public GpuIdBuilder {
|
|||||||
// `forallMappingSizes`.
|
// `forallMappingSizes`.
|
||||||
Value linearId = buildLinearThreadId(rewriter, loc, this->blockDimsOfr);
|
Value linearId = buildLinearThreadId(rewriter, loc, this->blockDimsOfr);
|
||||||
AffineExpr d0 = getAffineDimExpr(0, rewriter.getContext());
|
AffineExpr d0 = getAffineDimExpr(0, rewriter.getContext());
|
||||||
OpFoldResult warpIdOfr = makeComposedFoldedAffineApply(
|
OpFoldResult warpIdOfr = affine::makeComposedFoldedAffineApply(
|
||||||
rewriter, loc, d0.floorDiv(kWarpSize), {linearId});
|
rewriter, loc, d0.floorDiv(kWarpSize), {linearId});
|
||||||
Value warpId = getValueOrCreateConstantIndexOp(rewriter, loc, warpIdOfr);
|
Value warpId = getValueOrCreateConstantIndexOp(rewriter, loc, warpIdOfr);
|
||||||
// Sizes in [x, y, z] -> [z, y x] order to properly compute strides in
|
// Sizes in [x, y, z] -> [z, y x] order to properly compute strides in
|
||||||
@ -149,7 +149,8 @@ struct GpuWarpIdBuilder : public GpuIdBuilder {
|
|||||||
SmallVector<Value> ids;
|
SmallVector<Value> ids;
|
||||||
// Reverse back to be in [x, y, z] order.
|
// Reverse back to be in [x, y, z] order.
|
||||||
for (AffineExpr e : llvm::reverse(delinearizingExprs))
|
for (AffineExpr e : llvm::reverse(delinearizingExprs))
|
||||||
ids.push_back(makeComposedAffineApply(rewriter, loc, e, warpId));
|
ids.push_back(
|
||||||
|
affine::makeComposedAffineApply(rewriter, loc, e, warpId));
|
||||||
|
|
||||||
// clang-format off
|
// clang-format off
|
||||||
LDBG("----linearId: " << linearId);
|
LDBG("----linearId: " << linearId);
|
||||||
@ -204,7 +205,8 @@ struct GpuLinearIdBuilder : public GpuIdBuilder {
|
|||||||
SmallVector<Value> ids;
|
SmallVector<Value> ids;
|
||||||
// Reverse back to be in [x, y, z] order.
|
// Reverse back to be in [x, y, z] order.
|
||||||
for (AffineExpr e : llvm::reverse(delinearizingExprs))
|
for (AffineExpr e : llvm::reverse(delinearizingExprs))
|
||||||
ids.push_back(makeComposedAffineApply(rewriter, loc, e, linearId));
|
ids.push_back(
|
||||||
|
affine::makeComposedAffineApply(rewriter, loc, e, linearId));
|
||||||
|
|
||||||
// clang-format off
|
// clang-format off
|
||||||
LLVM_DEBUG(llvm::interleaveComma(reverseBasisSizes,
|
LLVM_DEBUG(llvm::interleaveComma(reverseBasisSizes,
|
||||||
|
@ -81,8 +81,8 @@ static void insertCopyLoops(ImplicitLocOpBuilder &b, Value from, Value to) {
|
|||||||
GPUDialect::getNumWorkgroupDimensions())))) {
|
GPUDialect::getNumWorkgroupDimensions())))) {
|
||||||
Value v = en.value();
|
Value v = en.value();
|
||||||
auto loop = cast<scf::ForOp>(v.getParentRegion()->getParentOp());
|
auto loop = cast<scf::ForOp>(v.getParentRegion()->getParentOp());
|
||||||
mapLoopToProcessorIds(loop, {threadIds[en.index()]},
|
affine::mapLoopToProcessorIds(loop, {threadIds[en.index()]},
|
||||||
{blockDims[en.index()]});
|
{blockDims[en.index()]});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -636,7 +636,7 @@ LinalgOp::reifyResultShapes(OpBuilder &b,
|
|||||||
Location loc = getOperation()->getLoc();
|
Location loc = getOperation()->getLoc();
|
||||||
IRRewriter rewriter(b);
|
IRRewriter rewriter(b);
|
||||||
SmallVector<OpFoldResult> allResultDimValues =
|
SmallVector<OpFoldResult> allResultDimValues =
|
||||||
makeComposedFoldedMultiResultAffineApply(
|
affine::makeComposedFoldedMultiResultAffineApply(
|
||||||
rewriter, loc, resultShapesFromInputShapesMap,
|
rewriter, loc, resultShapesFromInputShapesMap,
|
||||||
createFlatListOfOperandDims(b, loc));
|
createFlatListOfOperandDims(b, loc));
|
||||||
int64_t pos = 0;
|
int64_t pos = 0;
|
||||||
|
@ -580,7 +580,7 @@ struct FoldInsertPadIntoFill : public OpRewritePattern<tensor::InsertSliceOp> {
|
|||||||
// plus low padding sizes.
|
// plus low padding sizes.
|
||||||
SmallVector<OpFoldResult, 4> newOffsets;
|
SmallVector<OpFoldResult, 4> newOffsets;
|
||||||
for (const auto &p : llvm::zip(lowPads, oldOffsets)) {
|
for (const auto &p : llvm::zip(lowPads, oldOffsets)) {
|
||||||
newOffsets.push_back(makeComposedFoldedAffineApply(
|
newOffsets.push_back(affine::makeComposedFoldedAffineApply(
|
||||||
rewriter, loc, addMap, {std::get<0>(p), std::get<1>(p)}));
|
rewriter, loc, addMap, {std::get<0>(p), std::get<1>(p)}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1169,8 +1169,8 @@ DiagnosedSilenceableFailure transform::MultiTileSizesOp::applyToOne(
|
|||||||
AffineExpr s0 = builder.getAffineSymbolExpr(0);
|
AffineExpr s0 = builder.getAffineSymbolExpr(0);
|
||||||
AffineExpr s1 = builder.getAffineSymbolExpr(1);
|
AffineExpr s1 = builder.getAffineSymbolExpr(1);
|
||||||
Operation *splitPoint =
|
Operation *splitPoint =
|
||||||
makeComposedAffineApply(builder, target.getLoc(), s0 * s1,
|
affine::makeComposedAffineApply(builder, target.getLoc(), s0 * s1,
|
||||||
{spec->lowTileSize, spec->lowTripCount});
|
{spec->lowTileSize, spec->lowTripCount});
|
||||||
Operation *lowTileSize = spec->lowTileSize.getDefiningOp();
|
Operation *lowTileSize = spec->lowTileSize.getDefiningOp();
|
||||||
Operation *highTileSize = spec->highTileSize.getDefiningOp();
|
Operation *highTileSize = spec->highTileSize.getDefiningOp();
|
||||||
assert(lowTileSize && highTileSize && splitPoint &&
|
assert(lowTileSize && highTileSize && splitPoint &&
|
||||||
@ -1420,7 +1420,7 @@ packMatmulGreedily(RewriterBase &rewriter, LinalgOp linalgOp,
|
|||||||
AffineExpr d0, s0;
|
AffineExpr d0, s0;
|
||||||
bindDims(rewriter.getContext(), d0);
|
bindDims(rewriter.getContext(), d0);
|
||||||
bindSymbols(rewriter.getContext(), s0);
|
bindSymbols(rewriter.getContext(), s0);
|
||||||
adjustedPackedSizes.push_back(makeComposedFoldedAffineApply(
|
adjustedPackedSizes.push_back(affine::makeComposedFoldedAffineApply(
|
||||||
rewriter, genericOp->getLoc(), d0.ceilDiv(s0) * s0,
|
rewriter, genericOp->getLoc(), d0.ceilDiv(s0) * s0,
|
||||||
{loopRanges[adjustedPackedSizes.size()].size,
|
{loopRanges[adjustedPackedSizes.size()].size,
|
||||||
rewriter.getIndexAttr(paddedSizesNextMultipleOf[i])}));
|
rewriter.getIndexAttr(paddedSizesNextMultipleOf[i])}));
|
||||||
@ -1983,8 +1983,8 @@ transform::ScalarizeOp::applyToOne(LinalgOp target,
|
|||||||
TrackingListener listener(state, *this);
|
TrackingListener listener(state, *this);
|
||||||
IRRewriter rewriter(getContext(), &listener);
|
IRRewriter rewriter(getContext(), &listener);
|
||||||
SmallVector<OpFoldResult> shapeSizes =
|
SmallVector<OpFoldResult> shapeSizes =
|
||||||
makeComposedFoldedMultiResultAffineApply(rewriter, loc, map,
|
affine::makeComposedFoldedMultiResultAffineApply(rewriter, loc, map,
|
||||||
allShapeSizes);
|
allShapeSizes);
|
||||||
// If the shape size is dynamic, tile by 1.
|
// If the shape size is dynamic, tile by 1.
|
||||||
// Otherwise, do not tile (i.e. tile size 0).
|
// Otherwise, do not tile (i.e. tile size 0).
|
||||||
for (OpFoldResult shapeSize : shapeSizes) {
|
for (OpFoldResult shapeSize : shapeSizes) {
|
||||||
@ -3351,7 +3351,7 @@ public:
|
|||||||
void init() {
|
void init() {
|
||||||
declareDependentDialect<pdl::PDLDialect>();
|
declareDependentDialect<pdl::PDLDialect>();
|
||||||
declareDependentDialect<LinalgDialect>();
|
declareDependentDialect<LinalgDialect>();
|
||||||
declareGeneratedDialect<AffineDialect>();
|
declareGeneratedDialect<affine::AffineDialect>();
|
||||||
declareGeneratedDialect<arith::ArithDialect>();
|
declareGeneratedDialect<arith::ArithDialect>();
|
||||||
declareGeneratedDialect<scf::SCFDialect>();
|
declareGeneratedDialect<scf::SCFDialect>();
|
||||||
declareGeneratedDialect<vector::VectorDialect>();
|
declareGeneratedDialect<vector::VectorDialect>();
|
||||||
|
@ -96,7 +96,7 @@ struct BubbleUpExtractSliceOpPattern
|
|||||||
linalgOp, "failed to get loops map from shape sizes");
|
linalgOp, "failed to get loops map from shape sizes");
|
||||||
}
|
}
|
||||||
SmallVector<OpFoldResult> sizeBounds =
|
SmallVector<OpFoldResult> sizeBounds =
|
||||||
makeComposedFoldedMultiResultAffineApply(
|
affine::makeComposedFoldedMultiResultAffineApply(
|
||||||
rewriter, linalgLoc, shapeSizesToLoopsMap, allShapeSizes);
|
rewriter, linalgLoc, shapeSizesToLoopsMap, allShapeSizes);
|
||||||
|
|
||||||
// The offsets and sizes from the slice operation only give you the tile
|
// The offsets and sizes from the slice operation only give you the tile
|
||||||
|
@ -55,7 +55,7 @@ static SmallVector<Value> unrollIndex(OpBuilder &b, Location loc, Value index,
|
|||||||
for (int64_t f : factors)
|
for (int64_t f : factors)
|
||||||
basis.push_back(b.create<arith::ConstantOp>(loc, b.getIndexAttr(f)));
|
basis.push_back(b.create<arith::ConstantOp>(loc, b.getIndexAttr(f)));
|
||||||
FailureOr<SmallVector<Value>> multiIndex =
|
FailureOr<SmallVector<Value>> multiIndex =
|
||||||
delinearizeIndex(b, loc, index, basis);
|
affine::delinearizeIndex(b, loc, index, basis);
|
||||||
assert(!failed(multiIndex) && "Failed to linearize img2col index");
|
assert(!failed(multiIndex) && "Failed to linearize img2col index");
|
||||||
return *multiIndex;
|
return *multiIndex;
|
||||||
}
|
}
|
||||||
@ -68,7 +68,8 @@ static Value getConvolvedIndex(OpBuilder &b, Location loc, Value oIndex,
|
|||||||
AffineExpr oExpr, fExpr;
|
AffineExpr oExpr, fExpr;
|
||||||
bindSymbols(b.getContext(), oExpr, fExpr);
|
bindSymbols(b.getContext(), oExpr, fExpr);
|
||||||
AffineMap convMap = AffineMap::get(0, 2, stride * oExpr + fExpr);
|
AffineMap convMap = AffineMap::get(0, 2, stride * oExpr + fExpr);
|
||||||
return makeComposedAffineApply(b, loc, convMap, ValueRange{oIndex, fIndex});
|
return affine::makeComposedAffineApply(b, loc, convMap,
|
||||||
|
ValueRange{oIndex, fIndex});
|
||||||
}
|
}
|
||||||
|
|
||||||
FailureOr<std::pair<Operation *, Operation *>>
|
FailureOr<std::pair<Operation *, Operation *>>
|
||||||
|
@ -111,8 +111,8 @@ static SmallVector<OpFoldResult> getGenericOpLoopRange(OpBuilder &b,
|
|||||||
cast<LinalgOp>(op.getOperation()).createFlatListOfOperandDims(b, loc);
|
cast<LinalgOp>(op.getOperation()).createFlatListOfOperandDims(b, loc);
|
||||||
AffineMap map = op.getShapesToLoopsMap();
|
AffineMap map = op.getShapesToLoopsMap();
|
||||||
IRRewriter rewriter(b);
|
IRRewriter rewriter(b);
|
||||||
return makeComposedFoldedMultiResultAffineApply(rewriter, loc, map,
|
return affine::makeComposedFoldedMultiResultAffineApply(rewriter, loc, map,
|
||||||
allShapesSizes);
|
allShapesSizes);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Helper method to permute the list of `values` based on the `map`.
|
/// Helper method to permute the list of `values` based on the `map`.
|
||||||
|
@ -179,7 +179,7 @@ static void generateFusedElementwiseOpRegion(
|
|||||||
});
|
});
|
||||||
for (IndexOp indexOp :
|
for (IndexOp indexOp :
|
||||||
llvm::make_early_inc_range(producerBlock.getOps<IndexOp>())) {
|
llvm::make_early_inc_range(producerBlock.getOps<IndexOp>())) {
|
||||||
Value newIndex = rewriter.create<mlir::AffineApplyOp>(
|
Value newIndex = rewriter.create<affine::AffineApplyOp>(
|
||||||
producer.getLoc(),
|
producer.getLoc(),
|
||||||
consumerToProducerLoopsMap.getSubMap(indexOp.getDim()), fusedIndices);
|
consumerToProducerLoopsMap.getSubMap(indexOp.getDim()), fusedIndices);
|
||||||
mapper.map(indexOp.getResult(), newIndex);
|
mapper.map(indexOp.getResult(), newIndex);
|
||||||
@ -719,7 +719,7 @@ static void updateExpandedGenericOpRegion(PatternRewriter &rewriter,
|
|||||||
assert(!ShapedType::isDynamic(std::get<0>(it)));
|
assert(!ShapedType::isDynamic(std::get<0>(it)));
|
||||||
AffineExpr idx, acc;
|
AffineExpr idx, acc;
|
||||||
bindDims(rewriter.getContext(), idx, acc);
|
bindDims(rewriter.getContext(), idx, acc);
|
||||||
newIndex = rewriter.create<AffineApplyOp>(
|
newIndex = rewriter.create<affine::AffineApplyOp>(
|
||||||
indexOp.getLoc(), idx + acc * std::get<0>(it),
|
indexOp.getLoc(), idx + acc * std::get<0>(it),
|
||||||
ValueRange{std::get<1>(it), newIndex});
|
ValueRange{std::get<1>(it), newIndex});
|
||||||
}
|
}
|
||||||
@ -1871,7 +1871,7 @@ struct LinalgElementwiseOpFusionPass
|
|||||||
populateFoldReshapeOpsByExpansionPatterns(patterns, defaultControlFn);
|
populateFoldReshapeOpsByExpansionPatterns(patterns, defaultControlFn);
|
||||||
|
|
||||||
// General canonicalization patterns.
|
// General canonicalization patterns.
|
||||||
AffineApplyOp::getCanonicalizationPatterns(patterns, context);
|
affine::AffineApplyOp::getCanonicalizationPatterns(patterns, context);
|
||||||
GenericOp::getCanonicalizationPatterns(patterns, context);
|
GenericOp::getCanonicalizationPatterns(patterns, context);
|
||||||
tensor::ExpandShapeOp::getCanonicalizationPatterns(patterns, context);
|
tensor::ExpandShapeOp::getCanonicalizationPatterns(patterns, context);
|
||||||
tensor::CollapseShapeOp::getCanonicalizationPatterns(patterns, context);
|
tensor::CollapseShapeOp::getCanonicalizationPatterns(patterns, context);
|
||||||
|
@ -462,7 +462,7 @@ HoistPaddingAnalysis::getHoistedPackedTensorSizes(RewriterBase &rewriter,
|
|||||||
// of the enclosing loops.
|
// of the enclosing loops.
|
||||||
for (auto forOp : packingLoops) {
|
for (auto forOp : packingLoops) {
|
||||||
// Compute an upper bound `ubVal` for the upper bound of `forOp`.
|
// Compute an upper bound `ubVal` for the upper bound of `forOp`.
|
||||||
FailureOr<OpFoldResult> loopUb = reifyIndexValueBound(
|
FailureOr<OpFoldResult> loopUb = affine::reifyIndexValueBound(
|
||||||
rewriter, loc, presburger::BoundType::UB, forOp.getUpperBound(),
|
rewriter, loc, presburger::BoundType::UB, forOp.getUpperBound(),
|
||||||
/*stopCondition=*/
|
/*stopCondition=*/
|
||||||
[&](Value v, std::optional<int64_t> d) {
|
[&](Value v, std::optional<int64_t> d) {
|
||||||
@ -472,7 +472,8 @@ HoistPaddingAnalysis::getHoistedPackedTensorSizes(RewriterBase &rewriter,
|
|||||||
Operation *op = v.getDefiningOp();
|
Operation *op = v.getDefiningOp();
|
||||||
if (!op)
|
if (!op)
|
||||||
return true;
|
return true;
|
||||||
return !isa<AffineMinOp, AffineMaxOp, AffineApplyOp>(op);
|
return !isa<affine::AffineMinOp, affine::AffineMaxOp,
|
||||||
|
affine::AffineApplyOp>(op);
|
||||||
},
|
},
|
||||||
/*closedUB=*/true);
|
/*closedUB=*/true);
|
||||||
assert(succeeded(loopUb) && "could not get upper bound");
|
assert(succeeded(loopUb) && "could not get upper bound");
|
||||||
@ -485,7 +486,7 @@ HoistPaddingAnalysis::getHoistedPackedTensorSizes(RewriterBase &rewriter,
|
|||||||
AffineExpr lb, ub, step;
|
AffineExpr lb, ub, step;
|
||||||
bindDims(rewriter.getContext(), lb, ub);
|
bindDims(rewriter.getContext(), lb, ub);
|
||||||
bindSymbols(rewriter.getContext(), step);
|
bindSymbols(rewriter.getContext(), step);
|
||||||
Value res = rewriter.createOrFold<AffineApplyOp>(
|
Value res = rewriter.createOrFold<affine::AffineApplyOp>(
|
||||||
loc, (ub - lb).ceilDiv(step),
|
loc, (ub - lb).ceilDiv(step),
|
||||||
ValueRange{forOp.getLowerBound(), ubVal,
|
ValueRange{forOp.getLowerBound(), ubVal,
|
||||||
cast<scf::ForOp>(forOp).getStep()});
|
cast<scf::ForOp>(forOp).getStep()});
|
||||||
@ -519,7 +520,7 @@ static Value buildLoopIterationCount(RewriterBase &rewriter, scf::ForOp outer,
|
|||||||
Value ivVal = forOp.getInductionVar(), lbVal = forOp.getLowerBound(),
|
Value ivVal = forOp.getInductionVar(), lbVal = forOp.getLowerBound(),
|
||||||
stepVal = forOp.getStep();
|
stepVal = forOp.getStep();
|
||||||
auto loc = forOp->getLoc();
|
auto loc = forOp->getLoc();
|
||||||
return rewriter.createOrFold<AffineApplyOp>(
|
return rewriter.createOrFold<affine::AffineApplyOp>(
|
||||||
loc, (iv - lb).ceilDiv(step), ValueRange{ivVal, lbVal, stepVal});
|
loc, (iv - lb).ceilDiv(step), ValueRange{ivVal, lbVal, stepVal});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -94,7 +94,7 @@ void mlir::linalg::hoistRedundantVectorTransfers(func::FuncOp func) {
|
|||||||
auto loop = dyn_cast<LoopLikeOpInterface>(transferRead->getParentOp());
|
auto loop = dyn_cast<LoopLikeOpInterface>(transferRead->getParentOp());
|
||||||
LLVM_DEBUG(DBGS() << "Parent op: " << *transferRead->getParentOp()
|
LLVM_DEBUG(DBGS() << "Parent op: " << *transferRead->getParentOp()
|
||||||
<< "\n");
|
<< "\n");
|
||||||
if (!isa_and_nonnull<scf::ForOp, AffineForOp>(loop))
|
if (!isa_and_nonnull<scf::ForOp, affine::AffineForOp>(loop))
|
||||||
return WalkResult::advance();
|
return WalkResult::advance();
|
||||||
|
|
||||||
LLVM_DEBUG(DBGS() << "Candidate read: " << *transferRead.getOperation()
|
LLVM_DEBUG(DBGS() << "Candidate read: " << *transferRead.getOperation()
|
||||||
@ -200,7 +200,7 @@ void mlir::linalg::hoistRedundantVectorTransfers(func::FuncOp func) {
|
|||||||
// the walk.
|
// the walk.
|
||||||
return WalkResult::interrupt();
|
return WalkResult::interrupt();
|
||||||
})
|
})
|
||||||
.Case<AffineForOp>([&](AffineForOp affineForOp) {
|
.Case<affine::AffineForOp>([&](affine::AffineForOp affineForOp) {
|
||||||
auto newForOp = replaceForOpWithNewYields(
|
auto newForOp = replaceForOpWithNewYields(
|
||||||
b, affineForOp, transferRead.getVector(),
|
b, affineForOp, transferRead.getVector(),
|
||||||
SmallVector<Value>{transferWrite.getVector()},
|
SmallVector<Value>{transferWrite.getVector()},
|
||||||
|
@ -96,7 +96,7 @@ mlir::linalg::interchangeGenericOp(RewriterBase &rewriter, GenericOp genericOp,
|
|||||||
std::back_inserter(allIndices), [&](uint64_t dim) {
|
std::back_inserter(allIndices), [&](uint64_t dim) {
|
||||||
return rewriter.create<IndexOp>(indexOp->getLoc(), dim);
|
return rewriter.create<IndexOp>(indexOp->getLoc(), dim);
|
||||||
});
|
});
|
||||||
rewriter.replaceOpWithNewOp<AffineApplyOp>(
|
rewriter.replaceOpWithNewOp<affine::AffineApplyOp>(
|
||||||
indexOp, permutationMap.getSubMap(indexOp.getDim()), allIndices);
|
indexOp, permutationMap.getSubMap(indexOp.getDim()), allIndices);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -49,8 +49,8 @@ static SmallVector<Value> makeCanonicalAffineApplies(OpBuilder &b, Location loc,
|
|||||||
for (auto e : map.getResults()) {
|
for (auto e : map.getResults()) {
|
||||||
auto exprMap = AffineMap::get(dims, map.getNumSymbols(), e);
|
auto exprMap = AffineMap::get(dims, map.getNumSymbols(), e);
|
||||||
SmallVector<Value> operands(vals.begin(), vals.end());
|
SmallVector<Value> operands(vals.begin(), vals.end());
|
||||||
canonicalizeMapAndOperands(&exprMap, &operands);
|
affine::canonicalizeMapAndOperands(&exprMap, &operands);
|
||||||
res.push_back(b.create<AffineApplyOp>(loc, exprMap, operands));
|
res.push_back(b.create<affine::AffineApplyOp>(loc, exprMap, operands));
|
||||||
}
|
}
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
@ -189,7 +189,7 @@ static void replaceIndexOpsByInductionVariables(RewriterBase &rewriter,
|
|||||||
.Case([&](scf::ForOp forOp) {
|
.Case([&](scf::ForOp forOp) {
|
||||||
allIvs.push_back(forOp.getInductionVar());
|
allIvs.push_back(forOp.getInductionVar());
|
||||||
})
|
})
|
||||||
.Case([&](AffineForOp affineForOp) {
|
.Case([&](affine::AffineForOp affineForOp) {
|
||||||
allIvs.push_back(affineForOp.getInductionVar());
|
allIvs.push_back(affineForOp.getInductionVar());
|
||||||
})
|
})
|
||||||
.Default([&](Operation *op) { assert(false && "unexpected op"); });
|
.Default([&](Operation *op) { assert(false && "unexpected op"); });
|
||||||
@ -208,10 +208,12 @@ static void replaceIndexOpsByInductionVariables(RewriterBase &rewriter,
|
|||||||
template <typename LoopTy>
|
template <typename LoopTy>
|
||||||
static FailureOr<LinalgLoops> linalgOpToLoopsImpl(RewriterBase &rewriter,
|
static FailureOr<LinalgLoops> linalgOpToLoopsImpl(RewriterBase &rewriter,
|
||||||
LinalgOp linalgOp) {
|
LinalgOp linalgOp) {
|
||||||
using LoadOpTy = std::conditional_t<std::is_same<LoopTy, AffineForOp>::value,
|
using LoadOpTy =
|
||||||
AffineLoadOp, memref::LoadOp>;
|
std::conditional_t<std::is_same<LoopTy, affine::AffineForOp>::value,
|
||||||
using StoreOpTy = std::conditional_t<std::is_same<LoopTy, AffineForOp>::value,
|
affine::AffineLoadOp, memref::LoadOp>;
|
||||||
AffineStoreOp, memref::StoreOp>;
|
using StoreOpTy =
|
||||||
|
std::conditional_t<std::is_same<LoopTy, affine::AffineForOp>::value,
|
||||||
|
affine::AffineStoreOp, memref::StoreOp>;
|
||||||
|
|
||||||
// The flattened loopToOperandRangesMaps is expected to be an invertible
|
// The flattened loopToOperandRangesMaps is expected to be an invertible
|
||||||
// permutation map (which is asserted in the inverse calculation).
|
// permutation map (which is asserted in the inverse calculation).
|
||||||
@ -284,11 +286,11 @@ public:
|
|||||||
/// other cases, it is replaced by its unique operand.
|
/// other cases, it is replaced by its unique operand.
|
||||||
struct FoldAffineOp : public RewritePattern {
|
struct FoldAffineOp : public RewritePattern {
|
||||||
FoldAffineOp(MLIRContext *context)
|
FoldAffineOp(MLIRContext *context)
|
||||||
: RewritePattern(AffineApplyOp::getOperationName(), 0, context) {}
|
: RewritePattern(affine::AffineApplyOp::getOperationName(), 0, context) {}
|
||||||
|
|
||||||
LogicalResult matchAndRewrite(Operation *op,
|
LogicalResult matchAndRewrite(Operation *op,
|
||||||
PatternRewriter &rewriter) const override {
|
PatternRewriter &rewriter) const override {
|
||||||
AffineApplyOp affineApplyOp = cast<AffineApplyOp>(op);
|
auto affineApplyOp = cast<affine::AffineApplyOp>(op);
|
||||||
auto map = affineApplyOp.getAffineMap();
|
auto map = affineApplyOp.getAffineMap();
|
||||||
if (map.getNumResults() != 1 || map.getNumInputs() > 1)
|
if (map.getNumResults() != 1 || map.getNumInputs() > 1)
|
||||||
return failure();
|
return failure();
|
||||||
@ -316,7 +318,7 @@ static void lowerLinalgToLoopsImpl(func::FuncOp funcOp) {
|
|||||||
patterns.add<LinalgRewritePattern<LoopType>>(context);
|
patterns.add<LinalgRewritePattern<LoopType>>(context);
|
||||||
memref::DimOp::getCanonicalizationPatterns(patterns, context);
|
memref::DimOp::getCanonicalizationPatterns(patterns, context);
|
||||||
tensor::DimOp::getCanonicalizationPatterns(patterns, context);
|
tensor::DimOp::getCanonicalizationPatterns(patterns, context);
|
||||||
AffineApplyOp::getCanonicalizationPatterns(patterns, context);
|
affine::AffineApplyOp::getCanonicalizationPatterns(patterns, context);
|
||||||
patterns.add<FoldAffineOp>(context);
|
patterns.add<FoldAffineOp>(context);
|
||||||
// Just apply the patterns greedily.
|
// Just apply the patterns greedily.
|
||||||
(void)applyPatternsAndFoldGreedily(funcOp, std::move(patterns));
|
(void)applyPatternsAndFoldGreedily(funcOp, std::move(patterns));
|
||||||
@ -328,7 +330,7 @@ struct LowerToAffineLoops
|
|||||||
registry.insert<memref::MemRefDialect>();
|
registry.insert<memref::MemRefDialect>();
|
||||||
}
|
}
|
||||||
void runOnOperation() override {
|
void runOnOperation() override {
|
||||||
lowerLinalgToLoopsImpl<AffineForOp>(getOperation());
|
lowerLinalgToLoopsImpl<affine::AffineForOp>(getOperation());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -368,7 +370,7 @@ mlir::createConvertLinalgToAffineLoopsPass() {
|
|||||||
/// Emits a loop nest of `affine.for` with the proper body for `linalgOp`.
|
/// Emits a loop nest of `affine.for` with the proper body for `linalgOp`.
|
||||||
FailureOr<LinalgLoops>
|
FailureOr<LinalgLoops>
|
||||||
mlir::linalg::linalgOpToAffineLoops(RewriterBase &rewriter, LinalgOp linalgOp) {
|
mlir::linalg::linalgOpToAffineLoops(RewriterBase &rewriter, LinalgOp linalgOp) {
|
||||||
return linalgOpToLoopsImpl<AffineForOp>(rewriter, linalgOp);
|
return linalgOpToLoopsImpl<affine::AffineForOp>(rewriter, linalgOp);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Emits a loop nest of `scf.for` with the proper body for `linalgOp`.
|
/// Emits a loop nest of `scf.for` with the proper body for `linalgOp`.
|
||||||
|
@ -81,14 +81,14 @@ linalg::splitOp(RewriterBase &rewriter, TilingInterface op, unsigned dimension,
|
|||||||
// Adjust the split point so that it doesn't overflow the size.
|
// Adjust the split point so that it doesn't overflow the size.
|
||||||
AffineExpr d0, d1, d2;
|
AffineExpr d0, d1, d2;
|
||||||
bindDims(rewriter.getContext(), d0, d1, d2);
|
bindDims(rewriter.getContext(), d0, d1, d2);
|
||||||
OpFoldResult minSplitPoint = makeComposedFoldedAffineMin(
|
OpFoldResult minSplitPoint = affine::makeComposedFoldedAffineMin(
|
||||||
rewriter, op.getLoc(),
|
rewriter, op.getLoc(),
|
||||||
AffineMap::inferFromExprList(ArrayRef<AffineExpr>{d0, d1 + d2}).front(),
|
AffineMap::inferFromExprList(ArrayRef<AffineExpr>{d0, d1 + d2}).front(),
|
||||||
{splitPoint, offsets[dimension], sizes[dimension]});
|
{splitPoint, offsets[dimension], sizes[dimension]});
|
||||||
|
|
||||||
// Compute the size of the second part. Return early if the second part would
|
// Compute the size of the second part. Return early if the second part would
|
||||||
// have an empty iteration space.
|
// have an empty iteration space.
|
||||||
OpFoldResult remainingSize = makeComposedFoldedAffineApply(
|
OpFoldResult remainingSize = affine::makeComposedFoldedAffineApply(
|
||||||
rewriter, op.getLoc(), d0 + d1 - d2,
|
rewriter, op.getLoc(), d0 + d1 - d2,
|
||||||
{iterationSpace[dimension].offset, iterationSpace[dimension].size,
|
{iterationSpace[dimension].offset, iterationSpace[dimension].size,
|
||||||
minSplitPoint});
|
minSplitPoint});
|
||||||
@ -121,7 +121,7 @@ linalg::splitOp(RewriterBase &rewriter, TilingInterface op, unsigned dimension,
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Create the second part.
|
// Create the second part.
|
||||||
OpFoldResult totalOffset = makeComposedFoldedAffineApply(
|
OpFoldResult totalOffset = affine::makeComposedFoldedAffineApply(
|
||||||
rewriter, op.getLoc(), d0 + d1, {offsets[dimension], minSplitPoint});
|
rewriter, op.getLoc(), d0 + d1, {offsets[dimension], minSplitPoint});
|
||||||
SmallVector<Value> secondResults;
|
SmallVector<Value> secondResults;
|
||||||
TilingInterface secondPart =
|
TilingInterface secondPart =
|
||||||
|
@ -39,6 +39,7 @@ namespace mlir {
|
|||||||
} // namespace mlir
|
} // namespace mlir
|
||||||
|
|
||||||
using namespace mlir;
|
using namespace mlir;
|
||||||
|
using namespace mlir::affine;
|
||||||
using namespace mlir::linalg;
|
using namespace mlir::linalg;
|
||||||
using namespace mlir::scf;
|
using namespace mlir::scf;
|
||||||
|
|
||||||
@ -178,7 +179,7 @@ mlir::linalg::computeMultiTileSizes(OpBuilder &builder, LinalgOp op,
|
|||||||
AffineExpr s1 = b.getAffineSymbolExpr(1);
|
AffineExpr s1 = b.getAffineSymbolExpr(1);
|
||||||
AffineExpr s2 = b.getAffineSymbolExpr(2);
|
AffineExpr s2 = b.getAffineSymbolExpr(2);
|
||||||
auto apply = [&](AffineExpr expr, ValueRange values) -> Value {
|
auto apply = [&](AffineExpr expr, ValueRange values) -> Value {
|
||||||
return makeComposedAffineApply(b, b.getLoc(), expr, values);
|
return affine::makeComposedAffineApply(b, b.getLoc(), expr, values);
|
||||||
};
|
};
|
||||||
Value a = apply(s0.floorDiv(s1), {tripCount, divisorValue});
|
Value a = apply(s0.floorDiv(s1), {tripCount, divisorValue});
|
||||||
Value t = apply((s0 + s1 - 1).floorDiv(s1), {targetSizeValue, divisorValue});
|
Value t = apply((s0 + s1 - 1).floorDiv(s1), {targetSizeValue, divisorValue});
|
||||||
@ -228,7 +229,7 @@ static bool canOmitTileOffsetInBoundsCheck(OpFoldResult tileSize,
|
|||||||
/// Build an `affine_max` of all the `vals`.
|
/// Build an `affine_max` of all the `vals`.
|
||||||
static OpFoldResult buildMax(OpBuilder &b, Location loc,
|
static OpFoldResult buildMax(OpBuilder &b, Location loc,
|
||||||
ArrayRef<OpFoldResult> vals) {
|
ArrayRef<OpFoldResult> vals) {
|
||||||
return makeComposedFoldedAffineMax(
|
return affine::makeComposedFoldedAffineMax(
|
||||||
b, loc, AffineMap::getMultiDimIdentityMap(vals.size(), loc.getContext()),
|
b, loc, AffineMap::getMultiDimIdentityMap(vals.size(), loc.getContext()),
|
||||||
vals);
|
vals);
|
||||||
}
|
}
|
||||||
@ -236,7 +237,7 @@ static OpFoldResult buildMax(OpBuilder &b, Location loc,
|
|||||||
/// Build an `affine_min` of all the `vals`.
|
/// Build an `affine_min` of all the `vals`.
|
||||||
static OpFoldResult buildMin(OpBuilder &b, Location loc,
|
static OpFoldResult buildMin(OpBuilder &b, Location loc,
|
||||||
ArrayRef<OpFoldResult> vals) {
|
ArrayRef<OpFoldResult> vals) {
|
||||||
return makeComposedFoldedAffineMin(
|
return affine::makeComposedFoldedAffineMin(
|
||||||
b, loc, AffineMap::getMultiDimIdentityMap(vals.size(), loc.getContext()),
|
b, loc, AffineMap::getMultiDimIdentityMap(vals.size(), loc.getContext()),
|
||||||
vals);
|
vals);
|
||||||
}
|
}
|
||||||
@ -968,10 +969,10 @@ mlir::linalg::getLinalgTilingCanonicalizationPatterns(MLIRContext *ctx) {
|
|||||||
void mlir::linalg::populateLinalgTilingCanonicalizationPatterns(
|
void mlir::linalg::populateLinalgTilingCanonicalizationPatterns(
|
||||||
RewritePatternSet &patterns) {
|
RewritePatternSet &patterns) {
|
||||||
auto *ctx = patterns.getContext();
|
auto *ctx = patterns.getContext();
|
||||||
AffineApplyOp::getCanonicalizationPatterns(patterns, ctx);
|
affine::AffineApplyOp::getCanonicalizationPatterns(patterns, ctx);
|
||||||
AffineForOp::getCanonicalizationPatterns(patterns, ctx);
|
affine::AffineForOp::getCanonicalizationPatterns(patterns, ctx);
|
||||||
AffineMinOp::getCanonicalizationPatterns(patterns, ctx);
|
affine::AffineMinOp::getCanonicalizationPatterns(patterns, ctx);
|
||||||
AffineMaxOp::getCanonicalizationPatterns(patterns, ctx);
|
affine::AffineMaxOp::getCanonicalizationPatterns(patterns, ctx);
|
||||||
arith::ConstantIndexOp::getCanonicalizationPatterns(patterns, ctx);
|
arith::ConstantIndexOp::getCanonicalizationPatterns(patterns, ctx);
|
||||||
|
|
||||||
memref::SubViewOp::getCanonicalizationPatterns(patterns, ctx);
|
memref::SubViewOp::getCanonicalizationPatterns(patterns, ctx);
|
||||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user