[flang][OpenMP] Revert array element reduction lowering
Revert #196094 and its follow-up #209701. The expression override mechanism does not preserve reduction-object identity across all data environments, leading to incorrect lowering for procedure-local and nested private arrays. Restore the pre-#196094 lowering. Keep coverage showing that array-element constructs compile through the boxed-array reduction path. This was in response to this comment: https://github.com/llvm/llvm-project/pull/196094#issuecomment-5248085956 Fixing exactly the bug in the comment wasn't hard but AI code review found a large number of follow on bugs so I think the design needs a rethink, and definitely shouldn't be included in the LLVM release. Assisted-by: Codex
This commit is contained in:
1 parent
55feb0a3b6
commit
2162da3e36
10 files changed
+115
-561
No files matched your search
@@ -13,7 +13,6 @@
|
||||
#ifndef FORTRAN_LOWER_REDUCTIONPROCESSOR_H
|
||||
#define FORTRAN_LOWER_REDUCTIONPROCESSOR_H
|
||||
|
||||
#include "flang/Lower/AbstractConverter.h"
|
||||
#include "flang/Lower/OpenMP/Clauses.h"
|
||||
#include "flang/Optimizer/Builder/FIRBuilder.h"
|
||||
#include "flang/Optimizer/Dialect/FIRType.h"
|
||||
@@ -22,7 +21,6 @@
|
||||
#include "flang/Semantics/type.h"
|
||||
#include "mlir/IR/Location.h"
|
||||
#include "mlir/IR/Types.h"
|
||||
#include "llvm/ADT/ArrayRef.h"
|
||||
|
||||
namespace mlir {
|
||||
namespace omp {
|
||||
@@ -170,16 +168,8 @@ public:
|
||||
llvm::SmallVectorImpl<bool> &reduceVarByRef,
|
||||
llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
|
||||
const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
|
||||
llvm::ArrayRef<Object> reductionObjects, lower::SymMap &symMap,
|
||||
llvm::DenseMap<const semantics::Symbol *, mlir::Value>
|
||||
*reductionVarCache = nullptr);
|
||||
|
||||
/// Check if an expression is lowered as a Reduction object. This ensures
|
||||
/// reductions such as Array Elements are properly represented, rather than
|
||||
/// reducing the full array.
|
||||
// TODO support more types of objects
|
||||
// to avoid Reduction clauses being represented in FIR as full arrays.
|
||||
static bool isExpressionLoweredAsReductionObject(const Object *object);
|
||||
};
|
||||
|
||||
template <typename FloatOp, typename IntegerOp>
|
||||
|
||||
@@ -2416,8 +2416,7 @@ private:
|
||||
Fortran::lower::omp::ReductionProcessor rp;
|
||||
bool result = rp.processReductionArguments<fir::DeclareReductionOp>(
|
||||
toLocation(), *this, info.reduceOperatorList, reduceVars,
|
||||
reduceVarByRef, reductionDeclSymbols, info.reduceSymList,
|
||||
/*reductionObjects=*/{}, getSymbolMap());
|
||||
reduceVarByRef, reductionDeclSymbols, info.reduceSymList);
|
||||
if (!result)
|
||||
TODO(toLocation(), "Lowering unrecognised reduction type");
|
||||
|
||||
|
||||
@@ -975,20 +975,11 @@ private:
|
||||
mlir::Location loc;
|
||||
};
|
||||
|
||||
static mlir::Value
|
||||
findOverriddenExprValue(const Fortran::lower::ExprToValueMap &map,
|
||||
const Fortran::lower::SomeExpr &expr);
|
||||
|
||||
hlfir::EntityWithAttributes HlfirDesignatorBuilder::genDesignatorExpr(
|
||||
const Fortran::lower::SomeExpr &designatorExpr,
|
||||
bool vectorSubscriptDesignatorToValue) {
|
||||
// Expr<SomeType> plumbing to unwrap Designator<T> and call
|
||||
// gen(Designator<T>.u).
|
||||
if (const Fortran::lower::ExprToValueMap *map =
|
||||
getConverter().getExprOverrides()) {
|
||||
if (mlir::Value value = findOverriddenExprValue(*map, designatorExpr))
|
||||
return hlfir::EntityWithAttributes{value};
|
||||
}
|
||||
return Fortran::common::visit(
|
||||
[&](const auto &x) -> hlfir::EntityWithAttributes {
|
||||
using T = std::decay_t<decltype(x)>;
|
||||
@@ -1561,30 +1552,6 @@ static bool hasDeferredCharacterLength(const Fortran::semantics::Symbol &sym) {
|
||||
type->characterTypeSpec().length().isDeferred();
|
||||
}
|
||||
|
||||
static mlir::Value
|
||||
findOverriddenExprValue(const Fortran::lower::ExprToValueMap &map,
|
||||
const Fortran::lower::SomeExpr &expr) {
|
||||
if (auto match = map.find(&expr); match != map.end())
|
||||
return match->second;
|
||||
|
||||
// The map uses pointer identity, but the some expressions
|
||||
// (e.g. a(2)) may appear at multiple AST nodes with different addresses.
|
||||
// Fall back to structural comparison via ArrayRef::operator==.
|
||||
for (auto [key, value] : map) {
|
||||
if (Fortran::lower::isEqual(key, &expr))
|
||||
return value;
|
||||
auto keyRef = Fortran::evaluate::ExtractDataRef(*key);
|
||||
auto exprRef = Fortran::evaluate::ExtractDataRef(expr);
|
||||
if (keyRef && exprRef) {
|
||||
auto *keyArray = std::get_if<Fortran::evaluate::ArrayRef>(&keyRef->u);
|
||||
auto *exprArray = std::get_if<Fortran::evaluate::ArrayRef>(&exprRef->u);
|
||||
if (keyArray && exprArray && *keyArray == *exprArray)
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
/// Lower Expr to HLFIR.
|
||||
class HlfirBuilder {
|
||||
public:
|
||||
@@ -1598,12 +1565,12 @@ public:
|
||||
if (const Fortran::lower::ExprToValueMap *map =
|
||||
getConverter().getExprOverrides()) {
|
||||
if constexpr (std::is_same_v<T, Fortran::evaluate::SomeType>) {
|
||||
if (mlir::Value value = findOverriddenExprValue(*map, expr))
|
||||
return hlfir::EntityWithAttributes{value};
|
||||
if (auto match = map->find(&expr); match != map->end())
|
||||
return hlfir::EntityWithAttributes{match->second};
|
||||
} else {
|
||||
Fortran::lower::SomeExpr someExpr = toEvExpr(expr);
|
||||
if (mlir::Value value = findOverriddenExprValue(*map, someExpr))
|
||||
return hlfir::EntityWithAttributes{value};
|
||||
if (auto match = map->find(&someExpr); match != map->end())
|
||||
return hlfir::EntityWithAttributes{match->second};
|
||||
}
|
||||
}
|
||||
return Fortran::common::visit([&](const auto &x) { return gen(x); },
|
||||
@@ -1645,12 +1612,6 @@ private:
|
||||
template <typename T>
|
||||
hlfir::EntityWithAttributes
|
||||
gen(const Fortran::evaluate::Designator<T> &designator) {
|
||||
if (const Fortran::lower::ExprToValueMap *map =
|
||||
getConverter().getExprOverrides()) {
|
||||
Fortran::lower::SomeExpr someExpr = toEvExpr(designator);
|
||||
if (mlir::Value value = findOverriddenExprValue(*map, someExpr))
|
||||
return hlfir::EntityWithAttributes{value};
|
||||
}
|
||||
return HlfirDesignatorBuilder(getLoc(), getConverter(), getSymMap(),
|
||||
getStmtCtx())
|
||||
.gen(designator.u);
|
||||
|
||||
@@ -1680,7 +1680,7 @@ bool ClauseProcessor::processInReduction(
|
||||
currentLocation, converter,
|
||||
std::get<typename omp::clause::ReductionOperatorList>(clause.t),
|
||||
inReductionVars, inReduceVarByRef, inReductionDeclSymbols,
|
||||
inReductionSyms, inReductionObjects, converter.getSymbolMap()))
|
||||
inReductionSyms))
|
||||
TODO(currentLocation, "Lowering unrecognised reduction type");
|
||||
|
||||
// Copy local lists into the output.
|
||||
@@ -2098,8 +2098,7 @@ bool ClauseProcessor::processReduction(
|
||||
currentLocation, converter,
|
||||
std::get<typename omp::clause::ReductionOperatorList>(clause.t),
|
||||
reductionVars, reduceVarByRef, reductionDeclSymbols,
|
||||
reductionSyms, reductionObjects, converter.getSymbolMap(),
|
||||
reductionVarCache))
|
||||
reductionSyms, reductionVarCache))
|
||||
TODO(currentLocation, "Lowering unrecognised reduction type");
|
||||
// Copy local lists into the output.
|
||||
llvm::copy(reductionVars, std::back_inserter(result.reductionVars));
|
||||
@@ -2128,8 +2127,7 @@ bool ClauseProcessor::processTaskReduction(
|
||||
currentLocation, converter,
|
||||
std::get<typename omp::clause::ReductionOperatorList>(clause.t),
|
||||
taskReductionVars, taskReduceVarByRef, taskReductionDeclSymbols,
|
||||
taskReductionSyms, taskReductionObjects,
|
||||
converter.getSymbolMap()))
|
||||
taskReductionSyms))
|
||||
TODO(currentLocation, "Lowering unrecognised reduction type");
|
||||
// Copy local lists into the output.
|
||||
llvm::copy(taskReductionVars,
|
||||
|
||||
@@ -66,30 +66,27 @@ DataSharingProcessor::DataSharingProcessor(
|
||||
lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
|
||||
const List<Clause> &clauses, lower::pft::Evaluation &eval,
|
||||
bool shouldCollectPreDeterminedSymbols, bool useDelayedPrivatization,
|
||||
lower::SymMap &symTable, bool isTargetPrivatization,
|
||||
llvm::ArrayRef<const semantics::Symbol *> symbolsCoveredByReductionElements)
|
||||
lower::SymMap &symTable, bool isTargetPrivatization)
|
||||
: converter(converter), semaCtx(semaCtx),
|
||||
firOpBuilder(converter.getFirOpBuilder()), clauses(clauses), eval(eval),
|
||||
shouldCollectPreDeterminedSymbols(shouldCollectPreDeterminedSymbols),
|
||||
useDelayedPrivatization(useDelayedPrivatization), symTable(symTable),
|
||||
isTargetPrivatization(isTargetPrivatization), visitor(semaCtx) {
|
||||
this->symbolsCoveredByReductionElements.insert(
|
||||
symbolsCoveredByReductionElements.begin(),
|
||||
symbolsCoveredByReductionElements.end());
|
||||
eval.visit([&](const auto &functionParserNode) {
|
||||
parser::Walk(functionParserNode, visitor);
|
||||
});
|
||||
}
|
||||
|
||||
DataSharingProcessor::DataSharingProcessor(
|
||||
lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
|
||||
lower::pft::Evaluation &eval, bool useDelayedPrivatization,
|
||||
lower::SymMap &symTable, bool isTargetPrivatization,
|
||||
llvm::ArrayRef<const semantics::Symbol *> symbolsCoveredByReductionElements)
|
||||
: DataSharingProcessor(
|
||||
converter, semaCtx, {}, eval,
|
||||
/*shouldCollectPreDeterminedSymols=*/false, useDelayedPrivatization,
|
||||
symTable, isTargetPrivatization, symbolsCoveredByReductionElements) {}
|
||||
DataSharingProcessor::DataSharingProcessor(lower::AbstractConverter &converter,
|
||||
semantics::SemanticsContext &semaCtx,
|
||||
lower::pft::Evaluation &eval,
|
||||
bool useDelayedPrivatization,
|
||||
lower::SymMap &symTable,
|
||||
bool isTargetPrivatization)
|
||||
: DataSharingProcessor(converter, semaCtx, {}, eval,
|
||||
/*shouldCollectPreDeterminedSymols=*/false,
|
||||
useDelayedPrivatization, symTable,
|
||||
isTargetPrivatization) {}
|
||||
|
||||
void DataSharingProcessor::processStep1(
|
||||
mlir::omp::PrivateClauseOps *clauseOps,
|
||||
@@ -289,20 +286,6 @@ void DataSharingProcessor::collectSymbolsForPrivatization() {
|
||||
allPrivatizedSymbols.insert(sym);
|
||||
}
|
||||
|
||||
bool DataSharingProcessor::isCoveredByReductionElement(
|
||||
const semantics::Symbol *sym) const {
|
||||
if (symbolsCoveredByReductionElements.contains(sym) ||
|
||||
symbolsCoveredByReductionElements.contains(&sym->GetUltimate()))
|
||||
return true;
|
||||
|
||||
if (const auto *hostAssoc = sym->detailsIf<semantics::HostAssocDetails>())
|
||||
return symbolsCoveredByReductionElements.contains(&hostAssoc->symbol()) ||
|
||||
symbolsCoveredByReductionElements.contains(
|
||||
&hostAssoc->symbol().GetUltimate());
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool DataSharingProcessor::needBarrier() {
|
||||
// Emit implicit barrier to synchronize threads and avoid data races on
|
||||
// initialization of firstprivate variables and post-update of lastprivate
|
||||
@@ -519,11 +502,6 @@ void DataSharingProcessor::collectPrivatizedSymbols(
|
||||
return false;
|
||||
|
||||
if (collectImplicit) {
|
||||
// If all uses of a privatisaed variable are covered by an expr in a
|
||||
// reduction clause, these should be ignored.
|
||||
if (isCoveredByReductionElement(sym))
|
||||
return false;
|
||||
|
||||
// If we're a combined construct with a target region, implicit
|
||||
// firstprivate captures, should only belong to the target region
|
||||
// and not be added/captured by later directives. Parallel regions
|
||||
|
||||
@@ -19,8 +19,6 @@
|
||||
#include "flang/Parser/parse-tree.h"
|
||||
#include "flang/Semantics/symbol.h"
|
||||
#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
|
||||
#include "llvm/ADT/ArrayRef.h"
|
||||
#include "llvm/ADT/SmallPtrSet.h"
|
||||
#include <variant>
|
||||
|
||||
namespace mlir {
|
||||
@@ -109,8 +107,6 @@ private:
|
||||
bool useDelayedPrivatization;
|
||||
bool forceHeapAllocationForPrivateDynamicArrays = false;
|
||||
llvm::SmallPtrSet<const semantics::Symbol *, 16> mightHaveReadHostSym;
|
||||
llvm::SmallPtrSet<const semantics::Symbol *, 4>
|
||||
symbolsCoveredByReductionElements;
|
||||
lower::SymMap &symTable;
|
||||
bool isTargetPrivatization;
|
||||
OMPConstructSymbolVisitor visitor;
|
||||
@@ -131,7 +127,6 @@ private:
|
||||
const omp::ObjectList &objects,
|
||||
llvm::SetVector<const semantics::Symbol *> &symbolSet);
|
||||
void collectSymbolsForPrivatization();
|
||||
bool isCoveredByReductionElement(const semantics::Symbol *sym) const;
|
||||
void insertBarrier(mlir::omp::PrivateClauseOps *clauseOps);
|
||||
void collectDefaultSymbols();
|
||||
void collectImplicitSymbols();
|
||||
@@ -160,17 +155,13 @@ public:
|
||||
lower::pft::Evaluation &eval,
|
||||
bool shouldCollectPreDeterminedSymbols,
|
||||
bool useDelayedPrivatization, lower::SymMap &symTable,
|
||||
bool isTargetPrivatization = false,
|
||||
llvm::ArrayRef<const semantics::Symbol *>
|
||||
symbolsCoveredByReductionElements = {});
|
||||
bool isTargetPrivatization = false);
|
||||
|
||||
DataSharingProcessor(lower::AbstractConverter &converter,
|
||||
semantics::SemanticsContext &semaCtx,
|
||||
lower::pft::Evaluation &eval,
|
||||
bool useDelayedPrivatization, lower::SymMap &symTable,
|
||||
bool isTargetPrivatization = false,
|
||||
llvm::ArrayRef<const semantics::Symbol *>
|
||||
symbolsCoveredByReductionElements = {});
|
||||
bool isTargetPrivatization = false);
|
||||
|
||||
// Privatisation is split into two steps.
|
||||
// Step1 performs cloning of all privatisation clauses and copying for
|
||||
|
||||
@@ -834,31 +834,18 @@ static void bindEntryBlockArgs(lower::AbstractConverter &converter,
|
||||
llvm::ArrayRef<mlir::Value> vars,
|
||||
llvm::ArrayRef<mlir::BlockArgument> args) {
|
||||
llvm::SmallVector<const semantics::Symbol *> processedSyms;
|
||||
llvm::SmallVector<const Object *> processedObjects;
|
||||
for (const Object &object : objects) {
|
||||
const semantics::Symbol *sym = object.sym();
|
||||
if (const auto *commonDet =
|
||||
sym->detailsIf<semantics::CommonBlockDetails>()) {
|
||||
for (auto &mem : commonDet->objects()) {
|
||||
processedSyms.push_back(&*mem);
|
||||
processedObjects.push_back(&object);
|
||||
}
|
||||
llvm::transform(commonDet->objects(), std::back_inserter(processedSyms),
|
||||
[&](const auto &mem) { return &*mem; });
|
||||
} else {
|
||||
processedSyms.push_back(sym);
|
||||
processedObjects.push_back(&object);
|
||||
}
|
||||
}
|
||||
|
||||
assert(processedSyms.size() == processedObjects.size());
|
||||
for (auto [sym, var, arg, object] :
|
||||
llvm::zip_equal(processedSyms, vars, args, processedObjects)) {
|
||||
bool skipBind =
|
||||
ReductionProcessor::isExpressionLoweredAsReductionObject(object) ||
|
||||
(object && sym->Rank() > 0 &&
|
||||
!fir::unwrapUntilSeqType(arg.getType()));
|
||||
if (skipBind)
|
||||
continue;
|
||||
|
||||
for (auto [sym, var, arg] : llvm::zip_equal(processedSyms, vars, args))
|
||||
converter.bindSymbol(
|
||||
*sym,
|
||||
hlfir::translateToExtendedValue(
|
||||
@@ -866,7 +853,6 @@ static void bindEntryBlockArgs(lower::AbstractConverter &converter,
|
||||
/*contiguousHint=*/
|
||||
evaluate::IsSimplyContiguous(*sym, converter.getFoldingContext()))
|
||||
.first);
|
||||
}
|
||||
};
|
||||
|
||||
// Process in clause name alphabetical order to match block arguments order.
|
||||
@@ -1343,44 +1329,13 @@ genLoopVars(mlir::Operation *op, lower::AbstractConverter &converter,
|
||||
// next one would result in 'hlfir.declare' operations being introduced inside
|
||||
// of a wrapper, which is illegal.
|
||||
mlir::IRMapping mapper;
|
||||
llvm::SmallVector<std::pair<Object, mlir::Value>> mappedReductionObjects;
|
||||
auto mapEquivalentReductionObjects =
|
||||
[&](const ObjectEntryBlockArgsEntry &entry) {
|
||||
for (auto [object, var] : llvm::zip(entry.objects, entry.vars)) {
|
||||
for (auto [mappedObject, mappedValue] :
|
||||
llvm::reverse(mappedReductionObjects)) {
|
||||
if (object.id() == mappedObject.id()) {
|
||||
mapper.map(var, mappedValue);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
auto rememberReductionObjects =
|
||||
[&](const ObjectEntryBlockArgsEntry &entry,
|
||||
llvm::ArrayRef<mlir::BlockArgument> args) {
|
||||
for (auto [object, arg] : llvm::zip(entry.objects, args))
|
||||
mappedReductionObjects.emplace_back(object, arg);
|
||||
};
|
||||
|
||||
for (auto [argGeneratingOp, blockArgs] : wrapperArgs) {
|
||||
mapEquivalentReductionObjects(blockArgs.inReduction);
|
||||
mapEquivalentReductionObjects(blockArgs.reduction);
|
||||
mapEquivalentReductionObjects(blockArgs.taskReduction);
|
||||
|
||||
for (mlir::OpOperand &operand : argGeneratingOp->getOpOperands())
|
||||
operand.set(mapper.lookupOrDefault(operand.get()));
|
||||
|
||||
for (const auto [arg, var] : llvm::zip_equal(
|
||||
argGeneratingOp->getRegion(0).getArguments(), blockArgs.getVars()))
|
||||
mapper.map(var, arg);
|
||||
|
||||
rememberReductionObjects(blockArgs.inReduction,
|
||||
argGeneratingOp.getInReductionBlockArgs());
|
||||
rememberReductionObjects(blockArgs.reduction,
|
||||
argGeneratingOp.getReductionBlockArgs());
|
||||
rememberReductionObjects(blockArgs.taskReduction,
|
||||
argGeneratingOp.getTaskReductionBlockArgs());
|
||||
}
|
||||
|
||||
// Bind the entry block arguments of parent wrappers to the corresponding
|
||||
@@ -1653,186 +1608,6 @@ struct OpWithBodyGenInfo {
|
||||
bool privatize = true;
|
||||
};
|
||||
|
||||
static mlir::Value getReductionOverrideValue(fir::FirOpBuilder &builder,
|
||||
mlir::Location loc,
|
||||
const Object *object,
|
||||
mlir::BlockArgument arg) {
|
||||
if (hlfir::isFortranEntityWithAttributes(arg))
|
||||
return arg;
|
||||
|
||||
fir::FortranVariableFlagsAttr attributes;
|
||||
llvm::SmallVector<mlir::Value> typeParams;
|
||||
auto declareOp = hlfir::DeclareOp::create(
|
||||
builder, loc, arg, "omp.reduction.element", nullptr, typeParams, nullptr,
|
||||
nullptr, 0, attributes);
|
||||
return declareOp.getBase();
|
||||
}
|
||||
|
||||
static void
|
||||
addReductionObjectOverrides(fir::FirOpBuilder &builder, mlir::Location loc,
|
||||
lower::ExprToValueMap &overrides,
|
||||
const ObjectEntryBlockArgsEntry &entry,
|
||||
llvm::ArrayRef<mlir::BlockArgument> blockArgs) {
|
||||
if (entry.objects.empty())
|
||||
return;
|
||||
|
||||
for (auto pair : llvm::zip_equal(entry.objects, blockArgs)) {
|
||||
const Object &object = std::get<0>(pair);
|
||||
const mlir::BlockArgument &arg = std::get<1>(pair);
|
||||
if (!ReductionProcessor::isExpressionLoweredAsReductionObject(&object))
|
||||
continue;
|
||||
const SomeExpr *expr = &object.ref().value();
|
||||
|
||||
// Evict any outer-scope entry for the same array element so the
|
||||
// innermost scope always wins regardless of DenseMap iteration order.
|
||||
llvm::SmallVector<const SomeExpr *> toEvict;
|
||||
for (auto [key, value] : overrides) {
|
||||
if (Fortran::lower::isEqual(key, expr)) {
|
||||
toEvict.push_back(key);
|
||||
}
|
||||
}
|
||||
for (const SomeExpr *key : toEvict) {
|
||||
overrides.erase(key);
|
||||
}
|
||||
|
||||
overrides[expr] = getReductionOverrideValue(builder, loc, &object, arg);
|
||||
}
|
||||
}
|
||||
|
||||
static const semantics::Symbol *getArrayElementSymbol(const SomeExpr &expr) {
|
||||
std::optional<Fortran::evaluate::DataRef> dataRef =
|
||||
Fortran::evaluate::ExtractDataRef(expr);
|
||||
if (!dataRef)
|
||||
return nullptr;
|
||||
|
||||
if (const auto *arrayRef =
|
||||
std::get_if<Fortran::evaluate::ArrayRef>(&dataRef->u))
|
||||
return &arrayRef->GetLastSymbol();
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
static void
|
||||
addSymbolAliases(llvm::SmallVectorImpl<const semantics::Symbol *> &aliases,
|
||||
const semantics::Symbol *symbol) {
|
||||
aliases.push_back(symbol);
|
||||
aliases.push_back(&symbol->GetUltimate());
|
||||
if (const auto *hostAssoc =
|
||||
symbol->detailsIf<semantics::HostAssocDetails>()) {
|
||||
aliases.push_back(&hostAssoc->symbol());
|
||||
aliases.push_back(&hostAssoc->symbol().GetUltimate());
|
||||
}
|
||||
}
|
||||
|
||||
struct ArrayElementReductionUseCollector {
|
||||
explicit ArrayElementReductionUseCollector(
|
||||
const llvm::DenseMap<const semantics::Symbol *, const semantics::Symbol *>
|
||||
&aliasToReductionSymbol,
|
||||
llvm::DenseMap<const semantics::Symbol *,
|
||||
llvm::SmallVector<const SomeExpr *>>
|
||||
&reductionElementExprs)
|
||||
: aliasToReductionSymbol(aliasToReductionSymbol),
|
||||
reductionElementExprs(reductionElementExprs) {}
|
||||
|
||||
const llvm::DenseMap<const semantics::Symbol *, const semantics::Symbol *>
|
||||
&aliasToReductionSymbol;
|
||||
llvm::DenseMap<const semantics::Symbol *, llvm::SmallVector<const SomeExpr *>>
|
||||
&reductionElementExprs;
|
||||
llvm::SmallPtrSet<const semantics::Symbol *, 16> seen;
|
||||
llvm::SmallPtrSet<const semantics::Symbol *, 16> uncovered;
|
||||
|
||||
void classifyReductionElementUses(const SomeExpr &expr) {
|
||||
llvm::SmallPtrSet<const semantics::Symbol *, 4> exprCandidates;
|
||||
auto getReductionSymbol = [this](const semantics::Symbol &symbol) {
|
||||
auto it = aliasToReductionSymbol.find(&symbol);
|
||||
return it == aliasToReductionSymbol.end() ? nullptr : it->second;
|
||||
};
|
||||
for (const semantics::Symbol &symbol :
|
||||
Fortran::evaluate::CollectSymbols(expr))
|
||||
if (const semantics::Symbol *reductionSymbol = getReductionSymbol(symbol))
|
||||
exprCandidates.insert(reductionSymbol);
|
||||
if (exprCandidates.empty())
|
||||
return;
|
||||
|
||||
auto isCoveredReductionUse =
|
||||
[this](const semantics::Symbol *reductionSymbol, const SomeExpr &expr) {
|
||||
auto it = reductionElementExprs.find(reductionSymbol);
|
||||
return it != reductionElementExprs.end() &&
|
||||
llvm::any_of(it->second, [&](const SomeExpr *reductionExpr) {
|
||||
return Fortran::lower::isEqual(&expr, reductionExpr);
|
||||
});
|
||||
};
|
||||
llvm::SmallPtrSet<const semantics::Symbol *, 4> seenInExpr;
|
||||
for (const SomeExpr &designator :
|
||||
semantics::omp::GetTopLevelDesignators(expr)) {
|
||||
const semantics::Symbol *symbol = getArrayElementSymbol(designator);
|
||||
const semantics::Symbol *reductionSymbol =
|
||||
symbol ? getReductionSymbol(*symbol) : nullptr;
|
||||
if (!reductionSymbol)
|
||||
continue;
|
||||
|
||||
if (isCoveredReductionUse(reductionSymbol, designator)) {
|
||||
seen.insert(reductionSymbol);
|
||||
seenInExpr.insert(reductionSymbol);
|
||||
} else {
|
||||
uncovered.insert(reductionSymbol);
|
||||
}
|
||||
}
|
||||
|
||||
for (const semantics::Symbol *symbol : exprCandidates)
|
||||
if (!seenInExpr.contains(symbol))
|
||||
uncovered.insert(symbol);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool Pre(const T &node) {
|
||||
if constexpr (parser::HasTypedExpr<T>::value) {
|
||||
if (const SomeExpr *expr = semantics::GetExpr(nullptr, node)) {
|
||||
classifyReductionElementUses(*expr);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Pre(const parser::Name &name) { return false; }
|
||||
|
||||
template <typename T>
|
||||
void Post(const T &) {}
|
||||
};
|
||||
|
||||
static llvm::SmallVector<const semantics::Symbol *>
|
||||
getSymbolsCoveredByReductionElements(lower::pft::Evaluation &eval,
|
||||
llvm::ArrayRef<Object> reductionObjects) {
|
||||
llvm::DenseMap<const semantics::Symbol *, const semantics::Symbol *>
|
||||
aliasToReductionSymbol;
|
||||
llvm::DenseMap<const semantics::Symbol *, llvm::SmallVector<const SomeExpr *>>
|
||||
reductionElementExprs;
|
||||
for (const Object &object : reductionObjects) {
|
||||
if (!ReductionProcessor::isExpressionLoweredAsReductionObject(&object))
|
||||
continue;
|
||||
llvm::SmallVector<const semantics::Symbol *> aliases;
|
||||
addSymbolAliases(aliases, object.sym());
|
||||
for (const semantics::Symbol *alias : aliases)
|
||||
aliasToReductionSymbol[alias] = object.sym();
|
||||
reductionElementExprs[object.sym()].push_back(&*object.ref());
|
||||
}
|
||||
|
||||
if (reductionElementExprs.empty())
|
||||
return {};
|
||||
|
||||
ArrayElementReductionUseCollector collector(aliasToReductionSymbol,
|
||||
reductionElementExprs);
|
||||
eval.visit([&](const auto &node) { parser::Walk(node, collector); });
|
||||
|
||||
llvm::SmallVector<const semantics::Symbol *> suppressList;
|
||||
for (auto &[symbol, exprs] : reductionElementExprs)
|
||||
if (collector.seen.contains(symbol) &&
|
||||
!collector.uncovered.contains(symbol))
|
||||
suppressList.push_back(symbol);
|
||||
return suppressList;
|
||||
}
|
||||
|
||||
/// Create the body (block) for an OpenMP Operation.
|
||||
///
|
||||
/// \param [in] op - the operation the body belongs to.
|
||||
@@ -1917,27 +1692,6 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
|
||||
groupprivatizeVars(info.converter, info.eval);
|
||||
|
||||
if (!info.genSkeletonOnly) {
|
||||
lower::ExprToValueMap local;
|
||||
if (auto *old = info.converter.getExprOverrides())
|
||||
local.insert(old->begin(), old->end());
|
||||
if (info.blockArgs) {
|
||||
if (auto ompBlockArgOp =
|
||||
mlir::dyn_cast<mlir::omp::BlockArgOpenMPOpInterface>(op)) {
|
||||
addReductionObjectOverrides(firOpBuilder, info.loc, local,
|
||||
info.blockArgs->inReduction,
|
||||
ompBlockArgOp.getInReductionBlockArgs());
|
||||
addReductionObjectOverrides(firOpBuilder, info.loc, local,
|
||||
info.blockArgs->reduction,
|
||||
ompBlockArgOp.getReductionBlockArgs());
|
||||
addReductionObjectOverrides(firOpBuilder, info.loc, local,
|
||||
info.blockArgs->taskReduction,
|
||||
ompBlockArgOp.getTaskReductionBlockArgs());
|
||||
}
|
||||
}
|
||||
|
||||
auto *old = info.converter.getExprOverrides();
|
||||
info.converter.overrideExprValues(local.empty() ? old : &local);
|
||||
|
||||
if (ConstructQueue::const_iterator next = std::next(item);
|
||||
next != queue.end()) {
|
||||
genOMPDispatch(info.converter, info.symTable, info.semaCtx, info.eval,
|
||||
@@ -1953,8 +1707,6 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
|
||||
genNestedEvaluations(info.converter, info.eval);
|
||||
temp->erase();
|
||||
}
|
||||
|
||||
info.converter.overrideExprValues(old);
|
||||
}
|
||||
|
||||
// Get or create a unique exiting block from the given region, or
|
||||
@@ -2776,40 +2528,21 @@ genLoopNestOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
|
||||
const ObjectEntryBlockArgs &>>
|
||||
wrapperArgs,
|
||||
llvm::omp::Directive directive, DataSharingProcessor &dsp) {
|
||||
const lower::ExprToValueMap *oldOverrides = converter.getExprOverrides();
|
||||
lower::ExprToValueMap loopNestOverrides;
|
||||
auto ivCallback = [&](mlir::Operation *op) {
|
||||
genLoopVars(op, converter, loc, iv, wrapperArgs);
|
||||
if (oldOverrides)
|
||||
loopNestOverrides.insert(oldOverrides->begin(), oldOverrides->end());
|
||||
for (auto [argGeneratingOp, blockArgs] : wrapperArgs) {
|
||||
addReductionObjectOverrides(converter.getFirOpBuilder(), loc,
|
||||
loopNestOverrides, blockArgs.inReduction,
|
||||
argGeneratingOp.getInReductionBlockArgs());
|
||||
addReductionObjectOverrides(converter.getFirOpBuilder(), loc,
|
||||
loopNestOverrides, blockArgs.reduction,
|
||||
argGeneratingOp.getReductionBlockArgs());
|
||||
addReductionObjectOverrides(converter.getFirOpBuilder(), loc,
|
||||
loopNestOverrides, blockArgs.taskReduction,
|
||||
argGeneratingOp.getTaskReductionBlockArgs());
|
||||
}
|
||||
converter.overrideExprValues(
|
||||
loopNestOverrides.empty() ? oldOverrides : &loopNestOverrides);
|
||||
return llvm::SmallVector<const semantics::Symbol *>(iv);
|
||||
};
|
||||
|
||||
uint64_t nestValue = getCollapseValue(item->clauses);
|
||||
nestValue = nestValue < iv.size() ? iv.size() : nestValue;
|
||||
auto *nestedEval = getCollapsedLoopEval(eval, nestValue);
|
||||
auto loopNestOp = genOpWithBody<mlir::omp::LoopNestOp>(
|
||||
return genOpWithBody<mlir::omp::LoopNestOp>(
|
||||
OpWithBodyGenInfo(converter, symTable, semaCtx, loc, *nestedEval,
|
||||
directive)
|
||||
.setClauses(&item->clauses)
|
||||
.setDataSharingProcessor(&dsp)
|
||||
.setGenRegionEntryCb(ivCallback),
|
||||
queue, item, clauseOps);
|
||||
converter.overrideExprValues(oldOverrides);
|
||||
return loopNestOp;
|
||||
}
|
||||
|
||||
static mlir::omp::LoopOp
|
||||
@@ -3856,14 +3589,9 @@ genTaskOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
|
||||
.setClauses(&item->clauses),
|
||||
queue, item, clauseOps);
|
||||
|
||||
llvm::SmallVector<const semantics::Symbol *>
|
||||
symbolsCoveredByReductionElements =
|
||||
getSymbolsCoveredByReductionElements(eval, inReductionObjects);
|
||||
DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
|
||||
lower::omp::isLastItemInQueue(item, queue),
|
||||
/*useDelayedPrivatization=*/true, symTable,
|
||||
/*isTargetPrivatization=*/false,
|
||||
symbolsCoveredByReductionElements);
|
||||
/*useDelayedPrivatization=*/true, symTable);
|
||||
dsp.processStep1(&clauseOps);
|
||||
|
||||
ObjectEntryBlockArgs taskArgs;
|
||||
@@ -4126,17 +3854,9 @@ static mlir::omp::TaskloopContextOp genStandaloneTaskloop(
|
||||
|
||||
genTaskloopClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
|
||||
taskloopClauseOps, reductionObjects, inReductionObjects);
|
||||
llvm::SmallVector<Object> allReductionObjects;
|
||||
llvm::append_range(allReductionObjects, reductionObjects);
|
||||
llvm::append_range(allReductionObjects, inReductionObjects);
|
||||
llvm::SmallVector<const semantics::Symbol *>
|
||||
symbolsCoveredByReductionElements =
|
||||
getSymbolsCoveredByReductionElements(eval, allReductionObjects);
|
||||
DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
|
||||
/*shouldCollectPreDeterminedSymbols=*/true,
|
||||
enableDelayedPrivatization, symTable,
|
||||
/*isTargetPrivatization=*/false,
|
||||
symbolsCoveredByReductionElements);
|
||||
enableDelayedPrivatization, symTable);
|
||||
dsp.processStep1(&taskloopClauseOps);
|
||||
|
||||
mlir::omp::LoopNestOperands loopNestClauseOps;
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
#include "flang/Lower/Support/ReductionProcessor.h"
|
||||
|
||||
#include "flang/Lower/AbstractConverter.h"
|
||||
#include "flang/Lower/ConvertExprToHLFIR.h"
|
||||
#include "flang/Lower/ConvertType.h"
|
||||
#include "flang/Lower/OpenMP/Clauses.h"
|
||||
#include "flang/Lower/Support/PrivateReductionUtils.h"
|
||||
@@ -48,7 +47,6 @@ template bool ReductionProcessor::processReductionArguments<
|
||||
llvm::SmallVectorImpl<bool> &reduceVarByRef,
|
||||
llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
|
||||
const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
|
||||
llvm::ArrayRef<Object> reductionObjects, lower::SymMap &symMap,
|
||||
llvm::DenseMap<const semantics::Symbol *, mlir::Value> *reductionVarCache);
|
||||
|
||||
template bool ReductionProcessor::processReductionArguments<
|
||||
@@ -59,7 +57,6 @@ template bool ReductionProcessor::processReductionArguments<
|
||||
llvm::SmallVectorImpl<bool> &reduceVarByRef,
|
||||
llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
|
||||
const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
|
||||
llvm::ArrayRef<Object> reductionObjects, lower::SymMap &symMap,
|
||||
llvm::DenseMap<const semantics::Symbol *, mlir::Value> *reductionVarCache);
|
||||
|
||||
template mlir::omp::DeclareReductionOp
|
||||
@@ -382,18 +379,6 @@ mlir::Value ReductionProcessor::createScalarCombiner(
|
||||
return reductionOp;
|
||||
}
|
||||
|
||||
bool ReductionProcessor::isExpressionLoweredAsReductionObject(
|
||||
const Object *object) {
|
||||
if (!object || !object->ref())
|
||||
return false;
|
||||
const SomeExpr &expr = *object->ref();
|
||||
// Only genuine single array elements (rank 0) are lowered via the element
|
||||
// path. Array sections such as a(2:96) and vector subscripts have rank > 0;
|
||||
// lowering them here produces an unsupported sequence type and aborts in
|
||||
// PrivateReductionUtils. Let them fall back to the boxed whole-array path.
|
||||
return evaluate::IsArrayElement(expr) && expr.Rank() == 0;
|
||||
}
|
||||
|
||||
template <typename ParentDeclOpType>
|
||||
static void genYield(fir::FirOpBuilder &builder, mlir::Location loc,
|
||||
mlir::Value yieldedValue) {
|
||||
@@ -701,7 +686,6 @@ bool ReductionProcessor::processReductionArguments(
|
||||
llvm::SmallVectorImpl<bool> &reduceVarByRef,
|
||||
llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
|
||||
const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols,
|
||||
llvm::ArrayRef<Object> reductionObjects, lower::SymMap &symMap,
|
||||
llvm::DenseMap<const semantics::Symbol *, mlir::Value> *reductionVarCache) {
|
||||
fir::FirOpBuilder &builder = converter.getFirOpBuilder();
|
||||
|
||||
@@ -748,105 +732,79 @@ bool ReductionProcessor::processReductionArguments(
|
||||
builder.getRegion().getParentOfType<fir::DoConcurrentOp>());
|
||||
}
|
||||
|
||||
assert((reductionObjects.empty() ||
|
||||
reductionSymbols.size() == reductionObjects.size()) &&
|
||||
"mismatched reduction symbol and object lists");
|
||||
|
||||
for (unsigned i = 0; i < reductionSymbols.size(); ++i) {
|
||||
const Object *object =
|
||||
reductionObjects.empty() ? nullptr : &reductionObjects[i];
|
||||
const semantics::Symbol *symbol =
|
||||
object ? object->sym() : reductionSymbols[i];
|
||||
const SomeExpr *expr = object && object->ref() ? &*object->ref() : nullptr;
|
||||
const bool isObjectExpr =
|
||||
ReductionProcessor::isExpressionLoweredAsReductionObject(object);
|
||||
|
||||
for (const semantics::Symbol *symbol : reductionSymbols) {
|
||||
// If a cached reduction variable exists for this symbol, reuse it.
|
||||
// This ensures that composite constructs (e.g. DO SIMD) where both
|
||||
// the outer wrapper (wsloop) and inner wrapper (simd) process the same
|
||||
// reduction clause share the same SSA value, enabling genLoopVars()'s
|
||||
// IRMapping to correctly remap inner wrapper operands to outer wrapper
|
||||
// block arguments. Array element reductions are intentionally not cached:
|
||||
// block-argument object tracking maps their scoped uses.
|
||||
if (reductionVarCache && !isObjectExpr) {
|
||||
if (auto it = reductionVarCache->find(symbol);
|
||||
it != reductionVarCache->end()) {
|
||||
// block arguments.
|
||||
if (reductionVarCache) {
|
||||
auto it = reductionVarCache->find(symbol);
|
||||
if (it != reductionVarCache->end()) {
|
||||
reductionVars.push_back(it->second);
|
||||
reduceVarByRef.push_back(doReductionByRef(it->second));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
mlir::Value reductionVal;
|
||||
mlir::Type refTy;
|
||||
mlir::Value symVal = converter.getSymbolAddress(*symbol);
|
||||
|
||||
if (isObjectExpr) {
|
||||
StatementContext stmtCtx;
|
||||
hlfir::EntityWithAttributes entity = convertExprToHLFIR(
|
||||
converter.getCurrentLocation(), converter, *expr, symMap, stmtCtx);
|
||||
reductionVal = entity.getBase();
|
||||
// TODO Add support for Boxed and Sequenced types once these are supported
|
||||
refTy = reductionVal.getType();
|
||||
} else {
|
||||
mlir::Value symVal = converter.getSymbolAddress(*symbol);
|
||||
if (auto declOp = symVal.getDefiningOp<hlfir::DeclareOp>())
|
||||
symVal = declOp.getBase();
|
||||
|
||||
if (auto declOp = symVal.getDefiningOp<hlfir::DeclareOp>())
|
||||
symVal = declOp.getBase();
|
||||
mlir::Type eleType;
|
||||
auto refType = mlir::dyn_cast_or_null<fir::ReferenceType>(symVal.getType());
|
||||
if (refType)
|
||||
eleType = refType.getEleTy();
|
||||
else
|
||||
eleType = symVal.getType();
|
||||
|
||||
mlir::Type eleType;
|
||||
auto refType =
|
||||
mlir::dyn_cast_or_null<fir::ReferenceType>(symVal.getType());
|
||||
if (refType)
|
||||
eleType = refType.getEleTy();
|
||||
else
|
||||
eleType = symVal.getType();
|
||||
// all arrays must be boxed so that we have convenient access to all the
|
||||
// information needed to iterate over the array
|
||||
if (mlir::isa<fir::SequenceType>(eleType)) {
|
||||
// For Host associated symbols, use `SymbolBox` instead
|
||||
lower::SymbolBox symBox = converter.lookupOneLevelUpSymbol(*symbol);
|
||||
hlfir::Entity entity{symBox.getAddr()};
|
||||
entity = genVariableBox(currentLocation, builder, entity);
|
||||
mlir::Value box = entity.getBase();
|
||||
|
||||
// all arrays must be boxed so that we have convenient access to all the
|
||||
// information needed to iterate over the array
|
||||
if (mlir::isa<fir::SequenceType>(eleType)) {
|
||||
// For Host associated symbols, use `SymbolBox` instead
|
||||
lower::SymbolBox symBox = converter.lookupOneLevelUpSymbol(*symbol);
|
||||
hlfir::Entity entity{symBox.getAddr()};
|
||||
entity = genVariableBox(currentLocation, builder, entity);
|
||||
mlir::Value box = entity.getBase();
|
||||
// Always pass the box by reference so that the OpenMP dialect
|
||||
// verifiers don't need to know anything about fir.box
|
||||
auto alloca =
|
||||
fir::AllocaOp::create(builder, currentLocation, box.getType());
|
||||
fir::StoreOp::create(builder, currentLocation, box, alloca);
|
||||
|
||||
// Always pass the box by reference so that the OpenMP dialect
|
||||
// verifiers don't need to know anything about fir.box
|
||||
auto alloca =
|
||||
fir::AllocaOp::create(builder, currentLocation, box.getType());
|
||||
fir::StoreOp::create(builder, currentLocation, box, alloca);
|
||||
|
||||
symVal = alloca;
|
||||
} else if (mlir::isa<fir::BaseBoxType>(symVal.getType())) {
|
||||
// boxed arrays are passed as values not by reference. Unfortunately,
|
||||
// we can't pass a box by value to omp.redution_declare, so turn it
|
||||
// into a reference
|
||||
auto oldIP = builder.saveInsertionPoint();
|
||||
builder.setInsertionPointToStart(builder.getAllocaBlock());
|
||||
auto alloca =
|
||||
fir::AllocaOp::create(builder, currentLocation, symVal.getType());
|
||||
builder.restoreInsertionPoint(oldIP);
|
||||
fir::StoreOp::create(builder, currentLocation, symVal, alloca);
|
||||
symVal = alloca;
|
||||
}
|
||||
|
||||
// this isn't the same as the by-val and by-ref passing later in the
|
||||
// pipeline. Both styles assume that the variable is a reference at
|
||||
// this point
|
||||
assert(fir::isa_ref_type(symVal.getType()) &&
|
||||
"reduction input var is passed by reference");
|
||||
mlir::Type elementType = fir::dyn_cast_ptrEleTy(symVal.getType());
|
||||
const bool symIsVolatile = fir::isa_volatile_type(symVal.getType());
|
||||
refTy = fir::ReferenceType::get(elementType, symIsVolatile);
|
||||
reductionVal = symVal;
|
||||
symVal = alloca;
|
||||
} else if (mlir::isa<fir::BaseBoxType>(symVal.getType())) {
|
||||
// boxed arrays are passed as values not by reference. Unfortunately,
|
||||
// we can't pass a box by value to omp.redution_declare, so turn it
|
||||
// into a reference
|
||||
auto oldIP = builder.saveInsertionPoint();
|
||||
builder.setInsertionPointToStart(builder.getAllocaBlock());
|
||||
auto alloca =
|
||||
fir::AllocaOp::create(builder, currentLocation, symVal.getType());
|
||||
builder.restoreInsertionPoint(oldIP);
|
||||
fir::StoreOp::create(builder, currentLocation, symVal, alloca);
|
||||
symVal = alloca;
|
||||
}
|
||||
|
||||
// this isn't the same as the by-val and by-ref passing later in the
|
||||
// pipeline. Both styles assume that the variable is a reference at
|
||||
// this point
|
||||
assert(fir::isa_ref_type(symVal.getType()) &&
|
||||
"reduction input var is passed by reference");
|
||||
mlir::Type elementType = fir::dyn_cast_ptrEleTy(symVal.getType());
|
||||
const bool symIsVolatile = fir::isa_volatile_type(symVal.getType());
|
||||
mlir::Type refTy = fir::ReferenceType::get(elementType, symIsVolatile);
|
||||
|
||||
reductionVars.push_back(
|
||||
builder.createConvert(currentLocation, refTy, reductionVal));
|
||||
builder.createConvert(currentLocation, refTy, symVal));
|
||||
reduceVarByRef.push_back(doReductionByRef(reductionVars.back()));
|
||||
|
||||
// Cache the final SSA value for this symbol so that subsequent calls
|
||||
// (e.g. for the inner wrapper in a composite construct) reuse it.
|
||||
if (reductionVarCache && !isObjectExpr)
|
||||
if (reductionVarCache)
|
||||
reductionVarCache->try_emplace(symbol, reductionVars.back());
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,10 @@
|
||||
! RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=50 %s -o - | FileCheck %s --implicit-check-not=add_reduction_byref_box
|
||||
! RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=50 %s -o - | FileCheck %s --implicit-check-not=omp.reduction.element
|
||||
|
||||
! Array-element reductions currently use the whole-array boxed reduction path.
|
||||
! Keep lowering coverage for these constructs so that this limitation does not
|
||||
! hide failures to compile them.
|
||||
|
||||
! CHECK: omp.declare_reduction @[[BOX_RED:add_reduction_byref_box_4xi32]] : !fir.ref<!fir.box<!fir.array<4xi32>>>
|
||||
|
||||
subroutine reduction_literal(a, n)
|
||||
integer :: a(4), n
|
||||
@@ -9,10 +15,10 @@ subroutine reduction_literal(a, n)
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPreduction_literal
|
||||
! CHECK: omp.wsloop {{.*}} reduction(@add_reduction_i32 {{.*}} : !fir.ref<i32>) {
|
||||
! CHECK: hlfir.declare %arg{{[0-9]+}} {uniq_name = "omp.reduction.element"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
|
||||
! CHECK: fir.load %{{[0-9]+}}#0 : !fir.ref<i32>
|
||||
! CHECK: hlfir.assign {{.*}} to %{{[0-9]+}}#0 : i32, !fir.ref<i32>
|
||||
! CHECK: omp.wsloop {{.*}} reduction(byref @[[BOX_RED]] {{.*}} : !fir.ref<!fir.box<!fir.array<4xi32>>>) {
|
||||
! CHECK: hlfir.declare %{{.*}} {uniq_name = "_QFreduction_literalEa"} : (!fir.ref<!fir.box<!fir.array<4xi32>>>)
|
||||
! CHECK: fir.load %{{.*}} : !fir.ref<!fir.box<!fir.array<4xi32>>>
|
||||
! CHECK: hlfir.designate %{{.*}} (%c2) {{.*}} -> !fir.ref<i32>
|
||||
|
||||
subroutine reduction_multiple(a, n)
|
||||
integer :: a(4), n
|
||||
@@ -24,11 +30,9 @@ subroutine reduction_multiple(a, n)
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPreduction_multiple
|
||||
! CHECK: omp.wsloop {{.*}} reduction(@add_reduction_i32 {{.*}}, @add_reduction_i32 {{.*}} : !fir.ref<i32>, !fir.ref<i32>) {
|
||||
! CHECK: hlfir.declare %arg{{[0-9]+}} {uniq_name = "omp.reduction.element"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
|
||||
! CHECK: hlfir.declare %arg{{[0-9]+}} {uniq_name = "omp.reduction.element"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
|
||||
! CHECK: hlfir.assign {{.*}} to %{{[0-9]+}}#0 : i32, !fir.ref<i32>
|
||||
! CHECK: hlfir.assign {{.*}} to %{{[0-9]+}}#0 : i32, !fir.ref<i32>
|
||||
! CHECK: omp.wsloop {{.*}} reduction(byref @[[BOX_RED]] {{.*}}, byref @[[BOX_RED]] {{.*}} : !fir.ref<!fir.box<!fir.array<4xi32>>>, !fir.ref<!fir.box<!fir.array<4xi32>>>) {
|
||||
! CHECK: hlfir.designate %{{.*}} (%c2) {{.*}} -> !fir.ref<i32>
|
||||
! CHECK: hlfir.designate %{{.*}} (%c3) {{.*}} -> !fir.ref<i32>
|
||||
|
||||
subroutine reduction_arrays(a, b, n)
|
||||
integer :: a(4), b(4), n
|
||||
@@ -39,9 +43,10 @@ subroutine reduction_arrays(a, b, n)
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPreduction_arrays
|
||||
! CHECK: omp.wsloop {{.*}} reduction(@add_reduction_i32 {{.*}}, @add_reduction_i32 {{.*}} : !fir.ref<i32>, !fir.ref<i32>) {
|
||||
! CHECK: hlfir.declare %arg{{[0-9]+}} {uniq_name = "omp.reduction.element"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
|
||||
! CHECK: hlfir.declare %arg{{[0-9]+}} {uniq_name = "omp.reduction.element"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
|
||||
! CHECK: omp.wsloop {{.*}} reduction(byref @[[BOX_RED]] {{.*}}, byref @[[BOX_RED]] {{.*}} : !fir.ref<!fir.box<!fir.array<4xi32>>>, !fir.ref<!fir.box<!fir.array<4xi32>>>) {
|
||||
! CHECK: hlfir.declare %{{.*}} {uniq_name = "_QFreduction_arraysEa"} : (!fir.ref<!fir.box<!fir.array<4xi32>>>)
|
||||
! CHECK: hlfir.declare %{{.*}} {uniq_name = "_QFreduction_arraysEb"} : (!fir.ref<!fir.box<!fir.array<4xi32>>>)
|
||||
! CHECK: hlfir.designate %{{.*}} (%c2{{.*}}) {{.*}} -> !fir.ref<i32>
|
||||
|
||||
subroutine reduction_variable(a, n, j)
|
||||
integer :: a(4), n, j
|
||||
@@ -52,9 +57,9 @@ subroutine reduction_variable(a, n, j)
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPreduction_variable
|
||||
! CHECK: omp.wsloop {{.*}} reduction(@add_reduction_i32 {{.*}} : !fir.ref<i32>) {
|
||||
! CHECK: hlfir.declare %arg{{[0-9]+}} {uniq_name = "omp.reduction.element"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
|
||||
! CHECK: hlfir.assign {{.*}} to %{{[0-9]+}}#0 : i32, !fir.ref<i32>
|
||||
! CHECK: omp.wsloop {{.*}} reduction(byref @[[BOX_RED]] {{.*}} : !fir.ref<!fir.box<!fir.array<4xi32>>>) {
|
||||
! CHECK: hlfir.declare %{{.*}} {uniq_name = "_QFreduction_variableEa"} : (!fir.ref<!fir.box<!fir.array<4xi32>>>)
|
||||
! CHECK: hlfir.designate %{{.*}} (%{{.*}}) {{.*}} -> !fir.ref<i32>
|
||||
|
||||
subroutine reduction_do_simd(a, n)
|
||||
integer :: a(4), n
|
||||
@@ -65,10 +70,9 @@ subroutine reduction_do_simd(a, n)
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPreduction_do_simd
|
||||
! CHECK: omp.wsloop reduction(@add_reduction_i32 {{.*}} -> [[WSARG:%arg[0-9]+]] : !fir.ref<i32>) {
|
||||
! CHECK: omp.simd {{.*}} reduction(@add_reduction_i32 [[WSARG]] -> [[SIMDARG:%arg[0-9]+]] : !fir.ref<i32>) {
|
||||
! CHECK: hlfir.declare [[SIMDARG]] {uniq_name = "omp.reduction.element"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
|
||||
! CHECK: hlfir.assign {{.*}} to %{{[0-9]+}}#0 : i32, !fir.ref<i32>
|
||||
! CHECK: omp.wsloop reduction(byref @[[BOX_RED]] {{.*}} -> %[[WSARG:.*]] : !fir.ref<!fir.box<!fir.array<4xi32>>>) {
|
||||
! CHECK: omp.simd {{.*}} reduction(byref @[[BOX_RED]] %[[WSARG]] -> %{{.*}} : !fir.ref<!fir.box<!fir.array<4xi32>>>) {
|
||||
! CHECK: hlfir.designate %{{.*}} (%c2) {{.*}} -> !fir.ref<i32>
|
||||
|
||||
subroutine task_reduction_element(a)
|
||||
integer :: a(4)
|
||||
@@ -80,13 +84,9 @@ subroutine task_reduction_element(a)
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPtask_reduction_element
|
||||
! CHECK-NOT: _QFtask_reduction_elementEa_firstprivate_box_4xi32
|
||||
! CHECK: omp.taskgroup task_reduction(@add_reduction_i32 {{.*}} -> [[TGARG:%arg[0-9]+]] : !fir.ref<i32>) {
|
||||
! CHECK: [[TGDECL:%[0-9]+]]:2 = hlfir.declare [[TGARG]] {uniq_name = "omp.reduction.element"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
|
||||
! CHECK: omp.task in_reduction(@add_reduction_i32 [[TGDECL]]#0 -> [[TASKARG:%arg[0-9]+]] : !fir.ref<i32>)
|
||||
! CHECK: [[TASKDECL:%[0-9]+]]:2 = hlfir.declare [[TASKARG]] {uniq_name = "omp.reduction.element"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
|
||||
! CHECK: fir.load [[TASKDECL]]#0 : !fir.ref<i32>
|
||||
! CHECK: hlfir.assign {{.*}} to [[TASKDECL]]#0 : i32, !fir.ref<i32>
|
||||
! CHECK: omp.taskgroup task_reduction(byref @[[BOX_RED]] {{.*}} -> %{{.*}} : !fir.ref<!fir.box<!fir.array<4xi32>>>) {
|
||||
! CHECK: omp.task in_reduction(byref @[[BOX_RED]] {{.*}} -> %{{.*}} : !fir.ref<!fir.box<!fir.array<4xi32>>>) private({{.*}}_firstprivate_box_4xi32
|
||||
! CHECK: hlfir.designate %{{.*}} (%c2) {{.*}} -> !fir.ref<i32>
|
||||
|
||||
subroutine taskloop_in_reduction_element(a, n)
|
||||
integer :: a(4), n
|
||||
@@ -97,10 +97,8 @@ subroutine taskloop_in_reduction_element(a, n)
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPtaskloop_in_reduction_element
|
||||
! CHECK-NOT: _QFtaskloop_in_reduction_elementEa_firstprivate_box_4xi32
|
||||
! CHECK: omp.taskloop.context in_reduction(@add_reduction_i32 {{.*}} -> [[TLARG:%arg[0-9]+]] : !fir.ref<i32>)
|
||||
! CHECK: hlfir.declare [[TLARG]] {uniq_name = "omp.reduction.element"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
|
||||
! CHECK: hlfir.assign {{.*}} to %{{[0-9]+}}#0 : i32, !fir.ref<i32>
|
||||
! CHECK: omp.taskloop.context in_reduction(byref @[[BOX_RED]] {{.*}} : !fir.ref<!fir.box<!fir.array<4xi32>>>) private({{.*}}_firstprivate_box_4xi32
|
||||
! CHECK: hlfir.designate %{{.*}} (%c2) {{.*}} -> !fir.ref<i32>
|
||||
|
||||
subroutine taskloop_reduction_element(a, n)
|
||||
integer :: a(4), n
|
||||
@@ -111,10 +109,8 @@ subroutine taskloop_reduction_element(a, n)
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPtaskloop_reduction_element
|
||||
! CHECK-NOT: _QFtaskloop_reduction_elementEa_firstprivate_box_4xi32
|
||||
! CHECK: omp.taskloop.context {{.*}} reduction(@add_reduction_i32 {{.*}} -> [[TLRARG:%arg[0-9]+]] : !fir.ref<i32>)
|
||||
! CHECK: hlfir.declare [[TLRARG]] {uniq_name = "omp.reduction.element"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
|
||||
! CHECK: hlfir.assign {{.*}} to %{{[0-9]+}}#0 : i32, !fir.ref<i32>
|
||||
! CHECK: omp.taskloop.context private({{.*}}_firstprivate_box_4xi32{{.*}}) reduction(byref @[[BOX_RED]] {{.*}} : !fir.ref<!fir.box<!fir.array<4xi32>>>) {
|
||||
! CHECK: hlfir.designate %{{.*}} (%c2) {{.*}} -> !fir.ref<i32>
|
||||
|
||||
subroutine taskloop_reduction_mixed_use(a, n)
|
||||
integer :: a(4), n
|
||||
@@ -126,8 +122,9 @@ subroutine taskloop_reduction_mixed_use(a, n)
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPtaskloop_reduction_mixed_use
|
||||
! CHECK: omp.taskloop.context private({{.*}}@_QFtaskloop_reduction_mixed_useEa_firstprivate_box_4xi32{{.*}}) reduction(@add_reduction_i32 {{.*}} -> [[TLMARG:%arg[0-9]+]] : !fir.ref<i32>)
|
||||
! CHECK: hlfir.declare [[TLMARG]] {uniq_name = "omp.reduction.element"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
|
||||
! CHECK: omp.taskloop.context private({{.*}}_firstprivate_box_4xi32{{.*}}) reduction(byref @[[BOX_RED]] {{.*}} : !fir.ref<!fir.box<!fir.array<4xi32>>>) {
|
||||
! CHECK: hlfir.designate %{{.*}} (%c2) {{.*}} -> !fir.ref<i32>
|
||||
! CHECK: hlfir.designate %{{.*}} (%c1) {{.*}} -> !fir.ref<i32>
|
||||
|
||||
subroutine taskloop_reduction_nested_index_use(a, b, n)
|
||||
integer :: a(4), b(4), n
|
||||
@@ -139,5 +136,6 @@ subroutine taskloop_reduction_nested_index_use(a, b, n)
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPtaskloop_reduction_nested_index_use
|
||||
! CHECK: omp.taskloop.context private({{.*}}@_QFtaskloop_reduction_nested_index_useEa_firstprivate_box_4xi32{{.*}}) reduction(@add_reduction_i32 {{.*}} -> [[TLNARG:%arg[0-9]+]] : !fir.ref<i32>)
|
||||
! CHECK: hlfir.declare [[TLNARG]] {uniq_name = "omp.reduction.element"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
|
||||
! CHECK: omp.taskloop.context private({{.*}}_firstprivate_box_4xi32{{.*}}) reduction(byref @[[BOX_RED]] {{.*}} : !fir.ref<!fir.box<!fir.array<4xi32>>>) {
|
||||
! CHECK: hlfir.designate %{{.*}} (%c2) {{.*}} -> !fir.ref<i32>
|
||||
! CHECK: hlfir.designate %{{.*}} (%{{.*}}) {{.*}} -> !fir.ref<i32>
|
||||
@@ -1,39 +0,0 @@
|
||||
! RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=50 %s -o - | FileCheck %s --implicit-check-not=omp.reduction.element
|
||||
|
||||
! Regression test for reductions on array *sections* (e.g. a(2:96)).
|
||||
!
|
||||
! An array section has rank > 0, so it must be lowered using the boxed
|
||||
! whole-array reduction (@add_reduction_byref_box_*) and the section applied
|
||||
! via hlfir.designate inside the region. It must NOT be routed through the
|
||||
! single-element path (uniq_name = "omp.reduction.element"), which only supports
|
||||
! rank-0 references and otherwise aborts in PrivateReductionUtils with
|
||||
! "creating reduction/privatization init region for unsupported type".
|
||||
|
||||
subroutine reduction_array_section(a, n)
|
||||
integer :: a(100), n
|
||||
!$omp parallel do reduction(+: a(2:96))
|
||||
do i = 1, n
|
||||
a(2:96) = a(2:96) + i
|
||||
end do
|
||||
end subroutine
|
||||
|
||||
! CHECK: omp.declare_reduction @[[RED:add_reduction_byref_box_100xi32]] : !fir.ref<!fir.box<!fir.array<100xi32>>>
|
||||
|
||||
! CHECK-LABEL: func.func @_QPreduction_array_section
|
||||
! CHECK: omp.wsloop {{.*}} reduction(byref @[[RED]] %{{[0-9]+}} -> %[[ARG:.*]] : !fir.ref<!fir.box<!fir.array<100xi32>>>) {
|
||||
! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG]] {uniq_name = "_QFreduction_array_sectionEa"}
|
||||
! CHECK: %[[BOX:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.array<100xi32>>>
|
||||
! CHECK: hlfir.designate %[[BOX]] (%c2:%c96:%c1) {{.*}} -> !fir.ref<!fir.array<95xi32>>
|
||||
|
||||
subroutine reduction_array_section_simd(a, n)
|
||||
integer :: a(100), n
|
||||
!$omp parallel do simd reduction(+: a(2:96))
|
||||
do i = 1, n
|
||||
a(2:96) = a(2:96) + i
|
||||
end do
|
||||
end subroutine
|
||||
|
||||
! CHECK-LABEL: func.func @_QPreduction_array_section_simd
|
||||
! CHECK: omp.wsloop reduction(byref @[[RED]] %{{[0-9]+}} -> %[[WSARG:.*]] : !fir.ref<!fir.box<!fir.array<100xi32>>>) {
|
||||
! CHECK: omp.simd {{.*}} reduction(byref @[[RED]] %[[WSARG]] -> %[[SIMDARG:.*]] : !fir.ref<!fir.box<!fir.array<100xi32>>>) {
|
||||
! CHECK: hlfir.declare %[[SIMDARG]] {uniq_name = "_QFreduction_array_section_simdEa"}
|
||||
Reference in new issue
Block a user