File: | build/source/llvm/utils/TableGen/CodeGenDAGPatterns.cpp |
Warning: | line 2907, column 38 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- CodeGenDAGPatterns.cpp - Read DAG patterns from .td file -----------===// | ||||||
2 | // | ||||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||
6 | // | ||||||
7 | //===----------------------------------------------------------------------===// | ||||||
8 | // | ||||||
9 | // This file implements the CodeGenDAGPatterns class, which is used to read and | ||||||
10 | // represent the patterns present in a .td file for instructions. | ||||||
11 | // | ||||||
12 | //===----------------------------------------------------------------------===// | ||||||
13 | |||||||
14 | #include "CodeGenDAGPatterns.h" | ||||||
15 | #include "CodeGenInstruction.h" | ||||||
16 | #include "CodeGenRegisters.h" | ||||||
17 | #include "llvm/ADT/DenseSet.h" | ||||||
18 | #include "llvm/ADT/MapVector.h" | ||||||
19 | #include "llvm/ADT/STLExtras.h" | ||||||
20 | #include "llvm/ADT/SmallSet.h" | ||||||
21 | #include "llvm/ADT/SmallString.h" | ||||||
22 | #include "llvm/ADT/StringExtras.h" | ||||||
23 | #include "llvm/ADT/StringMap.h" | ||||||
24 | #include "llvm/ADT/Twine.h" | ||||||
25 | #include "llvm/Support/Debug.h" | ||||||
26 | #include "llvm/Support/ErrorHandling.h" | ||||||
27 | #include "llvm/Support/TypeSize.h" | ||||||
28 | #include "llvm/TableGen/Error.h" | ||||||
29 | #include "llvm/TableGen/Record.h" | ||||||
30 | #include <algorithm> | ||||||
31 | #include <cstdio> | ||||||
32 | #include <iterator> | ||||||
33 | #include <set> | ||||||
34 | using namespace llvm; | ||||||
35 | |||||||
36 | #define DEBUG_TYPE"dag-patterns" "dag-patterns" | ||||||
37 | |||||||
38 | static inline bool isIntegerOrPtr(MVT VT) { | ||||||
39 | return VT.isInteger() || VT == MVT::iPTR; | ||||||
40 | } | ||||||
41 | static inline bool isFloatingPoint(MVT VT) { | ||||||
42 | return VT.isFloatingPoint(); | ||||||
43 | } | ||||||
44 | static inline bool isVector(MVT VT) { | ||||||
45 | return VT.isVector(); | ||||||
46 | } | ||||||
47 | static inline bool isScalar(MVT VT) { | ||||||
48 | return !VT.isVector(); | ||||||
49 | } | ||||||
50 | static inline bool isScalarInteger(MVT VT) { | ||||||
51 | return VT.isScalarInteger(); | ||||||
52 | } | ||||||
53 | |||||||
54 | template <typename Predicate> | ||||||
55 | static bool berase_if(MachineValueTypeSet &S, Predicate P) { | ||||||
56 | bool Erased = false; | ||||||
57 | // It is ok to iterate over MachineValueTypeSet and remove elements from it | ||||||
58 | // at the same time. | ||||||
59 | for (MVT T : S) { | ||||||
60 | if (!P(T)) | ||||||
61 | continue; | ||||||
62 | Erased = true; | ||||||
63 | S.erase(T); | ||||||
64 | } | ||||||
65 | return Erased; | ||||||
66 | } | ||||||
67 | |||||||
68 | void MachineValueTypeSet::writeToStream(raw_ostream &OS) const { | ||||||
69 | SmallVector<MVT, 4> Types(begin(), end()); | ||||||
70 | array_pod_sort(Types.begin(), Types.end()); | ||||||
71 | |||||||
72 | OS << '['; | ||||||
73 | ListSeparator LS(" "); | ||||||
74 | for (const MVT &T : Types) | ||||||
75 | OS << LS << ValueTypeByHwMode::getMVTName(T); | ||||||
76 | OS << ']'; | ||||||
77 | } | ||||||
78 | |||||||
79 | // --- TypeSetByHwMode | ||||||
80 | |||||||
81 | // This is a parameterized type-set class. For each mode there is a list | ||||||
82 | // of types that are currently possible for a given tree node. Type | ||||||
83 | // inference will apply to each mode separately. | ||||||
84 | |||||||
85 | TypeSetByHwMode::TypeSetByHwMode(ArrayRef<ValueTypeByHwMode> VTList) { | ||||||
86 | for (const ValueTypeByHwMode &VVT : VTList) { | ||||||
87 | insert(VVT); | ||||||
88 | AddrSpaces.push_back(VVT.PtrAddrSpace); | ||||||
89 | } | ||||||
90 | } | ||||||
91 | |||||||
92 | bool TypeSetByHwMode::isValueTypeByHwMode(bool AllowEmpty) const { | ||||||
93 | for (const auto &I : *this) { | ||||||
94 | if (I.second.size() > 1) | ||||||
95 | return false; | ||||||
96 | if (!AllowEmpty && I.second.empty()) | ||||||
97 | return false; | ||||||
98 | } | ||||||
99 | return true; | ||||||
100 | } | ||||||
101 | |||||||
102 | ValueTypeByHwMode TypeSetByHwMode::getValueTypeByHwMode() const { | ||||||
103 | assert(isValueTypeByHwMode(true) &&(static_cast <bool> (isValueTypeByHwMode(true) && "The type set has multiple types for at least one HW mode") ? void (0) : __assert_fail ("isValueTypeByHwMode(true) && \"The type set has multiple types for at least one HW mode\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 104, __extension__ __PRETTY_FUNCTION__)) | ||||||
104 | "The type set has multiple types for at least one HW mode")(static_cast <bool> (isValueTypeByHwMode(true) && "The type set has multiple types for at least one HW mode") ? void (0) : __assert_fail ("isValueTypeByHwMode(true) && \"The type set has multiple types for at least one HW mode\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 104, __extension__ __PRETTY_FUNCTION__)); | ||||||
105 | ValueTypeByHwMode VVT; | ||||||
106 | auto ASI = AddrSpaces.begin(); | ||||||
107 | |||||||
108 | for (const auto &I : *this) { | ||||||
109 | MVT T = I.second.empty() ? MVT::Other : *I.second.begin(); | ||||||
110 | VVT.getOrCreateTypeForMode(I.first, T); | ||||||
111 | if (ASI != AddrSpaces.end()) | ||||||
112 | VVT.PtrAddrSpace = *ASI++; | ||||||
113 | } | ||||||
114 | return VVT; | ||||||
115 | } | ||||||
116 | |||||||
117 | bool TypeSetByHwMode::isPossible() const { | ||||||
118 | for (const auto &I : *this) | ||||||
119 | if (!I.second.empty()) | ||||||
120 | return true; | ||||||
121 | return false; | ||||||
122 | } | ||||||
123 | |||||||
124 | bool TypeSetByHwMode::insert(const ValueTypeByHwMode &VVT) { | ||||||
125 | bool Changed = false; | ||||||
126 | bool ContainsDefault = false; | ||||||
127 | MVT DT = MVT::Other; | ||||||
128 | |||||||
129 | for (const auto &P : VVT) { | ||||||
130 | unsigned M = P.first; | ||||||
131 | // Make sure there exists a set for each specific mode from VVT. | ||||||
132 | Changed |= getOrCreate(M).insert(P.second).second; | ||||||
133 | // Cache VVT's default mode. | ||||||
134 | if (DefaultMode == M) { | ||||||
135 | ContainsDefault = true; | ||||||
136 | DT = P.second; | ||||||
137 | } | ||||||
138 | } | ||||||
139 | |||||||
140 | // If VVT has a default mode, add the corresponding type to all | ||||||
141 | // modes in "this" that do not exist in VVT. | ||||||
142 | if (ContainsDefault) | ||||||
143 | for (auto &I : *this) | ||||||
144 | if (!VVT.hasMode(I.first)) | ||||||
145 | Changed |= I.second.insert(DT).second; | ||||||
146 | |||||||
147 | return Changed; | ||||||
148 | } | ||||||
149 | |||||||
150 | // Constrain the type set to be the intersection with VTS. | ||||||
151 | bool TypeSetByHwMode::constrain(const TypeSetByHwMode &VTS) { | ||||||
152 | bool Changed = false; | ||||||
153 | if (hasDefault()) { | ||||||
154 | for (const auto &I : VTS) { | ||||||
155 | unsigned M = I.first; | ||||||
156 | if (M == DefaultMode || hasMode(M)) | ||||||
157 | continue; | ||||||
158 | Map.insert({M, Map.at(DefaultMode)}); | ||||||
159 | Changed = true; | ||||||
160 | } | ||||||
161 | } | ||||||
162 | |||||||
163 | for (auto &I : *this) { | ||||||
164 | unsigned M = I.first; | ||||||
165 | SetType &S = I.second; | ||||||
166 | if (VTS.hasMode(M) || VTS.hasDefault()) { | ||||||
167 | Changed |= intersect(I.second, VTS.get(M)); | ||||||
168 | } else if (!S.empty()) { | ||||||
169 | S.clear(); | ||||||
170 | Changed = true; | ||||||
171 | } | ||||||
172 | } | ||||||
173 | return Changed; | ||||||
174 | } | ||||||
175 | |||||||
176 | template <typename Predicate> | ||||||
177 | bool TypeSetByHwMode::constrain(Predicate P) { | ||||||
178 | bool Changed = false; | ||||||
179 | for (auto &I : *this) | ||||||
180 | Changed |= berase_if(I.second, [&P](MVT VT) { return !P(VT); }); | ||||||
181 | return Changed; | ||||||
182 | } | ||||||
183 | |||||||
184 | template <typename Predicate> | ||||||
185 | bool TypeSetByHwMode::assign_if(const TypeSetByHwMode &VTS, Predicate P) { | ||||||
186 | assert(empty())(static_cast <bool> (empty()) ? void (0) : __assert_fail ("empty()", "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 186 , __extension__ __PRETTY_FUNCTION__)); | ||||||
187 | for (const auto &I : VTS) { | ||||||
188 | SetType &S = getOrCreate(I.first); | ||||||
189 | for (auto J : I.second) | ||||||
190 | if (P(J)) | ||||||
191 | S.insert(J); | ||||||
192 | } | ||||||
193 | return !empty(); | ||||||
194 | } | ||||||
195 | |||||||
196 | void TypeSetByHwMode::writeToStream(raw_ostream &OS) const { | ||||||
197 | SmallVector<unsigned, 4> Modes; | ||||||
198 | Modes.reserve(Map.size()); | ||||||
199 | |||||||
200 | for (const auto &I : *this) | ||||||
201 | Modes.push_back(I.first); | ||||||
202 | if (Modes.empty()) { | ||||||
203 | OS << "{}"; | ||||||
204 | return; | ||||||
205 | } | ||||||
206 | array_pod_sort(Modes.begin(), Modes.end()); | ||||||
207 | |||||||
208 | OS << '{'; | ||||||
209 | for (unsigned M : Modes) { | ||||||
210 | OS << ' ' << getModeName(M) << ':'; | ||||||
211 | get(M).writeToStream(OS); | ||||||
212 | } | ||||||
213 | OS << " }"; | ||||||
214 | } | ||||||
215 | |||||||
216 | bool TypeSetByHwMode::operator==(const TypeSetByHwMode &VTS) const { | ||||||
217 | // The isSimple call is much quicker than hasDefault - check this first. | ||||||
218 | bool IsSimple = isSimple(); | ||||||
219 | bool VTSIsSimple = VTS.isSimple(); | ||||||
220 | if (IsSimple && VTSIsSimple) | ||||||
221 | return getSimple() == VTS.getSimple(); | ||||||
222 | |||||||
223 | // Speedup: We have a default if the set is simple. | ||||||
224 | bool HaveDefault = IsSimple || hasDefault(); | ||||||
225 | bool VTSHaveDefault = VTSIsSimple || VTS.hasDefault(); | ||||||
226 | if (HaveDefault != VTSHaveDefault) | ||||||
227 | return false; | ||||||
228 | |||||||
229 | SmallSet<unsigned, 4> Modes; | ||||||
230 | for (auto &I : *this) | ||||||
231 | Modes.insert(I.first); | ||||||
232 | for (const auto &I : VTS) | ||||||
233 | Modes.insert(I.first); | ||||||
234 | |||||||
235 | if (HaveDefault) { | ||||||
236 | // Both sets have default mode. | ||||||
237 | for (unsigned M : Modes) { | ||||||
238 | if (get(M) != VTS.get(M)) | ||||||
239 | return false; | ||||||
240 | } | ||||||
241 | } else { | ||||||
242 | // Neither set has default mode. | ||||||
243 | for (unsigned M : Modes) { | ||||||
244 | // If there is no default mode, an empty set is equivalent to not having | ||||||
245 | // the corresponding mode. | ||||||
246 | bool NoModeThis = !hasMode(M) || get(M).empty(); | ||||||
247 | bool NoModeVTS = !VTS.hasMode(M) || VTS.get(M).empty(); | ||||||
248 | if (NoModeThis != NoModeVTS) | ||||||
249 | return false; | ||||||
250 | if (!NoModeThis) | ||||||
251 | if (get(M) != VTS.get(M)) | ||||||
252 | return false; | ||||||
253 | } | ||||||
254 | } | ||||||
255 | |||||||
256 | return true; | ||||||
257 | } | ||||||
258 | |||||||
259 | namespace llvm { | ||||||
260 | raw_ostream &operator<<(raw_ostream &OS, const MachineValueTypeSet &T) { | ||||||
261 | T.writeToStream(OS); | ||||||
262 | return OS; | ||||||
263 | } | ||||||
264 | raw_ostream &operator<<(raw_ostream &OS, const TypeSetByHwMode &T) { | ||||||
265 | T.writeToStream(OS); | ||||||
266 | return OS; | ||||||
267 | } | ||||||
268 | } | ||||||
269 | |||||||
270 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) | ||||||
271 | void TypeSetByHwMode::dump() const { | ||||||
272 | dbgs() << *this << '\n'; | ||||||
273 | } | ||||||
274 | |||||||
275 | bool TypeSetByHwMode::intersect(SetType &Out, const SetType &In) { | ||||||
276 | bool OutP = Out.count(MVT::iPTR), InP = In.count(MVT::iPTR); | ||||||
277 | // Complement of In. | ||||||
278 | auto CompIn = [&In](MVT T) -> bool { return !In.count(T); }; | ||||||
279 | |||||||
280 | if (OutP == InP) | ||||||
281 | return berase_if(Out, CompIn); | ||||||
282 | |||||||
283 | // Compute the intersection of scalars separately to account for only | ||||||
284 | // one set containing iPTR. | ||||||
285 | // The intersection of iPTR with a set of integer scalar types that does not | ||||||
286 | // include iPTR will result in the most specific scalar type: | ||||||
287 | // - iPTR is more specific than any set with two elements or more | ||||||
288 | // - iPTR is less specific than any single integer scalar type. | ||||||
289 | // For example | ||||||
290 | // { iPTR } * { i32 } -> { i32 } | ||||||
291 | // { iPTR } * { i32 i64 } -> { iPTR } | ||||||
292 | // and | ||||||
293 | // { iPTR i32 } * { i32 } -> { i32 } | ||||||
294 | // { iPTR i32 } * { i32 i64 } -> { i32 i64 } | ||||||
295 | // { iPTR i32 } * { i32 i64 i128 } -> { iPTR i32 } | ||||||
296 | |||||||
297 | // Let In' = elements only in In, Out' = elements only in Out, and | ||||||
298 | // IO = elements common to both. Normally IO would be returned as the result | ||||||
299 | // of the intersection, but we need to account for iPTR being a "wildcard" of | ||||||
300 | // sorts. Since elements in IO are those that match both sets exactly, they | ||||||
301 | // will all belong to the output. If any of the "leftovers" (i.e. In' or | ||||||
302 | // Out') contain iPTR, it means that the other set doesn't have it, but it | ||||||
303 | // could have (1) a more specific type, or (2) a set of types that is less | ||||||
304 | // specific. The "leftovers" from the other set is what we want to examine | ||||||
305 | // more closely. | ||||||
306 | |||||||
307 | auto subtract = [](const SetType &A, const SetType &B) { | ||||||
308 | SetType Diff = A; | ||||||
309 | berase_if(Diff, [&B](MVT T) { return B.count(T); }); | ||||||
310 | return Diff; | ||||||
311 | }; | ||||||
312 | |||||||
313 | if (InP) { | ||||||
314 | SetType OutOnly = subtract(Out, In); | ||||||
315 | if (OutOnly.empty()) { | ||||||
316 | // This means that Out \subset In, so no change to Out. | ||||||
317 | return false; | ||||||
318 | } | ||||||
319 | unsigned NumI = llvm::count_if(OutOnly, isScalarInteger); | ||||||
320 | if (NumI == 1 && OutOnly.size() == 1) { | ||||||
321 | // There is only one element in Out', and it happens to be a scalar | ||||||
322 | // integer that should be kept as a match for iPTR in In. | ||||||
323 | return false; | ||||||
324 | } | ||||||
325 | berase_if(Out, CompIn); | ||||||
326 | if (NumI == 1) { | ||||||
327 | // Replace the iPTR with the leftover scalar integer. | ||||||
328 | Out.insert(*llvm::find_if(OutOnly, isScalarInteger)); | ||||||
329 | } else if (NumI > 1) { | ||||||
330 | Out.insert(MVT::iPTR); | ||||||
331 | } | ||||||
332 | return true; | ||||||
333 | } | ||||||
334 | |||||||
335 | // OutP == true | ||||||
336 | SetType InOnly = subtract(In, Out); | ||||||
337 | unsigned SizeOut = Out.size(); | ||||||
338 | berase_if(Out, CompIn); // This will remove at least the iPTR. | ||||||
339 | unsigned NumI = llvm::count_if(InOnly, isScalarInteger); | ||||||
340 | if (NumI == 0) { | ||||||
341 | // iPTR deleted from Out. | ||||||
342 | return true; | ||||||
343 | } | ||||||
344 | if (NumI == 1) { | ||||||
345 | // Replace the iPTR with the leftover scalar integer. | ||||||
346 | Out.insert(*llvm::find_if(InOnly, isScalarInteger)); | ||||||
347 | return true; | ||||||
348 | } | ||||||
349 | |||||||
350 | // NumI > 1: Keep the iPTR in Out. | ||||||
351 | Out.insert(MVT::iPTR); | ||||||
352 | // If iPTR was the only element initially removed from Out, then Out | ||||||
353 | // has not changed. | ||||||
354 | return SizeOut != Out.size(); | ||||||
355 | } | ||||||
356 | |||||||
357 | bool TypeSetByHwMode::validate() const { | ||||||
358 | if (empty()) | ||||||
359 | return true; | ||||||
360 | bool AllEmpty = true; | ||||||
361 | for (const auto &I : *this) | ||||||
362 | AllEmpty &= I.second.empty(); | ||||||
363 | return !AllEmpty; | ||||||
364 | } | ||||||
365 | |||||||
366 | // --- TypeInfer | ||||||
367 | |||||||
368 | bool TypeInfer::MergeInTypeInfo(TypeSetByHwMode &Out, | ||||||
369 | const TypeSetByHwMode &In) { | ||||||
370 | ValidateOnExit _1(Out, *this); | ||||||
371 | In.validate(); | ||||||
372 | if (In.empty() || Out == In || TP.hasError()) | ||||||
373 | return false; | ||||||
374 | if (Out.empty()) { | ||||||
375 | Out = In; | ||||||
376 | return true; | ||||||
377 | } | ||||||
378 | |||||||
379 | bool Changed = Out.constrain(In); | ||||||
380 | if (Changed && Out.empty()) | ||||||
381 | TP.error("Type contradiction"); | ||||||
382 | |||||||
383 | return Changed; | ||||||
384 | } | ||||||
385 | |||||||
386 | bool TypeInfer::forceArbitrary(TypeSetByHwMode &Out) { | ||||||
387 | ValidateOnExit _1(Out, *this); | ||||||
388 | if (TP.hasError()) | ||||||
389 | return false; | ||||||
390 | assert(!Out.empty() && "cannot pick from an empty set")(static_cast <bool> (!Out.empty() && "cannot pick from an empty set" ) ? void (0) : __assert_fail ("!Out.empty() && \"cannot pick from an empty set\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 390, __extension__ __PRETTY_FUNCTION__)); | ||||||
391 | |||||||
392 | bool Changed = false; | ||||||
393 | for (auto &I : Out) { | ||||||
394 | TypeSetByHwMode::SetType &S = I.second; | ||||||
395 | if (S.size() <= 1) | ||||||
396 | continue; | ||||||
397 | MVT T = *S.begin(); // Pick the first element. | ||||||
398 | S.clear(); | ||||||
399 | S.insert(T); | ||||||
400 | Changed = true; | ||||||
401 | } | ||||||
402 | return Changed; | ||||||
403 | } | ||||||
404 | |||||||
405 | bool TypeInfer::EnforceInteger(TypeSetByHwMode &Out) { | ||||||
406 | ValidateOnExit _1(Out, *this); | ||||||
407 | if (TP.hasError()) | ||||||
408 | return false; | ||||||
409 | if (!Out.empty()) | ||||||
410 | return Out.constrain(isIntegerOrPtr); | ||||||
411 | |||||||
412 | return Out.assign_if(getLegalTypes(), isIntegerOrPtr); | ||||||
413 | } | ||||||
414 | |||||||
415 | bool TypeInfer::EnforceFloatingPoint(TypeSetByHwMode &Out) { | ||||||
416 | ValidateOnExit _1(Out, *this); | ||||||
417 | if (TP.hasError()) | ||||||
418 | return false; | ||||||
419 | if (!Out.empty()) | ||||||
420 | return Out.constrain(isFloatingPoint); | ||||||
421 | |||||||
422 | return Out.assign_if(getLegalTypes(), isFloatingPoint); | ||||||
423 | } | ||||||
424 | |||||||
425 | bool TypeInfer::EnforceScalar(TypeSetByHwMode &Out) { | ||||||
426 | ValidateOnExit _1(Out, *this); | ||||||
427 | if (TP.hasError()) | ||||||
428 | return false; | ||||||
429 | if (!Out.empty()) | ||||||
430 | return Out.constrain(isScalar); | ||||||
431 | |||||||
432 | return Out.assign_if(getLegalTypes(), isScalar); | ||||||
433 | } | ||||||
434 | |||||||
435 | bool TypeInfer::EnforceVector(TypeSetByHwMode &Out) { | ||||||
436 | ValidateOnExit _1(Out, *this); | ||||||
437 | if (TP.hasError()) | ||||||
438 | return false; | ||||||
439 | if (!Out.empty()) | ||||||
440 | return Out.constrain(isVector); | ||||||
441 | |||||||
442 | return Out.assign_if(getLegalTypes(), isVector); | ||||||
443 | } | ||||||
444 | |||||||
445 | bool TypeInfer::EnforceAny(TypeSetByHwMode &Out) { | ||||||
446 | ValidateOnExit _1(Out, *this); | ||||||
447 | if (TP.hasError() || !Out.empty()) | ||||||
448 | return false; | ||||||
449 | |||||||
450 | Out = getLegalTypes(); | ||||||
451 | return true; | ||||||
452 | } | ||||||
453 | |||||||
454 | template <typename Iter, typename Pred, typename Less> | ||||||
455 | static Iter min_if(Iter B, Iter E, Pred P, Less L) { | ||||||
456 | if (B == E) | ||||||
457 | return E; | ||||||
458 | Iter Min = E; | ||||||
459 | for (Iter I = B; I != E; ++I) { | ||||||
460 | if (!P(*I)) | ||||||
461 | continue; | ||||||
462 | if (Min == E || L(*I, *Min)) | ||||||
463 | Min = I; | ||||||
464 | } | ||||||
465 | return Min; | ||||||
466 | } | ||||||
467 | |||||||
468 | template <typename Iter, typename Pred, typename Less> | ||||||
469 | static Iter max_if(Iter B, Iter E, Pred P, Less L) { | ||||||
470 | if (B == E) | ||||||
471 | return E; | ||||||
472 | Iter Max = E; | ||||||
473 | for (Iter I = B; I != E; ++I) { | ||||||
474 | if (!P(*I)) | ||||||
475 | continue; | ||||||
476 | if (Max == E || L(*Max, *I)) | ||||||
477 | Max = I; | ||||||
478 | } | ||||||
479 | return Max; | ||||||
480 | } | ||||||
481 | |||||||
482 | /// Make sure that for each type in Small, there exists a larger type in Big. | ||||||
483 | bool TypeInfer::EnforceSmallerThan(TypeSetByHwMode &Small, TypeSetByHwMode &Big, | ||||||
484 | bool SmallIsVT) { | ||||||
485 | ValidateOnExit _1(Small, *this), _2(Big, *this); | ||||||
486 | if (TP.hasError()) | ||||||
487 | return false; | ||||||
488 | bool Changed = false; | ||||||
489 | |||||||
490 | assert((!SmallIsVT || !Small.empty()) &&(static_cast <bool> ((!SmallIsVT || !Small.empty()) && "Small should not be empty for SDTCisVTSmallerThanOp") ? void (0) : __assert_fail ("(!SmallIsVT || !Small.empty()) && \"Small should not be empty for SDTCisVTSmallerThanOp\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 491, __extension__ __PRETTY_FUNCTION__)) | ||||||
491 | "Small should not be empty for SDTCisVTSmallerThanOp")(static_cast <bool> ((!SmallIsVT || !Small.empty()) && "Small should not be empty for SDTCisVTSmallerThanOp") ? void (0) : __assert_fail ("(!SmallIsVT || !Small.empty()) && \"Small should not be empty for SDTCisVTSmallerThanOp\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 491, __extension__ __PRETTY_FUNCTION__)); | ||||||
492 | |||||||
493 | if (Small.empty()) | ||||||
494 | Changed |= EnforceAny(Small); | ||||||
495 | if (Big.empty()) | ||||||
496 | Changed |= EnforceAny(Big); | ||||||
497 | |||||||
498 | assert(Small.hasDefault() && Big.hasDefault())(static_cast <bool> (Small.hasDefault() && Big. hasDefault()) ? void (0) : __assert_fail ("Small.hasDefault() && Big.hasDefault()" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 498, __extension__ __PRETTY_FUNCTION__)); | ||||||
499 | |||||||
500 | SmallVector<unsigned, 4> Modes; | ||||||
501 | union_modes(Small, Big, Modes); | ||||||
502 | |||||||
503 | // 1. Only allow integer or floating point types and make sure that | ||||||
504 | // both sides are both integer or both floating point. | ||||||
505 | // 2. Make sure that either both sides have vector types, or neither | ||||||
506 | // of them does. | ||||||
507 | for (unsigned M : Modes) { | ||||||
508 | TypeSetByHwMode::SetType &S = Small.get(M); | ||||||
509 | TypeSetByHwMode::SetType &B = Big.get(M); | ||||||
510 | |||||||
511 | assert((!SmallIsVT || !S.empty()) && "Expected non-empty type")(static_cast <bool> ((!SmallIsVT || !S.empty()) && "Expected non-empty type") ? void (0) : __assert_fail ("(!SmallIsVT || !S.empty()) && \"Expected non-empty type\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 511, __extension__ __PRETTY_FUNCTION__)); | ||||||
512 | |||||||
513 | if (any_of(S, isIntegerOrPtr) && any_of(B, isIntegerOrPtr)) { | ||||||
514 | auto NotInt = [](MVT VT) { return !isIntegerOrPtr(VT); }; | ||||||
515 | Changed |= berase_if(S, NotInt); | ||||||
516 | Changed |= berase_if(B, NotInt); | ||||||
517 | } else if (any_of(S, isFloatingPoint) && any_of(B, isFloatingPoint)) { | ||||||
518 | auto NotFP = [](MVT VT) { return !isFloatingPoint(VT); }; | ||||||
519 | Changed |= berase_if(S, NotFP); | ||||||
520 | Changed |= berase_if(B, NotFP); | ||||||
521 | } else if (SmallIsVT && B.empty()) { | ||||||
522 | // B is empty and since S is a specific VT, it will never be empty. Don't | ||||||
523 | // report this as a change, just clear S and continue. This prevents an | ||||||
524 | // infinite loop. | ||||||
525 | S.clear(); | ||||||
526 | } else if (S.empty() || B.empty()) { | ||||||
527 | Changed = !S.empty() || !B.empty(); | ||||||
528 | S.clear(); | ||||||
529 | B.clear(); | ||||||
530 | } else { | ||||||
531 | TP.error("Incompatible types"); | ||||||
532 | return Changed; | ||||||
533 | } | ||||||
534 | |||||||
535 | if (none_of(S, isVector) || none_of(B, isVector)) { | ||||||
536 | Changed |= berase_if(S, isVector); | ||||||
537 | Changed |= berase_if(B, isVector); | ||||||
538 | } | ||||||
539 | } | ||||||
540 | |||||||
541 | auto LT = [](MVT A, MVT B) -> bool { | ||||||
542 | // Always treat non-scalable MVTs as smaller than scalable MVTs for the | ||||||
543 | // purposes of ordering. | ||||||
544 | auto ASize = std::make_tuple(A.isScalableVector(), A.getScalarSizeInBits(), | ||||||
545 | A.getSizeInBits().getKnownMinValue()); | ||||||
546 | auto BSize = std::make_tuple(B.isScalableVector(), B.getScalarSizeInBits(), | ||||||
547 | B.getSizeInBits().getKnownMinValue()); | ||||||
548 | return ASize < BSize; | ||||||
549 | }; | ||||||
550 | auto SameKindLE = [](MVT A, MVT B) -> bool { | ||||||
551 | // This function is used when removing elements: when a vector is compared | ||||||
552 | // to a non-vector or a scalable vector to any non-scalable MVT, it should | ||||||
553 | // return false (to avoid removal). | ||||||
554 | if (std::make_tuple(A.isVector(), A.isScalableVector()) != | ||||||
555 | std::make_tuple(B.isVector(), B.isScalableVector())) | ||||||
556 | return false; | ||||||
557 | |||||||
558 | return std::make_tuple(A.getScalarSizeInBits(), | ||||||
559 | A.getSizeInBits().getKnownMinValue()) <= | ||||||
560 | std::make_tuple(B.getScalarSizeInBits(), | ||||||
561 | B.getSizeInBits().getKnownMinValue()); | ||||||
562 | }; | ||||||
563 | |||||||
564 | for (unsigned M : Modes) { | ||||||
565 | TypeSetByHwMode::SetType &S = Small.get(M); | ||||||
566 | TypeSetByHwMode::SetType &B = Big.get(M); | ||||||
567 | // MinS = min scalar in Small, remove all scalars from Big that are | ||||||
568 | // smaller-or-equal than MinS. | ||||||
569 | auto MinS = min_if(S.begin(), S.end(), isScalar, LT); | ||||||
570 | if (MinS != S.end()) | ||||||
571 | Changed |= berase_if(B, std::bind(SameKindLE, | ||||||
572 | std::placeholders::_1, *MinS)); | ||||||
573 | |||||||
574 | // MaxS = max scalar in Big, remove all scalars from Small that are | ||||||
575 | // larger than MaxS. | ||||||
576 | auto MaxS = max_if(B.begin(), B.end(), isScalar, LT); | ||||||
577 | if (MaxS != B.end()) | ||||||
578 | Changed |= berase_if(S, std::bind(SameKindLE, | ||||||
579 | *MaxS, std::placeholders::_1)); | ||||||
580 | |||||||
581 | // MinV = min vector in Small, remove all vectors from Big that are | ||||||
582 | // smaller-or-equal than MinV. | ||||||
583 | auto MinV = min_if(S.begin(), S.end(), isVector, LT); | ||||||
584 | if (MinV != S.end()) | ||||||
585 | Changed |= berase_if(B, std::bind(SameKindLE, | ||||||
586 | std::placeholders::_1, *MinV)); | ||||||
587 | |||||||
588 | // MaxV = max vector in Big, remove all vectors from Small that are | ||||||
589 | // larger than MaxV. | ||||||
590 | auto MaxV = max_if(B.begin(), B.end(), isVector, LT); | ||||||
591 | if (MaxV != B.end()) | ||||||
592 | Changed |= berase_if(S, std::bind(SameKindLE, | ||||||
593 | *MaxV, std::placeholders::_1)); | ||||||
594 | } | ||||||
595 | |||||||
596 | return Changed; | ||||||
597 | } | ||||||
598 | |||||||
599 | /// 1. Ensure that for each type T in Vec, T is a vector type, and that | ||||||
600 | /// for each type U in Elem, U is a scalar type. | ||||||
601 | /// 2. Ensure that for each (scalar) type U in Elem, there exists a (vector) | ||||||
602 | /// type T in Vec, such that U is the element type of T. | ||||||
603 | bool TypeInfer::EnforceVectorEltTypeIs(TypeSetByHwMode &Vec, | ||||||
604 | TypeSetByHwMode &Elem) { | ||||||
605 | ValidateOnExit _1(Vec, *this), _2(Elem, *this); | ||||||
606 | if (TP.hasError()) | ||||||
607 | return false; | ||||||
608 | bool Changed = false; | ||||||
609 | |||||||
610 | if (Vec.empty()) | ||||||
611 | Changed |= EnforceVector(Vec); | ||||||
612 | if (Elem.empty()) | ||||||
613 | Changed |= EnforceScalar(Elem); | ||||||
614 | |||||||
615 | SmallVector<unsigned, 4> Modes; | ||||||
616 | union_modes(Vec, Elem, Modes); | ||||||
617 | for (unsigned M : Modes) { | ||||||
618 | TypeSetByHwMode::SetType &V = Vec.get(M); | ||||||
619 | TypeSetByHwMode::SetType &E = Elem.get(M); | ||||||
620 | |||||||
621 | Changed |= berase_if(V, isScalar); // Scalar = !vector | ||||||
622 | Changed |= berase_if(E, isVector); // Vector = !scalar | ||||||
623 | assert(!V.empty() && !E.empty())(static_cast <bool> (!V.empty() && !E.empty()) ? void (0) : __assert_fail ("!V.empty() && !E.empty()" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 623, __extension__ __PRETTY_FUNCTION__)); | ||||||
624 | |||||||
625 | MachineValueTypeSet VT, ST; | ||||||
626 | // Collect element types from the "vector" set. | ||||||
627 | for (MVT T : V) | ||||||
628 | VT.insert(T.getVectorElementType()); | ||||||
629 | // Collect scalar types from the "element" set. | ||||||
630 | for (MVT T : E) | ||||||
631 | ST.insert(T); | ||||||
632 | |||||||
633 | // Remove from V all (vector) types whose element type is not in S. | ||||||
634 | Changed |= berase_if(V, [&ST](MVT T) -> bool { | ||||||
635 | return !ST.count(T.getVectorElementType()); | ||||||
636 | }); | ||||||
637 | // Remove from E all (scalar) types, for which there is no corresponding | ||||||
638 | // type in V. | ||||||
639 | Changed |= berase_if(E, [&VT](MVT T) -> bool { return !VT.count(T); }); | ||||||
640 | } | ||||||
641 | |||||||
642 | return Changed; | ||||||
643 | } | ||||||
644 | |||||||
645 | bool TypeInfer::EnforceVectorEltTypeIs(TypeSetByHwMode &Vec, | ||||||
646 | const ValueTypeByHwMode &VVT) { | ||||||
647 | TypeSetByHwMode Tmp(VVT); | ||||||
648 | ValidateOnExit _1(Vec, *this), _2(Tmp, *this); | ||||||
649 | return EnforceVectorEltTypeIs(Vec, Tmp); | ||||||
650 | } | ||||||
651 | |||||||
652 | /// Ensure that for each type T in Sub, T is a vector type, and there | ||||||
653 | /// exists a type U in Vec such that U is a vector type with the same | ||||||
654 | /// element type as T and at least as many elements as T. | ||||||
655 | bool TypeInfer::EnforceVectorSubVectorTypeIs(TypeSetByHwMode &Vec, | ||||||
656 | TypeSetByHwMode &Sub) { | ||||||
657 | ValidateOnExit _1(Vec, *this), _2(Sub, *this); | ||||||
658 | if (TP.hasError()) | ||||||
659 | return false; | ||||||
660 | |||||||
661 | /// Return true if B is a suB-vector of P, i.e. P is a suPer-vector of B. | ||||||
662 | auto IsSubVec = [](MVT B, MVT P) -> bool { | ||||||
663 | if (!B.isVector() || !P.isVector()) | ||||||
664 | return false; | ||||||
665 | // Logically a <4 x i32> is a valid subvector of <n x 4 x i32> | ||||||
666 | // but until there are obvious use-cases for this, keep the | ||||||
667 | // types separate. | ||||||
668 | if (B.isScalableVector() != P.isScalableVector()) | ||||||
669 | return false; | ||||||
670 | if (B.getVectorElementType() != P.getVectorElementType()) | ||||||
671 | return false; | ||||||
672 | return B.getVectorMinNumElements() < P.getVectorMinNumElements(); | ||||||
673 | }; | ||||||
674 | |||||||
675 | /// Return true if S has no element (vector type) that T is a sub-vector of, | ||||||
676 | /// i.e. has the same element type as T and more elements. | ||||||
677 | auto NoSubV = [&IsSubVec](const TypeSetByHwMode::SetType &S, MVT T) -> bool { | ||||||
678 | for (auto I : S) | ||||||
679 | if (IsSubVec(T, I)) | ||||||
680 | return false; | ||||||
681 | return true; | ||||||
682 | }; | ||||||
683 | |||||||
684 | /// Return true if S has no element (vector type) that T is a super-vector | ||||||
685 | /// of, i.e. has the same element type as T and fewer elements. | ||||||
686 | auto NoSupV = [&IsSubVec](const TypeSetByHwMode::SetType &S, MVT T) -> bool { | ||||||
687 | for (auto I : S) | ||||||
688 | if (IsSubVec(I, T)) | ||||||
689 | return false; | ||||||
690 | return true; | ||||||
691 | }; | ||||||
692 | |||||||
693 | bool Changed = false; | ||||||
694 | |||||||
695 | if (Vec.empty()) | ||||||
696 | Changed |= EnforceVector(Vec); | ||||||
697 | if (Sub.empty()) | ||||||
698 | Changed |= EnforceVector(Sub); | ||||||
699 | |||||||
700 | SmallVector<unsigned, 4> Modes; | ||||||
701 | union_modes(Vec, Sub, Modes); | ||||||
702 | for (unsigned M : Modes) { | ||||||
703 | TypeSetByHwMode::SetType &S = Sub.get(M); | ||||||
704 | TypeSetByHwMode::SetType &V = Vec.get(M); | ||||||
705 | |||||||
706 | Changed |= berase_if(S, isScalar); | ||||||
707 | |||||||
708 | // Erase all types from S that are not sub-vectors of a type in V. | ||||||
709 | Changed |= berase_if(S, std::bind(NoSubV, V, std::placeholders::_1)); | ||||||
710 | |||||||
711 | // Erase all types from V that are not super-vectors of a type in S. | ||||||
712 | Changed |= berase_if(V, std::bind(NoSupV, S, std::placeholders::_1)); | ||||||
713 | } | ||||||
714 | |||||||
715 | return Changed; | ||||||
716 | } | ||||||
717 | |||||||
718 | /// 1. Ensure that V has a scalar type iff W has a scalar type. | ||||||
719 | /// 2. Ensure that for each vector type T in V, there exists a vector | ||||||
720 | /// type U in W, such that T and U have the same number of elements. | ||||||
721 | /// 3. Ensure that for each vector type U in W, there exists a vector | ||||||
722 | /// type T in V, such that T and U have the same number of elements | ||||||
723 | /// (reverse of 2). | ||||||
724 | bool TypeInfer::EnforceSameNumElts(TypeSetByHwMode &V, TypeSetByHwMode &W) { | ||||||
725 | ValidateOnExit _1(V, *this), _2(W, *this); | ||||||
726 | if (TP.hasError()) | ||||||
727 | return false; | ||||||
728 | |||||||
729 | bool Changed = false; | ||||||
730 | if (V.empty()) | ||||||
731 | Changed |= EnforceAny(V); | ||||||
732 | if (W.empty()) | ||||||
733 | Changed |= EnforceAny(W); | ||||||
734 | |||||||
735 | // An actual vector type cannot have 0 elements, so we can treat scalars | ||||||
736 | // as zero-length vectors. This way both vectors and scalars can be | ||||||
737 | // processed identically. | ||||||
738 | auto NoLength = [](const SmallDenseSet<ElementCount> &Lengths, | ||||||
739 | MVT T) -> bool { | ||||||
740 | return !Lengths.count(T.isVector() ? T.getVectorElementCount() | ||||||
741 | : ElementCount()); | ||||||
742 | }; | ||||||
743 | |||||||
744 | SmallVector<unsigned, 4> Modes; | ||||||
745 | union_modes(V, W, Modes); | ||||||
746 | for (unsigned M : Modes) { | ||||||
747 | TypeSetByHwMode::SetType &VS = V.get(M); | ||||||
748 | TypeSetByHwMode::SetType &WS = W.get(M); | ||||||
749 | |||||||
750 | SmallDenseSet<ElementCount> VN, WN; | ||||||
751 | for (MVT T : VS) | ||||||
752 | VN.insert(T.isVector() ? T.getVectorElementCount() : ElementCount()); | ||||||
753 | for (MVT T : WS) | ||||||
754 | WN.insert(T.isVector() ? T.getVectorElementCount() : ElementCount()); | ||||||
755 | |||||||
756 | Changed |= berase_if(VS, std::bind(NoLength, WN, std::placeholders::_1)); | ||||||
757 | Changed |= berase_if(WS, std::bind(NoLength, VN, std::placeholders::_1)); | ||||||
758 | } | ||||||
759 | return Changed; | ||||||
760 | } | ||||||
761 | |||||||
762 | namespace { | ||||||
763 | struct TypeSizeComparator { | ||||||
764 | bool operator()(const TypeSize &LHS, const TypeSize &RHS) const { | ||||||
765 | return std::make_tuple(LHS.isScalable(), LHS.getKnownMinValue()) < | ||||||
766 | std::make_tuple(RHS.isScalable(), RHS.getKnownMinValue()); | ||||||
767 | } | ||||||
768 | }; | ||||||
769 | } // end anonymous namespace | ||||||
770 | |||||||
771 | /// 1. Ensure that for each type T in A, there exists a type U in B, | ||||||
772 | /// such that T and U have equal size in bits. | ||||||
773 | /// 2. Ensure that for each type U in B, there exists a type T in A | ||||||
774 | /// such that T and U have equal size in bits (reverse of 1). | ||||||
775 | bool TypeInfer::EnforceSameSize(TypeSetByHwMode &A, TypeSetByHwMode &B) { | ||||||
776 | ValidateOnExit _1(A, *this), _2(B, *this); | ||||||
777 | if (TP.hasError()) | ||||||
778 | return false; | ||||||
779 | bool Changed = false; | ||||||
780 | if (A.empty()) | ||||||
781 | Changed |= EnforceAny(A); | ||||||
782 | if (B.empty()) | ||||||
783 | Changed |= EnforceAny(B); | ||||||
784 | |||||||
785 | typedef SmallSet<TypeSize, 2, TypeSizeComparator> TypeSizeSet; | ||||||
786 | |||||||
787 | auto NoSize = [](const TypeSizeSet &Sizes, MVT T) -> bool { | ||||||
788 | return !Sizes.count(T.getSizeInBits()); | ||||||
789 | }; | ||||||
790 | |||||||
791 | SmallVector<unsigned, 4> Modes; | ||||||
792 | union_modes(A, B, Modes); | ||||||
793 | for (unsigned M : Modes) { | ||||||
794 | TypeSetByHwMode::SetType &AS = A.get(M); | ||||||
795 | TypeSetByHwMode::SetType &BS = B.get(M); | ||||||
796 | TypeSizeSet AN, BN; | ||||||
797 | |||||||
798 | for (MVT T : AS) | ||||||
799 | AN.insert(T.getSizeInBits()); | ||||||
800 | for (MVT T : BS) | ||||||
801 | BN.insert(T.getSizeInBits()); | ||||||
802 | |||||||
803 | Changed |= berase_if(AS, std::bind(NoSize, BN, std::placeholders::_1)); | ||||||
804 | Changed |= berase_if(BS, std::bind(NoSize, AN, std::placeholders::_1)); | ||||||
805 | } | ||||||
806 | |||||||
807 | return Changed; | ||||||
808 | } | ||||||
809 | |||||||
810 | void TypeInfer::expandOverloads(TypeSetByHwMode &VTS) { | ||||||
811 | ValidateOnExit _1(VTS, *this); | ||||||
812 | const TypeSetByHwMode &Legal = getLegalTypes(); | ||||||
813 | assert(Legal.isSimple() && "Default-mode only expected")(static_cast <bool> (Legal.isSimple() && "Default-mode only expected" ) ? void (0) : __assert_fail ("Legal.isSimple() && \"Default-mode only expected\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 813, __extension__ __PRETTY_FUNCTION__)); | ||||||
814 | const TypeSetByHwMode::SetType &LegalTypes = Legal.getSimple(); | ||||||
815 | |||||||
816 | for (auto &I : VTS) | ||||||
817 | expandOverloads(I.second, LegalTypes); | ||||||
818 | } | ||||||
819 | |||||||
820 | void TypeInfer::expandOverloads(TypeSetByHwMode::SetType &Out, | ||||||
821 | const TypeSetByHwMode::SetType &Legal) { | ||||||
822 | if (Out.count(MVT::iPTRAny)) { | ||||||
823 | Out.erase(MVT::iPTRAny); | ||||||
824 | Out.insert(MVT::iPTR); | ||||||
825 | } else if (Out.count(MVT::iAny)) { | ||||||
826 | Out.erase(MVT::iAny); | ||||||
827 | for (MVT T : MVT::integer_valuetypes()) | ||||||
828 | if (Legal.count(T)) | ||||||
829 | Out.insert(T); | ||||||
830 | for (MVT T : MVT::integer_fixedlen_vector_valuetypes()) | ||||||
831 | if (Legal.count(T)) | ||||||
832 | Out.insert(T); | ||||||
833 | for (MVT T : MVT::integer_scalable_vector_valuetypes()) | ||||||
834 | if (Legal.count(T)) | ||||||
835 | Out.insert(T); | ||||||
836 | } else if (Out.count(MVT::fAny)) { | ||||||
837 | Out.erase(MVT::fAny); | ||||||
838 | for (MVT T : MVT::fp_valuetypes()) | ||||||
839 | if (Legal.count(T)) | ||||||
840 | Out.insert(T); | ||||||
841 | for (MVT T : MVT::fp_fixedlen_vector_valuetypes()) | ||||||
842 | if (Legal.count(T)) | ||||||
843 | Out.insert(T); | ||||||
844 | for (MVT T : MVT::fp_scalable_vector_valuetypes()) | ||||||
845 | if (Legal.count(T)) | ||||||
846 | Out.insert(T); | ||||||
847 | } else if (Out.count(MVT::vAny)) { | ||||||
848 | Out.erase(MVT::vAny); | ||||||
849 | for (MVT T : MVT::vector_valuetypes()) | ||||||
850 | if (Legal.count(T)) | ||||||
851 | Out.insert(T); | ||||||
852 | } else if (Out.count(MVT::Any)) { | ||||||
853 | Out.erase(MVT::Any); | ||||||
854 | for (MVT T : MVT::all_valuetypes()) | ||||||
855 | if (Legal.count(T)) | ||||||
856 | Out.insert(T); | ||||||
857 | } | ||||||
858 | } | ||||||
859 | |||||||
860 | const TypeSetByHwMode &TypeInfer::getLegalTypes() { | ||||||
861 | if (!LegalTypesCached) { | ||||||
862 | TypeSetByHwMode::SetType &LegalTypes = LegalCache.getOrCreate(DefaultMode); | ||||||
863 | // Stuff all types from all modes into the default mode. | ||||||
864 | const TypeSetByHwMode <S = TP.getDAGPatterns().getLegalTypes(); | ||||||
865 | for (const auto &I : LTS) | ||||||
866 | LegalTypes.insert(I.second); | ||||||
867 | LegalTypesCached = true; | ||||||
868 | } | ||||||
869 | assert(LegalCache.isSimple() && "Default-mode only expected")(static_cast <bool> (LegalCache.isSimple() && "Default-mode only expected" ) ? void (0) : __assert_fail ("LegalCache.isSimple() && \"Default-mode only expected\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 869, __extension__ __PRETTY_FUNCTION__)); | ||||||
870 | return LegalCache; | ||||||
871 | } | ||||||
872 | |||||||
873 | TypeInfer::ValidateOnExit::~ValidateOnExit() { | ||||||
874 | if (Infer.Validate && !VTS.validate()) { | ||||||
875 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | ||||||
876 | errs() << "Type set is empty for each HW mode:\n" | ||||||
877 | "possible type contradiction in the pattern below " | ||||||
878 | "(use -print-records with llvm-tblgen to see all " | ||||||
879 | "expanded records).\n"; | ||||||
880 | Infer.TP.dump(); | ||||||
881 | errs() << "Generated from record:\n"; | ||||||
882 | Infer.TP.getRecord()->dump(); | ||||||
883 | #endif | ||||||
884 | PrintFatalError(Infer.TP.getRecord()->getLoc(), | ||||||
885 | "Type set is empty for each HW mode in '" + | ||||||
886 | Infer.TP.getRecord()->getName() + "'"); | ||||||
887 | } | ||||||
888 | } | ||||||
889 | |||||||
890 | |||||||
891 | //===----------------------------------------------------------------------===// | ||||||
892 | // ScopedName Implementation | ||||||
893 | //===----------------------------------------------------------------------===// | ||||||
894 | |||||||
895 | bool ScopedName::operator==(const ScopedName &o) const { | ||||||
896 | return Scope == o.Scope && Identifier == o.Identifier; | ||||||
897 | } | ||||||
898 | |||||||
899 | bool ScopedName::operator!=(const ScopedName &o) const { | ||||||
900 | return !(*this == o); | ||||||
901 | } | ||||||
902 | |||||||
903 | |||||||
904 | //===----------------------------------------------------------------------===// | ||||||
905 | // TreePredicateFn Implementation | ||||||
906 | //===----------------------------------------------------------------------===// | ||||||
907 | |||||||
908 | /// TreePredicateFn constructor. Here 'N' is a subclass of PatFrag. | ||||||
909 | TreePredicateFn::TreePredicateFn(TreePattern *N) : PatFragRec(N) { | ||||||
910 | assert((static_cast <bool> ((!hasPredCode() || !hasImmCode()) && ".td file corrupt: can't have a node predicate *and* an imm predicate" ) ? void (0) : __assert_fail ("(!hasPredCode() || !hasImmCode()) && \".td file corrupt: can't have a node predicate *and* an imm predicate\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 912, __extension__ __PRETTY_FUNCTION__)) | ||||||
911 | (!hasPredCode() || !hasImmCode()) &&(static_cast <bool> ((!hasPredCode() || !hasImmCode()) && ".td file corrupt: can't have a node predicate *and* an imm predicate" ) ? void (0) : __assert_fail ("(!hasPredCode() || !hasImmCode()) && \".td file corrupt: can't have a node predicate *and* an imm predicate\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 912, __extension__ __PRETTY_FUNCTION__)) | ||||||
912 | ".td file corrupt: can't have a node predicate *and* an imm predicate")(static_cast <bool> ((!hasPredCode() || !hasImmCode()) && ".td file corrupt: can't have a node predicate *and* an imm predicate" ) ? void (0) : __assert_fail ("(!hasPredCode() || !hasImmCode()) && \".td file corrupt: can't have a node predicate *and* an imm predicate\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 912, __extension__ __PRETTY_FUNCTION__)); | ||||||
913 | } | ||||||
914 | |||||||
915 | bool TreePredicateFn::hasPredCode() const { | ||||||
916 | return isLoad() || isStore() || isAtomic() || hasNoUse() || | ||||||
917 | !PatFragRec->getRecord()->getValueAsString("PredicateCode").empty(); | ||||||
918 | } | ||||||
919 | |||||||
920 | std::string TreePredicateFn::getPredCode() const { | ||||||
921 | std::string Code; | ||||||
922 | |||||||
923 | if (!isLoad() && !isStore() && !isAtomic()) { | ||||||
924 | Record *MemoryVT = getMemoryVT(); | ||||||
925 | |||||||
926 | if (MemoryVT) | ||||||
927 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
928 | "MemoryVT requires IsLoad or IsStore"); | ||||||
929 | } | ||||||
930 | |||||||
931 | if (!isLoad() && !isStore()) { | ||||||
932 | if (isUnindexed()) | ||||||
933 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
934 | "IsUnindexed requires IsLoad or IsStore"); | ||||||
935 | |||||||
936 | Record *ScalarMemoryVT = getScalarMemoryVT(); | ||||||
937 | |||||||
938 | if (ScalarMemoryVT) | ||||||
939 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
940 | "ScalarMemoryVT requires IsLoad or IsStore"); | ||||||
941 | } | ||||||
942 | |||||||
943 | if (isLoad() + isStore() + isAtomic() > 1) | ||||||
944 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
945 | "IsLoad, IsStore, and IsAtomic are mutually exclusive"); | ||||||
946 | |||||||
947 | if (isLoad()) { | ||||||
948 | if (!isUnindexed() && !isNonExtLoad() && !isAnyExtLoad() && | ||||||
949 | !isSignExtLoad() && !isZeroExtLoad() && getMemoryVT() == nullptr && | ||||||
950 | getScalarMemoryVT() == nullptr && getAddressSpaces() == nullptr && | ||||||
951 | getMinAlignment() < 1) | ||||||
952 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
953 | "IsLoad cannot be used by itself"); | ||||||
954 | } else { | ||||||
955 | if (isNonExtLoad()) | ||||||
956 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
957 | "IsNonExtLoad requires IsLoad"); | ||||||
958 | if (isAnyExtLoad()) | ||||||
959 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
960 | "IsAnyExtLoad requires IsLoad"); | ||||||
961 | |||||||
962 | if (!isAtomic()) { | ||||||
963 | if (isSignExtLoad()) | ||||||
964 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
965 | "IsSignExtLoad requires IsLoad or IsAtomic"); | ||||||
966 | if (isZeroExtLoad()) | ||||||
967 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
968 | "IsZeroExtLoad requires IsLoad or IsAtomic"); | ||||||
969 | } | ||||||
970 | } | ||||||
971 | |||||||
972 | if (isStore()) { | ||||||
973 | if (!isUnindexed() && !isTruncStore() && !isNonTruncStore() && | ||||||
974 | getMemoryVT() == nullptr && getScalarMemoryVT() == nullptr && | ||||||
975 | getAddressSpaces() == nullptr && getMinAlignment() < 1) | ||||||
976 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
977 | "IsStore cannot be used by itself"); | ||||||
978 | } else { | ||||||
979 | if (isNonTruncStore()) | ||||||
980 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
981 | "IsNonTruncStore requires IsStore"); | ||||||
982 | if (isTruncStore()) | ||||||
983 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
984 | "IsTruncStore requires IsStore"); | ||||||
985 | } | ||||||
986 | |||||||
987 | if (isAtomic()) { | ||||||
988 | if (getMemoryVT() == nullptr && !isAtomicOrderingMonotonic() && | ||||||
989 | getAddressSpaces() == nullptr && | ||||||
990 | // FIXME: Should atomic loads be IsLoad, IsAtomic, or both? | ||||||
991 | !isZeroExtLoad() && !isSignExtLoad() && !isAtomicOrderingAcquire() && | ||||||
992 | !isAtomicOrderingRelease() && !isAtomicOrderingAcquireRelease() && | ||||||
993 | !isAtomicOrderingSequentiallyConsistent() && | ||||||
994 | !isAtomicOrderingAcquireOrStronger() && | ||||||
995 | !isAtomicOrderingReleaseOrStronger() && | ||||||
996 | !isAtomicOrderingWeakerThanAcquire() && | ||||||
997 | !isAtomicOrderingWeakerThanRelease()) | ||||||
998 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
999 | "IsAtomic cannot be used by itself"); | ||||||
1000 | } else { | ||||||
1001 | if (isAtomicOrderingMonotonic()) | ||||||
1002 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1003 | "IsAtomicOrderingMonotonic requires IsAtomic"); | ||||||
1004 | if (isAtomicOrderingAcquire()) | ||||||
1005 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1006 | "IsAtomicOrderingAcquire requires IsAtomic"); | ||||||
1007 | if (isAtomicOrderingRelease()) | ||||||
1008 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1009 | "IsAtomicOrderingRelease requires IsAtomic"); | ||||||
1010 | if (isAtomicOrderingAcquireRelease()) | ||||||
1011 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1012 | "IsAtomicOrderingAcquireRelease requires IsAtomic"); | ||||||
1013 | if (isAtomicOrderingSequentiallyConsistent()) | ||||||
1014 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1015 | "IsAtomicOrderingSequentiallyConsistent requires IsAtomic"); | ||||||
1016 | if (isAtomicOrderingAcquireOrStronger()) | ||||||
1017 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1018 | "IsAtomicOrderingAcquireOrStronger requires IsAtomic"); | ||||||
1019 | if (isAtomicOrderingReleaseOrStronger()) | ||||||
1020 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1021 | "IsAtomicOrderingReleaseOrStronger requires IsAtomic"); | ||||||
1022 | if (isAtomicOrderingWeakerThanAcquire()) | ||||||
1023 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1024 | "IsAtomicOrderingWeakerThanAcquire requires IsAtomic"); | ||||||
1025 | } | ||||||
1026 | |||||||
1027 | if (isLoad() || isStore() || isAtomic()) { | ||||||
1028 | if (ListInit *AddressSpaces = getAddressSpaces()) { | ||||||
1029 | Code += "unsigned AddrSpace = cast<MemSDNode>(N)->getAddressSpace();\n" | ||||||
1030 | " if ("; | ||||||
1031 | |||||||
1032 | ListSeparator LS(" && "); | ||||||
1033 | for (Init *Val : AddressSpaces->getValues()) { | ||||||
1034 | Code += LS; | ||||||
1035 | |||||||
1036 | IntInit *IntVal = dyn_cast<IntInit>(Val); | ||||||
1037 | if (!IntVal) { | ||||||
1038 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1039 | "AddressSpaces element must be integer"); | ||||||
1040 | } | ||||||
1041 | |||||||
1042 | Code += "AddrSpace != " + utostr(IntVal->getValue()); | ||||||
1043 | } | ||||||
1044 | |||||||
1045 | Code += ")\nreturn false;\n"; | ||||||
1046 | } | ||||||
1047 | |||||||
1048 | int64_t MinAlign = getMinAlignment(); | ||||||
1049 | if (MinAlign > 0) { | ||||||
1050 | Code += "if (cast<MemSDNode>(N)->getAlign() < Align("; | ||||||
1051 | Code += utostr(MinAlign); | ||||||
1052 | Code += "))\nreturn false;\n"; | ||||||
1053 | } | ||||||
1054 | |||||||
1055 | Record *MemoryVT = getMemoryVT(); | ||||||
1056 | |||||||
1057 | if (MemoryVT) | ||||||
1058 | Code += ("if (cast<MemSDNode>(N)->getMemoryVT() != MVT::" + | ||||||
1059 | MemoryVT->getName() + ") return false;\n") | ||||||
1060 | .str(); | ||||||
1061 | } | ||||||
1062 | |||||||
1063 | if (isAtomic() && isAtomicOrderingMonotonic()) | ||||||
1064 | Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != " | ||||||
1065 | "AtomicOrdering::Monotonic) return false;\n"; | ||||||
1066 | if (isAtomic() && isAtomicOrderingAcquire()) | ||||||
1067 | Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != " | ||||||
1068 | "AtomicOrdering::Acquire) return false;\n"; | ||||||
1069 | if (isAtomic() && isAtomicOrderingRelease()) | ||||||
1070 | Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != " | ||||||
1071 | "AtomicOrdering::Release) return false;\n"; | ||||||
1072 | if (isAtomic() && isAtomicOrderingAcquireRelease()) | ||||||
1073 | Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != " | ||||||
1074 | "AtomicOrdering::AcquireRelease) return false;\n"; | ||||||
1075 | if (isAtomic() && isAtomicOrderingSequentiallyConsistent()) | ||||||
1076 | Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != " | ||||||
1077 | "AtomicOrdering::SequentiallyConsistent) return false;\n"; | ||||||
1078 | |||||||
1079 | if (isAtomic() && isAtomicOrderingAcquireOrStronger()) | ||||||
1080 | Code += "if (!isAcquireOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) " | ||||||
1081 | "return false;\n"; | ||||||
1082 | if (isAtomic() && isAtomicOrderingWeakerThanAcquire()) | ||||||
1083 | Code += "if (isAcquireOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) " | ||||||
1084 | "return false;\n"; | ||||||
1085 | |||||||
1086 | if (isAtomic() && isAtomicOrderingReleaseOrStronger()) | ||||||
1087 | Code += "if (!isReleaseOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) " | ||||||
1088 | "return false;\n"; | ||||||
1089 | if (isAtomic() && isAtomicOrderingWeakerThanRelease()) | ||||||
1090 | Code += "if (isReleaseOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) " | ||||||
1091 | "return false;\n"; | ||||||
1092 | |||||||
1093 | // TODO: Handle atomic sextload/zextload normally when ATOMIC_LOAD is removed. | ||||||
1094 | if (isAtomic() && (isZeroExtLoad() || isSignExtLoad())) | ||||||
1095 | Code += "return false;\n"; | ||||||
1096 | |||||||
1097 | if (isLoad() || isStore()) { | ||||||
1098 | StringRef SDNodeName = isLoad() ? "LoadSDNode" : "StoreSDNode"; | ||||||
1099 | |||||||
1100 | if (isUnindexed()) | ||||||
1101 | Code += ("if (cast<" + SDNodeName + | ||||||
1102 | ">(N)->getAddressingMode() != ISD::UNINDEXED) " | ||||||
1103 | "return false;\n") | ||||||
1104 | .str(); | ||||||
1105 | |||||||
1106 | if (isLoad()) { | ||||||
1107 | if ((isNonExtLoad() + isAnyExtLoad() + isSignExtLoad() + | ||||||
1108 | isZeroExtLoad()) > 1) | ||||||
1109 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1110 | "IsNonExtLoad, IsAnyExtLoad, IsSignExtLoad, and " | ||||||
1111 | "IsZeroExtLoad are mutually exclusive"); | ||||||
1112 | if (isNonExtLoad()) | ||||||
1113 | Code += "if (cast<LoadSDNode>(N)->getExtensionType() != " | ||||||
1114 | "ISD::NON_EXTLOAD) return false;\n"; | ||||||
1115 | if (isAnyExtLoad()) | ||||||
1116 | Code += "if (cast<LoadSDNode>(N)->getExtensionType() != ISD::EXTLOAD) " | ||||||
1117 | "return false;\n"; | ||||||
1118 | if (isSignExtLoad()) | ||||||
1119 | Code += "if (cast<LoadSDNode>(N)->getExtensionType() != ISD::SEXTLOAD) " | ||||||
1120 | "return false;\n"; | ||||||
1121 | if (isZeroExtLoad()) | ||||||
1122 | Code += "if (cast<LoadSDNode>(N)->getExtensionType() != ISD::ZEXTLOAD) " | ||||||
1123 | "return false;\n"; | ||||||
1124 | } else { | ||||||
1125 | if ((isNonTruncStore() + isTruncStore()) > 1) | ||||||
1126 | PrintFatalError( | ||||||
1127 | getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1128 | "IsNonTruncStore, and IsTruncStore are mutually exclusive"); | ||||||
1129 | if (isNonTruncStore()) | ||||||
1130 | Code += | ||||||
1131 | " if (cast<StoreSDNode>(N)->isTruncatingStore()) return false;\n"; | ||||||
1132 | if (isTruncStore()) | ||||||
1133 | Code += | ||||||
1134 | " if (!cast<StoreSDNode>(N)->isTruncatingStore()) return false;\n"; | ||||||
1135 | } | ||||||
1136 | |||||||
1137 | Record *ScalarMemoryVT = getScalarMemoryVT(); | ||||||
1138 | |||||||
1139 | if (ScalarMemoryVT) | ||||||
1140 | Code += ("if (cast<" + SDNodeName + | ||||||
1141 | ">(N)->getMemoryVT().getScalarType() != MVT::" + | ||||||
1142 | ScalarMemoryVT->getName() + ") return false;\n") | ||||||
1143 | .str(); | ||||||
1144 | } | ||||||
1145 | |||||||
1146 | if (hasNoUse()) | ||||||
1147 | Code += "if (!SDValue(N, 0).use_empty()) return false;\n"; | ||||||
1148 | |||||||
1149 | std::string PredicateCode = | ||||||
1150 | std::string(PatFragRec->getRecord()->getValueAsString("PredicateCode")); | ||||||
1151 | |||||||
1152 | Code += PredicateCode; | ||||||
1153 | |||||||
1154 | if (PredicateCode.empty() && !Code.empty()) | ||||||
1155 | Code += "return true;\n"; | ||||||
1156 | |||||||
1157 | return Code; | ||||||
1158 | } | ||||||
1159 | |||||||
1160 | bool TreePredicateFn::hasImmCode() const { | ||||||
1161 | return !PatFragRec->getRecord()->getValueAsString("ImmediateCode").empty(); | ||||||
1162 | } | ||||||
1163 | |||||||
1164 | std::string TreePredicateFn::getImmCode() const { | ||||||
1165 | return std::string( | ||||||
1166 | PatFragRec->getRecord()->getValueAsString("ImmediateCode")); | ||||||
1167 | } | ||||||
1168 | |||||||
1169 | bool TreePredicateFn::immCodeUsesAPInt() const { | ||||||
1170 | return getOrigPatFragRecord()->getRecord()->getValueAsBit("IsAPInt"); | ||||||
1171 | } | ||||||
1172 | |||||||
1173 | bool TreePredicateFn::immCodeUsesAPFloat() const { | ||||||
1174 | bool Unset; | ||||||
1175 | // The return value will be false when IsAPFloat is unset. | ||||||
1176 | return getOrigPatFragRecord()->getRecord()->getValueAsBitOrUnset("IsAPFloat", | ||||||
1177 | Unset); | ||||||
1178 | } | ||||||
1179 | |||||||
1180 | bool TreePredicateFn::isPredefinedPredicateEqualTo(StringRef Field, | ||||||
1181 | bool Value) const { | ||||||
1182 | bool Unset; | ||||||
1183 | bool Result = | ||||||
1184 | getOrigPatFragRecord()->getRecord()->getValueAsBitOrUnset(Field, Unset); | ||||||
1185 | if (Unset) | ||||||
1186 | return false; | ||||||
1187 | return Result == Value; | ||||||
1188 | } | ||||||
1189 | bool TreePredicateFn::usesOperands() const { | ||||||
1190 | return isPredefinedPredicateEqualTo("PredicateCodeUsesOperands", true); | ||||||
1191 | } | ||||||
1192 | bool TreePredicateFn::hasNoUse() const { | ||||||
1193 | return isPredefinedPredicateEqualTo("HasNoUse", true); | ||||||
1194 | } | ||||||
1195 | bool TreePredicateFn::isLoad() const { | ||||||
1196 | return isPredefinedPredicateEqualTo("IsLoad", true); | ||||||
1197 | } | ||||||
1198 | bool TreePredicateFn::isStore() const { | ||||||
1199 | return isPredefinedPredicateEqualTo("IsStore", true); | ||||||
1200 | } | ||||||
1201 | bool TreePredicateFn::isAtomic() const { | ||||||
1202 | return isPredefinedPredicateEqualTo("IsAtomic", true); | ||||||
1203 | } | ||||||
1204 | bool TreePredicateFn::isUnindexed() const { | ||||||
1205 | return isPredefinedPredicateEqualTo("IsUnindexed", true); | ||||||
1206 | } | ||||||
1207 | bool TreePredicateFn::isNonExtLoad() const { | ||||||
1208 | return isPredefinedPredicateEqualTo("IsNonExtLoad", true); | ||||||
1209 | } | ||||||
1210 | bool TreePredicateFn::isAnyExtLoad() const { | ||||||
1211 | return isPredefinedPredicateEqualTo("IsAnyExtLoad", true); | ||||||
1212 | } | ||||||
1213 | bool TreePredicateFn::isSignExtLoad() const { | ||||||
1214 | return isPredefinedPredicateEqualTo("IsSignExtLoad", true); | ||||||
1215 | } | ||||||
1216 | bool TreePredicateFn::isZeroExtLoad() const { | ||||||
1217 | return isPredefinedPredicateEqualTo("IsZeroExtLoad", true); | ||||||
1218 | } | ||||||
1219 | bool TreePredicateFn::isNonTruncStore() const { | ||||||
1220 | return isPredefinedPredicateEqualTo("IsTruncStore", false); | ||||||
1221 | } | ||||||
1222 | bool TreePredicateFn::isTruncStore() const { | ||||||
1223 | return isPredefinedPredicateEqualTo("IsTruncStore", true); | ||||||
1224 | } | ||||||
1225 | bool TreePredicateFn::isAtomicOrderingMonotonic() const { | ||||||
1226 | return isPredefinedPredicateEqualTo("IsAtomicOrderingMonotonic", true); | ||||||
1227 | } | ||||||
1228 | bool TreePredicateFn::isAtomicOrderingAcquire() const { | ||||||
1229 | return isPredefinedPredicateEqualTo("IsAtomicOrderingAcquire", true); | ||||||
1230 | } | ||||||
1231 | bool TreePredicateFn::isAtomicOrderingRelease() const { | ||||||
1232 | return isPredefinedPredicateEqualTo("IsAtomicOrderingRelease", true); | ||||||
1233 | } | ||||||
1234 | bool TreePredicateFn::isAtomicOrderingAcquireRelease() const { | ||||||
1235 | return isPredefinedPredicateEqualTo("IsAtomicOrderingAcquireRelease", true); | ||||||
1236 | } | ||||||
1237 | bool TreePredicateFn::isAtomicOrderingSequentiallyConsistent() const { | ||||||
1238 | return isPredefinedPredicateEqualTo("IsAtomicOrderingSequentiallyConsistent", | ||||||
1239 | true); | ||||||
1240 | } | ||||||
1241 | bool TreePredicateFn::isAtomicOrderingAcquireOrStronger() const { | ||||||
1242 | return isPredefinedPredicateEqualTo("IsAtomicOrderingAcquireOrStronger", true); | ||||||
1243 | } | ||||||
1244 | bool TreePredicateFn::isAtomicOrderingWeakerThanAcquire() const { | ||||||
1245 | return isPredefinedPredicateEqualTo("IsAtomicOrderingAcquireOrStronger", false); | ||||||
1246 | } | ||||||
1247 | bool TreePredicateFn::isAtomicOrderingReleaseOrStronger() const { | ||||||
1248 | return isPredefinedPredicateEqualTo("IsAtomicOrderingReleaseOrStronger", true); | ||||||
1249 | } | ||||||
1250 | bool TreePredicateFn::isAtomicOrderingWeakerThanRelease() const { | ||||||
1251 | return isPredefinedPredicateEqualTo("IsAtomicOrderingReleaseOrStronger", false); | ||||||
1252 | } | ||||||
1253 | Record *TreePredicateFn::getMemoryVT() const { | ||||||
1254 | Record *R = getOrigPatFragRecord()->getRecord(); | ||||||
1255 | if (R->isValueUnset("MemoryVT")) | ||||||
1256 | return nullptr; | ||||||
1257 | return R->getValueAsDef("MemoryVT"); | ||||||
1258 | } | ||||||
1259 | |||||||
1260 | ListInit *TreePredicateFn::getAddressSpaces() const { | ||||||
1261 | Record *R = getOrigPatFragRecord()->getRecord(); | ||||||
1262 | if (R->isValueUnset("AddressSpaces")) | ||||||
1263 | return nullptr; | ||||||
1264 | return R->getValueAsListInit("AddressSpaces"); | ||||||
1265 | } | ||||||
1266 | |||||||
1267 | int64_t TreePredicateFn::getMinAlignment() const { | ||||||
1268 | Record *R = getOrigPatFragRecord()->getRecord(); | ||||||
1269 | if (R->isValueUnset("MinAlignment")) | ||||||
1270 | return 0; | ||||||
1271 | return R->getValueAsInt("MinAlignment"); | ||||||
1272 | } | ||||||
1273 | |||||||
1274 | Record *TreePredicateFn::getScalarMemoryVT() const { | ||||||
1275 | Record *R = getOrigPatFragRecord()->getRecord(); | ||||||
1276 | if (R->isValueUnset("ScalarMemoryVT")) | ||||||
1277 | return nullptr; | ||||||
1278 | return R->getValueAsDef("ScalarMemoryVT"); | ||||||
1279 | } | ||||||
1280 | bool TreePredicateFn::hasGISelPredicateCode() const { | ||||||
1281 | return !PatFragRec->getRecord() | ||||||
1282 | ->getValueAsString("GISelPredicateCode") | ||||||
1283 | .empty(); | ||||||
1284 | } | ||||||
1285 | std::string TreePredicateFn::getGISelPredicateCode() const { | ||||||
1286 | return std::string( | ||||||
1287 | PatFragRec->getRecord()->getValueAsString("GISelPredicateCode")); | ||||||
1288 | } | ||||||
1289 | |||||||
1290 | StringRef TreePredicateFn::getImmType() const { | ||||||
1291 | if (immCodeUsesAPInt()) | ||||||
1292 | return "const APInt &"; | ||||||
1293 | if (immCodeUsesAPFloat()) | ||||||
1294 | return "const APFloat &"; | ||||||
1295 | return "int64_t"; | ||||||
1296 | } | ||||||
1297 | |||||||
1298 | StringRef TreePredicateFn::getImmTypeIdentifier() const { | ||||||
1299 | if (immCodeUsesAPInt()) | ||||||
1300 | return "APInt"; | ||||||
1301 | if (immCodeUsesAPFloat()) | ||||||
1302 | return "APFloat"; | ||||||
1303 | return "I64"; | ||||||
1304 | } | ||||||
1305 | |||||||
1306 | /// isAlwaysTrue - Return true if this is a noop predicate. | ||||||
1307 | bool TreePredicateFn::isAlwaysTrue() const { | ||||||
1308 | return !hasPredCode() && !hasImmCode(); | ||||||
1309 | } | ||||||
1310 | |||||||
1311 | /// Return the name to use in the generated code to reference this, this is | ||||||
1312 | /// "Predicate_foo" if from a pattern fragment "foo". | ||||||
1313 | std::string TreePredicateFn::getFnName() const { | ||||||
1314 | return "Predicate_" + PatFragRec->getRecord()->getName().str(); | ||||||
1315 | } | ||||||
1316 | |||||||
1317 | /// getCodeToRunOnSDNode - Return the code for the function body that | ||||||
1318 | /// evaluates this predicate. The argument is expected to be in "Node", | ||||||
1319 | /// not N. This handles casting and conversion to a concrete node type as | ||||||
1320 | /// appropriate. | ||||||
1321 | std::string TreePredicateFn::getCodeToRunOnSDNode() const { | ||||||
1322 | // Handle immediate predicates first. | ||||||
1323 | std::string ImmCode = getImmCode(); | ||||||
1324 | if (!ImmCode.empty()) { | ||||||
1325 | if (isLoad()) | ||||||
1326 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1327 | "IsLoad cannot be used with ImmLeaf or its subclasses"); | ||||||
1328 | if (isStore()) | ||||||
1329 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1330 | "IsStore cannot be used with ImmLeaf or its subclasses"); | ||||||
1331 | if (isUnindexed()) | ||||||
1332 | PrintFatalError( | ||||||
1333 | getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1334 | "IsUnindexed cannot be used with ImmLeaf or its subclasses"); | ||||||
1335 | if (isNonExtLoad()) | ||||||
1336 | PrintFatalError( | ||||||
1337 | getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1338 | "IsNonExtLoad cannot be used with ImmLeaf or its subclasses"); | ||||||
1339 | if (isAnyExtLoad()) | ||||||
1340 | PrintFatalError( | ||||||
1341 | getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1342 | "IsAnyExtLoad cannot be used with ImmLeaf or its subclasses"); | ||||||
1343 | if (isSignExtLoad()) | ||||||
1344 | PrintFatalError( | ||||||
1345 | getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1346 | "IsSignExtLoad cannot be used with ImmLeaf or its subclasses"); | ||||||
1347 | if (isZeroExtLoad()) | ||||||
1348 | PrintFatalError( | ||||||
1349 | getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1350 | "IsZeroExtLoad cannot be used with ImmLeaf or its subclasses"); | ||||||
1351 | if (isNonTruncStore()) | ||||||
1352 | PrintFatalError( | ||||||
1353 | getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1354 | "IsNonTruncStore cannot be used with ImmLeaf or its subclasses"); | ||||||
1355 | if (isTruncStore()) | ||||||
1356 | PrintFatalError( | ||||||
1357 | getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1358 | "IsTruncStore cannot be used with ImmLeaf or its subclasses"); | ||||||
1359 | if (getMemoryVT()) | ||||||
1360 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1361 | "MemoryVT cannot be used with ImmLeaf or its subclasses"); | ||||||
1362 | if (getScalarMemoryVT()) | ||||||
1363 | PrintFatalError( | ||||||
1364 | getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1365 | "ScalarMemoryVT cannot be used with ImmLeaf or its subclasses"); | ||||||
1366 | |||||||
1367 | std::string Result = (" " + getImmType() + " Imm = ").str(); | ||||||
1368 | if (immCodeUsesAPFloat()) | ||||||
1369 | Result += "cast<ConstantFPSDNode>(Node)->getValueAPF();\n"; | ||||||
1370 | else if (immCodeUsesAPInt()) | ||||||
1371 | Result += "cast<ConstantSDNode>(Node)->getAPIntValue();\n"; | ||||||
1372 | else | ||||||
1373 | Result += "cast<ConstantSDNode>(Node)->getSExtValue();\n"; | ||||||
1374 | return Result + ImmCode; | ||||||
1375 | } | ||||||
1376 | |||||||
1377 | // Handle arbitrary node predicates. | ||||||
1378 | assert(hasPredCode() && "Don't have any predicate code!")(static_cast <bool> (hasPredCode() && "Don't have any predicate code!" ) ? void (0) : __assert_fail ("hasPredCode() && \"Don't have any predicate code!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 1378, __extension__ __PRETTY_FUNCTION__)); | ||||||
1379 | |||||||
1380 | // If this is using PatFrags, there are multiple trees to search. They should | ||||||
1381 | // all have the same class. FIXME: Is there a way to find a common | ||||||
1382 | // superclass? | ||||||
1383 | StringRef ClassName; | ||||||
1384 | for (const auto &Tree : PatFragRec->getTrees()) { | ||||||
1385 | StringRef TreeClassName; | ||||||
1386 | if (Tree->isLeaf()) | ||||||
1387 | TreeClassName = "SDNode"; | ||||||
1388 | else { | ||||||
1389 | Record *Op = Tree->getOperator(); | ||||||
1390 | const SDNodeInfo &Info = PatFragRec->getDAGPatterns().getSDNodeInfo(Op); | ||||||
1391 | TreeClassName = Info.getSDClassName(); | ||||||
1392 | } | ||||||
1393 | |||||||
1394 | if (ClassName.empty()) | ||||||
1395 | ClassName = TreeClassName; | ||||||
1396 | else if (ClassName != TreeClassName) { | ||||||
1397 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | ||||||
1398 | "PatFrags trees do not have consistent class"); | ||||||
1399 | } | ||||||
1400 | } | ||||||
1401 | |||||||
1402 | std::string Result; | ||||||
1403 | if (ClassName == "SDNode") | ||||||
1404 | Result = " SDNode *N = Node;\n"; | ||||||
1405 | else | ||||||
1406 | Result = " auto *N = cast<" + ClassName.str() + ">(Node);\n"; | ||||||
1407 | |||||||
1408 | return (Twine(Result) + " (void)N;\n" + getPredCode()).str(); | ||||||
1409 | } | ||||||
1410 | |||||||
1411 | //===----------------------------------------------------------------------===// | ||||||
1412 | // PatternToMatch implementation | ||||||
1413 | // | ||||||
1414 | |||||||
1415 | static bool isImmAllOnesAllZerosMatch(const TreePatternNode *P) { | ||||||
1416 | if (!P->isLeaf()) | ||||||
1417 | return false; | ||||||
1418 | DefInit *DI = dyn_cast<DefInit>(P->getLeafValue()); | ||||||
1419 | if (!DI) | ||||||
1420 | return false; | ||||||
1421 | |||||||
1422 | Record *R = DI->getDef(); | ||||||
1423 | return R->getName() == "immAllOnesV" || R->getName() == "immAllZerosV"; | ||||||
1424 | } | ||||||
1425 | |||||||
1426 | /// getPatternSize - Return the 'size' of this pattern. We want to match large | ||||||
1427 | /// patterns before small ones. This is used to determine the size of a | ||||||
1428 | /// pattern. | ||||||
1429 | static unsigned getPatternSize(const TreePatternNode *P, | ||||||
1430 | const CodeGenDAGPatterns &CGP) { | ||||||
1431 | unsigned Size = 3; // The node itself. | ||||||
1432 | // If the root node is a ConstantSDNode, increases its size. | ||||||
1433 | // e.g. (set R32:$dst, 0). | ||||||
1434 | if (P->isLeaf() && isa<IntInit>(P->getLeafValue())) | ||||||
1435 | Size += 2; | ||||||
1436 | |||||||
1437 | if (const ComplexPattern *AM = P->getComplexPatternInfo(CGP)) { | ||||||
1438 | Size += AM->getComplexity(); | ||||||
1439 | // We don't want to count any children twice, so return early. | ||||||
1440 | return Size; | ||||||
1441 | } | ||||||
1442 | |||||||
1443 | // If this node has some predicate function that must match, it adds to the | ||||||
1444 | // complexity of this node. | ||||||
1445 | if (!P->getPredicateCalls().empty()) | ||||||
1446 | ++Size; | ||||||
1447 | |||||||
1448 | // Count children in the count if they are also nodes. | ||||||
1449 | for (unsigned i = 0, e = P->getNumChildren(); i != e; ++i) { | ||||||
1450 | const TreePatternNode *Child = P->getChild(i); | ||||||
1451 | if (!Child->isLeaf() && Child->getNumTypes()) { | ||||||
1452 | const TypeSetByHwMode &T0 = Child->getExtType(0); | ||||||
1453 | // At this point, all variable type sets should be simple, i.e. only | ||||||
1454 | // have a default mode. | ||||||
1455 | if (T0.getMachineValueType() != MVT::Other) { | ||||||
1456 | Size += getPatternSize(Child, CGP); | ||||||
1457 | continue; | ||||||
1458 | } | ||||||
1459 | } | ||||||
1460 | if (Child->isLeaf()) { | ||||||
1461 | if (isa<IntInit>(Child->getLeafValue())) | ||||||
1462 | Size += 5; // Matches a ConstantSDNode (+3) and a specific value (+2). | ||||||
1463 | else if (Child->getComplexPatternInfo(CGP)) | ||||||
1464 | Size += getPatternSize(Child, CGP); | ||||||
1465 | else if (isImmAllOnesAllZerosMatch(Child)) | ||||||
1466 | Size += 4; // Matches a build_vector(+3) and a predicate (+1). | ||||||
1467 | else if (!Child->getPredicateCalls().empty()) | ||||||
1468 | ++Size; | ||||||
1469 | } | ||||||
1470 | } | ||||||
1471 | |||||||
1472 | return Size; | ||||||
1473 | } | ||||||
1474 | |||||||
1475 | /// Compute the complexity metric for the input pattern. This roughly | ||||||
1476 | /// corresponds to the number of nodes that are covered. | ||||||
1477 | int PatternToMatch:: | ||||||
1478 | getPatternComplexity(const CodeGenDAGPatterns &CGP) const { | ||||||
1479 | return getPatternSize(getSrcPattern(), CGP) + getAddedComplexity(); | ||||||
1480 | } | ||||||
1481 | |||||||
1482 | void PatternToMatch::getPredicateRecords( | ||||||
1483 | SmallVectorImpl<Record *> &PredicateRecs) const { | ||||||
1484 | for (Init *I : Predicates->getValues()) { | ||||||
1485 | if (DefInit *Pred = dyn_cast<DefInit>(I)) { | ||||||
1486 | Record *Def = Pred->getDef(); | ||||||
1487 | if (!Def->isSubClassOf("Predicate")) { | ||||||
1488 | #ifndef NDEBUG | ||||||
1489 | Def->dump(); | ||||||
1490 | #endif | ||||||
1491 | llvm_unreachable("Unknown predicate type!")::llvm::llvm_unreachable_internal("Unknown predicate type!", "llvm/utils/TableGen/CodeGenDAGPatterns.cpp" , 1491); | ||||||
1492 | } | ||||||
1493 | PredicateRecs.push_back(Def); | ||||||
1494 | } | ||||||
1495 | } | ||||||
1496 | // Sort so that different orders get canonicalized to the same string. | ||||||
1497 | llvm::sort(PredicateRecs, LessRecord()); | ||||||
1498 | } | ||||||
1499 | |||||||
1500 | /// getPredicateCheck - Return a single string containing all of this | ||||||
1501 | /// pattern's predicates concatenated with "&&" operators. | ||||||
1502 | /// | ||||||
1503 | std::string PatternToMatch::getPredicateCheck() const { | ||||||
1504 | SmallVector<Record *, 4> PredicateRecs; | ||||||
1505 | getPredicateRecords(PredicateRecs); | ||||||
1506 | |||||||
1507 | SmallString<128> PredicateCheck; | ||||||
1508 | raw_svector_ostream OS(PredicateCheck); | ||||||
1509 | ListSeparator LS(" && "); | ||||||
1510 | for (Record *Pred : PredicateRecs) { | ||||||
1511 | StringRef CondString = Pred->getValueAsString("CondString"); | ||||||
1512 | if (CondString.empty()) | ||||||
1513 | continue; | ||||||
1514 | OS << LS << '(' << CondString << ')'; | ||||||
1515 | } | ||||||
1516 | |||||||
1517 | if (!HwModeFeatures.empty()) | ||||||
1518 | OS << LS << HwModeFeatures; | ||||||
1519 | |||||||
1520 | return std::string(PredicateCheck); | ||||||
1521 | } | ||||||
1522 | |||||||
1523 | //===----------------------------------------------------------------------===// | ||||||
1524 | // SDTypeConstraint implementation | ||||||
1525 | // | ||||||
1526 | |||||||
1527 | SDTypeConstraint::SDTypeConstraint(Record *R, const CodeGenHwModes &CGH) { | ||||||
1528 | OperandNo = R->getValueAsInt("OperandNum"); | ||||||
1529 | |||||||
1530 | if (R->isSubClassOf("SDTCisVT")) { | ||||||
1531 | ConstraintType = SDTCisVT; | ||||||
1532 | VVT = getValueTypeByHwMode(R->getValueAsDef("VT"), CGH); | ||||||
1533 | for (const auto &P : VVT) | ||||||
1534 | if (P.second == MVT::isVoid) | ||||||
1535 | PrintFatalError(R->getLoc(), "Cannot use 'Void' as type to SDTCisVT"); | ||||||
1536 | } else if (R->isSubClassOf("SDTCisPtrTy")) { | ||||||
1537 | ConstraintType = SDTCisPtrTy; | ||||||
1538 | } else if (R->isSubClassOf("SDTCisInt")) { | ||||||
1539 | ConstraintType = SDTCisInt; | ||||||
1540 | } else if (R->isSubClassOf("SDTCisFP")) { | ||||||
1541 | ConstraintType = SDTCisFP; | ||||||
1542 | } else if (R->isSubClassOf("SDTCisVec")) { | ||||||
1543 | ConstraintType = SDTCisVec; | ||||||
1544 | } else if (R->isSubClassOf("SDTCisSameAs")) { | ||||||
1545 | ConstraintType = SDTCisSameAs; | ||||||
1546 | x.SDTCisSameAs_Info.OtherOperandNum = R->getValueAsInt("OtherOperandNum"); | ||||||
1547 | } else if (R->isSubClassOf("SDTCisVTSmallerThanOp")) { | ||||||
1548 | ConstraintType = SDTCisVTSmallerThanOp; | ||||||
1549 | x.SDTCisVTSmallerThanOp_Info.OtherOperandNum = | ||||||
1550 | R->getValueAsInt("OtherOperandNum"); | ||||||
1551 | } else if (R->isSubClassOf("SDTCisOpSmallerThanOp")) { | ||||||
1552 | ConstraintType = SDTCisOpSmallerThanOp; | ||||||
1553 | x.SDTCisOpSmallerThanOp_Info.BigOperandNum = | ||||||
1554 | R->getValueAsInt("BigOperandNum"); | ||||||
1555 | } else if (R->isSubClassOf("SDTCisEltOfVec")) { | ||||||
1556 | ConstraintType = SDTCisEltOfVec; | ||||||
1557 | x.SDTCisEltOfVec_Info.OtherOperandNum = R->getValueAsInt("OtherOpNum"); | ||||||
1558 | } else if (R->isSubClassOf("SDTCisSubVecOfVec")) { | ||||||
1559 | ConstraintType = SDTCisSubVecOfVec; | ||||||
1560 | x.SDTCisSubVecOfVec_Info.OtherOperandNum = | ||||||
1561 | R->getValueAsInt("OtherOpNum"); | ||||||
1562 | } else if (R->isSubClassOf("SDTCVecEltisVT")) { | ||||||
1563 | ConstraintType = SDTCVecEltisVT; | ||||||
1564 | VVT = getValueTypeByHwMode(R->getValueAsDef("VT"), CGH); | ||||||
1565 | for (const auto &P : VVT) { | ||||||
1566 | MVT T = P.second; | ||||||
1567 | if (T.isVector()) | ||||||
1568 | PrintFatalError(R->getLoc(), | ||||||
1569 | "Cannot use vector type as SDTCVecEltisVT"); | ||||||
1570 | if (!T.isInteger() && !T.isFloatingPoint()) | ||||||
1571 | PrintFatalError(R->getLoc(), "Must use integer or floating point type " | ||||||
1572 | "as SDTCVecEltisVT"); | ||||||
1573 | } | ||||||
1574 | } else if (R->isSubClassOf("SDTCisSameNumEltsAs")) { | ||||||
1575 | ConstraintType = SDTCisSameNumEltsAs; | ||||||
1576 | x.SDTCisSameNumEltsAs_Info.OtherOperandNum = | ||||||
1577 | R->getValueAsInt("OtherOperandNum"); | ||||||
1578 | } else if (R->isSubClassOf("SDTCisSameSizeAs")) { | ||||||
1579 | ConstraintType = SDTCisSameSizeAs; | ||||||
1580 | x.SDTCisSameSizeAs_Info.OtherOperandNum = | ||||||
1581 | R->getValueAsInt("OtherOperandNum"); | ||||||
1582 | } else { | ||||||
1583 | PrintFatalError(R->getLoc(), | ||||||
1584 | "Unrecognized SDTypeConstraint '" + R->getName() + "'!\n"); | ||||||
1585 | } | ||||||
1586 | } | ||||||
1587 | |||||||
1588 | /// getOperandNum - Return the node corresponding to operand #OpNo in tree | ||||||
1589 | /// N, and the result number in ResNo. | ||||||
1590 | static TreePatternNode *getOperandNum(unsigned OpNo, TreePatternNode *N, | ||||||
1591 | const SDNodeInfo &NodeInfo, | ||||||
1592 | unsigned &ResNo) { | ||||||
1593 | unsigned NumResults = NodeInfo.getNumResults(); | ||||||
1594 | if (OpNo < NumResults) { | ||||||
1595 | ResNo = OpNo; | ||||||
1596 | return N; | ||||||
1597 | } | ||||||
1598 | |||||||
1599 | OpNo -= NumResults; | ||||||
1600 | |||||||
1601 | if (OpNo >= N->getNumChildren()) { | ||||||
1602 | std::string S; | ||||||
1603 | raw_string_ostream OS(S); | ||||||
1604 | OS << "Invalid operand number in type constraint " | ||||||
1605 | << (OpNo+NumResults) << " "; | ||||||
1606 | N->print(OS); | ||||||
1607 | PrintFatalError(S); | ||||||
1608 | } | ||||||
1609 | |||||||
1610 | return N->getChild(OpNo); | ||||||
1611 | } | ||||||
1612 | |||||||
1613 | /// ApplyTypeConstraint - Given a node in a pattern, apply this type | ||||||
1614 | /// constraint to the nodes operands. This returns true if it makes a | ||||||
1615 | /// change, false otherwise. If a type contradiction is found, flag an error. | ||||||
1616 | bool SDTypeConstraint::ApplyTypeConstraint(TreePatternNode *N, | ||||||
1617 | const SDNodeInfo &NodeInfo, | ||||||
1618 | TreePattern &TP) const { | ||||||
1619 | if (TP.hasError()) | ||||||
1620 | return false; | ||||||
1621 | |||||||
1622 | unsigned ResNo = 0; // The result number being referenced. | ||||||
1623 | TreePatternNode *NodeToApply = getOperandNum(OperandNo, N, NodeInfo, ResNo); | ||||||
1624 | TypeInfer &TI = TP.getInfer(); | ||||||
1625 | |||||||
1626 | switch (ConstraintType) { | ||||||
1627 | case SDTCisVT: | ||||||
1628 | // Operand must be a particular type. | ||||||
1629 | return NodeToApply->UpdateNodeType(ResNo, VVT, TP); | ||||||
1630 | case SDTCisPtrTy: | ||||||
1631 | // Operand must be same as target pointer type. | ||||||
1632 | return NodeToApply->UpdateNodeType(ResNo, MVT::iPTR, TP); | ||||||
1633 | case SDTCisInt: | ||||||
1634 | // Require it to be one of the legal integer VTs. | ||||||
1635 | return TI.EnforceInteger(NodeToApply->getExtType(ResNo)); | ||||||
1636 | case SDTCisFP: | ||||||
1637 | // Require it to be one of the legal fp VTs. | ||||||
1638 | return TI.EnforceFloatingPoint(NodeToApply->getExtType(ResNo)); | ||||||
1639 | case SDTCisVec: | ||||||
1640 | // Require it to be one of the legal vector VTs. | ||||||
1641 | return TI.EnforceVector(NodeToApply->getExtType(ResNo)); | ||||||
1642 | case SDTCisSameAs: { | ||||||
1643 | unsigned OResNo = 0; | ||||||
1644 | TreePatternNode *OtherNode = | ||||||
1645 | getOperandNum(x.SDTCisSameAs_Info.OtherOperandNum, N, NodeInfo, OResNo); | ||||||
1646 | return (int)NodeToApply->UpdateNodeType(ResNo, | ||||||
1647 | OtherNode->getExtType(OResNo), TP) | | ||||||
1648 | (int)OtherNode->UpdateNodeType(OResNo, | ||||||
1649 | NodeToApply->getExtType(ResNo), TP); | ||||||
1650 | } | ||||||
1651 | case SDTCisVTSmallerThanOp: { | ||||||
1652 | // The NodeToApply must be a leaf node that is a VT. OtherOperandNum must | ||||||
1653 | // have an integer type that is smaller than the VT. | ||||||
1654 | if (!NodeToApply->isLeaf() || | ||||||
1655 | !isa<DefInit>(NodeToApply->getLeafValue()) || | ||||||
1656 | !cast<DefInit>(NodeToApply->getLeafValue())->getDef() | ||||||
1657 | ->isSubClassOf("ValueType")) { | ||||||
1658 | TP.error(N->getOperator()->getName() + " expects a VT operand!"); | ||||||
1659 | return false; | ||||||
1660 | } | ||||||
1661 | DefInit *DI = cast<DefInit>(NodeToApply->getLeafValue()); | ||||||
1662 | const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); | ||||||
1663 | auto VVT = getValueTypeByHwMode(DI->getDef(), T.getHwModes()); | ||||||
1664 | TypeSetByHwMode TypeListTmp(VVT); | ||||||
1665 | |||||||
1666 | unsigned OResNo = 0; | ||||||
1667 | TreePatternNode *OtherNode = | ||||||
1668 | getOperandNum(x.SDTCisVTSmallerThanOp_Info.OtherOperandNum, N, NodeInfo, | ||||||
1669 | OResNo); | ||||||
1670 | |||||||
1671 | return TI.EnforceSmallerThan(TypeListTmp, OtherNode->getExtType(OResNo), | ||||||
1672 | /*SmallIsVT*/ true); | ||||||
1673 | } | ||||||
1674 | case SDTCisOpSmallerThanOp: { | ||||||
1675 | unsigned BResNo = 0; | ||||||
1676 | TreePatternNode *BigOperand = | ||||||
1677 | getOperandNum(x.SDTCisOpSmallerThanOp_Info.BigOperandNum, N, NodeInfo, | ||||||
1678 | BResNo); | ||||||
1679 | return TI.EnforceSmallerThan(NodeToApply->getExtType(ResNo), | ||||||
1680 | BigOperand->getExtType(BResNo)); | ||||||
1681 | } | ||||||
1682 | case SDTCisEltOfVec: { | ||||||
1683 | unsigned VResNo = 0; | ||||||
1684 | TreePatternNode *VecOperand = | ||||||
1685 | getOperandNum(x.SDTCisEltOfVec_Info.OtherOperandNum, N, NodeInfo, | ||||||
1686 | VResNo); | ||||||
1687 | // Filter vector types out of VecOperand that don't have the right element | ||||||
1688 | // type. | ||||||
1689 | return TI.EnforceVectorEltTypeIs(VecOperand->getExtType(VResNo), | ||||||
1690 | NodeToApply->getExtType(ResNo)); | ||||||
1691 | } | ||||||
1692 | case SDTCisSubVecOfVec: { | ||||||
1693 | unsigned VResNo = 0; | ||||||
1694 | TreePatternNode *BigVecOperand = | ||||||
1695 | getOperandNum(x.SDTCisSubVecOfVec_Info.OtherOperandNum, N, NodeInfo, | ||||||
1696 | VResNo); | ||||||
1697 | |||||||
1698 | // Filter vector types out of BigVecOperand that don't have the | ||||||
1699 | // right subvector type. | ||||||
1700 | return TI.EnforceVectorSubVectorTypeIs(BigVecOperand->getExtType(VResNo), | ||||||
1701 | NodeToApply->getExtType(ResNo)); | ||||||
1702 | } | ||||||
1703 | case SDTCVecEltisVT: { | ||||||
1704 | return TI.EnforceVectorEltTypeIs(NodeToApply->getExtType(ResNo), VVT); | ||||||
1705 | } | ||||||
1706 | case SDTCisSameNumEltsAs: { | ||||||
1707 | unsigned OResNo = 0; | ||||||
1708 | TreePatternNode *OtherNode = | ||||||
1709 | getOperandNum(x.SDTCisSameNumEltsAs_Info.OtherOperandNum, | ||||||
1710 | N, NodeInfo, OResNo); | ||||||
1711 | return TI.EnforceSameNumElts(OtherNode->getExtType(OResNo), | ||||||
1712 | NodeToApply->getExtType(ResNo)); | ||||||
1713 | } | ||||||
1714 | case SDTCisSameSizeAs: { | ||||||
1715 | unsigned OResNo = 0; | ||||||
1716 | TreePatternNode *OtherNode = | ||||||
1717 | getOperandNum(x.SDTCisSameSizeAs_Info.OtherOperandNum, | ||||||
1718 | N, NodeInfo, OResNo); | ||||||
1719 | return TI.EnforceSameSize(OtherNode->getExtType(OResNo), | ||||||
1720 | NodeToApply->getExtType(ResNo)); | ||||||
1721 | } | ||||||
1722 | } | ||||||
1723 | llvm_unreachable("Invalid ConstraintType!")::llvm::llvm_unreachable_internal("Invalid ConstraintType!", "llvm/utils/TableGen/CodeGenDAGPatterns.cpp" , 1723); | ||||||
1724 | } | ||||||
1725 | |||||||
1726 | // Update the node type to match an instruction operand or result as specified | ||||||
1727 | // in the ins or outs lists on the instruction definition. Return true if the | ||||||
1728 | // type was actually changed. | ||||||
1729 | bool TreePatternNode::UpdateNodeTypeFromInst(unsigned ResNo, | ||||||
1730 | Record *Operand, | ||||||
1731 | TreePattern &TP) { | ||||||
1732 | // The 'unknown' operand indicates that types should be inferred from the | ||||||
1733 | // context. | ||||||
1734 | if (Operand->isSubClassOf("unknown_class")) | ||||||
1735 | return false; | ||||||
1736 | |||||||
1737 | // The Operand class specifies a type directly. | ||||||
1738 | if (Operand->isSubClassOf("Operand")) { | ||||||
1739 | Record *R = Operand->getValueAsDef("Type"); | ||||||
1740 | const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); | ||||||
1741 | return UpdateNodeType(ResNo, getValueTypeByHwMode(R, T.getHwModes()), TP); | ||||||
1742 | } | ||||||
1743 | |||||||
1744 | // PointerLikeRegClass has a type that is determined at runtime. | ||||||
1745 | if (Operand->isSubClassOf("PointerLikeRegClass")) | ||||||
1746 | return UpdateNodeType(ResNo, MVT::iPTR, TP); | ||||||
1747 | |||||||
1748 | // Both RegisterClass and RegisterOperand operands derive their types from a | ||||||
1749 | // register class def. | ||||||
1750 | Record *RC = nullptr; | ||||||
1751 | if (Operand->isSubClassOf("RegisterClass")) | ||||||
1752 | RC = Operand; | ||||||
1753 | else if (Operand->isSubClassOf("RegisterOperand")) | ||||||
1754 | RC = Operand->getValueAsDef("RegClass"); | ||||||
1755 | |||||||
1756 | assert(RC && "Unknown operand type")(static_cast <bool> (RC && "Unknown operand type" ) ? void (0) : __assert_fail ("RC && \"Unknown operand type\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 1756, __extension__ __PRETTY_FUNCTION__)); | ||||||
1757 | CodeGenTarget &Tgt = TP.getDAGPatterns().getTargetInfo(); | ||||||
1758 | return UpdateNodeType(ResNo, Tgt.getRegisterClass(RC).getValueTypes(), TP); | ||||||
1759 | } | ||||||
1760 | |||||||
1761 | bool TreePatternNode::ContainsUnresolvedType(TreePattern &TP) const { | ||||||
1762 | for (unsigned i = 0, e = Types.size(); i != e; ++i) | ||||||
1763 | if (!TP.getInfer().isConcrete(Types[i], true)) | ||||||
1764 | return true; | ||||||
1765 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) | ||||||
1766 | if (getChild(i)->ContainsUnresolvedType(TP)) | ||||||
1767 | return true; | ||||||
1768 | return false; | ||||||
1769 | } | ||||||
1770 | |||||||
1771 | bool TreePatternNode::hasProperTypeByHwMode() const { | ||||||
1772 | for (const TypeSetByHwMode &S : Types) | ||||||
1773 | if (!S.isSimple()) | ||||||
1774 | return true; | ||||||
1775 | for (const TreePatternNodePtr &C : Children) | ||||||
1776 | if (C->hasProperTypeByHwMode()) | ||||||
1777 | return true; | ||||||
1778 | return false; | ||||||
1779 | } | ||||||
1780 | |||||||
1781 | bool TreePatternNode::hasPossibleType() const { | ||||||
1782 | for (const TypeSetByHwMode &S : Types) | ||||||
1783 | if (!S.isPossible()) | ||||||
1784 | return false; | ||||||
1785 | for (const TreePatternNodePtr &C : Children) | ||||||
1786 | if (!C->hasPossibleType()) | ||||||
1787 | return false; | ||||||
1788 | return true; | ||||||
1789 | } | ||||||
1790 | |||||||
1791 | bool TreePatternNode::setDefaultMode(unsigned Mode) { | ||||||
1792 | for (TypeSetByHwMode &S : Types) { | ||||||
1793 | S.makeSimple(Mode); | ||||||
1794 | // Check if the selected mode had a type conflict. | ||||||
1795 | if (S.get(DefaultMode).empty()) | ||||||
1796 | return false; | ||||||
1797 | } | ||||||
1798 | for (const TreePatternNodePtr &C : Children) | ||||||
1799 | if (!C->setDefaultMode(Mode)) | ||||||
1800 | return false; | ||||||
1801 | return true; | ||||||
1802 | } | ||||||
1803 | |||||||
1804 | //===----------------------------------------------------------------------===// | ||||||
1805 | // SDNodeInfo implementation | ||||||
1806 | // | ||||||
1807 | SDNodeInfo::SDNodeInfo(Record *R, const CodeGenHwModes &CGH) : Def(R) { | ||||||
1808 | EnumName = R->getValueAsString("Opcode"); | ||||||
1809 | SDClassName = R->getValueAsString("SDClass"); | ||||||
1810 | Record *TypeProfile = R->getValueAsDef("TypeProfile"); | ||||||
1811 | NumResults = TypeProfile->getValueAsInt("NumResults"); | ||||||
1812 | NumOperands = TypeProfile->getValueAsInt("NumOperands"); | ||||||
1813 | |||||||
1814 | // Parse the properties. | ||||||
1815 | Properties = parseSDPatternOperatorProperties(R); | ||||||
1816 | |||||||
1817 | // Parse the type constraints. | ||||||
1818 | std::vector<Record*> ConstraintList = | ||||||
1819 | TypeProfile->getValueAsListOfDefs("Constraints"); | ||||||
1820 | for (Record *R : ConstraintList) | ||||||
1821 | TypeConstraints.emplace_back(R, CGH); | ||||||
1822 | } | ||||||
1823 | |||||||
1824 | /// getKnownType - If the type constraints on this node imply a fixed type | ||||||
1825 | /// (e.g. all stores return void, etc), then return it as an | ||||||
1826 | /// MVT::SimpleValueType. Otherwise, return EEVT::Other. | ||||||
1827 | MVT::SimpleValueType SDNodeInfo::getKnownType(unsigned ResNo) const { | ||||||
1828 | unsigned NumResults = getNumResults(); | ||||||
1829 | assert(NumResults <= 1 &&(static_cast <bool> (NumResults <= 1 && "We only work with nodes with zero or one result so far!" ) ? void (0) : __assert_fail ("NumResults <= 1 && \"We only work with nodes with zero or one result so far!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 1830, __extension__ __PRETTY_FUNCTION__)) | ||||||
1830 | "We only work with nodes with zero or one result so far!")(static_cast <bool> (NumResults <= 1 && "We only work with nodes with zero or one result so far!" ) ? void (0) : __assert_fail ("NumResults <= 1 && \"We only work with nodes with zero or one result so far!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 1830, __extension__ __PRETTY_FUNCTION__)); | ||||||
1831 | assert(ResNo == 0 && "Only handles single result nodes so far")(static_cast <bool> (ResNo == 0 && "Only handles single result nodes so far" ) ? void (0) : __assert_fail ("ResNo == 0 && \"Only handles single result nodes so far\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 1831, __extension__ __PRETTY_FUNCTION__)); | ||||||
1832 | |||||||
1833 | for (const SDTypeConstraint &Constraint : TypeConstraints) { | ||||||
1834 | // Make sure that this applies to the correct node result. | ||||||
1835 | if (Constraint.OperandNo >= NumResults) // FIXME: need value # | ||||||
1836 | continue; | ||||||
1837 | |||||||
1838 | switch (Constraint.ConstraintType) { | ||||||
1839 | default: break; | ||||||
1840 | case SDTypeConstraint::SDTCisVT: | ||||||
1841 | if (Constraint.VVT.isSimple()) | ||||||
1842 | return Constraint.VVT.getSimple().SimpleTy; | ||||||
1843 | break; | ||||||
1844 | case SDTypeConstraint::SDTCisPtrTy: | ||||||
1845 | return MVT::iPTR; | ||||||
1846 | } | ||||||
1847 | } | ||||||
1848 | return MVT::Other; | ||||||
1849 | } | ||||||
1850 | |||||||
1851 | //===----------------------------------------------------------------------===// | ||||||
1852 | // TreePatternNode implementation | ||||||
1853 | // | ||||||
1854 | |||||||
1855 | static unsigned GetNumNodeResults(Record *Operator, CodeGenDAGPatterns &CDP) { | ||||||
1856 | if (Operator->getName() == "set" || | ||||||
1857 | Operator->getName() == "implicit") | ||||||
1858 | return 0; // All return nothing. | ||||||
1859 | |||||||
1860 | if (Operator->isSubClassOf("Intrinsic")) | ||||||
1861 | return CDP.getIntrinsic(Operator).IS.RetVTs.size(); | ||||||
1862 | |||||||
1863 | if (Operator->isSubClassOf("SDNode")) | ||||||
1864 | return CDP.getSDNodeInfo(Operator).getNumResults(); | ||||||
1865 | |||||||
1866 | if (Operator->isSubClassOf("PatFrags")) { | ||||||
1867 | // If we've already parsed this pattern fragment, get it. Otherwise, handle | ||||||
1868 | // the forward reference case where one pattern fragment references another | ||||||
1869 | // before it is processed. | ||||||
1870 | if (TreePattern *PFRec = CDP.getPatternFragmentIfRead(Operator)) { | ||||||
1871 | // The number of results of a fragment with alternative records is the | ||||||
1872 | // maximum number of results across all alternatives. | ||||||
1873 | unsigned NumResults = 0; | ||||||
1874 | for (const auto &T : PFRec->getTrees()) | ||||||
1875 | NumResults = std::max(NumResults, T->getNumTypes()); | ||||||
1876 | return NumResults; | ||||||
1877 | } | ||||||
1878 | |||||||
1879 | ListInit *LI = Operator->getValueAsListInit("Fragments"); | ||||||
1880 | assert(LI && "Invalid Fragment")(static_cast <bool> (LI && "Invalid Fragment") ? void (0) : __assert_fail ("LI && \"Invalid Fragment\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 1880, __extension__ __PRETTY_FUNCTION__)); | ||||||
1881 | unsigned NumResults = 0; | ||||||
1882 | for (Init *I : LI->getValues()) { | ||||||
1883 | Record *Op = nullptr; | ||||||
1884 | if (DagInit *Dag = dyn_cast<DagInit>(I)) | ||||||
1885 | if (DefInit *DI = dyn_cast<DefInit>(Dag->getOperator())) | ||||||
1886 | Op = DI->getDef(); | ||||||
1887 | assert(Op && "Invalid Fragment")(static_cast <bool> (Op && "Invalid Fragment") ? void (0) : __assert_fail ("Op && \"Invalid Fragment\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 1887, __extension__ __PRETTY_FUNCTION__)); | ||||||
1888 | NumResults = std::max(NumResults, GetNumNodeResults(Op, CDP)); | ||||||
1889 | } | ||||||
1890 | return NumResults; | ||||||
1891 | } | ||||||
1892 | |||||||
1893 | if (Operator->isSubClassOf("Instruction")) { | ||||||
1894 | CodeGenInstruction &InstInfo = CDP.getTargetInfo().getInstruction(Operator); | ||||||
1895 | |||||||
1896 | unsigned NumDefsToAdd = InstInfo.Operands.NumDefs; | ||||||
1897 | |||||||
1898 | // Subtract any defaulted outputs. | ||||||
1899 | for (unsigned i = 0; i != InstInfo.Operands.NumDefs; ++i) { | ||||||
1900 | Record *OperandNode = InstInfo.Operands[i].Rec; | ||||||
1901 | |||||||
1902 | if (OperandNode->isSubClassOf("OperandWithDefaultOps") && | ||||||
1903 | !CDP.getDefaultOperand(OperandNode).DefaultOps.empty()) | ||||||
1904 | --NumDefsToAdd; | ||||||
1905 | } | ||||||
1906 | |||||||
1907 | // Add on one implicit def if it has a resolvable type. | ||||||
1908 | if (InstInfo.HasOneImplicitDefWithKnownVT(CDP.getTargetInfo()) !=MVT::Other) | ||||||
1909 | ++NumDefsToAdd; | ||||||
1910 | return NumDefsToAdd; | ||||||
1911 | } | ||||||
1912 | |||||||
1913 | if (Operator->isSubClassOf("SDNodeXForm")) | ||||||
1914 | return 1; // FIXME: Generalize SDNodeXForm | ||||||
1915 | |||||||
1916 | if (Operator->isSubClassOf("ValueType")) | ||||||
1917 | return 1; // A type-cast of one result. | ||||||
1918 | |||||||
1919 | if (Operator->isSubClassOf("ComplexPattern")) | ||||||
1920 | return 1; | ||||||
1921 | |||||||
1922 | errs() << *Operator; | ||||||
1923 | PrintFatalError("Unhandled node in GetNumNodeResults"); | ||||||
1924 | } | ||||||
1925 | |||||||
1926 | void TreePatternNode::print(raw_ostream &OS) const { | ||||||
1927 | if (isLeaf()) | ||||||
1928 | OS << *getLeafValue(); | ||||||
1929 | else | ||||||
1930 | OS << '(' << getOperator()->getName(); | ||||||
1931 | |||||||
1932 | for (unsigned i = 0, e = Types.size(); i != e; ++i) { | ||||||
1933 | OS << ':'; | ||||||
1934 | getExtType(i).writeToStream(OS); | ||||||
1935 | } | ||||||
1936 | |||||||
1937 | if (!isLeaf()) { | ||||||
1938 | if (getNumChildren() != 0) { | ||||||
1939 | OS << " "; | ||||||
1940 | ListSeparator LS; | ||||||
1941 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) { | ||||||
1942 | OS << LS; | ||||||
1943 | getChild(i)->print(OS); | ||||||
1944 | } | ||||||
1945 | } | ||||||
1946 | OS << ")"; | ||||||
1947 | } | ||||||
1948 | |||||||
1949 | for (const TreePredicateCall &Pred : PredicateCalls) { | ||||||
1950 | OS << "<<P:"; | ||||||
1951 | if (Pred.Scope) | ||||||
1952 | OS << Pred.Scope << ":"; | ||||||
1953 | OS << Pred.Fn.getFnName() << ">>"; | ||||||
1954 | } | ||||||
1955 | if (TransformFn) | ||||||
1956 | OS << "<<X:" << TransformFn->getName() << ">>"; | ||||||
1957 | if (!getName().empty()) | ||||||
1958 | OS << ":$" << getName(); | ||||||
1959 | |||||||
1960 | for (const ScopedName &Name : NamesAsPredicateArg) | ||||||
1961 | OS << ":$pred:" << Name.getScope() << ":" << Name.getIdentifier(); | ||||||
1962 | } | ||||||
1963 | void TreePatternNode::dump() const { | ||||||
1964 | print(errs()); | ||||||
1965 | } | ||||||
1966 | |||||||
1967 | /// isIsomorphicTo - Return true if this node is recursively | ||||||
1968 | /// isomorphic to the specified node. For this comparison, the node's | ||||||
1969 | /// entire state is considered. The assigned name is ignored, since | ||||||
1970 | /// nodes with differing names are considered isomorphic. However, if | ||||||
1971 | /// the assigned name is present in the dependent variable set, then | ||||||
1972 | /// the assigned name is considered significant and the node is | ||||||
1973 | /// isomorphic if the names match. | ||||||
1974 | bool TreePatternNode::isIsomorphicTo(const TreePatternNode *N, | ||||||
1975 | const MultipleUseVarSet &DepVars) const { | ||||||
1976 | if (N == this) return true; | ||||||
1977 | if (N->isLeaf() != isLeaf()) | ||||||
1978 | return false; | ||||||
1979 | |||||||
1980 | // Check operator of non-leaves early since it can be cheaper than checking | ||||||
1981 | // types. | ||||||
1982 | if (!isLeaf()) | ||||||
1983 | if (N->getOperator() != getOperator() || | ||||||
1984 | N->getNumChildren() != getNumChildren()) | ||||||
1985 | return false; | ||||||
1986 | |||||||
1987 | if (getExtTypes() != N->getExtTypes() || | ||||||
1988 | getPredicateCalls() != N->getPredicateCalls() || | ||||||
1989 | getTransformFn() != N->getTransformFn()) | ||||||
1990 | return false; | ||||||
1991 | |||||||
1992 | if (isLeaf()) { | ||||||
1993 | if (DefInit *DI = dyn_cast<DefInit>(getLeafValue())) { | ||||||
1994 | if (DefInit *NDI = dyn_cast<DefInit>(N->getLeafValue())) { | ||||||
1995 | return ((DI->getDef() == NDI->getDef()) && | ||||||
1996 | (!DepVars.contains(getName()) || getName() == N->getName())); | ||||||
1997 | } | ||||||
1998 | } | ||||||
1999 | return getLeafValue() == N->getLeafValue(); | ||||||
2000 | } | ||||||
2001 | |||||||
2002 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) | ||||||
2003 | if (!getChild(i)->isIsomorphicTo(N->getChild(i), DepVars)) | ||||||
2004 | return false; | ||||||
2005 | return true; | ||||||
2006 | } | ||||||
2007 | |||||||
2008 | /// clone - Make a copy of this tree and all of its children. | ||||||
2009 | /// | ||||||
2010 | TreePatternNodePtr TreePatternNode::clone() const { | ||||||
2011 | TreePatternNodePtr New; | ||||||
2012 | if (isLeaf()) { | ||||||
2013 | New = std::make_shared<TreePatternNode>(getLeafValue(), getNumTypes()); | ||||||
2014 | } else { | ||||||
2015 | std::vector<TreePatternNodePtr> CChildren; | ||||||
2016 | CChildren.reserve(Children.size()); | ||||||
2017 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) | ||||||
2018 | CChildren.push_back(getChild(i)->clone()); | ||||||
2019 | New = std::make_shared<TreePatternNode>(getOperator(), std::move(CChildren), | ||||||
2020 | getNumTypes()); | ||||||
2021 | } | ||||||
2022 | New->setName(getName()); | ||||||
2023 | New->setNamesAsPredicateArg(getNamesAsPredicateArg()); | ||||||
2024 | New->Types = Types; | ||||||
2025 | New->setPredicateCalls(getPredicateCalls()); | ||||||
2026 | New->setGISelFlagsRecord(getGISelFlagsRecord()); | ||||||
2027 | New->setTransformFn(getTransformFn()); | ||||||
2028 | return New; | ||||||
2029 | } | ||||||
2030 | |||||||
2031 | /// RemoveAllTypes - Recursively strip all the types of this tree. | ||||||
2032 | void TreePatternNode::RemoveAllTypes() { | ||||||
2033 | // Reset to unknown type. | ||||||
2034 | std::fill(Types.begin(), Types.end(), TypeSetByHwMode()); | ||||||
2035 | if (isLeaf()) return; | ||||||
2036 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) | ||||||
2037 | getChild(i)->RemoveAllTypes(); | ||||||
2038 | } | ||||||
2039 | |||||||
2040 | |||||||
2041 | /// SubstituteFormalArguments - Replace the formal arguments in this tree | ||||||
2042 | /// with actual values specified by ArgMap. | ||||||
2043 | void TreePatternNode::SubstituteFormalArguments( | ||||||
2044 | std::map<std::string, TreePatternNodePtr> &ArgMap) { | ||||||
2045 | if (isLeaf()) return; | ||||||
2046 | |||||||
2047 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) { | ||||||
2048 | TreePatternNode *Child = getChild(i); | ||||||
2049 | if (Child->isLeaf()) { | ||||||
2050 | Init *Val = Child->getLeafValue(); | ||||||
2051 | // Note that, when substituting into an output pattern, Val might be an | ||||||
2052 | // UnsetInit. | ||||||
2053 | if (isa<UnsetInit>(Val) || (isa<DefInit>(Val) && | ||||||
2054 | cast<DefInit>(Val)->getDef()->getName() == "node")) { | ||||||
2055 | // We found a use of a formal argument, replace it with its value. | ||||||
2056 | TreePatternNodePtr NewChild = ArgMap[Child->getName()]; | ||||||
2057 | assert(NewChild && "Couldn't find formal argument!")(static_cast <bool> (NewChild && "Couldn't find formal argument!" ) ? void (0) : __assert_fail ("NewChild && \"Couldn't find formal argument!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2057, __extension__ __PRETTY_FUNCTION__)); | ||||||
2058 | assert((Child->getPredicateCalls().empty() ||(static_cast <bool> ((Child->getPredicateCalls().empty () || NewChild->getPredicateCalls() == Child->getPredicateCalls ()) && "Non-empty child predicate clobbered!") ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || NewChild->getPredicateCalls() == Child->getPredicateCalls()) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2060, __extension__ __PRETTY_FUNCTION__)) | ||||||
2059 | NewChild->getPredicateCalls() == Child->getPredicateCalls()) &&(static_cast <bool> ((Child->getPredicateCalls().empty () || NewChild->getPredicateCalls() == Child->getPredicateCalls ()) && "Non-empty child predicate clobbered!") ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || NewChild->getPredicateCalls() == Child->getPredicateCalls()) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2060, __extension__ __PRETTY_FUNCTION__)) | ||||||
2060 | "Non-empty child predicate clobbered!")(static_cast <bool> ((Child->getPredicateCalls().empty () || NewChild->getPredicateCalls() == Child->getPredicateCalls ()) && "Non-empty child predicate clobbered!") ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || NewChild->getPredicateCalls() == Child->getPredicateCalls()) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2060, __extension__ __PRETTY_FUNCTION__)); | ||||||
2061 | setChild(i, std::move(NewChild)); | ||||||
2062 | } | ||||||
2063 | } else { | ||||||
2064 | getChild(i)->SubstituteFormalArguments(ArgMap); | ||||||
2065 | } | ||||||
2066 | } | ||||||
2067 | } | ||||||
2068 | |||||||
2069 | |||||||
2070 | /// InlinePatternFragments - If this pattern refers to any pattern | ||||||
2071 | /// fragments, return the set of inlined versions (this can be more than | ||||||
2072 | /// one if a PatFrags record has multiple alternatives). | ||||||
2073 | void TreePatternNode::InlinePatternFragments( | ||||||
2074 | const TreePatternNodePtr &T, TreePattern &TP, | ||||||
2075 | std::vector<TreePatternNodePtr> &OutAlternatives) { | ||||||
2076 | |||||||
2077 | if (TP.hasError()) | ||||||
2078 | return; | ||||||
2079 | |||||||
2080 | if (T->isLeaf()) { | ||||||
2081 | OutAlternatives.push_back(T); // nothing to do. | ||||||
2082 | return; | ||||||
2083 | } | ||||||
2084 | |||||||
2085 | Record *Op = T->getOperator(); | ||||||
2086 | |||||||
2087 | if (!Op->isSubClassOf("PatFrags")) { | ||||||
2088 | if (T->getNumChildren() == 0) { | ||||||
2089 | OutAlternatives.push_back(T); | ||||||
2090 | return; | ||||||
2091 | } | ||||||
2092 | |||||||
2093 | // Recursively inline children nodes. | ||||||
2094 | std::vector<std::vector<TreePatternNodePtr>> ChildAlternatives( | ||||||
2095 | T->getNumChildren()); | ||||||
2096 | for (unsigned i = 0, e = T->getNumChildren(); i != e; ++i) { | ||||||
2097 | TreePatternNodePtr Child = T->getChildShared(i); | ||||||
2098 | InlinePatternFragments(Child, TP, ChildAlternatives[i]); | ||||||
2099 | // If there are no alternatives for any child, there are no | ||||||
2100 | // alternatives for this expression as whole. | ||||||
2101 | if (ChildAlternatives[i].empty()) | ||||||
2102 | return; | ||||||
2103 | |||||||
2104 | assert((Child->getPredicateCalls().empty() ||(static_cast <bool> ((Child->getPredicateCalls().empty () || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child ->getPredicateCalls(); })) && "Non-empty child predicate clobbered!" ) ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child->getPredicateCalls(); })) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2110, __extension__ __PRETTY_FUNCTION__)) | ||||||
2105 | llvm::all_of(ChildAlternatives[i],(static_cast <bool> ((Child->getPredicateCalls().empty () || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child ->getPredicateCalls(); })) && "Non-empty child predicate clobbered!" ) ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child->getPredicateCalls(); })) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2110, __extension__ __PRETTY_FUNCTION__)) | ||||||
2106 | [&](const TreePatternNodePtr &NewChild) {(static_cast <bool> ((Child->getPredicateCalls().empty () || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child ->getPredicateCalls(); })) && "Non-empty child predicate clobbered!" ) ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child->getPredicateCalls(); })) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2110, __extension__ __PRETTY_FUNCTION__)) | ||||||
2107 | return NewChild->getPredicateCalls() ==(static_cast <bool> ((Child->getPredicateCalls().empty () || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child ->getPredicateCalls(); })) && "Non-empty child predicate clobbered!" ) ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child->getPredicateCalls(); })) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2110, __extension__ __PRETTY_FUNCTION__)) | ||||||
2108 | Child->getPredicateCalls();(static_cast <bool> ((Child->getPredicateCalls().empty () || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child ->getPredicateCalls(); })) && "Non-empty child predicate clobbered!" ) ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child->getPredicateCalls(); })) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2110, __extension__ __PRETTY_FUNCTION__)) | ||||||
2109 | })) &&(static_cast <bool> ((Child->getPredicateCalls().empty () || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child ->getPredicateCalls(); })) && "Non-empty child predicate clobbered!" ) ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child->getPredicateCalls(); })) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2110, __extension__ __PRETTY_FUNCTION__)) | ||||||
2110 | "Non-empty child predicate clobbered!")(static_cast <bool> ((Child->getPredicateCalls().empty () || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child ->getPredicateCalls(); })) && "Non-empty child predicate clobbered!" ) ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child->getPredicateCalls(); })) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2110, __extension__ __PRETTY_FUNCTION__)); | ||||||
2111 | } | ||||||
2112 | |||||||
2113 | // The end result is an all-pairs construction of the resultant pattern. | ||||||
2114 | std::vector<unsigned> Idxs(ChildAlternatives.size()); | ||||||
2115 | bool NotDone; | ||||||
2116 | do { | ||||||
2117 | // Create the variant and add it to the output list. | ||||||
2118 | std::vector<TreePatternNodePtr> NewChildren; | ||||||
2119 | NewChildren.reserve(ChildAlternatives.size()); | ||||||
2120 | for (unsigned i = 0, e = ChildAlternatives.size(); i != e; ++i) | ||||||
2121 | NewChildren.push_back(ChildAlternatives[i][Idxs[i]]); | ||||||
2122 | TreePatternNodePtr R = std::make_shared<TreePatternNode>( | ||||||
2123 | T->getOperator(), std::move(NewChildren), T->getNumTypes()); | ||||||
2124 | |||||||
2125 | // Copy over properties. | ||||||
2126 | R->setName(T->getName()); | ||||||
2127 | R->setNamesAsPredicateArg(T->getNamesAsPredicateArg()); | ||||||
2128 | R->setPredicateCalls(T->getPredicateCalls()); | ||||||
2129 | R->setGISelFlagsRecord(T->getGISelFlagsRecord()); | ||||||
2130 | R->setTransformFn(T->getTransformFn()); | ||||||
2131 | for (unsigned i = 0, e = T->getNumTypes(); i != e; ++i) | ||||||
2132 | R->setType(i, T->getExtType(i)); | ||||||
2133 | for (unsigned i = 0, e = T->getNumResults(); i != e; ++i) | ||||||
2134 | R->setResultIndex(i, T->getResultIndex(i)); | ||||||
2135 | |||||||
2136 | // Register alternative. | ||||||
2137 | OutAlternatives.push_back(R); | ||||||
2138 | |||||||
2139 | // Increment indices to the next permutation by incrementing the | ||||||
2140 | // indices from last index backward, e.g., generate the sequence | ||||||
2141 | // [0, 0], [0, 1], [1, 0], [1, 1]. | ||||||
2142 | int IdxsIdx; | ||||||
2143 | for (IdxsIdx = Idxs.size() - 1; IdxsIdx >= 0; --IdxsIdx) { | ||||||
2144 | if (++Idxs[IdxsIdx] == ChildAlternatives[IdxsIdx].size()) | ||||||
2145 | Idxs[IdxsIdx] = 0; | ||||||
2146 | else | ||||||
2147 | break; | ||||||
2148 | } | ||||||
2149 | NotDone = (IdxsIdx >= 0); | ||||||
2150 | } while (NotDone); | ||||||
2151 | |||||||
2152 | return; | ||||||
2153 | } | ||||||
2154 | |||||||
2155 | // Otherwise, we found a reference to a fragment. First, look up its | ||||||
2156 | // TreePattern record. | ||||||
2157 | TreePattern *Frag = TP.getDAGPatterns().getPatternFragment(Op); | ||||||
2158 | |||||||
2159 | // Verify that we are passing the right number of operands. | ||||||
2160 | if (Frag->getNumArgs() != T->getNumChildren()) { | ||||||
2161 | TP.error("'" + Op->getName() + "' fragment requires " + | ||||||
2162 | Twine(Frag->getNumArgs()) + " operands!"); | ||||||
2163 | return; | ||||||
2164 | } | ||||||
2165 | |||||||
2166 | TreePredicateFn PredFn(Frag); | ||||||
2167 | unsigned Scope = 0; | ||||||
2168 | if (TreePredicateFn(Frag).usesOperands()) | ||||||
2169 | Scope = TP.getDAGPatterns().allocateScope(); | ||||||
2170 | |||||||
2171 | // Compute the map of formal to actual arguments. | ||||||
2172 | std::map<std::string, TreePatternNodePtr> ArgMap; | ||||||
2173 | for (unsigned i = 0, e = Frag->getNumArgs(); i != e; ++i) { | ||||||
2174 | TreePatternNodePtr Child = T->getChildShared(i); | ||||||
2175 | if (Scope != 0) { | ||||||
2176 | Child = Child->clone(); | ||||||
2177 | Child->addNameAsPredicateArg(ScopedName(Scope, Frag->getArgName(i))); | ||||||
2178 | } | ||||||
2179 | ArgMap[Frag->getArgName(i)] = Child; | ||||||
2180 | } | ||||||
2181 | |||||||
2182 | // Loop over all fragment alternatives. | ||||||
2183 | for (const auto &Alternative : Frag->getTrees()) { | ||||||
2184 | TreePatternNodePtr FragTree = Alternative->clone(); | ||||||
2185 | |||||||
2186 | if (!PredFn.isAlwaysTrue()) | ||||||
2187 | FragTree->addPredicateCall(PredFn, Scope); | ||||||
2188 | |||||||
2189 | // Resolve formal arguments to their actual value. | ||||||
2190 | if (Frag->getNumArgs()) | ||||||
2191 | FragTree->SubstituteFormalArguments(ArgMap); | ||||||
2192 | |||||||
2193 | // Transfer types. Note that the resolved alternative may have fewer | ||||||
2194 | // (but not more) results than the PatFrags node. | ||||||
2195 | FragTree->setName(T->getName()); | ||||||
2196 | for (unsigned i = 0, e = FragTree->getNumTypes(); i != e; ++i) | ||||||
2197 | FragTree->UpdateNodeType(i, T->getExtType(i), TP); | ||||||
2198 | |||||||
2199 | if (Op->isSubClassOf("GISelFlags")) | ||||||
2200 | FragTree->setGISelFlagsRecord(Op); | ||||||
2201 | |||||||
2202 | // Transfer in the old predicates. | ||||||
2203 | for (const TreePredicateCall &Pred : T->getPredicateCalls()) | ||||||
2204 | FragTree->addPredicateCall(Pred); | ||||||
2205 | |||||||
2206 | // The fragment we inlined could have recursive inlining that is needed. See | ||||||
2207 | // if there are any pattern fragments in it and inline them as needed. | ||||||
2208 | InlinePatternFragments(FragTree, TP, OutAlternatives); | ||||||
2209 | } | ||||||
2210 | } | ||||||
2211 | |||||||
2212 | /// getImplicitType - Check to see if the specified record has an implicit | ||||||
2213 | /// type which should be applied to it. This will infer the type of register | ||||||
2214 | /// references from the register file information, for example. | ||||||
2215 | /// | ||||||
2216 | /// When Unnamed is set, return the type of a DAG operand with no name, such as | ||||||
2217 | /// the F8RC register class argument in: | ||||||
2218 | /// | ||||||
2219 | /// (COPY_TO_REGCLASS GPR:$src, F8RC) | ||||||
2220 | /// | ||||||
2221 | /// When Unnamed is false, return the type of a named DAG operand such as the | ||||||
2222 | /// GPR:$src operand above. | ||||||
2223 | /// | ||||||
2224 | static TypeSetByHwMode getImplicitType(Record *R, unsigned ResNo, | ||||||
2225 | bool NotRegisters, | ||||||
2226 | bool Unnamed, | ||||||
2227 | TreePattern &TP) { | ||||||
2228 | CodeGenDAGPatterns &CDP = TP.getDAGPatterns(); | ||||||
2229 | |||||||
2230 | // Check to see if this is a register operand. | ||||||
2231 | if (R->isSubClassOf("RegisterOperand")) { | ||||||
2232 | assert(ResNo == 0 && "Regoperand ref only has one result!")(static_cast <bool> (ResNo == 0 && "Regoperand ref only has one result!" ) ? void (0) : __assert_fail ("ResNo == 0 && \"Regoperand ref only has one result!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2232, __extension__ __PRETTY_FUNCTION__)); | ||||||
2233 | if (NotRegisters) | ||||||
2234 | return TypeSetByHwMode(); // Unknown. | ||||||
2235 | Record *RegClass = R->getValueAsDef("RegClass"); | ||||||
2236 | const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); | ||||||
2237 | return TypeSetByHwMode(T.getRegisterClass(RegClass).getValueTypes()); | ||||||
2238 | } | ||||||
2239 | |||||||
2240 | // Check to see if this is a register or a register class. | ||||||
2241 | if (R->isSubClassOf("RegisterClass")) { | ||||||
2242 | assert(ResNo == 0 && "Regclass ref only has one result!")(static_cast <bool> (ResNo == 0 && "Regclass ref only has one result!" ) ? void (0) : __assert_fail ("ResNo == 0 && \"Regclass ref only has one result!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2242, __extension__ __PRETTY_FUNCTION__)); | ||||||
2243 | // An unnamed register class represents itself as an i32 immediate, for | ||||||
2244 | // example on a COPY_TO_REGCLASS instruction. | ||||||
2245 | if (Unnamed) | ||||||
2246 | return TypeSetByHwMode(MVT::i32); | ||||||
2247 | |||||||
2248 | // In a named operand, the register class provides the possible set of | ||||||
2249 | // types. | ||||||
2250 | if (NotRegisters) | ||||||
2251 | return TypeSetByHwMode(); // Unknown. | ||||||
2252 | const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); | ||||||
2253 | return TypeSetByHwMode(T.getRegisterClass(R).getValueTypes()); | ||||||
2254 | } | ||||||
2255 | |||||||
2256 | if (R->isSubClassOf("PatFrags")) { | ||||||
2257 | assert(ResNo == 0 && "FIXME: PatFrag with multiple results?")(static_cast <bool> (ResNo == 0 && "FIXME: PatFrag with multiple results?" ) ? void (0) : __assert_fail ("ResNo == 0 && \"FIXME: PatFrag with multiple results?\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2257, __extension__ __PRETTY_FUNCTION__)); | ||||||
2258 | // Pattern fragment types will be resolved when they are inlined. | ||||||
2259 | return TypeSetByHwMode(); // Unknown. | ||||||
2260 | } | ||||||
2261 | |||||||
2262 | if (R->isSubClassOf("Register")) { | ||||||
2263 | assert(ResNo == 0 && "Registers only produce one result!")(static_cast <bool> (ResNo == 0 && "Registers only produce one result!" ) ? void (0) : __assert_fail ("ResNo == 0 && \"Registers only produce one result!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2263, __extension__ __PRETTY_FUNCTION__)); | ||||||
2264 | if (NotRegisters) | ||||||
2265 | return TypeSetByHwMode(); // Unknown. | ||||||
2266 | const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); | ||||||
2267 | return TypeSetByHwMode(T.getRegisterVTs(R)); | ||||||
2268 | } | ||||||
2269 | |||||||
2270 | if (R->isSubClassOf("SubRegIndex")) { | ||||||
2271 | assert(ResNo == 0 && "SubRegisterIndices only produce one result!")(static_cast <bool> (ResNo == 0 && "SubRegisterIndices only produce one result!" ) ? void (0) : __assert_fail ("ResNo == 0 && \"SubRegisterIndices only produce one result!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2271, __extension__ __PRETTY_FUNCTION__)); | ||||||
2272 | return TypeSetByHwMode(MVT::i32); | ||||||
2273 | } | ||||||
2274 | |||||||
2275 | if (R->isSubClassOf("ValueType")) { | ||||||
2276 | assert(ResNo == 0 && "This node only has one result!")(static_cast <bool> (ResNo == 0 && "This node only has one result!" ) ? void (0) : __assert_fail ("ResNo == 0 && \"This node only has one result!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2276, __extension__ __PRETTY_FUNCTION__)); | ||||||
2277 | // An unnamed VTSDNode represents itself as an MVT::Other immediate. | ||||||
2278 | // | ||||||
2279 | // (sext_inreg GPR:$src, i16) | ||||||
2280 | // ~~~ | ||||||
2281 | if (Unnamed) | ||||||
2282 | return TypeSetByHwMode(MVT::Other); | ||||||
2283 | // With a name, the ValueType simply provides the type of the named | ||||||
2284 | // variable. | ||||||
2285 | // | ||||||
2286 | // (sext_inreg i32:$src, i16) | ||||||
2287 | // ~~~~~~~~ | ||||||
2288 | if (NotRegisters) | ||||||
2289 | return TypeSetByHwMode(); // Unknown. | ||||||
2290 | const CodeGenHwModes &CGH = CDP.getTargetInfo().getHwModes(); | ||||||
2291 | return TypeSetByHwMode(getValueTypeByHwMode(R, CGH)); | ||||||
2292 | } | ||||||
2293 | |||||||
2294 | if (R->isSubClassOf("CondCode")) { | ||||||
2295 | assert(ResNo == 0 && "This node only has one result!")(static_cast <bool> (ResNo == 0 && "This node only has one result!" ) ? void (0) : __assert_fail ("ResNo == 0 && \"This node only has one result!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2295, __extension__ __PRETTY_FUNCTION__)); | ||||||
2296 | // Using a CondCodeSDNode. | ||||||
2297 | return TypeSetByHwMode(MVT::Other); | ||||||
2298 | } | ||||||
2299 | |||||||
2300 | if (R->isSubClassOf("ComplexPattern")) { | ||||||
2301 | assert(ResNo == 0 && "FIXME: ComplexPattern with multiple results?")(static_cast <bool> (ResNo == 0 && "FIXME: ComplexPattern with multiple results?" ) ? void (0) : __assert_fail ("ResNo == 0 && \"FIXME: ComplexPattern with multiple results?\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2301, __extension__ __PRETTY_FUNCTION__)); | ||||||
2302 | if (NotRegisters) | ||||||
2303 | return TypeSetByHwMode(); // Unknown. | ||||||
2304 | Record *T = CDP.getComplexPattern(R).getValueType(); | ||||||
2305 | const CodeGenHwModes &CGH = CDP.getTargetInfo().getHwModes(); | ||||||
2306 | return TypeSetByHwMode(getValueTypeByHwMode(T, CGH)); | ||||||
2307 | } | ||||||
2308 | if (R->isSubClassOf("PointerLikeRegClass")) { | ||||||
2309 | assert(ResNo == 0 && "Regclass can only have one result!")(static_cast <bool> (ResNo == 0 && "Regclass can only have one result!" ) ? void (0) : __assert_fail ("ResNo == 0 && \"Regclass can only have one result!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2309, __extension__ __PRETTY_FUNCTION__)); | ||||||
2310 | TypeSetByHwMode VTS(MVT::iPTR); | ||||||
2311 | TP.getInfer().expandOverloads(VTS); | ||||||
2312 | return VTS; | ||||||
2313 | } | ||||||
2314 | |||||||
2315 | if (R->getName() == "node" || R->getName() == "srcvalue" || | ||||||
2316 | R->getName() == "zero_reg" || R->getName() == "immAllOnesV" || | ||||||
2317 | R->getName() == "immAllZerosV" || R->getName() == "undef_tied_input") { | ||||||
2318 | // Placeholder. | ||||||
2319 | return TypeSetByHwMode(); // Unknown. | ||||||
2320 | } | ||||||
2321 | |||||||
2322 | if (R->isSubClassOf("Operand")) { | ||||||
2323 | const CodeGenHwModes &CGH = CDP.getTargetInfo().getHwModes(); | ||||||
2324 | Record *T = R->getValueAsDef("Type"); | ||||||
2325 | return TypeSetByHwMode(getValueTypeByHwMode(T, CGH)); | ||||||
2326 | } | ||||||
2327 | |||||||
2328 | TP.error("Unknown node flavor used in pattern: " + R->getName()); | ||||||
2329 | return TypeSetByHwMode(MVT::Other); | ||||||
2330 | } | ||||||
2331 | |||||||
2332 | |||||||
2333 | /// getIntrinsicInfo - If this node corresponds to an intrinsic, return the | ||||||
2334 | /// CodeGenIntrinsic information for it, otherwise return a null pointer. | ||||||
2335 | const CodeGenIntrinsic *TreePatternNode:: | ||||||
2336 | getIntrinsicInfo(const CodeGenDAGPatterns &CDP) const { | ||||||
2337 | if (getOperator() != CDP.get_intrinsic_void_sdnode() && | ||||||
2338 | getOperator() != CDP.get_intrinsic_w_chain_sdnode() && | ||||||
2339 | getOperator() != CDP.get_intrinsic_wo_chain_sdnode()) | ||||||
2340 | return nullptr; | ||||||
2341 | |||||||
2342 | unsigned IID = cast<IntInit>(getChild(0)->getLeafValue())->getValue(); | ||||||
2343 | return &CDP.getIntrinsicInfo(IID); | ||||||
2344 | } | ||||||
2345 | |||||||
2346 | /// getComplexPatternInfo - If this node corresponds to a ComplexPattern, | ||||||
2347 | /// return the ComplexPattern information, otherwise return null. | ||||||
2348 | const ComplexPattern * | ||||||
2349 | TreePatternNode::getComplexPatternInfo(const CodeGenDAGPatterns &CGP) const { | ||||||
2350 | Record *Rec; | ||||||
2351 | if (isLeaf()) { | ||||||
2352 | DefInit *DI = dyn_cast<DefInit>(getLeafValue()); | ||||||
2353 | if (!DI) | ||||||
2354 | return nullptr; | ||||||
2355 | Rec = DI->getDef(); | ||||||
2356 | } else | ||||||
2357 | Rec = getOperator(); | ||||||
2358 | |||||||
2359 | if (!Rec->isSubClassOf("ComplexPattern")) | ||||||
2360 | return nullptr; | ||||||
2361 | return &CGP.getComplexPattern(Rec); | ||||||
2362 | } | ||||||
2363 | |||||||
2364 | unsigned TreePatternNode::getNumMIResults(const CodeGenDAGPatterns &CGP) const { | ||||||
2365 | // A ComplexPattern specifically declares how many results it fills in. | ||||||
2366 | if (const ComplexPattern *CP = getComplexPatternInfo(CGP)) | ||||||
2367 | return CP->getNumOperands(); | ||||||
2368 | |||||||
2369 | // If MIOperandInfo is specified, that gives the count. | ||||||
2370 | if (isLeaf()) { | ||||||
2371 | DefInit *DI = dyn_cast<DefInit>(getLeafValue()); | ||||||
2372 | if (DI && DI->getDef()->isSubClassOf("Operand")) { | ||||||
2373 | DagInit *MIOps = DI->getDef()->getValueAsDag("MIOperandInfo"); | ||||||
2374 | if (MIOps->getNumArgs()) | ||||||
2375 | return MIOps->getNumArgs(); | ||||||
2376 | } | ||||||
2377 | } | ||||||
2378 | |||||||
2379 | // Otherwise there is just one result. | ||||||
2380 | return 1; | ||||||
2381 | } | ||||||
2382 | |||||||
2383 | /// NodeHasProperty - Return true if this node has the specified property. | ||||||
2384 | bool TreePatternNode::NodeHasProperty(SDNP Property, | ||||||
2385 | const CodeGenDAGPatterns &CGP) const { | ||||||
2386 | if (isLeaf()) { | ||||||
2387 | if (const ComplexPattern *CP = getComplexPatternInfo(CGP)) | ||||||
2388 | return CP->hasProperty(Property); | ||||||
2389 | |||||||
2390 | return false; | ||||||
2391 | } | ||||||
2392 | |||||||
2393 | if (Property != SDNPHasChain) { | ||||||
2394 | // The chain proprety is already present on the different intrinsic node | ||||||
2395 | // types (intrinsic_w_chain, intrinsic_void), and is not explicitly listed | ||||||
2396 | // on the intrinsic. Anything else is specific to the individual intrinsic. | ||||||
2397 | if (const CodeGenIntrinsic *Int = getIntrinsicInfo(CGP)) | ||||||
2398 | return Int->hasProperty(Property); | ||||||
2399 | } | ||||||
2400 | |||||||
2401 | if (!Operator->isSubClassOf("SDPatternOperator")) | ||||||
2402 | return false; | ||||||
2403 | |||||||
2404 | return CGP.getSDNodeInfo(Operator).hasProperty(Property); | ||||||
2405 | } | ||||||
2406 | |||||||
2407 | |||||||
2408 | |||||||
2409 | |||||||
2410 | /// TreeHasProperty - Return true if any node in this tree has the specified | ||||||
2411 | /// property. | ||||||
2412 | bool TreePatternNode::TreeHasProperty(SDNP Property, | ||||||
2413 | const CodeGenDAGPatterns &CGP) const { | ||||||
2414 | if (NodeHasProperty(Property, CGP)) | ||||||
2415 | return true; | ||||||
2416 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) | ||||||
2417 | if (getChild(i)->TreeHasProperty(Property, CGP)) | ||||||
2418 | return true; | ||||||
2419 | return false; | ||||||
2420 | } | ||||||
2421 | |||||||
2422 | /// isCommutativeIntrinsic - Return true if the node corresponds to a | ||||||
2423 | /// commutative intrinsic. | ||||||
2424 | bool | ||||||
2425 | TreePatternNode::isCommutativeIntrinsic(const CodeGenDAGPatterns &CDP) const { | ||||||
2426 | if (const CodeGenIntrinsic *Int = getIntrinsicInfo(CDP)) | ||||||
2427 | return Int->isCommutative; | ||||||
2428 | return false; | ||||||
2429 | } | ||||||
2430 | |||||||
2431 | static bool isOperandClass(const TreePatternNode *N, StringRef Class) { | ||||||
2432 | if (!N->isLeaf()) | ||||||
2433 | return N->getOperator()->isSubClassOf(Class); | ||||||
2434 | |||||||
2435 | DefInit *DI = dyn_cast<DefInit>(N->getLeafValue()); | ||||||
2436 | if (DI && DI->getDef()->isSubClassOf(Class)) | ||||||
2437 | return true; | ||||||
2438 | |||||||
2439 | return false; | ||||||
2440 | } | ||||||
2441 | |||||||
2442 | static void emitTooManyOperandsError(TreePattern &TP, | ||||||
2443 | StringRef InstName, | ||||||
2444 | unsigned Expected, | ||||||
2445 | unsigned Actual) { | ||||||
2446 | TP.error("Instruction '" + InstName + "' was provided " + Twine(Actual) + | ||||||
2447 | " operands but expected only " + Twine(Expected) + "!"); | ||||||
2448 | } | ||||||
2449 | |||||||
2450 | static void emitTooFewOperandsError(TreePattern &TP, | ||||||
2451 | StringRef InstName, | ||||||
2452 | unsigned Actual) { | ||||||
2453 | TP.error("Instruction '" + InstName + | ||||||
2454 | "' expects more than the provided " + Twine(Actual) + " operands!"); | ||||||
2455 | } | ||||||
2456 | |||||||
2457 | /// ApplyTypeConstraints - Apply all of the type constraints relevant to | ||||||
2458 | /// this node and its children in the tree. This returns true if it makes a | ||||||
2459 | /// change, false otherwise. If a type contradiction is found, flag an error. | ||||||
2460 | bool TreePatternNode::ApplyTypeConstraints(TreePattern &TP, bool NotRegisters) { | ||||||
2461 | if (TP.hasError()) | ||||||
2462 | return false; | ||||||
2463 | |||||||
2464 | CodeGenDAGPatterns &CDP = TP.getDAGPatterns(); | ||||||
2465 | if (isLeaf()) { | ||||||
2466 | if (DefInit *DI = dyn_cast<DefInit>(getLeafValue())) { | ||||||
2467 | // If it's a regclass or something else known, include the type. | ||||||
2468 | bool MadeChange = false; | ||||||
2469 | for (unsigned i = 0, e = Types.size(); i != e; ++i) | ||||||
2470 | MadeChange |= UpdateNodeType(i, getImplicitType(DI->getDef(), i, | ||||||
2471 | NotRegisters, | ||||||
2472 | !hasName(), TP), TP); | ||||||
2473 | return MadeChange; | ||||||
2474 | } | ||||||
2475 | |||||||
2476 | if (IntInit *II = dyn_cast<IntInit>(getLeafValue())) { | ||||||
2477 | assert(Types.size() == 1 && "Invalid IntInit")(static_cast <bool> (Types.size() == 1 && "Invalid IntInit" ) ? void (0) : __assert_fail ("Types.size() == 1 && \"Invalid IntInit\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2477, __extension__ __PRETTY_FUNCTION__)); | ||||||
2478 | |||||||
2479 | // Int inits are always integers. :) | ||||||
2480 | bool MadeChange = TP.getInfer().EnforceInteger(Types[0]); | ||||||
2481 | |||||||
2482 | if (!TP.getInfer().isConcrete(Types[0], false)) | ||||||
2483 | return MadeChange; | ||||||
2484 | |||||||
2485 | ValueTypeByHwMode VVT = TP.getInfer().getConcrete(Types[0], false); | ||||||
2486 | for (auto &P : VVT) { | ||||||
2487 | MVT::SimpleValueType VT = P.second.SimpleTy; | ||||||
2488 | if (VT == MVT::iPTR || VT == MVT::iPTRAny) | ||||||
2489 | continue; | ||||||
2490 | unsigned Size = MVT(VT).getFixedSizeInBits(); | ||||||
2491 | // Make sure that the value is representable for this type. | ||||||
2492 | if (Size >= 32) | ||||||
2493 | continue; | ||||||
2494 | // Check that the value doesn't use more bits than we have. It must | ||||||
2495 | // either be a sign- or zero-extended equivalent of the original. | ||||||
2496 | int64_t SignBitAndAbove = II->getValue() >> (Size - 1); | ||||||
2497 | if (SignBitAndAbove == -1 || SignBitAndAbove == 0 || | ||||||
2498 | SignBitAndAbove == 1) | ||||||
2499 | continue; | ||||||
2500 | |||||||
2501 | TP.error("Integer value '" + Twine(II->getValue()) + | ||||||
2502 | "' is out of range for type '" + getEnumName(VT) + "'!"); | ||||||
2503 | break; | ||||||
2504 | } | ||||||
2505 | return MadeChange; | ||||||
2506 | } | ||||||
2507 | |||||||
2508 | return false; | ||||||
2509 | } | ||||||
2510 | |||||||
2511 | if (const CodeGenIntrinsic *Int = getIntrinsicInfo(CDP)) { | ||||||
2512 | bool MadeChange = false; | ||||||
2513 | |||||||
2514 | // Apply the result type to the node. | ||||||
2515 | unsigned NumRetVTs = Int->IS.RetVTs.size(); | ||||||
2516 | unsigned NumParamVTs = Int->IS.ParamVTs.size(); | ||||||
2517 | |||||||
2518 | for (unsigned i = 0, e = NumRetVTs; i != e; ++i) | ||||||
2519 | MadeChange |= UpdateNodeType(i, Int->IS.RetVTs[i], TP); | ||||||
2520 | |||||||
2521 | if (getNumChildren() != NumParamVTs + 1) { | ||||||
2522 | TP.error("Intrinsic '" + Int->Name + "' expects " + Twine(NumParamVTs) + | ||||||
2523 | " operands, not " + Twine(getNumChildren() - 1) + " operands!"); | ||||||
2524 | return false; | ||||||
2525 | } | ||||||
2526 | |||||||
2527 | // Apply type info to the intrinsic ID. | ||||||
2528 | MadeChange |= getChild(0)->UpdateNodeType(0, MVT::iPTR, TP); | ||||||
2529 | |||||||
2530 | for (unsigned i = 0, e = getNumChildren()-1; i != e; ++i) { | ||||||
2531 | MadeChange |= getChild(i+1)->ApplyTypeConstraints(TP, NotRegisters); | ||||||
2532 | |||||||
2533 | MVT::SimpleValueType OpVT = Int->IS.ParamVTs[i]; | ||||||
2534 | assert(getChild(i+1)->getNumTypes() == 1 && "Unhandled case")(static_cast <bool> (getChild(i+1)->getNumTypes() == 1 && "Unhandled case") ? void (0) : __assert_fail ("getChild(i+1)->getNumTypes() == 1 && \"Unhandled case\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2534, __extension__ __PRETTY_FUNCTION__)); | ||||||
2535 | MadeChange |= getChild(i+1)->UpdateNodeType(0, OpVT, TP); | ||||||
2536 | } | ||||||
2537 | return MadeChange; | ||||||
2538 | } | ||||||
2539 | |||||||
2540 | if (getOperator()->isSubClassOf("SDNode")) { | ||||||
2541 | const SDNodeInfo &NI = CDP.getSDNodeInfo(getOperator()); | ||||||
2542 | |||||||
2543 | // Check that the number of operands is sane. Negative operands -> varargs. | ||||||
2544 | if (NI.getNumOperands() >= 0 && | ||||||
2545 | getNumChildren() != (unsigned)NI.getNumOperands()) { | ||||||
2546 | TP.error(getOperator()->getName() + " node requires exactly " + | ||||||
2547 | Twine(NI.getNumOperands()) + " operands!"); | ||||||
2548 | return false; | ||||||
2549 | } | ||||||
2550 | |||||||
2551 | bool MadeChange = false; | ||||||
2552 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) | ||||||
2553 | MadeChange |= getChild(i)->ApplyTypeConstraints(TP, NotRegisters); | ||||||
2554 | MadeChange |= NI.ApplyTypeConstraints(this, TP); | ||||||
2555 | return MadeChange; | ||||||
2556 | } | ||||||
2557 | |||||||
2558 | if (getOperator()->isSubClassOf("Instruction")) { | ||||||
2559 | const DAGInstruction &Inst = CDP.getInstruction(getOperator()); | ||||||
2560 | CodeGenInstruction &InstInfo = | ||||||
2561 | CDP.getTargetInfo().getInstruction(getOperator()); | ||||||
2562 | |||||||
2563 | bool MadeChange = false; | ||||||
2564 | |||||||
2565 | // Apply the result types to the node, these come from the things in the | ||||||
2566 | // (outs) list of the instruction. | ||||||
2567 | unsigned NumResultsToAdd = std::min(InstInfo.Operands.NumDefs, | ||||||
2568 | Inst.getNumResults()); | ||||||
2569 | for (unsigned ResNo = 0; ResNo != NumResultsToAdd; ++ResNo) | ||||||
2570 | MadeChange |= UpdateNodeTypeFromInst(ResNo, Inst.getResult(ResNo), TP); | ||||||
2571 | |||||||
2572 | // If the instruction has implicit defs, we apply the first one as a result. | ||||||
2573 | // FIXME: This sucks, it should apply all implicit defs. | ||||||
2574 | if (!InstInfo.ImplicitDefs.empty()) { | ||||||
2575 | unsigned ResNo = NumResultsToAdd; | ||||||
2576 | |||||||
2577 | // FIXME: Generalize to multiple possible types and multiple possible | ||||||
2578 | // ImplicitDefs. | ||||||
2579 | MVT::SimpleValueType VT = | ||||||
2580 | InstInfo.HasOneImplicitDefWithKnownVT(CDP.getTargetInfo()); | ||||||
2581 | |||||||
2582 | if (VT != MVT::Other) | ||||||
2583 | MadeChange |= UpdateNodeType(ResNo, VT, TP); | ||||||
2584 | } | ||||||
2585 | |||||||
2586 | // If this is an INSERT_SUBREG, constrain the source and destination VTs to | ||||||
2587 | // be the same. | ||||||
2588 | if (getOperator()->getName() == "INSERT_SUBREG") { | ||||||
2589 | assert(getChild(0)->getNumTypes() == 1 && "FIXME: Unhandled")(static_cast <bool> (getChild(0)->getNumTypes() == 1 && "FIXME: Unhandled") ? void (0) : __assert_fail ("getChild(0)->getNumTypes() == 1 && \"FIXME: Unhandled\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2589, __extension__ __PRETTY_FUNCTION__)); | ||||||
2590 | MadeChange |= UpdateNodeType(0, getChild(0)->getExtType(0), TP); | ||||||
2591 | MadeChange |= getChild(0)->UpdateNodeType(0, getExtType(0), TP); | ||||||
2592 | } else if (getOperator()->getName() == "REG_SEQUENCE") { | ||||||
2593 | // We need to do extra, custom typechecking for REG_SEQUENCE since it is | ||||||
2594 | // variadic. | ||||||
2595 | |||||||
2596 | unsigned NChild = getNumChildren(); | ||||||
2597 | if (NChild < 3) { | ||||||
2598 | TP.error("REG_SEQUENCE requires at least 3 operands!"); | ||||||
2599 | return false; | ||||||
2600 | } | ||||||
2601 | |||||||
2602 | if (NChild % 2 == 0) { | ||||||
2603 | TP.error("REG_SEQUENCE requires an odd number of operands!"); | ||||||
2604 | return false; | ||||||
2605 | } | ||||||
2606 | |||||||
2607 | if (!isOperandClass(getChild(0), "RegisterClass")) { | ||||||
2608 | TP.error("REG_SEQUENCE requires a RegisterClass for first operand!"); | ||||||
2609 | return false; | ||||||
2610 | } | ||||||
2611 | |||||||
2612 | for (unsigned I = 1; I < NChild; I += 2) { | ||||||
2613 | TreePatternNode *SubIdxChild = getChild(I + 1); | ||||||
2614 | if (!isOperandClass(SubIdxChild, "SubRegIndex")) { | ||||||
2615 | TP.error("REG_SEQUENCE requires a SubRegIndex for operand " + | ||||||
2616 | Twine(I + 1) + "!"); | ||||||
2617 | return false; | ||||||
2618 | } | ||||||
2619 | } | ||||||
2620 | } | ||||||
2621 | |||||||
2622 | unsigned NumResults = Inst.getNumResults(); | ||||||
2623 | unsigned NumFixedOperands = InstInfo.Operands.size(); | ||||||
2624 | |||||||
2625 | // If one or more operands with a default value appear at the end of the | ||||||
2626 | // formal operand list for an instruction, we allow them to be overridden | ||||||
2627 | // by optional operands provided in the pattern. | ||||||
2628 | // | ||||||
2629 | // But if an operand B without a default appears at any point after an | ||||||
2630 | // operand A with a default, then we don't allow A to be overridden, | ||||||
2631 | // because there would be no way to specify whether the next operand in | ||||||
2632 | // the pattern was intended to override A or skip it. | ||||||
2633 | unsigned NonOverridableOperands = NumFixedOperands; | ||||||
2634 | while (NonOverridableOperands > NumResults && | ||||||
2635 | CDP.operandHasDefault(InstInfo.Operands[NonOverridableOperands-1].Rec)) | ||||||
2636 | --NonOverridableOperands; | ||||||
2637 | |||||||
2638 | unsigned ChildNo = 0; | ||||||
2639 | assert(NumResults <= NumFixedOperands)(static_cast <bool> (NumResults <= NumFixedOperands) ? void (0) : __assert_fail ("NumResults <= NumFixedOperands" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2639, __extension__ __PRETTY_FUNCTION__)); | ||||||
2640 | for (unsigned i = NumResults, e = NumFixedOperands; i != e; ++i) { | ||||||
2641 | Record *OperandNode = InstInfo.Operands[i].Rec; | ||||||
2642 | |||||||
2643 | // If the operand has a default value, do we use it? We must use the | ||||||
2644 | // default if we've run out of children of the pattern DAG to consume, | ||||||
2645 | // or if the operand is followed by a non-defaulted one. | ||||||
2646 | if (CDP.operandHasDefault(OperandNode) && | ||||||
2647 | (i < NonOverridableOperands || ChildNo >= getNumChildren())) | ||||||
2648 | continue; | ||||||
2649 | |||||||
2650 | // If we have run out of child nodes and there _isn't_ a default | ||||||
2651 | // value we can use for the next operand, give an error. | ||||||
2652 | if (ChildNo >= getNumChildren()) { | ||||||
2653 | emitTooFewOperandsError(TP, getOperator()->getName(), getNumChildren()); | ||||||
2654 | return false; | ||||||
2655 | } | ||||||
2656 | |||||||
2657 | TreePatternNode *Child = getChild(ChildNo++); | ||||||
2658 | unsigned ChildResNo = 0; // Instructions always use res #0 of their op. | ||||||
2659 | |||||||
2660 | // If the operand has sub-operands, they may be provided by distinct | ||||||
2661 | // child patterns, so attempt to match each sub-operand separately. | ||||||
2662 | if (OperandNode->isSubClassOf("Operand")) { | ||||||
2663 | DagInit *MIOpInfo = OperandNode->getValueAsDag("MIOperandInfo"); | ||||||
2664 | if (unsigned NumArgs = MIOpInfo->getNumArgs()) { | ||||||
2665 | // But don't do that if the whole operand is being provided by | ||||||
2666 | // a single ComplexPattern-related Operand. | ||||||
2667 | |||||||
2668 | if (Child->getNumMIResults(CDP) < NumArgs) { | ||||||
2669 | // Match first sub-operand against the child we already have. | ||||||
2670 | Record *SubRec = cast<DefInit>(MIOpInfo->getArg(0))->getDef(); | ||||||
2671 | MadeChange |= | ||||||
2672 | Child->UpdateNodeTypeFromInst(ChildResNo, SubRec, TP); | ||||||
2673 | |||||||
2674 | // And the remaining sub-operands against subsequent children. | ||||||
2675 | for (unsigned Arg = 1; Arg < NumArgs; ++Arg) { | ||||||
2676 | if (ChildNo >= getNumChildren()) { | ||||||
2677 | emitTooFewOperandsError(TP, getOperator()->getName(), | ||||||
2678 | getNumChildren()); | ||||||
2679 | return false; | ||||||
2680 | } | ||||||
2681 | Child = getChild(ChildNo++); | ||||||
2682 | |||||||
2683 | SubRec = cast<DefInit>(MIOpInfo->getArg(Arg))->getDef(); | ||||||
2684 | MadeChange |= | ||||||
2685 | Child->UpdateNodeTypeFromInst(ChildResNo, SubRec, TP); | ||||||
2686 | } | ||||||
2687 | continue; | ||||||
2688 | } | ||||||
2689 | } | ||||||
2690 | } | ||||||
2691 | |||||||
2692 | // If we didn't match by pieces above, attempt to match the whole | ||||||
2693 | // operand now. | ||||||
2694 | MadeChange |= Child->UpdateNodeTypeFromInst(ChildResNo, OperandNode, TP); | ||||||
2695 | } | ||||||
2696 | |||||||
2697 | if (!InstInfo.Operands.isVariadic && ChildNo != getNumChildren()) { | ||||||
2698 | emitTooManyOperandsError(TP, getOperator()->getName(), | ||||||
2699 | ChildNo, getNumChildren()); | ||||||
2700 | return false; | ||||||
2701 | } | ||||||
2702 | |||||||
2703 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) | ||||||
2704 | MadeChange |= getChild(i)->ApplyTypeConstraints(TP, NotRegisters); | ||||||
2705 | return MadeChange; | ||||||
2706 | } | ||||||
2707 | |||||||
2708 | if (getOperator()->isSubClassOf("ComplexPattern")) { | ||||||
2709 | bool MadeChange = false; | ||||||
2710 | |||||||
2711 | if (!NotRegisters) { | ||||||
2712 | assert(Types.size() == 1 && "ComplexPatterns only produce one result!")(static_cast <bool> (Types.size() == 1 && "ComplexPatterns only produce one result!" ) ? void (0) : __assert_fail ("Types.size() == 1 && \"ComplexPatterns only produce one result!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2712, __extension__ __PRETTY_FUNCTION__)); | ||||||
2713 | Record *T = CDP.getComplexPattern(getOperator()).getValueType(); | ||||||
2714 | const CodeGenHwModes &CGH = CDP.getTargetInfo().getHwModes(); | ||||||
2715 | const ValueTypeByHwMode VVT = getValueTypeByHwMode(T, CGH); | ||||||
2716 | // TODO: AArch64 and AMDGPU use ComplexPattern<untyped, ...> and then | ||||||
2717 | // exclusively use those as non-leaf nodes with explicit type casts, so | ||||||
2718 | // for backwards compatibility we do no inference in that case. This is | ||||||
2719 | // not supported when the ComplexPattern is used as a leaf value, | ||||||
2720 | // however; this inconsistency should be resolved, either by adding this | ||||||
2721 | // case there or by altering the backends to not do this (e.g. using Any | ||||||
2722 | // instead may work). | ||||||
2723 | if (!VVT.isSimple() || VVT.getSimple() != MVT::Untyped) | ||||||
2724 | MadeChange |= UpdateNodeType(0, VVT, TP); | ||||||
2725 | } | ||||||
2726 | |||||||
2727 | for (unsigned i = 0; i < getNumChildren(); ++i) | ||||||
2728 | MadeChange |= getChild(i)->ApplyTypeConstraints(TP, NotRegisters); | ||||||
2729 | |||||||
2730 | return MadeChange; | ||||||
2731 | } | ||||||
2732 | |||||||
2733 | assert(getOperator()->isSubClassOf("SDNodeXForm") && "Unknown node type!")(static_cast <bool> (getOperator()->isSubClassOf("SDNodeXForm" ) && "Unknown node type!") ? void (0) : __assert_fail ("getOperator()->isSubClassOf(\"SDNodeXForm\") && \"Unknown node type!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2733, __extension__ __PRETTY_FUNCTION__)); | ||||||
2734 | |||||||
2735 | // Node transforms always take one operand. | ||||||
2736 | if (getNumChildren() != 1) { | ||||||
2737 | TP.error("Node transform '" + getOperator()->getName() + | ||||||
2738 | "' requires one operand!"); | ||||||
2739 | return false; | ||||||
2740 | } | ||||||
2741 | |||||||
2742 | bool MadeChange = getChild(0)->ApplyTypeConstraints(TP, NotRegisters); | ||||||
2743 | return MadeChange; | ||||||
2744 | } | ||||||
2745 | |||||||
2746 | /// OnlyOnRHSOfCommutative - Return true if this value is only allowed on the | ||||||
2747 | /// RHS of a commutative operation, not the on LHS. | ||||||
2748 | static bool OnlyOnRHSOfCommutative(TreePatternNode *N) { | ||||||
2749 | if (!N->isLeaf() && N->getOperator()->getName() == "imm") | ||||||
2750 | return true; | ||||||
2751 | if (N->isLeaf() && isa<IntInit>(N->getLeafValue())) | ||||||
2752 | return true; | ||||||
2753 | if (isImmAllOnesAllZerosMatch(N)) | ||||||
2754 | return true; | ||||||
2755 | return false; | ||||||
2756 | } | ||||||
2757 | |||||||
2758 | |||||||
2759 | /// canPatternMatch - If it is impossible for this pattern to match on this | ||||||
2760 | /// target, fill in Reason and return false. Otherwise, return true. This is | ||||||
2761 | /// used as a sanity check for .td files (to prevent people from writing stuff | ||||||
2762 | /// that can never possibly work), and to prevent the pattern permuter from | ||||||
2763 | /// generating stuff that is useless. | ||||||
2764 | bool TreePatternNode::canPatternMatch(std::string &Reason, | ||||||
2765 | const CodeGenDAGPatterns &CDP) { | ||||||
2766 | if (isLeaf()) return true; | ||||||
2767 | |||||||
2768 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) | ||||||
2769 | if (!getChild(i)->canPatternMatch(Reason, CDP)) | ||||||
2770 | return false; | ||||||
2771 | |||||||
2772 | // If this is an intrinsic, handle cases that would make it not match. For | ||||||
2773 | // example, if an operand is required to be an immediate. | ||||||
2774 | if (getOperator()->isSubClassOf("Intrinsic")) { | ||||||
2775 | // TODO: | ||||||
2776 | return true; | ||||||
2777 | } | ||||||
2778 | |||||||
2779 | if (getOperator()->isSubClassOf("ComplexPattern")) | ||||||
2780 | return true; | ||||||
2781 | |||||||
2782 | // If this node is a commutative operator, check that the LHS isn't an | ||||||
2783 | // immediate. | ||||||
2784 | const SDNodeInfo &NodeInfo = CDP.getSDNodeInfo(getOperator()); | ||||||
2785 | bool isCommIntrinsic = isCommutativeIntrinsic(CDP); | ||||||
2786 | if (NodeInfo.hasProperty(SDNPCommutative) || isCommIntrinsic) { | ||||||
2787 | // Scan all of the operands of the node and make sure that only the last one | ||||||
2788 | // is a constant node, unless the RHS also is. | ||||||
2789 | if (!OnlyOnRHSOfCommutative(getChild(getNumChildren()-1))) { | ||||||
2790 | unsigned Skip = isCommIntrinsic ? 1 : 0; // First operand is intrinsic id. | ||||||
2791 | for (unsigned i = Skip, e = getNumChildren()-1; i != e; ++i) | ||||||
2792 | if (OnlyOnRHSOfCommutative(getChild(i))) { | ||||||
2793 | Reason="Immediate value must be on the RHS of commutative operators!"; | ||||||
2794 | return false; | ||||||
2795 | } | ||||||
2796 | } | ||||||
2797 | } | ||||||
2798 | |||||||
2799 | return true; | ||||||
2800 | } | ||||||
2801 | |||||||
2802 | //===----------------------------------------------------------------------===// | ||||||
2803 | // TreePattern implementation | ||||||
2804 | // | ||||||
2805 | |||||||
2806 | TreePattern::TreePattern(Record *TheRec, ListInit *RawPat, bool isInput, | ||||||
2807 | CodeGenDAGPatterns &cdp) : TheRecord(TheRec), CDP(cdp), | ||||||
2808 | isInputPattern(isInput), HasError(false), | ||||||
2809 | Infer(*this) { | ||||||
2810 | for (Init *I : RawPat->getValues()) | ||||||
2811 | Trees.push_back(ParseTreePattern(I, "")); | ||||||
2812 | } | ||||||
2813 | |||||||
2814 | TreePattern::TreePattern(Record *TheRec, DagInit *Pat, bool isInput, | ||||||
2815 | CodeGenDAGPatterns &cdp) : TheRecord(TheRec), CDP(cdp), | ||||||
2816 | isInputPattern(isInput), HasError(false), | ||||||
2817 | Infer(*this) { | ||||||
2818 | Trees.push_back(ParseTreePattern(Pat, "")); | ||||||
2819 | } | ||||||
2820 | |||||||
2821 | TreePattern::TreePattern(Record *TheRec, TreePatternNodePtr Pat, bool isInput, | ||||||
2822 | CodeGenDAGPatterns &cdp) | ||||||
2823 | : TheRecord(TheRec), CDP(cdp), isInputPattern(isInput), HasError(false), | ||||||
2824 | Infer(*this) { | ||||||
2825 | Trees.push_back(Pat); | ||||||
2826 | } | ||||||
2827 | |||||||
2828 | void TreePattern::error(const Twine &Msg) { | ||||||
2829 | if (HasError) | ||||||
2830 | return; | ||||||
2831 | dump(); | ||||||
2832 | PrintError(TheRecord->getLoc(), "In " + TheRecord->getName() + ": " + Msg); | ||||||
2833 | HasError = true; | ||||||
2834 | } | ||||||
2835 | |||||||
2836 | void TreePattern::ComputeNamedNodes() { | ||||||
2837 | for (TreePatternNodePtr &Tree : Trees) | ||||||
2838 | ComputeNamedNodes(Tree.get()); | ||||||
2839 | } | ||||||
2840 | |||||||
2841 | void TreePattern::ComputeNamedNodes(TreePatternNode *N) { | ||||||
2842 | if (!N->getName().empty()) | ||||||
2843 | NamedNodes[N->getName()].push_back(N); | ||||||
2844 | |||||||
2845 | for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) | ||||||
2846 | ComputeNamedNodes(N->getChild(i)); | ||||||
2847 | } | ||||||
2848 | |||||||
2849 | TreePatternNodePtr TreePattern::ParseTreePattern(Init *TheInit, | ||||||
2850 | StringRef OpName) { | ||||||
2851 | RecordKeeper &RK = TheInit->getRecordKeeper(); | ||||||
2852 | if (DefInit *DI
| ||||||
2853 | Record *R = DI->getDef(); | ||||||
2854 | |||||||
2855 | // Direct reference to a leaf DagNode or PatFrag? Turn it into a | ||||||
2856 | // TreePatternNode of its own. For example: | ||||||
2857 | /// (foo GPR, imm) -> (foo GPR, (imm)) | ||||||
2858 | if (R->isSubClassOf("SDNode") || R->isSubClassOf("PatFrags")) | ||||||
2859 | return ParseTreePattern( | ||||||
2860 | DagInit::get(DI, nullptr, | ||||||
2861 | std::vector<std::pair<Init*, StringInit*> >()), | ||||||
2862 | OpName); | ||||||
2863 | |||||||
2864 | // Input argument? | ||||||
2865 | TreePatternNodePtr Res = std::make_shared<TreePatternNode>(DI, 1); | ||||||
2866 | if (R->getName() == "node" && !OpName.empty()) { | ||||||
2867 | if (OpName.empty()) | ||||||
2868 | error("'node' argument requires a name to match with operand list"); | ||||||
2869 | Args.push_back(std::string(OpName)); | ||||||
2870 | } | ||||||
2871 | |||||||
2872 | Res->setName(OpName); | ||||||
2873 | return Res; | ||||||
2874 | } | ||||||
2875 | |||||||
2876 | // ?:$name or just $name. | ||||||
2877 | if (isa<UnsetInit>(TheInit)) { | ||||||
2878 | if (OpName.empty()) | ||||||
2879 | error("'?' argument requires a name to match with operand list"); | ||||||
2880 | TreePatternNodePtr Res = std::make_shared<TreePatternNode>(TheInit, 1); | ||||||
2881 | Args.push_back(std::string(OpName)); | ||||||
2882 | Res->setName(OpName); | ||||||
2883 | return Res; | ||||||
2884 | } | ||||||
2885 | |||||||
2886 | if (isa<IntInit>(TheInit) || isa<BitInit>(TheInit)) { | ||||||
2887 | if (!OpName.empty()) | ||||||
2888 | error("Constant int or bit argument should not have a name!"); | ||||||
2889 | if (isa<BitInit>(TheInit)) | ||||||
2890 | TheInit = TheInit->convertInitializerTo(IntRecTy::get(RK)); | ||||||
2891 | return std::make_shared<TreePatternNode>(TheInit, 1); | ||||||
2892 | } | ||||||
2893 | |||||||
2894 | if (BitsInit *BI
| ||||||
2895 | // Turn this into an IntInit. | ||||||
2896 | Init *II = BI->convertInitializerTo(IntRecTy::get(RK)); | ||||||
2897 | if (!II || !isa<IntInit>(II)) | ||||||
2898 | error("Bits value must be constants!"); | ||||||
2899 | return ParseTreePattern(II, OpName); | ||||||
2900 | } | ||||||
2901 | |||||||
2902 | DagInit *Dag = dyn_cast<DagInit>(TheInit); | ||||||
2903 | if (!Dag
| ||||||
2904 | TheInit->print(errs()); | ||||||
2905 | error("Pattern has unexpected init kind!"); | ||||||
2906 | } | ||||||
2907 | DefInit *OpDef = dyn_cast<DefInit>(Dag->getOperator()); | ||||||
| |||||||
2908 | if (!OpDef) error("Pattern has unexpected operator type!"); | ||||||
2909 | Record *Operator = OpDef->getDef(); | ||||||
2910 | |||||||
2911 | if (Operator->isSubClassOf("ValueType")) { | ||||||
2912 | // If the operator is a ValueType, then this must be "type cast" of a leaf | ||||||
2913 | // node. | ||||||
2914 | if (Dag->getNumArgs() != 1) | ||||||
2915 | error("Type cast only takes one operand!"); | ||||||
2916 | |||||||
2917 | TreePatternNodePtr New = | ||||||
2918 | ParseTreePattern(Dag->getArg(0), Dag->getArgNameStr(0)); | ||||||
2919 | |||||||
2920 | // Apply the type cast. | ||||||
2921 | if (New->getNumTypes() != 1) | ||||||
2922 | error("Type cast can only have one type!"); | ||||||
2923 | const CodeGenHwModes &CGH = getDAGPatterns().getTargetInfo().getHwModes(); | ||||||
2924 | New->UpdateNodeType(0, getValueTypeByHwMode(Operator, CGH), *this); | ||||||
2925 | |||||||
2926 | if (!OpName.empty()) | ||||||
2927 | error("ValueType cast should not have a name!"); | ||||||
2928 | return New; | ||||||
2929 | } | ||||||
2930 | |||||||
2931 | // Verify that this is something that makes sense for an operator. | ||||||
2932 | if (!Operator->isSubClassOf("PatFrags") && | ||||||
2933 | !Operator->isSubClassOf("SDNode") && | ||||||
2934 | !Operator->isSubClassOf("Instruction") && | ||||||
2935 | !Operator->isSubClassOf("SDNodeXForm") && | ||||||
2936 | !Operator->isSubClassOf("Intrinsic") && | ||||||
2937 | !Operator->isSubClassOf("ComplexPattern") && | ||||||
2938 | Operator->getName() != "set" && | ||||||
2939 | Operator->getName() != "implicit") | ||||||
2940 | error("Unrecognized node '" + Operator->getName() + "'!"); | ||||||
2941 | |||||||
2942 | // Check to see if this is something that is illegal in an input pattern. | ||||||
2943 | if (isInputPattern) { | ||||||
2944 | if (Operator->isSubClassOf("Instruction") || | ||||||
2945 | Operator->isSubClassOf("SDNodeXForm")) | ||||||
2946 | error("Cannot use '" + Operator->getName() + "' in an input pattern!"); | ||||||
2947 | } else { | ||||||
2948 | if (Operator->isSubClassOf("Intrinsic")) | ||||||
2949 | error("Cannot use '" + Operator->getName() + "' in an output pattern!"); | ||||||
2950 | |||||||
2951 | if (Operator->isSubClassOf("SDNode") && | ||||||
2952 | Operator->getName() != "imm" && | ||||||
2953 | Operator->getName() != "timm" && | ||||||
2954 | Operator->getName() != "fpimm" && | ||||||
2955 | Operator->getName() != "tglobaltlsaddr" && | ||||||
2956 | Operator->getName() != "tconstpool" && | ||||||
2957 | Operator->getName() != "tjumptable" && | ||||||
2958 | Operator->getName() != "tframeindex" && | ||||||
2959 | Operator->getName() != "texternalsym" && | ||||||
2960 | Operator->getName() != "tblockaddress" && | ||||||
2961 | Operator->getName() != "tglobaladdr" && | ||||||
2962 | Operator->getName() != "bb" && | ||||||
2963 | Operator->getName() != "vt" && | ||||||
2964 | Operator->getName() != "mcsym") | ||||||
2965 | error("Cannot use '" + Operator->getName() + "' in an output pattern!"); | ||||||
2966 | } | ||||||
2967 | |||||||
2968 | std::vector<TreePatternNodePtr> Children; | ||||||
2969 | |||||||
2970 | // Parse all the operands. | ||||||
2971 | for (unsigned i = 0, e = Dag->getNumArgs(); i != e; ++i) | ||||||
2972 | Children.push_back(ParseTreePattern(Dag->getArg(i), Dag->getArgNameStr(i))); | ||||||
2973 | |||||||
2974 | // Get the actual number of results before Operator is converted to an intrinsic | ||||||
2975 | // node (which is hard-coded to have either zero or one result). | ||||||
2976 | unsigned NumResults = GetNumNodeResults(Operator, CDP); | ||||||
2977 | |||||||
2978 | // If the operator is an intrinsic, then this is just syntactic sugar for | ||||||
2979 | // (intrinsic_* <number>, ..children..). Pick the right intrinsic node, and | ||||||
2980 | // convert the intrinsic name to a number. | ||||||
2981 | if (Operator->isSubClassOf("Intrinsic")) { | ||||||
2982 | const CodeGenIntrinsic &Int = getDAGPatterns().getIntrinsic(Operator); | ||||||
2983 | unsigned IID = getDAGPatterns().getIntrinsicID(Operator)+1; | ||||||
2984 | |||||||
2985 | // If this intrinsic returns void, it must have side-effects and thus a | ||||||
2986 | // chain. | ||||||
2987 | if (Int.IS.RetVTs.empty()) | ||||||
2988 | Operator = getDAGPatterns().get_intrinsic_void_sdnode(); | ||||||
2989 | else if (!Int.ME.doesNotAccessMemory() || Int.hasSideEffects) | ||||||
2990 | // Has side-effects, requires chain. | ||||||
2991 | Operator = getDAGPatterns().get_intrinsic_w_chain_sdnode(); | ||||||
2992 | else // Otherwise, no chain. | ||||||
2993 | Operator = getDAGPatterns().get_intrinsic_wo_chain_sdnode(); | ||||||
2994 | |||||||
2995 | Children.insert(Children.begin(), std::make_shared<TreePatternNode>( | ||||||
2996 | IntInit::get(RK, IID), 1)); | ||||||
2997 | } | ||||||
2998 | |||||||
2999 | if (Operator->isSubClassOf("ComplexPattern")) { | ||||||
3000 | for (unsigned i = 0; i < Children.size(); ++i) { | ||||||
3001 | TreePatternNodePtr Child = Children[i]; | ||||||
3002 | |||||||
3003 | if (Child->getName().empty()) | ||||||
3004 | error("All arguments to a ComplexPattern must be named"); | ||||||
3005 | |||||||
3006 | // Check that the ComplexPattern uses are consistent: "(MY_PAT $a, $b)" | ||||||
3007 | // and "(MY_PAT $b, $a)" should not be allowed in the same pattern; | ||||||
3008 | // neither should "(MY_PAT_1 $a, $b)" and "(MY_PAT_2 $a, $b)". | ||||||
3009 | auto OperandId = std::make_pair(Operator, i); | ||||||
3010 | auto PrevOp = ComplexPatternOperands.find(Child->getName()); | ||||||
3011 | if (PrevOp != ComplexPatternOperands.end()) { | ||||||
3012 | if (PrevOp->getValue() != OperandId) | ||||||
3013 | error("All ComplexPattern operands must appear consistently: " | ||||||
3014 | "in the same order in just one ComplexPattern instance."); | ||||||
3015 | } else | ||||||
3016 | ComplexPatternOperands[Child->getName()] = OperandId; | ||||||
3017 | } | ||||||
3018 | } | ||||||
3019 | |||||||
3020 | TreePatternNodePtr Result = | ||||||
3021 | std::make_shared<TreePatternNode>(Operator, std::move(Children), | ||||||
3022 | NumResults); | ||||||
3023 | Result->setName(OpName); | ||||||
3024 | |||||||
3025 | if (Dag->getName()) { | ||||||
3026 | assert(Result->getName().empty())(static_cast <bool> (Result->getName().empty()) ? void (0) : __assert_fail ("Result->getName().empty()", "llvm/utils/TableGen/CodeGenDAGPatterns.cpp" , 3026, __extension__ __PRETTY_FUNCTION__)); | ||||||
3027 | Result->setName(Dag->getNameStr()); | ||||||
3028 | } | ||||||
3029 | return Result; | ||||||
3030 | } | ||||||
3031 | |||||||
3032 | /// SimplifyTree - See if we can simplify this tree to eliminate something that | ||||||
3033 | /// will never match in favor of something obvious that will. This is here | ||||||
3034 | /// strictly as a convenience to target authors because it allows them to write | ||||||
3035 | /// more type generic things and have useless type casts fold away. | ||||||
3036 | /// | ||||||
3037 | /// This returns true if any change is made. | ||||||
3038 | static bool SimplifyTree(TreePatternNodePtr &N) { | ||||||
3039 | if (N->isLeaf()) | ||||||
3040 | return false; | ||||||
3041 | |||||||
3042 | // If we have a bitconvert with a resolved type and if the source and | ||||||
3043 | // destination types are the same, then the bitconvert is useless, remove it. | ||||||
3044 | // | ||||||
3045 | // We make an exception if the types are completely empty. This can come up | ||||||
3046 | // when the pattern being simplified is in the Fragments list of a PatFrags, | ||||||
3047 | // so that the operand is just an untyped "node". In that situation we leave | ||||||
3048 | // bitconverts unsimplified, and simplify them later once the fragment is | ||||||
3049 | // expanded into its true context. | ||||||
3050 | if (N->getOperator()->getName() == "bitconvert" && | ||||||
3051 | N->getExtType(0).isValueTypeByHwMode(false) && | ||||||
3052 | !N->getExtType(0).empty() && | ||||||
3053 | N->getExtType(0) == N->getChild(0)->getExtType(0) && | ||||||
3054 | N->getName().empty()) { | ||||||
3055 | N = N->getChildShared(0); | ||||||
3056 | SimplifyTree(N); | ||||||
3057 | return true; | ||||||
3058 | } | ||||||
3059 | |||||||
3060 | // Walk all children. | ||||||
3061 | bool MadeChange = false; | ||||||
3062 | for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) { | ||||||
3063 | TreePatternNodePtr Child = N->getChildShared(i); | ||||||
3064 | MadeChange |= SimplifyTree(Child); | ||||||
3065 | N->setChild(i, std::move(Child)); | ||||||
3066 | } | ||||||
3067 | return MadeChange; | ||||||
3068 | } | ||||||
3069 | |||||||
3070 | |||||||
3071 | |||||||
3072 | /// InferAllTypes - Infer/propagate as many types throughout the expression | ||||||
3073 | /// patterns as possible. Return true if all types are inferred, false | ||||||
3074 | /// otherwise. Flags an error if a type contradiction is found. | ||||||
3075 | bool TreePattern:: | ||||||
3076 | InferAllTypes(const StringMap<SmallVector<TreePatternNode*,1> > *InNamedTypes) { | ||||||
3077 | if (NamedNodes.empty()) | ||||||
3078 | ComputeNamedNodes(); | ||||||
3079 | |||||||
3080 | bool MadeChange = true; | ||||||
3081 | while (MadeChange) { | ||||||
3082 | MadeChange = false; | ||||||
3083 | for (TreePatternNodePtr &Tree : Trees) { | ||||||
3084 | MadeChange |= Tree->ApplyTypeConstraints(*this, false); | ||||||
3085 | MadeChange |= SimplifyTree(Tree); | ||||||
3086 | } | ||||||
3087 | |||||||
3088 | // If there are constraints on our named nodes, apply them. | ||||||
3089 | for (auto &Entry : NamedNodes) { | ||||||
3090 | SmallVectorImpl<TreePatternNode*> &Nodes = Entry.second; | ||||||
3091 | |||||||
3092 | // If we have input named node types, propagate their types to the named | ||||||
3093 | // values here. | ||||||
3094 | if (InNamedTypes) { | ||||||
3095 | if (!InNamedTypes->count(Entry.getKey())) { | ||||||
3096 | error("Node '" + std::string(Entry.getKey()) + | ||||||
3097 | "' in output pattern but not input pattern"); | ||||||
3098 | return true; | ||||||
3099 | } | ||||||
3100 | |||||||
3101 | const SmallVectorImpl<TreePatternNode*> &InNodes = | ||||||
3102 | InNamedTypes->find(Entry.getKey())->second; | ||||||
3103 | |||||||
3104 | // The input types should be fully resolved by now. | ||||||
3105 | for (TreePatternNode *Node : Nodes) { | ||||||
3106 | // If this node is a register class, and it is the root of the pattern | ||||||
3107 | // then we're mapping something onto an input register. We allow | ||||||
3108 | // changing the type of the input register in this case. This allows | ||||||
3109 | // us to match things like: | ||||||
3110 | // def : Pat<(v1i64 (bitconvert(v2i32 DPR:$src))), (v1i64 DPR:$src)>; | ||||||
3111 | if (Node == Trees[0].get() && Node->isLeaf()) { | ||||||
3112 | DefInit *DI = dyn_cast<DefInit>(Node->getLeafValue()); | ||||||
3113 | if (DI && (DI->getDef()->isSubClassOf("RegisterClass") || | ||||||
3114 | DI->getDef()->isSubClassOf("RegisterOperand"))) | ||||||
3115 | continue; | ||||||
3116 | } | ||||||
3117 | |||||||
3118 | assert(Node->getNumTypes() == 1 &&(static_cast <bool> (Node->getNumTypes() == 1 && InNodes[0]->getNumTypes() == 1 && "FIXME: cannot name multiple result nodes yet" ) ? void (0) : __assert_fail ("Node->getNumTypes() == 1 && InNodes[0]->getNumTypes() == 1 && \"FIXME: cannot name multiple result nodes yet\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3120, __extension__ __PRETTY_FUNCTION__)) | ||||||
3119 | InNodes[0]->getNumTypes() == 1 &&(static_cast <bool> (Node->getNumTypes() == 1 && InNodes[0]->getNumTypes() == 1 && "FIXME: cannot name multiple result nodes yet" ) ? void (0) : __assert_fail ("Node->getNumTypes() == 1 && InNodes[0]->getNumTypes() == 1 && \"FIXME: cannot name multiple result nodes yet\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3120, __extension__ __PRETTY_FUNCTION__)) | ||||||
3120 | "FIXME: cannot name multiple result nodes yet")(static_cast <bool> (Node->getNumTypes() == 1 && InNodes[0]->getNumTypes() == 1 && "FIXME: cannot name multiple result nodes yet" ) ? void (0) : __assert_fail ("Node->getNumTypes() == 1 && InNodes[0]->getNumTypes() == 1 && \"FIXME: cannot name multiple result nodes yet\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3120, __extension__ __PRETTY_FUNCTION__)); | ||||||
3121 | MadeChange |= Node->UpdateNodeType(0, InNodes[0]->getExtType(0), | ||||||
3122 | *this); | ||||||
3123 | } | ||||||
3124 | } | ||||||
3125 | |||||||
3126 | // If there are multiple nodes with the same name, they must all have the | ||||||
3127 | // same type. | ||||||
3128 | if (Entry.second.size() > 1) { | ||||||
3129 | for (unsigned i = 0, e = Nodes.size()-1; i != e; ++i) { | ||||||
3130 | TreePatternNode *N1 = Nodes[i], *N2 = Nodes[i+1]; | ||||||
3131 | assert(N1->getNumTypes() == 1 && N2->getNumTypes() == 1 &&(static_cast <bool> (N1->getNumTypes() == 1 && N2->getNumTypes() == 1 && "FIXME: cannot name multiple result nodes yet" ) ? void (0) : __assert_fail ("N1->getNumTypes() == 1 && N2->getNumTypes() == 1 && \"FIXME: cannot name multiple result nodes yet\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3132, __extension__ __PRETTY_FUNCTION__)) | ||||||
3132 | "FIXME: cannot name multiple result nodes yet")(static_cast <bool> (N1->getNumTypes() == 1 && N2->getNumTypes() == 1 && "FIXME: cannot name multiple result nodes yet" ) ? void (0) : __assert_fail ("N1->getNumTypes() == 1 && N2->getNumTypes() == 1 && \"FIXME: cannot name multiple result nodes yet\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3132, __extension__ __PRETTY_FUNCTION__)); | ||||||
3133 | |||||||
3134 | MadeChange |= N1->UpdateNodeType(0, N2->getExtType(0), *this); | ||||||
3135 | MadeChange |= N2->UpdateNodeType(0, N1->getExtType(0), *this); | ||||||
3136 | } | ||||||
3137 | } | ||||||
3138 | } | ||||||
3139 | } | ||||||
3140 | |||||||
3141 | bool HasUnresolvedTypes = false; | ||||||
3142 | for (const TreePatternNodePtr &Tree : Trees) | ||||||
3143 | HasUnresolvedTypes |= Tree->ContainsUnresolvedType(*this); | ||||||
3144 | return !HasUnresolvedTypes; | ||||||
3145 | } | ||||||
3146 | |||||||
3147 | void TreePattern::print(raw_ostream &OS) const { | ||||||
3148 | OS << getRecord()->getName(); | ||||||
3149 | if (!Args.empty()) { | ||||||
3150 | OS << "("; | ||||||
3151 | ListSeparator LS; | ||||||
3152 | for (const std::string &Arg : Args) | ||||||
3153 | OS << LS << Arg; | ||||||
3154 | OS << ")"; | ||||||
3155 | } | ||||||
3156 | OS << ": "; | ||||||
3157 | |||||||
3158 | if (Trees.size() > 1) | ||||||
3159 | OS << "[\n"; | ||||||
3160 | for (const TreePatternNodePtr &Tree : Trees) { | ||||||
3161 | OS << "\t"; | ||||||
3162 | Tree->print(OS); | ||||||
3163 | OS << "\n"; | ||||||
3164 | } | ||||||
3165 | |||||||
3166 | if (Trees.size() > 1) | ||||||
3167 | OS << "]\n"; | ||||||
3168 | } | ||||||
3169 | |||||||
3170 | void TreePattern::dump() const { print(errs()); } | ||||||
3171 | |||||||
3172 | //===----------------------------------------------------------------------===// | ||||||
3173 | // CodeGenDAGPatterns implementation | ||||||
3174 | // | ||||||
3175 | |||||||
3176 | CodeGenDAGPatterns::CodeGenDAGPatterns(RecordKeeper &R, | ||||||
3177 | PatternRewriterFn PatternRewriter) | ||||||
3178 | : Records(R), Target(R), LegalVTS(Target.getLegalValueTypes()), | ||||||
3179 | PatternRewriter(PatternRewriter) { | ||||||
3180 | |||||||
3181 | Intrinsics = CodeGenIntrinsicTable(Records); | ||||||
3182 | ParseNodeInfo(); | ||||||
3183 | ParseNodeTransforms(); | ||||||
3184 | ParseComplexPatterns(); | ||||||
3185 | ParsePatternFragments(); | ||||||
3186 | ParseDefaultOperands(); | ||||||
3187 | ParseInstructions(); | ||||||
3188 | ParsePatternFragments(/*OutFrags*/true); | ||||||
3189 | ParsePatterns(); | ||||||
3190 | |||||||
3191 | // Generate variants. For example, commutative patterns can match | ||||||
3192 | // multiple ways. Add them to PatternsToMatch as well. | ||||||
3193 | GenerateVariants(); | ||||||
3194 | |||||||
3195 | // Break patterns with parameterized types into a series of patterns, | ||||||
3196 | // where each one has a fixed type and is predicated on the conditions | ||||||
3197 | // of the associated HW mode. | ||||||
3198 | ExpandHwModeBasedTypes(); | ||||||
3199 | |||||||
3200 | // Infer instruction flags. For example, we can detect loads, | ||||||
3201 | // stores, and side effects in many cases by examining an | ||||||
3202 | // instruction's pattern. | ||||||
3203 | InferInstructionFlags(); | ||||||
3204 | |||||||
3205 | // Verify that instruction flags match the patterns. | ||||||
3206 | VerifyInstructionFlags(); | ||||||
3207 | } | ||||||
3208 | |||||||
3209 | Record *CodeGenDAGPatterns::getSDNodeNamed(StringRef Name) const { | ||||||
3210 | Record *N = Records.getDef(Name); | ||||||
3211 | if (!N || !N->isSubClassOf("SDNode")) | ||||||
3212 | PrintFatalError("Error getting SDNode '" + Name + "'!"); | ||||||
3213 | |||||||
3214 | return N; | ||||||
3215 | } | ||||||
3216 | |||||||
3217 | // Parse all of the SDNode definitions for the target, populating SDNodes. | ||||||
3218 | void CodeGenDAGPatterns::ParseNodeInfo() { | ||||||
3219 | std::vector<Record*> Nodes = Records.getAllDerivedDefinitions("SDNode"); | ||||||
3220 | const CodeGenHwModes &CGH = getTargetInfo().getHwModes(); | ||||||
3221 | |||||||
3222 | while (!Nodes.empty()) { | ||||||
3223 | Record *R = Nodes.back(); | ||||||
3224 | SDNodes.insert(std::make_pair(R, SDNodeInfo(R, CGH))); | ||||||
3225 | Nodes.pop_back(); | ||||||
3226 | } | ||||||
3227 | |||||||
3228 | // Get the builtin intrinsic nodes. | ||||||
3229 | intrinsic_void_sdnode = getSDNodeNamed("intrinsic_void"); | ||||||
3230 | intrinsic_w_chain_sdnode = getSDNodeNamed("intrinsic_w_chain"); | ||||||
3231 | intrinsic_wo_chain_sdnode = getSDNodeNamed("intrinsic_wo_chain"); | ||||||
3232 | } | ||||||
3233 | |||||||
3234 | /// ParseNodeTransforms - Parse all SDNodeXForm instances into the SDNodeXForms | ||||||
3235 | /// map, and emit them to the file as functions. | ||||||
3236 | void CodeGenDAGPatterns::ParseNodeTransforms() { | ||||||
3237 | std::vector<Record*> Xforms = Records.getAllDerivedDefinitions("SDNodeXForm"); | ||||||
3238 | while (!Xforms.empty()) { | ||||||
3239 | Record *XFormNode = Xforms.back(); | ||||||
3240 | Record *SDNode = XFormNode->getValueAsDef("Opcode"); | ||||||
3241 | StringRef Code = XFormNode->getValueAsString("XFormFunction"); | ||||||
3242 | SDNodeXForms.insert( | ||||||
3243 | std::make_pair(XFormNode, NodeXForm(SDNode, std::string(Code)))); | ||||||
3244 | |||||||
3245 | Xforms.pop_back(); | ||||||
3246 | } | ||||||
3247 | } | ||||||
3248 | |||||||
3249 | void CodeGenDAGPatterns::ParseComplexPatterns() { | ||||||
3250 | std::vector<Record*> AMs = Records.getAllDerivedDefinitions("ComplexPattern"); | ||||||
3251 | while (!AMs.empty()) { | ||||||
3252 | ComplexPatterns.insert(std::make_pair(AMs.back(), AMs.back())); | ||||||
3253 | AMs.pop_back(); | ||||||
3254 | } | ||||||
3255 | } | ||||||
3256 | |||||||
3257 | |||||||
3258 | /// ParsePatternFragments - Parse all of the PatFrag definitions in the .td | ||||||
3259 | /// file, building up the PatternFragments map. After we've collected them all, | ||||||
3260 | /// inline fragments together as necessary, so that there are no references left | ||||||
3261 | /// inside a pattern fragment to a pattern fragment. | ||||||
3262 | /// | ||||||
3263 | void CodeGenDAGPatterns::ParsePatternFragments(bool OutFrags) { | ||||||
3264 | std::vector<Record*> Fragments = Records.getAllDerivedDefinitions("PatFrags"); | ||||||
3265 | |||||||
3266 | // First step, parse all of the fragments. | ||||||
3267 | for (Record *Frag : Fragments) { | ||||||
3268 | if (OutFrags != Frag->isSubClassOf("OutPatFrag")) | ||||||
3269 | continue; | ||||||
3270 | |||||||
3271 | ListInit *LI = Frag->getValueAsListInit("Fragments"); | ||||||
3272 | TreePattern *P = | ||||||
3273 | (PatternFragments[Frag] = std::make_unique<TreePattern>( | ||||||
3274 | Frag, LI, !Frag->isSubClassOf("OutPatFrag"), | ||||||
3275 | *this)).get(); | ||||||
3276 | |||||||
3277 | // Validate the argument list, converting it to set, to discard duplicates. | ||||||
3278 | std::vector<std::string> &Args = P->getArgList(); | ||||||
3279 | // Copy the args so we can take StringRefs to them. | ||||||
3280 | auto ArgsCopy = Args; | ||||||
3281 | SmallDenseSet<StringRef, 4> OperandsSet; | ||||||
3282 | OperandsSet.insert(ArgsCopy.begin(), ArgsCopy.end()); | ||||||
3283 | |||||||
3284 | if (OperandsSet.count("")) | ||||||
3285 | P->error("Cannot have unnamed 'node' values in pattern fragment!"); | ||||||
3286 | |||||||
3287 | // Parse the operands list. | ||||||
3288 | DagInit *OpsList = Frag->getValueAsDag("Operands"); | ||||||
3289 | DefInit *OpsOp = dyn_cast<DefInit>(OpsList->getOperator()); | ||||||
3290 | // Special cases: ops == outs == ins. Different names are used to | ||||||
3291 | // improve readability. | ||||||
3292 | if (!OpsOp || | ||||||
3293 | (OpsOp->getDef()->getName() != "ops" && | ||||||
3294 | OpsOp->getDef()->getName() != "outs" && | ||||||
3295 | OpsOp->getDef()->getName() != "ins")) | ||||||
3296 | P->error("Operands list should start with '(ops ... '!"); | ||||||
3297 | |||||||
3298 | // Copy over the arguments. | ||||||
3299 | Args.clear(); | ||||||
3300 | for (unsigned j = 0, e = OpsList->getNumArgs(); j != e; ++j) { | ||||||
3301 | if (!isa<DefInit>(OpsList->getArg(j)) || | ||||||
3302 | cast<DefInit>(OpsList->getArg(j))->getDef()->getName() != "node") | ||||||
3303 | P->error("Operands list should all be 'node' values."); | ||||||
3304 | if (!OpsList->getArgName(j)) | ||||||
3305 | P->error("Operands list should have names for each operand!"); | ||||||
3306 | StringRef ArgNameStr = OpsList->getArgNameStr(j); | ||||||
3307 | if (!OperandsSet.count(ArgNameStr)) | ||||||
3308 | P->error("'" + ArgNameStr + | ||||||
3309 | "' does not occur in pattern or was multiply specified!"); | ||||||
3310 | OperandsSet.erase(ArgNameStr); | ||||||
3311 | Args.push_back(std::string(ArgNameStr)); | ||||||
3312 | } | ||||||
3313 | |||||||
3314 | if (!OperandsSet.empty()) | ||||||
3315 | P->error("Operands list does not contain an entry for operand '" + | ||||||
3316 | *OperandsSet.begin() + "'!"); | ||||||
3317 | |||||||
3318 | // If there is a node transformation corresponding to this, keep track of | ||||||
3319 | // it. | ||||||
3320 | Record *Transform = Frag->getValueAsDef("OperandTransform"); | ||||||
3321 | if (!getSDNodeTransform(Transform).second.empty()) // not noop xform? | ||||||
3322 | for (const auto &T : P->getTrees()) | ||||||
3323 | T->setTransformFn(Transform); | ||||||
3324 | } | ||||||
3325 | |||||||
3326 | // Now that we've parsed all of the tree fragments, do a closure on them so | ||||||
3327 | // that there are not references to PatFrags left inside of them. | ||||||
3328 | for (Record *Frag : Fragments) { | ||||||
3329 | if (OutFrags != Frag->isSubClassOf("OutPatFrag")) | ||||||
3330 | continue; | ||||||
3331 | |||||||
3332 | TreePattern &ThePat = *PatternFragments[Frag]; | ||||||
3333 | ThePat.InlinePatternFragments(); | ||||||
3334 | |||||||
3335 | // Infer as many types as possible. Don't worry about it if we don't infer | ||||||
3336 | // all of them, some may depend on the inputs of the pattern. Also, don't | ||||||
3337 | // validate type sets; validation may cause spurious failures e.g. if a | ||||||
3338 | // fragment needs floating-point types but the current target does not have | ||||||
3339 | // any (this is only an error if that fragment is ever used!). | ||||||
3340 | { | ||||||
3341 | TypeInfer::SuppressValidation SV(ThePat.getInfer()); | ||||||
3342 | ThePat.InferAllTypes(); | ||||||
3343 | ThePat.resetError(); | ||||||
3344 | } | ||||||
3345 | |||||||
3346 | // If debugging, print out the pattern fragment result. | ||||||
3347 | LLVM_DEBUG(ThePat.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { ThePat.dump(); } } while (false); | ||||||
3348 | } | ||||||
3349 | } | ||||||
3350 | |||||||
3351 | void CodeGenDAGPatterns::ParseDefaultOperands() { | ||||||
3352 | std::vector<Record*> DefaultOps; | ||||||
3353 | DefaultOps = Records.getAllDerivedDefinitions("OperandWithDefaultOps"); | ||||||
3354 | |||||||
3355 | // Find some SDNode. | ||||||
3356 | assert(!SDNodes.empty() && "No SDNodes parsed?")(static_cast <bool> (!SDNodes.empty() && "No SDNodes parsed?" ) ? void (0) : __assert_fail ("!SDNodes.empty() && \"No SDNodes parsed?\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3356, __extension__ __PRETTY_FUNCTION__)); | ||||||
3357 | Init *SomeSDNode = DefInit::get(SDNodes.begin()->first); | ||||||
3358 | |||||||
3359 | for (unsigned i = 0, e = DefaultOps.size(); i != e; ++i) { | ||||||
3360 | DagInit *DefaultInfo = DefaultOps[i]->getValueAsDag("DefaultOps"); | ||||||
3361 | |||||||
3362 | // Clone the DefaultInfo dag node, changing the operator from 'ops' to | ||||||
3363 | // SomeSDnode so that we can parse this. | ||||||
3364 | std::vector<std::pair<Init*, StringInit*> > Ops; | ||||||
3365 | for (unsigned op = 0, e = DefaultInfo->getNumArgs(); op != e; ++op) | ||||||
3366 | Ops.push_back(std::make_pair(DefaultInfo->getArg(op), | ||||||
3367 | DefaultInfo->getArgName(op))); | ||||||
3368 | DagInit *DI = DagInit::get(SomeSDNode, nullptr, Ops); | ||||||
3369 | |||||||
3370 | // Create a TreePattern to parse this. | ||||||
3371 | TreePattern P(DefaultOps[i], DI, false, *this); | ||||||
3372 | assert(P.getNumTrees() == 1 && "This ctor can only produce one tree!")(static_cast <bool> (P.getNumTrees() == 1 && "This ctor can only produce one tree!" ) ? void (0) : __assert_fail ("P.getNumTrees() == 1 && \"This ctor can only produce one tree!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3372, __extension__ __PRETTY_FUNCTION__)); | ||||||
3373 | |||||||
3374 | // Copy the operands over into a DAGDefaultOperand. | ||||||
3375 | DAGDefaultOperand DefaultOpInfo; | ||||||
3376 | |||||||
3377 | const TreePatternNodePtr &T = P.getTree(0); | ||||||
3378 | for (unsigned op = 0, e = T->getNumChildren(); op != e; ++op) { | ||||||
3379 | TreePatternNodePtr TPN = T->getChildShared(op); | ||||||
3380 | while (TPN->ApplyTypeConstraints(P, false)) | ||||||
3381 | /* Resolve all types */; | ||||||
3382 | |||||||
3383 | if (TPN->ContainsUnresolvedType(P)) { | ||||||
3384 | PrintFatalError("Value #" + Twine(i) + " of OperandWithDefaultOps '" + | ||||||
3385 | DefaultOps[i]->getName() + | ||||||
3386 | "' doesn't have a concrete type!"); | ||||||
3387 | } | ||||||
3388 | DefaultOpInfo.DefaultOps.push_back(std::move(TPN)); | ||||||
3389 | } | ||||||
3390 | |||||||
3391 | // Insert it into the DefaultOperands map so we can find it later. | ||||||
3392 | DefaultOperands[DefaultOps[i]] = DefaultOpInfo; | ||||||
3393 | } | ||||||
3394 | } | ||||||
3395 | |||||||
3396 | /// HandleUse - Given "Pat" a leaf in the pattern, check to see if it is an | ||||||
3397 | /// instruction input. Return true if this is a real use. | ||||||
3398 | static bool HandleUse(TreePattern &I, TreePatternNodePtr Pat, | ||||||
3399 | std::map<std::string, TreePatternNodePtr> &InstInputs) { | ||||||
3400 | // No name -> not interesting. | ||||||
3401 | if (Pat->getName().empty()) { | ||||||
3402 | if (Pat->isLeaf()) { | ||||||
3403 | DefInit *DI = dyn_cast<DefInit>(Pat->getLeafValue()); | ||||||
3404 | if (DI && (DI->getDef()->isSubClassOf("RegisterClass") || | ||||||
3405 | DI->getDef()->isSubClassOf("RegisterOperand"))) | ||||||
3406 | I.error("Input " + DI->getDef()->getName() + " must be named!"); | ||||||
3407 | } | ||||||
3408 | return false; | ||||||
3409 | } | ||||||
3410 | |||||||
3411 | Record *Rec; | ||||||
3412 | if (Pat->isLeaf()) { | ||||||
3413 | DefInit *DI = dyn_cast<DefInit>(Pat->getLeafValue()); | ||||||
3414 | if (!DI) | ||||||
3415 | I.error("Input $" + Pat->getName() + " must be an identifier!"); | ||||||
3416 | Rec = DI->getDef(); | ||||||
3417 | } else { | ||||||
3418 | Rec = Pat->getOperator(); | ||||||
3419 | } | ||||||
3420 | |||||||
3421 | // SRCVALUE nodes are ignored. | ||||||
3422 | if (Rec->getName() == "srcvalue") | ||||||
3423 | return false; | ||||||
3424 | |||||||
3425 | TreePatternNodePtr &Slot = InstInputs[Pat->getName()]; | ||||||
3426 | if (!Slot) { | ||||||
3427 | Slot = Pat; | ||||||
3428 | return true; | ||||||
3429 | } | ||||||
3430 | Record *SlotRec; | ||||||
3431 | if (Slot->isLeaf()) { | ||||||
3432 | SlotRec = cast<DefInit>(Slot->getLeafValue())->getDef(); | ||||||
3433 | } else { | ||||||
3434 | assert(Slot->getNumChildren() == 0 && "can't be a use with children!")(static_cast <bool> (Slot->getNumChildren() == 0 && "can't be a use with children!") ? void (0) : __assert_fail ( "Slot->getNumChildren() == 0 && \"can't be a use with children!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3434, __extension__ __PRETTY_FUNCTION__)); | ||||||
3435 | SlotRec = Slot->getOperator(); | ||||||
3436 | } | ||||||
3437 | |||||||
3438 | // Ensure that the inputs agree if we've already seen this input. | ||||||
3439 | if (Rec != SlotRec) | ||||||
3440 | I.error("All $" + Pat->getName() + " inputs must agree with each other"); | ||||||
3441 | // Ensure that the types can agree as well. | ||||||
3442 | Slot->UpdateNodeType(0, Pat->getExtType(0), I); | ||||||
3443 | Pat->UpdateNodeType(0, Slot->getExtType(0), I); | ||||||
3444 | if (Slot->getExtTypes() != Pat->getExtTypes()) | ||||||
3445 | I.error("All $" + Pat->getName() + " inputs must agree with each other"); | ||||||
3446 | return true; | ||||||
3447 | } | ||||||
3448 | |||||||
3449 | /// FindPatternInputsAndOutputs - Scan the specified TreePatternNode (which is | ||||||
3450 | /// part of "I", the instruction), computing the set of inputs and outputs of | ||||||
3451 | /// the pattern. Report errors if we see anything naughty. | ||||||
3452 | void CodeGenDAGPatterns::FindPatternInputsAndOutputs( | ||||||
3453 | TreePattern &I, TreePatternNodePtr Pat, | ||||||
3454 | std::map<std::string, TreePatternNodePtr> &InstInputs, | ||||||
3455 | MapVector<std::string, TreePatternNodePtr, std::map<std::string, unsigned>> | ||||||
3456 | &InstResults, | ||||||
3457 | std::vector<Record *> &InstImpResults) { | ||||||
3458 | |||||||
3459 | // The instruction pattern still has unresolved fragments. For *named* | ||||||
3460 | // nodes we must resolve those here. This may not result in multiple | ||||||
3461 | // alternatives. | ||||||
3462 | if (!Pat->getName().empty()) { | ||||||
3463 | TreePattern SrcPattern(I.getRecord(), Pat, true, *this); | ||||||
3464 | SrcPattern.InlinePatternFragments(); | ||||||
3465 | SrcPattern.InferAllTypes(); | ||||||
3466 | Pat = SrcPattern.getOnlyTree(); | ||||||
3467 | } | ||||||
3468 | |||||||
3469 | if (Pat->isLeaf()) { | ||||||
3470 | bool isUse = HandleUse(I, Pat, InstInputs); | ||||||
3471 | if (!isUse && Pat->getTransformFn()) | ||||||
3472 | I.error("Cannot specify a transform function for a non-input value!"); | ||||||
3473 | return; | ||||||
3474 | } | ||||||
3475 | |||||||
3476 | if (Pat->getOperator()->getName() == "implicit") { | ||||||
3477 | for (unsigned i = 0, e = Pat->getNumChildren(); i != e; ++i) { | ||||||
3478 | TreePatternNode *Dest = Pat->getChild(i); | ||||||
3479 | if (!Dest->isLeaf()) | ||||||
3480 | I.error("implicitly defined value should be a register!"); | ||||||
3481 | |||||||
3482 | DefInit *Val = dyn_cast<DefInit>(Dest->getLeafValue()); | ||||||
3483 | if (!Val || !Val->getDef()->isSubClassOf("Register")) | ||||||
3484 | I.error("implicitly defined value should be a register!"); | ||||||
3485 | InstImpResults.push_back(Val->getDef()); | ||||||
3486 | } | ||||||
3487 | return; | ||||||
3488 | } | ||||||
3489 | |||||||
3490 | if (Pat->getOperator()->getName() != "set") { | ||||||
3491 | // If this is not a set, verify that the children nodes are not void typed, | ||||||
3492 | // and recurse. | ||||||
3493 | for (unsigned i = 0, e = Pat->getNumChildren(); i != e; ++i) { | ||||||
3494 | if (Pat->getChild(i)->getNumTypes() == 0) | ||||||
3495 | I.error("Cannot have void nodes inside of patterns!"); | ||||||
3496 | FindPatternInputsAndOutputs(I, Pat->getChildShared(i), InstInputs, | ||||||
3497 | InstResults, InstImpResults); | ||||||
3498 | } | ||||||
3499 | |||||||
3500 | // If this is a non-leaf node with no children, treat it basically as if | ||||||
3501 | // it were a leaf. This handles nodes like (imm). | ||||||
3502 | bool isUse = HandleUse(I, Pat, InstInputs); | ||||||
3503 | |||||||
3504 | if (!isUse && Pat->getTransformFn()) | ||||||
3505 | I.error("Cannot specify a transform function for a non-input value!"); | ||||||
3506 | return; | ||||||
3507 | } | ||||||
3508 | |||||||
3509 | // Otherwise, this is a set, validate and collect instruction results. | ||||||
3510 | if (Pat->getNumChildren() == 0) | ||||||
3511 | I.error("set requires operands!"); | ||||||
3512 | |||||||
3513 | if (Pat->getTransformFn()) | ||||||
3514 | I.error("Cannot specify a transform function on a set node!"); | ||||||
3515 | |||||||
3516 | // Check the set destinations. | ||||||
3517 | unsigned NumDests = Pat->getNumChildren()-1; | ||||||
3518 | for (unsigned i = 0; i != NumDests; ++i) { | ||||||
3519 | TreePatternNodePtr Dest = Pat->getChildShared(i); | ||||||
3520 | // For set destinations we also must resolve fragments here. | ||||||
3521 | TreePattern DestPattern(I.getRecord(), Dest, false, *this); | ||||||
3522 | DestPattern.InlinePatternFragments(); | ||||||
3523 | DestPattern.InferAllTypes(); | ||||||
3524 | Dest = DestPattern.getOnlyTree(); | ||||||
3525 | |||||||
3526 | if (!Dest->isLeaf()) | ||||||
3527 | I.error("set destination should be a register!"); | ||||||
3528 | |||||||
3529 | DefInit *Val = dyn_cast<DefInit>(Dest->getLeafValue()); | ||||||
3530 | if (!Val) { | ||||||
3531 | I.error("set destination should be a register!"); | ||||||
3532 | continue; | ||||||
3533 | } | ||||||
3534 | |||||||
3535 | if (Val->getDef()->isSubClassOf("RegisterClass") || | ||||||
3536 | Val->getDef()->isSubClassOf("ValueType") || | ||||||
3537 | Val->getDef()->isSubClassOf("RegisterOperand") || | ||||||
3538 | Val->getDef()->isSubClassOf("PointerLikeRegClass")) { | ||||||
3539 | if (Dest->getName().empty()) | ||||||
3540 | I.error("set destination must have a name!"); | ||||||
3541 | if (InstResults.count(Dest->getName())) | ||||||
3542 | I.error("cannot set '" + Dest->getName() + "' multiple times"); | ||||||
3543 | InstResults[Dest->getName()] = Dest; | ||||||
3544 | } else if (Val->getDef()->isSubClassOf("Register")) { | ||||||
3545 | InstImpResults.push_back(Val->getDef()); | ||||||
3546 | } else { | ||||||
3547 | I.error("set destination should be a register!"); | ||||||
3548 | } | ||||||
3549 | } | ||||||
3550 | |||||||
3551 | // Verify and collect info from the computation. | ||||||
3552 | FindPatternInputsAndOutputs(I, Pat->getChildShared(NumDests), InstInputs, | ||||||
3553 | InstResults, InstImpResults); | ||||||
3554 | } | ||||||
3555 | |||||||
3556 | //===----------------------------------------------------------------------===// | ||||||
3557 | // Instruction Analysis | ||||||
3558 | //===----------------------------------------------------------------------===// | ||||||
3559 | |||||||
3560 | class InstAnalyzer { | ||||||
3561 | const CodeGenDAGPatterns &CDP; | ||||||
3562 | public: | ||||||
3563 | bool hasSideEffects; | ||||||
3564 | bool mayStore; | ||||||
3565 | bool mayLoad; | ||||||
3566 | bool isBitcast; | ||||||
3567 | bool isVariadic; | ||||||
3568 | bool hasChain; | ||||||
3569 | |||||||
3570 | InstAnalyzer(const CodeGenDAGPatterns &cdp) | ||||||
3571 | : CDP(cdp), hasSideEffects(false), mayStore(false), mayLoad(false), | ||||||
3572 | isBitcast(false), isVariadic(false), hasChain(false) {} | ||||||
3573 | |||||||
3574 | void Analyze(const PatternToMatch &Pat) { | ||||||
3575 | const TreePatternNode *N = Pat.getSrcPattern(); | ||||||
3576 | AnalyzeNode(N); | ||||||
3577 | // These properties are detected only on the root node. | ||||||
3578 | isBitcast = IsNodeBitcast(N); | ||||||
3579 | } | ||||||
3580 | |||||||
3581 | private: | ||||||
3582 | bool IsNodeBitcast(const TreePatternNode *N) const { | ||||||
3583 | if (hasSideEffects || mayLoad || mayStore || isVariadic) | ||||||
3584 | return false; | ||||||
3585 | |||||||
3586 | if (N->isLeaf()) | ||||||
3587 | return false; | ||||||
3588 | if (N->getNumChildren() != 1 || !N->getChild(0)->isLeaf()) | ||||||
3589 | return false; | ||||||
3590 | |||||||
3591 | if (N->getOperator()->isSubClassOf("ComplexPattern")) | ||||||
3592 | return false; | ||||||
3593 | |||||||
3594 | const SDNodeInfo &OpInfo = CDP.getSDNodeInfo(N->getOperator()); | ||||||
3595 | if (OpInfo.getNumResults() != 1 || OpInfo.getNumOperands() != 1) | ||||||
3596 | return false; | ||||||
3597 | return OpInfo.getEnumName() == "ISD::BITCAST"; | ||||||
3598 | } | ||||||
3599 | |||||||
3600 | public: | ||||||
3601 | void AnalyzeNode(const TreePatternNode *N) { | ||||||
3602 | if (N->isLeaf()) { | ||||||
3603 | if (DefInit *DI = dyn_cast<DefInit>(N->getLeafValue())) { | ||||||
3604 | Record *LeafRec = DI->getDef(); | ||||||
3605 | // Handle ComplexPattern leaves. | ||||||
3606 | if (LeafRec->isSubClassOf("ComplexPattern")) { | ||||||
3607 | const ComplexPattern &CP = CDP.getComplexPattern(LeafRec); | ||||||
3608 | if (CP.hasProperty(SDNPMayStore)) mayStore = true; | ||||||
3609 | if (CP.hasProperty(SDNPMayLoad)) mayLoad = true; | ||||||
3610 | if (CP.hasProperty(SDNPSideEffect)) hasSideEffects = true; | ||||||
3611 | } | ||||||
3612 | } | ||||||
3613 | return; | ||||||
3614 | } | ||||||
3615 | |||||||
3616 | // Analyze children. | ||||||
3617 | for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) | ||||||
3618 | AnalyzeNode(N->getChild(i)); | ||||||
3619 | |||||||
3620 | // Notice properties of the node. | ||||||
3621 | if (N->NodeHasProperty(SDNPMayStore, CDP)) mayStore = true; | ||||||
3622 | if (N->NodeHasProperty(SDNPMayLoad, CDP)) mayLoad = true; | ||||||
3623 | if (N->NodeHasProperty(SDNPSideEffect, CDP)) hasSideEffects = true; | ||||||
3624 | if (N->NodeHasProperty(SDNPVariadic, CDP)) isVariadic = true; | ||||||
3625 | if (N->NodeHasProperty(SDNPHasChain, CDP)) hasChain = true; | ||||||
3626 | |||||||
3627 | if (const CodeGenIntrinsic *IntInfo = N->getIntrinsicInfo(CDP)) { | ||||||
3628 | ModRefInfo MR = IntInfo->ME.getModRef(); | ||||||
3629 | // If this is an intrinsic, analyze it. | ||||||
3630 | if (isRefSet(MR)) | ||||||
3631 | mayLoad = true; // These may load memory. | ||||||
3632 | |||||||
3633 | if (isModSet(MR)) | ||||||
3634 | mayStore = true; // Intrinsics that can write to memory are 'mayStore'. | ||||||
3635 | |||||||
3636 | // Consider intrinsics that don't specify any restrictions on memory | ||||||
3637 | // effects as having a side-effect. | ||||||
3638 | if (IntInfo->ME == MemoryEffects::unknown() || IntInfo->hasSideEffects) | ||||||
3639 | hasSideEffects = true; | ||||||
3640 | } | ||||||
3641 | } | ||||||
3642 | |||||||
3643 | }; | ||||||
3644 | |||||||
3645 | static bool InferFromPattern(CodeGenInstruction &InstInfo, | ||||||
3646 | const InstAnalyzer &PatInfo, | ||||||
3647 | Record *PatDef) { | ||||||
3648 | bool Error = false; | ||||||
3649 | |||||||
3650 | // Remember where InstInfo got its flags. | ||||||
3651 | if (InstInfo.hasUndefFlags()) | ||||||
3652 | InstInfo.InferredFrom = PatDef; | ||||||
3653 | |||||||
3654 | // Check explicitly set flags for consistency. | ||||||
3655 | if (InstInfo.hasSideEffects != PatInfo.hasSideEffects && | ||||||
3656 | !InstInfo.hasSideEffects_Unset) { | ||||||
3657 | // Allow explicitly setting hasSideEffects = 1 on instructions, even when | ||||||
3658 | // the pattern has no side effects. That could be useful for div/rem | ||||||
3659 | // instructions that may trap. | ||||||
3660 | if (!InstInfo.hasSideEffects) { | ||||||
3661 | Error = true; | ||||||
3662 | PrintError(PatDef->getLoc(), "Pattern doesn't match hasSideEffects = " + | ||||||
3663 | Twine(InstInfo.hasSideEffects)); | ||||||
3664 | } | ||||||
3665 | } | ||||||
3666 | |||||||
3667 | if (InstInfo.mayStore != PatInfo.mayStore && !InstInfo.mayStore_Unset) { | ||||||
3668 | Error = true; | ||||||
3669 | PrintError(PatDef->getLoc(), "Pattern doesn't match mayStore = " + | ||||||
3670 | Twine(InstInfo.mayStore)); | ||||||
3671 | } | ||||||
3672 | |||||||
3673 | if (InstInfo.mayLoad != PatInfo.mayLoad && !InstInfo.mayLoad_Unset) { | ||||||
3674 | // Allow explicitly setting mayLoad = 1, even when the pattern has no loads. | ||||||
3675 | // Some targets translate immediates to loads. | ||||||
3676 | if (!InstInfo.mayLoad) { | ||||||
3677 | Error = true; | ||||||
3678 | PrintError(PatDef->getLoc(), "Pattern doesn't match mayLoad = " + | ||||||
3679 | Twine(InstInfo.mayLoad)); | ||||||
3680 | } | ||||||
3681 | } | ||||||
3682 | |||||||
3683 | // Transfer inferred flags. | ||||||
3684 | InstInfo.hasSideEffects |= PatInfo.hasSideEffects; | ||||||
3685 | InstInfo.mayStore |= PatInfo.mayStore; | ||||||
3686 | InstInfo.mayLoad |= PatInfo.mayLoad; | ||||||
3687 | |||||||
3688 | // These flags are silently added without any verification. | ||||||
3689 | // FIXME: To match historical behavior of TableGen, for now add those flags | ||||||
3690 | // only when we're inferring from the primary instruction pattern. | ||||||
3691 | if (PatDef->isSubClassOf("Instruction")) { | ||||||
3692 | InstInfo.isBitcast |= PatInfo.isBitcast; | ||||||
3693 | InstInfo.hasChain |= PatInfo.hasChain; | ||||||
3694 | InstInfo.hasChain_Inferred = true; | ||||||
3695 | } | ||||||
3696 | |||||||
3697 | // Don't infer isVariadic. This flag means something different on SDNodes and | ||||||
3698 | // instructions. For example, a CALL SDNode is variadic because it has the | ||||||
3699 | // call arguments as operands, but a CALL instruction is not variadic - it | ||||||
3700 | // has argument registers as implicit, not explicit uses. | ||||||
3701 | |||||||
3702 | return Error; | ||||||
3703 | } | ||||||
3704 | |||||||
3705 | /// hasNullFragReference - Return true if the DAG has any reference to the | ||||||
3706 | /// null_frag operator. | ||||||
3707 | static bool hasNullFragReference(DagInit *DI) { | ||||||
3708 | DefInit *OpDef = dyn_cast<DefInit>(DI->getOperator()); | ||||||
3709 | if (!OpDef) return false; | ||||||
3710 | Record *Operator = OpDef->getDef(); | ||||||
3711 | |||||||
3712 | // If this is the null fragment, return true. | ||||||
3713 | if (Operator->getName() == "null_frag") return true; | ||||||
3714 | // If any of the arguments reference the null fragment, return true. | ||||||
3715 | for (unsigned i = 0, e = DI->getNumArgs(); i != e; ++i) { | ||||||
3716 | if (auto Arg = dyn_cast<DefInit>(DI->getArg(i))) | ||||||
3717 | if (Arg->getDef()->getName() == "null_frag") | ||||||
3718 | return true; | ||||||
3719 | DagInit *Arg = dyn_cast<DagInit>(DI->getArg(i)); | ||||||
3720 | if (Arg && hasNullFragReference(Arg)) | ||||||
3721 | return true; | ||||||
3722 | } | ||||||
3723 | |||||||
3724 | return false; | ||||||
3725 | } | ||||||
3726 | |||||||
3727 | /// hasNullFragReference - Return true if any DAG in the list references | ||||||
3728 | /// the null_frag operator. | ||||||
3729 | static bool hasNullFragReference(ListInit *LI) { | ||||||
3730 | for (Init *I : LI->getValues()) { | ||||||
3731 | DagInit *DI = dyn_cast<DagInit>(I); | ||||||
3732 | assert(DI && "non-dag in an instruction Pattern list?!")(static_cast <bool> (DI && "non-dag in an instruction Pattern list?!" ) ? void (0) : __assert_fail ("DI && \"non-dag in an instruction Pattern list?!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3732, __extension__ __PRETTY_FUNCTION__)); | ||||||
3733 | if (hasNullFragReference(DI)) | ||||||
3734 | return true; | ||||||
3735 | } | ||||||
3736 | return false; | ||||||
3737 | } | ||||||
3738 | |||||||
3739 | /// Get all the instructions in a tree. | ||||||
3740 | static void | ||||||
3741 | getInstructionsInTree(TreePatternNode *Tree, SmallVectorImpl<Record*> &Instrs) { | ||||||
3742 | if (Tree->isLeaf()) | ||||||
3743 | return; | ||||||
3744 | if (Tree->getOperator()->isSubClassOf("Instruction")) | ||||||
3745 | Instrs.push_back(Tree->getOperator()); | ||||||
3746 | for (unsigned i = 0, e = Tree->getNumChildren(); i != e; ++i) | ||||||
3747 | getInstructionsInTree(Tree->getChild(i), Instrs); | ||||||
3748 | } | ||||||
3749 | |||||||
3750 | /// Check the class of a pattern leaf node against the instruction operand it | ||||||
3751 | /// represents. | ||||||
3752 | static bool checkOperandClass(CGIOperandList::OperandInfo &OI, | ||||||
3753 | Record *Leaf) { | ||||||
3754 | if (OI.Rec == Leaf) | ||||||
3755 | return true; | ||||||
3756 | |||||||
3757 | // Allow direct value types to be used in instruction set patterns. | ||||||
3758 | // The type will be checked later. | ||||||
3759 | if (Leaf->isSubClassOf("ValueType")) | ||||||
3760 | return true; | ||||||
3761 | |||||||
3762 | // Patterns can also be ComplexPattern instances. | ||||||
3763 | if (Leaf->isSubClassOf("ComplexPattern")) | ||||||
3764 | return true; | ||||||
3765 | |||||||
3766 | return false; | ||||||
3767 | } | ||||||
3768 | |||||||
3769 | void CodeGenDAGPatterns::parseInstructionPattern( | ||||||
3770 | CodeGenInstruction &CGI, ListInit *Pat, DAGInstMap &DAGInsts) { | ||||||
3771 | |||||||
3772 | assert(!DAGInsts.count(CGI.TheDef) && "Instruction already parsed!")(static_cast <bool> (!DAGInsts.count(CGI.TheDef) && "Instruction already parsed!") ? void (0) : __assert_fail ("!DAGInsts.count(CGI.TheDef) && \"Instruction already parsed!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3772, __extension__ __PRETTY_FUNCTION__)); | ||||||
3773 | |||||||
3774 | // Parse the instruction. | ||||||
3775 | TreePattern I(CGI.TheDef, Pat, true, *this); | ||||||
3776 | |||||||
3777 | // InstInputs - Keep track of all of the inputs of the instruction, along | ||||||
3778 | // with the record they are declared as. | ||||||
3779 | std::map<std::string, TreePatternNodePtr> InstInputs; | ||||||
3780 | |||||||
3781 | // InstResults - Keep track of all the virtual registers that are 'set' | ||||||
3782 | // in the instruction, including what reg class they are. | ||||||
3783 | MapVector<std::string, TreePatternNodePtr, std::map<std::string, unsigned>> | ||||||
3784 | InstResults; | ||||||
3785 | |||||||
3786 | std::vector<Record*> InstImpResults; | ||||||
3787 | |||||||
3788 | // Verify that the top-level forms in the instruction are of void type, and | ||||||
3789 | // fill in the InstResults map. | ||||||
3790 | SmallString<32> TypesString; | ||||||
3791 | for (unsigned j = 0, e = I.getNumTrees(); j != e; ++j) { | ||||||
3792 | TypesString.clear(); | ||||||
3793 | TreePatternNodePtr Pat = I.getTree(j); | ||||||
3794 | if (Pat->getNumTypes() != 0) { | ||||||
3795 | raw_svector_ostream OS(TypesString); | ||||||
3796 | ListSeparator LS; | ||||||
3797 | for (unsigned k = 0, ke = Pat->getNumTypes(); k != ke; ++k) { | ||||||
3798 | OS << LS; | ||||||
3799 | Pat->getExtType(k).writeToStream(OS); | ||||||
3800 | } | ||||||
3801 | I.error("Top-level forms in instruction pattern should have" | ||||||
3802 | " void types, has types " + | ||||||
3803 | OS.str()); | ||||||
3804 | } | ||||||
3805 | |||||||
3806 | // Find inputs and outputs, and verify the structure of the uses/defs. | ||||||
3807 | FindPatternInputsAndOutputs(I, Pat, InstInputs, InstResults, | ||||||
3808 | InstImpResults); | ||||||
3809 | } | ||||||
3810 | |||||||
3811 | // Now that we have inputs and outputs of the pattern, inspect the operands | ||||||
3812 | // list for the instruction. This determines the order that operands are | ||||||
3813 | // added to the machine instruction the node corresponds to. | ||||||
3814 | unsigned NumResults = InstResults.size(); | ||||||
3815 | |||||||
3816 | // Parse the operands list from the (ops) list, validating it. | ||||||
3817 | assert(I.getArgList().empty() && "Args list should still be empty here!")(static_cast <bool> (I.getArgList().empty() && "Args list should still be empty here!" ) ? void (0) : __assert_fail ("I.getArgList().empty() && \"Args list should still be empty here!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3817, __extension__ __PRETTY_FUNCTION__)); | ||||||
3818 | |||||||
3819 | // Check that all of the results occur first in the list. | ||||||
3820 | std::vector<Record*> Results; | ||||||
3821 | std::vector<unsigned> ResultIndices; | ||||||
3822 | SmallVector<TreePatternNodePtr, 2> ResNodes; | ||||||
3823 | for (unsigned i = 0; i != NumResults; ++i) { | ||||||
3824 | if (i == CGI.Operands.size()) { | ||||||
3825 | const std::string &OpName = | ||||||
3826 | llvm::find_if( | ||||||
3827 | InstResults, | ||||||
3828 | [](const std::pair<std::string, TreePatternNodePtr> &P) { | ||||||
3829 | return P.second; | ||||||
3830 | }) | ||||||
3831 | ->first; | ||||||
3832 | |||||||
3833 | I.error("'" + OpName + "' set but does not appear in operand list!"); | ||||||
3834 | } | ||||||
3835 | |||||||
3836 | const std::string &OpName = CGI.Operands[i].Name; | ||||||
3837 | |||||||
3838 | // Check that it exists in InstResults. | ||||||
3839 | auto InstResultIter = InstResults.find(OpName); | ||||||
3840 | if (InstResultIter == InstResults.end() || !InstResultIter->second) | ||||||
3841 | I.error("Operand $" + OpName + " does not exist in operand list!"); | ||||||
3842 | |||||||
3843 | TreePatternNodePtr RNode = InstResultIter->second; | ||||||
3844 | Record *R = cast<DefInit>(RNode->getLeafValue())->getDef(); | ||||||
3845 | ResNodes.push_back(std::move(RNode)); | ||||||
3846 | if (!R) | ||||||
3847 | I.error("Operand $" + OpName + " should be a set destination: all " | ||||||
3848 | "outputs must occur before inputs in operand list!"); | ||||||
3849 | |||||||
3850 | if (!checkOperandClass(CGI.Operands[i], R)) | ||||||
3851 | I.error("Operand $" + OpName + " class mismatch!"); | ||||||
3852 | |||||||
3853 | // Remember the return type. | ||||||
3854 | Results.push_back(CGI.Operands[i].Rec); | ||||||
3855 | |||||||
3856 | // Remember the result index. | ||||||
3857 | ResultIndices.push_back(std::distance(InstResults.begin(), InstResultIter)); | ||||||
3858 | |||||||
3859 | // Okay, this one checks out. | ||||||
3860 | InstResultIter->second = nullptr; | ||||||
3861 | } | ||||||
3862 | |||||||
3863 | // Loop over the inputs next. | ||||||
3864 | std::vector<TreePatternNodePtr> ResultNodeOperands; | ||||||
3865 | std::vector<Record*> Operands; | ||||||
3866 | for (unsigned i = NumResults, e = CGI.Operands.size(); i != e; ++i) { | ||||||
3867 | CGIOperandList::OperandInfo &Op = CGI.Operands[i]; | ||||||
3868 | const std::string &OpName = Op.Name; | ||||||
3869 | if (OpName.empty()) | ||||||
3870 | I.error("Operand #" + Twine(i) + " in operands list has no name!"); | ||||||
3871 | |||||||
3872 | if (!InstInputs.count(OpName)) { | ||||||
3873 | // If this is an operand with a DefaultOps set filled in, we can ignore | ||||||
3874 | // this. When we codegen it, we will do so as always executed. | ||||||
3875 | if (Op.Rec->isSubClassOf("OperandWithDefaultOps")) { | ||||||
3876 | // Does it have a non-empty DefaultOps field? If so, ignore this | ||||||
3877 | // operand. | ||||||
3878 | if (!getDefaultOperand(Op.Rec).DefaultOps.empty()) | ||||||
3879 | continue; | ||||||
3880 | } | ||||||
3881 | I.error("Operand $" + OpName + | ||||||
3882 | " does not appear in the instruction pattern"); | ||||||
3883 | } | ||||||
3884 | TreePatternNodePtr InVal = InstInputs[OpName]; | ||||||
3885 | InstInputs.erase(OpName); // It occurred, remove from map. | ||||||
3886 | |||||||
3887 | if (InVal->isLeaf() && isa<DefInit>(InVal->getLeafValue())) { | ||||||
3888 | Record *InRec = cast<DefInit>(InVal->getLeafValue())->getDef(); | ||||||
3889 | if (!checkOperandClass(Op, InRec)) | ||||||
3890 | I.error("Operand $" + OpName + "'s register class disagrees" | ||||||
3891 | " between the operand and pattern"); | ||||||
3892 | } | ||||||
3893 | Operands.push_back(Op.Rec); | ||||||
3894 | |||||||
3895 | // Construct the result for the dest-pattern operand list. | ||||||
3896 | TreePatternNodePtr OpNode = InVal->clone(); | ||||||
3897 | |||||||
3898 | // No predicate is useful on the result. | ||||||
3899 | OpNode->clearPredicateCalls(); | ||||||
3900 | |||||||
3901 | // Promote the xform function to be an explicit node if set. | ||||||
3902 | if (Record *Xform = OpNode->getTransformFn()) { | ||||||
3903 | OpNode->setTransformFn(nullptr); | ||||||
3904 | std::vector<TreePatternNodePtr> Children; | ||||||
3905 | Children.push_back(OpNode); | ||||||
3906 | OpNode = std::make_shared<TreePatternNode>(Xform, std::move(Children), | ||||||
3907 | OpNode->getNumTypes()); | ||||||
3908 | } | ||||||
3909 | |||||||
3910 | ResultNodeOperands.push_back(std::move(OpNode)); | ||||||
3911 | } | ||||||
3912 | |||||||
3913 | if (!InstInputs.empty()) | ||||||
3914 | I.error("Input operand $" + InstInputs.begin()->first + | ||||||
3915 | " occurs in pattern but not in operands list!"); | ||||||
3916 | |||||||
3917 | TreePatternNodePtr ResultPattern = std::make_shared<TreePatternNode>( | ||||||
3918 | I.getRecord(), std::move(ResultNodeOperands), | ||||||
3919 | GetNumNodeResults(I.getRecord(), *this)); | ||||||
3920 | // Copy fully inferred output node types to instruction result pattern. | ||||||
3921 | for (unsigned i = 0; i != NumResults; ++i) { | ||||||
3922 | assert(ResNodes[i]->getNumTypes() == 1 && "FIXME: Unhandled")(static_cast <bool> (ResNodes[i]->getNumTypes() == 1 && "FIXME: Unhandled") ? void (0) : __assert_fail ("ResNodes[i]->getNumTypes() == 1 && \"FIXME: Unhandled\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3922, __extension__ __PRETTY_FUNCTION__)); | ||||||
3923 | ResultPattern->setType(i, ResNodes[i]->getExtType(0)); | ||||||
3924 | ResultPattern->setResultIndex(i, ResultIndices[i]); | ||||||
3925 | } | ||||||
3926 | |||||||
3927 | // FIXME: Assume only the first tree is the pattern. The others are clobber | ||||||
3928 | // nodes. | ||||||
3929 | TreePatternNodePtr Pattern = I.getTree(0); | ||||||
3930 | TreePatternNodePtr SrcPattern; | ||||||
3931 | if (Pattern->getOperator()->getName() == "set") { | ||||||
3932 | SrcPattern = Pattern->getChild(Pattern->getNumChildren()-1)->clone(); | ||||||
3933 | } else{ | ||||||
3934 | // Not a set (store or something?) | ||||||
3935 | SrcPattern = Pattern; | ||||||
3936 | } | ||||||
3937 | |||||||
3938 | // Create and insert the instruction. | ||||||
3939 | // FIXME: InstImpResults should not be part of DAGInstruction. | ||||||
3940 | Record *R = I.getRecord(); | ||||||
3941 | DAGInsts.try_emplace(R, std::move(Results), std::move(Operands), | ||||||
3942 | std::move(InstImpResults), SrcPattern, ResultPattern); | ||||||
3943 | |||||||
3944 | LLVM_DEBUG(I.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { I.dump(); } } while (false); | ||||||
3945 | } | ||||||
3946 | |||||||
3947 | /// ParseInstructions - Parse all of the instructions, inlining and resolving | ||||||
3948 | /// any fragments involved. This populates the Instructions list with fully | ||||||
3949 | /// resolved instructions. | ||||||
3950 | void CodeGenDAGPatterns::ParseInstructions() { | ||||||
3951 | std::vector<Record*> Instrs = Records.getAllDerivedDefinitions("Instruction"); | ||||||
3952 | |||||||
3953 | for (Record *Instr : Instrs) { | ||||||
3954 | ListInit *LI = nullptr; | ||||||
3955 | |||||||
3956 | if (isa<ListInit>(Instr->getValueInit("Pattern"))) | ||||||
3957 | LI = Instr->getValueAsListInit("Pattern"); | ||||||
3958 | |||||||
3959 | // If there is no pattern, only collect minimal information about the | ||||||
3960 | // instruction for its operand list. We have to assume that there is one | ||||||
3961 | // result, as we have no detailed info. A pattern which references the | ||||||
3962 | // null_frag operator is as-if no pattern were specified. Normally this | ||||||
3963 | // is from a multiclass expansion w/ a SDPatternOperator passed in as | ||||||
3964 | // null_frag. | ||||||
3965 | if (!LI || LI->empty() || hasNullFragReference(LI)) { | ||||||
3966 | std::vector<Record*> Results; | ||||||
3967 | std::vector<Record*> Operands; | ||||||
3968 | |||||||
3969 | CodeGenInstruction &InstInfo = Target.getInstruction(Instr); | ||||||
3970 | |||||||
3971 | if (InstInfo.Operands.size() != 0) { | ||||||
3972 | for (unsigned j = 0, e = InstInfo.Operands.NumDefs; j < e; ++j) | ||||||
3973 | Results.push_back(InstInfo.Operands[j].Rec); | ||||||
3974 | |||||||
3975 | // The rest are inputs. | ||||||
3976 | for (unsigned j = InstInfo.Operands.NumDefs, | ||||||
3977 | e = InstInfo.Operands.size(); j < e; ++j) | ||||||
3978 | Operands.push_back(InstInfo.Operands[j].Rec); | ||||||
3979 | } | ||||||
3980 | |||||||
3981 | // Create and insert the instruction. | ||||||
3982 | Instructions.try_emplace(Instr, std::move(Results), std::move(Operands), | ||||||
3983 | std::vector<Record *>()); | ||||||
3984 | continue; // no pattern. | ||||||
3985 | } | ||||||
3986 | |||||||
3987 | CodeGenInstruction &CGI = Target.getInstruction(Instr); | ||||||
3988 | parseInstructionPattern(CGI, LI, Instructions); | ||||||
3989 | } | ||||||
3990 | |||||||
3991 | // If we can, convert the instructions to be patterns that are matched! | ||||||
3992 | for (auto &Entry : Instructions) { | ||||||
3993 | Record *Instr = Entry.first; | ||||||
3994 | DAGInstruction &TheInst = Entry.second; | ||||||
3995 | TreePatternNodePtr SrcPattern = TheInst.getSrcPattern(); | ||||||
3996 | TreePatternNodePtr ResultPattern = TheInst.getResultPattern(); | ||||||
3997 | |||||||
3998 | if (SrcPattern && ResultPattern) { | ||||||
3999 | TreePattern Pattern(Instr, SrcPattern, true, *this); | ||||||
4000 | TreePattern Result(Instr, ResultPattern, false, *this); | ||||||
4001 | ParseOnePattern(Instr, Pattern, Result, TheInst.getImpResults()); | ||||||
4002 | } | ||||||
4003 | } | ||||||
4004 | } | ||||||
4005 | |||||||
4006 | typedef std::pair<TreePatternNode *, unsigned> NameRecord; | ||||||
4007 | |||||||
4008 | static void FindNames(TreePatternNode *P, | ||||||
4009 | std::map<std::string, NameRecord> &Names, | ||||||
4010 | TreePattern *PatternTop) { | ||||||
4011 | if (!P->getName().empty()) { | ||||||
4012 | NameRecord &Rec = Names[P->getName()]; | ||||||
4013 | // If this is the first instance of the name, remember the node. | ||||||
4014 | if (Rec.second++ == 0) | ||||||
4015 | Rec.first = P; | ||||||
4016 | else if (Rec.first->getExtTypes() != P->getExtTypes()) | ||||||
4017 | PatternTop->error("repetition of value: $" + P->getName() + | ||||||
4018 | " where different uses have different types!"); | ||||||
4019 | } | ||||||
4020 | |||||||
4021 | if (!P->isLeaf()) { | ||||||
4022 | for (unsigned i = 0, e = P->getNumChildren(); i != e; ++i) | ||||||
4023 | FindNames(P->getChild(i), Names, PatternTop); | ||||||
4024 | } | ||||||
4025 | } | ||||||
4026 | |||||||
4027 | void CodeGenDAGPatterns::AddPatternToMatch(TreePattern *Pattern, | ||||||
4028 | PatternToMatch &&PTM) { | ||||||
4029 | // Do some sanity checking on the pattern we're about to match. | ||||||
4030 | std::string Reason; | ||||||
4031 | if (!PTM.getSrcPattern()->canPatternMatch(Reason, *this)) { | ||||||
4032 | PrintWarning(Pattern->getRecord()->getLoc(), | ||||||
4033 | Twine("Pattern can never match: ") + Reason); | ||||||
4034 | return; | ||||||
4035 | } | ||||||
4036 | |||||||
4037 | // If the source pattern's root is a complex pattern, that complex pattern | ||||||
4038 | // must specify the nodes it can potentially match. | ||||||
4039 | if (const ComplexPattern *CP = | ||||||
4040 | PTM.getSrcPattern()->getComplexPatternInfo(*this)) | ||||||
4041 | if (CP->getRootNodes().empty()) | ||||||
4042 | Pattern->error("ComplexPattern at root must specify list of opcodes it" | ||||||
4043 | " could match"); | ||||||
4044 | |||||||
4045 | |||||||
4046 | // Find all of the named values in the input and output, ensure they have the | ||||||
4047 | // same type. | ||||||
4048 | std::map<std::string, NameRecord> SrcNames, DstNames; | ||||||
4049 | FindNames(PTM.getSrcPattern(), SrcNames, Pattern); | ||||||
4050 | FindNames(PTM.getDstPattern(), DstNames, Pattern); | ||||||
4051 | |||||||
4052 | // Scan all of the named values in the destination pattern, rejecting them if | ||||||
4053 | // they don't exist in the input pattern. | ||||||
4054 | for (const auto &Entry : DstNames) { | ||||||
4055 | if (SrcNames[Entry.first].first == nullptr) | ||||||
4056 | Pattern->error("Pattern has input without matching name in output: $" + | ||||||
4057 | Entry.first); | ||||||
4058 | } | ||||||
4059 | |||||||
4060 | // Scan all of the named values in the source pattern, rejecting them if the | ||||||
4061 | // name isn't used in the dest, and isn't used to tie two values together. | ||||||
4062 | for (const auto &Entry : SrcNames) | ||||||
4063 | if (DstNames[Entry.first].first == nullptr && | ||||||
4064 | SrcNames[Entry.first].second == 1) | ||||||
4065 | Pattern->error("Pattern has dead named input: $" + Entry.first); | ||||||
4066 | |||||||
4067 | PatternsToMatch.push_back(std::move(PTM)); | ||||||
4068 | } | ||||||
4069 | |||||||
4070 | void CodeGenDAGPatterns::InferInstructionFlags() { | ||||||
4071 | ArrayRef<const CodeGenInstruction*> Instructions = | ||||||
4072 | Target.getInstructionsByEnumValue(); | ||||||
4073 | |||||||
4074 | unsigned Errors = 0; | ||||||
4075 | |||||||
4076 | // Try to infer flags from all patterns in PatternToMatch. These include | ||||||
4077 | // both the primary instruction patterns (which always come first) and | ||||||
4078 | // patterns defined outside the instruction. | ||||||
4079 | for (const PatternToMatch &PTM : ptms()) { | ||||||
4080 | // We can only infer from single-instruction patterns, otherwise we won't | ||||||
4081 | // know which instruction should get the flags. | ||||||
4082 | SmallVector<Record*, 8> PatInstrs; | ||||||
4083 | getInstructionsInTree(PTM.getDstPattern(), PatInstrs); | ||||||
4084 | if (PatInstrs.size() != 1) | ||||||
4085 | continue; | ||||||
4086 | |||||||
4087 | // Get the single instruction. | ||||||
4088 | CodeGenInstruction &InstInfo = Target.getInstruction(PatInstrs.front()); | ||||||
4089 | |||||||
4090 | // Only infer properties from the first pattern. We'll verify the others. | ||||||
4091 | if (InstInfo.InferredFrom) | ||||||
4092 | continue; | ||||||
4093 | |||||||
4094 | InstAnalyzer PatInfo(*this); | ||||||
4095 | PatInfo.Analyze(PTM); | ||||||
4096 | Errors += InferFromPattern(InstInfo, PatInfo, PTM.getSrcRecord()); | ||||||
4097 | } | ||||||
4098 | |||||||
4099 | if (Errors) | ||||||
4100 | PrintFatalError("pattern conflicts"); | ||||||
4101 | |||||||
4102 | // If requested by the target, guess any undefined properties. | ||||||
4103 | if (Target.guessInstructionProperties()) { | ||||||
4104 | for (unsigned i = 0, e = Instructions.size(); i != e; ++i) { | ||||||
4105 | CodeGenInstruction *InstInfo = | ||||||
4106 | const_cast<CodeGenInstruction *>(Instructions[i]); | ||||||
4107 | if (InstInfo->InferredFrom) | ||||||
4108 | continue; | ||||||
4109 | // The mayLoad and mayStore flags default to false. | ||||||
4110 | // Conservatively assume hasSideEffects if it wasn't explicit. | ||||||
4111 | if (InstInfo->hasSideEffects_Unset) | ||||||
4112 | InstInfo->hasSideEffects = true; | ||||||
4113 | } | ||||||
4114 | return; | ||||||
4115 | } | ||||||
4116 | |||||||
4117 | // Complain about any flags that are still undefined. | ||||||
4118 | for (unsigned i = 0, e = Instructions.size(); i != e; ++i) { | ||||||
4119 | CodeGenInstruction *InstInfo = | ||||||
4120 | const_cast<CodeGenInstruction *>(Instructions[i]); | ||||||
4121 | if (InstInfo->InferredFrom) | ||||||
4122 | continue; | ||||||
4123 | if (InstInfo->hasSideEffects_Unset) | ||||||
4124 | PrintError(InstInfo->TheDef->getLoc(), | ||||||
4125 | "Can't infer hasSideEffects from patterns"); | ||||||
4126 | if (InstInfo->mayStore_Unset) | ||||||
4127 | PrintError(InstInfo->TheDef->getLoc(), | ||||||
4128 | "Can't infer mayStore from patterns"); | ||||||
4129 | if (InstInfo->mayLoad_Unset) | ||||||
4130 | PrintError(InstInfo->TheDef->getLoc(), | ||||||
4131 | "Can't infer mayLoad from patterns"); | ||||||
4132 | } | ||||||
4133 | } | ||||||
4134 | |||||||
4135 | |||||||
4136 | /// Verify instruction flags against pattern node properties. | ||||||
4137 | void CodeGenDAGPatterns::VerifyInstructionFlags() { | ||||||
4138 | unsigned Errors = 0; | ||||||
4139 | for (const PatternToMatch &PTM : ptms()) { | ||||||
4140 | SmallVector<Record*, 8> Instrs; | ||||||
4141 | getInstructionsInTree(PTM.getDstPattern(), Instrs); | ||||||
4142 | if (Instrs.empty()) | ||||||
4143 | continue; | ||||||
4144 | |||||||
4145 | // Count the number of instructions with each flag set. | ||||||
4146 | unsigned NumSideEffects = 0; | ||||||
4147 | unsigned NumStores = 0; | ||||||
4148 | unsigned NumLoads = 0; | ||||||
4149 | for (const Record *Instr : Instrs) { | ||||||
4150 | const CodeGenInstruction &InstInfo = Target.getInstruction(Instr); | ||||||
4151 | NumSideEffects += InstInfo.hasSideEffects; | ||||||
4152 | NumStores += InstInfo.mayStore; | ||||||
4153 | NumLoads += InstInfo.mayLoad; | ||||||
4154 | } | ||||||
4155 | |||||||
4156 | // Analyze the source pattern. | ||||||
4157 | InstAnalyzer PatInfo(*this); | ||||||
4158 | PatInfo.Analyze(PTM); | ||||||
4159 | |||||||
4160 | // Collect error messages. | ||||||
4161 | SmallVector<std::string, 4> Msgs; | ||||||
4162 | |||||||
4163 | // Check for missing flags in the output. | ||||||
4164 | // Permit extra flags for now at least. | ||||||
4165 | if (PatInfo.hasSideEffects && !NumSideEffects) | ||||||
4166 | Msgs.push_back("pattern has side effects, but hasSideEffects isn't set"); | ||||||
4167 | |||||||
4168 | // Don't verify store flags on instructions with side effects. At least for | ||||||
4169 | // intrinsics, side effects implies mayStore. | ||||||
4170 | if (!PatInfo.hasSideEffects && PatInfo.mayStore && !NumStores) | ||||||
4171 | Msgs.push_back("pattern may store, but mayStore isn't set"); | ||||||
4172 | |||||||
4173 | // Similarly, mayStore implies mayLoad on intrinsics. | ||||||
4174 | if (!PatInfo.mayStore && PatInfo.mayLoad && !NumLoads) | ||||||
4175 | Msgs.push_back("pattern may load, but mayLoad isn't set"); | ||||||
4176 | |||||||
4177 | // Print error messages. | ||||||
4178 | if (Msgs.empty()) | ||||||
4179 | continue; | ||||||
4180 | ++Errors; | ||||||
4181 | |||||||
4182 | for (const std::string &Msg : Msgs) | ||||||
4183 | PrintError(PTM.getSrcRecord()->getLoc(), Twine(Msg) + " on the " + | ||||||
4184 | (Instrs.size() == 1 ? | ||||||
4185 | "instruction" : "output instructions")); | ||||||
4186 | // Provide the location of the relevant instruction definitions. | ||||||
4187 | for (const Record *Instr : Instrs) { | ||||||
4188 | if (Instr != PTM.getSrcRecord()) | ||||||
4189 | PrintError(Instr->getLoc(), "defined here"); | ||||||
4190 | const CodeGenInstruction &InstInfo = Target.getInstruction(Instr); | ||||||
4191 | if (InstInfo.InferredFrom && | ||||||
4192 | InstInfo.InferredFrom != InstInfo.TheDef && | ||||||
4193 | InstInfo.InferredFrom != PTM.getSrcRecord()) | ||||||
4194 | PrintError(InstInfo.InferredFrom->getLoc(), "inferred from pattern"); | ||||||
4195 | } | ||||||
4196 | } | ||||||
4197 | if (Errors) | ||||||
4198 | PrintFatalError("Errors in DAG patterns"); | ||||||
4199 | } | ||||||
4200 | |||||||
4201 | /// Given a pattern result with an unresolved type, see if we can find one | ||||||
4202 | /// instruction with an unresolved result type. Force this result type to an | ||||||
4203 | /// arbitrary element if it's possible types to converge results. | ||||||
4204 | static bool ForceArbitraryInstResultType(TreePatternNode *N, TreePattern &TP) { | ||||||
4205 | if (N->isLeaf()) | ||||||
4206 | return false; | ||||||
4207 | |||||||
4208 | // Analyze children. | ||||||
4209 | for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) | ||||||
4210 | if (ForceArbitraryInstResultType(N->getChild(i), TP)) | ||||||
4211 | return true; | ||||||
4212 | |||||||
4213 | if (!N->getOperator()->isSubClassOf("Instruction")) | ||||||
4214 | return false; | ||||||
4215 | |||||||
4216 | // If this type is already concrete or completely unknown we can't do | ||||||
4217 | // anything. | ||||||
4218 | TypeInfer &TI = TP.getInfer(); | ||||||
4219 | for (unsigned i = 0, e = N->getNumTypes(); i != e; ++i) { | ||||||
4220 | if (N->getExtType(i).empty() || TI.isConcrete(N->getExtType(i), false)) | ||||||
4221 | continue; | ||||||
4222 | |||||||
4223 | // Otherwise, force its type to an arbitrary choice. | ||||||
4224 | if (TI.forceArbitrary(N->getExtType(i))) | ||||||
4225 | return true; | ||||||
4226 | } | ||||||
4227 | |||||||
4228 | return false; | ||||||
4229 | } | ||||||
4230 | |||||||
4231 | // Promote xform function to be an explicit node wherever set. | ||||||
4232 | static TreePatternNodePtr PromoteXForms(TreePatternNodePtr N) { | ||||||
4233 | if (Record *Xform = N->getTransformFn()) { | ||||||
4234 | N->setTransformFn(nullptr); | ||||||
4235 | std::vector<TreePatternNodePtr> Children; | ||||||
4236 | Children.push_back(PromoteXForms(N)); | ||||||
4237 | return std::make_shared<TreePatternNode>(Xform, std::move(Children), | ||||||
4238 | N->getNumTypes()); | ||||||
4239 | } | ||||||
4240 | |||||||
4241 | if (!N->isLeaf()) | ||||||
4242 | for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) { | ||||||
4243 | TreePatternNodePtr Child = N->getChildShared(i); | ||||||
4244 | N->setChild(i, PromoteXForms(Child)); | ||||||
4245 | } | ||||||
4246 | return N; | ||||||
4247 | } | ||||||
4248 | |||||||
4249 | void CodeGenDAGPatterns::ParseOnePattern(Record *TheDef, | ||||||
4250 | TreePattern &Pattern, TreePattern &Result, | ||||||
4251 | const std::vector<Record *> &InstImpResults) { | ||||||
4252 | |||||||
4253 | // Inline pattern fragments and expand multiple alternatives. | ||||||
4254 | Pattern.InlinePatternFragments(); | ||||||
4255 | Result.InlinePatternFragments(); | ||||||
4256 | |||||||
4257 | if (Result.getNumTrees() != 1) | ||||||
4258 | Result.error("Cannot use multi-alternative fragments in result pattern!"); | ||||||
4259 | |||||||
4260 | // Infer types. | ||||||
4261 | bool IterateInference; | ||||||
4262 | bool InferredAllPatternTypes, InferredAllResultTypes; | ||||||
4263 | do { | ||||||
4264 | // Infer as many types as possible. If we cannot infer all of them, we | ||||||
4265 | // can never do anything with this pattern: report it to the user. | ||||||
4266 | InferredAllPatternTypes = | ||||||
4267 | Pattern.InferAllTypes(&Pattern.getNamedNodesMap()); | ||||||
4268 | |||||||
4269 | // Infer as many types as possible. If we cannot infer all of them, we | ||||||
4270 | // can never do anything with this pattern: report it to the user. | ||||||
4271 | InferredAllResultTypes = | ||||||
4272 | Result.InferAllTypes(&Pattern.getNamedNodesMap()); | ||||||
4273 | |||||||
4274 | IterateInference = false; | ||||||
4275 | |||||||
4276 | // Apply the type of the result to the source pattern. This helps us | ||||||
4277 | // resolve cases where the input type is known to be a pointer type (which | ||||||
4278 | // is considered resolved), but the result knows it needs to be 32- or | ||||||
4279 | // 64-bits. Infer the other way for good measure. | ||||||
4280 | for (const auto &T : Pattern.getTrees()) | ||||||
4281 | for (unsigned i = 0, e = std::min(Result.getOnlyTree()->getNumTypes(), | ||||||
4282 | T->getNumTypes()); | ||||||
4283 | i != e; ++i) { | ||||||
4284 | IterateInference |= T->UpdateNodeType( | ||||||
4285 | i, Result.getOnlyTree()->getExtType(i), Result); | ||||||
4286 | IterateInference |= Result.getOnlyTree()->UpdateNodeType( | ||||||
4287 | i, T->getExtType(i), Result); | ||||||
4288 | } | ||||||
4289 | |||||||
4290 | // If our iteration has converged and the input pattern's types are fully | ||||||
4291 | // resolved but the result pattern is not fully resolved, we may have a | ||||||
4292 | // situation where we have two instructions in the result pattern and | ||||||
4293 | // the instructions require a common register class, but don't care about | ||||||
4294 | // what actual MVT is used. This is actually a bug in our modelling: | ||||||
4295 | // output patterns should have register classes, not MVTs. | ||||||
4296 | // | ||||||
4297 | // In any case, to handle this, we just go through and disambiguate some | ||||||
4298 | // arbitrary types to the result pattern's nodes. | ||||||
4299 | if (!IterateInference && InferredAllPatternTypes && | ||||||
4300 | !InferredAllResultTypes) | ||||||
4301 | IterateInference = | ||||||
4302 | ForceArbitraryInstResultType(Result.getTree(0).get(), Result); | ||||||
4303 | } while (IterateInference); | ||||||
4304 | |||||||
4305 | // Verify that we inferred enough types that we can do something with the | ||||||
4306 | // pattern and result. If these fire the user has to add type casts. | ||||||
4307 | if (!InferredAllPatternTypes) | ||||||
4308 | Pattern.error("Could not infer all types in pattern!"); | ||||||
4309 | if (!InferredAllResultTypes) { | ||||||
4310 | Pattern.dump(); | ||||||
4311 | Result.error("Could not infer all types in pattern result!"); | ||||||
4312 | } | ||||||
4313 | |||||||
4314 | // Promote xform function to be an explicit node wherever set. | ||||||
4315 | TreePatternNodePtr DstShared = PromoteXForms(Result.getOnlyTree()); | ||||||
4316 | |||||||
4317 | TreePattern Temp(Result.getRecord(), DstShared, false, *this); | ||||||
4318 | Temp.InferAllTypes(); | ||||||
4319 | |||||||
4320 | ListInit *Preds = TheDef->getValueAsListInit("Predicates"); | ||||||
4321 | int Complexity = TheDef->getValueAsInt("AddedComplexity"); | ||||||
4322 | |||||||
4323 | if (PatternRewriter) | ||||||
4324 | PatternRewriter(&Pattern); | ||||||
4325 | |||||||
4326 | // A pattern may end up with an "impossible" type, i.e. a situation | ||||||
4327 | // where all types have been eliminated for some node in this pattern. | ||||||
4328 | // This could occur for intrinsics that only make sense for a specific | ||||||
4329 | // value type, and use a specific register class. If, for some mode, | ||||||
4330 | // that register class does not accept that type, the type inference | ||||||
4331 | // will lead to a contradiction, which is not an error however, but | ||||||
4332 | // a sign that this pattern will simply never match. | ||||||
4333 | if (Temp.getOnlyTree()->hasPossibleType()) | ||||||
4334 | for (const auto &T : Pattern.getTrees()) | ||||||
4335 | if (T->hasPossibleType()) | ||||||
4336 | AddPatternToMatch(&Pattern, | ||||||
4337 | PatternToMatch(TheDef, Preds, T, Temp.getOnlyTree(), | ||||||
4338 | InstImpResults, Complexity, | ||||||
4339 | TheDef->getID())); | ||||||
4340 | } | ||||||
4341 | |||||||
4342 | void CodeGenDAGPatterns::ParsePatterns() { | ||||||
4343 | std::vector<Record*> Patterns = Records.getAllDerivedDefinitions("Pattern"); | ||||||
4344 | |||||||
4345 | for (Record *CurPattern : Patterns) { | ||||||
4346 | DagInit *Tree = CurPattern->getValueAsDag("PatternToMatch"); | ||||||
4347 | |||||||
4348 | // If the pattern references the null_frag, there's nothing to do. | ||||||
4349 | if (hasNullFragReference(Tree)) | ||||||
| |||||||
4350 | continue; | ||||||
4351 | |||||||
4352 | TreePattern Pattern(CurPattern, Tree, true, *this); | ||||||
4353 | |||||||
4354 | ListInit *LI = CurPattern->getValueAsListInit("ResultInstrs"); | ||||||
4355 | if (LI->empty()) continue; // no pattern. | ||||||
4356 | |||||||
4357 | // Parse the instruction. | ||||||
4358 | TreePattern Result(CurPattern, LI, false, *this); | ||||||
4359 | |||||||
4360 | if (Result.getNumTrees() != 1) | ||||||
4361 | Result.error("Cannot handle instructions producing instructions " | ||||||
4362 | "with temporaries yet!"); | ||||||
4363 | |||||||
4364 | // Validate that the input pattern is correct. | ||||||
4365 | std::map<std::string, TreePatternNodePtr> InstInputs; | ||||||
4366 | MapVector<std::string, TreePatternNodePtr, std::map<std::string, unsigned>> | ||||||
4367 | InstResults; | ||||||
4368 | std::vector<Record*> InstImpResults; | ||||||
4369 | for (unsigned j = 0, ee = Pattern.getNumTrees(); j != ee; ++j) | ||||||
4370 | FindPatternInputsAndOutputs(Pattern, Pattern.getTree(j), InstInputs, | ||||||
4371 | InstResults, InstImpResults); | ||||||
4372 | |||||||
4373 | ParseOnePattern(CurPattern, Pattern, Result, InstImpResults); | ||||||
4374 | } | ||||||
4375 | } | ||||||
4376 | |||||||
4377 | static void collectModes(std::set<unsigned> &Modes, const TreePatternNode *N) { | ||||||
4378 | for (const TypeSetByHwMode &VTS : N->getExtTypes()) | ||||||
4379 | for (const auto &I : VTS) | ||||||
4380 | Modes.insert(I.first); | ||||||
4381 | |||||||
4382 | for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) | ||||||
4383 | collectModes(Modes, N->getChild(i)); | ||||||
4384 | } | ||||||
4385 | |||||||
4386 | void CodeGenDAGPatterns::ExpandHwModeBasedTypes() { | ||||||
4387 | const CodeGenHwModes &CGH = getTargetInfo().getHwModes(); | ||||||
4388 | std::vector<PatternToMatch> Copy; | ||||||
4389 | PatternsToMatch.swap(Copy); | ||||||
4390 | |||||||
4391 | auto AppendPattern = [this](PatternToMatch &P, unsigned Mode, | ||||||
4392 | StringRef Check) { | ||||||
4393 | TreePatternNodePtr NewSrc = P.getSrcPattern()->clone(); | ||||||
4394 | TreePatternNodePtr NewDst = P.getDstPattern()->clone(); | ||||||
4395 | if (!NewSrc->setDefaultMode(Mode) || !NewDst->setDefaultMode(Mode)) { | ||||||
4396 | return; | ||||||
4397 | } | ||||||
4398 | |||||||
4399 | PatternsToMatch.emplace_back(P.getSrcRecord(), P.getPredicates(), | ||||||
4400 | std::move(NewSrc), std::move(NewDst), | ||||||
4401 | P.getDstRegs(), P.getAddedComplexity(), | ||||||
4402 | Record::getNewUID(Records), Mode, Check); | ||||||
4403 | }; | ||||||
4404 | |||||||
4405 | for (PatternToMatch &P : Copy) { | ||||||
4406 | TreePatternNodePtr SrcP = nullptr, DstP = nullptr; | ||||||
4407 | if (P.getSrcPattern()->hasProperTypeByHwMode()) | ||||||
4408 | SrcP = P.getSrcPatternShared(); | ||||||
4409 | if (P.getDstPattern()->hasProperTypeByHwMode()) | ||||||
4410 | DstP = P.getDstPatternShared(); | ||||||
4411 | if (!SrcP && !DstP) { | ||||||
4412 | PatternsToMatch.push_back(P); | ||||||
4413 | continue; | ||||||
4414 | } | ||||||
4415 | |||||||
4416 | std::set<unsigned> Modes; | ||||||
4417 | if (SrcP) | ||||||
4418 | collectModes(Modes, SrcP.get()); | ||||||
4419 | if (DstP) | ||||||
4420 | collectModes(Modes, DstP.get()); | ||||||
4421 | |||||||
4422 | // The predicate for the default mode needs to be constructed for each | ||||||
4423 | // pattern separately. | ||||||
4424 | // Since not all modes must be present in each pattern, if a mode m is | ||||||
4425 | // absent, then there is no point in constructing a check for m. If such | ||||||
4426 | // a check was created, it would be equivalent to checking the default | ||||||
4427 | // mode, except not all modes' predicates would be a part of the checking | ||||||
4428 | // code. The subsequently generated check for the default mode would then | ||||||
4429 | // have the exact same patterns, but a different predicate code. To avoid | ||||||
4430 | // duplicated patterns with different predicate checks, construct the | ||||||
4431 | // default check as a negation of all predicates that are actually present | ||||||
4432 | // in the source/destination patterns. | ||||||
4433 | SmallString<128> DefaultCheck; | ||||||
4434 | |||||||
4435 | for (unsigned M : Modes) { | ||||||
4436 | if (M == DefaultMode) | ||||||
4437 | continue; | ||||||
4438 | |||||||
4439 | // Fill the map entry for this mode. | ||||||
4440 | const HwMode &HM = CGH.getMode(M); | ||||||
4441 | AppendPattern(P, M, HM.Predicates); | ||||||
4442 | |||||||
4443 | // Add negations of the HM's predicates to the default predicate. | ||||||
4444 | if (!DefaultCheck.empty()) | ||||||
4445 | DefaultCheck += " && "; | ||||||
4446 | DefaultCheck += "!("; | ||||||
4447 | DefaultCheck += HM.Predicates; | ||||||
4448 | DefaultCheck += ")"; | ||||||
4449 | } | ||||||
4450 | |||||||
4451 | bool HasDefault = Modes.count(DefaultMode); | ||||||
4452 | if (HasDefault) | ||||||
4453 | AppendPattern(P, DefaultMode, DefaultCheck); | ||||||
4454 | } | ||||||
4455 | } | ||||||
4456 | |||||||
4457 | /// Dependent variable map for CodeGenDAGPattern variant generation | ||||||
4458 | typedef StringMap<int> DepVarMap; | ||||||
4459 | |||||||
4460 | static void FindDepVarsOf(TreePatternNode *N, DepVarMap &DepMap) { | ||||||
4461 | if (N->isLeaf()) { | ||||||
4462 | if (N->hasName() && isa<DefInit>(N->getLeafValue())) | ||||||
4463 | DepMap[N->getName()]++; | ||||||
4464 | } else { | ||||||
4465 | for (size_t i = 0, e = N->getNumChildren(); i != e; ++i) | ||||||
4466 | FindDepVarsOf(N->getChild(i), DepMap); | ||||||
4467 | } | ||||||
4468 | } | ||||||
4469 | |||||||
4470 | /// Find dependent variables within child patterns | ||||||
4471 | static void FindDepVars(TreePatternNode *N, MultipleUseVarSet &DepVars) { | ||||||
4472 | DepVarMap depcounts; | ||||||
4473 | FindDepVarsOf(N, depcounts); | ||||||
4474 | for (const auto &Pair : depcounts) { | ||||||
4475 | if (Pair.getValue() > 1) | ||||||
4476 | DepVars.insert(Pair.getKey()); | ||||||
4477 | } | ||||||
4478 | } | ||||||
4479 | |||||||
4480 | #ifndef NDEBUG | ||||||
4481 | /// Dump the dependent variable set: | ||||||
4482 | static void DumpDepVars(MultipleUseVarSet &DepVars) { | ||||||
4483 | if (DepVars.empty()) { | ||||||
4484 | LLVM_DEBUG(errs() << "<empty set>")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "<empty set>"; } } while (false); | ||||||
4485 | } else { | ||||||
4486 | LLVM_DEBUG(errs() << "[ ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "[ "; } } while (false); | ||||||
4487 | for (const auto &DepVar : DepVars) { | ||||||
4488 | LLVM_DEBUG(errs() << DepVar.getKey() << " ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << DepVar.getKey() << " " ; } } while (false); | ||||||
4489 | } | ||||||
4490 | LLVM_DEBUG(errs() << "]")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "]"; } } while (false); | ||||||
4491 | } | ||||||
4492 | } | ||||||
4493 | #endif | ||||||
4494 | |||||||
4495 | |||||||
4496 | /// CombineChildVariants - Given a bunch of permutations of each child of the | ||||||
4497 | /// 'operator' node, put them together in all possible ways. | ||||||
4498 | static void CombineChildVariants( | ||||||
4499 | TreePatternNodePtr Orig, | ||||||
4500 | const std::vector<std::vector<TreePatternNodePtr>> &ChildVariants, | ||||||
4501 | std::vector<TreePatternNodePtr> &OutVariants, CodeGenDAGPatterns &CDP, | ||||||
4502 | const MultipleUseVarSet &DepVars) { | ||||||
4503 | // Make sure that each operand has at least one variant to choose from. | ||||||
4504 | for (const auto &Variants : ChildVariants) | ||||||
4505 | if (Variants.empty()) | ||||||
4506 | return; | ||||||
4507 | |||||||
4508 | // The end result is an all-pairs construction of the resultant pattern. | ||||||
4509 | std::vector<unsigned> Idxs(ChildVariants.size()); | ||||||
4510 | bool NotDone; | ||||||
4511 | do { | ||||||
4512 | #ifndef NDEBUG | ||||||
4513 | LLVM_DEBUG(if (!Idxs.empty()) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { if (!Idxs.empty()) { errs() << Orig ->getOperator()->getName() << ": Idxs = [ "; for ( unsigned Idx : Idxs) { errs() << Idx << " "; } errs () << "]\n"; }; } } while (false) | ||||||
4514 | errs() << Orig->getOperator()->getName() << ": Idxs = [ ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { if (!Idxs.empty()) { errs() << Orig ->getOperator()->getName() << ": Idxs = [ "; for ( unsigned Idx : Idxs) { errs() << Idx << " "; } errs () << "]\n"; }; } } while (false) | ||||||
4515 | for (unsigned Idx : Idxs) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { if (!Idxs.empty()) { errs() << Orig ->getOperator()->getName() << ": Idxs = [ "; for ( unsigned Idx : Idxs) { errs() << Idx << " "; } errs () << "]\n"; }; } } while (false) | ||||||
4516 | errs() << Idx << " ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { if (!Idxs.empty()) { errs() << Orig ->getOperator()->getName() << ": Idxs = [ "; for ( unsigned Idx : Idxs) { errs() << Idx << " "; } errs () << "]\n"; }; } } while (false) | ||||||
4517 | }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { if (!Idxs.empty()) { errs() << Orig ->getOperator()->getName() << ": Idxs = [ "; for ( unsigned Idx : Idxs) { errs() << Idx << " "; } errs () << "]\n"; }; } } while (false) | ||||||
4518 | errs() << "]\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { if (!Idxs.empty()) { errs() << Orig ->getOperator()->getName() << ": Idxs = [ "; for ( unsigned Idx : Idxs) { errs() << Idx << " "; } errs () << "]\n"; }; } } while (false) | ||||||
4519 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { if (!Idxs.empty()) { errs() << Orig ->getOperator()->getName() << ": Idxs = [ "; for ( unsigned Idx : Idxs) { errs() << Idx << " "; } errs () << "]\n"; }; } } while (false); | ||||||
4520 | #endif | ||||||
4521 | // Create the variant and add it to the output list. | ||||||
4522 | std::vector<TreePatternNodePtr> NewChildren; | ||||||
4523 | for (unsigned i = 0, e = ChildVariants.size(); i != e; ++i) | ||||||
4524 | NewChildren.push_back(ChildVariants[i][Idxs[i]]); | ||||||
4525 | TreePatternNodePtr R = std::make_shared<TreePatternNode>( | ||||||
4526 | Orig->getOperator(), std::move(NewChildren), Orig->getNumTypes()); | ||||||
4527 | |||||||
4528 | // Copy over properties. | ||||||
4529 | R->setName(Orig->getName()); | ||||||
4530 | R->setNamesAsPredicateArg(Orig->getNamesAsPredicateArg()); | ||||||
4531 | R->setPredicateCalls(Orig->getPredicateCalls()); | ||||||
4532 | R->setGISelFlagsRecord(Orig->getGISelFlagsRecord()); | ||||||
4533 | R->setTransformFn(Orig->getTransformFn()); | ||||||
4534 | for (unsigned i = 0, e = Orig->getNumTypes(); i != e; ++i) | ||||||
4535 | R->setType(i, Orig->getExtType(i)); | ||||||
4536 | |||||||
4537 | // If this pattern cannot match, do not include it as a variant. | ||||||
4538 | std::string ErrString; | ||||||
4539 | // Scan to see if this pattern has already been emitted. We can get | ||||||
4540 | // duplication due to things like commuting: | ||||||
4541 | // (and GPRC:$a, GPRC:$b) -> (and GPRC:$b, GPRC:$a) | ||||||
4542 | // which are the same pattern. Ignore the dups. | ||||||
4543 | if (R->canPatternMatch(ErrString, CDP) && | ||||||
4544 | none_of(OutVariants, [&](TreePatternNodePtr Variant) { | ||||||
4545 | return R->isIsomorphicTo(Variant.get(), DepVars); | ||||||
4546 | })) | ||||||
4547 | OutVariants.push_back(R); | ||||||
4548 | |||||||
4549 | // Increment indices to the next permutation by incrementing the | ||||||
4550 | // indices from last index backward, e.g., generate the sequence | ||||||
4551 | // [0, 0], [0, 1], [1, 0], [1, 1]. | ||||||
4552 | int IdxsIdx; | ||||||
4553 | for (IdxsIdx = Idxs.size() - 1; IdxsIdx >= 0; --IdxsIdx) { | ||||||
4554 | if (++Idxs[IdxsIdx] == ChildVariants[IdxsIdx].size()) | ||||||
4555 | Idxs[IdxsIdx] = 0; | ||||||
4556 | else | ||||||
4557 | break; | ||||||
4558 | } | ||||||
4559 | NotDone = (IdxsIdx >= 0); | ||||||
4560 | } while (NotDone); | ||||||
4561 | } | ||||||
4562 | |||||||
4563 | /// CombineChildVariants - A helper function for binary operators. | ||||||
4564 | /// | ||||||
4565 | static void CombineChildVariants(TreePatternNodePtr Orig, | ||||||
4566 | const std::vector<TreePatternNodePtr> &LHS, | ||||||
4567 | const std::vector<TreePatternNodePtr> &RHS, | ||||||
4568 | std::vector<TreePatternNodePtr> &OutVariants, | ||||||
4569 | CodeGenDAGPatterns &CDP, | ||||||
4570 | const MultipleUseVarSet &DepVars) { | ||||||
4571 | std::vector<std::vector<TreePatternNodePtr>> ChildVariants; | ||||||
4572 | ChildVariants.push_back(LHS); | ||||||
4573 | ChildVariants.push_back(RHS); | ||||||
4574 | CombineChildVariants(Orig, ChildVariants, OutVariants, CDP, DepVars); | ||||||
4575 | } | ||||||
4576 | |||||||
4577 | static void | ||||||
4578 | GatherChildrenOfAssociativeOpcode(TreePatternNodePtr N, | ||||||
4579 | std::vector<TreePatternNodePtr> &Children) { | ||||||
4580 | assert(N->getNumChildren()==2 &&"Associative but doesn't have 2 children!")(static_cast <bool> (N->getNumChildren()==2 && "Associative but doesn't have 2 children!") ? void (0) : __assert_fail ("N->getNumChildren()==2 &&\"Associative but doesn't have 2 children!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 4580, __extension__ __PRETTY_FUNCTION__)); | ||||||
4581 | Record *Operator = N->getOperator(); | ||||||
4582 | |||||||
4583 | // Only permit raw nodes. | ||||||
4584 | if (!N->getName().empty() || !N->getPredicateCalls().empty() || | ||||||
4585 | N->getTransformFn()) { | ||||||
4586 | Children.push_back(N); | ||||||
4587 | return; | ||||||
4588 | } | ||||||
4589 | |||||||
4590 | if (N->getChild(0)->isLeaf() || N->getChild(0)->getOperator() != Operator) | ||||||
4591 | Children.push_back(N->getChildShared(0)); | ||||||
4592 | else | ||||||
4593 | GatherChildrenOfAssociativeOpcode(N->getChildShared(0), Children); | ||||||
4594 | |||||||
4595 | if (N->getChild(1)->isLeaf() || N->getChild(1)->getOperator() != Operator) | ||||||
4596 | Children.push_back(N->getChildShared(1)); | ||||||
4597 | else | ||||||
4598 | GatherChildrenOfAssociativeOpcode(N->getChildShared(1), Children); | ||||||
4599 | } | ||||||
4600 | |||||||
4601 | /// GenerateVariantsOf - Given a pattern N, generate all permutations we can of | ||||||
4602 | /// the (potentially recursive) pattern by using algebraic laws. | ||||||
4603 | /// | ||||||
4604 | static void GenerateVariantsOf(TreePatternNodePtr N, | ||||||
4605 | std::vector<TreePatternNodePtr> &OutVariants, | ||||||
4606 | CodeGenDAGPatterns &CDP, | ||||||
4607 | const MultipleUseVarSet &DepVars) { | ||||||
4608 | // We cannot permute leaves or ComplexPattern uses. | ||||||
4609 | if (N->isLeaf() || N->getOperator()->isSubClassOf("ComplexPattern")) { | ||||||
4610 | OutVariants.push_back(N); | ||||||
4611 | return; | ||||||
4612 | } | ||||||
4613 | |||||||
4614 | // Look up interesting info about the node. | ||||||
4615 | const SDNodeInfo &NodeInfo = CDP.getSDNodeInfo(N->getOperator()); | ||||||
4616 | |||||||
4617 | // If this node is associative, re-associate. | ||||||
4618 | if (NodeInfo.hasProperty(SDNPAssociative)) { | ||||||
4619 | // Re-associate by pulling together all of the linked operators | ||||||
4620 | std::vector<TreePatternNodePtr> MaximalChildren; | ||||||
4621 | GatherChildrenOfAssociativeOpcode(N, MaximalChildren); | ||||||
4622 | |||||||
4623 | // Only handle child sizes of 3. Otherwise we'll end up trying too many | ||||||
4624 | // permutations. | ||||||
4625 | if (MaximalChildren.size() == 3) { | ||||||
4626 | // Find the variants of all of our maximal children. | ||||||
4627 | std::vector<TreePatternNodePtr> AVariants, BVariants, CVariants; | ||||||
4628 | GenerateVariantsOf(MaximalChildren[0], AVariants, CDP, DepVars); | ||||||
4629 | GenerateVariantsOf(MaximalChildren[1], BVariants, CDP, DepVars); | ||||||
4630 | GenerateVariantsOf(MaximalChildren[2], CVariants, CDP, DepVars); | ||||||
4631 | |||||||
4632 | // There are only two ways we can permute the tree: | ||||||
4633 | // (A op B) op C and A op (B op C) | ||||||
4634 | // Within these forms, we can also permute A/B/C. | ||||||
4635 | |||||||
4636 | // Generate legal pair permutations of A/B/C. | ||||||
4637 | std::vector<TreePatternNodePtr> ABVariants; | ||||||
4638 | std::vector<TreePatternNodePtr> BAVariants; | ||||||
4639 | std::vector<TreePatternNodePtr> ACVariants; | ||||||
4640 | std::vector<TreePatternNodePtr> CAVariants; | ||||||
4641 | std::vector<TreePatternNodePtr> BCVariants; | ||||||
4642 | std::vector<TreePatternNodePtr> CBVariants; | ||||||
4643 | CombineChildVariants(N, AVariants, BVariants, ABVariants, CDP, DepVars); | ||||||
4644 | CombineChildVariants(N, BVariants, AVariants, BAVariants, CDP, DepVars); | ||||||
4645 | CombineChildVariants(N, AVariants, CVariants, ACVariants, CDP, DepVars); | ||||||
4646 | CombineChildVariants(N, CVariants, AVariants, CAVariants, CDP, DepVars); | ||||||
4647 | CombineChildVariants(N, BVariants, CVariants, BCVariants, CDP, DepVars); | ||||||
4648 | CombineChildVariants(N, CVariants, BVariants, CBVariants, CDP, DepVars); | ||||||
4649 | |||||||
4650 | // Combine those into the result: (x op x) op x | ||||||
4651 | CombineChildVariants(N, ABVariants, CVariants, OutVariants, CDP, DepVars); | ||||||
4652 | CombineChildVariants(N, BAVariants, CVariants, OutVariants, CDP, DepVars); | ||||||
4653 | CombineChildVariants(N, ACVariants, BVariants, OutVariants, CDP, DepVars); | ||||||
4654 | CombineChildVariants(N, CAVariants, BVariants, OutVariants, CDP, DepVars); | ||||||
4655 | CombineChildVariants(N, BCVariants, AVariants, OutVariants, CDP, DepVars); | ||||||
4656 | CombineChildVariants(N, CBVariants, AVariants, OutVariants, CDP, DepVars); | ||||||
4657 | |||||||
4658 | // Combine those into the result: x op (x op x) | ||||||
4659 | CombineChildVariants(N, CVariants, ABVariants, OutVariants, CDP, DepVars); | ||||||
4660 | CombineChildVariants(N, CVariants, BAVariants, OutVariants, CDP, DepVars); | ||||||
4661 | CombineChildVariants(N, BVariants, ACVariants, OutVariants, CDP, DepVars); | ||||||
4662 | CombineChildVariants(N, BVariants, CAVariants, OutVariants, CDP, DepVars); | ||||||
4663 | CombineChildVariants(N, AVariants, BCVariants, OutVariants, CDP, DepVars); | ||||||
4664 | CombineChildVariants(N, AVariants, CBVariants, OutVariants, CDP, DepVars); | ||||||
4665 | return; | ||||||
4666 | } | ||||||
4667 | } | ||||||
4668 | |||||||
4669 | // Compute permutations of all children. | ||||||
4670 | std::vector<std::vector<TreePatternNodePtr>> ChildVariants( | ||||||
4671 | N->getNumChildren()); | ||||||
4672 | for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) | ||||||
4673 | GenerateVariantsOf(N->getChildShared(i), ChildVariants[i], CDP, DepVars); | ||||||
4674 | |||||||
4675 | // Build all permutations based on how the children were formed. | ||||||
4676 | CombineChildVariants(N, ChildVariants, OutVariants, CDP, DepVars); | ||||||
4677 | |||||||
4678 | // If this node is commutative, consider the commuted order. | ||||||
4679 | bool isCommIntrinsic = N->isCommutativeIntrinsic(CDP); | ||||||
4680 | if (NodeInfo.hasProperty(SDNPCommutative) || isCommIntrinsic) { | ||||||
4681 | unsigned Skip = isCommIntrinsic ? 1 : 0; // First operand is intrinsic id. | ||||||
4682 | assert(N->getNumChildren() >= (2 + Skip) &&(static_cast <bool> (N->getNumChildren() >= (2 + Skip ) && "Commutative but doesn't have 2 children!") ? void (0) : __assert_fail ("N->getNumChildren() >= (2 + Skip) && \"Commutative but doesn't have 2 children!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 4683, __extension__ __PRETTY_FUNCTION__)) | ||||||
4683 | "Commutative but doesn't have 2 children!")(static_cast <bool> (N->getNumChildren() >= (2 + Skip ) && "Commutative but doesn't have 2 children!") ? void (0) : __assert_fail ("N->getNumChildren() >= (2 + Skip) && \"Commutative but doesn't have 2 children!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 4683, __extension__ __PRETTY_FUNCTION__)); | ||||||
4684 | // Don't allow commuting children which are actually register references. | ||||||
4685 | bool NoRegisters = true; | ||||||
4686 | unsigned i = 0 + Skip; | ||||||
4687 | unsigned e = 2 + Skip; | ||||||
4688 | for (; i != e; ++i) { | ||||||
4689 | TreePatternNode *Child = N->getChild(i); | ||||||
4690 | if (Child->isLeaf()) | ||||||
4691 | if (DefInit *DI = dyn_cast<DefInit>(Child->getLeafValue())) { | ||||||
4692 | Record *RR = DI->getDef(); | ||||||
4693 | if (RR->isSubClassOf("Register")) | ||||||
4694 | NoRegisters = false; | ||||||
4695 | } | ||||||
4696 | } | ||||||
4697 | // Consider the commuted order. | ||||||
4698 | if (NoRegisters) { | ||||||
4699 | // Swap the first two operands after the intrinsic id, if present. | ||||||
4700 | unsigned i = isCommIntrinsic ? 1 : 0; | ||||||
4701 | std::swap(ChildVariants[i], ChildVariants[i + 1]); | ||||||
4702 | CombineChildVariants(N, ChildVariants, OutVariants, CDP, DepVars); | ||||||
4703 | } | ||||||
4704 | } | ||||||
4705 | } | ||||||
4706 | |||||||
4707 | |||||||
4708 | // GenerateVariants - Generate variants. For example, commutative patterns can | ||||||
4709 | // match multiple ways. Add them to PatternsToMatch as well. | ||||||
4710 | void CodeGenDAGPatterns::GenerateVariants() { | ||||||
4711 | LLVM_DEBUG(errs() << "Generating instruction variants.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "Generating instruction variants.\n" ; } } while (false); | ||||||
4712 | |||||||
4713 | // Loop over all of the patterns we've collected, checking to see if we can | ||||||
4714 | // generate variants of the instruction, through the exploitation of | ||||||
4715 | // identities. This permits the target to provide aggressive matching without | ||||||
4716 | // the .td file having to contain tons of variants of instructions. | ||||||
4717 | // | ||||||
4718 | // Note that this loop adds new patterns to the PatternsToMatch list, but we | ||||||
4719 | // intentionally do not reconsider these. Any variants of added patterns have | ||||||
4720 | // already been added. | ||||||
4721 | // | ||||||
4722 | for (unsigned i = 0, e = PatternsToMatch.size(); i != e; ++i) { | ||||||
4723 | MultipleUseVarSet DepVars; | ||||||
4724 | std::vector<TreePatternNodePtr> Variants; | ||||||
4725 | FindDepVars(PatternsToMatch[i].getSrcPattern(), DepVars); | ||||||
4726 | LLVM_DEBUG(errs() << "Dependent/multiply used variables: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "Dependent/multiply used variables: " ; } } while (false); | ||||||
4727 | LLVM_DEBUG(DumpDepVars(DepVars))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { DumpDepVars(DepVars); } } while (false); | ||||||
4728 | LLVM_DEBUG(errs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "\n"; } } while (false); | ||||||
4729 | GenerateVariantsOf(PatternsToMatch[i].getSrcPatternShared(), Variants, | ||||||
4730 | *this, DepVars); | ||||||
4731 | |||||||
4732 | assert(PatternsToMatch[i].getHwModeFeatures().empty() &&(static_cast <bool> (PatternsToMatch[i].getHwModeFeatures ().empty() && "HwModes should not have been expanded yet!" ) ? void (0) : __assert_fail ("PatternsToMatch[i].getHwModeFeatures().empty() && \"HwModes should not have been expanded yet!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 4733, __extension__ __PRETTY_FUNCTION__)) | ||||||
4733 | "HwModes should not have been expanded yet!")(static_cast <bool> (PatternsToMatch[i].getHwModeFeatures ().empty() && "HwModes should not have been expanded yet!" ) ? void (0) : __assert_fail ("PatternsToMatch[i].getHwModeFeatures().empty() && \"HwModes should not have been expanded yet!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 4733, __extension__ __PRETTY_FUNCTION__)); | ||||||
4734 | |||||||
4735 | assert(!Variants.empty() && "Must create at least original variant!")(static_cast <bool> (!Variants.empty() && "Must create at least original variant!" ) ? void (0) : __assert_fail ("!Variants.empty() && \"Must create at least original variant!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 4735, __extension__ __PRETTY_FUNCTION__)); | ||||||
4736 | if (Variants.size() == 1) // No additional variants for this pattern. | ||||||
4737 | continue; | ||||||
4738 | |||||||
4739 | LLVM_DEBUG(errs() << "FOUND VARIANTS OF: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "FOUND VARIANTS OF: "; PatternsToMatch [i].getSrcPattern()->dump(); errs() << "\n"; } } while (false) | ||||||
4740 | PatternsToMatch[i].getSrcPattern()->dump(); errs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "FOUND VARIANTS OF: "; PatternsToMatch [i].getSrcPattern()->dump(); errs() << "\n"; } } while (false); | ||||||
4741 | |||||||
4742 | for (unsigned v = 0, e = Variants.size(); v != e; ++v) { | ||||||
4743 | TreePatternNodePtr Variant = Variants[v]; | ||||||
4744 | |||||||
4745 | LLVM_DEBUG(errs() << " VAR#" << v << ": "; Variant->dump();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << " VAR#" << v << ": "; Variant->dump(); errs() << "\n"; } } while (false ) | ||||||
4746 | errs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << " VAR#" << v << ": "; Variant->dump(); errs() << "\n"; } } while (false ); | ||||||
4747 | |||||||
4748 | // Scan to see if an instruction or explicit pattern already matches this. | ||||||
4749 | bool AlreadyExists = false; | ||||||
4750 | for (unsigned p = 0, e = PatternsToMatch.size(); p != e; ++p) { | ||||||
4751 | // Skip if the top level predicates do not match. | ||||||
4752 | if ((i != p) && (PatternsToMatch[i].getPredicates() != | ||||||
4753 | PatternsToMatch[p].getPredicates())) | ||||||
4754 | continue; | ||||||
4755 | // Check to see if this variant already exists. | ||||||
4756 | if (Variant->isIsomorphicTo(PatternsToMatch[p].getSrcPattern(), | ||||||
4757 | DepVars)) { | ||||||
4758 | LLVM_DEBUG(errs() << " *** ALREADY EXISTS, ignoring variant.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << " *** ALREADY EXISTS, ignoring variant.\n" ; } } while (false); | ||||||
4759 | AlreadyExists = true; | ||||||
4760 | break; | ||||||
4761 | } | ||||||
4762 | } | ||||||
4763 | // If we already have it, ignore the variant. | ||||||
4764 | if (AlreadyExists) continue; | ||||||
4765 | |||||||
4766 | // Otherwise, add it to the list of patterns we have. | ||||||
4767 | PatternsToMatch.emplace_back( | ||||||
4768 | PatternsToMatch[i].getSrcRecord(), PatternsToMatch[i].getPredicates(), | ||||||
4769 | Variant, PatternsToMatch[i].getDstPatternShared(), | ||||||
4770 | PatternsToMatch[i].getDstRegs(), | ||||||
4771 | PatternsToMatch[i].getAddedComplexity(), Record::getNewUID(Records), | ||||||
4772 | PatternsToMatch[i].getForceMode(), | ||||||
4773 | PatternsToMatch[i].getHwModeFeatures()); | ||||||
4774 | } | ||||||
4775 | |||||||
4776 | LLVM_DEBUG(errs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "\n"; } } while (false); | ||||||
4777 | } | ||||||
4778 | } |