File: | build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/llvm/utils/TableGen/CodeGenDAGPatterns.cpp |
Warning: | line 2876, column 22 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- CodeGenDAGPatterns.cpp - Read DAG patterns from .td file -----------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // This file implements the CodeGenDAGPatterns class, which is used to read and | |||
10 | // represent the patterns present in a .td file for instructions. | |||
11 | // | |||
12 | //===----------------------------------------------------------------------===// | |||
13 | ||||
14 | #include "CodeGenDAGPatterns.h" | |||
15 | #include "CodeGenInstruction.h" | |||
16 | #include "llvm/ADT/DenseSet.h" | |||
17 | #include "llvm/ADT/MapVector.h" | |||
18 | #include "llvm/ADT/STLExtras.h" | |||
19 | #include "llvm/ADT/SmallSet.h" | |||
20 | #include "llvm/ADT/SmallString.h" | |||
21 | #include "llvm/ADT/StringExtras.h" | |||
22 | #include "llvm/ADT/StringMap.h" | |||
23 | #include "llvm/ADT/Twine.h" | |||
24 | #include "llvm/Support/Debug.h" | |||
25 | #include "llvm/Support/ErrorHandling.h" | |||
26 | #include "llvm/Support/TypeSize.h" | |||
27 | #include "llvm/TableGen/Error.h" | |||
28 | #include "llvm/TableGen/Record.h" | |||
29 | #include <algorithm> | |||
30 | #include <cstdio> | |||
31 | #include <iterator> | |||
32 | #include <set> | |||
33 | using namespace llvm; | |||
34 | ||||
35 | #define DEBUG_TYPE"dag-patterns" "dag-patterns" | |||
36 | ||||
37 | static inline bool isIntegerOrPtr(MVT VT) { | |||
38 | return VT.isInteger() || VT == MVT::iPTR; | |||
39 | } | |||
40 | static inline bool isFloatingPoint(MVT VT) { | |||
41 | return VT.isFloatingPoint(); | |||
42 | } | |||
43 | static inline bool isVector(MVT VT) { | |||
44 | return VT.isVector(); | |||
45 | } | |||
46 | static inline bool isScalar(MVT VT) { | |||
47 | return !VT.isVector(); | |||
48 | } | |||
49 | ||||
50 | template <typename Predicate> | |||
51 | static bool berase_if(MachineValueTypeSet &S, Predicate P) { | |||
52 | bool Erased = false; | |||
53 | // It is ok to iterate over MachineValueTypeSet and remove elements from it | |||
54 | // at the same time. | |||
55 | for (MVT T : S) { | |||
56 | if (!P(T)) | |||
57 | continue; | |||
58 | Erased = true; | |||
59 | S.erase(T); | |||
60 | } | |||
61 | return Erased; | |||
62 | } | |||
63 | ||||
64 | // --- TypeSetByHwMode | |||
65 | ||||
66 | // This is a parameterized type-set class. For each mode there is a list | |||
67 | // of types that are currently possible for a given tree node. Type | |||
68 | // inference will apply to each mode separately. | |||
69 | ||||
70 | TypeSetByHwMode::TypeSetByHwMode(ArrayRef<ValueTypeByHwMode> VTList) { | |||
71 | for (const ValueTypeByHwMode &VVT : VTList) { | |||
72 | insert(VVT); | |||
73 | AddrSpaces.push_back(VVT.PtrAddrSpace); | |||
74 | } | |||
75 | } | |||
76 | ||||
77 | bool TypeSetByHwMode::isValueTypeByHwMode(bool AllowEmpty) const { | |||
78 | for (const auto &I : *this) { | |||
79 | if (I.second.size() > 1) | |||
80 | return false; | |||
81 | if (!AllowEmpty && I.second.empty()) | |||
82 | return false; | |||
83 | } | |||
84 | return true; | |||
85 | } | |||
86 | ||||
87 | ValueTypeByHwMode TypeSetByHwMode::getValueTypeByHwMode() const { | |||
88 | assert(isValueTypeByHwMode(true) &&(static_cast <bool> (isValueTypeByHwMode(true) && "The type set has multiple types for at least one HW mode") ? void (0) : __assert_fail ("isValueTypeByHwMode(true) && \"The type set has multiple types for at least one HW mode\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 89, __extension__ __PRETTY_FUNCTION__)) | |||
89 | "The type set has multiple types for at least one HW mode")(static_cast <bool> (isValueTypeByHwMode(true) && "The type set has multiple types for at least one HW mode") ? void (0) : __assert_fail ("isValueTypeByHwMode(true) && \"The type set has multiple types for at least one HW mode\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 89, __extension__ __PRETTY_FUNCTION__)); | |||
90 | ValueTypeByHwMode VVT; | |||
91 | auto ASI = AddrSpaces.begin(); | |||
92 | ||||
93 | for (const auto &I : *this) { | |||
94 | MVT T = I.second.empty() ? MVT::Other : *I.second.begin(); | |||
95 | VVT.getOrCreateTypeForMode(I.first, T); | |||
96 | if (ASI != AddrSpaces.end()) | |||
97 | VVT.PtrAddrSpace = *ASI++; | |||
98 | } | |||
99 | return VVT; | |||
100 | } | |||
101 | ||||
102 | bool TypeSetByHwMode::isPossible() const { | |||
103 | for (const auto &I : *this) | |||
104 | if (!I.second.empty()) | |||
105 | return true; | |||
106 | return false; | |||
107 | } | |||
108 | ||||
109 | bool TypeSetByHwMode::insert(const ValueTypeByHwMode &VVT) { | |||
110 | bool Changed = false; | |||
111 | bool ContainsDefault = false; | |||
112 | MVT DT = MVT::Other; | |||
113 | ||||
114 | for (const auto &P : VVT) { | |||
115 | unsigned M = P.first; | |||
116 | // Make sure there exists a set for each specific mode from VVT. | |||
117 | Changed |= getOrCreate(M).insert(P.second).second; | |||
118 | // Cache VVT's default mode. | |||
119 | if (DefaultMode == M) { | |||
120 | ContainsDefault = true; | |||
121 | DT = P.second; | |||
122 | } | |||
123 | } | |||
124 | ||||
125 | // If VVT has a default mode, add the corresponding type to all | |||
126 | // modes in "this" that do not exist in VVT. | |||
127 | if (ContainsDefault) | |||
128 | for (auto &I : *this) | |||
129 | if (!VVT.hasMode(I.first)) | |||
130 | Changed |= I.second.insert(DT).second; | |||
131 | ||||
132 | return Changed; | |||
133 | } | |||
134 | ||||
135 | // Constrain the type set to be the intersection with VTS. | |||
136 | bool TypeSetByHwMode::constrain(const TypeSetByHwMode &VTS) { | |||
137 | bool Changed = false; | |||
138 | if (hasDefault()) { | |||
139 | for (const auto &I : VTS) { | |||
140 | unsigned M = I.first; | |||
141 | if (M == DefaultMode || hasMode(M)) | |||
142 | continue; | |||
143 | Map.insert({M, Map.at(DefaultMode)}); | |||
144 | Changed = true; | |||
145 | } | |||
146 | } | |||
147 | ||||
148 | for (auto &I : *this) { | |||
149 | unsigned M = I.first; | |||
150 | SetType &S = I.second; | |||
151 | if (VTS.hasMode(M) || VTS.hasDefault()) { | |||
152 | Changed |= intersect(I.second, VTS.get(M)); | |||
153 | } else if (!S.empty()) { | |||
154 | S.clear(); | |||
155 | Changed = true; | |||
156 | } | |||
157 | } | |||
158 | return Changed; | |||
159 | } | |||
160 | ||||
161 | template <typename Predicate> | |||
162 | bool TypeSetByHwMode::constrain(Predicate P) { | |||
163 | bool Changed = false; | |||
164 | for (auto &I : *this) | |||
165 | Changed |= berase_if(I.second, [&P](MVT VT) { return !P(VT); }); | |||
166 | return Changed; | |||
167 | } | |||
168 | ||||
169 | template <typename Predicate> | |||
170 | bool TypeSetByHwMode::assign_if(const TypeSetByHwMode &VTS, Predicate P) { | |||
171 | assert(empty())(static_cast <bool> (empty()) ? void (0) : __assert_fail ("empty()", "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 171 , __extension__ __PRETTY_FUNCTION__)); | |||
172 | for (const auto &I : VTS) { | |||
173 | SetType &S = getOrCreate(I.first); | |||
174 | for (auto J : I.second) | |||
175 | if (P(J)) | |||
176 | S.insert(J); | |||
177 | } | |||
178 | return !empty(); | |||
179 | } | |||
180 | ||||
181 | void TypeSetByHwMode::writeToStream(raw_ostream &OS) const { | |||
182 | SmallVector<unsigned, 4> Modes; | |||
183 | Modes.reserve(Map.size()); | |||
184 | ||||
185 | for (const auto &I : *this) | |||
186 | Modes.push_back(I.first); | |||
187 | if (Modes.empty()) { | |||
188 | OS << "{}"; | |||
189 | return; | |||
190 | } | |||
191 | array_pod_sort(Modes.begin(), Modes.end()); | |||
192 | ||||
193 | OS << '{'; | |||
194 | for (unsigned M : Modes) { | |||
195 | OS << ' ' << getModeName(M) << ':'; | |||
196 | writeToStream(get(M), OS); | |||
197 | } | |||
198 | OS << " }"; | |||
199 | } | |||
200 | ||||
201 | void TypeSetByHwMode::writeToStream(const SetType &S, raw_ostream &OS) { | |||
202 | SmallVector<MVT, 4> Types(S.begin(), S.end()); | |||
203 | array_pod_sort(Types.begin(), Types.end()); | |||
204 | ||||
205 | OS << '['; | |||
206 | ListSeparator LS(" "); | |||
207 | for (const MVT &T : Types) | |||
208 | OS << LS << ValueTypeByHwMode::getMVTName(T); | |||
209 | OS << ']'; | |||
210 | } | |||
211 | ||||
212 | bool TypeSetByHwMode::operator==(const TypeSetByHwMode &VTS) const { | |||
213 | // The isSimple call is much quicker than hasDefault - check this first. | |||
214 | bool IsSimple = isSimple(); | |||
215 | bool VTSIsSimple = VTS.isSimple(); | |||
216 | if (IsSimple && VTSIsSimple) | |||
217 | return *begin() == *VTS.begin(); | |||
218 | ||||
219 | // Speedup: We have a default if the set is simple. | |||
220 | bool HaveDefault = IsSimple || hasDefault(); | |||
221 | bool VTSHaveDefault = VTSIsSimple || VTS.hasDefault(); | |||
222 | if (HaveDefault != VTSHaveDefault) | |||
223 | return false; | |||
224 | ||||
225 | SmallSet<unsigned, 4> Modes; | |||
226 | for (auto &I : *this) | |||
227 | Modes.insert(I.first); | |||
228 | for (const auto &I : VTS) | |||
229 | Modes.insert(I.first); | |||
230 | ||||
231 | if (HaveDefault) { | |||
232 | // Both sets have default mode. | |||
233 | for (unsigned M : Modes) { | |||
234 | if (get(M) != VTS.get(M)) | |||
235 | return false; | |||
236 | } | |||
237 | } else { | |||
238 | // Neither set has default mode. | |||
239 | for (unsigned M : Modes) { | |||
240 | // If there is no default mode, an empty set is equivalent to not having | |||
241 | // the corresponding mode. | |||
242 | bool NoModeThis = !hasMode(M) || get(M).empty(); | |||
243 | bool NoModeVTS = !VTS.hasMode(M) || VTS.get(M).empty(); | |||
244 | if (NoModeThis != NoModeVTS) | |||
245 | return false; | |||
246 | if (!NoModeThis) | |||
247 | if (get(M) != VTS.get(M)) | |||
248 | return false; | |||
249 | } | |||
250 | } | |||
251 | ||||
252 | return true; | |||
253 | } | |||
254 | ||||
255 | namespace llvm { | |||
256 | raw_ostream &operator<<(raw_ostream &OS, const TypeSetByHwMode &T) { | |||
257 | T.writeToStream(OS); | |||
258 | return OS; | |||
259 | } | |||
260 | } | |||
261 | ||||
262 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) | |||
263 | void TypeSetByHwMode::dump() const { | |||
264 | dbgs() << *this << '\n'; | |||
265 | } | |||
266 | ||||
267 | bool TypeSetByHwMode::intersect(SetType &Out, const SetType &In) { | |||
268 | bool OutP = Out.count(MVT::iPTR), InP = In.count(MVT::iPTR); | |||
269 | auto Int = [&In](MVT T) -> bool { return !In.count(T); }; | |||
270 | ||||
271 | if (OutP == InP) | |||
272 | return berase_if(Out, Int); | |||
273 | ||||
274 | // Compute the intersection of scalars separately to account for only | |||
275 | // one set containing iPTR. | |||
276 | // The intersection of iPTR with a set of integer scalar types that does not | |||
277 | // include iPTR will result in the most specific scalar type: | |||
278 | // - iPTR is more specific than any set with two elements or more | |||
279 | // - iPTR is less specific than any single integer scalar type. | |||
280 | // For example | |||
281 | // { iPTR } * { i32 } -> { i32 } | |||
282 | // { iPTR } * { i32 i64 } -> { iPTR } | |||
283 | // and | |||
284 | // { iPTR i32 } * { i32 } -> { i32 } | |||
285 | // { iPTR i32 } * { i32 i64 } -> { i32 i64 } | |||
286 | // { iPTR i32 } * { i32 i64 i128 } -> { iPTR i32 } | |||
287 | ||||
288 | // Compute the difference between the two sets in such a way that the | |||
289 | // iPTR is in the set that is being subtracted. This is to see if there | |||
290 | // are any extra scalars in the set without iPTR that are not in the | |||
291 | // set containing iPTR. Then the iPTR could be considered a "wildcard" | |||
292 | // matching these scalars. If there is only one such scalar, it would | |||
293 | // replace the iPTR, if there are more, the iPTR would be retained. | |||
294 | SetType Diff; | |||
295 | if (InP) { | |||
296 | Diff = Out; | |||
297 | berase_if(Diff, [&In](MVT T) { return In.count(T); }); | |||
298 | // Pre-remove these elements and rely only on InP/OutP to determine | |||
299 | // whether a change has been made. | |||
300 | berase_if(Out, [&Diff](MVT T) { return Diff.count(T); }); | |||
301 | } else { | |||
302 | Diff = In; | |||
303 | berase_if(Diff, [&Out](MVT T) { return Out.count(T); }); | |||
304 | Out.erase(MVT::iPTR); | |||
305 | } | |||
306 | ||||
307 | // The actual intersection. | |||
308 | bool Changed = berase_if(Out, Int); | |||
309 | unsigned NumD = Diff.size(); | |||
310 | if (NumD == 0) | |||
311 | return Changed; | |||
312 | ||||
313 | if (NumD == 1) { | |||
314 | Out.insert(*Diff.begin()); | |||
315 | // This is a change only if Out was the one with iPTR (which is now | |||
316 | // being replaced). | |||
317 | Changed |= OutP; | |||
318 | } else { | |||
319 | // Multiple elements from Out are now replaced with iPTR. | |||
320 | Out.insert(MVT::iPTR); | |||
321 | Changed |= !OutP; | |||
322 | } | |||
323 | return Changed; | |||
324 | } | |||
325 | ||||
326 | bool TypeSetByHwMode::validate() const { | |||
327 | #ifndef NDEBUG | |||
328 | if (empty()) | |||
329 | return true; | |||
330 | bool AllEmpty = true; | |||
331 | for (const auto &I : *this) | |||
332 | AllEmpty &= I.second.empty(); | |||
333 | return !AllEmpty; | |||
334 | #endif | |||
335 | return true; | |||
336 | } | |||
337 | ||||
338 | // --- TypeInfer | |||
339 | ||||
340 | bool TypeInfer::MergeInTypeInfo(TypeSetByHwMode &Out, | |||
341 | const TypeSetByHwMode &In) { | |||
342 | ValidateOnExit _1(Out, *this); | |||
343 | In.validate(); | |||
344 | if (In.empty() || Out == In || TP.hasError()) | |||
345 | return false; | |||
346 | if (Out.empty()) { | |||
347 | Out = In; | |||
348 | return true; | |||
349 | } | |||
350 | ||||
351 | bool Changed = Out.constrain(In); | |||
352 | if (Changed && Out.empty()) | |||
353 | TP.error("Type contradiction"); | |||
354 | ||||
355 | return Changed; | |||
356 | } | |||
357 | ||||
358 | bool TypeInfer::forceArbitrary(TypeSetByHwMode &Out) { | |||
359 | ValidateOnExit _1(Out, *this); | |||
360 | if (TP.hasError()) | |||
361 | return false; | |||
362 | assert(!Out.empty() && "cannot pick from an empty set")(static_cast <bool> (!Out.empty() && "cannot pick from an empty set" ) ? void (0) : __assert_fail ("!Out.empty() && \"cannot pick from an empty set\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 362, __extension__ __PRETTY_FUNCTION__)); | |||
363 | ||||
364 | bool Changed = false; | |||
365 | for (auto &I : Out) { | |||
366 | TypeSetByHwMode::SetType &S = I.second; | |||
367 | if (S.size() <= 1) | |||
368 | continue; | |||
369 | MVT T = *S.begin(); // Pick the first element. | |||
370 | S.clear(); | |||
371 | S.insert(T); | |||
372 | Changed = true; | |||
373 | } | |||
374 | return Changed; | |||
375 | } | |||
376 | ||||
377 | bool TypeInfer::EnforceInteger(TypeSetByHwMode &Out) { | |||
378 | ValidateOnExit _1(Out, *this); | |||
379 | if (TP.hasError()) | |||
380 | return false; | |||
381 | if (!Out.empty()) | |||
382 | return Out.constrain(isIntegerOrPtr); | |||
383 | ||||
384 | return Out.assign_if(getLegalTypes(), isIntegerOrPtr); | |||
385 | } | |||
386 | ||||
387 | bool TypeInfer::EnforceFloatingPoint(TypeSetByHwMode &Out) { | |||
388 | ValidateOnExit _1(Out, *this); | |||
389 | if (TP.hasError()) | |||
390 | return false; | |||
391 | if (!Out.empty()) | |||
392 | return Out.constrain(isFloatingPoint); | |||
393 | ||||
394 | return Out.assign_if(getLegalTypes(), isFloatingPoint); | |||
395 | } | |||
396 | ||||
397 | bool TypeInfer::EnforceScalar(TypeSetByHwMode &Out) { | |||
398 | ValidateOnExit _1(Out, *this); | |||
399 | if (TP.hasError()) | |||
400 | return false; | |||
401 | if (!Out.empty()) | |||
402 | return Out.constrain(isScalar); | |||
403 | ||||
404 | return Out.assign_if(getLegalTypes(), isScalar); | |||
405 | } | |||
406 | ||||
407 | bool TypeInfer::EnforceVector(TypeSetByHwMode &Out) { | |||
408 | ValidateOnExit _1(Out, *this); | |||
409 | if (TP.hasError()) | |||
410 | return false; | |||
411 | if (!Out.empty()) | |||
412 | return Out.constrain(isVector); | |||
413 | ||||
414 | return Out.assign_if(getLegalTypes(), isVector); | |||
415 | } | |||
416 | ||||
417 | bool TypeInfer::EnforceAny(TypeSetByHwMode &Out) { | |||
418 | ValidateOnExit _1(Out, *this); | |||
419 | if (TP.hasError() || !Out.empty()) | |||
420 | return false; | |||
421 | ||||
422 | Out = getLegalTypes(); | |||
423 | return true; | |||
424 | } | |||
425 | ||||
426 | template <typename Iter, typename Pred, typename Less> | |||
427 | static Iter min_if(Iter B, Iter E, Pred P, Less L) { | |||
428 | if (B == E) | |||
429 | return E; | |||
430 | Iter Min = E; | |||
431 | for (Iter I = B; I != E; ++I) { | |||
432 | if (!P(*I)) | |||
433 | continue; | |||
434 | if (Min == E || L(*I, *Min)) | |||
435 | Min = I; | |||
436 | } | |||
437 | return Min; | |||
438 | } | |||
439 | ||||
440 | template <typename Iter, typename Pred, typename Less> | |||
441 | static Iter max_if(Iter B, Iter E, Pred P, Less L) { | |||
442 | if (B == E) | |||
443 | return E; | |||
444 | Iter Max = E; | |||
445 | for (Iter I = B; I != E; ++I) { | |||
446 | if (!P(*I)) | |||
447 | continue; | |||
448 | if (Max == E || L(*Max, *I)) | |||
449 | Max = I; | |||
450 | } | |||
451 | return Max; | |||
452 | } | |||
453 | ||||
454 | /// Make sure that for each type in Small, there exists a larger type in Big. | |||
455 | bool TypeInfer::EnforceSmallerThan(TypeSetByHwMode &Small, TypeSetByHwMode &Big, | |||
456 | bool SmallIsVT) { | |||
457 | ValidateOnExit _1(Small, *this), _2(Big, *this); | |||
458 | if (TP.hasError()) | |||
459 | return false; | |||
460 | bool Changed = false; | |||
461 | ||||
462 | assert((!SmallIsVT || !Small.empty()) &&(static_cast <bool> ((!SmallIsVT || !Small.empty()) && "Small should not be empty for SDTCisVTSmallerThanOp") ? void (0) : __assert_fail ("(!SmallIsVT || !Small.empty()) && \"Small should not be empty for SDTCisVTSmallerThanOp\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 463, __extension__ __PRETTY_FUNCTION__)) | |||
463 | "Small should not be empty for SDTCisVTSmallerThanOp")(static_cast <bool> ((!SmallIsVT || !Small.empty()) && "Small should not be empty for SDTCisVTSmallerThanOp") ? void (0) : __assert_fail ("(!SmallIsVT || !Small.empty()) && \"Small should not be empty for SDTCisVTSmallerThanOp\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 463, __extension__ __PRETTY_FUNCTION__)); | |||
464 | ||||
465 | if (Small.empty()) | |||
466 | Changed |= EnforceAny(Small); | |||
467 | if (Big.empty()) | |||
468 | Changed |= EnforceAny(Big); | |||
469 | ||||
470 | assert(Small.hasDefault() && Big.hasDefault())(static_cast <bool> (Small.hasDefault() && Big. hasDefault()) ? void (0) : __assert_fail ("Small.hasDefault() && Big.hasDefault()" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 470, __extension__ __PRETTY_FUNCTION__)); | |||
471 | ||||
472 | SmallVector<unsigned, 4> Modes; | |||
473 | union_modes(Small, Big, Modes); | |||
474 | ||||
475 | // 1. Only allow integer or floating point types and make sure that | |||
476 | // both sides are both integer or both floating point. | |||
477 | // 2. Make sure that either both sides have vector types, or neither | |||
478 | // of them does. | |||
479 | for (unsigned M : Modes) { | |||
480 | TypeSetByHwMode::SetType &S = Small.get(M); | |||
481 | TypeSetByHwMode::SetType &B = Big.get(M); | |||
482 | ||||
483 | assert((!SmallIsVT || !S.empty()) && "Expected non-empty type")(static_cast <bool> ((!SmallIsVT || !S.empty()) && "Expected non-empty type") ? void (0) : __assert_fail ("(!SmallIsVT || !S.empty()) && \"Expected non-empty type\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 483, __extension__ __PRETTY_FUNCTION__)); | |||
484 | ||||
485 | if (any_of(S, isIntegerOrPtr) && any_of(B, isIntegerOrPtr)) { | |||
486 | auto NotInt = [](MVT VT) { return !isIntegerOrPtr(VT); }; | |||
487 | Changed |= berase_if(S, NotInt); | |||
488 | Changed |= berase_if(B, NotInt); | |||
489 | } else if (any_of(S, isFloatingPoint) && any_of(B, isFloatingPoint)) { | |||
490 | auto NotFP = [](MVT VT) { return !isFloatingPoint(VT); }; | |||
491 | Changed |= berase_if(S, NotFP); | |||
492 | Changed |= berase_if(B, NotFP); | |||
493 | } else if (SmallIsVT && B.empty()) { | |||
494 | // B is empty and since S is a specific VT, it will never be empty. Don't | |||
495 | // report this as a change, just clear S and continue. This prevents an | |||
496 | // infinite loop. | |||
497 | S.clear(); | |||
498 | } else if (S.empty() || B.empty()) { | |||
499 | Changed = !S.empty() || !B.empty(); | |||
500 | S.clear(); | |||
501 | B.clear(); | |||
502 | } else { | |||
503 | TP.error("Incompatible types"); | |||
504 | return Changed; | |||
505 | } | |||
506 | ||||
507 | if (none_of(S, isVector) || none_of(B, isVector)) { | |||
508 | Changed |= berase_if(S, isVector); | |||
509 | Changed |= berase_if(B, isVector); | |||
510 | } | |||
511 | } | |||
512 | ||||
513 | auto LT = [](MVT A, MVT B) -> bool { | |||
514 | // Always treat non-scalable MVTs as smaller than scalable MVTs for the | |||
515 | // purposes of ordering. | |||
516 | auto ASize = std::make_tuple(A.isScalableVector(), A.getScalarSizeInBits(), | |||
517 | A.getSizeInBits().getKnownMinSize()); | |||
518 | auto BSize = std::make_tuple(B.isScalableVector(), B.getScalarSizeInBits(), | |||
519 | B.getSizeInBits().getKnownMinSize()); | |||
520 | return ASize < BSize; | |||
521 | }; | |||
522 | auto SameKindLE = [](MVT A, MVT B) -> bool { | |||
523 | // This function is used when removing elements: when a vector is compared | |||
524 | // to a non-vector or a scalable vector to any non-scalable MVT, it should | |||
525 | // return false (to avoid removal). | |||
526 | if (std::make_tuple(A.isVector(), A.isScalableVector()) != | |||
527 | std::make_tuple(B.isVector(), B.isScalableVector())) | |||
528 | return false; | |||
529 | ||||
530 | return std::make_tuple(A.getScalarSizeInBits(), | |||
531 | A.getSizeInBits().getKnownMinSize()) <= | |||
532 | std::make_tuple(B.getScalarSizeInBits(), | |||
533 | B.getSizeInBits().getKnownMinSize()); | |||
534 | }; | |||
535 | ||||
536 | for (unsigned M : Modes) { | |||
537 | TypeSetByHwMode::SetType &S = Small.get(M); | |||
538 | TypeSetByHwMode::SetType &B = Big.get(M); | |||
539 | // MinS = min scalar in Small, remove all scalars from Big that are | |||
540 | // smaller-or-equal than MinS. | |||
541 | auto MinS = min_if(S.begin(), S.end(), isScalar, LT); | |||
542 | if (MinS != S.end()) | |||
543 | Changed |= berase_if(B, std::bind(SameKindLE, | |||
544 | std::placeholders::_1, *MinS)); | |||
545 | ||||
546 | // MaxS = max scalar in Big, remove all scalars from Small that are | |||
547 | // larger than MaxS. | |||
548 | auto MaxS = max_if(B.begin(), B.end(), isScalar, LT); | |||
549 | if (MaxS != B.end()) | |||
550 | Changed |= berase_if(S, std::bind(SameKindLE, | |||
551 | *MaxS, std::placeholders::_1)); | |||
552 | ||||
553 | // MinV = min vector in Small, remove all vectors from Big that are | |||
554 | // smaller-or-equal than MinV. | |||
555 | auto MinV = min_if(S.begin(), S.end(), isVector, LT); | |||
556 | if (MinV != S.end()) | |||
557 | Changed |= berase_if(B, std::bind(SameKindLE, | |||
558 | std::placeholders::_1, *MinV)); | |||
559 | ||||
560 | // MaxV = max vector in Big, remove all vectors from Small that are | |||
561 | // larger than MaxV. | |||
562 | auto MaxV = max_if(B.begin(), B.end(), isVector, LT); | |||
563 | if (MaxV != B.end()) | |||
564 | Changed |= berase_if(S, std::bind(SameKindLE, | |||
565 | *MaxV, std::placeholders::_1)); | |||
566 | } | |||
567 | ||||
568 | return Changed; | |||
569 | } | |||
570 | ||||
571 | /// 1. Ensure that for each type T in Vec, T is a vector type, and that | |||
572 | /// for each type U in Elem, U is a scalar type. | |||
573 | /// 2. Ensure that for each (scalar) type U in Elem, there exists a (vector) | |||
574 | /// type T in Vec, such that U is the element type of T. | |||
575 | bool TypeInfer::EnforceVectorEltTypeIs(TypeSetByHwMode &Vec, | |||
576 | TypeSetByHwMode &Elem) { | |||
577 | ValidateOnExit _1(Vec, *this), _2(Elem, *this); | |||
578 | if (TP.hasError()) | |||
579 | return false; | |||
580 | bool Changed = false; | |||
581 | ||||
582 | if (Vec.empty()) | |||
583 | Changed |= EnforceVector(Vec); | |||
584 | if (Elem.empty()) | |||
585 | Changed |= EnforceScalar(Elem); | |||
586 | ||||
587 | SmallVector<unsigned, 4> Modes; | |||
588 | union_modes(Vec, Elem, Modes); | |||
589 | for (unsigned M : Modes) { | |||
590 | TypeSetByHwMode::SetType &V = Vec.get(M); | |||
591 | TypeSetByHwMode::SetType &E = Elem.get(M); | |||
592 | ||||
593 | Changed |= berase_if(V, isScalar); // Scalar = !vector | |||
594 | Changed |= berase_if(E, isVector); // Vector = !scalar | |||
595 | assert(!V.empty() && !E.empty())(static_cast <bool> (!V.empty() && !E.empty()) ? void (0) : __assert_fail ("!V.empty() && !E.empty()" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 595, __extension__ __PRETTY_FUNCTION__)); | |||
596 | ||||
597 | MachineValueTypeSet VT, ST; | |||
598 | // Collect element types from the "vector" set. | |||
599 | for (MVT T : V) | |||
600 | VT.insert(T.getVectorElementType()); | |||
601 | // Collect scalar types from the "element" set. | |||
602 | for (MVT T : E) | |||
603 | ST.insert(T); | |||
604 | ||||
605 | // Remove from V all (vector) types whose element type is not in S. | |||
606 | Changed |= berase_if(V, [&ST](MVT T) -> bool { | |||
607 | return !ST.count(T.getVectorElementType()); | |||
608 | }); | |||
609 | // Remove from E all (scalar) types, for which there is no corresponding | |||
610 | // type in V. | |||
611 | Changed |= berase_if(E, [&VT](MVT T) -> bool { return !VT.count(T); }); | |||
612 | } | |||
613 | ||||
614 | return Changed; | |||
615 | } | |||
616 | ||||
617 | bool TypeInfer::EnforceVectorEltTypeIs(TypeSetByHwMode &Vec, | |||
618 | const ValueTypeByHwMode &VVT) { | |||
619 | TypeSetByHwMode Tmp(VVT); | |||
620 | ValidateOnExit _1(Vec, *this), _2(Tmp, *this); | |||
621 | return EnforceVectorEltTypeIs(Vec, Tmp); | |||
622 | } | |||
623 | ||||
624 | /// Ensure that for each type T in Sub, T is a vector type, and there | |||
625 | /// exists a type U in Vec such that U is a vector type with the same | |||
626 | /// element type as T and at least as many elements as T. | |||
627 | bool TypeInfer::EnforceVectorSubVectorTypeIs(TypeSetByHwMode &Vec, | |||
628 | TypeSetByHwMode &Sub) { | |||
629 | ValidateOnExit _1(Vec, *this), _2(Sub, *this); | |||
630 | if (TP.hasError()) | |||
631 | return false; | |||
632 | ||||
633 | /// Return true if B is a suB-vector of P, i.e. P is a suPer-vector of B. | |||
634 | auto IsSubVec = [](MVT B, MVT P) -> bool { | |||
635 | if (!B.isVector() || !P.isVector()) | |||
636 | return false; | |||
637 | // Logically a <4 x i32> is a valid subvector of <n x 4 x i32> | |||
638 | // but until there are obvious use-cases for this, keep the | |||
639 | // types separate. | |||
640 | if (B.isScalableVector() != P.isScalableVector()) | |||
641 | return false; | |||
642 | if (B.getVectorElementType() != P.getVectorElementType()) | |||
643 | return false; | |||
644 | return B.getVectorMinNumElements() < P.getVectorMinNumElements(); | |||
645 | }; | |||
646 | ||||
647 | /// Return true if S has no element (vector type) that T is a sub-vector of, | |||
648 | /// i.e. has the same element type as T and more elements. | |||
649 | auto NoSubV = [&IsSubVec](const TypeSetByHwMode::SetType &S, MVT T) -> bool { | |||
650 | for (auto I : S) | |||
651 | if (IsSubVec(T, I)) | |||
652 | return false; | |||
653 | return true; | |||
654 | }; | |||
655 | ||||
656 | /// Return true if S has no element (vector type) that T is a super-vector | |||
657 | /// of, i.e. has the same element type as T and fewer elements. | |||
658 | auto NoSupV = [&IsSubVec](const TypeSetByHwMode::SetType &S, MVT T) -> bool { | |||
659 | for (auto I : S) | |||
660 | if (IsSubVec(I, T)) | |||
661 | return false; | |||
662 | return true; | |||
663 | }; | |||
664 | ||||
665 | bool Changed = false; | |||
666 | ||||
667 | if (Vec.empty()) | |||
668 | Changed |= EnforceVector(Vec); | |||
669 | if (Sub.empty()) | |||
670 | Changed |= EnforceVector(Sub); | |||
671 | ||||
672 | SmallVector<unsigned, 4> Modes; | |||
673 | union_modes(Vec, Sub, Modes); | |||
674 | for (unsigned M : Modes) { | |||
675 | TypeSetByHwMode::SetType &S = Sub.get(M); | |||
676 | TypeSetByHwMode::SetType &V = Vec.get(M); | |||
677 | ||||
678 | Changed |= berase_if(S, isScalar); | |||
679 | ||||
680 | // Erase all types from S that are not sub-vectors of a type in V. | |||
681 | Changed |= berase_if(S, std::bind(NoSubV, V, std::placeholders::_1)); | |||
682 | ||||
683 | // Erase all types from V that are not super-vectors of a type in S. | |||
684 | Changed |= berase_if(V, std::bind(NoSupV, S, std::placeholders::_1)); | |||
685 | } | |||
686 | ||||
687 | return Changed; | |||
688 | } | |||
689 | ||||
690 | /// 1. Ensure that V has a scalar type iff W has a scalar type. | |||
691 | /// 2. Ensure that for each vector type T in V, there exists a vector | |||
692 | /// type U in W, such that T and U have the same number of elements. | |||
693 | /// 3. Ensure that for each vector type U in W, there exists a vector | |||
694 | /// type T in V, such that T and U have the same number of elements | |||
695 | /// (reverse of 2). | |||
696 | bool TypeInfer::EnforceSameNumElts(TypeSetByHwMode &V, TypeSetByHwMode &W) { | |||
697 | ValidateOnExit _1(V, *this), _2(W, *this); | |||
698 | if (TP.hasError()) | |||
699 | return false; | |||
700 | ||||
701 | bool Changed = false; | |||
702 | if (V.empty()) | |||
703 | Changed |= EnforceAny(V); | |||
704 | if (W.empty()) | |||
705 | Changed |= EnforceAny(W); | |||
706 | ||||
707 | // An actual vector type cannot have 0 elements, so we can treat scalars | |||
708 | // as zero-length vectors. This way both vectors and scalars can be | |||
709 | // processed identically. | |||
710 | auto NoLength = [](const SmallDenseSet<ElementCount> &Lengths, | |||
711 | MVT T) -> bool { | |||
712 | return !Lengths.count(T.isVector() ? T.getVectorElementCount() | |||
713 | : ElementCount::getNull()); | |||
714 | }; | |||
715 | ||||
716 | SmallVector<unsigned, 4> Modes; | |||
717 | union_modes(V, W, Modes); | |||
718 | for (unsigned M : Modes) { | |||
719 | TypeSetByHwMode::SetType &VS = V.get(M); | |||
720 | TypeSetByHwMode::SetType &WS = W.get(M); | |||
721 | ||||
722 | SmallDenseSet<ElementCount> VN, WN; | |||
723 | for (MVT T : VS) | |||
724 | VN.insert(T.isVector() ? T.getVectorElementCount() | |||
725 | : ElementCount::getNull()); | |||
726 | for (MVT T : WS) | |||
727 | WN.insert(T.isVector() ? T.getVectorElementCount() | |||
728 | : ElementCount::getNull()); | |||
729 | ||||
730 | Changed |= berase_if(VS, std::bind(NoLength, WN, std::placeholders::_1)); | |||
731 | Changed |= berase_if(WS, std::bind(NoLength, VN, std::placeholders::_1)); | |||
732 | } | |||
733 | return Changed; | |||
734 | } | |||
735 | ||||
736 | namespace { | |||
737 | struct TypeSizeComparator { | |||
738 | bool operator()(const TypeSize &LHS, const TypeSize &RHS) const { | |||
739 | return std::make_tuple(LHS.isScalable(), LHS.getKnownMinValue()) < | |||
740 | std::make_tuple(RHS.isScalable(), RHS.getKnownMinValue()); | |||
741 | } | |||
742 | }; | |||
743 | } // end anonymous namespace | |||
744 | ||||
745 | /// 1. Ensure that for each type T in A, there exists a type U in B, | |||
746 | /// such that T and U have equal size in bits. | |||
747 | /// 2. Ensure that for each type U in B, there exists a type T in A | |||
748 | /// such that T and U have equal size in bits (reverse of 1). | |||
749 | bool TypeInfer::EnforceSameSize(TypeSetByHwMode &A, TypeSetByHwMode &B) { | |||
750 | ValidateOnExit _1(A, *this), _2(B, *this); | |||
751 | if (TP.hasError()) | |||
752 | return false; | |||
753 | bool Changed = false; | |||
754 | if (A.empty()) | |||
755 | Changed |= EnforceAny(A); | |||
756 | if (B.empty()) | |||
757 | Changed |= EnforceAny(B); | |||
758 | ||||
759 | typedef SmallSet<TypeSize, 2, TypeSizeComparator> TypeSizeSet; | |||
760 | ||||
761 | auto NoSize = [](const TypeSizeSet &Sizes, MVT T) -> bool { | |||
762 | return !Sizes.count(T.getSizeInBits()); | |||
763 | }; | |||
764 | ||||
765 | SmallVector<unsigned, 4> Modes; | |||
766 | union_modes(A, B, Modes); | |||
767 | for (unsigned M : Modes) { | |||
768 | TypeSetByHwMode::SetType &AS = A.get(M); | |||
769 | TypeSetByHwMode::SetType &BS = B.get(M); | |||
770 | TypeSizeSet AN, BN; | |||
771 | ||||
772 | for (MVT T : AS) | |||
773 | AN.insert(T.getSizeInBits()); | |||
774 | for (MVT T : BS) | |||
775 | BN.insert(T.getSizeInBits()); | |||
776 | ||||
777 | Changed |= berase_if(AS, std::bind(NoSize, BN, std::placeholders::_1)); | |||
778 | Changed |= berase_if(BS, std::bind(NoSize, AN, std::placeholders::_1)); | |||
779 | } | |||
780 | ||||
781 | return Changed; | |||
782 | } | |||
783 | ||||
784 | void TypeInfer::expandOverloads(TypeSetByHwMode &VTS) { | |||
785 | ValidateOnExit _1(VTS, *this); | |||
786 | const TypeSetByHwMode &Legal = getLegalTypes(); | |||
787 | assert(Legal.isDefaultOnly() && "Default-mode only expected")(static_cast <bool> (Legal.isDefaultOnly() && "Default-mode only expected" ) ? void (0) : __assert_fail ("Legal.isDefaultOnly() && \"Default-mode only expected\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 787, __extension__ __PRETTY_FUNCTION__)); | |||
788 | const TypeSetByHwMode::SetType &LegalTypes = Legal.get(DefaultMode); | |||
789 | ||||
790 | for (auto &I : VTS) | |||
791 | expandOverloads(I.second, LegalTypes); | |||
792 | } | |||
793 | ||||
794 | void TypeInfer::expandOverloads(TypeSetByHwMode::SetType &Out, | |||
795 | const TypeSetByHwMode::SetType &Legal) { | |||
796 | std::set<MVT> Ovs; | |||
797 | for (MVT T : Out) { | |||
798 | if (!T.isOverloaded()) | |||
799 | continue; | |||
800 | ||||
801 | Ovs.insert(T); | |||
802 | // MachineValueTypeSet allows iteration and erasing. | |||
803 | Out.erase(T); | |||
804 | } | |||
805 | ||||
806 | for (MVT Ov : Ovs) { | |||
807 | switch (Ov.SimpleTy) { | |||
808 | case MVT::iPTRAny: | |||
809 | Out.insert(MVT::iPTR); | |||
810 | return; | |||
811 | case MVT::iAny: | |||
812 | for (MVT T : MVT::integer_valuetypes()) | |||
813 | if (Legal.count(T)) | |||
814 | Out.insert(T); | |||
815 | for (MVT T : MVT::integer_fixedlen_vector_valuetypes()) | |||
816 | if (Legal.count(T)) | |||
817 | Out.insert(T); | |||
818 | for (MVT T : MVT::integer_scalable_vector_valuetypes()) | |||
819 | if (Legal.count(T)) | |||
820 | Out.insert(T); | |||
821 | return; | |||
822 | case MVT::fAny: | |||
823 | for (MVT T : MVT::fp_valuetypes()) | |||
824 | if (Legal.count(T)) | |||
825 | Out.insert(T); | |||
826 | for (MVT T : MVT::fp_fixedlen_vector_valuetypes()) | |||
827 | if (Legal.count(T)) | |||
828 | Out.insert(T); | |||
829 | for (MVT T : MVT::fp_scalable_vector_valuetypes()) | |||
830 | if (Legal.count(T)) | |||
831 | Out.insert(T); | |||
832 | return; | |||
833 | case MVT::vAny: | |||
834 | for (MVT T : MVT::vector_valuetypes()) | |||
835 | if (Legal.count(T)) | |||
836 | Out.insert(T); | |||
837 | return; | |||
838 | case MVT::Any: | |||
839 | for (MVT T : MVT::all_valuetypes()) | |||
840 | if (Legal.count(T)) | |||
841 | Out.insert(T); | |||
842 | return; | |||
843 | default: | |||
844 | break; | |||
845 | } | |||
846 | } | |||
847 | } | |||
848 | ||||
849 | const TypeSetByHwMode &TypeInfer::getLegalTypes() { | |||
850 | if (!LegalTypesCached) { | |||
851 | TypeSetByHwMode::SetType &LegalTypes = LegalCache.getOrCreate(DefaultMode); | |||
852 | // Stuff all types from all modes into the default mode. | |||
853 | const TypeSetByHwMode <S = TP.getDAGPatterns().getLegalTypes(); | |||
854 | for (const auto &I : LTS) | |||
855 | LegalTypes.insert(I.second); | |||
856 | LegalTypesCached = true; | |||
857 | } | |||
858 | assert(LegalCache.isDefaultOnly() && "Default-mode only expected")(static_cast <bool> (LegalCache.isDefaultOnly() && "Default-mode only expected") ? void (0) : __assert_fail ("LegalCache.isDefaultOnly() && \"Default-mode only expected\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 858, __extension__ __PRETTY_FUNCTION__)); | |||
859 | return LegalCache; | |||
860 | } | |||
861 | ||||
862 | #ifndef NDEBUG | |||
863 | TypeInfer::ValidateOnExit::~ValidateOnExit() { | |||
864 | if (Infer.Validate && !VTS.validate()) { | |||
865 | dbgs() << "Type set is empty for each HW mode:\n" | |||
866 | "possible type contradiction in the pattern below " | |||
867 | "(use -print-records with llvm-tblgen to see all " | |||
868 | "expanded records).\n"; | |||
869 | Infer.TP.dump(); | |||
870 | dbgs() << "Generated from record:\n"; | |||
871 | Infer.TP.getRecord()->dump(); | |||
872 | PrintFatalError(Infer.TP.getRecord()->getLoc(), | |||
873 | "Type set is empty for each HW mode in '" + | |||
874 | Infer.TP.getRecord()->getName() + "'"); | |||
875 | } | |||
876 | } | |||
877 | #endif | |||
878 | ||||
879 | ||||
880 | //===----------------------------------------------------------------------===// | |||
881 | // ScopedName Implementation | |||
882 | //===----------------------------------------------------------------------===// | |||
883 | ||||
884 | bool ScopedName::operator==(const ScopedName &o) const { | |||
885 | return Scope == o.Scope && Identifier == o.Identifier; | |||
886 | } | |||
887 | ||||
888 | bool ScopedName::operator!=(const ScopedName &o) const { | |||
889 | return !(*this == o); | |||
890 | } | |||
891 | ||||
892 | ||||
893 | //===----------------------------------------------------------------------===// | |||
894 | // TreePredicateFn Implementation | |||
895 | //===----------------------------------------------------------------------===// | |||
896 | ||||
897 | /// TreePredicateFn constructor. Here 'N' is a subclass of PatFrag. | |||
898 | TreePredicateFn::TreePredicateFn(TreePattern *N) : PatFragRec(N) { | |||
899 | assert((static_cast <bool> ((!hasPredCode() || !hasImmCode()) && ".td file corrupt: can't have a node predicate *and* an imm predicate" ) ? void (0) : __assert_fail ("(!hasPredCode() || !hasImmCode()) && \".td file corrupt: can't have a node predicate *and* an imm predicate\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 901, __extension__ __PRETTY_FUNCTION__)) | |||
900 | (!hasPredCode() || !hasImmCode()) &&(static_cast <bool> ((!hasPredCode() || !hasImmCode()) && ".td file corrupt: can't have a node predicate *and* an imm predicate" ) ? void (0) : __assert_fail ("(!hasPredCode() || !hasImmCode()) && \".td file corrupt: can't have a node predicate *and* an imm predicate\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 901, __extension__ __PRETTY_FUNCTION__)) | |||
901 | ".td file corrupt: can't have a node predicate *and* an imm predicate")(static_cast <bool> ((!hasPredCode() || !hasImmCode()) && ".td file corrupt: can't have a node predicate *and* an imm predicate" ) ? void (0) : __assert_fail ("(!hasPredCode() || !hasImmCode()) && \".td file corrupt: can't have a node predicate *and* an imm predicate\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 901, __extension__ __PRETTY_FUNCTION__)); | |||
902 | } | |||
903 | ||||
904 | bool TreePredicateFn::hasPredCode() const { | |||
905 | return isLoad() || isStore() || isAtomic() || | |||
906 | !PatFragRec->getRecord()->getValueAsString("PredicateCode").empty(); | |||
907 | } | |||
908 | ||||
909 | std::string TreePredicateFn::getPredCode() const { | |||
910 | std::string Code; | |||
911 | ||||
912 | if (!isLoad() && !isStore() && !isAtomic()) { | |||
913 | Record *MemoryVT = getMemoryVT(); | |||
914 | ||||
915 | if (MemoryVT) | |||
916 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
917 | "MemoryVT requires IsLoad or IsStore"); | |||
918 | } | |||
919 | ||||
920 | if (!isLoad() && !isStore()) { | |||
921 | if (isUnindexed()) | |||
922 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
923 | "IsUnindexed requires IsLoad or IsStore"); | |||
924 | ||||
925 | Record *ScalarMemoryVT = getScalarMemoryVT(); | |||
926 | ||||
927 | if (ScalarMemoryVT) | |||
928 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
929 | "ScalarMemoryVT requires IsLoad or IsStore"); | |||
930 | } | |||
931 | ||||
932 | if (isLoad() + isStore() + isAtomic() > 1) | |||
933 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
934 | "IsLoad, IsStore, and IsAtomic are mutually exclusive"); | |||
935 | ||||
936 | if (isLoad()) { | |||
937 | if (!isUnindexed() && !isNonExtLoad() && !isAnyExtLoad() && | |||
938 | !isSignExtLoad() && !isZeroExtLoad() && getMemoryVT() == nullptr && | |||
939 | getScalarMemoryVT() == nullptr && getAddressSpaces() == nullptr && | |||
940 | getMinAlignment() < 1) | |||
941 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
942 | "IsLoad cannot be used by itself"); | |||
943 | } else { | |||
944 | if (isNonExtLoad()) | |||
945 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
946 | "IsNonExtLoad requires IsLoad"); | |||
947 | if (isAnyExtLoad()) | |||
948 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
949 | "IsAnyExtLoad requires IsLoad"); | |||
950 | if (isSignExtLoad()) | |||
951 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
952 | "IsSignExtLoad requires IsLoad"); | |||
953 | if (isZeroExtLoad()) | |||
954 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
955 | "IsZeroExtLoad requires IsLoad"); | |||
956 | } | |||
957 | ||||
958 | if (isStore()) { | |||
959 | if (!isUnindexed() && !isTruncStore() && !isNonTruncStore() && | |||
960 | getMemoryVT() == nullptr && getScalarMemoryVT() == nullptr && | |||
961 | getAddressSpaces() == nullptr && getMinAlignment() < 1) | |||
962 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
963 | "IsStore cannot be used by itself"); | |||
964 | } else { | |||
965 | if (isNonTruncStore()) | |||
966 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
967 | "IsNonTruncStore requires IsStore"); | |||
968 | if (isTruncStore()) | |||
969 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
970 | "IsTruncStore requires IsStore"); | |||
971 | } | |||
972 | ||||
973 | if (isAtomic()) { | |||
974 | if (getMemoryVT() == nullptr && !isAtomicOrderingMonotonic() && | |||
975 | getAddressSpaces() == nullptr && | |||
976 | !isAtomicOrderingAcquire() && !isAtomicOrderingRelease() && | |||
977 | !isAtomicOrderingAcquireRelease() && | |||
978 | !isAtomicOrderingSequentiallyConsistent() && | |||
979 | !isAtomicOrderingAcquireOrStronger() && | |||
980 | !isAtomicOrderingReleaseOrStronger() && | |||
981 | !isAtomicOrderingWeakerThanAcquire() && | |||
982 | !isAtomicOrderingWeakerThanRelease()) | |||
983 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
984 | "IsAtomic cannot be used by itself"); | |||
985 | } else { | |||
986 | if (isAtomicOrderingMonotonic()) | |||
987 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
988 | "IsAtomicOrderingMonotonic requires IsAtomic"); | |||
989 | if (isAtomicOrderingAcquire()) | |||
990 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
991 | "IsAtomicOrderingAcquire requires IsAtomic"); | |||
992 | if (isAtomicOrderingRelease()) | |||
993 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
994 | "IsAtomicOrderingRelease requires IsAtomic"); | |||
995 | if (isAtomicOrderingAcquireRelease()) | |||
996 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
997 | "IsAtomicOrderingAcquireRelease requires IsAtomic"); | |||
998 | if (isAtomicOrderingSequentiallyConsistent()) | |||
999 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1000 | "IsAtomicOrderingSequentiallyConsistent requires IsAtomic"); | |||
1001 | if (isAtomicOrderingAcquireOrStronger()) | |||
1002 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1003 | "IsAtomicOrderingAcquireOrStronger requires IsAtomic"); | |||
1004 | if (isAtomicOrderingReleaseOrStronger()) | |||
1005 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1006 | "IsAtomicOrderingReleaseOrStronger requires IsAtomic"); | |||
1007 | if (isAtomicOrderingWeakerThanAcquire()) | |||
1008 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1009 | "IsAtomicOrderingWeakerThanAcquire requires IsAtomic"); | |||
1010 | } | |||
1011 | ||||
1012 | if (isLoad() || isStore() || isAtomic()) { | |||
1013 | if (ListInit *AddressSpaces = getAddressSpaces()) { | |||
1014 | Code += "unsigned AddrSpace = cast<MemSDNode>(N)->getAddressSpace();\n" | |||
1015 | " if ("; | |||
1016 | ||||
1017 | ListSeparator LS(" && "); | |||
1018 | for (Init *Val : AddressSpaces->getValues()) { | |||
1019 | Code += LS; | |||
1020 | ||||
1021 | IntInit *IntVal = dyn_cast<IntInit>(Val); | |||
1022 | if (!IntVal) { | |||
1023 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1024 | "AddressSpaces element must be integer"); | |||
1025 | } | |||
1026 | ||||
1027 | Code += "AddrSpace != " + utostr(IntVal->getValue()); | |||
1028 | } | |||
1029 | ||||
1030 | Code += ")\nreturn false;\n"; | |||
1031 | } | |||
1032 | ||||
1033 | int64_t MinAlign = getMinAlignment(); | |||
1034 | if (MinAlign > 0) { | |||
1035 | Code += "if (cast<MemSDNode>(N)->getAlign() < Align("; | |||
1036 | Code += utostr(MinAlign); | |||
1037 | Code += "))\nreturn false;\n"; | |||
1038 | } | |||
1039 | ||||
1040 | Record *MemoryVT = getMemoryVT(); | |||
1041 | ||||
1042 | if (MemoryVT) | |||
1043 | Code += ("if (cast<MemSDNode>(N)->getMemoryVT() != MVT::" + | |||
1044 | MemoryVT->getName() + ") return false;\n") | |||
1045 | .str(); | |||
1046 | } | |||
1047 | ||||
1048 | if (isAtomic() && isAtomicOrderingMonotonic()) | |||
1049 | Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != " | |||
1050 | "AtomicOrdering::Monotonic) return false;\n"; | |||
1051 | if (isAtomic() && isAtomicOrderingAcquire()) | |||
1052 | Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != " | |||
1053 | "AtomicOrdering::Acquire) return false;\n"; | |||
1054 | if (isAtomic() && isAtomicOrderingRelease()) | |||
1055 | Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != " | |||
1056 | "AtomicOrdering::Release) return false;\n"; | |||
1057 | if (isAtomic() && isAtomicOrderingAcquireRelease()) | |||
1058 | Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != " | |||
1059 | "AtomicOrdering::AcquireRelease) return false;\n"; | |||
1060 | if (isAtomic() && isAtomicOrderingSequentiallyConsistent()) | |||
1061 | Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != " | |||
1062 | "AtomicOrdering::SequentiallyConsistent) return false;\n"; | |||
1063 | ||||
1064 | if (isAtomic() && isAtomicOrderingAcquireOrStronger()) | |||
1065 | Code += "if (!isAcquireOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) " | |||
1066 | "return false;\n"; | |||
1067 | if (isAtomic() && isAtomicOrderingWeakerThanAcquire()) | |||
1068 | Code += "if (isAcquireOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) " | |||
1069 | "return false;\n"; | |||
1070 | ||||
1071 | if (isAtomic() && isAtomicOrderingReleaseOrStronger()) | |||
1072 | Code += "if (!isReleaseOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) " | |||
1073 | "return false;\n"; | |||
1074 | if (isAtomic() && isAtomicOrderingWeakerThanRelease()) | |||
1075 | Code += "if (isReleaseOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) " | |||
1076 | "return false;\n"; | |||
1077 | ||||
1078 | if (isLoad() || isStore()) { | |||
1079 | StringRef SDNodeName = isLoad() ? "LoadSDNode" : "StoreSDNode"; | |||
1080 | ||||
1081 | if (isUnindexed()) | |||
1082 | Code += ("if (cast<" + SDNodeName + | |||
1083 | ">(N)->getAddressingMode() != ISD::UNINDEXED) " | |||
1084 | "return false;\n") | |||
1085 | .str(); | |||
1086 | ||||
1087 | if (isLoad()) { | |||
1088 | if ((isNonExtLoad() + isAnyExtLoad() + isSignExtLoad() + | |||
1089 | isZeroExtLoad()) > 1) | |||
1090 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1091 | "IsNonExtLoad, IsAnyExtLoad, IsSignExtLoad, and " | |||
1092 | "IsZeroExtLoad are mutually exclusive"); | |||
1093 | if (isNonExtLoad()) | |||
1094 | Code += "if (cast<LoadSDNode>(N)->getExtensionType() != " | |||
1095 | "ISD::NON_EXTLOAD) return false;\n"; | |||
1096 | if (isAnyExtLoad()) | |||
1097 | Code += "if (cast<LoadSDNode>(N)->getExtensionType() != ISD::EXTLOAD) " | |||
1098 | "return false;\n"; | |||
1099 | if (isSignExtLoad()) | |||
1100 | Code += "if (cast<LoadSDNode>(N)->getExtensionType() != ISD::SEXTLOAD) " | |||
1101 | "return false;\n"; | |||
1102 | if (isZeroExtLoad()) | |||
1103 | Code += "if (cast<LoadSDNode>(N)->getExtensionType() != ISD::ZEXTLOAD) " | |||
1104 | "return false;\n"; | |||
1105 | } else { | |||
1106 | if ((isNonTruncStore() + isTruncStore()) > 1) | |||
1107 | PrintFatalError( | |||
1108 | getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1109 | "IsNonTruncStore, and IsTruncStore are mutually exclusive"); | |||
1110 | if (isNonTruncStore()) | |||
1111 | Code += | |||
1112 | " if (cast<StoreSDNode>(N)->isTruncatingStore()) return false;\n"; | |||
1113 | if (isTruncStore()) | |||
1114 | Code += | |||
1115 | " if (!cast<StoreSDNode>(N)->isTruncatingStore()) return false;\n"; | |||
1116 | } | |||
1117 | ||||
1118 | Record *ScalarMemoryVT = getScalarMemoryVT(); | |||
1119 | ||||
1120 | if (ScalarMemoryVT) | |||
1121 | Code += ("if (cast<" + SDNodeName + | |||
1122 | ">(N)->getMemoryVT().getScalarType() != MVT::" + | |||
1123 | ScalarMemoryVT->getName() + ") return false;\n") | |||
1124 | .str(); | |||
1125 | } | |||
1126 | ||||
1127 | std::string PredicateCode = | |||
1128 | std::string(PatFragRec->getRecord()->getValueAsString("PredicateCode")); | |||
1129 | ||||
1130 | Code += PredicateCode; | |||
1131 | ||||
1132 | if (PredicateCode.empty() && !Code.empty()) | |||
1133 | Code += "return true;\n"; | |||
1134 | ||||
1135 | return Code; | |||
1136 | } | |||
1137 | ||||
1138 | bool TreePredicateFn::hasImmCode() const { | |||
1139 | return !PatFragRec->getRecord()->getValueAsString("ImmediateCode").empty(); | |||
1140 | } | |||
1141 | ||||
1142 | std::string TreePredicateFn::getImmCode() const { | |||
1143 | return std::string( | |||
1144 | PatFragRec->getRecord()->getValueAsString("ImmediateCode")); | |||
1145 | } | |||
1146 | ||||
1147 | bool TreePredicateFn::immCodeUsesAPInt() const { | |||
1148 | return getOrigPatFragRecord()->getRecord()->getValueAsBit("IsAPInt"); | |||
1149 | } | |||
1150 | ||||
1151 | bool TreePredicateFn::immCodeUsesAPFloat() const { | |||
1152 | bool Unset; | |||
1153 | // The return value will be false when IsAPFloat is unset. | |||
1154 | return getOrigPatFragRecord()->getRecord()->getValueAsBitOrUnset("IsAPFloat", | |||
1155 | Unset); | |||
1156 | } | |||
1157 | ||||
1158 | bool TreePredicateFn::isPredefinedPredicateEqualTo(StringRef Field, | |||
1159 | bool Value) const { | |||
1160 | bool Unset; | |||
1161 | bool Result = | |||
1162 | getOrigPatFragRecord()->getRecord()->getValueAsBitOrUnset(Field, Unset); | |||
1163 | if (Unset) | |||
1164 | return false; | |||
1165 | return Result == Value; | |||
1166 | } | |||
1167 | bool TreePredicateFn::usesOperands() const { | |||
1168 | return isPredefinedPredicateEqualTo("PredicateCodeUsesOperands", true); | |||
1169 | } | |||
1170 | bool TreePredicateFn::isLoad() const { | |||
1171 | return isPredefinedPredicateEqualTo("IsLoad", true); | |||
1172 | } | |||
1173 | bool TreePredicateFn::isStore() const { | |||
1174 | return isPredefinedPredicateEqualTo("IsStore", true); | |||
1175 | } | |||
1176 | bool TreePredicateFn::isAtomic() const { | |||
1177 | return isPredefinedPredicateEqualTo("IsAtomic", true); | |||
1178 | } | |||
1179 | bool TreePredicateFn::isUnindexed() const { | |||
1180 | return isPredefinedPredicateEqualTo("IsUnindexed", true); | |||
1181 | } | |||
1182 | bool TreePredicateFn::isNonExtLoad() const { | |||
1183 | return isPredefinedPredicateEqualTo("IsNonExtLoad", true); | |||
1184 | } | |||
1185 | bool TreePredicateFn::isAnyExtLoad() const { | |||
1186 | return isPredefinedPredicateEqualTo("IsAnyExtLoad", true); | |||
1187 | } | |||
1188 | bool TreePredicateFn::isSignExtLoad() const { | |||
1189 | return isPredefinedPredicateEqualTo("IsSignExtLoad", true); | |||
1190 | } | |||
1191 | bool TreePredicateFn::isZeroExtLoad() const { | |||
1192 | return isPredefinedPredicateEqualTo("IsZeroExtLoad", true); | |||
1193 | } | |||
1194 | bool TreePredicateFn::isNonTruncStore() const { | |||
1195 | return isPredefinedPredicateEqualTo("IsTruncStore", false); | |||
1196 | } | |||
1197 | bool TreePredicateFn::isTruncStore() const { | |||
1198 | return isPredefinedPredicateEqualTo("IsTruncStore", true); | |||
1199 | } | |||
1200 | bool TreePredicateFn::isAtomicOrderingMonotonic() const { | |||
1201 | return isPredefinedPredicateEqualTo("IsAtomicOrderingMonotonic", true); | |||
1202 | } | |||
1203 | bool TreePredicateFn::isAtomicOrderingAcquire() const { | |||
1204 | return isPredefinedPredicateEqualTo("IsAtomicOrderingAcquire", true); | |||
1205 | } | |||
1206 | bool TreePredicateFn::isAtomicOrderingRelease() const { | |||
1207 | return isPredefinedPredicateEqualTo("IsAtomicOrderingRelease", true); | |||
1208 | } | |||
1209 | bool TreePredicateFn::isAtomicOrderingAcquireRelease() const { | |||
1210 | return isPredefinedPredicateEqualTo("IsAtomicOrderingAcquireRelease", true); | |||
1211 | } | |||
1212 | bool TreePredicateFn::isAtomicOrderingSequentiallyConsistent() const { | |||
1213 | return isPredefinedPredicateEqualTo("IsAtomicOrderingSequentiallyConsistent", | |||
1214 | true); | |||
1215 | } | |||
1216 | bool TreePredicateFn::isAtomicOrderingAcquireOrStronger() const { | |||
1217 | return isPredefinedPredicateEqualTo("IsAtomicOrderingAcquireOrStronger", true); | |||
1218 | } | |||
1219 | bool TreePredicateFn::isAtomicOrderingWeakerThanAcquire() const { | |||
1220 | return isPredefinedPredicateEqualTo("IsAtomicOrderingAcquireOrStronger", false); | |||
1221 | } | |||
1222 | bool TreePredicateFn::isAtomicOrderingReleaseOrStronger() const { | |||
1223 | return isPredefinedPredicateEqualTo("IsAtomicOrderingReleaseOrStronger", true); | |||
1224 | } | |||
1225 | bool TreePredicateFn::isAtomicOrderingWeakerThanRelease() const { | |||
1226 | return isPredefinedPredicateEqualTo("IsAtomicOrderingReleaseOrStronger", false); | |||
1227 | } | |||
1228 | Record *TreePredicateFn::getMemoryVT() const { | |||
1229 | Record *R = getOrigPatFragRecord()->getRecord(); | |||
1230 | if (R->isValueUnset("MemoryVT")) | |||
1231 | return nullptr; | |||
1232 | return R->getValueAsDef("MemoryVT"); | |||
1233 | } | |||
1234 | ||||
1235 | ListInit *TreePredicateFn::getAddressSpaces() const { | |||
1236 | Record *R = getOrigPatFragRecord()->getRecord(); | |||
1237 | if (R->isValueUnset("AddressSpaces")) | |||
1238 | return nullptr; | |||
1239 | return R->getValueAsListInit("AddressSpaces"); | |||
1240 | } | |||
1241 | ||||
1242 | int64_t TreePredicateFn::getMinAlignment() const { | |||
1243 | Record *R = getOrigPatFragRecord()->getRecord(); | |||
1244 | if (R->isValueUnset("MinAlignment")) | |||
1245 | return 0; | |||
1246 | return R->getValueAsInt("MinAlignment"); | |||
1247 | } | |||
1248 | ||||
1249 | Record *TreePredicateFn::getScalarMemoryVT() const { | |||
1250 | Record *R = getOrigPatFragRecord()->getRecord(); | |||
1251 | if (R->isValueUnset("ScalarMemoryVT")) | |||
1252 | return nullptr; | |||
1253 | return R->getValueAsDef("ScalarMemoryVT"); | |||
1254 | } | |||
1255 | bool TreePredicateFn::hasGISelPredicateCode() const { | |||
1256 | return !PatFragRec->getRecord() | |||
1257 | ->getValueAsString("GISelPredicateCode") | |||
1258 | .empty(); | |||
1259 | } | |||
1260 | std::string TreePredicateFn::getGISelPredicateCode() const { | |||
1261 | return std::string( | |||
1262 | PatFragRec->getRecord()->getValueAsString("GISelPredicateCode")); | |||
1263 | } | |||
1264 | ||||
1265 | StringRef TreePredicateFn::getImmType() const { | |||
1266 | if (immCodeUsesAPInt()) | |||
1267 | return "const APInt &"; | |||
1268 | if (immCodeUsesAPFloat()) | |||
1269 | return "const APFloat &"; | |||
1270 | return "int64_t"; | |||
1271 | } | |||
1272 | ||||
1273 | StringRef TreePredicateFn::getImmTypeIdentifier() const { | |||
1274 | if (immCodeUsesAPInt()) | |||
1275 | return "APInt"; | |||
1276 | if (immCodeUsesAPFloat()) | |||
1277 | return "APFloat"; | |||
1278 | return "I64"; | |||
1279 | } | |||
1280 | ||||
1281 | /// isAlwaysTrue - Return true if this is a noop predicate. | |||
1282 | bool TreePredicateFn::isAlwaysTrue() const { | |||
1283 | return !hasPredCode() && !hasImmCode(); | |||
1284 | } | |||
1285 | ||||
1286 | /// Return the name to use in the generated code to reference this, this is | |||
1287 | /// "Predicate_foo" if from a pattern fragment "foo". | |||
1288 | std::string TreePredicateFn::getFnName() const { | |||
1289 | return "Predicate_" + PatFragRec->getRecord()->getName().str(); | |||
1290 | } | |||
1291 | ||||
1292 | /// getCodeToRunOnSDNode - Return the code for the function body that | |||
1293 | /// evaluates this predicate. The argument is expected to be in "Node", | |||
1294 | /// not N. This handles casting and conversion to a concrete node type as | |||
1295 | /// appropriate. | |||
1296 | std::string TreePredicateFn::getCodeToRunOnSDNode() const { | |||
1297 | // Handle immediate predicates first. | |||
1298 | std::string ImmCode = getImmCode(); | |||
1299 | if (!ImmCode.empty()) { | |||
1300 | if (isLoad()) | |||
1301 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1302 | "IsLoad cannot be used with ImmLeaf or its subclasses"); | |||
1303 | if (isStore()) | |||
1304 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1305 | "IsStore cannot be used with ImmLeaf or its subclasses"); | |||
1306 | if (isUnindexed()) | |||
1307 | PrintFatalError( | |||
1308 | getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1309 | "IsUnindexed cannot be used with ImmLeaf or its subclasses"); | |||
1310 | if (isNonExtLoad()) | |||
1311 | PrintFatalError( | |||
1312 | getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1313 | "IsNonExtLoad cannot be used with ImmLeaf or its subclasses"); | |||
1314 | if (isAnyExtLoad()) | |||
1315 | PrintFatalError( | |||
1316 | getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1317 | "IsAnyExtLoad cannot be used with ImmLeaf or its subclasses"); | |||
1318 | if (isSignExtLoad()) | |||
1319 | PrintFatalError( | |||
1320 | getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1321 | "IsSignExtLoad cannot be used with ImmLeaf or its subclasses"); | |||
1322 | if (isZeroExtLoad()) | |||
1323 | PrintFatalError( | |||
1324 | getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1325 | "IsZeroExtLoad cannot be used with ImmLeaf or its subclasses"); | |||
1326 | if (isNonTruncStore()) | |||
1327 | PrintFatalError( | |||
1328 | getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1329 | "IsNonTruncStore cannot be used with ImmLeaf or its subclasses"); | |||
1330 | if (isTruncStore()) | |||
1331 | PrintFatalError( | |||
1332 | getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1333 | "IsTruncStore cannot be used with ImmLeaf or its subclasses"); | |||
1334 | if (getMemoryVT()) | |||
1335 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1336 | "MemoryVT cannot be used with ImmLeaf or its subclasses"); | |||
1337 | if (getScalarMemoryVT()) | |||
1338 | PrintFatalError( | |||
1339 | getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1340 | "ScalarMemoryVT cannot be used with ImmLeaf or its subclasses"); | |||
1341 | ||||
1342 | std::string Result = (" " + getImmType() + " Imm = ").str(); | |||
1343 | if (immCodeUsesAPFloat()) | |||
1344 | Result += "cast<ConstantFPSDNode>(Node)->getValueAPF();\n"; | |||
1345 | else if (immCodeUsesAPInt()) | |||
1346 | Result += "cast<ConstantSDNode>(Node)->getAPIntValue();\n"; | |||
1347 | else | |||
1348 | Result += "cast<ConstantSDNode>(Node)->getSExtValue();\n"; | |||
1349 | return Result + ImmCode; | |||
1350 | } | |||
1351 | ||||
1352 | // Handle arbitrary node predicates. | |||
1353 | assert(hasPredCode() && "Don't have any predicate code!")(static_cast <bool> (hasPredCode() && "Don't have any predicate code!" ) ? void (0) : __assert_fail ("hasPredCode() && \"Don't have any predicate code!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 1353, __extension__ __PRETTY_FUNCTION__)); | |||
1354 | ||||
1355 | // If this is using PatFrags, there are multiple trees to search. They should | |||
1356 | // all have the same class. FIXME: Is there a way to find a common | |||
1357 | // superclass? | |||
1358 | StringRef ClassName; | |||
1359 | for (const auto &Tree : PatFragRec->getTrees()) { | |||
1360 | StringRef TreeClassName; | |||
1361 | if (Tree->isLeaf()) | |||
1362 | TreeClassName = "SDNode"; | |||
1363 | else { | |||
1364 | Record *Op = Tree->getOperator(); | |||
1365 | const SDNodeInfo &Info = PatFragRec->getDAGPatterns().getSDNodeInfo(Op); | |||
1366 | TreeClassName = Info.getSDClassName(); | |||
1367 | } | |||
1368 | ||||
1369 | if (ClassName.empty()) | |||
1370 | ClassName = TreeClassName; | |||
1371 | else if (ClassName != TreeClassName) { | |||
1372 | PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), | |||
1373 | "PatFrags trees do not have consistent class"); | |||
1374 | } | |||
1375 | } | |||
1376 | ||||
1377 | std::string Result; | |||
1378 | if (ClassName == "SDNode") | |||
1379 | Result = " SDNode *N = Node;\n"; | |||
1380 | else | |||
1381 | Result = " auto *N = cast<" + ClassName.str() + ">(Node);\n"; | |||
1382 | ||||
1383 | return (Twine(Result) + " (void)N;\n" + getPredCode()).str(); | |||
1384 | } | |||
1385 | ||||
1386 | //===----------------------------------------------------------------------===// | |||
1387 | // PatternToMatch implementation | |||
1388 | // | |||
1389 | ||||
1390 | static bool isImmAllOnesAllZerosMatch(const TreePatternNode *P) { | |||
1391 | if (!P->isLeaf()) | |||
1392 | return false; | |||
1393 | DefInit *DI = dyn_cast<DefInit>(P->getLeafValue()); | |||
1394 | if (!DI) | |||
1395 | return false; | |||
1396 | ||||
1397 | Record *R = DI->getDef(); | |||
1398 | return R->getName() == "immAllOnesV" || R->getName() == "immAllZerosV"; | |||
1399 | } | |||
1400 | ||||
1401 | /// getPatternSize - Return the 'size' of this pattern. We want to match large | |||
1402 | /// patterns before small ones. This is used to determine the size of a | |||
1403 | /// pattern. | |||
1404 | static unsigned getPatternSize(const TreePatternNode *P, | |||
1405 | const CodeGenDAGPatterns &CGP) { | |||
1406 | unsigned Size = 3; // The node itself. | |||
1407 | // If the root node is a ConstantSDNode, increases its size. | |||
1408 | // e.g. (set R32:$dst, 0). | |||
1409 | if (P->isLeaf() && isa<IntInit>(P->getLeafValue())) | |||
1410 | Size += 2; | |||
1411 | ||||
1412 | if (const ComplexPattern *AM = P->getComplexPatternInfo(CGP)) { | |||
1413 | Size += AM->getComplexity(); | |||
1414 | // We don't want to count any children twice, so return early. | |||
1415 | return Size; | |||
1416 | } | |||
1417 | ||||
1418 | // If this node has some predicate function that must match, it adds to the | |||
1419 | // complexity of this node. | |||
1420 | if (!P->getPredicateCalls().empty()) | |||
1421 | ++Size; | |||
1422 | ||||
1423 | // Count children in the count if they are also nodes. | |||
1424 | for (unsigned i = 0, e = P->getNumChildren(); i != e; ++i) { | |||
1425 | const TreePatternNode *Child = P->getChild(i); | |||
1426 | if (!Child->isLeaf() && Child->getNumTypes()) { | |||
1427 | const TypeSetByHwMode &T0 = Child->getExtType(0); | |||
1428 | // At this point, all variable type sets should be simple, i.e. only | |||
1429 | // have a default mode. | |||
1430 | if (T0.getMachineValueType() != MVT::Other) { | |||
1431 | Size += getPatternSize(Child, CGP); | |||
1432 | continue; | |||
1433 | } | |||
1434 | } | |||
1435 | if (Child->isLeaf()) { | |||
1436 | if (isa<IntInit>(Child->getLeafValue())) | |||
1437 | Size += 5; // Matches a ConstantSDNode (+3) and a specific value (+2). | |||
1438 | else if (Child->getComplexPatternInfo(CGP)) | |||
1439 | Size += getPatternSize(Child, CGP); | |||
1440 | else if (isImmAllOnesAllZerosMatch(Child)) | |||
1441 | Size += 4; // Matches a build_vector(+3) and a predicate (+1). | |||
1442 | else if (!Child->getPredicateCalls().empty()) | |||
1443 | ++Size; | |||
1444 | } | |||
1445 | } | |||
1446 | ||||
1447 | return Size; | |||
1448 | } | |||
1449 | ||||
1450 | /// Compute the complexity metric for the input pattern. This roughly | |||
1451 | /// corresponds to the number of nodes that are covered. | |||
1452 | int PatternToMatch:: | |||
1453 | getPatternComplexity(const CodeGenDAGPatterns &CGP) const { | |||
1454 | return getPatternSize(getSrcPattern(), CGP) + getAddedComplexity(); | |||
1455 | } | |||
1456 | ||||
1457 | void PatternToMatch::getPredicateRecords( | |||
1458 | SmallVectorImpl<Record *> &PredicateRecs) const { | |||
1459 | for (Init *I : Predicates->getValues()) { | |||
1460 | if (DefInit *Pred = dyn_cast<DefInit>(I)) { | |||
1461 | Record *Def = Pred->getDef(); | |||
1462 | if (!Def->isSubClassOf("Predicate")) { | |||
1463 | #ifndef NDEBUG | |||
1464 | Def->dump(); | |||
1465 | #endif | |||
1466 | llvm_unreachable("Unknown predicate type!")::llvm::llvm_unreachable_internal("Unknown predicate type!", "llvm/utils/TableGen/CodeGenDAGPatterns.cpp" , 1466); | |||
1467 | } | |||
1468 | PredicateRecs.push_back(Def); | |||
1469 | } | |||
1470 | } | |||
1471 | // Sort so that different orders get canonicalized to the same string. | |||
1472 | llvm::sort(PredicateRecs, LessRecord()); | |||
1473 | } | |||
1474 | ||||
1475 | /// getPredicateCheck - Return a single string containing all of this | |||
1476 | /// pattern's predicates concatenated with "&&" operators. | |||
1477 | /// | |||
1478 | std::string PatternToMatch::getPredicateCheck() const { | |||
1479 | SmallVector<Record *, 4> PredicateRecs; | |||
1480 | getPredicateRecords(PredicateRecs); | |||
1481 | ||||
1482 | SmallString<128> PredicateCheck; | |||
1483 | for (Record *Pred : PredicateRecs) { | |||
1484 | StringRef CondString = Pred->getValueAsString("CondString"); | |||
1485 | if (CondString.empty()) | |||
1486 | continue; | |||
1487 | if (!PredicateCheck.empty()) | |||
1488 | PredicateCheck += " && "; | |||
1489 | PredicateCheck += "("; | |||
1490 | PredicateCheck += CondString; | |||
1491 | PredicateCheck += ")"; | |||
1492 | } | |||
1493 | ||||
1494 | if (!HwModeFeatures.empty()) { | |||
1495 | if (!PredicateCheck.empty()) | |||
1496 | PredicateCheck += " && "; | |||
1497 | PredicateCheck += HwModeFeatures; | |||
1498 | } | |||
1499 | ||||
1500 | return std::string(PredicateCheck); | |||
1501 | } | |||
1502 | ||||
1503 | //===----------------------------------------------------------------------===// | |||
1504 | // SDTypeConstraint implementation | |||
1505 | // | |||
1506 | ||||
1507 | SDTypeConstraint::SDTypeConstraint(Record *R, const CodeGenHwModes &CGH) { | |||
1508 | OperandNo = R->getValueAsInt("OperandNum"); | |||
1509 | ||||
1510 | if (R->isSubClassOf("SDTCisVT")) { | |||
1511 | ConstraintType = SDTCisVT; | |||
1512 | VVT = getValueTypeByHwMode(R->getValueAsDef("VT"), CGH); | |||
1513 | for (const auto &P : VVT) | |||
1514 | if (P.second == MVT::isVoid) | |||
1515 | PrintFatalError(R->getLoc(), "Cannot use 'Void' as type to SDTCisVT"); | |||
1516 | } else if (R->isSubClassOf("SDTCisPtrTy")) { | |||
1517 | ConstraintType = SDTCisPtrTy; | |||
1518 | } else if (R->isSubClassOf("SDTCisInt")) { | |||
1519 | ConstraintType = SDTCisInt; | |||
1520 | } else if (R->isSubClassOf("SDTCisFP")) { | |||
1521 | ConstraintType = SDTCisFP; | |||
1522 | } else if (R->isSubClassOf("SDTCisVec")) { | |||
1523 | ConstraintType = SDTCisVec; | |||
1524 | } else if (R->isSubClassOf("SDTCisSameAs")) { | |||
1525 | ConstraintType = SDTCisSameAs; | |||
1526 | x.SDTCisSameAs_Info.OtherOperandNum = R->getValueAsInt("OtherOperandNum"); | |||
1527 | } else if (R->isSubClassOf("SDTCisVTSmallerThanOp")) { | |||
1528 | ConstraintType = SDTCisVTSmallerThanOp; | |||
1529 | x.SDTCisVTSmallerThanOp_Info.OtherOperandNum = | |||
1530 | R->getValueAsInt("OtherOperandNum"); | |||
1531 | } else if (R->isSubClassOf("SDTCisOpSmallerThanOp")) { | |||
1532 | ConstraintType = SDTCisOpSmallerThanOp; | |||
1533 | x.SDTCisOpSmallerThanOp_Info.BigOperandNum = | |||
1534 | R->getValueAsInt("BigOperandNum"); | |||
1535 | } else if (R->isSubClassOf("SDTCisEltOfVec")) { | |||
1536 | ConstraintType = SDTCisEltOfVec; | |||
1537 | x.SDTCisEltOfVec_Info.OtherOperandNum = R->getValueAsInt("OtherOpNum"); | |||
1538 | } else if (R->isSubClassOf("SDTCisSubVecOfVec")) { | |||
1539 | ConstraintType = SDTCisSubVecOfVec; | |||
1540 | x.SDTCisSubVecOfVec_Info.OtherOperandNum = | |||
1541 | R->getValueAsInt("OtherOpNum"); | |||
1542 | } else if (R->isSubClassOf("SDTCVecEltisVT")) { | |||
1543 | ConstraintType = SDTCVecEltisVT; | |||
1544 | VVT = getValueTypeByHwMode(R->getValueAsDef("VT"), CGH); | |||
1545 | for (const auto &P : VVT) { | |||
1546 | MVT T = P.second; | |||
1547 | if (T.isVector()) | |||
1548 | PrintFatalError(R->getLoc(), | |||
1549 | "Cannot use vector type as SDTCVecEltisVT"); | |||
1550 | if (!T.isInteger() && !T.isFloatingPoint()) | |||
1551 | PrintFatalError(R->getLoc(), "Must use integer or floating point type " | |||
1552 | "as SDTCVecEltisVT"); | |||
1553 | } | |||
1554 | } else if (R->isSubClassOf("SDTCisSameNumEltsAs")) { | |||
1555 | ConstraintType = SDTCisSameNumEltsAs; | |||
1556 | x.SDTCisSameNumEltsAs_Info.OtherOperandNum = | |||
1557 | R->getValueAsInt("OtherOperandNum"); | |||
1558 | } else if (R->isSubClassOf("SDTCisSameSizeAs")) { | |||
1559 | ConstraintType = SDTCisSameSizeAs; | |||
1560 | x.SDTCisSameSizeAs_Info.OtherOperandNum = | |||
1561 | R->getValueAsInt("OtherOperandNum"); | |||
1562 | } else { | |||
1563 | PrintFatalError(R->getLoc(), | |||
1564 | "Unrecognized SDTypeConstraint '" + R->getName() + "'!\n"); | |||
1565 | } | |||
1566 | } | |||
1567 | ||||
1568 | /// getOperandNum - Return the node corresponding to operand #OpNo in tree | |||
1569 | /// N, and the result number in ResNo. | |||
1570 | static TreePatternNode *getOperandNum(unsigned OpNo, TreePatternNode *N, | |||
1571 | const SDNodeInfo &NodeInfo, | |||
1572 | unsigned &ResNo) { | |||
1573 | unsigned NumResults = NodeInfo.getNumResults(); | |||
1574 | if (OpNo < NumResults) { | |||
1575 | ResNo = OpNo; | |||
1576 | return N; | |||
1577 | } | |||
1578 | ||||
1579 | OpNo -= NumResults; | |||
1580 | ||||
1581 | if (OpNo >= N->getNumChildren()) { | |||
1582 | std::string S; | |||
1583 | raw_string_ostream OS(S); | |||
1584 | OS << "Invalid operand number in type constraint " | |||
1585 | << (OpNo+NumResults) << " "; | |||
1586 | N->print(OS); | |||
1587 | PrintFatalError(S); | |||
1588 | } | |||
1589 | ||||
1590 | return N->getChild(OpNo); | |||
1591 | } | |||
1592 | ||||
1593 | /// ApplyTypeConstraint - Given a node in a pattern, apply this type | |||
1594 | /// constraint to the nodes operands. This returns true if it makes a | |||
1595 | /// change, false otherwise. If a type contradiction is found, flag an error. | |||
1596 | bool SDTypeConstraint::ApplyTypeConstraint(TreePatternNode *N, | |||
1597 | const SDNodeInfo &NodeInfo, | |||
1598 | TreePattern &TP) const { | |||
1599 | if (TP.hasError()) | |||
1600 | return false; | |||
1601 | ||||
1602 | unsigned ResNo = 0; // The result number being referenced. | |||
1603 | TreePatternNode *NodeToApply = getOperandNum(OperandNo, N, NodeInfo, ResNo); | |||
1604 | TypeInfer &TI = TP.getInfer(); | |||
1605 | ||||
1606 | switch (ConstraintType) { | |||
1607 | case SDTCisVT: | |||
1608 | // Operand must be a particular type. | |||
1609 | return NodeToApply->UpdateNodeType(ResNo, VVT, TP); | |||
1610 | case SDTCisPtrTy: | |||
1611 | // Operand must be same as target pointer type. | |||
1612 | return NodeToApply->UpdateNodeType(ResNo, MVT::iPTR, TP); | |||
1613 | case SDTCisInt: | |||
1614 | // Require it to be one of the legal integer VTs. | |||
1615 | return TI.EnforceInteger(NodeToApply->getExtType(ResNo)); | |||
1616 | case SDTCisFP: | |||
1617 | // Require it to be one of the legal fp VTs. | |||
1618 | return TI.EnforceFloatingPoint(NodeToApply->getExtType(ResNo)); | |||
1619 | case SDTCisVec: | |||
1620 | // Require it to be one of the legal vector VTs. | |||
1621 | return TI.EnforceVector(NodeToApply->getExtType(ResNo)); | |||
1622 | case SDTCisSameAs: { | |||
1623 | unsigned OResNo = 0; | |||
1624 | TreePatternNode *OtherNode = | |||
1625 | getOperandNum(x.SDTCisSameAs_Info.OtherOperandNum, N, NodeInfo, OResNo); | |||
1626 | return (int)NodeToApply->UpdateNodeType(ResNo, | |||
1627 | OtherNode->getExtType(OResNo), TP) | | |||
1628 | (int)OtherNode->UpdateNodeType(OResNo, | |||
1629 | NodeToApply->getExtType(ResNo), TP); | |||
1630 | } | |||
1631 | case SDTCisVTSmallerThanOp: { | |||
1632 | // The NodeToApply must be a leaf node that is a VT. OtherOperandNum must | |||
1633 | // have an integer type that is smaller than the VT. | |||
1634 | if (!NodeToApply->isLeaf() || | |||
1635 | !isa<DefInit>(NodeToApply->getLeafValue()) || | |||
1636 | !cast<DefInit>(NodeToApply->getLeafValue())->getDef() | |||
1637 | ->isSubClassOf("ValueType")) { | |||
1638 | TP.error(N->getOperator()->getName() + " expects a VT operand!"); | |||
1639 | return false; | |||
1640 | } | |||
1641 | DefInit *DI = cast<DefInit>(NodeToApply->getLeafValue()); | |||
1642 | const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); | |||
1643 | auto VVT = getValueTypeByHwMode(DI->getDef(), T.getHwModes()); | |||
1644 | TypeSetByHwMode TypeListTmp(VVT); | |||
1645 | ||||
1646 | unsigned OResNo = 0; | |||
1647 | TreePatternNode *OtherNode = | |||
1648 | getOperandNum(x.SDTCisVTSmallerThanOp_Info.OtherOperandNum, N, NodeInfo, | |||
1649 | OResNo); | |||
1650 | ||||
1651 | return TI.EnforceSmallerThan(TypeListTmp, OtherNode->getExtType(OResNo), | |||
1652 | /*SmallIsVT*/ true); | |||
1653 | } | |||
1654 | case SDTCisOpSmallerThanOp: { | |||
1655 | unsigned BResNo = 0; | |||
1656 | TreePatternNode *BigOperand = | |||
1657 | getOperandNum(x.SDTCisOpSmallerThanOp_Info.BigOperandNum, N, NodeInfo, | |||
1658 | BResNo); | |||
1659 | return TI.EnforceSmallerThan(NodeToApply->getExtType(ResNo), | |||
1660 | BigOperand->getExtType(BResNo)); | |||
1661 | } | |||
1662 | case SDTCisEltOfVec: { | |||
1663 | unsigned VResNo = 0; | |||
1664 | TreePatternNode *VecOperand = | |||
1665 | getOperandNum(x.SDTCisEltOfVec_Info.OtherOperandNum, N, NodeInfo, | |||
1666 | VResNo); | |||
1667 | // Filter vector types out of VecOperand that don't have the right element | |||
1668 | // type. | |||
1669 | return TI.EnforceVectorEltTypeIs(VecOperand->getExtType(VResNo), | |||
1670 | NodeToApply->getExtType(ResNo)); | |||
1671 | } | |||
1672 | case SDTCisSubVecOfVec: { | |||
1673 | unsigned VResNo = 0; | |||
1674 | TreePatternNode *BigVecOperand = | |||
1675 | getOperandNum(x.SDTCisSubVecOfVec_Info.OtherOperandNum, N, NodeInfo, | |||
1676 | VResNo); | |||
1677 | ||||
1678 | // Filter vector types out of BigVecOperand that don't have the | |||
1679 | // right subvector type. | |||
1680 | return TI.EnforceVectorSubVectorTypeIs(BigVecOperand->getExtType(VResNo), | |||
1681 | NodeToApply->getExtType(ResNo)); | |||
1682 | } | |||
1683 | case SDTCVecEltisVT: { | |||
1684 | return TI.EnforceVectorEltTypeIs(NodeToApply->getExtType(ResNo), VVT); | |||
1685 | } | |||
1686 | case SDTCisSameNumEltsAs: { | |||
1687 | unsigned OResNo = 0; | |||
1688 | TreePatternNode *OtherNode = | |||
1689 | getOperandNum(x.SDTCisSameNumEltsAs_Info.OtherOperandNum, | |||
1690 | N, NodeInfo, OResNo); | |||
1691 | return TI.EnforceSameNumElts(OtherNode->getExtType(OResNo), | |||
1692 | NodeToApply->getExtType(ResNo)); | |||
1693 | } | |||
1694 | case SDTCisSameSizeAs: { | |||
1695 | unsigned OResNo = 0; | |||
1696 | TreePatternNode *OtherNode = | |||
1697 | getOperandNum(x.SDTCisSameSizeAs_Info.OtherOperandNum, | |||
1698 | N, NodeInfo, OResNo); | |||
1699 | return TI.EnforceSameSize(OtherNode->getExtType(OResNo), | |||
1700 | NodeToApply->getExtType(ResNo)); | |||
1701 | } | |||
1702 | } | |||
1703 | llvm_unreachable("Invalid ConstraintType!")::llvm::llvm_unreachable_internal("Invalid ConstraintType!", "llvm/utils/TableGen/CodeGenDAGPatterns.cpp" , 1703); | |||
1704 | } | |||
1705 | ||||
1706 | // Update the node type to match an instruction operand or result as specified | |||
1707 | // in the ins or outs lists on the instruction definition. Return true if the | |||
1708 | // type was actually changed. | |||
1709 | bool TreePatternNode::UpdateNodeTypeFromInst(unsigned ResNo, | |||
1710 | Record *Operand, | |||
1711 | TreePattern &TP) { | |||
1712 | // The 'unknown' operand indicates that types should be inferred from the | |||
1713 | // context. | |||
1714 | if (Operand->isSubClassOf("unknown_class")) | |||
1715 | return false; | |||
1716 | ||||
1717 | // The Operand class specifies a type directly. | |||
1718 | if (Operand->isSubClassOf("Operand")) { | |||
1719 | Record *R = Operand->getValueAsDef("Type"); | |||
1720 | const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); | |||
1721 | return UpdateNodeType(ResNo, getValueTypeByHwMode(R, T.getHwModes()), TP); | |||
1722 | } | |||
1723 | ||||
1724 | // PointerLikeRegClass has a type that is determined at runtime. | |||
1725 | if (Operand->isSubClassOf("PointerLikeRegClass")) | |||
1726 | return UpdateNodeType(ResNo, MVT::iPTR, TP); | |||
1727 | ||||
1728 | // Both RegisterClass and RegisterOperand operands derive their types from a | |||
1729 | // register class def. | |||
1730 | Record *RC = nullptr; | |||
1731 | if (Operand->isSubClassOf("RegisterClass")) | |||
1732 | RC = Operand; | |||
1733 | else if (Operand->isSubClassOf("RegisterOperand")) | |||
1734 | RC = Operand->getValueAsDef("RegClass"); | |||
1735 | ||||
1736 | assert(RC && "Unknown operand type")(static_cast <bool> (RC && "Unknown operand type" ) ? void (0) : __assert_fail ("RC && \"Unknown operand type\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 1736, __extension__ __PRETTY_FUNCTION__)); | |||
1737 | CodeGenTarget &Tgt = TP.getDAGPatterns().getTargetInfo(); | |||
1738 | return UpdateNodeType(ResNo, Tgt.getRegisterClass(RC).getValueTypes(), TP); | |||
1739 | } | |||
1740 | ||||
1741 | bool TreePatternNode::ContainsUnresolvedType(TreePattern &TP) const { | |||
1742 | for (unsigned i = 0, e = Types.size(); i != e; ++i) | |||
1743 | if (!TP.getInfer().isConcrete(Types[i], true)) | |||
1744 | return true; | |||
1745 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) | |||
1746 | if (getChild(i)->ContainsUnresolvedType(TP)) | |||
1747 | return true; | |||
1748 | return false; | |||
1749 | } | |||
1750 | ||||
1751 | bool TreePatternNode::hasProperTypeByHwMode() const { | |||
1752 | for (const TypeSetByHwMode &S : Types) | |||
1753 | if (!S.isDefaultOnly()) | |||
1754 | return true; | |||
1755 | for (const TreePatternNodePtr &C : Children) | |||
1756 | if (C->hasProperTypeByHwMode()) | |||
1757 | return true; | |||
1758 | return false; | |||
1759 | } | |||
1760 | ||||
1761 | bool TreePatternNode::hasPossibleType() const { | |||
1762 | for (const TypeSetByHwMode &S : Types) | |||
1763 | if (!S.isPossible()) | |||
1764 | return false; | |||
1765 | for (const TreePatternNodePtr &C : Children) | |||
1766 | if (!C->hasPossibleType()) | |||
1767 | return false; | |||
1768 | return true; | |||
1769 | } | |||
1770 | ||||
1771 | bool TreePatternNode::setDefaultMode(unsigned Mode) { | |||
1772 | for (TypeSetByHwMode &S : Types) { | |||
1773 | S.makeSimple(Mode); | |||
1774 | // Check if the selected mode had a type conflict. | |||
1775 | if (S.get(DefaultMode).empty()) | |||
1776 | return false; | |||
1777 | } | |||
1778 | for (const TreePatternNodePtr &C : Children) | |||
1779 | if (!C->setDefaultMode(Mode)) | |||
1780 | return false; | |||
1781 | return true; | |||
1782 | } | |||
1783 | ||||
1784 | //===----------------------------------------------------------------------===// | |||
1785 | // SDNodeInfo implementation | |||
1786 | // | |||
1787 | SDNodeInfo::SDNodeInfo(Record *R, const CodeGenHwModes &CGH) : Def(R) { | |||
1788 | EnumName = R->getValueAsString("Opcode"); | |||
1789 | SDClassName = R->getValueAsString("SDClass"); | |||
1790 | Record *TypeProfile = R->getValueAsDef("TypeProfile"); | |||
1791 | NumResults = TypeProfile->getValueAsInt("NumResults"); | |||
1792 | NumOperands = TypeProfile->getValueAsInt("NumOperands"); | |||
1793 | ||||
1794 | // Parse the properties. | |||
1795 | Properties = parseSDPatternOperatorProperties(R); | |||
1796 | ||||
1797 | // Parse the type constraints. | |||
1798 | std::vector<Record*> ConstraintList = | |||
1799 | TypeProfile->getValueAsListOfDefs("Constraints"); | |||
1800 | for (Record *R : ConstraintList) | |||
1801 | TypeConstraints.emplace_back(R, CGH); | |||
1802 | } | |||
1803 | ||||
1804 | /// getKnownType - If the type constraints on this node imply a fixed type | |||
1805 | /// (e.g. all stores return void, etc), then return it as an | |||
1806 | /// MVT::SimpleValueType. Otherwise, return EEVT::Other. | |||
1807 | MVT::SimpleValueType SDNodeInfo::getKnownType(unsigned ResNo) const { | |||
1808 | unsigned NumResults = getNumResults(); | |||
1809 | assert(NumResults <= 1 &&(static_cast <bool> (NumResults <= 1 && "We only work with nodes with zero or one result so far!" ) ? void (0) : __assert_fail ("NumResults <= 1 && \"We only work with nodes with zero or one result so far!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 1810, __extension__ __PRETTY_FUNCTION__)) | |||
1810 | "We only work with nodes with zero or one result so far!")(static_cast <bool> (NumResults <= 1 && "We only work with nodes with zero or one result so far!" ) ? void (0) : __assert_fail ("NumResults <= 1 && \"We only work with nodes with zero or one result so far!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 1810, __extension__ __PRETTY_FUNCTION__)); | |||
1811 | assert(ResNo == 0 && "Only handles single result nodes so far")(static_cast <bool> (ResNo == 0 && "Only handles single result nodes so far" ) ? void (0) : __assert_fail ("ResNo == 0 && \"Only handles single result nodes so far\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 1811, __extension__ __PRETTY_FUNCTION__)); | |||
1812 | ||||
1813 | for (const SDTypeConstraint &Constraint : TypeConstraints) { | |||
1814 | // Make sure that this applies to the correct node result. | |||
1815 | if (Constraint.OperandNo >= NumResults) // FIXME: need value # | |||
1816 | continue; | |||
1817 | ||||
1818 | switch (Constraint.ConstraintType) { | |||
1819 | default: break; | |||
1820 | case SDTypeConstraint::SDTCisVT: | |||
1821 | if (Constraint.VVT.isSimple()) | |||
1822 | return Constraint.VVT.getSimple().SimpleTy; | |||
1823 | break; | |||
1824 | case SDTypeConstraint::SDTCisPtrTy: | |||
1825 | return MVT::iPTR; | |||
1826 | } | |||
1827 | } | |||
1828 | return MVT::Other; | |||
1829 | } | |||
1830 | ||||
1831 | //===----------------------------------------------------------------------===// | |||
1832 | // TreePatternNode implementation | |||
1833 | // | |||
1834 | ||||
1835 | static unsigned GetNumNodeResults(Record *Operator, CodeGenDAGPatterns &CDP) { | |||
1836 | if (Operator->getName() == "set" || | |||
1837 | Operator->getName() == "implicit") | |||
1838 | return 0; // All return nothing. | |||
1839 | ||||
1840 | if (Operator->isSubClassOf("Intrinsic")) | |||
1841 | return CDP.getIntrinsic(Operator).IS.RetVTs.size(); | |||
1842 | ||||
1843 | if (Operator->isSubClassOf("SDNode")) | |||
1844 | return CDP.getSDNodeInfo(Operator).getNumResults(); | |||
1845 | ||||
1846 | if (Operator->isSubClassOf("PatFrags")) { | |||
1847 | // If we've already parsed this pattern fragment, get it. Otherwise, handle | |||
1848 | // the forward reference case where one pattern fragment references another | |||
1849 | // before it is processed. | |||
1850 | if (TreePattern *PFRec = CDP.getPatternFragmentIfRead(Operator)) { | |||
1851 | // The number of results of a fragment with alternative records is the | |||
1852 | // maximum number of results across all alternatives. | |||
1853 | unsigned NumResults = 0; | |||
1854 | for (const auto &T : PFRec->getTrees()) | |||
1855 | NumResults = std::max(NumResults, T->getNumTypes()); | |||
1856 | return NumResults; | |||
1857 | } | |||
1858 | ||||
1859 | ListInit *LI = Operator->getValueAsListInit("Fragments"); | |||
1860 | assert(LI && "Invalid Fragment")(static_cast <bool> (LI && "Invalid Fragment") ? void (0) : __assert_fail ("LI && \"Invalid Fragment\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 1860, __extension__ __PRETTY_FUNCTION__)); | |||
1861 | unsigned NumResults = 0; | |||
1862 | for (Init *I : LI->getValues()) { | |||
1863 | Record *Op = nullptr; | |||
1864 | if (DagInit *Dag = dyn_cast<DagInit>(I)) | |||
1865 | if (DefInit *DI = dyn_cast<DefInit>(Dag->getOperator())) | |||
1866 | Op = DI->getDef(); | |||
1867 | assert(Op && "Invalid Fragment")(static_cast <bool> (Op && "Invalid Fragment") ? void (0) : __assert_fail ("Op && \"Invalid Fragment\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 1867, __extension__ __PRETTY_FUNCTION__)); | |||
1868 | NumResults = std::max(NumResults, GetNumNodeResults(Op, CDP)); | |||
1869 | } | |||
1870 | return NumResults; | |||
1871 | } | |||
1872 | ||||
1873 | if (Operator->isSubClassOf("Instruction")) { | |||
1874 | CodeGenInstruction &InstInfo = CDP.getTargetInfo().getInstruction(Operator); | |||
1875 | ||||
1876 | unsigned NumDefsToAdd = InstInfo.Operands.NumDefs; | |||
1877 | ||||
1878 | // Subtract any defaulted outputs. | |||
1879 | for (unsigned i = 0; i != InstInfo.Operands.NumDefs; ++i) { | |||
1880 | Record *OperandNode = InstInfo.Operands[i].Rec; | |||
1881 | ||||
1882 | if (OperandNode->isSubClassOf("OperandWithDefaultOps") && | |||
1883 | !CDP.getDefaultOperand(OperandNode).DefaultOps.empty()) | |||
1884 | --NumDefsToAdd; | |||
1885 | } | |||
1886 | ||||
1887 | // Add on one implicit def if it has a resolvable type. | |||
1888 | if (InstInfo.HasOneImplicitDefWithKnownVT(CDP.getTargetInfo()) !=MVT::Other) | |||
1889 | ++NumDefsToAdd; | |||
1890 | return NumDefsToAdd; | |||
1891 | } | |||
1892 | ||||
1893 | if (Operator->isSubClassOf("SDNodeXForm")) | |||
1894 | return 1; // FIXME: Generalize SDNodeXForm | |||
1895 | ||||
1896 | if (Operator->isSubClassOf("ValueType")) | |||
1897 | return 1; // A type-cast of one result. | |||
1898 | ||||
1899 | if (Operator->isSubClassOf("ComplexPattern")) | |||
1900 | return 1; | |||
1901 | ||||
1902 | errs() << *Operator; | |||
1903 | PrintFatalError("Unhandled node in GetNumNodeResults"); | |||
1904 | } | |||
1905 | ||||
1906 | void TreePatternNode::print(raw_ostream &OS) const { | |||
1907 | if (isLeaf()) | |||
1908 | OS << *getLeafValue(); | |||
1909 | else | |||
1910 | OS << '(' << getOperator()->getName(); | |||
1911 | ||||
1912 | for (unsigned i = 0, e = Types.size(); i != e; ++i) { | |||
1913 | OS << ':'; | |||
1914 | getExtType(i).writeToStream(OS); | |||
1915 | } | |||
1916 | ||||
1917 | if (!isLeaf()) { | |||
1918 | if (getNumChildren() != 0) { | |||
1919 | OS << " "; | |||
1920 | ListSeparator LS; | |||
1921 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) { | |||
1922 | OS << LS; | |||
1923 | getChild(i)->print(OS); | |||
1924 | } | |||
1925 | } | |||
1926 | OS << ")"; | |||
1927 | } | |||
1928 | ||||
1929 | for (const TreePredicateCall &Pred : PredicateCalls) { | |||
1930 | OS << "<<P:"; | |||
1931 | if (Pred.Scope) | |||
1932 | OS << Pred.Scope << ":"; | |||
1933 | OS << Pred.Fn.getFnName() << ">>"; | |||
1934 | } | |||
1935 | if (TransformFn) | |||
1936 | OS << "<<X:" << TransformFn->getName() << ">>"; | |||
1937 | if (!getName().empty()) | |||
1938 | OS << ":$" << getName(); | |||
1939 | ||||
1940 | for (const ScopedName &Name : NamesAsPredicateArg) | |||
1941 | OS << ":$pred:" << Name.getScope() << ":" << Name.getIdentifier(); | |||
1942 | } | |||
1943 | void TreePatternNode::dump() const { | |||
1944 | print(errs()); | |||
1945 | } | |||
1946 | ||||
1947 | /// isIsomorphicTo - Return true if this node is recursively | |||
1948 | /// isomorphic to the specified node. For this comparison, the node's | |||
1949 | /// entire state is considered. The assigned name is ignored, since | |||
1950 | /// nodes with differing names are considered isomorphic. However, if | |||
1951 | /// the assigned name is present in the dependent variable set, then | |||
1952 | /// the assigned name is considered significant and the node is | |||
1953 | /// isomorphic if the names match. | |||
1954 | bool TreePatternNode::isIsomorphicTo(const TreePatternNode *N, | |||
1955 | const MultipleUseVarSet &DepVars) const { | |||
1956 | if (N == this) return true; | |||
1957 | if (N->isLeaf() != isLeaf() || getExtTypes() != N->getExtTypes() || | |||
1958 | getPredicateCalls() != N->getPredicateCalls() || | |||
1959 | getTransformFn() != N->getTransformFn()) | |||
1960 | return false; | |||
1961 | ||||
1962 | if (isLeaf()) { | |||
1963 | if (DefInit *DI = dyn_cast<DefInit>(getLeafValue())) { | |||
1964 | if (DefInit *NDI = dyn_cast<DefInit>(N->getLeafValue())) { | |||
1965 | return ((DI->getDef() == NDI->getDef()) | |||
1966 | && (DepVars.find(getName()) == DepVars.end() | |||
1967 | || getName() == N->getName())); | |||
1968 | } | |||
1969 | } | |||
1970 | return getLeafValue() == N->getLeafValue(); | |||
1971 | } | |||
1972 | ||||
1973 | if (N->getOperator() != getOperator() || | |||
1974 | N->getNumChildren() != getNumChildren()) return false; | |||
1975 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) | |||
1976 | if (!getChild(i)->isIsomorphicTo(N->getChild(i), DepVars)) | |||
1977 | return false; | |||
1978 | return true; | |||
1979 | } | |||
1980 | ||||
1981 | /// clone - Make a copy of this tree and all of its children. | |||
1982 | /// | |||
1983 | TreePatternNodePtr TreePatternNode::clone() const { | |||
1984 | TreePatternNodePtr New; | |||
1985 | if (isLeaf()) { | |||
1986 | New = std::make_shared<TreePatternNode>(getLeafValue(), getNumTypes()); | |||
1987 | } else { | |||
1988 | std::vector<TreePatternNodePtr> CChildren; | |||
1989 | CChildren.reserve(Children.size()); | |||
1990 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) | |||
1991 | CChildren.push_back(getChild(i)->clone()); | |||
1992 | New = std::make_shared<TreePatternNode>(getOperator(), std::move(CChildren), | |||
1993 | getNumTypes()); | |||
1994 | } | |||
1995 | New->setName(getName()); | |||
1996 | New->setNamesAsPredicateArg(getNamesAsPredicateArg()); | |||
1997 | New->Types = Types; | |||
1998 | New->setPredicateCalls(getPredicateCalls()); | |||
1999 | New->setTransformFn(getTransformFn()); | |||
2000 | return New; | |||
2001 | } | |||
2002 | ||||
2003 | /// RemoveAllTypes - Recursively strip all the types of this tree. | |||
2004 | void TreePatternNode::RemoveAllTypes() { | |||
2005 | // Reset to unknown type. | |||
2006 | std::fill(Types.begin(), Types.end(), TypeSetByHwMode()); | |||
2007 | if (isLeaf()) return; | |||
2008 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) | |||
2009 | getChild(i)->RemoveAllTypes(); | |||
2010 | } | |||
2011 | ||||
2012 | ||||
2013 | /// SubstituteFormalArguments - Replace the formal arguments in this tree | |||
2014 | /// with actual values specified by ArgMap. | |||
2015 | void TreePatternNode::SubstituteFormalArguments( | |||
2016 | std::map<std::string, TreePatternNodePtr> &ArgMap) { | |||
2017 | if (isLeaf()) return; | |||
2018 | ||||
2019 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) { | |||
2020 | TreePatternNode *Child = getChild(i); | |||
2021 | if (Child->isLeaf()) { | |||
2022 | Init *Val = Child->getLeafValue(); | |||
2023 | // Note that, when substituting into an output pattern, Val might be an | |||
2024 | // UnsetInit. | |||
2025 | if (isa<UnsetInit>(Val) || (isa<DefInit>(Val) && | |||
2026 | cast<DefInit>(Val)->getDef()->getName() == "node")) { | |||
2027 | // We found a use of a formal argument, replace it with its value. | |||
2028 | TreePatternNodePtr NewChild = ArgMap[Child->getName()]; | |||
2029 | assert(NewChild && "Couldn't find formal argument!")(static_cast <bool> (NewChild && "Couldn't find formal argument!" ) ? void (0) : __assert_fail ("NewChild && \"Couldn't find formal argument!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2029, __extension__ __PRETTY_FUNCTION__)); | |||
2030 | assert((Child->getPredicateCalls().empty() ||(static_cast <bool> ((Child->getPredicateCalls().empty () || NewChild->getPredicateCalls() == Child->getPredicateCalls ()) && "Non-empty child predicate clobbered!") ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || NewChild->getPredicateCalls() == Child->getPredicateCalls()) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2032, __extension__ __PRETTY_FUNCTION__)) | |||
2031 | NewChild->getPredicateCalls() == Child->getPredicateCalls()) &&(static_cast <bool> ((Child->getPredicateCalls().empty () || NewChild->getPredicateCalls() == Child->getPredicateCalls ()) && "Non-empty child predicate clobbered!") ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || NewChild->getPredicateCalls() == Child->getPredicateCalls()) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2032, __extension__ __PRETTY_FUNCTION__)) | |||
2032 | "Non-empty child predicate clobbered!")(static_cast <bool> ((Child->getPredicateCalls().empty () || NewChild->getPredicateCalls() == Child->getPredicateCalls ()) && "Non-empty child predicate clobbered!") ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || NewChild->getPredicateCalls() == Child->getPredicateCalls()) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2032, __extension__ __PRETTY_FUNCTION__)); | |||
2033 | setChild(i, std::move(NewChild)); | |||
2034 | } | |||
2035 | } else { | |||
2036 | getChild(i)->SubstituteFormalArguments(ArgMap); | |||
2037 | } | |||
2038 | } | |||
2039 | } | |||
2040 | ||||
2041 | ||||
2042 | /// InlinePatternFragments - If this pattern refers to any pattern | |||
2043 | /// fragments, return the set of inlined versions (this can be more than | |||
2044 | /// one if a PatFrags record has multiple alternatives). | |||
2045 | void TreePatternNode::InlinePatternFragments( | |||
2046 | TreePatternNodePtr T, TreePattern &TP, | |||
2047 | std::vector<TreePatternNodePtr> &OutAlternatives) { | |||
2048 | ||||
2049 | if (TP.hasError()) | |||
2050 | return; | |||
2051 | ||||
2052 | if (isLeaf()) { | |||
2053 | OutAlternatives.push_back(T); // nothing to do. | |||
2054 | return; | |||
2055 | } | |||
2056 | ||||
2057 | Record *Op = getOperator(); | |||
2058 | ||||
2059 | if (!Op->isSubClassOf("PatFrags")) { | |||
2060 | if (getNumChildren() == 0) { | |||
2061 | OutAlternatives.push_back(T); | |||
2062 | return; | |||
2063 | } | |||
2064 | ||||
2065 | // Recursively inline children nodes. | |||
2066 | std::vector<std::vector<TreePatternNodePtr> > ChildAlternatives; | |||
2067 | ChildAlternatives.resize(getNumChildren()); | |||
2068 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) { | |||
2069 | TreePatternNodePtr Child = getChildShared(i); | |||
2070 | Child->InlinePatternFragments(Child, TP, ChildAlternatives[i]); | |||
2071 | // If there are no alternatives for any child, there are no | |||
2072 | // alternatives for this expression as whole. | |||
2073 | if (ChildAlternatives[i].empty()) | |||
2074 | return; | |||
2075 | ||||
2076 | assert((Child->getPredicateCalls().empty() ||(static_cast <bool> ((Child->getPredicateCalls().empty () || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child ->getPredicateCalls(); })) && "Non-empty child predicate clobbered!" ) ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child->getPredicateCalls(); })) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2082, __extension__ __PRETTY_FUNCTION__)) | |||
2077 | llvm::all_of(ChildAlternatives[i],(static_cast <bool> ((Child->getPredicateCalls().empty () || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child ->getPredicateCalls(); })) && "Non-empty child predicate clobbered!" ) ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child->getPredicateCalls(); })) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2082, __extension__ __PRETTY_FUNCTION__)) | |||
2078 | [&](const TreePatternNodePtr &NewChild) {(static_cast <bool> ((Child->getPredicateCalls().empty () || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child ->getPredicateCalls(); })) && "Non-empty child predicate clobbered!" ) ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child->getPredicateCalls(); })) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2082, __extension__ __PRETTY_FUNCTION__)) | |||
2079 | return NewChild->getPredicateCalls() ==(static_cast <bool> ((Child->getPredicateCalls().empty () || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child ->getPredicateCalls(); })) && "Non-empty child predicate clobbered!" ) ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child->getPredicateCalls(); })) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2082, __extension__ __PRETTY_FUNCTION__)) | |||
2080 | Child->getPredicateCalls();(static_cast <bool> ((Child->getPredicateCalls().empty () || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child ->getPredicateCalls(); })) && "Non-empty child predicate clobbered!" ) ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child->getPredicateCalls(); })) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2082, __extension__ __PRETTY_FUNCTION__)) | |||
2081 | })) &&(static_cast <bool> ((Child->getPredicateCalls().empty () || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child ->getPredicateCalls(); })) && "Non-empty child predicate clobbered!" ) ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child->getPredicateCalls(); })) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2082, __extension__ __PRETTY_FUNCTION__)) | |||
2082 | "Non-empty child predicate clobbered!")(static_cast <bool> ((Child->getPredicateCalls().empty () || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child ->getPredicateCalls(); })) && "Non-empty child predicate clobbered!" ) ? void (0) : __assert_fail ("(Child->getPredicateCalls().empty() || llvm::all_of(ChildAlternatives[i], [&](const TreePatternNodePtr &NewChild) { return NewChild->getPredicateCalls() == Child->getPredicateCalls(); })) && \"Non-empty child predicate clobbered!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2082, __extension__ __PRETTY_FUNCTION__)); | |||
2083 | } | |||
2084 | ||||
2085 | // The end result is an all-pairs construction of the resultant pattern. | |||
2086 | std::vector<unsigned> Idxs; | |||
2087 | Idxs.resize(ChildAlternatives.size()); | |||
2088 | bool NotDone; | |||
2089 | do { | |||
2090 | // Create the variant and add it to the output list. | |||
2091 | std::vector<TreePatternNodePtr> NewChildren; | |||
2092 | for (unsigned i = 0, e = ChildAlternatives.size(); i != e; ++i) | |||
2093 | NewChildren.push_back(ChildAlternatives[i][Idxs[i]]); | |||
2094 | TreePatternNodePtr R = std::make_shared<TreePatternNode>( | |||
2095 | getOperator(), std::move(NewChildren), getNumTypes()); | |||
2096 | ||||
2097 | // Copy over properties. | |||
2098 | R->setName(getName()); | |||
2099 | R->setNamesAsPredicateArg(getNamesAsPredicateArg()); | |||
2100 | R->setPredicateCalls(getPredicateCalls()); | |||
2101 | R->setTransformFn(getTransformFn()); | |||
2102 | for (unsigned i = 0, e = getNumTypes(); i != e; ++i) | |||
2103 | R->setType(i, getExtType(i)); | |||
2104 | for (unsigned i = 0, e = getNumResults(); i != e; ++i) | |||
2105 | R->setResultIndex(i, getResultIndex(i)); | |||
2106 | ||||
2107 | // Register alternative. | |||
2108 | OutAlternatives.push_back(R); | |||
2109 | ||||
2110 | // Increment indices to the next permutation by incrementing the | |||
2111 | // indices from last index backward, e.g., generate the sequence | |||
2112 | // [0, 0], [0, 1], [1, 0], [1, 1]. | |||
2113 | int IdxsIdx; | |||
2114 | for (IdxsIdx = Idxs.size() - 1; IdxsIdx >= 0; --IdxsIdx) { | |||
2115 | if (++Idxs[IdxsIdx] == ChildAlternatives[IdxsIdx].size()) | |||
2116 | Idxs[IdxsIdx] = 0; | |||
2117 | else | |||
2118 | break; | |||
2119 | } | |||
2120 | NotDone = (IdxsIdx >= 0); | |||
2121 | } while (NotDone); | |||
2122 | ||||
2123 | return; | |||
2124 | } | |||
2125 | ||||
2126 | // Otherwise, we found a reference to a fragment. First, look up its | |||
2127 | // TreePattern record. | |||
2128 | TreePattern *Frag = TP.getDAGPatterns().getPatternFragment(Op); | |||
2129 | ||||
2130 | // Verify that we are passing the right number of operands. | |||
2131 | if (Frag->getNumArgs() != Children.size()) { | |||
2132 | TP.error("'" + Op->getName() + "' fragment requires " + | |||
2133 | Twine(Frag->getNumArgs()) + " operands!"); | |||
2134 | return; | |||
2135 | } | |||
2136 | ||||
2137 | TreePredicateFn PredFn(Frag); | |||
2138 | unsigned Scope = 0; | |||
2139 | if (TreePredicateFn(Frag).usesOperands()) | |||
2140 | Scope = TP.getDAGPatterns().allocateScope(); | |||
2141 | ||||
2142 | // Compute the map of formal to actual arguments. | |||
2143 | std::map<std::string, TreePatternNodePtr> ArgMap; | |||
2144 | for (unsigned i = 0, e = Frag->getNumArgs(); i != e; ++i) { | |||
2145 | TreePatternNodePtr Child = getChildShared(i); | |||
2146 | if (Scope != 0) { | |||
2147 | Child = Child->clone(); | |||
2148 | Child->addNameAsPredicateArg(ScopedName(Scope, Frag->getArgName(i))); | |||
2149 | } | |||
2150 | ArgMap[Frag->getArgName(i)] = Child; | |||
2151 | } | |||
2152 | ||||
2153 | // Loop over all fragment alternatives. | |||
2154 | for (const auto &Alternative : Frag->getTrees()) { | |||
2155 | TreePatternNodePtr FragTree = Alternative->clone(); | |||
2156 | ||||
2157 | if (!PredFn.isAlwaysTrue()) | |||
2158 | FragTree->addPredicateCall(PredFn, Scope); | |||
2159 | ||||
2160 | // Resolve formal arguments to their actual value. | |||
2161 | if (Frag->getNumArgs()) | |||
2162 | FragTree->SubstituteFormalArguments(ArgMap); | |||
2163 | ||||
2164 | // Transfer types. Note that the resolved alternative may have fewer | |||
2165 | // (but not more) results than the PatFrags node. | |||
2166 | FragTree->setName(getName()); | |||
2167 | for (unsigned i = 0, e = FragTree->getNumTypes(); i != e; ++i) | |||
2168 | FragTree->UpdateNodeType(i, getExtType(i), TP); | |||
2169 | ||||
2170 | // Transfer in the old predicates. | |||
2171 | for (const TreePredicateCall &Pred : getPredicateCalls()) | |||
2172 | FragTree->addPredicateCall(Pred); | |||
2173 | ||||
2174 | // The fragment we inlined could have recursive inlining that is needed. See | |||
2175 | // if there are any pattern fragments in it and inline them as needed. | |||
2176 | FragTree->InlinePatternFragments(FragTree, TP, OutAlternatives); | |||
2177 | } | |||
2178 | } | |||
2179 | ||||
2180 | /// getImplicitType - Check to see if the specified record has an implicit | |||
2181 | /// type which should be applied to it. This will infer the type of register | |||
2182 | /// references from the register file information, for example. | |||
2183 | /// | |||
2184 | /// When Unnamed is set, return the type of a DAG operand with no name, such as | |||
2185 | /// the F8RC register class argument in: | |||
2186 | /// | |||
2187 | /// (COPY_TO_REGCLASS GPR:$src, F8RC) | |||
2188 | /// | |||
2189 | /// When Unnamed is false, return the type of a named DAG operand such as the | |||
2190 | /// GPR:$src operand above. | |||
2191 | /// | |||
2192 | static TypeSetByHwMode getImplicitType(Record *R, unsigned ResNo, | |||
2193 | bool NotRegisters, | |||
2194 | bool Unnamed, | |||
2195 | TreePattern &TP) { | |||
2196 | CodeGenDAGPatterns &CDP = TP.getDAGPatterns(); | |||
2197 | ||||
2198 | // Check to see if this is a register operand. | |||
2199 | if (R->isSubClassOf("RegisterOperand")) { | |||
2200 | assert(ResNo == 0 && "Regoperand ref only has one result!")(static_cast <bool> (ResNo == 0 && "Regoperand ref only has one result!" ) ? void (0) : __assert_fail ("ResNo == 0 && \"Regoperand ref only has one result!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2200, __extension__ __PRETTY_FUNCTION__)); | |||
2201 | if (NotRegisters) | |||
2202 | return TypeSetByHwMode(); // Unknown. | |||
2203 | Record *RegClass = R->getValueAsDef("RegClass"); | |||
2204 | const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); | |||
2205 | return TypeSetByHwMode(T.getRegisterClass(RegClass).getValueTypes()); | |||
2206 | } | |||
2207 | ||||
2208 | // Check to see if this is a register or a register class. | |||
2209 | if (R->isSubClassOf("RegisterClass")) { | |||
2210 | assert(ResNo == 0 && "Regclass ref only has one result!")(static_cast <bool> (ResNo == 0 && "Regclass ref only has one result!" ) ? void (0) : __assert_fail ("ResNo == 0 && \"Regclass ref only has one result!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2210, __extension__ __PRETTY_FUNCTION__)); | |||
2211 | // An unnamed register class represents itself as an i32 immediate, for | |||
2212 | // example on a COPY_TO_REGCLASS instruction. | |||
2213 | if (Unnamed) | |||
2214 | return TypeSetByHwMode(MVT::i32); | |||
2215 | ||||
2216 | // In a named operand, the register class provides the possible set of | |||
2217 | // types. | |||
2218 | if (NotRegisters) | |||
2219 | return TypeSetByHwMode(); // Unknown. | |||
2220 | const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); | |||
2221 | return TypeSetByHwMode(T.getRegisterClass(R).getValueTypes()); | |||
2222 | } | |||
2223 | ||||
2224 | if (R->isSubClassOf("PatFrags")) { | |||
2225 | assert(ResNo == 0 && "FIXME: PatFrag with multiple results?")(static_cast <bool> (ResNo == 0 && "FIXME: PatFrag with multiple results?" ) ? void (0) : __assert_fail ("ResNo == 0 && \"FIXME: PatFrag with multiple results?\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2225, __extension__ __PRETTY_FUNCTION__)); | |||
2226 | // Pattern fragment types will be resolved when they are inlined. | |||
2227 | return TypeSetByHwMode(); // Unknown. | |||
2228 | } | |||
2229 | ||||
2230 | if (R->isSubClassOf("Register")) { | |||
2231 | assert(ResNo == 0 && "Registers only produce one result!")(static_cast <bool> (ResNo == 0 && "Registers only produce one result!" ) ? void (0) : __assert_fail ("ResNo == 0 && \"Registers only produce one result!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2231, __extension__ __PRETTY_FUNCTION__)); | |||
2232 | if (NotRegisters) | |||
2233 | return TypeSetByHwMode(); // Unknown. | |||
2234 | const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); | |||
2235 | return TypeSetByHwMode(T.getRegisterVTs(R)); | |||
2236 | } | |||
2237 | ||||
2238 | if (R->isSubClassOf("SubRegIndex")) { | |||
2239 | assert(ResNo == 0 && "SubRegisterIndices only produce one result!")(static_cast <bool> (ResNo == 0 && "SubRegisterIndices only produce one result!" ) ? void (0) : __assert_fail ("ResNo == 0 && \"SubRegisterIndices only produce one result!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2239, __extension__ __PRETTY_FUNCTION__)); | |||
2240 | return TypeSetByHwMode(MVT::i32); | |||
2241 | } | |||
2242 | ||||
2243 | if (R->isSubClassOf("ValueType")) { | |||
2244 | assert(ResNo == 0 && "This node only has one result!")(static_cast <bool> (ResNo == 0 && "This node only has one result!" ) ? void (0) : __assert_fail ("ResNo == 0 && \"This node only has one result!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2244, __extension__ __PRETTY_FUNCTION__)); | |||
2245 | // An unnamed VTSDNode represents itself as an MVT::Other immediate. | |||
2246 | // | |||
2247 | // (sext_inreg GPR:$src, i16) | |||
2248 | // ~~~ | |||
2249 | if (Unnamed) | |||
2250 | return TypeSetByHwMode(MVT::Other); | |||
2251 | // With a name, the ValueType simply provides the type of the named | |||
2252 | // variable. | |||
2253 | // | |||
2254 | // (sext_inreg i32:$src, i16) | |||
2255 | // ~~~~~~~~ | |||
2256 | if (NotRegisters) | |||
2257 | return TypeSetByHwMode(); // Unknown. | |||
2258 | const CodeGenHwModes &CGH = CDP.getTargetInfo().getHwModes(); | |||
2259 | return TypeSetByHwMode(getValueTypeByHwMode(R, CGH)); | |||
2260 | } | |||
2261 | ||||
2262 | if (R->isSubClassOf("CondCode")) { | |||
2263 | assert(ResNo == 0 && "This node only has one result!")(static_cast <bool> (ResNo == 0 && "This node only has one result!" ) ? void (0) : __assert_fail ("ResNo == 0 && \"This node only has one result!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2263, __extension__ __PRETTY_FUNCTION__)); | |||
2264 | // Using a CondCodeSDNode. | |||
2265 | return TypeSetByHwMode(MVT::Other); | |||
2266 | } | |||
2267 | ||||
2268 | if (R->isSubClassOf("ComplexPattern")) { | |||
2269 | assert(ResNo == 0 && "FIXME: ComplexPattern with multiple results?")(static_cast <bool> (ResNo == 0 && "FIXME: ComplexPattern with multiple results?" ) ? void (0) : __assert_fail ("ResNo == 0 && \"FIXME: ComplexPattern with multiple results?\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2269, __extension__ __PRETTY_FUNCTION__)); | |||
2270 | if (NotRegisters) | |||
2271 | return TypeSetByHwMode(); // Unknown. | |||
2272 | Record *T = CDP.getComplexPattern(R).getValueType(); | |||
2273 | const CodeGenHwModes &CGH = CDP.getTargetInfo().getHwModes(); | |||
2274 | return TypeSetByHwMode(getValueTypeByHwMode(T, CGH)); | |||
2275 | } | |||
2276 | if (R->isSubClassOf("PointerLikeRegClass")) { | |||
2277 | assert(ResNo == 0 && "Regclass can only have one result!")(static_cast <bool> (ResNo == 0 && "Regclass can only have one result!" ) ? void (0) : __assert_fail ("ResNo == 0 && \"Regclass can only have one result!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2277, __extension__ __PRETTY_FUNCTION__)); | |||
2278 | TypeSetByHwMode VTS(MVT::iPTR); | |||
2279 | TP.getInfer().expandOverloads(VTS); | |||
2280 | return VTS; | |||
2281 | } | |||
2282 | ||||
2283 | if (R->getName() == "node" || R->getName() == "srcvalue" || | |||
2284 | R->getName() == "zero_reg" || R->getName() == "immAllOnesV" || | |||
2285 | R->getName() == "immAllZerosV" || R->getName() == "undef_tied_input") { | |||
2286 | // Placeholder. | |||
2287 | return TypeSetByHwMode(); // Unknown. | |||
2288 | } | |||
2289 | ||||
2290 | if (R->isSubClassOf("Operand")) { | |||
2291 | const CodeGenHwModes &CGH = CDP.getTargetInfo().getHwModes(); | |||
2292 | Record *T = R->getValueAsDef("Type"); | |||
2293 | return TypeSetByHwMode(getValueTypeByHwMode(T, CGH)); | |||
2294 | } | |||
2295 | ||||
2296 | TP.error("Unknown node flavor used in pattern: " + R->getName()); | |||
2297 | return TypeSetByHwMode(MVT::Other); | |||
2298 | } | |||
2299 | ||||
2300 | ||||
2301 | /// getIntrinsicInfo - If this node corresponds to an intrinsic, return the | |||
2302 | /// CodeGenIntrinsic information for it, otherwise return a null pointer. | |||
2303 | const CodeGenIntrinsic *TreePatternNode:: | |||
2304 | getIntrinsicInfo(const CodeGenDAGPatterns &CDP) const { | |||
2305 | if (getOperator() != CDP.get_intrinsic_void_sdnode() && | |||
2306 | getOperator() != CDP.get_intrinsic_w_chain_sdnode() && | |||
2307 | getOperator() != CDP.get_intrinsic_wo_chain_sdnode()) | |||
2308 | return nullptr; | |||
2309 | ||||
2310 | unsigned IID = cast<IntInit>(getChild(0)->getLeafValue())->getValue(); | |||
2311 | return &CDP.getIntrinsicInfo(IID); | |||
2312 | } | |||
2313 | ||||
2314 | /// getComplexPatternInfo - If this node corresponds to a ComplexPattern, | |||
2315 | /// return the ComplexPattern information, otherwise return null. | |||
2316 | const ComplexPattern * | |||
2317 | TreePatternNode::getComplexPatternInfo(const CodeGenDAGPatterns &CGP) const { | |||
2318 | Record *Rec; | |||
2319 | if (isLeaf()) { | |||
2320 | DefInit *DI = dyn_cast<DefInit>(getLeafValue()); | |||
2321 | if (!DI) | |||
2322 | return nullptr; | |||
2323 | Rec = DI->getDef(); | |||
2324 | } else | |||
2325 | Rec = getOperator(); | |||
2326 | ||||
2327 | if (!Rec->isSubClassOf("ComplexPattern")) | |||
2328 | return nullptr; | |||
2329 | return &CGP.getComplexPattern(Rec); | |||
2330 | } | |||
2331 | ||||
2332 | unsigned TreePatternNode::getNumMIResults(const CodeGenDAGPatterns &CGP) const { | |||
2333 | // A ComplexPattern specifically declares how many results it fills in. | |||
2334 | if (const ComplexPattern *CP = getComplexPatternInfo(CGP)) | |||
2335 | return CP->getNumOperands(); | |||
2336 | ||||
2337 | // If MIOperandInfo is specified, that gives the count. | |||
2338 | if (isLeaf()) { | |||
2339 | DefInit *DI = dyn_cast<DefInit>(getLeafValue()); | |||
2340 | if (DI && DI->getDef()->isSubClassOf("Operand")) { | |||
2341 | DagInit *MIOps = DI->getDef()->getValueAsDag("MIOperandInfo"); | |||
2342 | if (MIOps->getNumArgs()) | |||
2343 | return MIOps->getNumArgs(); | |||
2344 | } | |||
2345 | } | |||
2346 | ||||
2347 | // Otherwise there is just one result. | |||
2348 | return 1; | |||
2349 | } | |||
2350 | ||||
2351 | /// NodeHasProperty - Return true if this node has the specified property. | |||
2352 | bool TreePatternNode::NodeHasProperty(SDNP Property, | |||
2353 | const CodeGenDAGPatterns &CGP) const { | |||
2354 | if (isLeaf()) { | |||
2355 | if (const ComplexPattern *CP = getComplexPatternInfo(CGP)) | |||
2356 | return CP->hasProperty(Property); | |||
2357 | ||||
2358 | return false; | |||
2359 | } | |||
2360 | ||||
2361 | if (Property != SDNPHasChain) { | |||
2362 | // The chain proprety is already present on the different intrinsic node | |||
2363 | // types (intrinsic_w_chain, intrinsic_void), and is not explicitly listed | |||
2364 | // on the intrinsic. Anything else is specific to the individual intrinsic. | |||
2365 | if (const CodeGenIntrinsic *Int = getIntrinsicInfo(CGP)) | |||
2366 | return Int->hasProperty(Property); | |||
2367 | } | |||
2368 | ||||
2369 | if (!Operator->isSubClassOf("SDPatternOperator")) | |||
2370 | return false; | |||
2371 | ||||
2372 | return CGP.getSDNodeInfo(Operator).hasProperty(Property); | |||
2373 | } | |||
2374 | ||||
2375 | ||||
2376 | ||||
2377 | ||||
2378 | /// TreeHasProperty - Return true if any node in this tree has the specified | |||
2379 | /// property. | |||
2380 | bool TreePatternNode::TreeHasProperty(SDNP Property, | |||
2381 | const CodeGenDAGPatterns &CGP) const { | |||
2382 | if (NodeHasProperty(Property, CGP)) | |||
2383 | return true; | |||
2384 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) | |||
2385 | if (getChild(i)->TreeHasProperty(Property, CGP)) | |||
2386 | return true; | |||
2387 | return false; | |||
2388 | } | |||
2389 | ||||
2390 | /// isCommutativeIntrinsic - Return true if the node corresponds to a | |||
2391 | /// commutative intrinsic. | |||
2392 | bool | |||
2393 | TreePatternNode::isCommutativeIntrinsic(const CodeGenDAGPatterns &CDP) const { | |||
2394 | if (const CodeGenIntrinsic *Int = getIntrinsicInfo(CDP)) | |||
2395 | return Int->isCommutative; | |||
2396 | return false; | |||
2397 | } | |||
2398 | ||||
2399 | static bool isOperandClass(const TreePatternNode *N, StringRef Class) { | |||
2400 | if (!N->isLeaf()) | |||
2401 | return N->getOperator()->isSubClassOf(Class); | |||
2402 | ||||
2403 | DefInit *DI = dyn_cast<DefInit>(N->getLeafValue()); | |||
2404 | if (DI && DI->getDef()->isSubClassOf(Class)) | |||
2405 | return true; | |||
2406 | ||||
2407 | return false; | |||
2408 | } | |||
2409 | ||||
2410 | static void emitTooManyOperandsError(TreePattern &TP, | |||
2411 | StringRef InstName, | |||
2412 | unsigned Expected, | |||
2413 | unsigned Actual) { | |||
2414 | TP.error("Instruction '" + InstName + "' was provided " + Twine(Actual) + | |||
2415 | " operands but expected only " + Twine(Expected) + "!"); | |||
2416 | } | |||
2417 | ||||
2418 | static void emitTooFewOperandsError(TreePattern &TP, | |||
2419 | StringRef InstName, | |||
2420 | unsigned Actual) { | |||
2421 | TP.error("Instruction '" + InstName + | |||
2422 | "' expects more than the provided " + Twine(Actual) + " operands!"); | |||
2423 | } | |||
2424 | ||||
2425 | /// ApplyTypeConstraints - Apply all of the type constraints relevant to | |||
2426 | /// this node and its children in the tree. This returns true if it makes a | |||
2427 | /// change, false otherwise. If a type contradiction is found, flag an error. | |||
2428 | bool TreePatternNode::ApplyTypeConstraints(TreePattern &TP, bool NotRegisters) { | |||
2429 | if (TP.hasError()) | |||
2430 | return false; | |||
2431 | ||||
2432 | CodeGenDAGPatterns &CDP = TP.getDAGPatterns(); | |||
2433 | if (isLeaf()) { | |||
2434 | if (DefInit *DI = dyn_cast<DefInit>(getLeafValue())) { | |||
2435 | // If it's a regclass or something else known, include the type. | |||
2436 | bool MadeChange = false; | |||
2437 | for (unsigned i = 0, e = Types.size(); i != e; ++i) | |||
2438 | MadeChange |= UpdateNodeType(i, getImplicitType(DI->getDef(), i, | |||
2439 | NotRegisters, | |||
2440 | !hasName(), TP), TP); | |||
2441 | return MadeChange; | |||
2442 | } | |||
2443 | ||||
2444 | if (IntInit *II = dyn_cast<IntInit>(getLeafValue())) { | |||
2445 | assert(Types.size() == 1 && "Invalid IntInit")(static_cast <bool> (Types.size() == 1 && "Invalid IntInit" ) ? void (0) : __assert_fail ("Types.size() == 1 && \"Invalid IntInit\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2445, __extension__ __PRETTY_FUNCTION__)); | |||
2446 | ||||
2447 | // Int inits are always integers. :) | |||
2448 | bool MadeChange = TP.getInfer().EnforceInteger(Types[0]); | |||
2449 | ||||
2450 | if (!TP.getInfer().isConcrete(Types[0], false)) | |||
2451 | return MadeChange; | |||
2452 | ||||
2453 | ValueTypeByHwMode VVT = TP.getInfer().getConcrete(Types[0], false); | |||
2454 | for (auto &P : VVT) { | |||
2455 | MVT::SimpleValueType VT = P.second.SimpleTy; | |||
2456 | if (VT == MVT::iPTR || VT == MVT::iPTRAny) | |||
2457 | continue; | |||
2458 | unsigned Size = MVT(VT).getFixedSizeInBits(); | |||
2459 | // Make sure that the value is representable for this type. | |||
2460 | if (Size >= 32) | |||
2461 | continue; | |||
2462 | // Check that the value doesn't use more bits than we have. It must | |||
2463 | // either be a sign- or zero-extended equivalent of the original. | |||
2464 | int64_t SignBitAndAbove = II->getValue() >> (Size - 1); | |||
2465 | if (SignBitAndAbove == -1 || SignBitAndAbove == 0 || | |||
2466 | SignBitAndAbove == 1) | |||
2467 | continue; | |||
2468 | ||||
2469 | TP.error("Integer value '" + Twine(II->getValue()) + | |||
2470 | "' is out of range for type '" + getEnumName(VT) + "'!"); | |||
2471 | break; | |||
2472 | } | |||
2473 | return MadeChange; | |||
2474 | } | |||
2475 | ||||
2476 | return false; | |||
2477 | } | |||
2478 | ||||
2479 | if (const CodeGenIntrinsic *Int = getIntrinsicInfo(CDP)) { | |||
2480 | bool MadeChange = false; | |||
2481 | ||||
2482 | // Apply the result type to the node. | |||
2483 | unsigned NumRetVTs = Int->IS.RetVTs.size(); | |||
2484 | unsigned NumParamVTs = Int->IS.ParamVTs.size(); | |||
2485 | ||||
2486 | for (unsigned i = 0, e = NumRetVTs; i != e; ++i) | |||
2487 | MadeChange |= UpdateNodeType(i, Int->IS.RetVTs[i], TP); | |||
2488 | ||||
2489 | if (getNumChildren() != NumParamVTs + 1) { | |||
2490 | TP.error("Intrinsic '" + Int->Name + "' expects " + Twine(NumParamVTs) + | |||
2491 | " operands, not " + Twine(getNumChildren() - 1) + " operands!"); | |||
2492 | return false; | |||
2493 | } | |||
2494 | ||||
2495 | // Apply type info to the intrinsic ID. | |||
2496 | MadeChange |= getChild(0)->UpdateNodeType(0, MVT::iPTR, TP); | |||
2497 | ||||
2498 | for (unsigned i = 0, e = getNumChildren()-1; i != e; ++i) { | |||
2499 | MadeChange |= getChild(i+1)->ApplyTypeConstraints(TP, NotRegisters); | |||
2500 | ||||
2501 | MVT::SimpleValueType OpVT = Int->IS.ParamVTs[i]; | |||
2502 | assert(getChild(i+1)->getNumTypes() == 1 && "Unhandled case")(static_cast <bool> (getChild(i+1)->getNumTypes() == 1 && "Unhandled case") ? void (0) : __assert_fail ("getChild(i+1)->getNumTypes() == 1 && \"Unhandled case\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2502, __extension__ __PRETTY_FUNCTION__)); | |||
2503 | MadeChange |= getChild(i+1)->UpdateNodeType(0, OpVT, TP); | |||
2504 | } | |||
2505 | return MadeChange; | |||
2506 | } | |||
2507 | ||||
2508 | if (getOperator()->isSubClassOf("SDNode")) { | |||
2509 | const SDNodeInfo &NI = CDP.getSDNodeInfo(getOperator()); | |||
2510 | ||||
2511 | // Check that the number of operands is sane. Negative operands -> varargs. | |||
2512 | if (NI.getNumOperands() >= 0 && | |||
2513 | getNumChildren() != (unsigned)NI.getNumOperands()) { | |||
2514 | TP.error(getOperator()->getName() + " node requires exactly " + | |||
2515 | Twine(NI.getNumOperands()) + " operands!"); | |||
2516 | return false; | |||
2517 | } | |||
2518 | ||||
2519 | bool MadeChange = false; | |||
2520 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) | |||
2521 | MadeChange |= getChild(i)->ApplyTypeConstraints(TP, NotRegisters); | |||
2522 | MadeChange |= NI.ApplyTypeConstraints(this, TP); | |||
2523 | return MadeChange; | |||
2524 | } | |||
2525 | ||||
2526 | if (getOperator()->isSubClassOf("Instruction")) { | |||
2527 | const DAGInstruction &Inst = CDP.getInstruction(getOperator()); | |||
2528 | CodeGenInstruction &InstInfo = | |||
2529 | CDP.getTargetInfo().getInstruction(getOperator()); | |||
2530 | ||||
2531 | bool MadeChange = false; | |||
2532 | ||||
2533 | // Apply the result types to the node, these come from the things in the | |||
2534 | // (outs) list of the instruction. | |||
2535 | unsigned NumResultsToAdd = std::min(InstInfo.Operands.NumDefs, | |||
2536 | Inst.getNumResults()); | |||
2537 | for (unsigned ResNo = 0; ResNo != NumResultsToAdd; ++ResNo) | |||
2538 | MadeChange |= UpdateNodeTypeFromInst(ResNo, Inst.getResult(ResNo), TP); | |||
2539 | ||||
2540 | // If the instruction has implicit defs, we apply the first one as a result. | |||
2541 | // FIXME: This sucks, it should apply all implicit defs. | |||
2542 | if (!InstInfo.ImplicitDefs.empty()) { | |||
2543 | unsigned ResNo = NumResultsToAdd; | |||
2544 | ||||
2545 | // FIXME: Generalize to multiple possible types and multiple possible | |||
2546 | // ImplicitDefs. | |||
2547 | MVT::SimpleValueType VT = | |||
2548 | InstInfo.HasOneImplicitDefWithKnownVT(CDP.getTargetInfo()); | |||
2549 | ||||
2550 | if (VT != MVT::Other) | |||
2551 | MadeChange |= UpdateNodeType(ResNo, VT, TP); | |||
2552 | } | |||
2553 | ||||
2554 | // If this is an INSERT_SUBREG, constrain the source and destination VTs to | |||
2555 | // be the same. | |||
2556 | if (getOperator()->getName() == "INSERT_SUBREG") { | |||
2557 | assert(getChild(0)->getNumTypes() == 1 && "FIXME: Unhandled")(static_cast <bool> (getChild(0)->getNumTypes() == 1 && "FIXME: Unhandled") ? void (0) : __assert_fail ("getChild(0)->getNumTypes() == 1 && \"FIXME: Unhandled\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2557, __extension__ __PRETTY_FUNCTION__)); | |||
2558 | MadeChange |= UpdateNodeType(0, getChild(0)->getExtType(0), TP); | |||
2559 | MadeChange |= getChild(0)->UpdateNodeType(0, getExtType(0), TP); | |||
2560 | } else if (getOperator()->getName() == "REG_SEQUENCE") { | |||
2561 | // We need to do extra, custom typechecking for REG_SEQUENCE since it is | |||
2562 | // variadic. | |||
2563 | ||||
2564 | unsigned NChild = getNumChildren(); | |||
2565 | if (NChild < 3) { | |||
2566 | TP.error("REG_SEQUENCE requires at least 3 operands!"); | |||
2567 | return false; | |||
2568 | } | |||
2569 | ||||
2570 | if (NChild % 2 == 0) { | |||
2571 | TP.error("REG_SEQUENCE requires an odd number of operands!"); | |||
2572 | return false; | |||
2573 | } | |||
2574 | ||||
2575 | if (!isOperandClass(getChild(0), "RegisterClass")) { | |||
2576 | TP.error("REG_SEQUENCE requires a RegisterClass for first operand!"); | |||
2577 | return false; | |||
2578 | } | |||
2579 | ||||
2580 | for (unsigned I = 1; I < NChild; I += 2) { | |||
2581 | TreePatternNode *SubIdxChild = getChild(I + 1); | |||
2582 | if (!isOperandClass(SubIdxChild, "SubRegIndex")) { | |||
2583 | TP.error("REG_SEQUENCE requires a SubRegIndex for operand " + | |||
2584 | Twine(I + 1) + "!"); | |||
2585 | return false; | |||
2586 | } | |||
2587 | } | |||
2588 | } | |||
2589 | ||||
2590 | unsigned NumResults = Inst.getNumResults(); | |||
2591 | unsigned NumFixedOperands = InstInfo.Operands.size(); | |||
2592 | ||||
2593 | // If one or more operands with a default value appear at the end of the | |||
2594 | // formal operand list for an instruction, we allow them to be overridden | |||
2595 | // by optional operands provided in the pattern. | |||
2596 | // | |||
2597 | // But if an operand B without a default appears at any point after an | |||
2598 | // operand A with a default, then we don't allow A to be overridden, | |||
2599 | // because there would be no way to specify whether the next operand in | |||
2600 | // the pattern was intended to override A or skip it. | |||
2601 | unsigned NonOverridableOperands = NumFixedOperands; | |||
2602 | while (NonOverridableOperands > NumResults && | |||
2603 | CDP.operandHasDefault(InstInfo.Operands[NonOverridableOperands-1].Rec)) | |||
2604 | --NonOverridableOperands; | |||
2605 | ||||
2606 | unsigned ChildNo = 0; | |||
2607 | assert(NumResults <= NumFixedOperands)(static_cast <bool> (NumResults <= NumFixedOperands) ? void (0) : __assert_fail ("NumResults <= NumFixedOperands" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2607, __extension__ __PRETTY_FUNCTION__)); | |||
2608 | for (unsigned i = NumResults, e = NumFixedOperands; i != e; ++i) { | |||
2609 | Record *OperandNode = InstInfo.Operands[i].Rec; | |||
2610 | ||||
2611 | // If the operand has a default value, do we use it? We must use the | |||
2612 | // default if we've run out of children of the pattern DAG to consume, | |||
2613 | // or if the operand is followed by a non-defaulted one. | |||
2614 | if (CDP.operandHasDefault(OperandNode) && | |||
2615 | (i < NonOverridableOperands || ChildNo >= getNumChildren())) | |||
2616 | continue; | |||
2617 | ||||
2618 | // If we have run out of child nodes and there _isn't_ a default | |||
2619 | // value we can use for the next operand, give an error. | |||
2620 | if (ChildNo >= getNumChildren()) { | |||
2621 | emitTooFewOperandsError(TP, getOperator()->getName(), getNumChildren()); | |||
2622 | return false; | |||
2623 | } | |||
2624 | ||||
2625 | TreePatternNode *Child = getChild(ChildNo++); | |||
2626 | unsigned ChildResNo = 0; // Instructions always use res #0 of their op. | |||
2627 | ||||
2628 | // If the operand has sub-operands, they may be provided by distinct | |||
2629 | // child patterns, so attempt to match each sub-operand separately. | |||
2630 | if (OperandNode->isSubClassOf("Operand")) { | |||
2631 | DagInit *MIOpInfo = OperandNode->getValueAsDag("MIOperandInfo"); | |||
2632 | if (unsigned NumArgs = MIOpInfo->getNumArgs()) { | |||
2633 | // But don't do that if the whole operand is being provided by | |||
2634 | // a single ComplexPattern-related Operand. | |||
2635 | ||||
2636 | if (Child->getNumMIResults(CDP) < NumArgs) { | |||
2637 | // Match first sub-operand against the child we already have. | |||
2638 | Record *SubRec = cast<DefInit>(MIOpInfo->getArg(0))->getDef(); | |||
2639 | MadeChange |= | |||
2640 | Child->UpdateNodeTypeFromInst(ChildResNo, SubRec, TP); | |||
2641 | ||||
2642 | // And the remaining sub-operands against subsequent children. | |||
2643 | for (unsigned Arg = 1; Arg < NumArgs; ++Arg) { | |||
2644 | if (ChildNo >= getNumChildren()) { | |||
2645 | emitTooFewOperandsError(TP, getOperator()->getName(), | |||
2646 | getNumChildren()); | |||
2647 | return false; | |||
2648 | } | |||
2649 | Child = getChild(ChildNo++); | |||
2650 | ||||
2651 | SubRec = cast<DefInit>(MIOpInfo->getArg(Arg))->getDef(); | |||
2652 | MadeChange |= | |||
2653 | Child->UpdateNodeTypeFromInst(ChildResNo, SubRec, TP); | |||
2654 | } | |||
2655 | continue; | |||
2656 | } | |||
2657 | } | |||
2658 | } | |||
2659 | ||||
2660 | // If we didn't match by pieces above, attempt to match the whole | |||
2661 | // operand now. | |||
2662 | MadeChange |= Child->UpdateNodeTypeFromInst(ChildResNo, OperandNode, TP); | |||
2663 | } | |||
2664 | ||||
2665 | if (!InstInfo.Operands.isVariadic && ChildNo != getNumChildren()) { | |||
2666 | emitTooManyOperandsError(TP, getOperator()->getName(), | |||
2667 | ChildNo, getNumChildren()); | |||
2668 | return false; | |||
2669 | } | |||
2670 | ||||
2671 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) | |||
2672 | MadeChange |= getChild(i)->ApplyTypeConstraints(TP, NotRegisters); | |||
2673 | return MadeChange; | |||
2674 | } | |||
2675 | ||||
2676 | if (getOperator()->isSubClassOf("ComplexPattern")) { | |||
2677 | bool MadeChange = false; | |||
2678 | ||||
2679 | if (!NotRegisters) { | |||
2680 | assert(Types.size() == 1 && "ComplexPatterns only produce one result!")(static_cast <bool> (Types.size() == 1 && "ComplexPatterns only produce one result!" ) ? void (0) : __assert_fail ("Types.size() == 1 && \"ComplexPatterns only produce one result!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2680, __extension__ __PRETTY_FUNCTION__)); | |||
2681 | Record *T = CDP.getComplexPattern(getOperator()).getValueType(); | |||
2682 | const CodeGenHwModes &CGH = CDP.getTargetInfo().getHwModes(); | |||
2683 | const ValueTypeByHwMode VVT = getValueTypeByHwMode(T, CGH); | |||
2684 | // TODO: AArch64 and AMDGPU use ComplexPattern<untyped, ...> and then | |||
2685 | // exclusively use those as non-leaf nodes with explicit type casts, so | |||
2686 | // for backwards compatibility we do no inference in that case. This is | |||
2687 | // not supported when the ComplexPattern is used as a leaf value, | |||
2688 | // however; this inconsistency should be resolved, either by adding this | |||
2689 | // case there or by altering the backends to not do this (e.g. using Any | |||
2690 | // instead may work). | |||
2691 | if (!VVT.isSimple() || VVT.getSimple() != MVT::Untyped) | |||
2692 | MadeChange |= UpdateNodeType(0, VVT, TP); | |||
2693 | } | |||
2694 | ||||
2695 | for (unsigned i = 0; i < getNumChildren(); ++i) | |||
2696 | MadeChange |= getChild(i)->ApplyTypeConstraints(TP, NotRegisters); | |||
2697 | ||||
2698 | return MadeChange; | |||
2699 | } | |||
2700 | ||||
2701 | assert(getOperator()->isSubClassOf("SDNodeXForm") && "Unknown node type!")(static_cast <bool> (getOperator()->isSubClassOf("SDNodeXForm" ) && "Unknown node type!") ? void (0) : __assert_fail ("getOperator()->isSubClassOf(\"SDNodeXForm\") && \"Unknown node type!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 2701, __extension__ __PRETTY_FUNCTION__)); | |||
2702 | ||||
2703 | // Node transforms always take one operand. | |||
2704 | if (getNumChildren() != 1) { | |||
2705 | TP.error("Node transform '" + getOperator()->getName() + | |||
2706 | "' requires one operand!"); | |||
2707 | return false; | |||
2708 | } | |||
2709 | ||||
2710 | bool MadeChange = getChild(0)->ApplyTypeConstraints(TP, NotRegisters); | |||
2711 | return MadeChange; | |||
2712 | } | |||
2713 | ||||
2714 | /// OnlyOnRHSOfCommutative - Return true if this value is only allowed on the | |||
2715 | /// RHS of a commutative operation, not the on LHS. | |||
2716 | static bool OnlyOnRHSOfCommutative(TreePatternNode *N) { | |||
2717 | if (!N->isLeaf() && N->getOperator()->getName() == "imm") | |||
2718 | return true; | |||
2719 | if (N->isLeaf() && isa<IntInit>(N->getLeafValue())) | |||
2720 | return true; | |||
2721 | if (isImmAllOnesAllZerosMatch(N)) | |||
2722 | return true; | |||
2723 | return false; | |||
2724 | } | |||
2725 | ||||
2726 | ||||
2727 | /// canPatternMatch - If it is impossible for this pattern to match on this | |||
2728 | /// target, fill in Reason and return false. Otherwise, return true. This is | |||
2729 | /// used as a sanity check for .td files (to prevent people from writing stuff | |||
2730 | /// that can never possibly work), and to prevent the pattern permuter from | |||
2731 | /// generating stuff that is useless. | |||
2732 | bool TreePatternNode::canPatternMatch(std::string &Reason, | |||
2733 | const CodeGenDAGPatterns &CDP) { | |||
2734 | if (isLeaf()) return true; | |||
2735 | ||||
2736 | for (unsigned i = 0, e = getNumChildren(); i != e; ++i) | |||
2737 | if (!getChild(i)->canPatternMatch(Reason, CDP)) | |||
2738 | return false; | |||
2739 | ||||
2740 | // If this is an intrinsic, handle cases that would make it not match. For | |||
2741 | // example, if an operand is required to be an immediate. | |||
2742 | if (getOperator()->isSubClassOf("Intrinsic")) { | |||
2743 | // TODO: | |||
2744 | return true; | |||
2745 | } | |||
2746 | ||||
2747 | if (getOperator()->isSubClassOf("ComplexPattern")) | |||
2748 | return true; | |||
2749 | ||||
2750 | // If this node is a commutative operator, check that the LHS isn't an | |||
2751 | // immediate. | |||
2752 | const SDNodeInfo &NodeInfo = CDP.getSDNodeInfo(getOperator()); | |||
2753 | bool isCommIntrinsic = isCommutativeIntrinsic(CDP); | |||
2754 | if (NodeInfo.hasProperty(SDNPCommutative) || isCommIntrinsic) { | |||
2755 | // Scan all of the operands of the node and make sure that only the last one | |||
2756 | // is a constant node, unless the RHS also is. | |||
2757 | if (!OnlyOnRHSOfCommutative(getChild(getNumChildren()-1))) { | |||
2758 | unsigned Skip = isCommIntrinsic ? 1 : 0; // First operand is intrinsic id. | |||
2759 | for (unsigned i = Skip, e = getNumChildren()-1; i != e; ++i) | |||
2760 | if (OnlyOnRHSOfCommutative(getChild(i))) { | |||
2761 | Reason="Immediate value must be on the RHS of commutative operators!"; | |||
2762 | return false; | |||
2763 | } | |||
2764 | } | |||
2765 | } | |||
2766 | ||||
2767 | return true; | |||
2768 | } | |||
2769 | ||||
2770 | //===----------------------------------------------------------------------===// | |||
2771 | // TreePattern implementation | |||
2772 | // | |||
2773 | ||||
2774 | TreePattern::TreePattern(Record *TheRec, ListInit *RawPat, bool isInput, | |||
2775 | CodeGenDAGPatterns &cdp) : TheRecord(TheRec), CDP(cdp), | |||
2776 | isInputPattern(isInput), HasError(false), | |||
2777 | Infer(*this) { | |||
2778 | for (Init *I : RawPat->getValues()) | |||
2779 | Trees.push_back(ParseTreePattern(I, "")); | |||
2780 | } | |||
2781 | ||||
2782 | TreePattern::TreePattern(Record *TheRec, DagInit *Pat, bool isInput, | |||
2783 | CodeGenDAGPatterns &cdp) : TheRecord(TheRec), CDP(cdp), | |||
2784 | isInputPattern(isInput), HasError(false), | |||
2785 | Infer(*this) { | |||
2786 | Trees.push_back(ParseTreePattern(Pat, "")); | |||
2787 | } | |||
2788 | ||||
2789 | TreePattern::TreePattern(Record *TheRec, TreePatternNodePtr Pat, bool isInput, | |||
2790 | CodeGenDAGPatterns &cdp) | |||
2791 | : TheRecord(TheRec), CDP(cdp), isInputPattern(isInput), HasError(false), | |||
2792 | Infer(*this) { | |||
2793 | Trees.push_back(Pat); | |||
2794 | } | |||
2795 | ||||
2796 | void TreePattern::error(const Twine &Msg) { | |||
2797 | if (HasError) | |||
2798 | return; | |||
2799 | dump(); | |||
2800 | PrintError(TheRecord->getLoc(), "In " + TheRecord->getName() + ": " + Msg); | |||
2801 | HasError = true; | |||
2802 | } | |||
2803 | ||||
2804 | void TreePattern::ComputeNamedNodes() { | |||
2805 | for (TreePatternNodePtr &Tree : Trees) | |||
2806 | ComputeNamedNodes(Tree.get()); | |||
2807 | } | |||
2808 | ||||
2809 | void TreePattern::ComputeNamedNodes(TreePatternNode *N) { | |||
2810 | if (!N->getName().empty()) | |||
2811 | NamedNodes[N->getName()].push_back(N); | |||
2812 | ||||
2813 | for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) | |||
2814 | ComputeNamedNodes(N->getChild(i)); | |||
2815 | } | |||
2816 | ||||
2817 | TreePatternNodePtr TreePattern::ParseTreePattern(Init *TheInit, | |||
2818 | StringRef OpName) { | |||
2819 | if (DefInit *DI
| |||
| ||||
2820 | Record *R = DI->getDef(); | |||
2821 | ||||
2822 | // Direct reference to a leaf DagNode or PatFrag? Turn it into a | |||
2823 | // TreePatternNode of its own. For example: | |||
2824 | /// (foo GPR, imm) -> (foo GPR, (imm)) | |||
2825 | if (R->isSubClassOf("SDNode") || R->isSubClassOf("PatFrags")) | |||
2826 | return ParseTreePattern( | |||
2827 | DagInit::get(DI, nullptr, | |||
2828 | std::vector<std::pair<Init*, StringInit*> >()), | |||
2829 | OpName); | |||
2830 | ||||
2831 | // Input argument? | |||
2832 | TreePatternNodePtr Res = std::make_shared<TreePatternNode>(DI, 1); | |||
2833 | if (R->getName() == "node" && !OpName.empty()) { | |||
2834 | if (OpName.empty()) | |||
2835 | error("'node' argument requires a name to match with operand list"); | |||
2836 | Args.push_back(std::string(OpName)); | |||
2837 | } | |||
2838 | ||||
2839 | Res->setName(OpName); | |||
2840 | return Res; | |||
2841 | } | |||
2842 | ||||
2843 | // ?:$name or just $name. | |||
2844 | if (isa<UnsetInit>(TheInit)) { | |||
2845 | if (OpName.empty()) | |||
2846 | error("'?' argument requires a name to match with operand list"); | |||
2847 | TreePatternNodePtr Res = std::make_shared<TreePatternNode>(TheInit, 1); | |||
2848 | Args.push_back(std::string(OpName)); | |||
2849 | Res->setName(OpName); | |||
2850 | return Res; | |||
2851 | } | |||
2852 | ||||
2853 | if (isa<IntInit>(TheInit) || isa<BitInit>(TheInit)) { | |||
2854 | if (!OpName.empty()) | |||
2855 | error("Constant int or bit argument should not have a name!"); | |||
2856 | if (isa<BitInit>(TheInit)) | |||
2857 | TheInit = TheInit->convertInitializerTo(IntRecTy::get()); | |||
2858 | return std::make_shared<TreePatternNode>(TheInit, 1); | |||
2859 | } | |||
2860 | ||||
2861 | if (BitsInit *BI
| |||
2862 | // Turn this into an IntInit. | |||
2863 | Init *II = BI->convertInitializerTo(IntRecTy::get()); | |||
2864 | if (!II || !isa<IntInit>(II)) | |||
2865 | error("Bits value must be constants!"); | |||
2866 | return ParseTreePattern(II, OpName); | |||
2867 | } | |||
2868 | ||||
2869 | DagInit *Dag = dyn_cast<DagInit>(TheInit); | |||
2870 | if (!Dag
| |||
2871 | TheInit->print(errs()); | |||
2872 | error("Pattern has unexpected init kind!"); | |||
2873 | } | |||
2874 | DefInit *OpDef = dyn_cast<DefInit>(Dag->getOperator()); | |||
2875 | if (!OpDef
| |||
2876 | Record *Operator = OpDef->getDef(); | |||
| ||||
2877 | ||||
2878 | if (Operator->isSubClassOf("ValueType")) { | |||
2879 | // If the operator is a ValueType, then this must be "type cast" of a leaf | |||
2880 | // node. | |||
2881 | if (Dag->getNumArgs() != 1) | |||
2882 | error("Type cast only takes one operand!"); | |||
2883 | ||||
2884 | TreePatternNodePtr New = | |||
2885 | ParseTreePattern(Dag->getArg(0), Dag->getArgNameStr(0)); | |||
2886 | ||||
2887 | // Apply the type cast. | |||
2888 | if (New->getNumTypes() != 1) | |||
2889 | error("Type cast can only have one type!"); | |||
2890 | const CodeGenHwModes &CGH = getDAGPatterns().getTargetInfo().getHwModes(); | |||
2891 | New->UpdateNodeType(0, getValueTypeByHwMode(Operator, CGH), *this); | |||
2892 | ||||
2893 | if (!OpName.empty()) | |||
2894 | error("ValueType cast should not have a name!"); | |||
2895 | return New; | |||
2896 | } | |||
2897 | ||||
2898 | // Verify that this is something that makes sense for an operator. | |||
2899 | if (!Operator->isSubClassOf("PatFrags") && | |||
2900 | !Operator->isSubClassOf("SDNode") && | |||
2901 | !Operator->isSubClassOf("Instruction") && | |||
2902 | !Operator->isSubClassOf("SDNodeXForm") && | |||
2903 | !Operator->isSubClassOf("Intrinsic") && | |||
2904 | !Operator->isSubClassOf("ComplexPattern") && | |||
2905 | Operator->getName() != "set" && | |||
2906 | Operator->getName() != "implicit") | |||
2907 | error("Unrecognized node '" + Operator->getName() + "'!"); | |||
2908 | ||||
2909 | // Check to see if this is something that is illegal in an input pattern. | |||
2910 | if (isInputPattern) { | |||
2911 | if (Operator->isSubClassOf("Instruction") || | |||
2912 | Operator->isSubClassOf("SDNodeXForm")) | |||
2913 | error("Cannot use '" + Operator->getName() + "' in an input pattern!"); | |||
2914 | } else { | |||
2915 | if (Operator->isSubClassOf("Intrinsic")) | |||
2916 | error("Cannot use '" + Operator->getName() + "' in an output pattern!"); | |||
2917 | ||||
2918 | if (Operator->isSubClassOf("SDNode") && | |||
2919 | Operator->getName() != "imm" && | |||
2920 | Operator->getName() != "timm" && | |||
2921 | Operator->getName() != "fpimm" && | |||
2922 | Operator->getName() != "tglobaltlsaddr" && | |||
2923 | Operator->getName() != "tconstpool" && | |||
2924 | Operator->getName() != "tjumptable" && | |||
2925 | Operator->getName() != "tframeindex" && | |||
2926 | Operator->getName() != "texternalsym" && | |||
2927 | Operator->getName() != "tblockaddress" && | |||
2928 | Operator->getName() != "tglobaladdr" && | |||
2929 | Operator->getName() != "bb" && | |||
2930 | Operator->getName() != "vt" && | |||
2931 | Operator->getName() != "mcsym") | |||
2932 | error("Cannot use '" + Operator->getName() + "' in an output pattern!"); | |||
2933 | } | |||
2934 | ||||
2935 | std::vector<TreePatternNodePtr> Children; | |||
2936 | ||||
2937 | // Parse all the operands. | |||
2938 | for (unsigned i = 0, e = Dag->getNumArgs(); i != e; ++i) | |||
2939 | Children.push_back(ParseTreePattern(Dag->getArg(i), Dag->getArgNameStr(i))); | |||
2940 | ||||
2941 | // Get the actual number of results before Operator is converted to an intrinsic | |||
2942 | // node (which is hard-coded to have either zero or one result). | |||
2943 | unsigned NumResults = GetNumNodeResults(Operator, CDP); | |||
2944 | ||||
2945 | // If the operator is an intrinsic, then this is just syntactic sugar for | |||
2946 | // (intrinsic_* <number>, ..children..). Pick the right intrinsic node, and | |||
2947 | // convert the intrinsic name to a number. | |||
2948 | if (Operator->isSubClassOf("Intrinsic")) { | |||
2949 | const CodeGenIntrinsic &Int = getDAGPatterns().getIntrinsic(Operator); | |||
2950 | unsigned IID = getDAGPatterns().getIntrinsicID(Operator)+1; | |||
2951 | ||||
2952 | // If this intrinsic returns void, it must have side-effects and thus a | |||
2953 | // chain. | |||
2954 | if (Int.IS.RetVTs.empty()) | |||
2955 | Operator = getDAGPatterns().get_intrinsic_void_sdnode(); | |||
2956 | else if (Int.ModRef != CodeGenIntrinsic::NoMem || Int.hasSideEffects) | |||
2957 | // Has side-effects, requires chain. | |||
2958 | Operator = getDAGPatterns().get_intrinsic_w_chain_sdnode(); | |||
2959 | else // Otherwise, no chain. | |||
2960 | Operator = getDAGPatterns().get_intrinsic_wo_chain_sdnode(); | |||
2961 | ||||
2962 | Children.insert(Children.begin(), | |||
2963 | std::make_shared<TreePatternNode>(IntInit::get(IID), 1)); | |||
2964 | } | |||
2965 | ||||
2966 | if (Operator->isSubClassOf("ComplexPattern")) { | |||
2967 | for (unsigned i = 0; i < Children.size(); ++i) { | |||
2968 | TreePatternNodePtr Child = Children[i]; | |||
2969 | ||||
2970 | if (Child->getName().empty()) | |||
2971 | error("All arguments to a ComplexPattern must be named"); | |||
2972 | ||||
2973 | // Check that the ComplexPattern uses are consistent: "(MY_PAT $a, $b)" | |||
2974 | // and "(MY_PAT $b, $a)" should not be allowed in the same pattern; | |||
2975 | // neither should "(MY_PAT_1 $a, $b)" and "(MY_PAT_2 $a, $b)". | |||
2976 | auto OperandId = std::make_pair(Operator, i); | |||
2977 | auto PrevOp = ComplexPatternOperands.find(Child->getName()); | |||
2978 | if (PrevOp != ComplexPatternOperands.end()) { | |||
2979 | if (PrevOp->getValue() != OperandId) | |||
2980 | error("All ComplexPattern operands must appear consistently: " | |||
2981 | "in the same order in just one ComplexPattern instance."); | |||
2982 | } else | |||
2983 | ComplexPatternOperands[Child->getName()] = OperandId; | |||
2984 | } | |||
2985 | } | |||
2986 | ||||
2987 | TreePatternNodePtr Result = | |||
2988 | std::make_shared<TreePatternNode>(Operator, std::move(Children), | |||
2989 | NumResults); | |||
2990 | Result->setName(OpName); | |||
2991 | ||||
2992 | if (Dag->getName()) { | |||
2993 | assert(Result->getName().empty())(static_cast <bool> (Result->getName().empty()) ? void (0) : __assert_fail ("Result->getName().empty()", "llvm/utils/TableGen/CodeGenDAGPatterns.cpp" , 2993, __extension__ __PRETTY_FUNCTION__)); | |||
2994 | Result->setName(Dag->getNameStr()); | |||
2995 | } | |||
2996 | return Result; | |||
2997 | } | |||
2998 | ||||
2999 | /// SimplifyTree - See if we can simplify this tree to eliminate something that | |||
3000 | /// will never match in favor of something obvious that will. This is here | |||
3001 | /// strictly as a convenience to target authors because it allows them to write | |||
3002 | /// more type generic things and have useless type casts fold away. | |||
3003 | /// | |||
3004 | /// This returns true if any change is made. | |||
3005 | static bool SimplifyTree(TreePatternNodePtr &N) { | |||
3006 | if (N->isLeaf()) | |||
3007 | return false; | |||
3008 | ||||
3009 | // If we have a bitconvert with a resolved type and if the source and | |||
3010 | // destination types are the same, then the bitconvert is useless, remove it. | |||
3011 | // | |||
3012 | // We make an exception if the types are completely empty. This can come up | |||
3013 | // when the pattern being simplified is in the Fragments list of a PatFrags, | |||
3014 | // so that the operand is just an untyped "node". In that situation we leave | |||
3015 | // bitconverts unsimplified, and simplify them later once the fragment is | |||
3016 | // expanded into its true context. | |||
3017 | if (N->getOperator()->getName() == "bitconvert" && | |||
3018 | N->getExtType(0).isValueTypeByHwMode(false) && | |||
3019 | !N->getExtType(0).empty() && | |||
3020 | N->getExtType(0) == N->getChild(0)->getExtType(0) && | |||
3021 | N->getName().empty()) { | |||
3022 | N = N->getChildShared(0); | |||
3023 | SimplifyTree(N); | |||
3024 | return true; | |||
3025 | } | |||
3026 | ||||
3027 | // Walk all children. | |||
3028 | bool MadeChange = false; | |||
3029 | for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) { | |||
3030 | TreePatternNodePtr Child = N->getChildShared(i); | |||
3031 | MadeChange |= SimplifyTree(Child); | |||
3032 | N->setChild(i, std::move(Child)); | |||
3033 | } | |||
3034 | return MadeChange; | |||
3035 | } | |||
3036 | ||||
3037 | ||||
3038 | ||||
3039 | /// InferAllTypes - Infer/propagate as many types throughout the expression | |||
3040 | /// patterns as possible. Return true if all types are inferred, false | |||
3041 | /// otherwise. Flags an error if a type contradiction is found. | |||
3042 | bool TreePattern:: | |||
3043 | InferAllTypes(const StringMap<SmallVector<TreePatternNode*,1> > *InNamedTypes) { | |||
3044 | if (NamedNodes.empty()) | |||
3045 | ComputeNamedNodes(); | |||
3046 | ||||
3047 | bool MadeChange = true; | |||
3048 | while (MadeChange) { | |||
3049 | MadeChange = false; | |||
3050 | for (TreePatternNodePtr &Tree : Trees) { | |||
3051 | MadeChange |= Tree->ApplyTypeConstraints(*this, false); | |||
3052 | MadeChange |= SimplifyTree(Tree); | |||
3053 | } | |||
3054 | ||||
3055 | // If there are constraints on our named nodes, apply them. | |||
3056 | for (auto &Entry : NamedNodes) { | |||
3057 | SmallVectorImpl<TreePatternNode*> &Nodes = Entry.second; | |||
3058 | ||||
3059 | // If we have input named node types, propagate their types to the named | |||
3060 | // values here. | |||
3061 | if (InNamedTypes) { | |||
3062 | if (!InNamedTypes->count(Entry.getKey())) { | |||
3063 | error("Node '" + std::string(Entry.getKey()) + | |||
3064 | "' in output pattern but not input pattern"); | |||
3065 | return true; | |||
3066 | } | |||
3067 | ||||
3068 | const SmallVectorImpl<TreePatternNode*> &InNodes = | |||
3069 | InNamedTypes->find(Entry.getKey())->second; | |||
3070 | ||||
3071 | // The input types should be fully resolved by now. | |||
3072 | for (TreePatternNode *Node : Nodes) { | |||
3073 | // If this node is a register class, and it is the root of the pattern | |||
3074 | // then we're mapping something onto an input register. We allow | |||
3075 | // changing the type of the input register in this case. This allows | |||
3076 | // us to match things like: | |||
3077 | // def : Pat<(v1i64 (bitconvert(v2i32 DPR:$src))), (v1i64 DPR:$src)>; | |||
3078 | if (Node == Trees[0].get() && Node->isLeaf()) { | |||
3079 | DefInit *DI = dyn_cast<DefInit>(Node->getLeafValue()); | |||
3080 | if (DI && (DI->getDef()->isSubClassOf("RegisterClass") || | |||
3081 | DI->getDef()->isSubClassOf("RegisterOperand"))) | |||
3082 | continue; | |||
3083 | } | |||
3084 | ||||
3085 | assert(Node->getNumTypes() == 1 &&(static_cast <bool> (Node->getNumTypes() == 1 && InNodes[0]->getNumTypes() == 1 && "FIXME: cannot name multiple result nodes yet" ) ? void (0) : __assert_fail ("Node->getNumTypes() == 1 && InNodes[0]->getNumTypes() == 1 && \"FIXME: cannot name multiple result nodes yet\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3087, __extension__ __PRETTY_FUNCTION__)) | |||
3086 | InNodes[0]->getNumTypes() == 1 &&(static_cast <bool> (Node->getNumTypes() == 1 && InNodes[0]->getNumTypes() == 1 && "FIXME: cannot name multiple result nodes yet" ) ? void (0) : __assert_fail ("Node->getNumTypes() == 1 && InNodes[0]->getNumTypes() == 1 && \"FIXME: cannot name multiple result nodes yet\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3087, __extension__ __PRETTY_FUNCTION__)) | |||
3087 | "FIXME: cannot name multiple result nodes yet")(static_cast <bool> (Node->getNumTypes() == 1 && InNodes[0]->getNumTypes() == 1 && "FIXME: cannot name multiple result nodes yet" ) ? void (0) : __assert_fail ("Node->getNumTypes() == 1 && InNodes[0]->getNumTypes() == 1 && \"FIXME: cannot name multiple result nodes yet\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3087, __extension__ __PRETTY_FUNCTION__)); | |||
3088 | MadeChange |= Node->UpdateNodeType(0, InNodes[0]->getExtType(0), | |||
3089 | *this); | |||
3090 | } | |||
3091 | } | |||
3092 | ||||
3093 | // If there are multiple nodes with the same name, they must all have the | |||
3094 | // same type. | |||
3095 | if (Entry.second.size() > 1) { | |||
3096 | for (unsigned i = 0, e = Nodes.size()-1; i != e; ++i) { | |||
3097 | TreePatternNode *N1 = Nodes[i], *N2 = Nodes[i+1]; | |||
3098 | assert(N1->getNumTypes() == 1 && N2->getNumTypes() == 1 &&(static_cast <bool> (N1->getNumTypes() == 1 && N2->getNumTypes() == 1 && "FIXME: cannot name multiple result nodes yet" ) ? void (0) : __assert_fail ("N1->getNumTypes() == 1 && N2->getNumTypes() == 1 && \"FIXME: cannot name multiple result nodes yet\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3099, __extension__ __PRETTY_FUNCTION__)) | |||
3099 | "FIXME: cannot name multiple result nodes yet")(static_cast <bool> (N1->getNumTypes() == 1 && N2->getNumTypes() == 1 && "FIXME: cannot name multiple result nodes yet" ) ? void (0) : __assert_fail ("N1->getNumTypes() == 1 && N2->getNumTypes() == 1 && \"FIXME: cannot name multiple result nodes yet\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3099, __extension__ __PRETTY_FUNCTION__)); | |||
3100 | ||||
3101 | MadeChange |= N1->UpdateNodeType(0, N2->getExtType(0), *this); | |||
3102 | MadeChange |= N2->UpdateNodeType(0, N1->getExtType(0), *this); | |||
3103 | } | |||
3104 | } | |||
3105 | } | |||
3106 | } | |||
3107 | ||||
3108 | bool HasUnresolvedTypes = false; | |||
3109 | for (const TreePatternNodePtr &Tree : Trees) | |||
3110 | HasUnresolvedTypes |= Tree->ContainsUnresolvedType(*this); | |||
3111 | return !HasUnresolvedTypes; | |||
3112 | } | |||
3113 | ||||
3114 | void TreePattern::print(raw_ostream &OS) const { | |||
3115 | OS << getRecord()->getName(); | |||
3116 | if (!Args.empty()) { | |||
3117 | OS << "("; | |||
3118 | ListSeparator LS; | |||
3119 | for (const std::string &Arg : Args) | |||
3120 | OS << LS << Arg; | |||
3121 | OS << ")"; | |||
3122 | } | |||
3123 | OS << ": "; | |||
3124 | ||||
3125 | if (Trees.size() > 1) | |||
3126 | OS << "[\n"; | |||
3127 | for (const TreePatternNodePtr &Tree : Trees) { | |||
3128 | OS << "\t"; | |||
3129 | Tree->print(OS); | |||
3130 | OS << "\n"; | |||
3131 | } | |||
3132 | ||||
3133 | if (Trees.size() > 1) | |||
3134 | OS << "]\n"; | |||
3135 | } | |||
3136 | ||||
3137 | void TreePattern::dump() const { print(errs()); } | |||
3138 | ||||
3139 | //===----------------------------------------------------------------------===// | |||
3140 | // CodeGenDAGPatterns implementation | |||
3141 | // | |||
3142 | ||||
3143 | CodeGenDAGPatterns::CodeGenDAGPatterns(RecordKeeper &R, | |||
3144 | PatternRewriterFn PatternRewriter) | |||
3145 | : Records(R), Target(R), LegalVTS(Target.getLegalValueTypes()), | |||
3146 | PatternRewriter(PatternRewriter) { | |||
3147 | ||||
3148 | Intrinsics = CodeGenIntrinsicTable(Records); | |||
3149 | ParseNodeInfo(); | |||
3150 | ParseNodeTransforms(); | |||
3151 | ParseComplexPatterns(); | |||
3152 | ParsePatternFragments(); | |||
3153 | ParseDefaultOperands(); | |||
3154 | ParseInstructions(); | |||
3155 | ParsePatternFragments(/*OutFrags*/true); | |||
3156 | ParsePatterns(); | |||
3157 | ||||
3158 | // Generate variants. For example, commutative patterns can match | |||
3159 | // multiple ways. Add them to PatternsToMatch as well. | |||
3160 | GenerateVariants(); | |||
3161 | ||||
3162 | // Break patterns with parameterized types into a series of patterns, | |||
3163 | // where each one has a fixed type and is predicated on the conditions | |||
3164 | // of the associated HW mode. | |||
3165 | ExpandHwModeBasedTypes(); | |||
3166 | ||||
3167 | // Infer instruction flags. For example, we can detect loads, | |||
3168 | // stores, and side effects in many cases by examining an | |||
3169 | // instruction's pattern. | |||
3170 | InferInstructionFlags(); | |||
3171 | ||||
3172 | // Verify that instruction flags match the patterns. | |||
3173 | VerifyInstructionFlags(); | |||
3174 | } | |||
3175 | ||||
3176 | Record *CodeGenDAGPatterns::getSDNodeNamed(StringRef Name) const { | |||
3177 | Record *N = Records.getDef(Name); | |||
3178 | if (!N || !N->isSubClassOf("SDNode")) | |||
3179 | PrintFatalError("Error getting SDNode '" + Name + "'!"); | |||
3180 | ||||
3181 | return N; | |||
3182 | } | |||
3183 | ||||
3184 | // Parse all of the SDNode definitions for the target, populating SDNodes. | |||
3185 | void CodeGenDAGPatterns::ParseNodeInfo() { | |||
3186 | std::vector<Record*> Nodes = Records.getAllDerivedDefinitions("SDNode"); | |||
3187 | const CodeGenHwModes &CGH = getTargetInfo().getHwModes(); | |||
3188 | ||||
3189 | while (!Nodes.empty()) { | |||
3190 | Record *R = Nodes.back(); | |||
3191 | SDNodes.insert(std::make_pair(R, SDNodeInfo(R, CGH))); | |||
3192 | Nodes.pop_back(); | |||
3193 | } | |||
3194 | ||||
3195 | // Get the builtin intrinsic nodes. | |||
3196 | intrinsic_void_sdnode = getSDNodeNamed("intrinsic_void"); | |||
3197 | intrinsic_w_chain_sdnode = getSDNodeNamed("intrinsic_w_chain"); | |||
3198 | intrinsic_wo_chain_sdnode = getSDNodeNamed("intrinsic_wo_chain"); | |||
3199 | } | |||
3200 | ||||
3201 | /// ParseNodeTransforms - Parse all SDNodeXForm instances into the SDNodeXForms | |||
3202 | /// map, and emit them to the file as functions. | |||
3203 | void CodeGenDAGPatterns::ParseNodeTransforms() { | |||
3204 | std::vector<Record*> Xforms = Records.getAllDerivedDefinitions("SDNodeXForm"); | |||
3205 | while (!Xforms.empty()) { | |||
3206 | Record *XFormNode = Xforms.back(); | |||
3207 | Record *SDNode = XFormNode->getValueAsDef("Opcode"); | |||
3208 | StringRef Code = XFormNode->getValueAsString("XFormFunction"); | |||
3209 | SDNodeXForms.insert( | |||
3210 | std::make_pair(XFormNode, NodeXForm(SDNode, std::string(Code)))); | |||
3211 | ||||
3212 | Xforms.pop_back(); | |||
3213 | } | |||
3214 | } | |||
3215 | ||||
3216 | void CodeGenDAGPatterns::ParseComplexPatterns() { | |||
3217 | std::vector<Record*> AMs = Records.getAllDerivedDefinitions("ComplexPattern"); | |||
3218 | while (!AMs.empty()) { | |||
3219 | ComplexPatterns.insert(std::make_pair(AMs.back(), AMs.back())); | |||
3220 | AMs.pop_back(); | |||
3221 | } | |||
3222 | } | |||
3223 | ||||
3224 | ||||
3225 | /// ParsePatternFragments - Parse all of the PatFrag definitions in the .td | |||
3226 | /// file, building up the PatternFragments map. After we've collected them all, | |||
3227 | /// inline fragments together as necessary, so that there are no references left | |||
3228 | /// inside a pattern fragment to a pattern fragment. | |||
3229 | /// | |||
3230 | void CodeGenDAGPatterns::ParsePatternFragments(bool OutFrags) { | |||
3231 | std::vector<Record*> Fragments = Records.getAllDerivedDefinitions("PatFrags"); | |||
3232 | ||||
3233 | // First step, parse all of the fragments. | |||
3234 | for (Record *Frag : Fragments) { | |||
3235 | if (OutFrags != Frag->isSubClassOf("OutPatFrag")) | |||
3236 | continue; | |||
3237 | ||||
3238 | ListInit *LI = Frag->getValueAsListInit("Fragments"); | |||
3239 | TreePattern *P = | |||
3240 | (PatternFragments[Frag] = std::make_unique<TreePattern>( | |||
3241 | Frag, LI, !Frag->isSubClassOf("OutPatFrag"), | |||
3242 | *this)).get(); | |||
3243 | ||||
3244 | // Validate the argument list, converting it to set, to discard duplicates. | |||
3245 | std::vector<std::string> &Args = P->getArgList(); | |||
3246 | // Copy the args so we can take StringRefs to them. | |||
3247 | auto ArgsCopy = Args; | |||
3248 | SmallDenseSet<StringRef, 4> OperandsSet; | |||
3249 | OperandsSet.insert(ArgsCopy.begin(), ArgsCopy.end()); | |||
3250 | ||||
3251 | if (OperandsSet.count("")) | |||
3252 | P->error("Cannot have unnamed 'node' values in pattern fragment!"); | |||
3253 | ||||
3254 | // Parse the operands list. | |||
3255 | DagInit *OpsList = Frag->getValueAsDag("Operands"); | |||
3256 | DefInit *OpsOp = dyn_cast<DefInit>(OpsList->getOperator()); | |||
3257 | // Special cases: ops == outs == ins. Different names are used to | |||
3258 | // improve readability. | |||
3259 | if (!OpsOp || | |||
3260 | (OpsOp->getDef()->getName() != "ops" && | |||
3261 | OpsOp->getDef()->getName() != "outs" && | |||
3262 | OpsOp->getDef()->getName() != "ins")) | |||
3263 | P->error("Operands list should start with '(ops ... '!"); | |||
3264 | ||||
3265 | // Copy over the arguments. | |||
3266 | Args.clear(); | |||
3267 | for (unsigned j = 0, e = OpsList->getNumArgs(); j != e; ++j) { | |||
3268 | if (!isa<DefInit>(OpsList->getArg(j)) || | |||
3269 | cast<DefInit>(OpsList->getArg(j))->getDef()->getName() != "node") | |||
3270 | P->error("Operands list should all be 'node' values."); | |||
3271 | if (!OpsList->getArgName(j)) | |||
3272 | P->error("Operands list should have names for each operand!"); | |||
3273 | StringRef ArgNameStr = OpsList->getArgNameStr(j); | |||
3274 | if (!OperandsSet.count(ArgNameStr)) | |||
3275 | P->error("'" + ArgNameStr + | |||
3276 | "' does not occur in pattern or was multiply specified!"); | |||
3277 | OperandsSet.erase(ArgNameStr); | |||
3278 | Args.push_back(std::string(ArgNameStr)); | |||
3279 | } | |||
3280 | ||||
3281 | if (!OperandsSet.empty()) | |||
3282 | P->error("Operands list does not contain an entry for operand '" + | |||
3283 | *OperandsSet.begin() + "'!"); | |||
3284 | ||||
3285 | // If there is a node transformation corresponding to this, keep track of | |||
3286 | // it. | |||
3287 | Record *Transform = Frag->getValueAsDef("OperandTransform"); | |||
3288 | if (!getSDNodeTransform(Transform).second.empty()) // not noop xform? | |||
3289 | for (const auto &T : P->getTrees()) | |||
3290 | T->setTransformFn(Transform); | |||
3291 | } | |||
3292 | ||||
3293 | // Now that we've parsed all of the tree fragments, do a closure on them so | |||
3294 | // that there are not references to PatFrags left inside of them. | |||
3295 | for (Record *Frag : Fragments) { | |||
3296 | if (OutFrags != Frag->isSubClassOf("OutPatFrag")) | |||
3297 | continue; | |||
3298 | ||||
3299 | TreePattern &ThePat = *PatternFragments[Frag]; | |||
3300 | ThePat.InlinePatternFragments(); | |||
3301 | ||||
3302 | // Infer as many types as possible. Don't worry about it if we don't infer | |||
3303 | // all of them, some may depend on the inputs of the pattern. Also, don't | |||
3304 | // validate type sets; validation may cause spurious failures e.g. if a | |||
3305 | // fragment needs floating-point types but the current target does not have | |||
3306 | // any (this is only an error if that fragment is ever used!). | |||
3307 | { | |||
3308 | TypeInfer::SuppressValidation SV(ThePat.getInfer()); | |||
3309 | ThePat.InferAllTypes(); | |||
3310 | ThePat.resetError(); | |||
3311 | } | |||
3312 | ||||
3313 | // If debugging, print out the pattern fragment result. | |||
3314 | LLVM_DEBUG(ThePat.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { ThePat.dump(); } } while (false); | |||
3315 | } | |||
3316 | } | |||
3317 | ||||
3318 | void CodeGenDAGPatterns::ParseDefaultOperands() { | |||
3319 | std::vector<Record*> DefaultOps; | |||
3320 | DefaultOps = Records.getAllDerivedDefinitions("OperandWithDefaultOps"); | |||
3321 | ||||
3322 | // Find some SDNode. | |||
3323 | assert(!SDNodes.empty() && "No SDNodes parsed?")(static_cast <bool> (!SDNodes.empty() && "No SDNodes parsed?" ) ? void (0) : __assert_fail ("!SDNodes.empty() && \"No SDNodes parsed?\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3323, __extension__ __PRETTY_FUNCTION__)); | |||
3324 | Init *SomeSDNode = DefInit::get(SDNodes.begin()->first); | |||
3325 | ||||
3326 | for (unsigned i = 0, e = DefaultOps.size(); i != e; ++i) { | |||
3327 | DagInit *DefaultInfo = DefaultOps[i]->getValueAsDag("DefaultOps"); | |||
3328 | ||||
3329 | // Clone the DefaultInfo dag node, changing the operator from 'ops' to | |||
3330 | // SomeSDnode so that we can parse this. | |||
3331 | std::vector<std::pair<Init*, StringInit*> > Ops; | |||
3332 | for (unsigned op = 0, e = DefaultInfo->getNumArgs(); op != e; ++op) | |||
3333 | Ops.push_back(std::make_pair(DefaultInfo->getArg(op), | |||
3334 | DefaultInfo->getArgName(op))); | |||
3335 | DagInit *DI = DagInit::get(SomeSDNode, nullptr, Ops); | |||
3336 | ||||
3337 | // Create a TreePattern to parse this. | |||
3338 | TreePattern P(DefaultOps[i], DI, false, *this); | |||
3339 | assert(P.getNumTrees() == 1 && "This ctor can only produce one tree!")(static_cast <bool> (P.getNumTrees() == 1 && "This ctor can only produce one tree!" ) ? void (0) : __assert_fail ("P.getNumTrees() == 1 && \"This ctor can only produce one tree!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3339, __extension__ __PRETTY_FUNCTION__)); | |||
3340 | ||||
3341 | // Copy the operands over into a DAGDefaultOperand. | |||
3342 | DAGDefaultOperand DefaultOpInfo; | |||
3343 | ||||
3344 | const TreePatternNodePtr &T = P.getTree(0); | |||
3345 | for (unsigned op = 0, e = T->getNumChildren(); op != e; ++op) { | |||
3346 | TreePatternNodePtr TPN = T->getChildShared(op); | |||
3347 | while (TPN->ApplyTypeConstraints(P, false)) | |||
3348 | /* Resolve all types */; | |||
3349 | ||||
3350 | if (TPN->ContainsUnresolvedType(P)) { | |||
3351 | PrintFatalError("Value #" + Twine(i) + " of OperandWithDefaultOps '" + | |||
3352 | DefaultOps[i]->getName() + | |||
3353 | "' doesn't have a concrete type!"); | |||
3354 | } | |||
3355 | DefaultOpInfo.DefaultOps.push_back(std::move(TPN)); | |||
3356 | } | |||
3357 | ||||
3358 | // Insert it into the DefaultOperands map so we can find it later. | |||
3359 | DefaultOperands[DefaultOps[i]] = DefaultOpInfo; | |||
3360 | } | |||
3361 | } | |||
3362 | ||||
3363 | /// HandleUse - Given "Pat" a leaf in the pattern, check to see if it is an | |||
3364 | /// instruction input. Return true if this is a real use. | |||
3365 | static bool HandleUse(TreePattern &I, TreePatternNodePtr Pat, | |||
3366 | std::map<std::string, TreePatternNodePtr> &InstInputs) { | |||
3367 | // No name -> not interesting. | |||
3368 | if (Pat->getName().empty()) { | |||
3369 | if (Pat->isLeaf()) { | |||
3370 | DefInit *DI = dyn_cast<DefInit>(Pat->getLeafValue()); | |||
3371 | if (DI && (DI->getDef()->isSubClassOf("RegisterClass") || | |||
3372 | DI->getDef()->isSubClassOf("RegisterOperand"))) | |||
3373 | I.error("Input " + DI->getDef()->getName() + " must be named!"); | |||
3374 | } | |||
3375 | return false; | |||
3376 | } | |||
3377 | ||||
3378 | Record *Rec; | |||
3379 | if (Pat->isLeaf()) { | |||
3380 | DefInit *DI = dyn_cast<DefInit>(Pat->getLeafValue()); | |||
3381 | if (!DI) | |||
3382 | I.error("Input $" + Pat->getName() + " must be an identifier!"); | |||
3383 | Rec = DI->getDef(); | |||
3384 | } else { | |||
3385 | Rec = Pat->getOperator(); | |||
3386 | } | |||
3387 | ||||
3388 | // SRCVALUE nodes are ignored. | |||
3389 | if (Rec->getName() == "srcvalue") | |||
3390 | return false; | |||
3391 | ||||
3392 | TreePatternNodePtr &Slot = InstInputs[Pat->getName()]; | |||
3393 | if (!Slot) { | |||
3394 | Slot = Pat; | |||
3395 | return true; | |||
3396 | } | |||
3397 | Record *SlotRec; | |||
3398 | if (Slot->isLeaf()) { | |||
3399 | SlotRec = cast<DefInit>(Slot->getLeafValue())->getDef(); | |||
3400 | } else { | |||
3401 | assert(Slot->getNumChildren() == 0 && "can't be a use with children!")(static_cast <bool> (Slot->getNumChildren() == 0 && "can't be a use with children!") ? void (0) : __assert_fail ( "Slot->getNumChildren() == 0 && \"can't be a use with children!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3401, __extension__ __PRETTY_FUNCTION__)); | |||
3402 | SlotRec = Slot->getOperator(); | |||
3403 | } | |||
3404 | ||||
3405 | // Ensure that the inputs agree if we've already seen this input. | |||
3406 | if (Rec != SlotRec) | |||
3407 | I.error("All $" + Pat->getName() + " inputs must agree with each other"); | |||
3408 | // Ensure that the types can agree as well. | |||
3409 | Slot->UpdateNodeType(0, Pat->getExtType(0), I); | |||
3410 | Pat->UpdateNodeType(0, Slot->getExtType(0), I); | |||
3411 | if (Slot->getExtTypes() != Pat->getExtTypes()) | |||
3412 | I.error("All $" + Pat->getName() + " inputs must agree with each other"); | |||
3413 | return true; | |||
3414 | } | |||
3415 | ||||
3416 | /// FindPatternInputsAndOutputs - Scan the specified TreePatternNode (which is | |||
3417 | /// part of "I", the instruction), computing the set of inputs and outputs of | |||
3418 | /// the pattern. Report errors if we see anything naughty. | |||
3419 | void CodeGenDAGPatterns::FindPatternInputsAndOutputs( | |||
3420 | TreePattern &I, TreePatternNodePtr Pat, | |||
3421 | std::map<std::string, TreePatternNodePtr> &InstInputs, | |||
3422 | MapVector<std::string, TreePatternNodePtr, std::map<std::string, unsigned>> | |||
3423 | &InstResults, | |||
3424 | std::vector<Record *> &InstImpResults) { | |||
3425 | ||||
3426 | // The instruction pattern still has unresolved fragments. For *named* | |||
3427 | // nodes we must resolve those here. This may not result in multiple | |||
3428 | // alternatives. | |||
3429 | if (!Pat->getName().empty()) { | |||
3430 | TreePattern SrcPattern(I.getRecord(), Pat, true, *this); | |||
3431 | SrcPattern.InlinePatternFragments(); | |||
3432 | SrcPattern.InferAllTypes(); | |||
3433 | Pat = SrcPattern.getOnlyTree(); | |||
3434 | } | |||
3435 | ||||
3436 | if (Pat->isLeaf()) { | |||
3437 | bool isUse = HandleUse(I, Pat, InstInputs); | |||
3438 | if (!isUse && Pat->getTransformFn()) | |||
3439 | I.error("Cannot specify a transform function for a non-input value!"); | |||
3440 | return; | |||
3441 | } | |||
3442 | ||||
3443 | if (Pat->getOperator()->getName() == "implicit") { | |||
3444 | for (unsigned i = 0, e = Pat->getNumChildren(); i != e; ++i) { | |||
3445 | TreePatternNode *Dest = Pat->getChild(i); | |||
3446 | if (!Dest->isLeaf()) | |||
3447 | I.error("implicitly defined value should be a register!"); | |||
3448 | ||||
3449 | DefInit *Val = dyn_cast<DefInit>(Dest->getLeafValue()); | |||
3450 | if (!Val || !Val->getDef()->isSubClassOf("Register")) | |||
3451 | I.error("implicitly defined value should be a register!"); | |||
3452 | InstImpResults.push_back(Val->getDef()); | |||
3453 | } | |||
3454 | return; | |||
3455 | } | |||
3456 | ||||
3457 | if (Pat->getOperator()->getName() != "set") { | |||
3458 | // If this is not a set, verify that the children nodes are not void typed, | |||
3459 | // and recurse. | |||
3460 | for (unsigned i = 0, e = Pat->getNumChildren(); i != e; ++i) { | |||
3461 | if (Pat->getChild(i)->getNumTypes() == 0) | |||
3462 | I.error("Cannot have void nodes inside of patterns!"); | |||
3463 | FindPatternInputsAndOutputs(I, Pat->getChildShared(i), InstInputs, | |||
3464 | InstResults, InstImpResults); | |||
3465 | } | |||
3466 | ||||
3467 | // If this is a non-leaf node with no children, treat it basically as if | |||
3468 | // it were a leaf. This handles nodes like (imm). | |||
3469 | bool isUse = HandleUse(I, Pat, InstInputs); | |||
3470 | ||||
3471 | if (!isUse && Pat->getTransformFn()) | |||
3472 | I.error("Cannot specify a transform function for a non-input value!"); | |||
3473 | return; | |||
3474 | } | |||
3475 | ||||
3476 | // Otherwise, this is a set, validate and collect instruction results. | |||
3477 | if (Pat->getNumChildren() == 0) | |||
3478 | I.error("set requires operands!"); | |||
3479 | ||||
3480 | if (Pat->getTransformFn()) | |||
3481 | I.error("Cannot specify a transform function on a set node!"); | |||
3482 | ||||
3483 | // Check the set destinations. | |||
3484 | unsigned NumDests = Pat->getNumChildren()-1; | |||
3485 | for (unsigned i = 0; i != NumDests; ++i) { | |||
3486 | TreePatternNodePtr Dest = Pat->getChildShared(i); | |||
3487 | // For set destinations we also must resolve fragments here. | |||
3488 | TreePattern DestPattern(I.getRecord(), Dest, false, *this); | |||
3489 | DestPattern.InlinePatternFragments(); | |||
3490 | DestPattern.InferAllTypes(); | |||
3491 | Dest = DestPattern.getOnlyTree(); | |||
3492 | ||||
3493 | if (!Dest->isLeaf()) | |||
3494 | I.error("set destination should be a register!"); | |||
3495 | ||||
3496 | DefInit *Val = dyn_cast<DefInit>(Dest->getLeafValue()); | |||
3497 | if (!Val) { | |||
3498 | I.error("set destination should be a register!"); | |||
3499 | continue; | |||
3500 | } | |||
3501 | ||||
3502 | if (Val->getDef()->isSubClassOf("RegisterClass") || | |||
3503 | Val->getDef()->isSubClassOf("ValueType") || | |||
3504 | Val->getDef()->isSubClassOf("RegisterOperand") || | |||
3505 | Val->getDef()->isSubClassOf("PointerLikeRegClass")) { | |||
3506 | if (Dest->getName().empty()) | |||
3507 | I.error("set destination must have a name!"); | |||
3508 | if (InstResults.count(Dest->getName())) | |||
3509 | I.error("cannot set '" + Dest->getName() + "' multiple times"); | |||
3510 | InstResults[Dest->getName()] = Dest; | |||
3511 | } else if (Val->getDef()->isSubClassOf("Register")) { | |||
3512 | InstImpResults.push_back(Val->getDef()); | |||
3513 | } else { | |||
3514 | I.error("set destination should be a register!"); | |||
3515 | } | |||
3516 | } | |||
3517 | ||||
3518 | // Verify and collect info from the computation. | |||
3519 | FindPatternInputsAndOutputs(I, Pat->getChildShared(NumDests), InstInputs, | |||
3520 | InstResults, InstImpResults); | |||
3521 | } | |||
3522 | ||||
3523 | //===----------------------------------------------------------------------===// | |||
3524 | // Instruction Analysis | |||
3525 | //===----------------------------------------------------------------------===// | |||
3526 | ||||
3527 | class InstAnalyzer { | |||
3528 | const CodeGenDAGPatterns &CDP; | |||
3529 | public: | |||
3530 | bool hasSideEffects; | |||
3531 | bool mayStore; | |||
3532 | bool mayLoad; | |||
3533 | bool isBitcast; | |||
3534 | bool isVariadic; | |||
3535 | bool hasChain; | |||
3536 | ||||
3537 | InstAnalyzer(const CodeGenDAGPatterns &cdp) | |||
3538 | : CDP(cdp), hasSideEffects(false), mayStore(false), mayLoad(false), | |||
3539 | isBitcast(false), isVariadic(false), hasChain(false) {} | |||
3540 | ||||
3541 | void Analyze(const PatternToMatch &Pat) { | |||
3542 | const TreePatternNode *N = Pat.getSrcPattern(); | |||
3543 | AnalyzeNode(N); | |||
3544 | // These properties are detected only on the root node. | |||
3545 | isBitcast = IsNodeBitcast(N); | |||
3546 | } | |||
3547 | ||||
3548 | private: | |||
3549 | bool IsNodeBitcast(const TreePatternNode *N) const { | |||
3550 | if (hasSideEffects || mayLoad || mayStore || isVariadic) | |||
3551 | return false; | |||
3552 | ||||
3553 | if (N->isLeaf()) | |||
3554 | return false; | |||
3555 | if (N->getNumChildren() != 1 || !N->getChild(0)->isLeaf()) | |||
3556 | return false; | |||
3557 | ||||
3558 | if (N->getOperator()->isSubClassOf("ComplexPattern")) | |||
3559 | return false; | |||
3560 | ||||
3561 | const SDNodeInfo &OpInfo = CDP.getSDNodeInfo(N->getOperator()); | |||
3562 | if (OpInfo.getNumResults() != 1 || OpInfo.getNumOperands() != 1) | |||
3563 | return false; | |||
3564 | return OpInfo.getEnumName() == "ISD::BITCAST"; | |||
3565 | } | |||
3566 | ||||
3567 | public: | |||
3568 | void AnalyzeNode(const TreePatternNode *N) { | |||
3569 | if (N->isLeaf()) { | |||
3570 | if (DefInit *DI = dyn_cast<DefInit>(N->getLeafValue())) { | |||
3571 | Record *LeafRec = DI->getDef(); | |||
3572 | // Handle ComplexPattern leaves. | |||
3573 | if (LeafRec->isSubClassOf("ComplexPattern")) { | |||
3574 | const ComplexPattern &CP = CDP.getComplexPattern(LeafRec); | |||
3575 | if (CP.hasProperty(SDNPMayStore)) mayStore = true; | |||
3576 | if (CP.hasProperty(SDNPMayLoad)) mayLoad = true; | |||
3577 | if (CP.hasProperty(SDNPSideEffect)) hasSideEffects = true; | |||
3578 | } | |||
3579 | } | |||
3580 | return; | |||
3581 | } | |||
3582 | ||||
3583 | // Analyze children. | |||
3584 | for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) | |||
3585 | AnalyzeNode(N->getChild(i)); | |||
3586 | ||||
3587 | // Notice properties of the node. | |||
3588 | if (N->NodeHasProperty(SDNPMayStore, CDP)) mayStore = true; | |||
3589 | if (N->NodeHasProperty(SDNPMayLoad, CDP)) mayLoad = true; | |||
3590 | if (N->NodeHasProperty(SDNPSideEffect, CDP)) hasSideEffects = true; | |||
3591 | if (N->NodeHasProperty(SDNPVariadic, CDP)) isVariadic = true; | |||
3592 | if (N->NodeHasProperty(SDNPHasChain, CDP)) hasChain = true; | |||
3593 | ||||
3594 | if (const CodeGenIntrinsic *IntInfo = N->getIntrinsicInfo(CDP)) { | |||
3595 | // If this is an intrinsic, analyze it. | |||
3596 | if (IntInfo->ModRef & CodeGenIntrinsic::MR_Ref) | |||
3597 | mayLoad = true;// These may load memory. | |||
3598 | ||||
3599 | if (IntInfo->ModRef & CodeGenIntrinsic::MR_Mod) | |||
3600 | mayStore = true;// Intrinsics that can write to memory are 'mayStore'. | |||
3601 | ||||
3602 | if (IntInfo->ModRef >= CodeGenIntrinsic::ReadWriteMem || | |||
3603 | IntInfo->hasSideEffects) | |||
3604 | // ReadWriteMem intrinsics can have other strange effects. | |||
3605 | hasSideEffects = true; | |||
3606 | } | |||
3607 | } | |||
3608 | ||||
3609 | }; | |||
3610 | ||||
3611 | static bool InferFromPattern(CodeGenInstruction &InstInfo, | |||
3612 | const InstAnalyzer &PatInfo, | |||
3613 | Record *PatDef) { | |||
3614 | bool Error = false; | |||
3615 | ||||
3616 | // Remember where InstInfo got its flags. | |||
3617 | if (InstInfo.hasUndefFlags()) | |||
3618 | InstInfo.InferredFrom = PatDef; | |||
3619 | ||||
3620 | // Check explicitly set flags for consistency. | |||
3621 | if (InstInfo.hasSideEffects != PatInfo.hasSideEffects && | |||
3622 | !InstInfo.hasSideEffects_Unset) { | |||
3623 | // Allow explicitly setting hasSideEffects = 1 on instructions, even when | |||
3624 | // the pattern has no side effects. That could be useful for div/rem | |||
3625 | // instructions that may trap. | |||
3626 | if (!InstInfo.hasSideEffects) { | |||
3627 | Error = true; | |||
3628 | PrintError(PatDef->getLoc(), "Pattern doesn't match hasSideEffects = " + | |||
3629 | Twine(InstInfo.hasSideEffects)); | |||
3630 | } | |||
3631 | } | |||
3632 | ||||
3633 | if (InstInfo.mayStore != PatInfo.mayStore && !InstInfo.mayStore_Unset) { | |||
3634 | Error = true; | |||
3635 | PrintError(PatDef->getLoc(), "Pattern doesn't match mayStore = " + | |||
3636 | Twine(InstInfo.mayStore)); | |||
3637 | } | |||
3638 | ||||
3639 | if (InstInfo.mayLoad != PatInfo.mayLoad && !InstInfo.mayLoad_Unset) { | |||
3640 | // Allow explicitly setting mayLoad = 1, even when the pattern has no loads. | |||
3641 | // Some targets translate immediates to loads. | |||
3642 | if (!InstInfo.mayLoad) { | |||
3643 | Error = true; | |||
3644 | PrintError(PatDef->getLoc(), "Pattern doesn't match mayLoad = " + | |||
3645 | Twine(InstInfo.mayLoad)); | |||
3646 | } | |||
3647 | } | |||
3648 | ||||
3649 | // Transfer inferred flags. | |||
3650 | InstInfo.hasSideEffects |= PatInfo.hasSideEffects; | |||
3651 | InstInfo.mayStore |= PatInfo.mayStore; | |||
3652 | InstInfo.mayLoad |= PatInfo.mayLoad; | |||
3653 | ||||
3654 | // These flags are silently added without any verification. | |||
3655 | // FIXME: To match historical behavior of TableGen, for now add those flags | |||
3656 | // only when we're inferring from the primary instruction pattern. | |||
3657 | if (PatDef->isSubClassOf("Instruction")) { | |||
3658 | InstInfo.isBitcast |= PatInfo.isBitcast; | |||
3659 | InstInfo.hasChain |= PatInfo.hasChain; | |||
3660 | InstInfo.hasChain_Inferred = true; | |||
3661 | } | |||
3662 | ||||
3663 | // Don't infer isVariadic. This flag means something different on SDNodes and | |||
3664 | // instructions. For example, a CALL SDNode is variadic because it has the | |||
3665 | // call arguments as operands, but a CALL instruction is not variadic - it | |||
3666 | // has argument registers as implicit, not explicit uses. | |||
3667 | ||||
3668 | return Error; | |||
3669 | } | |||
3670 | ||||
3671 | /// hasNullFragReference - Return true if the DAG has any reference to the | |||
3672 | /// null_frag operator. | |||
3673 | static bool hasNullFragReference(DagInit *DI) { | |||
3674 | DefInit *OpDef = dyn_cast<DefInit>(DI->getOperator()); | |||
3675 | if (!OpDef) return false; | |||
3676 | Record *Operator = OpDef->getDef(); | |||
3677 | ||||
3678 | // If this is the null fragment, return true. | |||
3679 | if (Operator->getName() == "null_frag") return true; | |||
3680 | // If any of the arguments reference the null fragment, return true. | |||
3681 | for (unsigned i = 0, e = DI->getNumArgs(); i != e; ++i) { | |||
3682 | if (auto Arg = dyn_cast<DefInit>(DI->getArg(i))) | |||
3683 | if (Arg->getDef()->getName() == "null_frag") | |||
3684 | return true; | |||
3685 | DagInit *Arg = dyn_cast<DagInit>(DI->getArg(i)); | |||
3686 | if (Arg && hasNullFragReference(Arg)) | |||
3687 | return true; | |||
3688 | } | |||
3689 | ||||
3690 | return false; | |||
3691 | } | |||
3692 | ||||
3693 | /// hasNullFragReference - Return true if any DAG in the list references | |||
3694 | /// the null_frag operator. | |||
3695 | static bool hasNullFragReference(ListInit *LI) { | |||
3696 | for (Init *I : LI->getValues()) { | |||
3697 | DagInit *DI = dyn_cast<DagInit>(I); | |||
3698 | assert(DI && "non-dag in an instruction Pattern list?!")(static_cast <bool> (DI && "non-dag in an instruction Pattern list?!" ) ? void (0) : __assert_fail ("DI && \"non-dag in an instruction Pattern list?!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3698, __extension__ __PRETTY_FUNCTION__)); | |||
3699 | if (hasNullFragReference(DI)) | |||
3700 | return true; | |||
3701 | } | |||
3702 | return false; | |||
3703 | } | |||
3704 | ||||
3705 | /// Get all the instructions in a tree. | |||
3706 | static void | |||
3707 | getInstructionsInTree(TreePatternNode *Tree, SmallVectorImpl<Record*> &Instrs) { | |||
3708 | if (Tree->isLeaf()) | |||
3709 | return; | |||
3710 | if (Tree->getOperator()->isSubClassOf("Instruction")) | |||
3711 | Instrs.push_back(Tree->getOperator()); | |||
3712 | for (unsigned i = 0, e = Tree->getNumChildren(); i != e; ++i) | |||
3713 | getInstructionsInTree(Tree->getChild(i), Instrs); | |||
3714 | } | |||
3715 | ||||
3716 | /// Check the class of a pattern leaf node against the instruction operand it | |||
3717 | /// represents. | |||
3718 | static bool checkOperandClass(CGIOperandList::OperandInfo &OI, | |||
3719 | Record *Leaf) { | |||
3720 | if (OI.Rec == Leaf) | |||
3721 | return true; | |||
3722 | ||||
3723 | // Allow direct value types to be used in instruction set patterns. | |||
3724 | // The type will be checked later. | |||
3725 | if (Leaf->isSubClassOf("ValueType")) | |||
3726 | return true; | |||
3727 | ||||
3728 | // Patterns can also be ComplexPattern instances. | |||
3729 | if (Leaf->isSubClassOf("ComplexPattern")) | |||
3730 | return true; | |||
3731 | ||||
3732 | return false; | |||
3733 | } | |||
3734 | ||||
3735 | void CodeGenDAGPatterns::parseInstructionPattern( | |||
3736 | CodeGenInstruction &CGI, ListInit *Pat, DAGInstMap &DAGInsts) { | |||
3737 | ||||
3738 | assert(!DAGInsts.count(CGI.TheDef) && "Instruction already parsed!")(static_cast <bool> (!DAGInsts.count(CGI.TheDef) && "Instruction already parsed!") ? void (0) : __assert_fail ("!DAGInsts.count(CGI.TheDef) && \"Instruction already parsed!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3738, __extension__ __PRETTY_FUNCTION__)); | |||
3739 | ||||
3740 | // Parse the instruction. | |||
3741 | TreePattern I(CGI.TheDef, Pat, true, *this); | |||
3742 | ||||
3743 | // InstInputs - Keep track of all of the inputs of the instruction, along | |||
3744 | // with the record they are declared as. | |||
3745 | std::map<std::string, TreePatternNodePtr> InstInputs; | |||
3746 | ||||
3747 | // InstResults - Keep track of all the virtual registers that are 'set' | |||
3748 | // in the instruction, including what reg class they are. | |||
3749 | MapVector<std::string, TreePatternNodePtr, std::map<std::string, unsigned>> | |||
3750 | InstResults; | |||
3751 | ||||
3752 | std::vector<Record*> InstImpResults; | |||
3753 | ||||
3754 | // Verify that the top-level forms in the instruction are of void type, and | |||
3755 | // fill in the InstResults map. | |||
3756 | SmallString<32> TypesString; | |||
3757 | for (unsigned j = 0, e = I.getNumTrees(); j != e; ++j) { | |||
3758 | TypesString.clear(); | |||
3759 | TreePatternNodePtr Pat = I.getTree(j); | |||
3760 | if (Pat->getNumTypes() != 0) { | |||
3761 | raw_svector_ostream OS(TypesString); | |||
3762 | ListSeparator LS; | |||
3763 | for (unsigned k = 0, ke = Pat->getNumTypes(); k != ke; ++k) { | |||
3764 | OS << LS; | |||
3765 | Pat->getExtType(k).writeToStream(OS); | |||
3766 | } | |||
3767 | I.error("Top-level forms in instruction pattern should have" | |||
3768 | " void types, has types " + | |||
3769 | OS.str()); | |||
3770 | } | |||
3771 | ||||
3772 | // Find inputs and outputs, and verify the structure of the uses/defs. | |||
3773 | FindPatternInputsAndOutputs(I, Pat, InstInputs, InstResults, | |||
3774 | InstImpResults); | |||
3775 | } | |||
3776 | ||||
3777 | // Now that we have inputs and outputs of the pattern, inspect the operands | |||
3778 | // list for the instruction. This determines the order that operands are | |||
3779 | // added to the machine instruction the node corresponds to. | |||
3780 | unsigned NumResults = InstResults.size(); | |||
3781 | ||||
3782 | // Parse the operands list from the (ops) list, validating it. | |||
3783 | assert(I.getArgList().empty() && "Args list should still be empty here!")(static_cast <bool> (I.getArgList().empty() && "Args list should still be empty here!" ) ? void (0) : __assert_fail ("I.getArgList().empty() && \"Args list should still be empty here!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3783, __extension__ __PRETTY_FUNCTION__)); | |||
3784 | ||||
3785 | // Check that all of the results occur first in the list. | |||
3786 | std::vector<Record*> Results; | |||
3787 | std::vector<unsigned> ResultIndices; | |||
3788 | SmallVector<TreePatternNodePtr, 2> ResNodes; | |||
3789 | for (unsigned i = 0; i != NumResults; ++i) { | |||
3790 | if (i == CGI.Operands.size()) { | |||
3791 | const std::string &OpName = | |||
3792 | llvm::find_if( | |||
3793 | InstResults, | |||
3794 | [](const std::pair<std::string, TreePatternNodePtr> &P) { | |||
3795 | return P.second; | |||
3796 | }) | |||
3797 | ->first; | |||
3798 | ||||
3799 | I.error("'" + OpName + "' set but does not appear in operand list!"); | |||
3800 | } | |||
3801 | ||||
3802 | const std::string &OpName = CGI.Operands[i].Name; | |||
3803 | ||||
3804 | // Check that it exists in InstResults. | |||
3805 | auto InstResultIter = InstResults.find(OpName); | |||
3806 | if (InstResultIter == InstResults.end() || !InstResultIter->second) | |||
3807 | I.error("Operand $" + OpName + " does not exist in operand list!"); | |||
3808 | ||||
3809 | TreePatternNodePtr RNode = InstResultIter->second; | |||
3810 | Record *R = cast<DefInit>(RNode->getLeafValue())->getDef(); | |||
3811 | ResNodes.push_back(std::move(RNode)); | |||
3812 | if (!R) | |||
3813 | I.error("Operand $" + OpName + " should be a set destination: all " | |||
3814 | "outputs must occur before inputs in operand list!"); | |||
3815 | ||||
3816 | if (!checkOperandClass(CGI.Operands[i], R)) | |||
3817 | I.error("Operand $" + OpName + " class mismatch!"); | |||
3818 | ||||
3819 | // Remember the return type. | |||
3820 | Results.push_back(CGI.Operands[i].Rec); | |||
3821 | ||||
3822 | // Remember the result index. | |||
3823 | ResultIndices.push_back(std::distance(InstResults.begin(), InstResultIter)); | |||
3824 | ||||
3825 | // Okay, this one checks out. | |||
3826 | InstResultIter->second = nullptr; | |||
3827 | } | |||
3828 | ||||
3829 | // Loop over the inputs next. | |||
3830 | std::vector<TreePatternNodePtr> ResultNodeOperands; | |||
3831 | std::vector<Record*> Operands; | |||
3832 | for (unsigned i = NumResults, e = CGI.Operands.size(); i != e; ++i) { | |||
3833 | CGIOperandList::OperandInfo &Op = CGI.Operands[i]; | |||
3834 | const std::string &OpName = Op.Name; | |||
3835 | if (OpName.empty()) | |||
3836 | I.error("Operand #" + Twine(i) + " in operands list has no name!"); | |||
3837 | ||||
3838 | if (!InstInputs.count(OpName)) { | |||
3839 | // If this is an operand with a DefaultOps set filled in, we can ignore | |||
3840 | // this. When we codegen it, we will do so as always executed. | |||
3841 | if (Op.Rec->isSubClassOf("OperandWithDefaultOps")) { | |||
3842 | // Does it have a non-empty DefaultOps field? If so, ignore this | |||
3843 | // operand. | |||
3844 | if (!getDefaultOperand(Op.Rec).DefaultOps.empty()) | |||
3845 | continue; | |||
3846 | } | |||
3847 | I.error("Operand $" + OpName + | |||
3848 | " does not appear in the instruction pattern"); | |||
3849 | } | |||
3850 | TreePatternNodePtr InVal = InstInputs[OpName]; | |||
3851 | InstInputs.erase(OpName); // It occurred, remove from map. | |||
3852 | ||||
3853 | if (InVal->isLeaf() && isa<DefInit>(InVal->getLeafValue())) { | |||
3854 | Record *InRec = cast<DefInit>(InVal->getLeafValue())->getDef(); | |||
3855 | if (!checkOperandClass(Op, InRec)) | |||
3856 | I.error("Operand $" + OpName + "'s register class disagrees" | |||
3857 | " between the operand and pattern"); | |||
3858 | } | |||
3859 | Operands.push_back(Op.Rec); | |||
3860 | ||||
3861 | // Construct the result for the dest-pattern operand list. | |||
3862 | TreePatternNodePtr OpNode = InVal->clone(); | |||
3863 | ||||
3864 | // No predicate is useful on the result. | |||
3865 | OpNode->clearPredicateCalls(); | |||
3866 | ||||
3867 | // Promote the xform function to be an explicit node if set. | |||
3868 | if (Record *Xform = OpNode->getTransformFn()) { | |||
3869 | OpNode->setTransformFn(nullptr); | |||
3870 | std::vector<TreePatternNodePtr> Children; | |||
3871 | Children.push_back(OpNode); | |||
3872 | OpNode = std::make_shared<TreePatternNode>(Xform, std::move(Children), | |||
3873 | OpNode->getNumTypes()); | |||
3874 | } | |||
3875 | ||||
3876 | ResultNodeOperands.push_back(std::move(OpNode)); | |||
3877 | } | |||
3878 | ||||
3879 | if (!InstInputs.empty()) | |||
3880 | I.error("Input operand $" + InstInputs.begin()->first + | |||
3881 | " occurs in pattern but not in operands list!"); | |||
3882 | ||||
3883 | TreePatternNodePtr ResultPattern = std::make_shared<TreePatternNode>( | |||
3884 | I.getRecord(), std::move(ResultNodeOperands), | |||
3885 | GetNumNodeResults(I.getRecord(), *this)); | |||
3886 | // Copy fully inferred output node types to instruction result pattern. | |||
3887 | for (unsigned i = 0; i != NumResults; ++i) { | |||
3888 | assert(ResNodes[i]->getNumTypes() == 1 && "FIXME: Unhandled")(static_cast <bool> (ResNodes[i]->getNumTypes() == 1 && "FIXME: Unhandled") ? void (0) : __assert_fail ("ResNodes[i]->getNumTypes() == 1 && \"FIXME: Unhandled\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 3888, __extension__ __PRETTY_FUNCTION__)); | |||
3889 | ResultPattern->setType(i, ResNodes[i]->getExtType(0)); | |||
3890 | ResultPattern->setResultIndex(i, ResultIndices[i]); | |||
3891 | } | |||
3892 | ||||
3893 | // FIXME: Assume only the first tree is the pattern. The others are clobber | |||
3894 | // nodes. | |||
3895 | TreePatternNodePtr Pattern = I.getTree(0); | |||
3896 | TreePatternNodePtr SrcPattern; | |||
3897 | if (Pattern->getOperator()->getName() == "set") { | |||
3898 | SrcPattern = Pattern->getChild(Pattern->getNumChildren()-1)->clone(); | |||
3899 | } else{ | |||
3900 | // Not a set (store or something?) | |||
3901 | SrcPattern = Pattern; | |||
3902 | } | |||
3903 | ||||
3904 | // Create and insert the instruction. | |||
3905 | // FIXME: InstImpResults should not be part of DAGInstruction. | |||
3906 | Record *R = I.getRecord(); | |||
3907 | DAGInsts.emplace(std::piecewise_construct, std::forward_as_tuple(R), | |||
3908 | std::forward_as_tuple(Results, Operands, InstImpResults, | |||
3909 | SrcPattern, ResultPattern)); | |||
3910 | ||||
3911 | LLVM_DEBUG(I.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { I.dump(); } } while (false); | |||
3912 | } | |||
3913 | ||||
3914 | /// ParseInstructions - Parse all of the instructions, inlining and resolving | |||
3915 | /// any fragments involved. This populates the Instructions list with fully | |||
3916 | /// resolved instructions. | |||
3917 | void CodeGenDAGPatterns::ParseInstructions() { | |||
3918 | std::vector<Record*> Instrs = Records.getAllDerivedDefinitions("Instruction"); | |||
3919 | ||||
3920 | for (Record *Instr : Instrs) { | |||
3921 | ListInit *LI = nullptr; | |||
3922 | ||||
3923 | if (isa<ListInit>(Instr->getValueInit("Pattern"))) | |||
3924 | LI = Instr->getValueAsListInit("Pattern"); | |||
3925 | ||||
3926 | // If there is no pattern, only collect minimal information about the | |||
3927 | // instruction for its operand list. We have to assume that there is one | |||
3928 | // result, as we have no detailed info. A pattern which references the | |||
3929 | // null_frag operator is as-if no pattern were specified. Normally this | |||
3930 | // is from a multiclass expansion w/ a SDPatternOperator passed in as | |||
3931 | // null_frag. | |||
3932 | if (!LI || LI->empty() || hasNullFragReference(LI)) { | |||
3933 | std::vector<Record*> Results; | |||
3934 | std::vector<Record*> Operands; | |||
3935 | ||||
3936 | CodeGenInstruction &InstInfo = Target.getInstruction(Instr); | |||
3937 | ||||
3938 | if (InstInfo.Operands.size() != 0) { | |||
3939 | for (unsigned j = 0, e = InstInfo.Operands.NumDefs; j < e; ++j) | |||
3940 | Results.push_back(InstInfo.Operands[j].Rec); | |||
3941 | ||||
3942 | // The rest are inputs. | |||
3943 | for (unsigned j = InstInfo.Operands.NumDefs, | |||
3944 | e = InstInfo.Operands.size(); j < e; ++j) | |||
3945 | Operands.push_back(InstInfo.Operands[j].Rec); | |||
3946 | } | |||
3947 | ||||
3948 | // Create and insert the instruction. | |||
3949 | std::vector<Record*> ImpResults; | |||
3950 | Instructions.insert(std::make_pair(Instr, | |||
3951 | DAGInstruction(Results, Operands, ImpResults))); | |||
3952 | continue; // no pattern. | |||
3953 | } | |||
3954 | ||||
3955 | CodeGenInstruction &CGI = Target.getInstruction(Instr); | |||
3956 | parseInstructionPattern(CGI, LI, Instructions); | |||
3957 | } | |||
3958 | ||||
3959 | // If we can, convert the instructions to be patterns that are matched! | |||
3960 | for (auto &Entry : Instructions) { | |||
3961 | Record *Instr = Entry.first; | |||
3962 | DAGInstruction &TheInst = Entry.second; | |||
3963 | TreePatternNodePtr SrcPattern = TheInst.getSrcPattern(); | |||
3964 | TreePatternNodePtr ResultPattern = TheInst.getResultPattern(); | |||
3965 | ||||
3966 | if (SrcPattern && ResultPattern) { | |||
3967 | TreePattern Pattern(Instr, SrcPattern, true, *this); | |||
3968 | TreePattern Result(Instr, ResultPattern, false, *this); | |||
3969 | ParseOnePattern(Instr, Pattern, Result, TheInst.getImpResults()); | |||
3970 | } | |||
3971 | } | |||
3972 | } | |||
3973 | ||||
3974 | typedef std::pair<TreePatternNode *, unsigned> NameRecord; | |||
3975 | ||||
3976 | static void FindNames(TreePatternNode *P, | |||
3977 | std::map<std::string, NameRecord> &Names, | |||
3978 | TreePattern *PatternTop) { | |||
3979 | if (!P->getName().empty()) { | |||
3980 | NameRecord &Rec = Names[P->getName()]; | |||
3981 | // If this is the first instance of the name, remember the node. | |||
3982 | if (Rec.second++ == 0) | |||
3983 | Rec.first = P; | |||
3984 | else if (Rec.first->getExtTypes() != P->getExtTypes()) | |||
3985 | PatternTop->error("repetition of value: $" + P->getName() + | |||
3986 | " where different uses have different types!"); | |||
3987 | } | |||
3988 | ||||
3989 | if (!P->isLeaf()) { | |||
3990 | for (unsigned i = 0, e = P->getNumChildren(); i != e; ++i) | |||
3991 | FindNames(P->getChild(i), Names, PatternTop); | |||
3992 | } | |||
3993 | } | |||
3994 | ||||
3995 | void CodeGenDAGPatterns::AddPatternToMatch(TreePattern *Pattern, | |||
3996 | PatternToMatch &&PTM) { | |||
3997 | // Do some sanity checking on the pattern we're about to match. | |||
3998 | std::string Reason; | |||
3999 | if (!PTM.getSrcPattern()->canPatternMatch(Reason, *this)) { | |||
4000 | PrintWarning(Pattern->getRecord()->getLoc(), | |||
4001 | Twine("Pattern can never match: ") + Reason); | |||
4002 | return; | |||
4003 | } | |||
4004 | ||||
4005 | // If the source pattern's root is a complex pattern, that complex pattern | |||
4006 | // must specify the nodes it can potentially match. | |||
4007 | if (const ComplexPattern *CP = | |||
4008 | PTM.getSrcPattern()->getComplexPatternInfo(*this)) | |||
4009 | if (CP->getRootNodes().empty()) | |||
4010 | Pattern->error("ComplexPattern at root must specify list of opcodes it" | |||
4011 | " could match"); | |||
4012 | ||||
4013 | ||||
4014 | // Find all of the named values in the input and output, ensure they have the | |||
4015 | // same type. | |||
4016 | std::map<std::string, NameRecord> SrcNames, DstNames; | |||
4017 | FindNames(PTM.getSrcPattern(), SrcNames, Pattern); | |||
4018 | FindNames(PTM.getDstPattern(), DstNames, Pattern); | |||
4019 | ||||
4020 | // Scan all of the named values in the destination pattern, rejecting them if | |||
4021 | // they don't exist in the input pattern. | |||
4022 | for (const auto &Entry : DstNames) { | |||
4023 | if (SrcNames[Entry.first].first == nullptr) | |||
4024 | Pattern->error("Pattern has input without matching name in output: $" + | |||
4025 | Entry.first); | |||
4026 | } | |||
4027 | ||||
4028 | // Scan all of the named values in the source pattern, rejecting them if the | |||
4029 | // name isn't used in the dest, and isn't used to tie two values together. | |||
4030 | for (const auto &Entry : SrcNames) | |||
4031 | if (DstNames[Entry.first].first == nullptr && | |||
4032 | SrcNames[Entry.first].second == 1) | |||
4033 | Pattern->error("Pattern has dead named input: $" + Entry.first); | |||
4034 | ||||
4035 | PatternsToMatch.push_back(std::move(PTM)); | |||
4036 | } | |||
4037 | ||||
4038 | void CodeGenDAGPatterns::InferInstructionFlags() { | |||
4039 | ArrayRef<const CodeGenInstruction*> Instructions = | |||
4040 | Target.getInstructionsByEnumValue(); | |||
4041 | ||||
4042 | unsigned Errors = 0; | |||
4043 | ||||
4044 | // Try to infer flags from all patterns in PatternToMatch. These include | |||
4045 | // both the primary instruction patterns (which always come first) and | |||
4046 | // patterns defined outside the instruction. | |||
4047 | for (const PatternToMatch &PTM : ptms()) { | |||
4048 | // We can only infer from single-instruction patterns, otherwise we won't | |||
4049 | // know which instruction should get the flags. | |||
4050 | SmallVector<Record*, 8> PatInstrs; | |||
4051 | getInstructionsInTree(PTM.getDstPattern(), PatInstrs); | |||
4052 | if (PatInstrs.size() != 1) | |||
4053 | continue; | |||
4054 | ||||
4055 | // Get the single instruction. | |||
4056 | CodeGenInstruction &InstInfo = Target.getInstruction(PatInstrs.front()); | |||
4057 | ||||
4058 | // Only infer properties from the first pattern. We'll verify the others. | |||
4059 | if (InstInfo.InferredFrom) | |||
4060 | continue; | |||
4061 | ||||
4062 | InstAnalyzer PatInfo(*this); | |||
4063 | PatInfo.Analyze(PTM); | |||
4064 | Errors += InferFromPattern(InstInfo, PatInfo, PTM.getSrcRecord()); | |||
4065 | } | |||
4066 | ||||
4067 | if (Errors) | |||
4068 | PrintFatalError("pattern conflicts"); | |||
4069 | ||||
4070 | // If requested by the target, guess any undefined properties. | |||
4071 | if (Target.guessInstructionProperties()) { | |||
4072 | for (unsigned i = 0, e = Instructions.size(); i != e; ++i) { | |||
4073 | CodeGenInstruction *InstInfo = | |||
4074 | const_cast<CodeGenInstruction *>(Instructions[i]); | |||
4075 | if (InstInfo->InferredFrom) | |||
4076 | continue; | |||
4077 | // The mayLoad and mayStore flags default to false. | |||
4078 | // Conservatively assume hasSideEffects if it wasn't explicit. | |||
4079 | if (InstInfo->hasSideEffects_Unset) | |||
4080 | InstInfo->hasSideEffects = true; | |||
4081 | } | |||
4082 | return; | |||
4083 | } | |||
4084 | ||||
4085 | // Complain about any flags that are still undefined. | |||
4086 | for (unsigned i = 0, e = Instructions.size(); i != e; ++i) { | |||
4087 | CodeGenInstruction *InstInfo = | |||
4088 | const_cast<CodeGenInstruction *>(Instructions[i]); | |||
4089 | if (InstInfo->InferredFrom) | |||
4090 | continue; | |||
4091 | if (InstInfo->hasSideEffects_Unset) | |||
4092 | PrintError(InstInfo->TheDef->getLoc(), | |||
4093 | "Can't infer hasSideEffects from patterns"); | |||
4094 | if (InstInfo->mayStore_Unset) | |||
4095 | PrintError(InstInfo->TheDef->getLoc(), | |||
4096 | "Can't infer mayStore from patterns"); | |||
4097 | if (InstInfo->mayLoad_Unset) | |||
4098 | PrintError(InstInfo->TheDef->getLoc(), | |||
4099 | "Can't infer mayLoad from patterns"); | |||
4100 | } | |||
4101 | } | |||
4102 | ||||
4103 | ||||
4104 | /// Verify instruction flags against pattern node properties. | |||
4105 | void CodeGenDAGPatterns::VerifyInstructionFlags() { | |||
4106 | unsigned Errors = 0; | |||
4107 | for (const PatternToMatch &PTM : ptms()) { | |||
4108 | SmallVector<Record*, 8> Instrs; | |||
4109 | getInstructionsInTree(PTM.getDstPattern(), Instrs); | |||
4110 | if (Instrs.empty()) | |||
4111 | continue; | |||
4112 | ||||
4113 | // Count the number of instructions with each flag set. | |||
4114 | unsigned NumSideEffects = 0; | |||
4115 | unsigned NumStores = 0; | |||
4116 | unsigned NumLoads = 0; | |||
4117 | for (const Record *Instr : Instrs) { | |||
4118 | const CodeGenInstruction &InstInfo = Target.getInstruction(Instr); | |||
4119 | NumSideEffects += InstInfo.hasSideEffects; | |||
4120 | NumStores += InstInfo.mayStore; | |||
4121 | NumLoads += InstInfo.mayLoad; | |||
4122 | } | |||
4123 | ||||
4124 | // Analyze the source pattern. | |||
4125 | InstAnalyzer PatInfo(*this); | |||
4126 | PatInfo.Analyze(PTM); | |||
4127 | ||||
4128 | // Collect error messages. | |||
4129 | SmallVector<std::string, 4> Msgs; | |||
4130 | ||||
4131 | // Check for missing flags in the output. | |||
4132 | // Permit extra flags for now at least. | |||
4133 | if (PatInfo.hasSideEffects && !NumSideEffects) | |||
4134 | Msgs.push_back("pattern has side effects, but hasSideEffects isn't set"); | |||
4135 | ||||
4136 | // Don't verify store flags on instructions with side effects. At least for | |||
4137 | // intrinsics, side effects implies mayStore. | |||
4138 | if (!PatInfo.hasSideEffects && PatInfo.mayStore && !NumStores) | |||
4139 | Msgs.push_back("pattern may store, but mayStore isn't set"); | |||
4140 | ||||
4141 | // Similarly, mayStore implies mayLoad on intrinsics. | |||
4142 | if (!PatInfo.mayStore && PatInfo.mayLoad && !NumLoads) | |||
4143 | Msgs.push_back("pattern may load, but mayLoad isn't set"); | |||
4144 | ||||
4145 | // Print error messages. | |||
4146 | if (Msgs.empty()) | |||
4147 | continue; | |||
4148 | ++Errors; | |||
4149 | ||||
4150 | for (const std::string &Msg : Msgs) | |||
4151 | PrintError(PTM.getSrcRecord()->getLoc(), Twine(Msg) + " on the " + | |||
4152 | (Instrs.size() == 1 ? | |||
4153 | "instruction" : "output instructions")); | |||
4154 | // Provide the location of the relevant instruction definitions. | |||
4155 | for (const Record *Instr : Instrs) { | |||
4156 | if (Instr != PTM.getSrcRecord()) | |||
4157 | PrintError(Instr->getLoc(), "defined here"); | |||
4158 | const CodeGenInstruction &InstInfo = Target.getInstruction(Instr); | |||
4159 | if (InstInfo.InferredFrom && | |||
4160 | InstInfo.InferredFrom != InstInfo.TheDef && | |||
4161 | InstInfo.InferredFrom != PTM.getSrcRecord()) | |||
4162 | PrintError(InstInfo.InferredFrom->getLoc(), "inferred from pattern"); | |||
4163 | } | |||
4164 | } | |||
4165 | if (Errors) | |||
4166 | PrintFatalError("Errors in DAG patterns"); | |||
4167 | } | |||
4168 | ||||
4169 | /// Given a pattern result with an unresolved type, see if we can find one | |||
4170 | /// instruction with an unresolved result type. Force this result type to an | |||
4171 | /// arbitrary element if it's possible types to converge results. | |||
4172 | static bool ForceArbitraryInstResultType(TreePatternNode *N, TreePattern &TP) { | |||
4173 | if (N->isLeaf()) | |||
4174 | return false; | |||
4175 | ||||
4176 | // Analyze children. | |||
4177 | for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) | |||
4178 | if (ForceArbitraryInstResultType(N->getChild(i), TP)) | |||
4179 | return true; | |||
4180 | ||||
4181 | if (!N->getOperator()->isSubClassOf("Instruction")) | |||
4182 | return false; | |||
4183 | ||||
4184 | // If this type is already concrete or completely unknown we can't do | |||
4185 | // anything. | |||
4186 | TypeInfer &TI = TP.getInfer(); | |||
4187 | for (unsigned i = 0, e = N->getNumTypes(); i != e; ++i) { | |||
4188 | if (N->getExtType(i).empty() || TI.isConcrete(N->getExtType(i), false)) | |||
4189 | continue; | |||
4190 | ||||
4191 | // Otherwise, force its type to an arbitrary choice. | |||
4192 | if (TI.forceArbitrary(N->getExtType(i))) | |||
4193 | return true; | |||
4194 | } | |||
4195 | ||||
4196 | return false; | |||
4197 | } | |||
4198 | ||||
4199 | // Promote xform function to be an explicit node wherever set. | |||
4200 | static TreePatternNodePtr PromoteXForms(TreePatternNodePtr N) { | |||
4201 | if (Record *Xform = N->getTransformFn()) { | |||
4202 | N->setTransformFn(nullptr); | |||
4203 | std::vector<TreePatternNodePtr> Children; | |||
4204 | Children.push_back(PromoteXForms(N)); | |||
4205 | return std::make_shared<TreePatternNode>(Xform, std::move(Children), | |||
4206 | N->getNumTypes()); | |||
4207 | } | |||
4208 | ||||
4209 | if (!N->isLeaf()) | |||
4210 | for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) { | |||
4211 | TreePatternNodePtr Child = N->getChildShared(i); | |||
4212 | N->setChild(i, PromoteXForms(Child)); | |||
4213 | } | |||
4214 | return N; | |||
4215 | } | |||
4216 | ||||
4217 | void CodeGenDAGPatterns::ParseOnePattern(Record *TheDef, | |||
4218 | TreePattern &Pattern, TreePattern &Result, | |||
4219 | const std::vector<Record *> &InstImpResults) { | |||
4220 | ||||
4221 | // Inline pattern fragments and expand multiple alternatives. | |||
4222 | Pattern.InlinePatternFragments(); | |||
4223 | Result.InlinePatternFragments(); | |||
4224 | ||||
4225 | if (Result.getNumTrees() != 1) | |||
4226 | Result.error("Cannot use multi-alternative fragments in result pattern!"); | |||
4227 | ||||
4228 | // Infer types. | |||
4229 | bool IterateInference; | |||
4230 | bool InferredAllPatternTypes, InferredAllResultTypes; | |||
4231 | do { | |||
4232 | // Infer as many types as possible. If we cannot infer all of them, we | |||
4233 | // can never do anything with this pattern: report it to the user. | |||
4234 | InferredAllPatternTypes = | |||
4235 | Pattern.InferAllTypes(&Pattern.getNamedNodesMap()); | |||
4236 | ||||
4237 | // Infer as many types as possible. If we cannot infer all of them, we | |||
4238 | // can never do anything with this pattern: report it to the user. | |||
4239 | InferredAllResultTypes = | |||
4240 | Result.InferAllTypes(&Pattern.getNamedNodesMap()); | |||
4241 | ||||
4242 | IterateInference = false; | |||
4243 | ||||
4244 | // Apply the type of the result to the source pattern. This helps us | |||
4245 | // resolve cases where the input type is known to be a pointer type (which | |||
4246 | // is considered resolved), but the result knows it needs to be 32- or | |||
4247 | // 64-bits. Infer the other way for good measure. | |||
4248 | for (const auto &T : Pattern.getTrees()) | |||
4249 | for (unsigned i = 0, e = std::min(Result.getOnlyTree()->getNumTypes(), | |||
4250 | T->getNumTypes()); | |||
4251 | i != e; ++i) { | |||
4252 | IterateInference |= T->UpdateNodeType( | |||
4253 | i, Result.getOnlyTree()->getExtType(i), Result); | |||
4254 | IterateInference |= Result.getOnlyTree()->UpdateNodeType( | |||
4255 | i, T->getExtType(i), Result); | |||
4256 | } | |||
4257 | ||||
4258 | // If our iteration has converged and the input pattern's types are fully | |||
4259 | // resolved but the result pattern is not fully resolved, we may have a | |||
4260 | // situation where we have two instructions in the result pattern and | |||
4261 | // the instructions require a common register class, but don't care about | |||
4262 | // what actual MVT is used. This is actually a bug in our modelling: | |||
4263 | // output patterns should have register classes, not MVTs. | |||
4264 | // | |||
4265 | // In any case, to handle this, we just go through and disambiguate some | |||
4266 | // arbitrary types to the result pattern's nodes. | |||
4267 | if (!IterateInference && InferredAllPatternTypes && | |||
4268 | !InferredAllResultTypes) | |||
4269 | IterateInference = | |||
4270 | ForceArbitraryInstResultType(Result.getTree(0).get(), Result); | |||
4271 | } while (IterateInference); | |||
4272 | ||||
4273 | // Verify that we inferred enough types that we can do something with the | |||
4274 | // pattern and result. If these fire the user has to add type casts. | |||
4275 | if (!InferredAllPatternTypes) | |||
4276 | Pattern.error("Could not infer all types in pattern!"); | |||
4277 | if (!InferredAllResultTypes) { | |||
4278 | Pattern.dump(); | |||
4279 | Result.error("Could not infer all types in pattern result!"); | |||
4280 | } | |||
4281 | ||||
4282 | // Promote xform function to be an explicit node wherever set. | |||
4283 | TreePatternNodePtr DstShared = PromoteXForms(Result.getOnlyTree()); | |||
4284 | ||||
4285 | TreePattern Temp(Result.getRecord(), DstShared, false, *this); | |||
4286 | Temp.InferAllTypes(); | |||
4287 | ||||
4288 | ListInit *Preds = TheDef->getValueAsListInit("Predicates"); | |||
4289 | int Complexity = TheDef->getValueAsInt("AddedComplexity"); | |||
4290 | ||||
4291 | if (PatternRewriter) | |||
4292 | PatternRewriter(&Pattern); | |||
4293 | ||||
4294 | // A pattern may end up with an "impossible" type, i.e. a situation | |||
4295 | // where all types have been eliminated for some node in this pattern. | |||
4296 | // This could occur for intrinsics that only make sense for a specific | |||
4297 | // value type, and use a specific register class. If, for some mode, | |||
4298 | // that register class does not accept that type, the type inference | |||
4299 | // will lead to a contradiction, which is not an error however, but | |||
4300 | // a sign that this pattern will simply never match. | |||
4301 | if (Temp.getOnlyTree()->hasPossibleType()) | |||
4302 | for (const auto &T : Pattern.getTrees()) | |||
4303 | if (T->hasPossibleType()) | |||
4304 | AddPatternToMatch(&Pattern, | |||
4305 | PatternToMatch(TheDef, Preds, T, Temp.getOnlyTree(), | |||
4306 | InstImpResults, Complexity, | |||
4307 | TheDef->getID())); | |||
4308 | } | |||
4309 | ||||
4310 | void CodeGenDAGPatterns::ParsePatterns() { | |||
4311 | std::vector<Record*> Patterns = Records.getAllDerivedDefinitions("Pattern"); | |||
4312 | ||||
4313 | for (Record *CurPattern : Patterns) { | |||
4314 | DagInit *Tree = CurPattern->getValueAsDag("PatternToMatch"); | |||
4315 | ||||
4316 | // If the pattern references the null_frag, there's nothing to do. | |||
4317 | if (hasNullFragReference(Tree)) | |||
4318 | continue; | |||
4319 | ||||
4320 | TreePattern Pattern(CurPattern, Tree, true, *this); | |||
4321 | ||||
4322 | ListInit *LI = CurPattern->getValueAsListInit("ResultInstrs"); | |||
4323 | if (LI->empty()) continue; // no pattern. | |||
4324 | ||||
4325 | // Parse the instruction. | |||
4326 | TreePattern Result(CurPattern, LI, false, *this); | |||
4327 | ||||
4328 | if (Result.getNumTrees() != 1) | |||
4329 | Result.error("Cannot handle instructions producing instructions " | |||
4330 | "with temporaries yet!"); | |||
4331 | ||||
4332 | // Validate that the input pattern is correct. | |||
4333 | std::map<std::string, TreePatternNodePtr> InstInputs; | |||
4334 | MapVector<std::string, TreePatternNodePtr, std::map<std::string, unsigned>> | |||
4335 | InstResults; | |||
4336 | std::vector<Record*> InstImpResults; | |||
4337 | for (unsigned j = 0, ee = Pattern.getNumTrees(); j != ee; ++j) | |||
4338 | FindPatternInputsAndOutputs(Pattern, Pattern.getTree(j), InstInputs, | |||
4339 | InstResults, InstImpResults); | |||
4340 | ||||
4341 | ParseOnePattern(CurPattern, Pattern, Result, InstImpResults); | |||
4342 | } | |||
4343 | } | |||
4344 | ||||
4345 | static void collectModes(std::set<unsigned> &Modes, const TreePatternNode *N) { | |||
4346 | for (const TypeSetByHwMode &VTS : N->getExtTypes()) | |||
4347 | for (const auto &I : VTS) | |||
4348 | Modes.insert(I.first); | |||
4349 | ||||
4350 | for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) | |||
4351 | collectModes(Modes, N->getChild(i)); | |||
4352 | } | |||
4353 | ||||
4354 | void CodeGenDAGPatterns::ExpandHwModeBasedTypes() { | |||
4355 | const CodeGenHwModes &CGH = getTargetInfo().getHwModes(); | |||
4356 | std::vector<PatternToMatch> Copy; | |||
4357 | PatternsToMatch.swap(Copy); | |||
4358 | ||||
4359 | auto AppendPattern = [this](PatternToMatch &P, unsigned Mode, | |||
4360 | StringRef Check) { | |||
4361 | TreePatternNodePtr NewSrc = P.getSrcPattern()->clone(); | |||
4362 | TreePatternNodePtr NewDst = P.getDstPattern()->clone(); | |||
4363 | if (!NewSrc->setDefaultMode(Mode) || !NewDst->setDefaultMode(Mode)) { | |||
4364 | return; | |||
4365 | } | |||
4366 | ||||
4367 | PatternsToMatch.emplace_back(P.getSrcRecord(), P.getPredicates(), | |||
4368 | std::move(NewSrc), std::move(NewDst), | |||
4369 | P.getDstRegs(), P.getAddedComplexity(), | |||
4370 | Record::getNewUID(), Mode, Check); | |||
4371 | }; | |||
4372 | ||||
4373 | for (PatternToMatch &P : Copy) { | |||
4374 | TreePatternNodePtr SrcP = nullptr, DstP = nullptr; | |||
4375 | if (P.getSrcPattern()->hasProperTypeByHwMode()) | |||
4376 | SrcP = P.getSrcPatternShared(); | |||
4377 | if (P.getDstPattern()->hasProperTypeByHwMode()) | |||
4378 | DstP = P.getDstPatternShared(); | |||
4379 | if (!SrcP && !DstP) { | |||
4380 | PatternsToMatch.push_back(P); | |||
4381 | continue; | |||
4382 | } | |||
4383 | ||||
4384 | std::set<unsigned> Modes; | |||
4385 | if (SrcP) | |||
4386 | collectModes(Modes, SrcP.get()); | |||
4387 | if (DstP) | |||
4388 | collectModes(Modes, DstP.get()); | |||
4389 | ||||
4390 | // The predicate for the default mode needs to be constructed for each | |||
4391 | // pattern separately. | |||
4392 | // Since not all modes must be present in each pattern, if a mode m is | |||
4393 | // absent, then there is no point in constructing a check for m. If such | |||
4394 | // a check was created, it would be equivalent to checking the default | |||
4395 | // mode, except not all modes' predicates would be a part of the checking | |||
4396 | // code. The subsequently generated check for the default mode would then | |||
4397 | // have the exact same patterns, but a different predicate code. To avoid | |||
4398 | // duplicated patterns with different predicate checks, construct the | |||
4399 | // default check as a negation of all predicates that are actually present | |||
4400 | // in the source/destination patterns. | |||
4401 | SmallString<128> DefaultCheck; | |||
4402 | ||||
4403 | for (unsigned M : Modes) { | |||
4404 | if (M == DefaultMode) | |||
4405 | continue; | |||
4406 | ||||
4407 | // Fill the map entry for this mode. | |||
4408 | const HwMode &HM = CGH.getMode(M); | |||
4409 | AppendPattern(P, M, "(MF->getSubtarget().checkFeatures(\"" + HM.Features + "\"))"); | |||
4410 | ||||
4411 | // Add negations of the HM's predicates to the default predicate. | |||
4412 | if (!DefaultCheck.empty()) | |||
4413 | DefaultCheck += " && "; | |||
4414 | DefaultCheck += "(!(MF->getSubtarget().checkFeatures(\""; | |||
4415 | DefaultCheck += HM.Features; | |||
4416 | DefaultCheck += "\")))"; | |||
4417 | } | |||
4418 | ||||
4419 | bool HasDefault = Modes.count(DefaultMode); | |||
4420 | if (HasDefault) | |||
4421 | AppendPattern(P, DefaultMode, DefaultCheck); | |||
4422 | } | |||
4423 | } | |||
4424 | ||||
4425 | /// Dependent variable map for CodeGenDAGPattern variant generation | |||
4426 | typedef StringMap<int> DepVarMap; | |||
4427 | ||||
4428 | static void FindDepVarsOf(TreePatternNode *N, DepVarMap &DepMap) { | |||
4429 | if (N->isLeaf()) { | |||
4430 | if (N->hasName() && isa<DefInit>(N->getLeafValue())) | |||
4431 | DepMap[N->getName()]++; | |||
4432 | } else { | |||
4433 | for (size_t i = 0, e = N->getNumChildren(); i != e; ++i) | |||
4434 | FindDepVarsOf(N->getChild(i), DepMap); | |||
4435 | } | |||
4436 | } | |||
4437 | ||||
4438 | /// Find dependent variables within child patterns | |||
4439 | static void FindDepVars(TreePatternNode *N, MultipleUseVarSet &DepVars) { | |||
4440 | DepVarMap depcounts; | |||
4441 | FindDepVarsOf(N, depcounts); | |||
4442 | for (const auto &Pair : depcounts) { | |||
4443 | if (Pair.getValue() > 1) | |||
4444 | DepVars.insert(Pair.getKey()); | |||
4445 | } | |||
4446 | } | |||
4447 | ||||
4448 | #ifndef NDEBUG | |||
4449 | /// Dump the dependent variable set: | |||
4450 | static void DumpDepVars(MultipleUseVarSet &DepVars) { | |||
4451 | if (DepVars.empty()) { | |||
4452 | LLVM_DEBUG(errs() << "<empty set>")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "<empty set>"; } } while (false); | |||
4453 | } else { | |||
4454 | LLVM_DEBUG(errs() << "[ ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "[ "; } } while (false); | |||
4455 | for (const auto &DepVar : DepVars) { | |||
4456 | LLVM_DEBUG(errs() << DepVar.getKey() << " ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << DepVar.getKey() << " " ; } } while (false); | |||
4457 | } | |||
4458 | LLVM_DEBUG(errs() << "]")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "]"; } } while (false); | |||
4459 | } | |||
4460 | } | |||
4461 | #endif | |||
4462 | ||||
4463 | ||||
4464 | /// CombineChildVariants - Given a bunch of permutations of each child of the | |||
4465 | /// 'operator' node, put them together in all possible ways. | |||
4466 | static void CombineChildVariants( | |||
4467 | TreePatternNodePtr Orig, | |||
4468 | const std::vector<std::vector<TreePatternNodePtr>> &ChildVariants, | |||
4469 | std::vector<TreePatternNodePtr> &OutVariants, CodeGenDAGPatterns &CDP, | |||
4470 | const MultipleUseVarSet &DepVars) { | |||
4471 | // Make sure that each operand has at least one variant to choose from. | |||
4472 | for (const auto &Variants : ChildVariants) | |||
4473 | if (Variants.empty()) | |||
4474 | return; | |||
4475 | ||||
4476 | // The end result is an all-pairs construction of the resultant pattern. | |||
4477 | std::vector<unsigned> Idxs; | |||
4478 | Idxs.resize(ChildVariants.size()); | |||
4479 | bool NotDone; | |||
4480 | do { | |||
4481 | #ifndef NDEBUG | |||
4482 | LLVM_DEBUG(if (!Idxs.empty()) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { if (!Idxs.empty()) { errs() << Orig ->getOperator()->getName() << ": Idxs = [ "; for ( unsigned Idx : Idxs) { errs() << Idx << " "; } errs () << "]\n"; }; } } while (false) | |||
4483 | errs() << Orig->getOperator()->getName() << ": Idxs = [ ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { if (!Idxs.empty()) { errs() << Orig ->getOperator()->getName() << ": Idxs = [ "; for ( unsigned Idx : Idxs) { errs() << Idx << " "; } errs () << "]\n"; }; } } while (false) | |||
4484 | for (unsigned Idx : Idxs) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { if (!Idxs.empty()) { errs() << Orig ->getOperator()->getName() << ": Idxs = [ "; for ( unsigned Idx : Idxs) { errs() << Idx << " "; } errs () << "]\n"; }; } } while (false) | |||
4485 | errs() << Idx << " ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { if (!Idxs.empty()) { errs() << Orig ->getOperator()->getName() << ": Idxs = [ "; for ( unsigned Idx : Idxs) { errs() << Idx << " "; } errs () << "]\n"; }; } } while (false) | |||
4486 | }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { if (!Idxs.empty()) { errs() << Orig ->getOperator()->getName() << ": Idxs = [ "; for ( unsigned Idx : Idxs) { errs() << Idx << " "; } errs () << "]\n"; }; } } while (false) | |||
4487 | errs() << "]\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { if (!Idxs.empty()) { errs() << Orig ->getOperator()->getName() << ": Idxs = [ "; for ( unsigned Idx : Idxs) { errs() << Idx << " "; } errs () << "]\n"; }; } } while (false) | |||
4488 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { if (!Idxs.empty()) { errs() << Orig ->getOperator()->getName() << ": Idxs = [ "; for ( unsigned Idx : Idxs) { errs() << Idx << " "; } errs () << "]\n"; }; } } while (false); | |||
4489 | #endif | |||
4490 | // Create the variant and add it to the output list. | |||
4491 | std::vector<TreePatternNodePtr> NewChildren; | |||
4492 | for (unsigned i = 0, e = ChildVariants.size(); i != e; ++i) | |||
4493 | NewChildren.push_back(ChildVariants[i][Idxs[i]]); | |||
4494 | TreePatternNodePtr R = std::make_shared<TreePatternNode>( | |||
4495 | Orig->getOperator(), std::move(NewChildren), Orig->getNumTypes()); | |||
4496 | ||||
4497 | // Copy over properties. | |||
4498 | R->setName(Orig->getName()); | |||
4499 | R->setNamesAsPredicateArg(Orig->getNamesAsPredicateArg()); | |||
4500 | R->setPredicateCalls(Orig->getPredicateCalls()); | |||
4501 | R->setTransformFn(Orig->getTransformFn()); | |||
4502 | for (unsigned i = 0, e = Orig->getNumTypes(); i != e; ++i) | |||
4503 | R->setType(i, Orig->getExtType(i)); | |||
4504 | ||||
4505 | // If this pattern cannot match, do not include it as a variant. | |||
4506 | std::string ErrString; | |||
4507 | // Scan to see if this pattern has already been emitted. We can get | |||
4508 | // duplication due to things like commuting: | |||
4509 | // (and GPRC:$a, GPRC:$b) -> (and GPRC:$b, GPRC:$a) | |||
4510 | // which are the same pattern. Ignore the dups. | |||
4511 | if (R->canPatternMatch(ErrString, CDP) && | |||
4512 | none_of(OutVariants, [&](TreePatternNodePtr Variant) { | |||
4513 | return R->isIsomorphicTo(Variant.get(), DepVars); | |||
4514 | })) | |||
4515 | OutVariants.push_back(R); | |||
4516 | ||||
4517 | // Increment indices to the next permutation by incrementing the | |||
4518 | // indices from last index backward, e.g., generate the sequence | |||
4519 | // [0, 0], [0, 1], [1, 0], [1, 1]. | |||
4520 | int IdxsIdx; | |||
4521 | for (IdxsIdx = Idxs.size() - 1; IdxsIdx >= 0; --IdxsIdx) { | |||
4522 | if (++Idxs[IdxsIdx] == ChildVariants[IdxsIdx].size()) | |||
4523 | Idxs[IdxsIdx] = 0; | |||
4524 | else | |||
4525 | break; | |||
4526 | } | |||
4527 | NotDone = (IdxsIdx >= 0); | |||
4528 | } while (NotDone); | |||
4529 | } | |||
4530 | ||||
4531 | /// CombineChildVariants - A helper function for binary operators. | |||
4532 | /// | |||
4533 | static void CombineChildVariants(TreePatternNodePtr Orig, | |||
4534 | const std::vector<TreePatternNodePtr> &LHS, | |||
4535 | const std::vector<TreePatternNodePtr> &RHS, | |||
4536 | std::vector<TreePatternNodePtr> &OutVariants, | |||
4537 | CodeGenDAGPatterns &CDP, | |||
4538 | const MultipleUseVarSet &DepVars) { | |||
4539 | std::vector<std::vector<TreePatternNodePtr>> ChildVariants; | |||
4540 | ChildVariants.push_back(LHS); | |||
4541 | ChildVariants.push_back(RHS); | |||
4542 | CombineChildVariants(Orig, ChildVariants, OutVariants, CDP, DepVars); | |||
4543 | } | |||
4544 | ||||
4545 | static void | |||
4546 | GatherChildrenOfAssociativeOpcode(TreePatternNodePtr N, | |||
4547 | std::vector<TreePatternNodePtr> &Children) { | |||
4548 | assert(N->getNumChildren()==2 &&"Associative but doesn't have 2 children!")(static_cast <bool> (N->getNumChildren()==2 && "Associative but doesn't have 2 children!") ? void (0) : __assert_fail ("N->getNumChildren()==2 &&\"Associative but doesn't have 2 children!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 4548, __extension__ __PRETTY_FUNCTION__)); | |||
4549 | Record *Operator = N->getOperator(); | |||
4550 | ||||
4551 | // Only permit raw nodes. | |||
4552 | if (!N->getName().empty() || !N->getPredicateCalls().empty() || | |||
4553 | N->getTransformFn()) { | |||
4554 | Children.push_back(N); | |||
4555 | return; | |||
4556 | } | |||
4557 | ||||
4558 | if (N->getChild(0)->isLeaf() || N->getChild(0)->getOperator() != Operator) | |||
4559 | Children.push_back(N->getChildShared(0)); | |||
4560 | else | |||
4561 | GatherChildrenOfAssociativeOpcode(N->getChildShared(0), Children); | |||
4562 | ||||
4563 | if (N->getChild(1)->isLeaf() || N->getChild(1)->getOperator() != Operator) | |||
4564 | Children.push_back(N->getChildShared(1)); | |||
4565 | else | |||
4566 | GatherChildrenOfAssociativeOpcode(N->getChildShared(1), Children); | |||
4567 | } | |||
4568 | ||||
4569 | /// GenerateVariantsOf - Given a pattern N, generate all permutations we can of | |||
4570 | /// the (potentially recursive) pattern by using algebraic laws. | |||
4571 | /// | |||
4572 | static void GenerateVariantsOf(TreePatternNodePtr N, | |||
4573 | std::vector<TreePatternNodePtr> &OutVariants, | |||
4574 | CodeGenDAGPatterns &CDP, | |||
4575 | const MultipleUseVarSet &DepVars) { | |||
4576 | // We cannot permute leaves or ComplexPattern uses. | |||
4577 | if (N->isLeaf() || N->getOperator()->isSubClassOf("ComplexPattern")) { | |||
4578 | OutVariants.push_back(N); | |||
4579 | return; | |||
4580 | } | |||
4581 | ||||
4582 | // Look up interesting info about the node. | |||
4583 | const SDNodeInfo &NodeInfo = CDP.getSDNodeInfo(N->getOperator()); | |||
4584 | ||||
4585 | // If this node is associative, re-associate. | |||
4586 | if (NodeInfo.hasProperty(SDNPAssociative)) { | |||
4587 | // Re-associate by pulling together all of the linked operators | |||
4588 | std::vector<TreePatternNodePtr> MaximalChildren; | |||
4589 | GatherChildrenOfAssociativeOpcode(N, MaximalChildren); | |||
4590 | ||||
4591 | // Only handle child sizes of 3. Otherwise we'll end up trying too many | |||
4592 | // permutations. | |||
4593 | if (MaximalChildren.size() == 3) { | |||
4594 | // Find the variants of all of our maximal children. | |||
4595 | std::vector<TreePatternNodePtr> AVariants, BVariants, CVariants; | |||
4596 | GenerateVariantsOf(MaximalChildren[0], AVariants, CDP, DepVars); | |||
4597 | GenerateVariantsOf(MaximalChildren[1], BVariants, CDP, DepVars); | |||
4598 | GenerateVariantsOf(MaximalChildren[2], CVariants, CDP, DepVars); | |||
4599 | ||||
4600 | // There are only two ways we can permute the tree: | |||
4601 | // (A op B) op C and A op (B op C) | |||
4602 | // Within these forms, we can also permute A/B/C. | |||
4603 | ||||
4604 | // Generate legal pair permutations of A/B/C. | |||
4605 | std::vector<TreePatternNodePtr> ABVariants; | |||
4606 | std::vector<TreePatternNodePtr> BAVariants; | |||
4607 | std::vector<TreePatternNodePtr> ACVariants; | |||
4608 | std::vector<TreePatternNodePtr> CAVariants; | |||
4609 | std::vector<TreePatternNodePtr> BCVariants; | |||
4610 | std::vector<TreePatternNodePtr> CBVariants; | |||
4611 | CombineChildVariants(N, AVariants, BVariants, ABVariants, CDP, DepVars); | |||
4612 | CombineChildVariants(N, BVariants, AVariants, BAVariants, CDP, DepVars); | |||
4613 | CombineChildVariants(N, AVariants, CVariants, ACVariants, CDP, DepVars); | |||
4614 | CombineChildVariants(N, CVariants, AVariants, CAVariants, CDP, DepVars); | |||
4615 | CombineChildVariants(N, BVariants, CVariants, BCVariants, CDP, DepVars); | |||
4616 | CombineChildVariants(N, CVariants, BVariants, CBVariants, CDP, DepVars); | |||
4617 | ||||
4618 | // Combine those into the result: (x op x) op x | |||
4619 | CombineChildVariants(N, ABVariants, CVariants, OutVariants, CDP, DepVars); | |||
4620 | CombineChildVariants(N, BAVariants, CVariants, OutVariants, CDP, DepVars); | |||
4621 | CombineChildVariants(N, ACVariants, BVariants, OutVariants, CDP, DepVars); | |||
4622 | CombineChildVariants(N, CAVariants, BVariants, OutVariants, CDP, DepVars); | |||
4623 | CombineChildVariants(N, BCVariants, AVariants, OutVariants, CDP, DepVars); | |||
4624 | CombineChildVariants(N, CBVariants, AVariants, OutVariants, CDP, DepVars); | |||
4625 | ||||
4626 | // Combine those into the result: x op (x op x) | |||
4627 | CombineChildVariants(N, CVariants, ABVariants, OutVariants, CDP, DepVars); | |||
4628 | CombineChildVariants(N, CVariants, BAVariants, OutVariants, CDP, DepVars); | |||
4629 | CombineChildVariants(N, BVariants, ACVariants, OutVariants, CDP, DepVars); | |||
4630 | CombineChildVariants(N, BVariants, CAVariants, OutVariants, CDP, DepVars); | |||
4631 | CombineChildVariants(N, AVariants, BCVariants, OutVariants, CDP, DepVars); | |||
4632 | CombineChildVariants(N, AVariants, CBVariants, OutVariants, CDP, DepVars); | |||
4633 | return; | |||
4634 | } | |||
4635 | } | |||
4636 | ||||
4637 | // Compute permutations of all children. | |||
4638 | std::vector<std::vector<TreePatternNodePtr>> ChildVariants; | |||
4639 | ChildVariants.resize(N->getNumChildren()); | |||
4640 | for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) | |||
4641 | GenerateVariantsOf(N->getChildShared(i), ChildVariants[i], CDP, DepVars); | |||
4642 | ||||
4643 | // Build all permutations based on how the children were formed. | |||
4644 | CombineChildVariants(N, ChildVariants, OutVariants, CDP, DepVars); | |||
4645 | ||||
4646 | // If this node is commutative, consider the commuted order. | |||
4647 | bool isCommIntrinsic = N->isCommutativeIntrinsic(CDP); | |||
4648 | if (NodeInfo.hasProperty(SDNPCommutative) || isCommIntrinsic) { | |||
4649 | unsigned Skip = isCommIntrinsic ? 1 : 0; // First operand is intrinsic id. | |||
4650 | assert(N->getNumChildren() >= (2 + Skip) &&(static_cast <bool> (N->getNumChildren() >= (2 + Skip ) && "Commutative but doesn't have 2 children!") ? void (0) : __assert_fail ("N->getNumChildren() >= (2 + Skip) && \"Commutative but doesn't have 2 children!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 4651, __extension__ __PRETTY_FUNCTION__)) | |||
4651 | "Commutative but doesn't have 2 children!")(static_cast <bool> (N->getNumChildren() >= (2 + Skip ) && "Commutative but doesn't have 2 children!") ? void (0) : __assert_fail ("N->getNumChildren() >= (2 + Skip) && \"Commutative but doesn't have 2 children!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 4651, __extension__ __PRETTY_FUNCTION__)); | |||
4652 | // Don't allow commuting children which are actually register references. | |||
4653 | bool NoRegisters = true; | |||
4654 | unsigned i = 0 + Skip; | |||
4655 | unsigned e = 2 + Skip; | |||
4656 | for (; i != e; ++i) { | |||
4657 | TreePatternNode *Child = N->getChild(i); | |||
4658 | if (Child->isLeaf()) | |||
4659 | if (DefInit *DI = dyn_cast<DefInit>(Child->getLeafValue())) { | |||
4660 | Record *RR = DI->getDef(); | |||
4661 | if (RR->isSubClassOf("Register")) | |||
4662 | NoRegisters = false; | |||
4663 | } | |||
4664 | } | |||
4665 | // Consider the commuted order. | |||
4666 | if (NoRegisters) { | |||
4667 | std::vector<std::vector<TreePatternNodePtr>> Variants; | |||
4668 | unsigned i = 0; | |||
4669 | if (isCommIntrinsic) | |||
4670 | Variants.push_back(std::move(ChildVariants[i++])); // Intrinsic id. | |||
4671 | Variants.push_back(std::move(ChildVariants[i + 1])); | |||
4672 | Variants.push_back(std::move(ChildVariants[i])); | |||
4673 | i += 2; | |||
4674 | // Remaining operands are not commuted. | |||
4675 | for (; i != N->getNumChildren(); ++i) | |||
4676 | Variants.push_back(std::move(ChildVariants[i])); | |||
4677 | CombineChildVariants(N, Variants, OutVariants, CDP, DepVars); | |||
4678 | } | |||
4679 | } | |||
4680 | } | |||
4681 | ||||
4682 | ||||
4683 | // GenerateVariants - Generate variants. For example, commutative patterns can | |||
4684 | // match multiple ways. Add them to PatternsToMatch as well. | |||
4685 | void CodeGenDAGPatterns::GenerateVariants() { | |||
4686 | LLVM_DEBUG(errs() << "Generating instruction variants.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "Generating instruction variants.\n" ; } } while (false); | |||
4687 | ||||
4688 | // Loop over all of the patterns we've collected, checking to see if we can | |||
4689 | // generate variants of the instruction, through the exploitation of | |||
4690 | // identities. This permits the target to provide aggressive matching without | |||
4691 | // the .td file having to contain tons of variants of instructions. | |||
4692 | // | |||
4693 | // Note that this loop adds new patterns to the PatternsToMatch list, but we | |||
4694 | // intentionally do not reconsider these. Any variants of added patterns have | |||
4695 | // already been added. | |||
4696 | // | |||
4697 | for (unsigned i = 0, e = PatternsToMatch.size(); i != e; ++i) { | |||
4698 | MultipleUseVarSet DepVars; | |||
4699 | std::vector<TreePatternNodePtr> Variants; | |||
4700 | FindDepVars(PatternsToMatch[i].getSrcPattern(), DepVars); | |||
4701 | LLVM_DEBUG(errs() << "Dependent/multiply used variables: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "Dependent/multiply used variables: " ; } } while (false); | |||
4702 | LLVM_DEBUG(DumpDepVars(DepVars))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { DumpDepVars(DepVars); } } while (false); | |||
4703 | LLVM_DEBUG(errs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "\n"; } } while (false); | |||
4704 | GenerateVariantsOf(PatternsToMatch[i].getSrcPatternShared(), Variants, | |||
4705 | *this, DepVars); | |||
4706 | ||||
4707 | assert(PatternsToMatch[i].getHwModeFeatures().empty() &&(static_cast <bool> (PatternsToMatch[i].getHwModeFeatures ().empty() && "HwModes should not have been expanded yet!" ) ? void (0) : __assert_fail ("PatternsToMatch[i].getHwModeFeatures().empty() && \"HwModes should not have been expanded yet!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 4708, __extension__ __PRETTY_FUNCTION__)) | |||
4708 | "HwModes should not have been expanded yet!")(static_cast <bool> (PatternsToMatch[i].getHwModeFeatures ().empty() && "HwModes should not have been expanded yet!" ) ? void (0) : __assert_fail ("PatternsToMatch[i].getHwModeFeatures().empty() && \"HwModes should not have been expanded yet!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 4708, __extension__ __PRETTY_FUNCTION__)); | |||
4709 | ||||
4710 | assert(!Variants.empty() && "Must create at least original variant!")(static_cast <bool> (!Variants.empty() && "Must create at least original variant!" ) ? void (0) : __assert_fail ("!Variants.empty() && \"Must create at least original variant!\"" , "llvm/utils/TableGen/CodeGenDAGPatterns.cpp", 4710, __extension__ __PRETTY_FUNCTION__)); | |||
4711 | if (Variants.size() == 1) // No additional variants for this pattern. | |||
4712 | continue; | |||
4713 | ||||
4714 | LLVM_DEBUG(errs() << "FOUND VARIANTS OF: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "FOUND VARIANTS OF: "; PatternsToMatch [i].getSrcPattern()->dump(); errs() << "\n"; } } while (false) | |||
4715 | PatternsToMatch[i].getSrcPattern()->dump(); errs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "FOUND VARIANTS OF: "; PatternsToMatch [i].getSrcPattern()->dump(); errs() << "\n"; } } while (false); | |||
4716 | ||||
4717 | for (unsigned v = 0, e = Variants.size(); v != e; ++v) { | |||
4718 | TreePatternNodePtr Variant = Variants[v]; | |||
4719 | ||||
4720 | LLVM_DEBUG(errs() << " VAR#" << v << ": "; Variant->dump();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << " VAR#" << v << ": "; Variant->dump(); errs() << "\n"; } } while (false ) | |||
4721 | errs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << " VAR#" << v << ": "; Variant->dump(); errs() << "\n"; } } while (false ); | |||
4722 | ||||
4723 | // Scan to see if an instruction or explicit pattern already matches this. | |||
4724 | bool AlreadyExists = false; | |||
4725 | for (unsigned p = 0, e = PatternsToMatch.size(); p != e; ++p) { | |||
4726 | // Skip if the top level predicates do not match. | |||
4727 | if ((i != p) && (PatternsToMatch[i].getPredicates() != | |||
4728 | PatternsToMatch[p].getPredicates())) | |||
4729 | continue; | |||
4730 | // Check to see if this variant already exists. | |||
4731 | if (Variant->isIsomorphicTo(PatternsToMatch[p].getSrcPattern(), | |||
4732 | DepVars)) { | |||
4733 | LLVM_DEBUG(errs() << " *** ALREADY EXISTS, ignoring variant.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << " *** ALREADY EXISTS, ignoring variant.\n" ; } } while (false); | |||
4734 | AlreadyExists = true; | |||
4735 | break; | |||
4736 | } | |||
4737 | } | |||
4738 | // If we already have it, ignore the variant. | |||
4739 | if (AlreadyExists) continue; | |||
4740 | ||||
4741 | // Otherwise, add it to the list of patterns we have. | |||
4742 | PatternsToMatch.emplace_back( | |||
4743 | PatternsToMatch[i].getSrcRecord(), PatternsToMatch[i].getPredicates(), | |||
4744 | Variant, PatternsToMatch[i].getDstPatternShared(), | |||
4745 | PatternsToMatch[i].getDstRegs(), | |||
4746 | PatternsToMatch[i].getAddedComplexity(), Record::getNewUID(), | |||
4747 | PatternsToMatch[i].getForceMode(), | |||
4748 | PatternsToMatch[i].getHwModeFeatures()); | |||
4749 | } | |||
4750 | ||||
4751 | LLVM_DEBUG(errs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dag-patterns")) { errs() << "\n"; } } while (false); | |||
4752 | } | |||
4753 | } |