20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/Support/Debug.h"
24 #define DEBUG_TYPE "format-token-breaker"
29 static const char *
const Blanks =
" \t\v\f\r";
44 static const char *
const KnownPrefixes[] = {
45 "///<",
"//!<",
"///",
"//",
"//!"};
46 StringRef LongestPrefix;
47 for (StringRef KnownPrefix : KnownPrefixes) {
48 if (Comment.startswith(KnownPrefix)) {
49 size_t PrefixLength = KnownPrefix.size();
50 while (PrefixLength < Comment.size() && Comment[PrefixLength] ==
' ')
52 if (PrefixLength > LongestPrefix.size())
53 LongestPrefix = Comment.substr(0, PrefixLength);
60 unsigned ContentStartColumn,
64 if (ColumnLimit <= ContentStartColumn + 1)
67 unsigned MaxSplit = ColumnLimit - ContentStartColumn + 1;
68 unsigned MaxSplitBytes = 0;
70 for (
unsigned NumChars = 0;
71 NumChars < MaxSplit && MaxSplitBytes < Text.size();) {
72 unsigned BytesInChar =
76 ContentStartColumn, TabWidth,
Encoding);
77 MaxSplitBytes += BytesInChar;
80 StringRef::size_type SpaceOffset = Text.find_last_of(
Blanks, MaxSplitBytes);
84 static llvm::Regex kNumberedListRegexp = llvm::Regex(
"^[1-9][0-9]?\\.");
85 if (SpaceOffset != StringRef::npos &&
86 kNumberedListRegexp.match(Text.substr(SpaceOffset).ltrim(
Blanks)))
87 SpaceOffset = Text.find_last_of(
Blanks, SpaceOffset);
89 if (SpaceOffset == StringRef::npos ||
91 Text.find_last_not_of(
Blanks, SpaceOffset) == StringRef::npos) {
94 StringRef::size_type FirstNonWhitespace = Text.find_first_not_of(
Blanks);
95 if (FirstNonWhitespace == StringRef::npos)
98 SpaceOffset = Text.find_first_of(
99 Blanks, std::max<unsigned>(MaxSplitBytes, FirstNonWhitespace));
101 if (SpaceOffset != StringRef::npos && SpaceOffset != 0) {
102 StringRef BeforeCut = Text.substr(0, SpaceOffset).rtrim(
Blanks);
103 StringRef AfterCut = Text.substr(SpaceOffset).ltrim(
Blanks);
105 AfterCut.begin() - BeforeCut.end());
116 if (ColumnLimit <= UsedColumns)
118 unsigned MaxSplit = ColumnLimit - UsedColumns;
119 StringRef::size_type SpaceOffset = 0;
120 StringRef::size_type SlashOffset = 0;
121 StringRef::size_type WordStartOffset = 0;
122 StringRef::size_type SplitPoint = 0;
123 for (
unsigned Chars = 0;;) {
125 if (Text[0] ==
'\\') {
131 Text.substr(0, Advance), UsedColumns + Chars, TabWidth,
Encoding);
134 if (Chars > MaxSplit || Text.size() <= Advance)
138 SpaceOffset = SplitPoint;
140 SlashOffset = SplitPoint;
142 WordStartOffset = SplitPoint;
144 SplitPoint += Advance;
145 Text = Text.substr(Advance);
148 if (SpaceOffset != 0)
150 if (SlashOffset != 0)
152 if (WordStartOffset != 0)
160 assert((Token.
is(TT_BlockComment) || Token.
is(TT_LineComment)) &&
161 "formatting regions are switched by comment tokens");
162 StringRef Content = Token.
TokenText.substr(2).ltrim();
163 return Content.startswith(
"clang-format on") ||
164 Content.startswith(
"clang-format off");
178 return RemainingTokenColumns + 1 - Split.second;
184 unsigned LineIndex,
unsigned TailOffset,
185 StringRef::size_type
Length)
const {
193 const FormatToken &Tok,
unsigned StartColumn, StringRef Prefix,
197 StartColumn(StartColumn), Prefix(Prefix), Postfix(Postfix) {
200 Prefix.size(), Tok.
TokenText.size() - Prefix.size() - Postfix.size());
204 const FormatToken &Tok,
unsigned StartColumn, StringRef Prefix,
212 unsigned ColumnLimit,
213 llvm::Regex &CommentPragmasRegex)
const {
228 unsigned StartColumn,
233 StartColumn(StartColumn) {}
239 unsigned ColumnLimit,
240 llvm::Regex &CommentPragmasRegex)
const {
242 if (CommentPragmasRegex.match(
Content[LineIndex]))
243 return Split(StringRef::npos, 0);
252 StringRef
Text =
Content[LineIndex].substr(TailOffset);
257 unsigned BreakOffsetInToken =
259 unsigned CharsToRemove = Split.second;
261 tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove,
"",
"",
267 unsigned PreviousEndColumn,
268 unsigned ColumnLimit)
const {
269 unsigned ReflowStartColumn = PreviousEndColumn + ReflowPrefix.size();
270 StringRef TrimmedText = Text.rtrim(
Blanks);
274 TrimmedText, ReflowStartColumn,
280 FullWidth <= ColumnLimit
281 ?
Split(TrimmedText.size(), Text.size() - TrimmedText.size())
290 if (ReflowSplit.first != StringRef::npos &&
292 PreviousEndColumn + ReflowPrefix.size() +
300 return Split(StringRef::npos, 0);
308 Content = Content.trim(
Blanks);
312 "@",
"TODO",
"FIXME",
"XXX",
"-# ",
"- ",
"+ ",
"* " };
313 bool hasSpecialMeaningPrefix =
false;
314 for (StringRef Prefix : kSpecialMeaningPrefixes) {
315 if (Content.startswith(Prefix)) {
316 hasSpecialMeaningPrefix =
true;
324 static llvm::Regex kNumberedListRegexp = llvm::Regex(
"^[1-9][0-9]?\\. ");
325 hasSpecialMeaningPrefix = hasSpecialMeaningPrefix ||
326 kNumberedListRegexp.match(Content);
331 return Content.size() >= 2 && !hasSpecialMeaningPrefix &&
332 !Content.endswith(
"\\") &&
340 unsigned OriginalStartColumn,
bool FirstInLine,
bool InPPDirective,
343 assert(
Tok.
is(TT_BlockComment) &&
344 "block comment section must start with a block comment");
347 assert(TokenText.startswith(
"/*") && TokenText.endswith(
"*/"));
348 TokenText.substr(2, TokenText.size() - 4).split(
Lines,
"\n");
350 int IndentDelta = StartColumn - OriginalStartColumn;
357 for (
size_t i = 1; i <
Lines.size(); ++i)
358 adjustWhitespace(i, IndentDelta);
362 DecorationColumn = StartColumn + 1;
375 if (
Lines.size() == 1 && !FirstInLine) {
383 for (
size_t i = 1, e =
Lines.size(); i < e && !Decoration.empty(); ++i) {
385 if (i + 1 == e &&
Content[i].empty())
387 if (!
Content[i].empty() && i + 1 != e &&
388 Decoration.startswith(
Content[i]))
390 while (!
Content[i].startswith(Decoration))
391 Decoration = Decoration.substr(0, Decoration.size() - 1);
394 LastLineNeedsDecoration =
true;
396 for (
size_t i = 1, e =
Lines.size(); i < e; ++i) {
402 LastLineNeedsDecoration =
false;
404 if (e >= 2 && !Decoration.empty()) {
407 }
else if (Decoration.empty()) {
419 unsigned DecorationSize = Decoration.startswith(
Content[i])
422 if (DecorationSize) {
426 if (!Decoration.startswith(
Content[i]))
431 std::max<unsigned>(IndentAtLineBreak, Decoration.size());
434 llvm::dbgs() <<
"IndentAtLineBreak " << IndentAtLineBreak <<
"\n";
435 for (
size_t i = 0; i <
Lines.size(); ++i) {
436 llvm::dbgs() << i <<
" |" <<
Content[i] <<
"| "
438 <<
"IN=" << (
Content[i].data() -
Lines[i].data()) <<
"\n";
443 void BreakableBlockComment::adjustWhitespace(
unsigned LineIndex,
450 size_t EndOfPreviousLine =
Lines[LineIndex - 1].size();
456 Lines[LineIndex - 1].find_last_not_of(
Blanks, EndOfPreviousLine);
457 if (EndOfPreviousLine == StringRef::npos)
458 EndOfPreviousLine = 0;
462 size_t StartOfLine =
Lines[LineIndex].find_first_not_of(
Blanks);
463 if (StartOfLine == StringRef::npos)
464 StartOfLine =
Lines[LineIndex].rtrim(
"\r\n").size();
466 StringRef Whitespace =
Lines[LineIndex].substr(0, StartOfLine);
468 size_t PreviousContentOffset =
469 Content[LineIndex - 1].data() -
Lines[LineIndex - 1].data();
471 PreviousContentOffset, EndOfPreviousLine - PreviousContentOffset);
472 Content[LineIndex] =
Lines[LineIndex].substr(StartOfLine);
481 unsigned LineIndex,
unsigned TailOffset,
482 StringRef::size_type
Length)
const {
483 unsigned ContentStartColumn = getContentStartColumn(LineIndex, TailOffset);
484 unsigned LineLength =
486 Content[LineIndex].substr(TailOffset, Length),
489 if (LineIndex + 1 ==
Lines.size()) {
494 if (
Content[LineIndex].substr(TailOffset, Length).empty()) {
495 LineLength -= Decoration.size();
504 StringRef
Text =
Content[LineIndex].substr(TailOffset);
505 StringRef Prefix = Decoration;
509 unsigned LocalIndentAtLineBreak = IndentAtLineBreak;
510 if (LineIndex + 1 ==
Lines.size() &&
511 Text.size() == Split.first + Split.second) {
514 if (LocalIndentAtLineBreak >= 2)
515 LocalIndentAtLineBreak -= 2;
519 unsigned BreakOffsetInToken =
521 unsigned CharsToRemove = Split.second;
522 assert(LocalIndentAtLineBreak >= Prefix.size());
524 tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove,
"", Prefix,
526 LocalIndentAtLineBreak - Prefix.size());
531 unsigned PreviousEndColumn,
532 unsigned ColumnLimit,
533 llvm::Regex &CommentPragmasRegex)
const {
534 if (!
mayReflow(LineIndex, CommentPragmasRegex))
535 return Split(StringRef::npos, 0);
541 unsigned BreakableBlockComment::getReflownColumn(
544 unsigned PreviousEndColumn)
const {
547 unsigned PostfixLength = (LineIndex + 1 ==
Lines.size() ? 2 : 0);
550 unsigned ReflownColumn =
554 return ReflownColumn;
558 unsigned LineIndex,
unsigned TailOffset,
559 unsigned PreviousEndColumn,
560 unsigned ColumnLimit,
561 Split SplitBefore)
const {
562 if (SplitBefore.first == StringRef::npos ||
567 SplitBefore.first + SplitBefore.second <
568 Content[LineIndex].ltrim().size()) {
574 StringRef TrimmedContent = Content[LineIndex].ltrim(
Blanks);
575 unsigned ReflownColumn =
576 getReflownColumn(TrimmedContent, LineIndex, PreviousEndColumn);
577 if (ReflownColumn <= ColumnLimit) {
578 return ReflownColumn;
584 unsigned LineIndex,
unsigned PreviousEndColumn,
unsigned ColumnLimit,
586 if (LineIndex == 0)
return;
587 StringRef TrimmedContent = Content[LineIndex].ltrim(
Blanks);
588 if (SplitBefore.first != StringRef::npos) {
591 "Reflowing whitespace within a token");
594 unsigned WhitespaceOffsetInToken = Content[LineIndex - 1].data() +
595 Content[LineIndex - 1].size() -
597 unsigned WhitespaceLength = TrimmedContent.data() -
599 WhitespaceOffsetInToken;
601 tokenAt(LineIndex), WhitespaceOffsetInToken,
602 WhitespaceLength,
"",
610 Split BreakSplit = SplitBefore;
611 BreakSplit.first += TrimmedContent.data() - Content[LineIndex].data();
612 unsigned ReflownColumn =
613 getReflownColumn(TrimmedContent, LineIndex, PreviousEndColumn);
614 if (ReflownColumn > ColumnLimit) {
615 insertBreak(LineIndex, 0, BreakSplit, Whitespaces);
622 StringRef Prefix = Decoration;
623 if (Content[LineIndex].empty()) {
624 if (LineIndex + 1 ==
Lines.size()) {
625 if (!LastLineNeedsDecoration) {
630 }
else if (!Decoration.empty()) {
633 Prefix = Prefix.substr(0, 1);
638 Prefix = Prefix.substr(0, 1);
643 unsigned WhitespaceOffsetInToken = Content[LineIndex - 1].data() +
644 Content[LineIndex - 1].size() -
646 unsigned WhitespaceLength = Content[LineIndex].data() -
648 WhitespaceOffsetInToken;
650 tokenAt(LineIndex), WhitespaceOffsetInToken, WhitespaceLength,
"", Prefix,
655 llvm::Regex &CommentPragmasRegex)
const {
658 StringRef IndentContent = Content[LineIndex];
659 if (
Lines[LineIndex].ltrim(
Blanks).startswith(
"*")) {
660 IndentContent =
Lines[LineIndex].ltrim(
Blanks).substr(1);
662 return LineIndex > 0 && !CommentPragmasRegex.match(IndentContent) &&
668 BreakableBlockComment::getContentStartColumn(
unsigned LineIndex,
669 unsigned TailOffset)
const {
672 return IndentAtLineBreak;
678 unsigned OriginalStartColumn,
bool FirstInLine,
bool InPPDirective,
681 assert(
Tok.
is(TT_LineComment) &&
682 "line comment section must start with a line comment");
685 CurrentTok && CurrentTok->
is(TT_LineComment);
686 CurrentTok = CurrentTok->Next) {
687 LastLineTok = LineTok;
688 StringRef TokenText(CurrentTok->TokenText);
689 assert(TokenText.startswith(
"//"));
690 size_t FirstLineIndex =
Lines.size();
691 TokenText.split(
Lines,
"\n");
692 Content.resize(
Lines.size());
694 OriginalContentColumn.resize(
Lines.size());
696 Prefix.resize(
Lines.size());
697 OriginalPrefix.resize(
Lines.size());
698 for (
size_t i = FirstLineIndex, e =
Lines.size(); i < e; ++i) {
701 StringRef IndentPrefix =
703 assert(IndentPrefix.startswith(
"//"));
704 OriginalPrefix[i] = Prefix[i] = IndentPrefix;
705 if (
Lines[i].size() > Prefix[i].size() &&
707 if (Prefix[i] ==
"//")
709 else if (Prefix[i] ==
"///")
711 else if (Prefix[i] ==
"//!")
713 else if (Prefix[i] ==
"///<")
715 else if (Prefix[i] ==
"//!<")
720 Content[i] =
Lines[i].substr(IndentPrefix.size());
721 OriginalContentColumn[i] =
735 size_t EndOfLine = Content[i].find_last_not_of(
Blanks);
736 if (EndOfLine == StringRef::npos)
737 EndOfLine = Content[i].size();
740 Content[i] = Content[i].substr(0, EndOfLine);
742 LineTok = CurrentTok->Next;
743 if (CurrentTok->Next && !CurrentTok->Next->ContinuesLineCommentSection) {
764 unsigned LineIndex,
unsigned TailOffset,
765 StringRef::size_type
Length)
const {
766 unsigned ContentStartColumn =
768 : OriginalContentColumn[LineIndex]);
770 Content[LineIndex].substr(TailOffset, Length),
777 StringRef
Text = Content[LineIndex].substr(TailOffset);
780 unsigned BreakOffsetInToken =
782 unsigned CharsToRemove = Split.second;
785 unsigned IndentAtLineBreak = OriginalContentColumn[LineIndex] +
786 Prefix[LineIndex].size() -
787 OriginalPrefix[LineIndex].size();
788 assert(IndentAtLineBreak >= Prefix[LineIndex].size());
790 tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove,
"",
792 IndentAtLineBreak - Prefix[LineIndex].size());
796 unsigned LineIndex,
unsigned PreviousEndColumn,
unsigned ColumnLimit,
797 llvm::Regex &CommentPragmasRegex)
const {
798 if (!
mayReflow(LineIndex, CommentPragmasRegex))
799 return Split(StringRef::npos, 0);
805 unsigned LineIndex,
unsigned TailOffset,
806 unsigned PreviousEndColumn,
807 unsigned ColumnLimit,
808 Split SplitBefore)
const {
809 if (SplitBefore.first == StringRef::npos ||
810 SplitBefore.first + SplitBefore.second < Content[LineIndex].size()) {
815 unsigned StartColumn = PreviousEndColumn +
ReflowPrefix.size();
824 unsigned LineIndex,
unsigned PreviousEndColumn,
unsigned ColumnLimit,
832 if (LineIndex > 0 &&
Tokens[LineIndex] !=
Tokens[LineIndex - 1]) {
833 if (SplitBefore.first != StringRef::npos) {
840 unsigned WhitespaceLength =
856 unsigned LineColumn =
858 (Content[LineIndex].data() -
Lines[LineIndex].data()) +
859 (OriginalPrefix[LineIndex].size() - Prefix[LineIndex].size());
872 if (OriginalPrefix[LineIndex] != Prefix[LineIndex]) {
876 assert(Prefix[LineIndex] == (OriginalPrefix[LineIndex] +
" ").str() &&
877 "Expecting a line comment prefix to differ from original by at most "
880 tokenAt(LineIndex), OriginalPrefix[LineIndex].size(), 0,
"",
"",
886 if (SplitBefore.first != StringRef::npos &&
887 SplitBefore.first + SplitBefore.second < Content[LineIndex].size()) {
888 insertBreak(LineIndex, 0, SplitBefore, Whitespaces);
899 unsigned LineIndex, llvm::Regex &CommentPragmasRegex)
const {
902 StringRef IndentContent = Content[LineIndex];
903 if (
Lines[LineIndex].startswith(
"//")) {
904 IndentContent =
Lines[LineIndex].substr(2);
906 return LineIndex > 0 && !CommentPragmasRegex.match(IndentContent) &&
909 OriginalPrefix[LineIndex] == OriginalPrefix[LineIndex - 1];
913 BreakableLineCommentSection::getContentStartColumn(
unsigned LineIndex,
914 unsigned TailOffset)
const {
915 if (TailOffset != 0) {
916 return OriginalContentColumn[LineIndex];
Declares BreakableToken, BreakableStringLiteral, BreakableComment, BreakableBlockComment and Breakabl...
Token - This structure provides full information about a lexed token.
static LLVM_READONLY bool isAlphanumeric(unsigned char c)
Return true if this character is an ASCII letter or digit: [a-zA-Z0-9].
static LLVM_READONLY bool isPunctuation(unsigned char c)
Return true if this character is an ASCII punctuation character.
char __ovld __cnfn max(char x, char y)
Returns y if x < y, otherwise it returns x.
This file implements an indenter that manages the indentation of continuations.