LLVM  16.0.0git
FileCollector.cpp
Go to the documentation of this file.
1 //===-- FileCollector.cpp ---------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
10 #include "llvm/ADT/SmallString.h"
11 #include "llvm/ADT/Twine.h"
13 #include "llvm/Support/Path.h"
14 #include "llvm/Support/Process.h"
15 
16 using namespace llvm;
17 
20 
21 void FileCollectorBase::addFile(const Twine &File) {
22  std::lock_guard<std::mutex> lock(Mutex);
23  std::string FileStr = File.str();
24  if (markAsSeen(FileStr))
25  addFileImpl(FileStr);
26 }
27 
30  std::error_code EC;
32 }
33 
34 static bool isCaseSensitivePath(StringRef Path) {
35  SmallString<256> TmpDest = Path, UpperDest, RealDest;
36 
37  // Remove component traversals, links, etc.
38  if (sys::fs::real_path(Path, TmpDest))
39  return true; // Current default value in vfs.yaml
40  Path = TmpDest;
41 
42  // Change path to all upper case and ask for its real path, if the latter
43  // exists and is equal to path, it's not case sensitive. Default to case
44  // sensitive in the absence of real_path, since this is the YAMLVFSWriter
45  // default.
46  UpperDest = Path.upper();
47  if (!sys::fs::real_path(UpperDest, RealDest) && Path.equals(RealDest))
48  return false;
49  return true;
50 }
51 
52 FileCollector::FileCollector(std::string Root, std::string OverlayRoot)
53  : Root(Root), OverlayRoot(OverlayRoot) {
54  assert(sys::path::is_absolute(Root) && "Root not absolute");
55  assert(sys::path::is_absolute(OverlayRoot) && "OverlayRoot not absolute");
56 }
57 
58 void FileCollector::PathCanonicalizer::updateWithRealPath(
59  SmallVectorImpl<char> &Path) {
60  StringRef SrcPath(Path.begin(), Path.size());
61  StringRef Filename = sys::path::filename(SrcPath);
62  StringRef Directory = sys::path::parent_path(SrcPath);
63 
64  // Use real_path to fix any symbolic link component present in the directory
65  // part of the path, caching the search because computing the real path is
66  // expensive.
67  SmallString<256> RealPath;
68  auto DirWithSymlink = CachedDirs.find(Directory);
69  if (DirWithSymlink == CachedDirs.end()) {
70  // FIXME: Should this be a call to FileSystem::getRealpath(), in some
71  // cases? What if there is nothing on disk?
72  if (sys::fs::real_path(Directory, RealPath))
73  return;
74  CachedDirs[Directory] = std::string(RealPath.str());
75  } else {
76  RealPath = DirWithSymlink->second;
77  }
78 
79  // Finish recreating the path by appending the original filename, since we
80  // don't need to resolve symlinks in the filename.
81  //
82  // FIXME: If we can cope with this, maybe we can cope without calling
83  // getRealPath() at all when there's no ".." component.
84  sys::path::append(RealPath, Filename);
85 
86  // Swap to create the output.
87  Path.swap(RealPath);
88 }
89 
90 /// Make Path absolute.
91 static void makeAbsolute(SmallVectorImpl<char> &Path) {
92  // We need an absolute src path to append to the root.
94 
95  // Canonicalize src to a native path to avoid mixed separator styles.
96  sys::path::native(Path);
97 
98  // Remove redundant leading "./" pieces and consecutive separators.
99  Path.erase(Path.begin(), sys::path::remove_leading_dotslash(
100  StringRef(Path.begin(), Path.size()))
101  .begin());
102 }
103 
106  PathStorage Paths;
107  Paths.VirtualPath = SrcPath;
108  makeAbsolute(Paths.VirtualPath);
109 
110  // If a ".." component is present after a symlink component, remove_dots may
111  // lead to the wrong real destination path. Let the source be canonicalized
112  // like that but make sure we always use the real path for the destination.
113  Paths.CopyFrom = Paths.VirtualPath;
114  updateWithRealPath(Paths.CopyFrom);
115 
116  // Canonicalize the virtual path by removing "..", "." components.
117  sys::path::remove_dots(Paths.VirtualPath, /*remove_dot_dot=*/true);
118 
119  return Paths;
120 }
121 
124 
125  SmallString<256> DstPath = StringRef(Root);
127 
128  // Always map a canonical src path to its real path into the YAML, by doing
129  // this we map different virtual src paths to the same entry in the VFS
130  // overlay, which is a way to emulate symlink inside the VFS; this is also
131  // needed for correctness, not doing that can lead to module redefinition
132  // errors.
133  addFileToMapping(Paths.VirtualPath, DstPath);
134 }
135 
139  std::error_code &EC) {
140  auto It = FS->dir_begin(Dir, EC);
141  if (EC)
142  return It;
143  addFile(Dir);
144  for (; !EC && It != llvm::vfs::directory_iterator(); It.increment(EC)) {
145  if (It->type() == sys::fs::file_type::regular_file ||
146  It->type() == sys::fs::file_type::directory_file ||
147  It->type() == sys::fs::file_type::symlink_file) {
148  addFile(It->path());
149  }
150  }
151  if (EC)
152  return It;
153  // Return a new iterator.
154  return FS->dir_begin(Dir, EC);
155 }
156 
157 /// Set the access and modification time for the given file from the given
158 /// status object.
159 static std::error_code
161  const sys::fs::file_status &Stat) {
162  int FD;
163 
164  if (auto EC =
166  return EC;
167 
169  FD, Stat.getLastAccessedTime(), Stat.getLastModificationTime()))
170  return EC;
171 
172  if (auto EC = sys::Process::SafelyCloseFileDescriptor(FD))
173  return EC;
174 
175  return {};
176 }
177 
178 std::error_code FileCollector::copyFiles(bool StopOnError) {
179  auto Err = sys::fs::create_directories(Root, /*IgnoreExisting=*/true);
180  if (Err) {
181  return Err;
182  }
183 
184  std::lock_guard<std::mutex> lock(Mutex);
185 
186  for (auto &entry : VFSWriter.getMappings()) {
187  // Get the status of the original file/directory.
189  if (std::error_code EC = sys::fs::status(entry.VPath, Stat)) {
190  if (StopOnError)
191  return EC;
192  continue;
193  }
194 
195  // Continue if the file doesn't exist.
196  if (Stat.type() == sys::fs::file_type::file_not_found)
197  continue;
198 
199  // Create directory tree.
200  if (std::error_code EC =
202  /*IgnoreExisting=*/true)) {
203  if (StopOnError)
204  return EC;
205  }
206 
207  if (Stat.type() == sys::fs::file_type::directory_file) {
208  // Construct a directory when it's just a directory entry.
209  if (std::error_code EC =
211  /*IgnoreExisting=*/true)) {
212  if (StopOnError)
213  return EC;
214  }
215  continue;
216  }
217 
218  // Copy file over.
219  if (std::error_code EC = sys::fs::copy_file(entry.VPath, entry.RPath)) {
220  if (StopOnError)
221  return EC;
222  }
223 
224  // Copy over permissions.
225  if (auto perms = sys::fs::getPermissions(entry.VPath)) {
226  if (std::error_code EC = sys::fs::setPermissions(entry.RPath, *perms)) {
227  if (StopOnError)
228  return EC;
229  }
230  }
231 
232  // Copy over modification time.
234  }
235  return {};
236 }
237 
238 std::error_code FileCollector::writeMapping(StringRef MappingFile) {
239  std::lock_guard<std::mutex> lock(Mutex);
240 
244 
245  std::error_code EC;
246  raw_fd_ostream os(MappingFile, EC, sys::fs::OF_TextWithCRLF);
247  if (EC)
248  return EC;
249 
250  VFSWriter.write(os);
251 
252  return {};
253 }
254 
255 namespace llvm {
256 
258 public:
260  std::shared_ptr<FileCollector> Collector)
261  : FS(std::move(FS)), Collector(std::move(Collector)) {}
262 
264  auto Result = FS->status(Path);
265  if (Result && Result->exists())
266  Collector->addFile(Path);
267  return Result;
268  }
269 
271  openFileForRead(const Twine &Path) override {
272  auto Result = FS->openFileForRead(Path);
273  if (Result && *Result)
274  Collector->addFile(Path);
275  return Result;
276  }
277 
279  std::error_code &EC) override {
280  return Collector->addDirectoryImpl(Dir, FS, EC);
281  }
282 
283  std::error_code getRealPath(const Twine &Path,
284  SmallVectorImpl<char> &Output) const override {
285  auto EC = FS->getRealPath(Path, Output);
286  if (!EC) {
287  Collector->addFile(Path);
288  if (Output.size() > 0)
289  Collector->addFile(Output);
290  }
291  return EC;
292  }
293 
294  std::error_code isLocal(const Twine &Path, bool &Result) override {
295  return FS->isLocal(Path, Result);
296  }
297 
299  return FS->getCurrentWorkingDirectory();
300  }
301 
302  std::error_code setCurrentWorkingDirectory(const llvm::Twine &Path) override {
303  return FS->setCurrentWorkingDirectory(Path);
304  }
305 
306 private:
308  std::shared_ptr<FileCollector> Collector;
309 };
310 
311 } // namespace llvm
312 
315  std::shared_ptr<FileCollector> Collector) {
317 }
llvm::sys::fs::openFileForWrite
std::error_code openFileForWrite(const Twine &Name, int &ResultFD, CreationDisposition Disp=CD_CreateAlways, OpenFlags Flags=OF_None, unsigned Mode=0666)
Opens the file with the given name in a write-only or read-write mode, returning its open file descri...
Definition: FileSystem.h:1065
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
FileSystem.h
llvm::FileCollector::Root
const std::string Root
The directory where collected files are copied to in copyFiles().
Definition: FileCollector.h:132
FileCollector.h
llvm::FileCollectorBase::markAsSeen
bool markAsSeen(StringRef Path)
Definition: FileCollector.h:31
llvm::sys::path::remove_leading_dotslash
StringRef remove_leading_dotslash(StringRef path, Style style=Style::native)
Remove redundant leading "./" pieces and consecutive separators.
Definition: Path.cpp:703
Path.h
llvm::sys::path::is_absolute
bool is_absolute(const Twine &path, Style style=Style::native)
Is path absolute?
Definition: Path.cpp:671
llvm::FileCollector::OverlayRoot
const std::string OverlayRoot
The root directory where the VFS overlay lives.
Definition: FileCollector.h:135
llvm::FileCollector::addFileImpl
void addFileImpl(StringRef SrcPath) override
Definition: FileCollector.cpp:122
isCaseSensitivePath
static bool isCaseSensitivePath(StringRef Path)
Definition: FileCollector.cpp:34
llvm::FileCollector::createCollectorVFS
static IntrusiveRefCntPtr< vfs::FileSystem > createCollectorVFS(IntrusiveRefCntPtr< vfs::FileSystem > BaseFS, std::shared_ptr< FileCollector > Collector)
Create a VFS that uses Collector to collect files accessed via BaseFS.
Definition: FileCollector.cpp:314
llvm::StringMap::end
iterator end()
Definition: StringMap.h:204
llvm::sys::fs::file_type::file_not_found
@ file_not_found
llvm::vfs::directory_iterator
An input iterator over the entries in a virtual path, similar to llvm::sys::fs::directory_iterator.
Definition: VirtualFileSystem.h:182
llvm::sys::fs::file_type::regular_file
@ regular_file
llvm::sys::fs::CD_OpenExisting
@ CD_OpenExisting
CD_OpenExisting - When opening a file:
Definition: FileSystem.h:743
llvm::X86AS::FS
@ FS
Definition: X86.h:200
llvm::sys::path::native
void native(const Twine &path, SmallVectorImpl< char > &result, Style style=Style::native)
Convert path to the native form.
Definition: Path.cpp:540
llvm::FileCollector::PathCanonicalizer::PathStorage
Definition: FileCollector.h:75
makeAbsolute
static void makeAbsolute(SmallVectorImpl< char > &Path)
Make Path absolute.
Definition: FileCollector.cpp:91
llvm::FileCollectorFileSystem::status
llvm::ErrorOr< llvm::vfs::Status > status(const Twine &Path) override
Get the status of the entry at Path, if one exists.
Definition: FileCollector.cpp:263
llvm::StringMap::find
iterator find(StringRef Key)
Definition: StringMap.h:217
llvm::sys::fs::setLastAccessAndModificationTime
std::error_code setLastAccessAndModificationTime(int FD, TimePoint<> AccessTime, TimePoint<> ModificationTime)
Set the file modification and access time.
llvm::FileCollectorFileSystem::isLocal
std::error_code isLocal(const Twine &Path, bool &Result) override
Is the file mounted on a local filesystem?
Definition: FileCollector.cpp:294
copyAccessAndModificationTime
static std::error_code copyAccessAndModificationTime(StringRef Filename, const sys::fs::file_status &Stat)
Set the access and modification time for the given file from the given status object.
Definition: FileCollector.cpp:160
Collector
Register Usage Information Collector
Definition: RegUsageInfoCollector.cpp:73
llvm::sys::path::append
void append(SmallVectorImpl< char > &path, const Twine &a, const Twine &b="", const Twine &c="", const Twine &d="")
Append to path.
Definition: Path.cpp:456
llvm::FileCollectorBase::addFileImpl
virtual void addFileImpl(StringRef SrcPath)=0
Process.h
llvm::sys::fs::file_type::symlink_file
@ symlink_file
SmallString.h
llvm::FileCollector::writeMapping
std::error_code writeMapping(StringRef MappingFile)
Write the yaml mapping (for the VFS) to the given file.
Definition: FileCollector.cpp:238
Twine.h
llvm::sys::fs::real_path
std::error_code real_path(const Twine &path, SmallVectorImpl< char > &output, bool expand_tilde=false)
Collapse all .
llvm::FileCollector::addDirectoryImpl
llvm::vfs::directory_iterator addDirectoryImpl(const llvm::Twine &Dir, IntrusiveRefCntPtr< vfs::FileSystem > FS, std::error_code &EC) override
Definition: FileCollector.cpp:137
llvm::sys::fs::create_directories
std::error_code create_directories(const Twine &path, bool IgnoreExisting=true, perms Perms=owner_all|group_all)
Create all the non-existent directories in path.
Definition: Path.cpp:967
llvm::FileCollectorFileSystem::getRealPath
std::error_code getRealPath(const Twine &Path, SmallVectorImpl< char > &Output) const override
Gets real path of Path e.g.
Definition: FileCollector.cpp:283
llvm::FileCollector::Canonicalizer
PathCanonicalizer Canonicalizer
Helper utility for canonicalizing paths.
Definition: FileCollector.h:141
llvm::FileCollectorFileSystem::FileCollectorFileSystem
FileCollectorFileSystem(IntrusiveRefCntPtr< vfs::FileSystem > FS, std::shared_ptr< FileCollector > Collector)
Definition: FileCollector.cpp:259
llvm::FileCollectorBase::~FileCollectorBase
virtual ~FileCollectorBase()
llvm::pdb::PDB_ColorItem::Path
@ Path
llvm::sys::SmartMutex< false >
llvm::SmallString< 256 >
llvm::sys::fs::OF_TextWithCRLF
@ OF_TextWithCRLF
The file should be opened in text mode and use a carriage linefeed '\r '.
Definition: FileSystem.h:770
llvm::FileCollector::copyFiles
std::error_code copyFiles(bool StopOnError=true)
Copy the files into the root directory.
Definition: FileCollector.cpp:178
llvm::sys::Process::SafelyCloseFileDescriptor
static std::error_code SafelyCloseFileDescriptor(int FD)
llvm::sys::fs::file_status
Represents the result of a call to sys::fs::status().
Definition: FileSystem.h:226
llvm::sys::path::remove_dots
bool remove_dots(SmallVectorImpl< char > &path, bool remove_dot_dot=false, Style style=Style::native)
In-place remove any '.
Definition: Path.cpp:715
llvm::vfs::getRealFileSystem
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
Definition: VirtualFileSystem.cpp:377
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::sys::path::parent_path
StringRef parent_path(StringRef path, Style style=Style::native)
Get parent path.
Definition: Path.cpp:467
llvm::move
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1861
llvm::FileCollectorFileSystem
Definition: FileCollector.cpp:257
llvm::sys::fs::setPermissions
std::error_code setPermissions(const Twine &Path, perms Permissions)
Set file permissions.
llvm::FileCollectorFileSystem::dir_begin
llvm::vfs::directory_iterator dir_begin(const llvm::Twine &Dir, std::error_code &EC) override
Get a directory_iterator for Dir.
Definition: FileCollector.cpp:278
llvm::sys::fs::make_absolute
void make_absolute(const Twine &current_directory, SmallVectorImpl< char > &path)
Make path an absolute path.
Definition: Path.cpp:906
llvm::FileCollectorBase::addFile
void addFile(const Twine &file)
Definition: FileCollector.cpp:21
llvm::FileCollectorFileSystem::setCurrentWorkingDirectory
std::error_code setCurrentWorkingDirectory(const llvm::Twine &Path) override
Set the working directory.
Definition: FileCollector.cpp:302
llvm::FileCollectorFileSystem::getCurrentWorkingDirectory
llvm::ErrorOr< std::string > getCurrentWorkingDirectory() const override
Get the working directory of this file system.
Definition: FileCollector.cpp:298
llvm::vfs::YAMLVFSWriter::setCaseSensitivity
void setCaseSensitivity(bool CaseSensitive)
Definition: VirtualFileSystem.h:1031
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::sys::fs::getPermissions
ErrorOr< perms > getPermissions(const Twine &Path)
Get file permissions.
Definition: Path.cpp:1151
llvm::vfs::YAMLVFSWriter::write
void write(llvm::raw_ostream &OS)
Definition: VirtualFileSystem.cpp:2750
llvm::raw_fd_ostream
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:441
llvm::vfs::YAMLVFSWriter::setOverlayDir
void setOverlayDir(StringRef OverlayDirectory)
Definition: VirtualFileSystem.h:1037
llvm::FileCollector::PathCanonicalizer::PathStorage::VirtualPath
SmallString< 256 > VirtualPath
Definition: FileCollector.h:77
llvm::vfs::YAMLVFSWriter::setUseExternalNames
void setUseExternalNames(bool UseExtNames)
Definition: VirtualFileSystem.h:1035
llvm::sys::fs::file_type::directory_file
@ directory_file
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
llvm::vfs::FileSystem
The virtual file system interface.
Definition: VirtualFileSystem.h:268
std
Definition: BitVector.h:851
llvm::FileCollectorBase::FileCollectorBase
FileCollectorBase()
llvm::sys::fs::is_directory
bool is_directory(const basic_file_status &status)
Does status represent a directory?
Definition: Path.cpp:1092
llvm::FileCollectorBase::addDirectoryImpl
virtual llvm::vfs::directory_iterator addDirectoryImpl(const llvm::Twine &Dir, IntrusiveRefCntPtr< vfs::FileSystem > FS, std::error_code &EC)=0
llvm::FileCollector::PathCanonicalizer::PathStorage::CopyFrom
SmallString< 256 > CopyFrom
Definition: FileCollector.h:76
llvm::SmallString::str
StringRef str() const
Explicit conversion to StringRef.
Definition: SmallString.h:260
llvm::sys::fs::perms
perms
Definition: FileSystem.h:86
llvm::sys::path::filename
StringRef filename(StringRef path, Style style=Style::native)
Get filename.
Definition: Path.cpp:577
llvm::FileCollector::PathCanonicalizer::canonicalize
PathStorage canonicalize(StringRef SrcPath)
Canonicalize a pair of virtual and real paths.
Definition: FileCollector.cpp:105
llvm::vfs::YAMLVFSWriter::getMappings
const std::vector< YAMLVFSEntry > & getMappings() const
Definition: VirtualFileSystem.h:1042
llvm::sys::fs::status
std::error_code status(const Twine &path, file_status &result, bool follow=true)
Get file status as if by POSIX stat().
llvm::ErrorOr
Represents either an error or a value T.
Definition: ErrorOr.h:56
llvm::FileCollectorFileSystem::openFileForRead
llvm::ErrorOr< std::unique_ptr< llvm::vfs::File > > openFileForRead(const Twine &Path) override
Get a File object for the file at Path, if one exists.
Definition: FileCollector.cpp:271
llvm::SmallVectorImpl< char >
llvm::FileCollector::FileCollector
FileCollector(std::string Root, std::string OverlayRoot)
Root is the directory where collected files are will be stored.
Definition: FileCollector.cpp:52
llvm::sys::path::relative_path
StringRef relative_path(StringRef path, Style style=Style::native)
Get relative path.
Definition: Path.cpp:413
llvm::FileCollectorBase::addDirectory
void addDirectory(const Twine &Dir)
Definition: FileCollector.cpp:28
llvm::StringRef::begin
iterator begin() const
Definition: StringRef.h:111
llvm::FileCollector::VFSWriter
vfs::YAMLVFSWriter VFSWriter
The yaml mapping writer.
Definition: FileCollector.h:138
llvm::IntrusiveRefCntPtr
A smart pointer to a reference-counted object that inherits from RefCountedBase or ThreadSafeRefCount...
Definition: IntrusiveRefCntPtr.h:168
entry
print Instructions which execute on loop entry
Definition: MustExecute.cpp:346
llvm::sys::fs::copy_file
std::error_code copy_file(const Twine &From, const Twine &To)
Copy the contents of From to To.
Definition: Path.cpp:1016