LLVM  12.0.0git
TarWriter.cpp
Go to the documentation of this file.
1 //===-- TarWriter.cpp - Tar archive file creator --------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // TarWriter class provides a feature to create a tar archive file.
10 //
11 // I put emphasis on simplicity over comprehensiveness when implementing this
12 // class because we don't need a full-fledged archive file generator in LLVM
13 // at the moment.
14 //
15 // The filename field in the Unix V7 tar header is 100 bytes. Longer filenames
16 // are stored using the PAX extension. The PAX header is standardized in
17 // POSIX.1-2001.
18 //
19 // The struct definition of UstarHeader is copied from
20 // https://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5
21 //
22 //===----------------------------------------------------------------------===//
23 
24 #include "llvm/Support/TarWriter.h"
25 #include "llvm/ADT/StringRef.h"
28 #include "llvm/Support/Path.h"
29 
30 using namespace llvm;
31 
32 // Each file in an archive must be aligned to this block size.
33 static const int BlockSize = 512;
34 
35 struct UstarHeader {
36  char Name[100];
37  char Mode[8];
38  char Uid[8];
39  char Gid[8];
40  char Size[12];
41  char Mtime[12];
42  char Checksum[8];
43  char TypeFlag;
44  char Linkname[100];
45  char Magic[6];
46  char Version[2];
47  char Uname[32];
48  char Gname[32];
49  char DevMajor[8];
50  char DevMinor[8];
51  char Prefix[155];
52  char Pad[12];
53 };
54 static_assert(sizeof(UstarHeader) == BlockSize, "invalid Ustar header");
55 
57  UstarHeader Hdr = {};
58  memcpy(Hdr.Magic, "ustar", 5); // Ustar magic
59  memcpy(Hdr.Version, "00", 2); // Ustar version
60  return Hdr;
61 }
62 
63 // A PAX attribute is in the form of "<length> <key>=<value>\n"
64 // where <length> is the length of the entire string including
65 // the length field itself. An example string is this.
66 //
67 // 25 ctime=1084839148.1212\n
68 //
69 // This function create such string.
70 static std::string formatPax(StringRef Key, StringRef Val) {
71  int Len = Key.size() + Val.size() + 3; // +3 for " ", "=" and "\n"
72 
73  // We need to compute total size twice because appending
74  // a length field could change total size by one.
75  int Total = Len + Twine(Len).str().size();
76  Total = Len + Twine(Total).str().size();
77  return (Twine(Total) + " " + Key + "=" + Val + "\n").str();
78 }
79 
80 // Headers in tar files must be aligned to 512 byte boundaries.
81 // This function forwards the current file position to the next boundary.
82 static void pad(raw_fd_ostream &OS) {
83  uint64_t Pos = OS.tell();
84  OS.seek(alignTo(Pos, BlockSize));
85 }
86 
87 // Computes a checksum for a tar header.
88 static void computeChecksum(UstarHeader &Hdr) {
89  // Before computing a checksum, checksum field must be
90  // filled with space characters.
91  memset(Hdr.Checksum, ' ', sizeof(Hdr.Checksum));
92 
93  // Compute a checksum and set it to the checksum field.
94  unsigned Chksum = 0;
95  for (size_t I = 0; I < sizeof(Hdr); ++I)
96  Chksum += reinterpret_cast<uint8_t *>(&Hdr)[I];
97  snprintf(Hdr.Checksum, sizeof(Hdr.Checksum), "%06o", Chksum);
98 }
99 
100 // Create a tar header and write it to a given output stream.
101 static void writePaxHeader(raw_fd_ostream &OS, StringRef Path) {
102  // A PAX header consists of a 512-byte header followed
103  // by key-value strings. First, create key-value strings.
104  std::string PaxAttr = formatPax("path", Path);
105 
106  // Create a 512-byte header.
108  snprintf(Hdr.Size, sizeof(Hdr.Size), "%011zo", PaxAttr.size());
109  Hdr.TypeFlag = 'x'; // PAX magic
110  computeChecksum(Hdr);
111 
112  // Write them down.
113  OS << StringRef(reinterpret_cast<char *>(&Hdr), sizeof(Hdr));
114  OS << PaxAttr;
115  pad(OS);
116 }
117 
118 // Path fits in a Ustar header if
119 //
120 // - Path is less than 100 characters long, or
121 // - Path is in the form of "<prefix>/<name>" where <prefix> is less
122 // than or equal to 155 characters long and <name> is less than 100
123 // characters long. Both <prefix> and <name> can contain extra '/'.
124 //
125 // If Path fits in a Ustar header, updates Prefix and Name and returns true.
126 // Otherwise, returns false.
128  if (Path.size() < sizeof(UstarHeader::Name)) {
129  Prefix = "";
130  Name = Path;
131  return true;
132  }
133 
134  // tar 1.13 and earlier unconditionally look at the tar header interpreted
135  // as an 'oldgnu_header', which has an 'isextended' byte at offset 482 in the
136  // header, corresponding to offset 137 in the prefix. That's the version of
137  // tar in gnuwin, so only use 137 of the 155 bytes in the prefix. This means
138  // we'll need a pax header after 237 bytes of path instead of after 255,
139  // but in return paths up to 237 bytes work with gnuwin, instead of just
140  // 137 bytes of directory + 100 bytes of basename previously.
141  // (tar-1.13 also doesn't support pax headers, but in practice all paths in
142  // llvm's test suite are short enough for that to not matter.)
143  const int MaxPrefix = 137;
144  size_t Sep = Path.rfind('/', MaxPrefix + 1);
145  if (Sep == StringRef::npos)
146  return false;
147  if (Path.size() - Sep - 1 >= sizeof(UstarHeader::Name))
148  return false;
149 
150  Prefix = Path.substr(0, Sep);
151  Name = Path.substr(Sep + 1);
152  return true;
153 }
154 
155 // The PAX header is an extended format, so a PAX header needs
156 // to be followed by a "real" header.
158  StringRef Name, size_t Size) {
160  memcpy(Hdr.Name, Name.data(), Name.size());
161  memcpy(Hdr.Mode, "0000664", 8);
162  snprintf(Hdr.Size, sizeof(Hdr.Size), "%011zo", Size);
163  memcpy(Hdr.Prefix, Prefix.data(), Prefix.size());
164  computeChecksum(Hdr);
165  OS << StringRef(reinterpret_cast<char *>(&Hdr), sizeof(Hdr));
166 }
167 
168 // Creates a TarWriter instance and returns it.
170  StringRef BaseDir) {
171  using namespace sys::fs;
172  int FD;
173  if (std::error_code EC =
174  openFileForWrite(OutputPath, FD, CD_CreateAlways, OF_None))
175  return make_error<StringError>("cannot open " + OutputPath, EC);
176  return std::unique_ptr<TarWriter>(new TarWriter(FD, BaseDir));
177 }
178 
179 TarWriter::TarWriter(int FD, StringRef BaseDir)
180  : OS(FD, /*shouldClose=*/true, /*unbuffered=*/false),
181  BaseDir(std::string(BaseDir)) {}
182 
183 // Append a given file to an archive.
185  // Write Path and Data.
186  std::string Fullpath = BaseDir + "/" + sys::path::convert_to_slash(Path);
187 
188  // We do not want to include the same file more than once.
189  if (!Files.insert(Fullpath).second)
190  return;
191 
193  StringRef Name;
194  if (splitUstar(Fullpath, Prefix, Name)) {
195  writeUstarHeader(OS, Prefix, Name, Data.size());
196  } else {
197  writePaxHeader(OS, Fullpath);
198  writeUstarHeader(OS, "", "", Data.size());
199  }
200 
201  OS << Data;
202  pad(OS);
203 
204  // POSIX requires tar archives end with two null blocks.
205  // Here, we write the terminator and then seek back, so that
206  // the file being output is terminated correctly at any moment.
207  uint64_t Pos = OS.tell();
208  OS << std::string(BlockSize * 2, '\0');
209  OS.seek(Pos);
210  OS.flush();
211 }
char Version[2]
Definition: TarWriter.cpp:46
char Magic[6]
Definition: TarWriter.cpp:45
SI Whole Quad Mode
This class represents lattice values for constants.
Definition: AllocatorList.h:23
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
char TypeFlag
Definition: TarWriter.cpp:43
LLVM_NODISCARD size_t rfind(char C, size_t From=npos) const
Search for the last character C in the string.
Definition: StringRef.h:378
uint64_t seek(uint64_t off)
Flushes the stream and repositions the underlying file descriptor position to the offset specified fr...
std::string convert_to_slash(StringRef path, Style style=Style::native)
Replaces backslashes with slashes if Windows.
Definition: Path.cpp:568
char Name[100]
Definition: TarWriter.cpp:36
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
char Mode[8]
Definition: TarWriter.cpp:37
Tagged union holding either a T or a Error.
Definition: APFloat.h:42
LLVM_NODISCARD StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:612
static constexpr size_t npos
Definition: StringRef.h:59
Key
PAL metadata keys.
char Size[12]
Definition: TarWriter.cpp:40
static bool splitUstar(StringRef Path, StringRef &Prefix, StringRef &Name)
Definition: TarWriter.cpp:127
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:160
static void computeChecksum(UstarHeader &Hdr)
Definition: TarWriter.cpp:88
static void pad(raw_fd_ostream &OS)
Definition: TarWriter.cpp:82
char Checksum[8]
Definition: TarWriter.cpp:42
static UstarHeader makeUstarHeader()
Definition: TarWriter.cpp:56
static void writePaxHeader(raw_fd_ostream &OS, StringRef Path)
Definition: TarWriter.cpp:101
static Expected< std::unique_ptr< TarWriter > > create(StringRef OutputPath, StringRef BaseDir)
Definition: TarWriter.cpp:169
static std::string formatPax(StringRef Key, StringRef Val)
Definition: TarWriter.cpp:70
std::error_code openFileForWrite(const Twine &Name, int &ResultFD, CreationDisposition Disp=CD_CreateAlways, OpenFlags Flags=OF_None, unsigned Mode=0666)
Opens the file with the given name in a write-only or read-write mode, returning its open file descri...
Definition: FileSystem.h:1032
static const char *const Magic
Definition: Archive.cpp:41
static const int BlockSize
Definition: TarWriter.cpp:33
static void writeUstarHeader(raw_fd_ostream &OS, StringRef Prefix, StringRef Name, size_t Size)
Definition: TarWriter.cpp:157
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:158
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:408
#define I(x, y, z)
Definition: MD5.cpp:59
uint32_t Size
Definition: Profile.cpp:46
std::string str() const
Return the twine contents as a std::string.
Definition: Twine.cpp:17
LLVM_NODISCARD const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:152
CD_CreateAlways - When opening a file:
Definition: FileSystem.h:738
uint64_t tell() const
tell - Return the current offset with the file.
Definition: raw_ostream.h:118
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:57
void append(StringRef Path, StringRef Data)
Definition: TarWriter.cpp:184
char Prefix[155]
Definition: TarWriter.cpp:51
const uint64_t Version
Definition: InstrProf.h:996