files.cpp

/*
 * Copyright (C) 2017 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "cuttlefish/common/libs/utils/files.h"

#ifdef __linux__
#include <linux/fiemap.h>
#include <linux/fs.h>
#include <sys/sendfile.h>
#endif

#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <ftw.h>
#include <libgen.h>
#include <sched.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/select.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/uio.h>
#include <unistd.h>

#include <array>
#include <chrono>
#include <fstream>
#include <ios>
#include <iosfwd>
#include <memory>
#include <string>
#include <string_view>
#include <vector>

#include <android-base/file.h>
#include <android-base/macros.h>
#include "absl/strings/str_join.h"
#include "absl/strings/str_split.h"
#include "absl/log/check.h"
#include "absl/log/log.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_join.h"
#include "fmt/format.h"

#include "cuttlefish/common/libs/fs/shared_buf.h"
#include "cuttlefish/common/libs/fs/shared_fd.h"
#include "cuttlefish/common/libs/utils/environment.h"
#include "cuttlefish/common/libs/utils/in_sandbox.h"
#include "cuttlefish/common/libs/utils/users.h"
#include "cuttlefish/posix/rename.h"
#include "cuttlefish/posix/strerror.h"
#include "cuttlefish/result/result.h"

#ifdef __APPLE__
#define off64_t off_t
#define ftruncate64 ftruncate
#endif

namespace cuttlefish {

bool FileExists(const std::string& path, bool follow_symlinks) {
  struct stat st {};
  return (follow_symlinks ? stat : lstat)(path.c_str(), &st) == 0;
}

Result<dev_t> FileDeviceId(const std::string& path) {
  struct stat out;
  CF_EXPECTF(
      stat(path.c_str(), &out) == 0,
      "stat() failed trying to retrieve device ID information for \"{}\" "
      "with error: {}",
      path, strerror(errno));
  return out.st_dev;
}

Result<bool> CanHardLink(const std::string& source,
                         const std::string& destination) {
  return CF_EXPECT(FileDeviceId(source)) ==
         CF_EXPECT(FileDeviceId(destination));
}

Result<ino_t> FileInodeNumber(const std::string& path) {
  struct stat out;
  CF_EXPECTF(
      stat(path.c_str(), &out) == 0,
      "stat() failed trying to retrieve inode num information for \"{}\" "
      "with error: {}",
      path, strerror(errno));
  return out.st_ino;
}

Result<bool> AreHardLinked(const std::string& source,
                           const std::string& destination) {
  return (CF_EXPECT(FileDeviceId(source)) ==
          CF_EXPECT(FileDeviceId(destination))) &&
         (CF_EXPECT(FileInodeNumber(source)) ==
          CF_EXPECT(FileInodeNumber(destination)));
}

Result<bool> AreFilesIdentical(const std::string& path1,
                               const std::string& path2) {
  struct stat st1;
  struct stat st2;
  CF_EXPECTF(stat(path1.c_str(), &st1) == 0, "stat failed for {}", path1);
  CF_EXPECTF(stat(path2.c_str(), &st2) == 0, "stat failed for {}", path2);
  if (st1.st_size != st2.st_size) {
    return false;
  }

  SharedFD fd1 = SharedFD::Open(path1, O_RDONLY);
  SharedFD fd2 = SharedFD::Open(path2, O_RDONLY);
  CF_EXPECTF(fd1->IsOpen(), "Failed to open \"{}\"", path1);
  CF_EXPECTF(fd2->IsOpen(), "Failed to open \"{}\"", path2);

  char buf1[4096];
  char buf2[4096];
  while (true) {
    auto r1 = fd1->Read(buf1, sizeof(buf1));
    auto r2 = fd2->Read(buf2, sizeof(buf2));
    CF_EXPECTF(r1 >= 0, "Read failed for \"{}\"", path1);
    CF_EXPECTF(r2 >= 0, "Read failed for \"{}\"", path2);
    if (r1 != r2) {
      return false;
    }
    if (r1 == 0) {
      break;
    }
    if (memcmp(buf1, buf2, r1) != 0) {
      return false;
    }
  }
  return true;
}

Result<std::string> LinkOrCopy(const std::string& target,
                               const std::string& destination,
                               const bool overwrite_existing) {
  if (FileExists(destination)) {
    if (CF_EXPECT(AreHardLinked(target, destination))) {
      return destination;
    }
    if (!overwrite_existing) {
      if (CF_EXPECT(AreFilesIdentical(target, destination))) {
        return destination;
      }
      return CF_ERRF(
          "Cannot link/copy from \"{}\" to \"{}\", the second file already "
          "exists and is different from the first",
          target, destination);
    }
    LOG(WARNING) << "Overwriting existing file \"" << destination
                 << "\" with \"" << target << "\" from the cache";
    CF_EXPECTF(unlink(destination.c_str()) == 0,
               "Failed to unlink \"{}\" with error: {}", destination,
               strerror(errno));
  }
  if (link(target.c_str(), destination.c_str()) == 0) {
    VLOG(1) << "Created hard link from \"" << target << "\" to \""
            << destination << "\"";
    return destination;
  }
  CF_EXPECTF(Copy(target, destination), "Failed to copy \"{}\" to \"{}\"",
             target, destination);
  VLOG(1) << "Copied file from \"" << target << "\" to \"" << destination
          << "\"";

  return destination;
}

bool FileHasContent(const std::string& path) {
  return FileSize(path) > 0;
}

Result<void> LinkOrCopyDirectoryContentsRecursively(
    const std::string& source, const std::string& destination) {
  CF_EXPECTF(IsDirectory(source), "Source '{}' is not a directory", source);

  CF_EXPECT(EnsureDirectoryExists(destination, 0755));

  auto linker_or_copier =
      [&source,
       &destination](const std::string& filepath) mutable -> Result<void> {
    const std::string src_path = filepath;
    const std::string dst_path =
        destination + "/" + filepath.substr(source.size() + 1);
    if (IsDirectory(src_path)) {
      CF_EXPECT(EnsureDirectoryExists(dst_path));
      return {};
    }
    const bool overwrite_existing = true;
    CF_EXPECT(LinkOrCopy(src_path, dst_path, overwrite_existing));
    return {};
  };
  CF_EXPECT(WalkDirectory(source, linker_or_copier));

  return {};
}

Result<void> MoveDirectoryContents(const std::string& source,
                                   const std::string& destination) {
  CF_EXPECTF(IsDirectory(source), "Source '{}' is not a directory", source);
  CF_EXPECT(EnsureDirectoryExists(destination));

  bool should_rename = CF_EXPECT(CanRename(source, destination));
  std::vector<std::string> contents = CF_EXPECT(DirectoryContents(source));
  for (const std::string& filepath : contents) {
    std::string src_filepath = source + "/" + filepath;
    std::string dst_filepath = destination + "/" + filepath;
    if (should_rename) {
      CF_EXPECT(Rename(src_filepath, dst_filepath));
    } else {
      CF_EXPECT(
          Copy(src_filepath, dst_filepath),
          "copy " << src_filepath << " to " << dst_filepath << " failed.");
    }
  }

  return {};
}

Result<std::vector<std::string>> DirectoryContents(const std::string& path) {
  std::vector<std::string> ret;
  std::unique_ptr<DIR, int(*)(DIR*)> dir(opendir(path.c_str()), closedir);
  CF_EXPECTF(dir != nullptr, "Could not read from dir \"{}\"", path);
  struct dirent* ent{};
  while ((ent = readdir(dir.get()))) {
    if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) {
      continue;
    }
    ret.emplace_back(ent->d_name);
  }
  return ret;
}

Result<std::vector<std::string>> DirectoryContentsPaths(
    const std::string& path) {
  std::vector<std::string> result = CF_EXPECT(DirectoryContents(path));
  for (std::string& filename : result) {
    filename = fmt::format("{}/{}", path, filename);
  }
  return result;
}

bool DirectoryExists(const std::string& path, bool follow_symlinks) {
  struct stat st {};
  if ((follow_symlinks ? stat : lstat)(path.c_str(), &st) == -1) {
    return false;
  }
  if ((st.st_mode & S_IFMT) != S_IFDIR) {
    return false;
  }
  return true;
}

Result<void> EnsureDirectoryExists(const std::string& directory_path,
                                   const mode_t mode,
                                   const std::string& group_name) {
  if (DirectoryExists(directory_path, /* follow_symlinks */ true)) {
    return {};
  }
  if (FileExists(directory_path, false) && !FileExists(directory_path, true)) {
    // directory_path is a link to a path that doesn't exist. This could happen
    // after executing certain cvd subcommands.
    CF_EXPECT(RemoveFile(directory_path),
              "Can't remove broken link: " << directory_path);
  }
  const auto parent_dir = android::base::Dirname(directory_path);
  if (parent_dir.size() > 1) {
    CF_EXPECT(EnsureDirectoryExists(parent_dir, mode, group_name));
  }
  VLOG(1) << "Setting up " << directory_path;
  if (mkdir(directory_path.c_str(), mode) < 0 && errno != EEXIST) {
    return CF_ERRNO("Failed to create directory: \"" << directory_path << "\""
                                                     << strerror(errno));
  }
  // TODO(schuffelen): Find an alternative for host-sandboxing mode
  if (InSandbox()) {
    return {};
  }

  CF_EXPECTF(chmod(directory_path.c_str(), mode) == 0,
             "Failed to set permission on {}: {}", directory_path,
             strerror(errno));

  if (!group_name.empty()) {
    CF_EXPECT(ChangeGroup(directory_path, group_name));
  }

  return {};
}

Result<void> ChangeGroup(const std::string& path,
                         const std::string& group_name) {
  auto groupId = GroupIdFromName(group_name);

  if (groupId == -1) {
    return CF_ERR("Failed to get group id: ") << group_name;
  }

  if (chown(path.c_str(), -1, groupId) != 0) {
    return CF_ERRNO("Failed to set group for path: "
                    << path << ", " << group_name << ", " << strerror(errno));
  }

  return {};
}

bool CanAccess(const std::string& path, const int mode) {
  return access(path.c_str(), mode) == 0;
}

Result<bool> IsDirectoryEmpty(const std::string& path) {
  std::unique_ptr<DIR, int (*)(DIR*)> direc(opendir(path.c_str()), closedir);
  CF_EXPECTF(direc.get(), "opendir('{}') failed: {}", path, StrError(errno));

  int cnt = 0;
  while (::readdir(direc.get())) {
    cnt++;
    if (cnt > 2) {
      return false;
    }
  }
  return true;
}

Result<void> RecursivelyRemoveDirectory(const std::string& path) {
  // Copied from libbase TemporaryDir destructor.
  auto callback = [](const char* child, const struct stat*, int file_type,
                     struct FTW*) -> int {
    switch (file_type) {
      case FTW_D:
      case FTW_DP:
      case FTW_DNR:
        if (rmdir(child) == -1) {
          PLOG(ERROR) << "rmdir " << child;
          return -1;
        }
        break;
      case FTW_NS:
      default:
        if (rmdir(child) != -1) {
          break;
        }
        // FALLTHRU (for gcc, lint, pcc, etc; and following for clang)
        FALLTHROUGH_INTENDED;
      case FTW_F:
      case FTW_SL:
      case FTW_SLN:
        if (unlink(child) == -1) {
          PLOG(ERROR) << "unlink " << child;
          return -1;
        }
        break;
    }
    return 0;
  };

  if (nftw(path.c_str(), callback, 128, FTW_DEPTH | FTW_PHYS) < 0) {
    return CF_ERRNO("Failed to remove directory \""
                    << path << "\": " << strerror(errno));
  }
  return {};
}

bool Copy(const std::string& from, const std::string& to) {
  SharedFD fd_from = SharedFD::Open(from, O_RDONLY);
  SharedFD fd_to = SharedFD::Open(to, O_WRONLY | O_CREAT | O_TRUNC, 0644);

  if (!fd_from->IsOpen() || !fd_to->IsOpen()) {
    return false;
  }

  off_t farthest_seek = fd_from->LSeek(0, SEEK_END);
  if (farthest_seek == -1) {
    LOG(ERROR) << "Could not lseek in \"" << from
               << "\": " << fd_from->StrError();
    return false;
  }
  if (fd_to->Truncate(farthest_seek) < 0) {
    LOG(ERROR) << "Failed to truncate " << to << ": " << fd_to->StrError();
  }
  off_t offset = 0;
  while (offset < farthest_seek) {
    off_t new_offset = fd_from->LSeek(offset, SEEK_HOLE);
    if (new_offset == -1) {
      if (fd_from->GetErrno() == ENXIO) {
        return true;
      }
      LOG(ERROR) << "Could not lseek in \"" << from
                 << "\": " << fd_from->StrError();
      return false;
    }
    auto data_bytes = new_offset - offset;
    if (fd_to->LSeek(offset, SEEK_SET) < 0) {
      LOG(ERROR) << "lseek() on " << to << " failed: " << fd_to->StrError();
      return false;
    }
    if (!fd_to->SendFile(*fd_from, &offset, data_bytes)) {
      LOG(ERROR) << "SendFile failed: " << fd_to->StrError();
      return false;
    }
    CHECK_EQ(offset, new_offset);

    if (offset >= farthest_seek) {
      return true;
    }
    new_offset = fd_from->LSeek(offset, SEEK_DATA);
    if (new_offset == -1) {
      if (fd_from->GetErrno() == ENXIO) {
        return true;
      }
      LOG(ERROR) << "Could not lseek in \"" << from
                 << "\": " << fd_from->StrError();
      return false;
    }
    offset = new_offset;
  }
  return true;
}

std::string AbsolutePath(std::string_view path) {
  if (path.empty()) {
    return {};
  }
  if (path[0] == '/') {
    return std::string(path);
  }

  Result<std::string> real_cwd = RealPath(".");
  if (!real_cwd.ok()) {
    LOG(WARNING) << "Could not get real path for current directory \".\": "
                 << real_cwd.error();
    return {};
  }
  return absl::StrCat(*real_cwd, "/", path);
}

Result<std::string> RealPath(const std::string& path) {
  std::array<char, PATH_MAX> buffer{};
  char* res;
  do {
    res = realpath(path.c_str(), buffer.data());
  } while (res == nullptr && errno == EINTR);
  CF_EXPECTF(res != nullptr, "Could not get real path for path \"{}\": {}",
             path, StrError(errno));
  return std::string(buffer.data());
}

off_t FileSize(const std::string& path) {
  struct stat st {};
  if (stat(path.c_str(), &st) == -1) {
    return 0;
  }
  return st.st_size;
}

Result<uid_t> FileOwner(const std::string& path) {
  struct stat st{};
  if (stat(path.c_str(), &st) == -1) {
    return CF_ERRF("Failed to stat file '{}' : {}", path, StrError(errno));
  }
  return st.st_uid;
}

bool MakeFileExecutable(const std::string& path) {
  VLOG(0) << "Making " << path << " executable";
  return chmod(path.c_str(), S_IRWXU) == 0;
}

Result<std::chrono::system_clock::time_point> FileModificationTime(
    const std::string& path) {
  struct stat st;
  CF_EXPECTF(stat(path.c_str(), &st) == 0,
             "stat() failed retrieving file modification time on \"{}\" with "
             "error: {}",
             path, strerror(errno));
#ifdef __linux__
  std::chrono::seconds seconds(st.st_mtim.tv_sec);
#elif defined(__APPLE__)
  std::chrono::seconds seconds(st.st_mtimespec.tv_sec);
#else
#error "Unsupported operating system"
#endif
  return std::chrono::system_clock::time_point(seconds);
}

Result<std::string> RenameFile(const std::string& current_filepath,
                               const std::string& target_filepath) {
  if (current_filepath != target_filepath) {
    CF_EXPECT(Rename(current_filepath, target_filepath));
  }
  return target_filepath;
}

Result<void> RemoveFile(const std::string& file) {
  VLOG(0) << "Removing file " << file;
  if (remove(file.c_str()) != 0) {
    return CF_ERRF("Failed to remove file '{}' : {}", file,
                   StrError(errno));
  }
  return {};
}

std::string ReadFile(const std::string& file) {
  std::string contents;
  std::ifstream in(file, std::ios::in | std::ios::binary);
  in.seekg(0, std::ios::end);
  if (in.fail()) {
    // TODO(schuffelen): Return a failing Result instead
    return "";
  }
  if (in.tellg() == std::ifstream::pos_type(-1)) {
    PLOG(ERROR) << "Failed to seek on " << file;
    return "";
  }
  contents.resize(in.tellg());
  in.seekg(0, std::ios::beg);
  if (!contents.empty()) {
    in.read(&contents[0], contents.size());
  }
  in.close();
  return(contents);
}

Result<std::string> ReadFileContents(const std::string& filepath) {
  CF_EXPECTF(FileExists(filepath), "The file at \"{}\" does not exist.",
             filepath);
  auto file = SharedFD::Open(filepath, O_RDONLY);
  CF_EXPECTF(file->IsOpen(), "Failed to open file \"{}\".  Error: {}\n",
             filepath, file->StrError());
  std::string file_content;
  auto size = ReadAll(file, &file_content);
  CF_EXPECTF(size >= 0, "Failed to read file contents.  Error: {}\n",
             file->StrError());
  return file_content;
}
Result<void> WriteNewFile(const std::string& filepath, std::string_view content,
                          mode_t mode) {
  CF_EXPECTF(!FileExists(filepath), "File already exists: {}", filepath);
  SharedFD file_fd = SharedFD::Open(filepath, O_CREAT | O_WRONLY, mode);
  CF_EXPECTF(file_fd->IsOpen(), "Failed to open file \"{}\" for writing: {}",
             filepath, file_fd->StrError());
  const auto written_size = WriteAll(file_fd, content);
  CF_EXPECTF(written_size == content.size(),
             "Failed to write all content to file. Error:\n",
             file_fd->StrError());
  return {};
}

std::string CurrentDirectory() {
  std::vector<char> process_wd(1 << 12, ' ');
  while (getcwd(process_wd.data(), process_wd.size()) == nullptr) {
    if (errno == ERANGE) {
      process_wd.resize(process_wd.size() * 2, ' ');
    } else {
      PLOG(ERROR) << "getcwd failed";
      return "";
    }
  }
  // Will find the null terminator and size the string appropriately.
  return std::string(process_wd.data());
}

FileSizes SparseFileSizes(const std::string& path) {
  auto fd = SharedFD::Open(path, O_RDONLY);
  if (!fd->IsOpen()) {
    LOG(ERROR) << "Could not open \"" << path << "\": " << fd->StrError();
    return {};
  }
  off_t farthest_seek = fd->LSeek(0, SEEK_END);
  VLOG(1) << "Farthest seek: " << farthest_seek;
  if (farthest_seek == -1) {
    LOG(ERROR) << "Could not lseek in \"" << path << "\": " << fd->StrError();
    return {};
  }
  off_t data_bytes = 0;
  off_t offset = 0;
  while (offset < farthest_seek) {
    off_t new_offset = fd->LSeek(offset, SEEK_HOLE);
    if (new_offset == -1) {
      // ENXIO is returned when there are no more blocks of this type coming.
      if (fd->GetErrno() == ENXIO) {
        break;
      } else {
        LOG(ERROR) << "Could not lseek in \"" << path << "\": " << fd->StrError();
        return {};
      }
    } else {
      data_bytes += new_offset - offset;
      offset = new_offset;
    }
    if (offset >= farthest_seek) {
      break;
    }
    new_offset = fd->LSeek(offset, SEEK_DATA);
    if (new_offset == -1) {
      // ENXIO is returned when there are no more blocks of this type coming.
      if (fd->GetErrno() == ENXIO) {
        break;
      } else {
        LOG(ERROR) << "Could not lseek in \"" << path << "\": " << fd->StrError();
        return {};
      }
    } else {
      offset = new_offset;
    }
  }
  return (FileSizes) { .sparse_size = farthest_seek, .disk_size = data_bytes };
}

bool FileIsSocket(const std::string& path) {
  struct stat st {};
  return stat(path.c_str(), &st) == 0 && S_ISSOCK(st.st_mode);
}

Result<std::string> FindFile(const std::string& path,
                             const std::string& target_name) {
  std::string ret;
  auto callback = [&ret, &target_name](
                      const std::string& filename) mutable -> Result<void> {
    if (android::base::Basename(filename) == target_name) {
      ret = filename;
    }
    return {};
  };
  CF_EXPECT(WalkDirectory(path, callback));
  CF_EXPECTF(!ret.empty(), "No file matching '{}' found in '{}'", target_name,
             path);
  return ret;
}

// Recursively enumerate files in |dir|, and invoke the callback function with
// path to each file/directory.
Result<void> WalkDirectory(const std::string& dir,
                           const WalkDirectoryCallback& callback) {
  for (const std::string& filename : CF_EXPECT(DirectoryContents(dir))) {
    auto file_path = dir + "/";
    file_path.append(filename);
    CF_EXPECT(callback(file_path));
    if (DirectoryExists(file_path)) {
      CF_EXPECT(WalkDirectory(file_path, callback));
    }
  }
  return {};
}

std::vector<std::string> Path(const std::string& env_name) {
  // TODO: Assumes a SUS system. Elsewhere we may need to change the delimiter.
  // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html
  return absl::StrSplit(StringFromEnv(env_name).value_or(""), ':');
}

Result<std::string> Search(const std::vector<std::string>& path,
                           std::string_view name) {
  // TODO: Assumes a SUS system. Elsewhere we may need to change the delimiter.
  // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html
  for (const auto& dir : path) {
    std::string abs_path = absl::StrCat(dir, "/", name);
    if (FileExists(abs_path)) {
      return abs_path;
    }
  }
  return CF_ERR("Not found: ") << name << ", path " << absl::StrJoin(path, ":");
}

Result<SharedFD> CreateOrReuseAndDrainFifo(const std::string& path, mode_t mode) {
  struct stat st {};
  bool existed = false;
  if (TEMP_FAILURE_RETRY(stat(path.c_str(), &st)) != 0) {
    CF_EXPECTF(TEMP_FAILURE_RETRY(mkfifo(path.c_str(), mode)) == 0,
               "Failed to mkfifo('{}', {:o}): {}", path, mode,
               ::cuttlefish::StrError(errno));
  } else {
    CF_EXPECTF(S_ISFIFO(st.st_mode),
               "File at '{}' exists but is not a FIFO", path);
    existed = true;
  }

  auto ret = SharedFD::Open(path, O_RDWR);
  CF_EXPECTF(ret->IsOpen(), "Failed to open '{}': '{}'", path, ret->StrError());

  if (existed) {
    int flags = ret->Fcntl(F_GETFL, 0);
    if (flags >= 0) {
      ret->Fcntl(F_SETFL, flags | O_NONBLOCK);
      char buf[4096];
      while (ret->Read(buf, sizeof(buf)) > 0) {
        // Reading while there is data to read
      }
      ret->Fcntl(F_SETFL, flags);
    }
  }

  return ret;
}

}  // namespace cuttlefish