aurostd_xfile.cpp

Go to the documentation of this file.

// ***************************************************************************
// *                                                                         *
// *           Aflow STEFANO CURTAROLO - Duke University 2003-2024           *
// *                                                                         *
// ***************************************************************************
// This file hold function dealing with files
// hagen.eckert@duke.edu
 
#ifndef _AUROSTD_XFILE_CPP_
#define _AUROSTD_XFILE_CPP_
 
#include "aurostd_xfile.h"
 
#include <algorithm>
#include <cmath>
#include <cstddef>
#include <cstdlib>
#include <ctime>
#include <deque>
#include <filesystem>
#include <fstream>
#include <ios>
#include <iostream>
#include <regex>
#include <sstream>
 
#include <archive.h>
#include <archive_entry.h>
 
#include <sys/stat.h>
 
#include "aurostd.h"
#include "aurostd_defs.h"
#include "aurostd_time.h"
#include "aurostd_xerror.h"
#include "aurostd_xrandom.h"
#include "aurostd_xscalar.h"
 
#include "aflow_xhost.h" // todo required for XPID use and XHOST.DEBUG use
 
using std::cerr;
using std::endl;
 
namespace aurostd {
  bool DirectoryMake(const string& directory_raw) {
    const string directory = CleanFileName(directory_raw);
    if (!std::filesystem::is_directory(directory)) {
      return std::filesystem::create_directories(directory);
    }
    return true;
  }

  [[deprecated("Should be rewritten without using system calls")]]
  bool SSH_DirectoryMake(const string& user, const string& machine, const string& directory_raw) {  // "" compliant SC20190401
    const bool LDEBUG = (false || XHOST.DEBUG);
    const string Directory(aurostd::CleanFileName(directory_raw));
    std::deque<string> tokens;
    string dir_tmp;
    string command;
    aurostd::string2tokens(Directory, tokens, "/");
    if (Directory.at(0) == '/') {
      tokens[0] = "/" + tokens[0]; // fix the root thing
    }
    for (const auto& token : tokens) {
      dir_tmp += token + "/";
      command = "ssh " + user + "@" + machine + " mkdir \"" + dir_tmp + "\"";
      if (LDEBUG) {
        cerr << "SSH_DirectoryMake creating directory=" << command << endl;
      }
      aurostd::execute(command);
    }
    return true;
  }

  bool SubDirectoryLS(const string& directory_raw, vector<string>& vsubd) {
    const string directory = CleanFileName(directory_raw);
    if (directory.empty()) {
      return false;
    }
    vsubd.clear();
    vsubd.push_back(directory);
    for (const std::filesystem::directory_entry& dir_entry : std::filesystem::directory_iterator(directory)) {
      if (dir_entry.is_directory()) {
        vsubd.push_back(dir_entry.path().filename().string());
      }
    }
    return true;
  }

  bool DirectoryLS(const string& directory_raw, vector<string>& vfiles) {
    const string directory = CleanFileName(directory_raw);
    if (directory.empty()) {
      return false;
    }
    vfiles.clear();
    for (const std::filesystem::directory_entry& dir_entry : std::filesystem::directory_iterator(directory)) {
      vfiles.push_back(dir_entry.path().filename().string());
    }
    return true;
  }
 
  bool DirectoryLS(const string& directory_raw, deque<string>& vfiles) {
    vector<string> _vfiles;
    if (DirectoryLS(directory_raw, _vfiles)) {
      vfiles = aurostd::vector2deque(_vfiles);
      return true;
    }
    return false;
  }

  string dirname(const string& file) {
    return std::filesystem::path(aurostd::CleanFileName(file)).parent_path();
  }

  string basename(const string& file) {
    return std::filesystem::path(aurostd::CleanFileName(file)).stem();
  }

  string GetFileExtension(const string& FileName) {
    return std::filesystem::path(FileName).extension().string();
  }

  string CleanFileName(const string& fileIN) {
    const bool LDEBUG = (false || XHOST.DEBUG);
    if (fileIN.empty()) {
      return fileIN;
    }
    // ME20211001
    // Remove any control characters (below ASCII 32) while copying. This is useful
    // when fileIN is read from a file, which can have all sorts of junk and causes
    // FileExist to break. We cannot use RemoveControlCodeCharactersFromString()
    // because it keeps tabs and linebreaks and cannot use CleanStringASCII because
    // it keeps control characters.
    string fileOUT;
    for (const char c : fileIN) {
      if (c > 31) {
        fileOUT += c;
      }
    }
    if (fileOUT.find('~') != string::npos) {
      aurostd::StringSubstInPlace(fileOUT, "~", getenv("HOME"));
    }
    // ME20200922 - Cleaning // must be in a while loop, or it won't clean e.g. ///
    while (fileOUT.find("//") != string::npos) {
      aurostd::StringSubstInPlace(fileOUT, "//", "/");
    }
    aurostd::StringSubstInPlace(fileOUT, "/./", "/");
 
    if (LDEBUG) {
      cerr << __AFLOW_FUNC__ << " " << fileOUT << endl;
    }
    return fileOUT;
  }

  string CleanFileName(const fs::path& fileIN) {
    return CleanFileName(fileIN.string());
  }

  bool DirectoryLocked(const string& directory, const string& LOCK) {
    if (FileExist(directory + "/" + LOCK)) {
      return true;
    }
    if (FileExist(directory + "/" + LOCK + _LOCK_LINK_SUFFIX_)) {
      return true;
    }
    if (FileExist(directory + "/" + LOCK + ".xz")) {
      return true;
    }
    if (FileExist(directory + "/" + LOCK + ".gz")) {
      return true;
    }
    if (FileExist(directory + "/" + LOCK + ".bz2")) {
      return true;
    }
    return false;
  }
 
  bool DirectoryLocked(const string& directory) {
    return DirectoryLocked(directory, "LOCK");
  }

  bool DirectorySkipped(const string& directory) {
    if (FileExist(directory + "/SKIP")) {
      return true;
    }
    if (FileExist(directory + "/SKIP.xz")) {
      return true;
    }
    if (FileExist(directory + "/SKIP.gz")) {
      return true;
    }
    if (FileExist(directory + "/SKIP.bz2")) {
      return true;
    }
    return false;
  }

  bool DirectoryWritable(const string& directory_raw) {  // "" compliant SC20190401
    const string Directory(aurostd::CleanFileName(directory_raw));
    const string filename = aurostd::TmpFileCreate("DirectoryWritable", Directory, true); // SD20220223 - uses TmpFileCreate
    const bool writable = aurostd::string2file("DirectoryWritable", filename);
    if (!writable || !FileExist(filename)) {
      return false;
    }
    aurostd::RemoveFile(filename);
    return true;
  }

  bool IsDirectory(const string& path) {
    return std::filesystem::is_directory(aurostd::CleanFileName(path));
  }

  //***************************************************************************//
  // aurostd::IsCompressed
  //***************************************************************************//
  bool IsCompressed(const string& FileNameIn) {
    const std::filesystem::path path(FileNameIn);
    archive* a;
    archive_entry* entry;
    bool check = aurostd::FileNotEmpty(FileNameIn);
    int r;
 
    // initialized the new archive
    a = archive_read_new();
    if (path.extension() == ".zip") {
      archive_read_support_format_zip(a);
    } else {
      archive_read_support_format_raw(a);
      archive_read_support_format_empty(a);
      archive_read_support_filter_all(a);
    }
 
    // open the archive
    r = archive_read_open_filename(a, FileNameIn.c_str(), 16384);
    if (r != ARCHIVE_OK) {
      check = false;
    }
 
    if (check) {
      // check if the content is valid (only quick check - file could be corrupt later)
      archive_read_next_header(a, &entry);
      if ((static_cast<std::string>(archive_filter_name(a, r)) == "none") && (static_cast<std::string>(archive_format_name(a)) == "raw")) {
        check = false;
      }
    }
    archive_read_free(a);
    return check;
  }

  bool IsCompressed(const string& FileNameIn, string& FileNameOut) {
    const std::filesystem::path path(FileNameIn);
    FileNameOut = path.parent_path().string() + "/" + path.stem().string();
    return IsCompressed(FileNameIn);
  }

  bool FileExist(const string& FileName) {
    try {
      return std::filesystem::exists(aurostd::CleanFileName(FileName));
    } catch (std::filesystem::filesystem_error& e) {
      return false;
    }
  }

  bool FileExist(const string& FileName, string& FileNameOut) {
    if (FileExist(FileName)) {
      FileNameOut = FileName;
      return true;
    }
    return false;
  }

  bool CompressFileExist(const string& FileName) {
    string FileNameOut;
    return CompressFileExist(FileName, FileNameOut);
  }

  bool CompressFileExist(const string& FileNameRaw, string& FileNameOut) {
    const string FileName = aurostd::CleanFileName(FileNameRaw);
    if (FileExist(FileName)) {
      FileNameOut = FileName;
      return true;
    }
    if (FileExist(FileName + ".xz")) {
      FileNameOut = FileName + ".xz";
      return true;
    }
    if (FileExist(FileName + ".bz2")) {
      FileNameOut = FileName + ".bz2";
      return true;
    }
    if (FileExist(FileName + ".gz")) {
      FileNameOut = FileName + ".gz";
      return true;
    }
    if (FileExist(FileName + ".zip")) {
      FileNameOut = FileName + ".zip";
      return true;
    }
    return false;
  }

  bool CompressFileWithinList(const vector<string>& list, const string& input) {
    string output;
    return CompressFileWithinList(list, input, output);
  }

  bool CompressFileWithinList(const vector<string>& list, const string& input, string& output) {
    output = "";
    for (const string& entry : list) {
      for (const string& suffix : compression_suffix) {
        if (entry == input + suffix) {
          output = input + suffix;
          return true;
        }
      }
    }
    return false;
  }

  bool FileEmpty(const string& FileNameRaw) {
    const string FileName(aurostd::CleanFileName(FileNameRaw));
    if (!FileExist(FileName)) {
      return true;  // does not exist hence empty
    }
    ifstream FileStream(FileName);
    return (FileStream.peek() == ifstream::traits_type::eof());
  }

  bool FileNotEmpty(const string& FileName) {
    return !FileEmpty(FileName);
  }

  long int GetTimestampModified(const string& FileNameRaw) {
    const string FileName(aurostd::CleanFileName(FileNameRaw));
    if (!FileExist(FileName)) {
      return 0;
    }
    time_t tm = 0;
    struct stat file_stat{};
    if (stat(FileName.c_str(), &file_stat) == 0) {
      tm = file_stat.st_mtime;
    }
    return static_cast<long int>(tm);
  }

  // gets modification time and returns SECONDS since now (as long int)
  long int SecondsSinceFileModified(const string& FileNameRaw) {
    const string FileName(aurostd::CleanFileName(FileNameRaw));
    if (!FileExist(FileName)) {
      return 0;
    }
    const long int tmod_file = GetTimestampModified(FileName);
    // instead, write a new file and take the difference in the time stamps
    // solution inspired by ME - create a temporary file IN THE CURRENT DIRECTORY (within NFS) and check time deltas
    const string dir = aurostd::dirname(FileName);
    const string tmpfile = aurostd::TmpFileCreate("timestamp", dir, true); // put in current directory, make it hidden
    if (!aurostd::string2file("timestamp", tmpfile)) {
      return 0;
    }  // write the file out, need a better fix here, perhaps write to /tmp?
    const long int tmod_tmp = aurostd::GetTimestampModified(tmpfile);
    aurostd::RemoveFile(tmpfile);
    return max(static_cast<long int>(0), tmod_tmp - tmod_file); // max ensures that if something goes wrong, we return 0
  }

  unsigned long long int FileSize(const string& FileName) {
    return std::filesystem::file_size(aurostd::CleanFileName(FileName));
  }

  void InFileExistCheck(const string& routine, const string& FileNameRaw, const ifstream& file_to_check) {
    const string FileName(aurostd::CleanFileName(FileNameRaw));
    if (!file_to_check) {
      const string message = "In routine " + routine + ". Cannot open file " + FileName + ".";
      throw aurostd::xerror(__AFLOW_FILE__, __AFLOW_FUNC__, message, _FILE_ERROR_);
    }
  }

  string TmpStrCreate(const string& identifier_raw, const string& tmpdir_raw, const bool hidden, const bool directory) {
    string identifier = identifier_raw;
    if (identifier.empty()) {
      identifier = "tmp";
    } // CO20210624
    string tmpdir = tmpdir_raw;
    if (tmpdir.empty()) {
      tmpdir = XHOST.tmpfs;
    } // CO20210315
    string str = tmpdir + "/" + (hidden ? "." : "") + "_aflow_" + identifier + "." + XHOST.user + ".pid" + XHOST.ostrPID.str() + ".tid" + XHOST.ostrTID.str() + ".a" + AFLOW_VERSION +
                 ".rnd" + aurostd::utype2string(uint(std::floor((double) 100000 * aurostd::ran0()))) + ".u" + aurostd::utype2string(uint((double) aurostd::get_useconds())) + (directory ? "_" : ".") + "tmp"; // CO20200502 - threadID
    str = aurostd::CleanFileName(str);
    return str;
  }

  string TmpFileCreate(const string& identifier, const string& tmpdir, const bool hidden) { // CO20210315
    return TmpStrCreate(identifier, tmpdir, hidden, false); // directory=false -> creating a file
  }

  string TmpDirectoryCreate(const string& identifier, const string& tmpdir, const bool hidden) { // CO20210315
    string dir = TmpStrCreate(identifier, tmpdir, hidden, true); // directory=true -> creating a directory
    DirectoryMake(dir);
    return dir;
  }

  bool CopyFile(const string& file_from, const string& file_to) {
    const std::filesystem::path from_path = CleanFileName(file_from);
    const std::filesystem::path to_path = CleanFileName(file_to);
 
    if (!std::filesystem::is_regular_file(from_path)) {
      return false;
    }
    if (std::filesystem::is_regular_file(to_path)) { // equivalence check can only be performed if both paths point to a file
      if (std::filesystem::equivalent(from_path, to_path)) {
        return false;
      }
    }
    return std::filesystem::copy_file(from_path, to_path, std::filesystem::copy_options::overwrite_existing);
  }

  bool LinkFile(const string& file_from, const string& file_to, const bool soft) {
    const string from_clean = CleanFileName(file_from);
    const string to_clean = CleanFileName(file_to);
    const std::filesystem::path from_path = from_clean;
    const std::filesystem::path to_path = to_clean;
    if (from_path.empty() || to_path.empty() || !std::filesystem::exists(from_path) || std::filesystem::exists(to_path) || !std::filesystem::exists(to_path.parent_path())) {
      return false;
    }
    if (soft) {
      std::filesystem::create_symlink(from_clean, to_clean);
      return std::filesystem::is_symlink(to_clean);
    }
    std::filesystem::create_hard_link(from_clean, to_clean);
    return std::filesystem::is_regular_file(to_clean);
  }

  bool file2directory(const string& file, const string& destination) {
    const std::filesystem::path old_path = CleanFileName(file);
    const std::filesystem::path dir_path = CleanFileName(destination);
    if (!std::filesystem::is_regular_file(old_path)) {
      return false;
    }
    if (!std::filesystem::is_directory(dir_path)) {
      return false;
    }
    try {
      std::filesystem::rename(old_path, dir_path / old_path.filename());
    } catch (std::filesystem::filesystem_error& e) {
      std::filesystem::copy(old_path, dir_path / old_path.filename(), std::filesystem::copy_options::overwrite_existing);
      std::filesystem::remove(old_path);
    }
    return true;
  }

  bool file2directory(const vector<string>& files, const string& destination) {
    for (const auto& file : files) {
      if (!file2directory(file, destination)) {
        return false;
      }
    }
 
    return true;
  }

  bool file2file(const string& file, const string& destination) {
    const std::filesystem::path old_path = CleanFileName(file);
    const std::filesystem::path new_path = CleanFileName(destination);
    if (!std::filesystem::is_regular_file(old_path)) {
      return false;
    }
    try {
      std::filesystem::rename(old_path, new_path);
    } catch (std::filesystem::filesystem_error& e) {
      std::filesystem::copy(old_path, new_path, std::filesystem::copy_options::overwrite_existing);
      std::filesystem::remove(old_path);
    }
    return true;
  }

  bool RemoveFile(const string& file) {
    const std::filesystem::path path(aurostd::CleanFileName(file));
    if (std::filesystem::is_regular_file(path)) {
      return std::filesystem::remove(path);
    }
    return false;
  }

  bool RemoveFile(const string& directory, const std::regex& regex_pattern) {
    const std::filesystem::path path(aurostd::CleanFileName(directory));
    for (const std::filesystem::directory_entry& dir_entry : std::filesystem::directory_iterator(std::filesystem::path(aurostd::CleanFileName(directory)))) {
      if (dir_entry.is_regular_file() && std::regex_match(dir_entry.path().filename().string(), regex_pattern)) {
        std::filesystem::remove(dir_entry.path());
      }
    }
    return true;
  }

  bool RemoveFiles(const string& directory) {
    for (const auto& dir_entry : std::filesystem::directory_iterator(aurostd::CleanFileName(directory))) {
      if (dir_entry.is_regular_file()) {
        std::filesystem::remove(dir_entry.path());
      }
    }
    return true;
  }

  bool RemoveFile(const vector<string>& files) {
    for (const auto& file : files) {
      if (!RemoveFile(file)) {
        return false;
      }
    }
    return true;
  }

  bool RemoveDirectory(const string& directory) {
    const std::filesystem::path path(aurostd::CleanFileName(directory));
    if (std::filesystem::is_directory(path)) {
      std::filesystem::remove_all(path);
    }
    return false;
  }
 
  compression_type GetCompressionType(const fs::path& path) {
    const string extension = aurostd::GetFileExtension(path);
    const auto it = std::find(compression_suffix.begin(), compression_suffix.end(), extension);
    if (it == compression_suffix.end()) {
      return compression_type::None;
    }
    const compression_type ct = static_cast<compression_type>(it - compression_suffix.begin());
    return ct;
  }

  size_t file2string(const string& FileNameIN, string& StringIN) {
    const string FileNameINClean = aurostd::CleanFileName(FileNameIN);
    if (GetCompressionType(FileNameINClean)) {
      return compressfile2string(FileNameINClean, StringIN);
    }
    const std::ifstream open_file(FileNameINClean);
    std::stringstream buffer;
    buffer << open_file.rdbuf();
    StringIN = buffer.str();
    return StringIN.length();
  }

  string file2string(const string& FileNameIN) {
    string StringIN;
    file2string(FileNameIN, StringIN);
    return StringIN;
  }

  size_t compressfile2string(const string& FileNameIN, string& content) {
    const string file(aurostd::CleanFileName(FileNameIN));
    std::stringstream ss;
    compressfile2stringstream(file, ss);
    content = ss.str();
    return content.length();
  }

  string compressfile2string(const string& FileNameIN) {
    string StringIN;
    compressfile2string(FileNameIN, StringIN);
    return StringIN;
  }

  size_t file2vectorstring(const string& FileNameIN, vector<string>& vline, bool consecutive, bool trim_edges) {
    return aurostd::string2vectorstring(file2string(aurostd::CleanFileName(FileNameIN)), vline, consecutive, trim_edges);
  }

  size_t file2vectorstring(const string& FileNameIN, deque<string>& vline) {
    return aurostd::string2dequestring(file2string(aurostd::CleanFileName(FileNameIN)), vline);
  }

  size_t compressfile2vectorstring(const string& FileNameIN, vector<string>& vline, bool consecutive, bool trim_edges) {
    return aurostd::string2vectorstring(compressfile2string(aurostd::CleanFileName(FileNameIN)), vline, consecutive, trim_edges);
  }

  size_t compressfile2vectorstring(const string& FileNameIN, deque<string>& vline) {
    return aurostd::string2dequestring(compressfile2string(aurostd::CleanFileName(FileNameIN)), vline);
  }

  void file2stringstream(const string& FileNameIN, stringstream& StringstreamIN) {
    const string FileNameINClean = aurostd::CleanFileName(FileNameIN);
    if (GetCompressionType(FileNameINClean)) {
      compressfile2stringstream(FileNameINClean, StringstreamIN);
    } else {
      const std::ifstream open_file(FileNameINClean);
      StringstreamIN << open_file.rdbuf();
    }
  }

  void compressfile2stringstream(const string& file_raw, stringstream& content) {
    const string file(aurostd::CleanFileName(file_raw));
    static constexpr size_t buff_len = 16384;
    thread_local char buff[buff_len + 1];
    archive* a = archive_read_new();
    archive_entry* entry;
 
    // initialized the archive to be read
 
    if (aurostd::GetFileExtension(file) == ".zip") {
      archive_read_support_format_zip(a);
    } else {
      archive_read_support_format_raw(a);
      archive_read_support_format_empty(a);
      archive_read_support_filter_all(a);
    }
 
    // open the archive
    if (const int r = archive_read_open_filename(a, file.c_str(), buff_len); r != ARCHIVE_OK) {
      const std::string message = "Problem loading output: ";
      throw aurostd::xerror(__AFLOW_FILE__, __AFLOW_FUNC__, message + archive_error_string(a), _FILE_NOT_FOUND_);
    }
    archive_read_next_header(a, &entry);
 
    // prepare the stringstream and write the data
    aurostd::StringstreamClean(content);
    size_t len = archive_read_data(a, buff, buff_len);
    while (len > 0) { // read until archive is empty
      buff[len] = '\0'; // ensure proper termination of the c style string
      content << buff;
      len = archive_read_data(a, buff, buff_len);
    }
 
    // free objects created in this function
    archive_read_free(a);
  }

  bool string2file(const string& StringOUTPUT, const std::string& FileNameRawOUTPUT, const compression_type ct, const string& mode) {
    const fs::path FileNameOUTPUT = CleanFileName(FileNameRawOUTPUT);
    try {
      fs::create_directories(fs::absolute(FileNameOUTPUT).parent_path());
    } catch (fs::filesystem_error& e) {
      return false;
    }
    bool writable = true; // CO20190808 - captures whether we can open/write file
 
    if (mode == "POST" || mode == "APPEND" || mode == "PRE") {
      string FileINPUT;
      if (fs::exists(FileNameOUTPUT)) {
        aurostd::file2string(FileNameOUTPUT, FileINPUT);
      }
      if (ct) {
        if (mode == "POST" || mode == "APPEND") {
          return string2compressfile(StringOUTPUT + FileINPUT, FileNameOUTPUT, ct);
        }
        return string2compressfile(FileINPUT + StringOUTPUT, FileNameOUTPUT, ct);
      }
      ofstream FileOUTPUT;
      FileOUTPUT.open(FileNameOUTPUT.c_str(), std::ios::out);
      writable = FileOUTPUT.is_open(); // CO20190808 - captures whether we can open/write file
      if (mode == "POST" || mode == "APPEND") {
        FileOUTPUT << FileINPUT;
      }
      FileOUTPUT << StringOUTPUT;
      if (mode == "PRE") {
        FileOUTPUT << FileINPUT;
      }
      FileOUTPUT.flush();
      FileOUTPUT.clear();
      FileOUTPUT.close();
      return writable; // return false if something got messed up //CO20190808 - captures whether we can open/write file
    }
 
    if (mode == "WRITE" || mode.empty()) {
      if (ct) {
        return string2compressfile(StringOUTPUT, FileNameOUTPUT, ct);
      }
      ofstream FileOUTPUT;
      FileOUTPUT.open(FileNameOUTPUT.c_str(), std::ios::out);
      writable = FileOUTPUT.is_open(); // CO20190808 - captures whether we can open/write file
      FileOUTPUT << StringOUTPUT;
      FileOUTPUT.flush();
      FileOUTPUT.clear();
      FileOUTPUT.close();
      return writable; // return false if something got messed up //CO20190808 - captures whether we can open/write file
    }
    return false;
  }

  bool string2compressfile(const string& content, const string& file_raw, const compression_type ct) {
    fs::path file(aurostd::CleanFileName(file_raw));
    try {
      fs::create_directories(fs::absolute(file).parent_path());
    } catch (fs::filesystem_error& e) {
      return false;
    }
 
    archive* a;
    archive_entry* entry;
    int r;
 
    // initialized the new archive
    a = archive_write_new();
    entry = archive_entry_new();
 
    // select compression and archive format
    aurostd::CompressInit(a, ct, true);
 
    // open the new archive
    if (const std::string ext = file.extension(); ext != compression_suffix[ct]) {
      file = file.replace_extension(ext + compression_suffix[ct]);
    }
 
    r = archive_write_open_filename(a, file.c_str());
    if (r != ARCHIVE_OK) {
      const std::string message = "Problem preparing output: ";
      throw aurostd::xerror(__AFLOW_FILE__, __AFLOW_FUNC__, message + archive_error_string(a), _FILE_NOT_FOUND_);
    }
 
    // configure new entry
    archive_entry_set_pathname(entry, file.c_str());
    archive_entry_set_size(entry, content.size());
    archive_entry_set_filetype(entry, AE_IFREG);
 
    // save file information to archive
    archive_write_header(a, entry);
 
    // write string to archive
    archive_write_data(a, content.c_str(), content.size());
 
    // free objects created in this function
    archive_entry_free(entry);
    archive_write_close(a);
    archive_write_free(a);
    return true;
  }

  bool stringstream2file(const stringstream& StringstreamOUTPUT, const string& FileNameOUTPUT, const compression_type ct, const string& mode) {
    return string2file(StringstreamOUTPUT.str(), FileNameOUTPUT, ct, mode);
  }

  bool stringstream2compressfile(const stringstream& content, const string& file, const compression_type ct) {
    return string2compressfile(content.str(), file, ct);
  }

  template <template <class...> class vtype> bool vectorstring2file(const vtype<string>& vline, const string& FileNameOUT, const compression_type ct) {
    const fs::path file = aurostd::CleanFileName(FileNameOUT);
 
    try {
      fs::create_directories(fs::absolute(file).parent_path());
    } catch (fs::filesystem_error& e) {
      return false;
    }
 
    if (ct) {
      std::stringstream full_string;
      for (const auto& iline : vline) {
        full_string << iline << endl;
      }
      return stringstream2compressfile(full_string, file, ct);
    }
    ofstream FileOUT;
    FileOUT.open(file.c_str(), std::ios::out);
    const bool writable = FileOUT.is_open(); // CO20190808 - captures whether we can open/write file
    for (const auto& iline : vline) {
      FileOUT << iline << endl;
    }
    FileOUT.flush();
    FileOUT.clear();
    FileOUT.close();
    return writable;
  }
  template bool vectorstring2file(const vector<string>& vline, const string& FileNameOUT, compression_type ct);
  template bool vectorstring2file(const deque<string>& vline, const string& FileNameOUT, compression_type ct);

  void CompressionErrorHandling(archive* a, const int r, const std::string& message) {
    if (r != ARCHIVE_OK) {
      if (r == ARCHIVE_WARN) {
        const string full_message = "LibArchive Warning during " + message + ": ";
        if (XHOST.DEBUG) {
          cerr << full_message << archive_error_string(a) << " (not a fatal error)" << endl;
        }
      } else {
        const string full_message = "LibArchive Error during " + message + ": ";
        throw aurostd::xerror(__AFLOW_FILE__, __AFLOW_FUNC__, full_message + archive_error_string(a), _RUNTIME_INIT_);
      }
    }
  }

  void CompressInit(archive* a, const compression_type ct, const bool single) {
    int r = ARCHIVE_OK;
    const std::string error_message = "initialization";
    switch (ct) {
      case BZ2:
        if (single) {
          r = archive_write_set_format_raw(a);
        } else {
          r = archive_write_set_format_pax_restricted(a);
        }
        CompressionErrorHandling(a, r, error_message);
        r = archive_write_add_filter_bzip2(a);
        CompressionErrorHandling(a, r, error_message);
        r = archive_write_set_options(a, "compression-level=9");
        CompressionErrorHandling(a, r, error_message);
        break;
      case GZ:
        if (single) {
          r = archive_write_set_format_raw(a);
        } else {
          r = archive_write_set_format_pax_restricted(a);
        }
        CompressionErrorHandling(a, r, error_message);
        r = archive_write_add_filter_gzip(a);
        CompressionErrorHandling(a, r, error_message);
        r = archive_write_set_options(a, "compression-level=9");
        CompressionErrorHandling(a, r, error_message);
        break;
      case XZ:
        if (single) {
          r = archive_write_set_format_raw(a);
        } else {
          r = archive_write_set_format_pax_restricted(a);
        }
        CompressionErrorHandling(a, r, error_message);
        r = archive_write_add_filter_xz(a);
        CompressionErrorHandling(a, r, error_message);
        r = archive_write_set_options(a, "compression-level=9,threads=1");
        CompressionErrorHandling(a, r, error_message);
        break;
      case ZIP:
        r = archive_write_set_format_zip(a);
        CompressionErrorHandling(a, r, error_message);
        if (single) {
          r = archive_write_set_options(a, "compression=deflate,zip64");
        } else {
          r = archive_write_set_options(a, "compression=store,zip64");
        }
        CompressionErrorHandling(a, r, error_message);
        break;
      case ZSTD:
        if (single) {
          r = archive_write_set_format_raw(a);
        } else {
          r = archive_write_set_format_pax_restricted(a);
        }
        CompressionErrorHandling(a, r, error_message);
        r = archive_write_add_filter_zstd(a);
        CompressionErrorHandling(a, r, error_message);
        r = archive_write_set_options(a, "compression-level=20");
        CompressionErrorHandling(a, r, error_message);
        break;
      default: throw aurostd::xerror(__AFLOW_FILE__, __AFLOW_FUNC__, "Unknown archive format", _INPUT_UNKNOWN_);
    }
  }

  void DecompressFile(const string& file_raw, const string& outfile_raw, const bool keep) {
    const string file(aurostd::CleanFileName(file_raw));
    string outfile(aurostd::CleanFileName(outfile_raw));
    const std::filesystem::path path(file);
    thread_local char buff[16384];
    archive* a;
    archive_entry* entry;
    size_t len;
    int r = ARCHIVE_OK;
    const std::string error_message = "decompression of a single file";
 
    // initialized the new archive and the file entries
    a = archive_read_new();
    if (path.extension() == ".zip") {
      r = archive_read_support_format_zip(a);
      CompressionErrorHandling(a, r, error_message);
    } else {
      r = archive_read_support_format_raw(a);
      CompressionErrorHandling(a, r, error_message);
      r = archive_read_support_format_empty(a);
      CompressionErrorHandling(a, r, error_message);
      r = archive_read_support_filter_all(a);
      CompressionErrorHandling(a, r, error_message);
    }
 
    // open the archive
    r = archive_read_open_filename(a, file.c_str(), sizeof(buff));
    CompressionErrorHandling(a, r, error_message);
    if (outfile.empty()) {
      outfile = path.parent_path().string() + "/" + path.stem().string();
    }
    r = archive_read_next_header(a, &entry);
    if (r == ARCHIVE_EOF) { // create empty file if archive has no data
      std::fstream fs(outfile, std::ios::out);
      fs.close();
    } else {
      CompressionErrorHandling(a, r, error_message);
      // open the new file and write it in chunks
      std::fstream fs(outfile, std::ios::out);
      len = archive_read_data(a, buff, sizeof(buff));
      while (len > 0) { // read until archive is empty
        fs.write(buff, len);
        len = archive_read_data(a, buff, sizeof(buff));
      }
      fs.close();
    }
 
    // free objects created in this function
    if (!keep) {
      aurostd::RemoveFile(file);
    }
    r = archive_read_free(a);
    CompressionErrorHandling(a, r, error_message);
  }
 
  void DecompressFile(const string& file_raw, const bool keep) {
    DecompressFile(file_raw, "", keep);
  }

  void DecompressFiles(const string& archive_file, const string& directory, const bool keep) {
    const string archive_file_clean(aurostd::CleanFileName(archive_file));
    string directory_clean(aurostd::CleanFileName(directory));
    std::filesystem::path path(archive_file_clean);
    thread_local char buff[16384];
    archive* a;
    archive_entry* entry;
    size_t len;
    int r = ARCHIVE_OK;
    const std::string error_message = "decompression of multiple files";
 
    // initialized the new archive and the file entries
    a = archive_read_new();
    if (path.extension() == ".zip") {
      r = archive_read_support_format_zip(a);
      CompressionErrorHandling(a, r, error_message);
    } else {
      r = archive_read_support_format_tar(a);
      CompressionErrorHandling(a, r, error_message);
      r = archive_read_support_filter_all(a);
      CompressionErrorHandling(a, r, error_message);
    }
 
    // open the archive
    r = archive_read_open_filename(a, archive_file_clean.c_str(), sizeof(buff));
    CompressionErrorHandling(a, r, error_message);
 
    // open the new files and write it in chunks
    if (directory_clean.empty()) {
      directory_clean = aurostd::getPWD();
    }
    while (archive_read_next_header(a, &entry) == ARCHIVE_OK) {
      if ((static_cast<std::string>(archive_filter_name(a, r)) == "none") && (static_cast<std::string>(archive_format_name(a)) == "raw")) {
        continue;
      }
      path = std::filesystem::path(directory_clean + "/" + archive_entry_pathname(entry));
      if (!std::filesystem::is_directory(path.parent_path())) {
        std::filesystem::create_directories(path.parent_path());
      }
      std::fstream fs(path.string(), std::ios::out);
      len = archive_read_data(a, buff, sizeof(buff));
      while (len > 0) { // read until archive is empty
        fs.write(buff, len);
        len = archive_read_data(a, buff, sizeof(buff));
      }
      fs.close();
    }
 
    // free objects created in this function
    if (!keep) {
      aurostd::RemoveFile(archive_file_clean);
    }
    r = archive_read_free(a);
    CompressionErrorHandling(a, r, error_message);
  }
 
  void DecompressFiles(const string& archive, const bool keep) {
    DecompressFiles(archive, "", keep);
  }

  void CompressFile(const string& file_raw, const compression_type ct, const bool keep) {
    const fs::path file(aurostd::CleanFileName(file_raw));
 
    thread_local char buff[16384];
    archive* a;
    archive* disk;
    archive_entry* entry;
    int r = ARCHIVE_OK;
    const std::string error_message = "compression of a single file";
 
    // initialized the new archive, the disk reader and the file entries
    a = archive_write_new();
    entry = archive_entry_new();
    disk = archive_read_disk_new();
    r = archive_read_disk_set_standard_lookup(disk);
    CompressionErrorHandling(a, r, error_message);
 
    // select compression and archive format
    aurostd::CompressInit(a, ct, true);
 
    // open the disk reader
    r = archive_read_disk_open(disk, file.c_str());
    CompressionErrorHandling(a, r, error_message);
 
    // open the new archive
    fs::path outfile = file;
    outfile.replace_extension(static_cast<string>(file.extension()) + compression_suffix[ct]);
 
    r = archive_write_open_filename(a, outfile.c_str());
    CompressionErrorHandling(a, r, error_message);
 
    // load information of file from disk
    r = archive_read_next_header2(disk, entry);
    CompressionErrorHandling(a, r, error_message);
    r = archive_read_disk_descend(disk);
    CompressionErrorHandling(a, r, error_message);
 
    // save file information to archive
    r = archive_write_header(a, entry);
    CompressionErrorHandling(a, r, error_message);
 
    // open the input file and write it in chunks to new archive
    std::ifstream is(archive_entry_sourcepath(entry), std::ifstream::binary);
    while (is) {
      is.read(buff, sizeof(buff));
      archive_write_data(a, buff, is.gcount());
    }
    is.close();
 
    // free objects created in this function
    if (!keep) {
      aurostd::RemoveFile(file);
    }
    archive_entry_free(entry);
    r = archive_read_free(disk);
    CompressionErrorHandling(a, r, error_message);
    r = archive_write_free(a);
    CompressionErrorHandling(a, r, error_message);
  }

  fs::path CompressFiles(const vector<string>& files_raw, const fs::path& relative_to_path, const string& archive_name, const compression_type ct, const bool keep) {
    thread_local char buff[16384];
    archive* a;
    archive* disk;
    archive_entry* entry;
    int r = ARCHIVE_OK;
    const std::string error_message = "compression of multiple files";
    vector<string> files;
    files.reserve(files_raw.size());
    for (const auto& f : files_raw) {
      files.push_back(aurostd::CleanFileName(f));
    }
 
    // initialized the new archive and the disk reader
    a = archive_write_new();
 
    // select compression and archive format
    aurostd::CompressInit(a, ct, false);
 
    // open the new archive
    string tar_suffix = ".tar";
    if (ct == compression_type::ZIP) {
      tar_suffix = "";
    }
    fs::path outfile = archive_name + tar_suffix + compression_suffix[ct];
    try {
      fs::create_directories(fs::absolute(outfile).parent_path());
    } catch (fs::filesystem_error& e) {
      const string error = e.what();
      const string message = "Parent folder could not be created: " + static_cast<string>(outfile.parent_path()) + "\n" + error;
      throw aurostd::xerror(__AFLOW_FILE__, __AFLOW_FUNC__, message, _FILE_ERROR_);
    }
 
    r = archive_write_open_filename(a, outfile.c_str());
    CompressionErrorHandling(a, r, error_message);
 
    for (const auto& file : files) {
      // initialized the file entries
      entry = archive_entry_new();
 
      // initialized the disk reader
      disk = archive_read_disk_new();
      r = archive_read_disk_set_standard_lookup(disk);
      CompressionErrorHandling(a, r, error_message);
 
      // open the disk reader
      r = archive_read_disk_open(disk, file.c_str());
      CompressionErrorHandling(a, r, error_message);
 
      // load information of file from disk
      r = archive_read_next_header2(disk, entry);
      CompressionErrorHandling(a, r, error_message);
      r = archive_read_disk_descend(disk);
      CompressionErrorHandling(a, r, error_message);
 
      // HE20240907 change path information to be relative
      const std::string original_path_raw = archive_entry_pathname(entry);
      const fs::path original_path(original_path_raw);
      // using proximate to avoid empty paths in case `relative_to_path` and `original_path` are not compatible
      const fs::path new_path = fs::proximate(original_path, relative_to_path);
      const std::string new_path_raw = new_path.string();
      archive_entry_set_pathname(entry, new_path_raw.c_str());
 
      // save file information to archive
      r = archive_write_header(a, entry);
      CompressionErrorHandling(a, r, error_message);
 
      // open the input file and write it in chunks to new archive
      std::ifstream is(archive_entry_sourcepath(entry), std::ifstream::binary);
      while (is) {
        is.read(buff, sizeof(buff));
        archive_write_data(a, buff, is.gcount());
      }
      is.close();
 
      // free read objects
      if (!keep) {
        aurostd::RemoveFile(file);
      }
      archive_entry_free(entry);
      r = archive_read_free(disk);
      CompressionErrorHandling(a, r, error_message);
    }
 
    // free write objects
    r = archive_write_close(a);
    CompressionErrorHandling(a, r, error_message);
    r = archive_write_free(a);
    CompressionErrorHandling(a, r, error_message);
 
    return outfile;
  }

  bool MatchCompressed(const string& CompressedFileName, const string& FileNameOUT) {
    if (!aurostd::IsCompressed(CompressedFileName) && !aurostd::IsCompressed(FileNameOUT)) {
      return true;
    }
    if (aurostd::IsCompressed(FileNameOUT)) {
      if (aurostd::GetFileExtension(CompressedFileName) == aurostd::GetFileExtension(FileNameOUT)) {
        return true;
      }
      return false;
    }
    const compression_type ct = aurostd::GetCompressionType(CompressedFileName);
    if (ct == compression_type::None) {
      return false;
    }
    aurostd::CompressFile(FileNameOUT, ct);
    return true;
  }

  bool compressfile2tempfile(const string& FileNameIN, string& FileNameOUT) {
    bool tempfile_created = false;
    return compressfile2tempfile(FileNameIN, FileNameOUT, tempfile_created);
  }

  bool compressfile2tempfile(const string& FileNameRawIN, string& FileNameOUT, bool& tempfile_created) {
    string FileNameIN;
    tempfile_created = false;
    if (!(aurostd::FileExist(FileNameRawIN, FileNameIN) || aurostd::CompressFileExist(FileNameRawIN, FileNameIN))) {
      return false;
    }
    if (!aurostd::IsCompressed(FileNameIN)) {
      FileNameOUT = FileNameIN;
      return true;
    }
    tempfile_created = true;
    FileNameOUT = aurostd::TmpStrCreate();
    aurostd::DecompressFile(FileNameIN, FileNameOUT, true);
    return true;
  }

  void compressfile2compressfile(const string& file, const compression_type ct, const bool keep) {
    const std::filesystem::path path(file);
    thread_local char buff[16384];
    archive* a;
    archive* a_new;
    archive_entry* entry;
    size_t len;
    int r = ARCHIVE_OK;
    const std::string error_message = "re-compressing of a single file";
 
    // initialized the new archive, the disk and the file entries
    a = archive_read_new();
    if (path.extension() == ".zip") {
      r = archive_read_support_format_zip(a);
      CompressionErrorHandling(a, r, error_message);
    } else {
      r = archive_read_support_format_raw(a);
      CompressionErrorHandling(a, r, error_message);
      r = archive_read_support_format_empty(a);
      CompressionErrorHandling(a, r, error_message);
      r = archive_read_support_filter_all(a);
      CompressionErrorHandling(a, r, error_message);
    }
    a_new = archive_write_new();
 
    // select compression and archive format
    aurostd::CompressInit(a_new, ct, true);
 
    // open the archive
    r = archive_read_open_filename(a, file.c_str(), sizeof(buff));
    CompressionErrorHandling(a, r, error_message);
    r = archive_read_next_header(a, &entry);
    CompressionErrorHandling(a, r, error_message);
 
    // open the new archive
    const string outfile = path.parent_path().string() + "/" + path.stem().string() + compression_suffix[ct];
    r = archive_write_open_filename(a_new, outfile.c_str());
    CompressionErrorHandling(a, r, error_message);
 
    // save file information to archive
    r = archive_write_header(a_new, entry);
    CompressionErrorHandling(a, r, error_message);
 
    // open the input archive and write it in chunks to new archive
    len = archive_read_data(a, buff, sizeof(buff));
    while (len > 0) { // read until archive is empty
      archive_write_data(a_new, buff, len);
      len = archive_read_data(a, buff, sizeof(buff));
    }
 
    // free objects created in this function
    if (!keep) {
      aurostd::RemoveFile(file);
    }
    r = archive_write_free(a_new);
    CompressionErrorHandling(a, r, error_message);
    r = archive_write_free(a);
    CompressionErrorHandling(a, r, error_message);
  }

  void compressfiles2compressfiles(const string& directory, const compression_type ct, const bool keep) {
    const string directory_clean = aurostd::CleanFileName(directory);
    if (directory_clean.empty()) {
      return;
    }
    const string& ext_new = compression_suffix[ct];
    string ext = ext_new;
    for (const std::filesystem::directory_entry& dir_entry : std::filesystem::directory_iterator(directory_clean)) {
      ext = dir_entry.path().extension().string();
      if ((ext == ".bz2" || ext == ".gz" || ext == ".xz" || ext == ".zip" || ext == ".zstd") && (ext != ext_new)) {
        aurostd::compressfile2compressfile(dir_entry.path().string(), ct, keep);
      }
    }
  }
 
} // namespace aurostd
 
#endif //_AUROSTD_XFILE_CPP_