Copy a file in a sane, safe and efficient way

Question

I search for a good way to copy a file (binary or text). I've written several samples, everyone works. But I want hear the opinion of seasoned programmers.

I missing good examples and search a way which works with C++.

ANSI-C-WAY

#include <iostream>
#include <cstdio>    // fopen, fclose, fread, fwrite, BUFSIZ
#include <ctime>
using namespace std;

int main() {
    clock_t start, end;
    start = clock();

    // BUFSIZE default is 8192 bytes
    // BUFSIZE of 1 means one chareter at time
    // good values should fit to blocksize, like 1024 or 4096
    // higher values reduce number of system calls
    // size_t BUFFER_SIZE = 4096;

    char buf[BUFSIZ];
    size_t size;

    FILE* source = fopen("from.ogv", "rb");
    FILE* dest = fopen("to.ogv", "wb");

    // clean and more secure
    // feof(FILE* stream) returns non-zero if the end of file indicator for stream is set

    while (size = fread(buf, 1, BUFSIZ, source)) {
        fwrite(buf, 1, size, dest);
    }

    fclose(source);
    fclose(dest);

    end = clock();

    cout << "CLOCKS_PER_SEC " << CLOCKS_PER_SEC << "\n";
    cout << "CPU-TIME START " << start << "\n";
    cout << "CPU-TIME END " << end << "\n";
    cout << "CPU-TIME END - START " << end - start << "\n";
    cout << "TIME(SEC) " << static_cast<double>(end - start) / CLOCKS_PER_SEC << "\n";

    return 0;
}

POSIX-WAY (K&R use this in "The C programming language", more low-level)

#include <iostream>
#include <fcntl.h>   // open
#include <unistd.h>  // read, write, close
#include <cstdio>    // BUFSIZ
#include <ctime>
using namespace std;

int main() {
    clock_t start, end;
    start = clock();

    // BUFSIZE defaults to 8192
    // BUFSIZE of 1 means one chareter at time
    // good values should fit to blocksize, like 1024 or 4096
    // higher values reduce number of system calls
    // size_t BUFFER_SIZE = 4096;

    char buf[BUFSIZ];
    size_t size;

    int source = open("from.ogv", O_RDONLY, 0);
    int dest = open("to.ogv", O_WRONLY | O_CREAT /*| O_TRUNC/**/, 0644);

    while ((size = read(source, buf, BUFSIZ)) > 0) {
        write(dest, buf, size);
    }

    close(source);
    close(dest);

    end = clock();

    cout << "CLOCKS_PER_SEC " << CLOCKS_PER_SEC << "\n";
    cout << "CPU-TIME START " << start << "\n";
    cout << "CPU-TIME END " << end << "\n";
    cout << "CPU-TIME END - START " << end - start << "\n";
    cout << "TIME(SEC) " << static_cast<double>(end - start) / CLOCKS_PER_SEC << "\n";

    return 0;
}

KISS-C++-Streambuffer-WAY

#include <iostream>
#include <fstream>
#include <ctime>
using namespace std;

int main() {
    clock_t start, end;
    start = clock();

    ifstream source("from.ogv", ios::binary);
    ofstream dest("to.ogv", ios::binary);

    dest << source.rdbuf();

    source.close();
    dest.close();

    end = clock();

    cout << "CLOCKS_PER_SEC " << CLOCKS_PER_SEC << "\n";
    cout << "CPU-TIME START " << start << "\n";
    cout << "CPU-TIME END " << end << "\n";
    cout << "CPU-TIME END - START " <<  end - start << "\n";
    cout << "TIME(SEC) " << static_cast<double>(end - start) / CLOCKS_PER_SEC << "\n";

    return 0;
}

COPY-ALGORITHM-C++-WAY

#include <iostream>
#include <fstream>
#include <ctime>
#include <algorithm>
#include <iterator>
using namespace std;

int main() {
    clock_t start, end;
    start = clock();

    ifstream source("from.ogv", ios::binary);
    ofstream dest("to.ogv", ios::binary);

    istreambuf_iterator<char> begin_source(source);
    istreambuf_iterator<char> end_source;
    ostreambuf_iterator<char> begin_dest(dest); 
    copy(begin_source, end_source, begin_dest);

    source.close();
    dest.close();

    end = clock();

    cout << "CLOCKS_PER_SEC " << CLOCKS_PER_SEC << "\n";
    cout << "CPU-TIME START " << start << "\n";
    cout << "CPU-TIME END " << end << "\n";
    cout << "CPU-TIME END - START " <<  end - start << "\n";
    cout << "TIME(SEC) " << static_cast<double>(end - start) / CLOCKS_PER_SEC << "\n";

    return 0;
}

OWN-BUFFER-C++-WAY

#include <iostream>
#include <fstream>
#include <ctime>
using namespace std;

int main() {
    clock_t start, end;
    start = clock();

    ifstream source("from.ogv", ios::binary);
    ofstream dest("to.ogv", ios::binary);

    // file size
    source.seekg(0, ios::end);
    ifstream::pos_type size = source.tellg();
    source.seekg(0);
    // allocate memory for buffer
    char* buffer = new char[size];

    // copy file    
    source.read(buffer, size);
    dest.write(buffer, size);

    // clean up
    delete[] buffer;
    source.close();
    dest.close();

    end = clock();

    cout << "CLOCKS_PER_SEC " << CLOCKS_PER_SEC << "\n";
    cout << "CPU-TIME START " << start << "\n";
    cout << "CPU-TIME END " << end << "\n";
    cout << "CPU-TIME END - START " <<  end - start << "\n";
    cout << "TIME(SEC) " << static_cast<double>(end - start) / CLOCKS_PER_SEC << "\n";

    return 0;
}

LINUX-WAY // requires kernel >= 2.6.33

#include <iostream>
#include <sys/sendfile.h>  // sendfile
#include <fcntl.h>         // open
#include <unistd.h>        // close
#include <sys/stat.h>      // fstat
#include <sys/types.h>     // fstat
#include <ctime>
using namespace std;

int main() {
    clock_t start, end;
    start = clock();

    int source = open("from.ogv", O_RDONLY, 0);
    int dest = open("to.ogv", O_WRONLY | O_CREAT /*| O_TRUNC/**/, 0644);

    // struct required, rationale: function stat() exists also
    struct stat stat_source;
    fstat(source, &stat_source);

    sendfile(dest, source, 0, stat_source.st_size);

    close(source);
    close(dest);

    end = clock();

    cout << "CLOCKS_PER_SEC " << CLOCKS_PER_SEC << "\n";
    cout << "CPU-TIME START " << start << "\n";
    cout << "CPU-TIME END " << end << "\n";
    cout << "CPU-TIME END - START " <<  end - start << "\n";
    cout << "TIME(SEC) " << static_cast<double>(end - start) / CLOCKS_PER_SEC << "\n";

    return 0;
}

Environment

• GNU/LINUX (Archlinux)
• Kernel 3.3
• GLIBC-2.15, LIBSTDC++ 4.7 (GCC-LIBS), GCC 4.7, Coreutils 8.16
• Using RUNLEVEL 3 (Multiuser, Network, Terminal, no GUI)
• INTEL SSD-Postville 80 GB, filled up to 50%
• Copy a 270 MB OGG-VIDEO-FILE

Steps to reproduce

 1. $ rm from.ogg
 2. $ reboot                           # kernel and filesystem buffers are in regular
 3. $ (time ./program) &>> report.txt  # executes program, redirects output of program and append to file
 4. $ sha256sum *.ogv                  # checksum
 5. $ rm to.ogg                        # remove copy, but no sync, kernel and fileystem buffers are used
 6. $ (time ./program) &>> report.txt  # executes program, redirects output of program and append to file

Results (CPU TIME used)

Program  Description                 UNBUFFERED|BUFFERED
ANSI C   (fread/frwite)                 490,000|260,000  
POSIX    (K&R, read/write)              450,000|230,000  
FSTREAM  (KISS, Streambuffer)           500,000|270,000 
FSTREAM  (Algorithm, copy)              500,000|270,000
FSTREAM  (OWN-BUFFER)                   500,000|340,000  
SENDFILE (native LINUX, sendfile)       410,000|200,000  

Filesize doesn't change.
sha256sum print the same results.
The video file is still playable.

Questions

• What method would you prefer?
• Do you know better solutions?
• Do you see any mistakes in my code?

• Do you know a reason to avoid a solution?

• FSTREAM (KISS, Streambuffer)
  I really like this one, because it is really short and simple. As far is I know the operator << is overloaded for rdbuf() and doesn't convert anything. Correct?

Thanks

Update 1
I changed the source in all samples in that way, that the open and close of the file descriptors is include in the measurement of clock(). Their are no other significant changes in the source code. The results doesn't changed! I also used time to double-check my results.

Update 2
ANSI C sample changed: The condition of the while-loop doesn't call any longer feof() instead I moved fread() into the condition. It looks like, the code runs now 10,000 clocks faster.

Measurement changed: The former results were always buffered, because I repeated the old command line rm to.ogv && sync && time ./program for each program a few times. Now I reboot the system for every program. The unbuffered results are new and show no surprise. The unbuffered results didn't changed really.

If i don't delete the old copy, the programs react different. Overwriting a existing file buffered is faster with POSIX and SENDFILE, all other programs are slower. Maybe the options truncate or create have a impact on this behaviour. But overwriting existing files with the same copy is not a real world use-case.

Performing the copy with cp takes 0.44 seconds unbuffered und 0.30 seconds buffered. So cp is a little bit slower than the POSIX sample. Looks fine for me.

Maybe I add also samples and results of mmap() and copy_file() from boost::filesystem.

Update 3
I've put this also on a blog page and extended it a little bit. Including splice(), which is a low-level function from the Linux kernel. Maybe more samples with Java will follow. http://www.ttyhoney.com/blog/?page_id=69

Answer 1

Copy a file in a sane way:

#include <fstream>

int main()
{
    std::ifstream  src("from.ogv", std::ios::binary);
    std::ofstream  dst("to.ogv",   std::ios::binary);

    dst << src.rdbuf();
}

This is so simple and intuitive to read it is worth the extra cost. If we were doing it a lot, better to fall back on OS calls to the file system. I am sure boost has a copy file method in its filesystem class.

There is a C method for interacting with the file system:

#include <copyfile.h>

int
copyfile(const char *from, const char *to, copyfile_state_t state, copyfile_flags_t flags);

Answer 2

With C++17 the standard way to copy a file will be including the <filesystem> header and using:

bool copy_file( const std::filesystem::path& from,
                const std::filesystem::path& to);

bool copy_file( const std::filesystem::path& from,
                const std::filesystem::path& to,
                std::filesystem::copy_options options);

The first form is equivalent to the second one with copy_options::none used as options (see also copy_file).

The filesystem library was originally developed as boost.filesystem and finally merged to ISO C++ as of C++17.

Answer 3

Too many!

The "ANSI C" way buffer is redundant, since a FILE is already buffered. (The size of this internal buffer is what BUFSIZ actually defines.)

The "OWN-BUFFER-C++-WAY" will be slow as it goes through fstream, which does a lot of virtual dispatching, and again maintains internal buffers or each stream object. (The "COPY-ALGORITHM-C++-WAY" does not suffer this, as the streambuf_iterator class bypasses the stream layer.)

I prefer the "COPY-ALGORITHM-C++-WAY", but without constructing an fstream, just create bare std::filebuf instances when no actual formatting is needed.

For raw performance, you can't beat POSIX file descriptors. It's ugly but portable and fast on any platform.

The Linux way appears to be incredibly fast — perhaps the OS let the function return before I/O was finished? In any case, that's not portable enough for many applications.

EDIT: Ah, "native Linux" may be improving performance by interleaving reads and writes with asynchronous I/O. Letting commands pile up can help the disk driver decide when is best to seek. You might try Boost Asio or pthreads for comparison. As for "can't beat POSIX file descriptors"… well that's true if you're doing anything with the data, not just blindly copying.

Answer 4

I want to make the very important note that the LINUX method using sendfile() has a major problem in that it can not copy files more than 2GB in size! I had implemented it following this question and was hitting problems because I was using it to copy HDF5 files that were many GB in size.

http://man7.org/linux/man-pages/man2/sendfile.2.html

sendfile() will transfer at most 0x7ffff000 (2,147,479,552) bytes, returning the number of bytes actually transferred. (This is true on both 32-bit and 64-bit systems.)

Answer 5

Qt has a method for copying files:

#include <QFile>
QFile::copy("originalFile.example","copiedFile.example");

Note that to use this you have to install Qt (instructions here) and include it in your project (if you're using Windows and you're not an administrator, you can download Qt here instead). Also see this answer.

Answer 6

For those who like boost:

boost::filesystem::path mySourcePath("foo.bar");
boost::filesystem::path myTargetPath("bar.foo");

// Variant 1: Overwrite existing
boost::filesystem::copy_file(mySourcePath, myTargetPath, boost::filesystem::copy_option::overwrite_if_exists);

// Variant 2: Fail if exists
boost::filesystem::copy_file(mySourcePath, myTargetPath, boost::filesystem::copy_option::fail_if_exists);

Note that boost::filesystem::path is also available as wpath for Unicode. And that you could also use

using namespace boost::filesystem

if you do not like those long type names

Answer 7

I'm not quite sure what a "good way" of copying a file is, but assuming "good" means "fast", I could broaden the subject a little.

Current operating systems have long been optimized to deal with run of the mill file copy. No clever bit of code will beat that. It is possible that some variant of your copy techniques will prove faster in some test scenario, but they most likely would fare worse in other cases.

Typically, the sendfile function probably returns before the write has been committed, thus giving the impression of being faster than the rest. I haven't read the code, but it is most certainly because it allocates its own dedicated buffer, trading memory for time. And the reason why it won't work for files bigger than 2Gb.

As long as you're dealing with a small number of files, everything occurs inside various buffers (the C++ runtime's first if you use iostream, the OS internal ones, apparently a file-sized extra buffer in the case of sendfile). Actual storage media is only accessed once enough data has been moved around to be worth the trouble of spinning a hard disk.

I suppose you could slightly improve performances in specific cases. Off the top of my head:

• If you're copying a huge file on the same disk, using a buffer bigger than the OS's might improve things a bit (but we're probably talking about gigabytes here).
• If you want to copy the same file on two different physical destinations you will probably be faster opening the three files at once than calling two copy_file sequentially (though you'll hardly notice the difference as long as the file fits in the OS cache)
• If you're dealing with lots of tiny files on an HDD you might want to read them in batches to minimize seeking time (though the OS already caches directory entries to avoid seeking like crazy and tiny files will likely reduce disk bandwidth dramatically anyway).

But all that is outside the scope of a general purpose file copy function.

So in my arguably seasoned programmer's opinion, a C++ file copy should just use the C++17 file_copy dedicated function, unless more is known about the context where the file copy occurs and some clever strategies can be devised to outsmart the OS.