Writing and Reading buffers

v1
Brett 2023-01-29 22:27:48 -05:00
parent 060e1a8efb
commit 38e6abe4e3
7 changed files with 368 additions and 153 deletions

View File

@ -1,5 +1,5 @@
/opt/netbrains/clion/clion-2022.1.3/bin/cmake/linux/bin/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_MAKE_PROGRAM=/opt/netbrains/clion/clion-2022.1.3/bin/ninja/linux/ninja -DBUILD_TESTS:BOOL=ON -G Ninja -S /home/brett/Documents/code/c++/BLT -B /home/brett/Documents/code/c++/BLT/cmake-build-release
Standard Files /home/brett/Documents/code/c++/BLT/src/blt/std/format.cpp;/home/brett/Documents/code/c++/BLT/src/blt/std/logging.cpp;/home/brett/Documents/code/c++/BLT/src/blt/std/system.cpp
Standard Files /home/brett/Documents/code/c++/BLT/src/blt/std/filesystem.cpp;/home/brett/Documents/code/c++/BLT/src/blt/std/format.cpp;/home/brett/Documents/code/c++/BLT/src/blt/std/logging.cpp;/home/brett/Documents/code/c++/BLT/src/blt/std/system.cpp
Profiler Files /home/brett/Documents/code/c++/BLT/src/blt/profiling/profiler.cpp
Source: /home/brett/Documents/code/c++/BLT
Current Source: /home/brett/Documents/code/c++/BLT

View File

@ -49,6 +49,13 @@ cmake_ninja_workdir = /home/brett/Documents/code/c++/BLT/cmake-build-release/
build cmake_object_order_depends_target_BLT: phony || CMakeFiles/BLT.dir
build CMakeFiles/BLT.dir/src/blt/std/filesystem.cpp.o: CXX_COMPILER__BLT_Release /home/brett/Documents/code/c++/BLT/src/blt/std/filesystem.cpp || cmake_object_order_depends_target_BLT
DEP_FILE = CMakeFiles/BLT.dir/src/blt/std/filesystem.cpp.o.d
FLAGS = -O3 -DNDEBUG -fdiagnostics-color=always
INCLUDES = -I/home/brett/Documents/code/c++/BLT/include -I/home/brett/Documents/code/c++/BLT/cmake-build-release/config
OBJECT_DIR = CMakeFiles/BLT.dir
OBJECT_FILE_DIR = CMakeFiles/BLT.dir/src/blt/std
build CMakeFiles/BLT.dir/src/blt/std/format.cpp.o: CXX_COMPILER__BLT_Release /home/brett/Documents/code/c++/BLT/src/blt/std/format.cpp || cmake_object_order_depends_target_BLT
DEP_FILE = CMakeFiles/BLT.dir/src/blt/std/format.cpp.o.d
FLAGS = -O3 -DNDEBUG -fdiagnostics-color=always
@ -99,7 +106,7 @@ build CMakeFiles/BLT.dir/src/blt/nbt/nbt_block.cpp.o: CXX_COMPILER__BLT_Release
#############################################
# Link the static library libBLT.a
build libBLT.a: CXX_STATIC_LIBRARY_LINKER__BLT_Release CMakeFiles/BLT.dir/src/blt/std/format.cpp.o CMakeFiles/BLT.dir/src/blt/std/logging.cpp.o CMakeFiles/BLT.dir/src/blt/std/system.cpp.o CMakeFiles/BLT.dir/src/blt/profiling/profiler.cpp.o CMakeFiles/BLT.dir/src/blt/nbt/nbt.cpp.o CMakeFiles/BLT.dir/src/blt/nbt/nbt_block.cpp.o
build libBLT.a: CXX_STATIC_LIBRARY_LINKER__BLT_Release CMakeFiles/BLT.dir/src/blt/std/filesystem.cpp.o CMakeFiles/BLT.dir/src/blt/std/format.cpp.o CMakeFiles/BLT.dir/src/blt/std/logging.cpp.o CMakeFiles/BLT.dir/src/blt/std/system.cpp.o CMakeFiles/BLT.dir/src/blt/profiling/profiler.cpp.o CMakeFiles/BLT.dir/src/blt/nbt/nbt.cpp.o CMakeFiles/BLT.dir/src/blt/nbt/nbt_block.cpp.o
LANGUAGE_COMPILE_FLAGS = -O3 -DNDEBUG
OBJECT_DIR = CMakeFiles/BLT.dir
POST_BUILD = :

View File

@ -7,49 +7,19 @@
#ifndef BLT_TESTS_NBT_H
#define BLT_TESTS_NBT_H
#include <fstream>
#include <ios>
#include "blt/std/format.h"
#include "blt/std/filesystem.h"
namespace blt::nbt {
void writeUTF8String(std::fstream& stream, const std::string& str);
std::string readUTF8String(std::fstream& stream);
class NBTByteReader {
public:
virtual void readBytes(char* buffer, size_t bytes) = 0;
};
class NBTByteFStreamReader : public NBTByteReader {
private:
std::fstream& m_stream;
size_t m_bufferSize;
char* m_buffer;
size_t readIndex = 0;
public:
explicit NBTByteFStreamReader(std::fstream& stream, size_t bufferSize):
m_stream(stream), m_bufferSize(bufferSize), m_buffer(new char[bufferSize]) {}
explicit NBTByteFStreamReader(NBTByteFStreamReader& copy) = delete;
explicit NBTByteFStreamReader(NBTByteFStreamReader&& move) = delete;
NBTByteFStreamReader& operator=(const NBTByteFStreamReader& copy) = delete;
void readBytes(char* buffer, size_t bytes) override;
~NBTByteFStreamReader() {
delete[] m_buffer;
}
};
class NBTDecoder {
private:
NBTByteReader* m_reader;
blt::fs::block_reader* m_reader;
public:
explicit NBTDecoder(NBTByteReader* reader): m_reader(reader) {}
explicit NBTDecoder(blt::fs::block_reader* reader): m_reader(reader) {}
};

View File

@ -0,0 +1,146 @@
/*
* Created by Brett on 29/01/23.
* Licensed under GNU General Public License V3.0
* See LICENSE file for license detail
*/
#ifndef BLT_FILESYSTEM_H
#define BLT_FILESYSTEM_H
#include <fstream>
#include <ios>
namespace blt::fs {
/**
* Creates an encapsulation of a T array which will be automatically deleted when this object goes out of scope.
* This is a simple buffer meant to be used only inside of a function and not moved around, with a few minor exceptions.
* @tparam T type that is stored in buffer eg char
*/
template<typename T>
struct scoped_buffer {
T* buffer;
unsigned long size;
explicit scoped_buffer(unsigned long size): size(size) {
buffer = new T[size];
}
scoped_buffer(scoped_buffer& copy) = delete;
scoped_buffer(scoped_buffer&& move) = delete;
scoped_buffer operator=(scoped_buffer& copyAssignment) = delete;
scoped_buffer operator=(scoped_buffer&& moveAssignment) = delete;
inline T& operator[](unsigned long index) const {
return buffer[index];
}
~scoped_buffer() {
delete[] buffer;
}
};
/**
* A simple interface which provides a way of reading the next block of data from a resource.
* The interface provides a single function "read" which will read a specified number of bytes into the buffer.
* The implementation for this could be fstreams, zlib deflate, or any method filesystem access.
* Reading of a large number of bytes ( > block size) is guaranteed to not significantly increase the read time and will likely result in a
* direct passthrough to the underlying system. Small reads will be buffered, hence the name "block" reader.
*/
class block_reader {
protected:
unsigned long m_bufferSize;
public:
explicit block_reader(unsigned long bufferSize): m_bufferSize(bufferSize) {}
/**
* Reads bytes from the internal filesystem implementation
* @param buffer buffer to copy the read bytes into
* @param bytes number of bytes to read
* @return status code. non-zero return codes indicates a failure has occurred.
*/
virtual int read(char* buffer, size_t bytes) = 0;
};
/**
* A buffered block writer without a definite backend implementation. Exactly the same as a block_reader but for writing to the filesystem.
*/
class block_writer {
protected:
unsigned long m_bufferSize;
public:
explicit block_writer(unsigned long bufferSize): m_bufferSize(bufferSize) {}
/**
* Writes the bytes to the filesystem backend implementation
* @param buffer bytes to write
* @param bytes number of bytes to write
* @return non-zero code if failure
*/
virtual int write(char* buffer, size_t bytes) = 0;
/**
* Ensures that the internal buffer is written to the filesystem.
*/
virtual void flush() = 0;
};
/**
* fstream implementation of the block reader.
*/
class fstream_block_reader : public block_reader {
private:
std::fstream& m_stream;
char* m_buffer;
size_t readIndex = 0;
public:
explicit fstream_block_reader(std::fstream& stream, size_t bufferSize):
m_stream(stream), m_buffer(new char[bufferSize]), block_reader(bufferSize) {}
explicit fstream_block_reader(fstream_block_reader& copy) = delete;
explicit fstream_block_reader(fstream_block_reader&& move) = delete;
fstream_block_reader& operator=(const fstream_block_reader& copy) = delete;
fstream_block_reader& operator=(const fstream_block_reader&& move) = delete;
int read(char* buffer, size_t bytes) override;
~fstream_block_reader() {
delete[] m_buffer;
}
};
class fstream_block_writer : public block_writer {
private:
std::fstream& m_stream;
char* m_buffer;
size_t writeIndex = 0;
public:
explicit fstream_block_writer(std::fstream& stream, size_t bufferSize):
m_stream(stream), m_buffer(new char[bufferSize]), block_writer(bufferSize) {}
explicit fstream_block_writer(fstream_block_writer& copy) = delete;
explicit fstream_block_writer(fstream_block_writer&& move) = delete;
fstream_block_writer& operator=(const fstream_block_writer& copy) = delete;
fstream_block_writer& operator=(const fstream_block_writer&& move) = delete;
int write(char* buffer, size_t bytes) override;
void flush() override;
~fstream_block_writer() {
delete[] m_buffer;
}
};
}
#endif //BLT_FILESYSTEM_H

View File

@ -4,7 +4,6 @@
* See LICENSE file for license detail
*/
#include <blt/nbt/nbt.h>
#include <cstring>
namespace blt::nbt {
void writeUTF8String(std::fstream& stream, const std::string& str) {
@ -28,23 +27,4 @@ namespace blt::nbt {
delete[] str.characters;
return std::move(strOut);
}
void NBTByteFStreamReader::readBytes(char* buffer, size_t bytes) {
if (readIndex == 0)
m_stream.read(m_buffer, (long) m_bufferSize);
if (readIndex + bytes >= m_bufferSize) {
// copy out all the data from the current buffer
auto bytesLeft = m_bufferSize - readIndex;
memcpy(buffer, m_buffer + readIndex, bytesLeft);
readIndex = 0;
bytes -= bytesLeft;
// now to prevent large scale reading in small blocks, we should just read the entire thing into the buffer.
m_stream.read(buffer + bytesLeft, (long) bytes);
} else {
// but in the case that the size of the data read is small, we should read in blocks and copy from that buffer
// that should be quicker since file operations are slow.
std::memcpy(buffer, m_buffer + readIndex, bytes);
readIndex += bytes;
}
}
}

View File

@ -0,0 +1,46 @@
/*
* Created by Brett on 29/01/23.
* Licensed under GNU General Public License V3.0
* See LICENSE file for license detail
*/
#include <blt/std/filesystem.h>
#include <cstring>
int blt::fs::fstream_block_reader::read(char* buffer, size_t bytes) {
if (readIndex == 0)
m_stream.read(m_buffer, (long) m_bufferSize);
if (readIndex + bytes >= m_bufferSize) {
// copy out all the data from the current buffer
auto bytesLeft = m_bufferSize - readIndex;
memcpy(buffer, m_buffer + readIndex, bytesLeft);
readIndex = 0;
// now to prevent large scale reading in small blocks, we should just read the entire thing into the buffer.
m_stream.read(buffer + bytesLeft, (long) (bytes - bytesLeft));
} else {
// but in the case that the size of the data read is small, we should read in blocks and copy from that buffer
// that should be quicker since file operations are slow.
std::memcpy(buffer, m_buffer + readIndex, bytes);
readIndex += bytes;
}
return 0;
}
int blt::fs::fstream_block_writer::write(char* buffer, size_t bytes) {
if (writeIndex + bytes >= m_bufferSize) {
// in an attempt to stay efficient we write out the old buffer and the new buffer
// since there is a good chance there is more than a buffer's worth of data being written
// otherwise the buffer is almost full and can be written anyway. (this might be bad for performance especially if the FS wants round numbers)
m_stream.write(m_buffer, (long) writeIndex);
writeIndex = 0;
m_stream.write(buffer, (long) bytes);
} else {
std::memcpy(m_buffer + writeIndex, buffer, bytes);
writeIndex += bytes;
}
return 0;
}
void blt::fs::fstream_block_writer::flush() {
m_stream.write(m_buffer, (long) writeIndex);
writeIndex = 0;
}

View File

@ -11,27 +11,14 @@
#include <blt/profiling/profiler.h>
#include <blt/std/logging.h>
#include <blt/std/format.h>
#include <blt/std/filesystem.h>
struct nbt_scoped_buffer {
char* buffer;
size_t bufferSize;
explicit nbt_scoped_buffer(size_t bufferSize): bufferSize(bufferSize) {
buffer = new char[bufferSize];
}
~nbt_scoped_buffer(){
delete[] buffer;
}
inline char operator[](size_t index) const {
return buffer[index];
}
};
inline bool readLargeBlockUsingNBTBufferedReader(const std::string& file, const nbt_scoped_buffer& bufferToCompare, size_t bufferSize) {
nbt_scoped_buffer read_buffer{bufferToCompare.bufferSize};
inline bool readLargeBlockUsingNBTBufferedReader(const std::string& file, const blt::fs::scoped_buffer<char>& bufferToCompare, size_t bufferSize) {
blt::fs::scoped_buffer<char> read_buffer{bufferToCompare.size};
std::fstream largeBlockInputLarge(file, std::ios::in | std::ios::binary);
blt::nbt::NBTByteFStreamReader byteLargeBlockInputLarge(largeBlockInputLarge, bufferSize);
byteLargeBlockInputLarge.readBytes(read_buffer.buffer, bufferToCompare.bufferSize);
for (int i = 0; i < bufferToCompare.bufferSize; i++) {
blt::fs::fstream_block_reader byteLargeBlockInputLarge(largeBlockInputLarge, bufferSize);
byteLargeBlockInputLarge.read(read_buffer.buffer, bufferToCompare.size);
for (int i = 0; i < bufferToCompare.size; i++) {
if (read_buffer[i] != bufferToCompare.buffer[i])
return false;
}
@ -39,6 +26,162 @@ inline bool readLargeBlockUsingNBTBufferedReader(const std::string& file, const
return true;
}
inline bool readIndividualUsingNBTBufferedReader(const std::string& file, const blt::fs::scoped_buffer<char>& bufferToCompare, size_t bufferSize) {
std::fstream largeBlockInput(file, std::ios::in | std::ios::binary);
blt::fs::fstream_block_reader byteLargeBlockInput(largeBlockInput, bufferSize);
for (int i = 0; i < bufferToCompare.size; i++) {
char byte;
byteLargeBlockInput.read(&byte, 1);
if (byte != bufferToCompare[i]) {
return false;
}
}
largeBlockInput.close();
return true;
}
inline void nbt_read_tests(){
constexpr auto bufferSize = 1024 * 1024;
blt::fs::scoped_buffer<char> buffer{bufferSize};
char* read_buffer = new char[bufferSize];
char* read_block_buffer = new char[bufferSize];
bool fstream_indv_correct = true;
bool fstream_large_correct = true;
for (int i = 0; i < bufferSize; i++)
buffer.buffer[i] = i+1;
BLT_START_INTERVAL("nbt read", "Raw Write");
std::fstream largeOutput("HeyThere.txt", std::ios::out | std::ios::binary);
largeOutput.write(buffer.buffer, bufferSize);
largeOutput.flush();
largeOutput.close();
BLT_END_INTERVAL("nbt read", "Raw Write");
BLT_START_INTERVAL("nbt read", "Raw Read Individual");
std::fstream largeInput("HeyThere.txt", std::ios::in | std::ios::binary);
for (int i = 0; i < bufferSize; i++) {
char byte;
largeInput.read(&byte, 1);
if (byte != buffer[i]) {
fstream_indv_correct = false;
break;
}
}
largeInput.close();
BLT_END_INTERVAL("nbt read", "Raw Read Individual");
BLT_START_INTERVAL("nbt read", "Raw Read Large");
std::fstream largeInputLarge("HeyThere.txt", std::ios::in | std::ios::binary);
largeInputLarge.read(read_buffer, bufferSize);
for (int i = 0; i < bufferSize; i++) {
if (read_buffer[i] != buffer[i])
fstream_large_correct = false;
}
largeInputLarge.close();
BLT_END_INTERVAL("nbt read", "Raw Read Large");
BLT_INFO("FStream Read Correctly? %s;", fstream_indv_correct ? "True" : "False");
BLT_INFO("FStream Large Read Correctly? %s;", fstream_large_correct ? "True" : "False");
for (int i = 0; i < 8; i++) {
auto size = (size_t) std::pow(2, 11 + i);
auto size_str = std::to_string(size);
auto profiler_string = "Block Read Individual " + size_str;
bool nbt_block_correct = true;
BLT_START_INTERVAL("nbt read individual", profiler_string);
nbt_block_correct = readIndividualUsingNBTBufferedReader("HeyThere.txt", buffer, size);
BLT_END_INTERVAL("nbt read individual", profiler_string);
BLT_INFO("NBT Individual Block %s Stream Correctly? %s;\n", size_str.c_str(), nbt_block_correct ? "True" : "False");
}
for (int i = 0; i < 8; i++) {
auto size = (size_t) std::pow(2, 11 + i);
auto size_str = std::to_string(size);
auto profiler_string = "Block Read " + size_str;
bool nbt_block_large_correct = true;
BLT_START_INTERVAL("nbt read block", profiler_string);
nbt_block_large_correct = readLargeBlockUsingNBTBufferedReader("HeyThere.txt", buffer, size);
BLT_END_INTERVAL("nbt read block", profiler_string);
BLT_INFO("NBT Block %s Stream Correctly? %s;\n", size_str.c_str(), nbt_block_large_correct ? "True" : "False");
}
BLT_PRINT_ORDERED("nbt read");
BLT_TRACE("");
BLT_PRINT_ORDERED("nbt read block");
BLT_TRACE("");
BLT_PRINT_ORDERED("nbt read individual");
delete[] read_buffer;
delete[] read_block_buffer;
}
inline void nbt_write_tests(){
constexpr auto bufferSize = 1024 * 1024;
blt::fs::scoped_buffer<char> buffer{bufferSize};
blt::fs::scoped_buffer<char> read_buffer{bufferSize};
for (int i = 0; i < bufferSize; i++)
buffer.buffer[i] = i+1;
std::fstream fileOutput("IAmAFile.txt", std::ios::binary | std::ios::out);
for (int i = 0; i < 8; i++) {
auto size = (size_t) std::pow(2, 11 + i);
auto size_str = std::to_string(size);
auto profiler_string = "Writer " + size_str;
blt::fs::fstream_block_writer writer(fileOutput, size);
BLT_START_INTERVAL("nbt write block", profiler_string);
writer.write(buffer.buffer, buffer.size);
BLT_END_INTERVAL("nbt write block", profiler_string);
BLT_START_INTERVAL("nbt write individual", profiler_string);
for (int j = 0; j < bufferSize; j++) {
writer.write(&buffer.buffer[j], 1);
}
BLT_END_INTERVAL("nbt write individual", profiler_string);
}
fileOutput.flush();
std::fstream fileInput("IAmAFile.txt", std::ios::binary | std::ios::in);
for (int i = 0; i < 8; i++) {
auto size = (size_t) std::pow(2, 11 + i);
auto size_str = std::to_string(size);
bool results = true;
fileInput.read(read_buffer.buffer, bufferSize);
for (int j = 0; j < bufferSize; j++) {
if (buffer[j] != read_buffer[j]) {
results = false;
BLT_FATAL("Error occurred at size %d and index %d", size, j);
break;
}
}
BLT_INFO("NBT %s Block Write Correctly? %s;\n", size_str.c_str(), results ? "True" : "False");
results = true;
fileInput.read(read_buffer.buffer, bufferSize);
for (int j = 0; j < bufferSize; j++) {
if (buffer[j] != read_buffer[j]) {
results = false;
BLT_FATAL("Error occurred at size %d and index %d", size, j);
break;
}
}
BLT_INFO("NBT %s Individual Write Correctly? %s;\n", size_str.c_str(), results ? "True" : "False");
}
BLT_PRINT_ORDERED("nbt write individual");
BLT_TRACE("");
BLT_PRINT_ORDERED("nbt write block");
}
inline void nbt_tests(){
std::fstream testOutput("Hello.txt", std::ios::out | std::ios::binary);
@ -69,85 +212,8 @@ inline void nbt_tests(){
testInput.close();
BLT_INFO("%d, %c, %d, %d, %d, %s", testByteIn[0], testByteIn[1], testByteIn[2], testShortIn, testIntIn, strIn.c_str());
constexpr auto bufferSize = 1024 * 128;
nbt_scoped_buffer buffer{bufferSize};
char* read_buffer = new char[bufferSize];
char* read_block_buffer = new char[bufferSize];
bool fstream_indv_correct = true;
bool fstream_large_correct = true;
bool nbt_block_indv_correct = true;
for (int i = 0; i < bufferSize; i++)
buffer.buffer[i] = i+1;
BLT_START_INTERVAL("nbt", "Raw Write");
std::fstream largeOutput("HeyThere.txt", std::ios::out | std::ios::binary);
largeOutput.write(buffer.buffer, bufferSize);
largeOutput.flush();
largeOutput.close();
BLT_END_INTERVAL("nbt", "Raw Write");
BLT_START_INTERVAL("nbt", "Raw Read Individual");
std::fstream largeInput("HeyThere.txt", std::ios::in | std::ios::binary);
for (int i = 0; i < bufferSize; i++) {
char byte;
largeInput.read(&byte, 1);
if (byte != buffer[i]) {
fstream_indv_correct = false;
break;
}
}
largeInput.close();
BLT_END_INTERVAL("nbt", "Raw Read Individual");
BLT_START_INTERVAL("nbt", "Block Read Individual");
std::fstream largeBlockInput("HeyThere.txt", std::ios::in | std::ios::binary);
blt::nbt::NBTByteFStreamReader byteLargeBlockInput(largeBlockInput, 1024 * 8);
for (int i = 0; i < bufferSize; i++) {
char byte;
byteLargeBlockInput.readBytes(&byte, 1);
if (byte != buffer[i]) {
nbt_block_indv_correct = false;
break;
}
}
largeBlockInput.close();
BLT_END_INTERVAL("nbt", "Block Read Individual");
BLT_START_INTERVAL("nbt", "Raw Read Large");
std::fstream largeInputLarge("HeyThere.txt", std::ios::in | std::ios::binary);
largeInputLarge.read(read_buffer, bufferSize);
for (int i = 0; i < bufferSize; i++) {
if (read_buffer[i] != buffer[i])
fstream_large_correct = false;
}
largeInputLarge.close();
BLT_END_INTERVAL("nbt", "Raw Read Large");
BLT_INFO("FStream Read Correctly? %s;", fstream_indv_correct ? "True" : "False");
BLT_INFO("FStream Large Read Correctly? %s;", fstream_large_correct ? "True" : "False");
BLT_INFO("NBT Block Stream Correctly? %s;\n", nbt_block_indv_correct ? "True" : "False");
for (int i = 0; i < 8; i++) {
auto size = (size_t) std::pow(2, 11 + i);
auto size_str = std::to_string(size);
auto profiler_string = "Block Read " + size_str;
bool nbt_block_large_correct = true;
BLT_START_INTERVAL("nbt", profiler_string);
nbt_block_large_correct = readLargeBlockUsingNBTBufferedReader("HeyThere.txt", buffer, size);
BLT_END_INTERVAL("nbt", profiler_string);
BLT_INFO("NBT Block %s Stream Correctly? %s;\n", size_str.c_str(), nbt_block_large_correct ? "True" : "False");
}
BLT_PRINT_ORDERED("nbt");
delete[] read_buffer;
delete[] read_block_buffer;
nbt_read_tests();
nbt_write_tests();
}