diff --git a/src/client/FileChunkingWriter.java b/src/client/ChunkedCompressedChecksumFileWriter.java similarity index 58% rename from src/client/FileChunkingWriter.java rename to src/client/ChunkedCompressedChecksumFileWriter.java index cd9844a..c16eecc 100644 --- a/src/client/FileChunkingWriter.java +++ b/src/client/ChunkedCompressedChecksumFileWriter.java @@ -8,18 +8,18 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; -public class FileChunkingWriter { +public class ChunkedCompressedChecksumFileWriter { - private final DataOutputStream writer; + private final DataOutputStream networkStreamWriter; private final StreamingXXHash64 streamHash; - private final DataInputStream fileReader; + private final DataInputStream fileInputReader; private final int bufferSize; private final long seed; - public FileChunkingWriter(DataOutputStream writer, DataInputStream fileReader, int bufferSize, long seed){ - this.writer = writer; + public ChunkedCompressedChecksumFileWriter(DataOutputStream networkStreamWriter, DataInputStream fileInputReader, int bufferSize, long seed){ + this.networkStreamWriter = networkStreamWriter; this.streamHash = FileUtil.XX_HASH_FACTORY.newStreamingHash64(seed); - this.fileReader = fileReader; + this.fileInputReader = fileInputReader; this.bufferSize = bufferSize; this.seed = seed; } @@ -37,23 +37,27 @@ public class FileChunkingWriter { ArrayData compressed = compress(uncompressed); // write data - writer.writeInt(uncompressed.length); - writer.writeInt(compressed.getActualLength()); - writer.writeLong(hash); - writer.write(compressed.getData(), 0, compressed.getActualLength()); - writer.flush(); + writeHeader(uncompressed.length, compressed.getActualLength(), hash); + networkStreamWriter.write(compressed.getData(), 0, compressed.getActualLength()); + networkStreamWriter.flush(); } public void close() throws IOException { - writer.writeInt(0); - writer.writeLong(streamHash.getValue()); - writer.flush(); + networkStreamWriter.writeInt(0); + networkStreamWriter.writeLong(streamHash.getValue()); + networkStreamWriter.flush(); + } + + private void writeHeader(int uncompressed, int compressed, long hash) throws IOException { + networkStreamWriter.writeInt(uncompressed); + networkStreamWriter.writeInt(compressed); + networkStreamWriter.writeLong(hash); } private byte[] readSome() throws IOException { - byte[] readBytes = new byte[Integer.min(fileReader.available(), bufferSize)]; + byte[] readBytes = new byte[Integer.min(fileInputReader.available(), bufferSize)]; - int totalRead = fileReader.read(readBytes); + int totalRead = fileInputReader.read(readBytes); assert(readBytes.length == totalRead); return readBytes; diff --git a/src/client/Client.java b/src/client/Client.java index 72fe0eb..26d4852 100644 --- a/src/client/Client.java +++ b/src/client/Client.java @@ -1,12 +1,8 @@ package client; -import net.jpountz.lz4.LZ4BlockInputStream; -import net.jpountz.lz4.LZ4BlockOutputStream; -import net.jpountz.lz4.LZ4FrameInputStream; -import net.jpountz.lz4.LZ4FrameOutputStream; import server.Server; import shared.ExceptionLogger; -import shared.FileHeader; +import shared.FileUtil; import java.io.*; import java.net.Socket; @@ -14,12 +10,6 @@ import java.util.*; public class Client { - public static class ClientInvalidUsageException extends RuntimeException { - public ClientInvalidUsageException(String str){ - super(str); - } - } - private final Socket serverConnection; private final DataOutputStream out; private final DataInputStream in; @@ -30,19 +20,13 @@ public class Client { in = new DataInputStream(new BufferedInputStream(serverConnection.getInputStream())); } - void sendFile(String path){ - if (new File(path).isDirectory()) - throw new ClientInvalidUsageException("Unable to send directory. Did you mean sendDir()?"); + public Client sendFile(String path){ System.out.println("Sending path " + path); - new FileHeader(path).write(out); - try { - out.flush(); - } catch (IOException e) { - ExceptionLogger.log(e); - } + FileUtil.write(path, out); + return this; } - void sendDir(String path){ + public Client sendDir(String path){ File p = new File(path); ArrayDeque filesToCheck = new ArrayDeque<>(Arrays.asList(Objects.requireNonNull(p.listFiles()))); while (!filesToCheck.isEmpty()) { @@ -52,9 +36,10 @@ public class Client { } else sendFile(f.getPath()); } + return this; } - void close(){ + public void close(){ try { in.close(); out.close(); @@ -66,7 +51,8 @@ public class Client { public static void main(String[] args) { try { - new Client("localhost", Server.SERVER_PORT).sendDir("in/"); + new Client("localhost", Server.SERVER_PORT).sendDir("in/").close(); + //new Client("localhost", Server.SERVER_PORT).sendFile("in/ihaveafile.txt").close(); } catch (Exception e){ ExceptionLogger.log(e); } diff --git a/src/server/ChunkedCompressedChecksumFileReader.java b/src/server/ChunkedCompressedChecksumFileReader.java new file mode 100644 index 0000000..1902fcb --- /dev/null +++ b/src/server/ChunkedCompressedChecksumFileReader.java @@ -0,0 +1,69 @@ +package server; + +import net.jpountz.xxhash.StreamingXXHash64; +import shared.ArrayData; +import shared.FileUtil; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Paths; + +public class ChunkedCompressedChecksumFileReader { + + private final DataInputStream networkStreamReader; + private final StreamingXXHash64 streamHash; + private final DataOutputStream fileOutputWriter; + private final long seed; + + public ChunkedCompressedChecksumFileReader(DataInputStream networkStreamReader, String fileOutputPath, long seed) throws IOException { + this.networkStreamReader = networkStreamReader; + this.streamHash = FileUtil.XX_HASH_FACTORY.newStreamingHash64(seed); + this.fileOutputWriter = new DataOutputStream(new BufferedOutputStream(Files.newOutputStream(Paths.get(fileOutputPath)))); + this.seed = seed; + } + + public FileHeader readChunk() throws IOException { + FileHeader header = readHeader(); + if (header.getUncompressed() == 0) + return header; + byte[] data = readSome(header); + byte[] decompressed = decompress(header, data); + hash(header, decompressed); + fileOutputWriter.write(decompressed, 0, decompressed.length); + return header; + } + + public void close() throws IOException { + long streamHash = networkStreamReader.readLong(); + if (streamHash != this.streamHash.getValue()) + throw new RuntimeException("Stream total hash doesn't match the client's sent hash!"); + fileOutputWriter.flush(); + fileOutputWriter.close(); + } + + private FileHeader readHeader() throws IOException { + return new FileHeader().read(networkStreamReader); + } + + private byte[] readSome(FileHeader header) throws IOException { + byte[] data = new byte[header.getCompressed()]; + int amount = networkStreamReader.read(data, 0, header.getCompressed()); + assert (header.getCompressed() == amount); + return data; + } + + private byte[] decompress(FileHeader header, byte[] data) { + byte[] restored = new byte[header.getUncompressed()]; + int len = FileUtil.DECOMPRESSOR.decompress(data, 0, restored, 0, header.getUncompressed()); + assert (header.getUncompressed() == len); + return restored; + } + + private void hash(FileHeader header, byte[] data) { + long computedHash = FileUtil.HASH_64.hash(data, 0, data.length, seed); + streamHash.update(data, 0, data.length); + if (computedHash != header.getHash()) + throw new RuntimeException("Computed hash doesn't match sent hash! File corrupted?"); + } + +} diff --git a/src/server/Connection.java b/src/server/Connection.java index d9ebe67..785ec85 100644 --- a/src/server/Connection.java +++ b/src/server/Connection.java @@ -1,11 +1,7 @@ package server; -import net.jpountz.lz4.LZ4BlockInputStream; -import net.jpountz.lz4.LZ4BlockOutputStream; -import net.jpountz.lz4.LZ4FrameInputStream; -import net.jpountz.lz4.LZ4FrameOutputStream; import shared.ExceptionLogger; -import shared.FileHeader; +import shared.FileUtil; import java.io.*; import java.net.Socket; @@ -37,8 +33,8 @@ public class Connection implements Runnable { if (in.available() > 0) { byte command = in.readByte(); - if (command == FileHeader.COMMAND.WRITE.type) - FileHeader.receive(in); + if (command == FileUtil.COMMAND.WRITE.type) + FileUtil.receive(in); } } catch (IOException e) { throw new RuntimeException(e); diff --git a/src/server/FileHeader.java b/src/server/FileHeader.java new file mode 100644 index 0000000..6119808 --- /dev/null +++ b/src/server/FileHeader.java @@ -0,0 +1,36 @@ +package server; + +import shared.ExceptionLogger; + +import java.io.DataInputStream; +import java.io.IOException; + +public class FileHeader { + + private int uncompressed; + private int compressed; + private long hash; + + public FileHeader() {} + + public FileHeader read(DataInputStream reader) throws IOException{ + uncompressed = reader.readInt(); + if (uncompressed == 0) + return this; + compressed = reader.readInt(); + hash = reader.readLong(); + return this; + } + + public int getUncompressed() { + return uncompressed; + } + + public int getCompressed() { + return compressed; + } + + public long getHash() { + return hash; + } +} diff --git a/src/shared/FileHeader.java b/src/shared/FileHeader.java deleted file mode 100644 index 0422ce3..0000000 --- a/src/shared/FileHeader.java +++ /dev/null @@ -1,111 +0,0 @@ -package shared; - -import client.Client; -import client.FileChunkingWriter; -import net.jpountz.xxhash.StreamingXXHash64; - -import java.io.*; -import java.nio.file.Files; -import java.nio.file.Paths; - -public class FileHeader { - - public enum COMMAND { - WRITE((byte) 1); - public final byte type; - - COMMAND(byte type) { - this.type = type; - } - } - - private final String relative_path; - private final String full_path; - - public FileHeader(String path) { - File pf = new File(path); - if (!pf.exists()) - throw new Client.ClientInvalidUsageException("Unable to send a file which doesn't exist!"); - if (pf.isDirectory()) - throw new Client.ClientInvalidUsageException("Path is a directory unable to send!"); - String workingDirectory = System.getProperty("user.dir"); - this.full_path = path; - this.relative_path = path.replace(workingDirectory, ""); - System.out.println(relative_path); - } - - public void write(DataOutputStream dataOut) { - try { - DataInputStream fileReader = new DataInputStream(new BufferedInputStream(Files.newInputStream(Paths.get(full_path)))); - - dataOut.writeByte(COMMAND.WRITE.type); - dataOut.writeUTF(relative_path); - - FileChunkingWriter writer = new FileChunkingWriter(dataOut, fileReader, FileUtil.READER_SIZE, FileUtil.SEED); - - while (fileReader.available() > 0) - writer.processChunk(); - - writer.close(); - fileReader.close(); - } catch (Exception e) { - ExceptionLogger.log(e); - } - } - - public static void receive(DataInputStream reader) { - try { - String path = createPath(reader.readUTF()); - System.out.println("Writing to file: " + path); - - DataOutputStream writer = new DataOutputStream(new BufferedOutputStream(Files.newOutputStream(Paths.get(path)))); - - StreamingXXHash64 computedStreamHash = FileUtil.XX_HASH_FACTORY.newStreamingHash64(FileUtil.SEED); - while (true) { - int uncompressed_size = reader.readInt(); - - if (uncompressed_size <= 0) - break; - - int compressed_size = reader.readInt(); - long hash = reader.readLong(); - byte[] data = new byte[compressed_size]; - int amount = reader.read(data, 0, compressed_size); - - assert(amount == compressed_size); - - byte[] restored = new byte[uncompressed_size]; - int len = FileUtil.DECOMPRESSOR.decompress(data, 0, restored, 0, uncompressed_size); - - assert(len == uncompressed_size); - - long computedHash = FileUtil.HASH_64.hash(restored, 0, uncompressed_size, FileUtil.SEED); - computedStreamHash.update(restored, 0, uncompressed_size); - - if (hash != computedHash) - throw new RuntimeException(hash + " HELP! " + computedHash); - - writer.write(restored, 0, uncompressed_size); - } - long streamHash = reader.readLong(); - if (computedStreamHash.getValue() != streamHash) - throw new RuntimeException("HELP 22"); - writer.flush(); - writer.close(); - } catch (Exception e) { - ExceptionLogger.log(e); - } - } - - private static String createPath(String userFile) { - String[] pathParts = userFile.split("/"); - String userDirectory = userFile.replace(pathParts[pathParts.length - 1], ""); - - File ld = new File(System.getProperty("user.dir") + "/write/" + userDirectory); - if (!ld.exists() && !ld.mkdirs()) - System.out.println("Failed to create directory"); - - return System.getProperty("user.dir") + "/write/" + userFile; - } - -} diff --git a/src/shared/FileUtil.java b/src/shared/FileUtil.java index 0c9ebdb..7f97730 100644 --- a/src/shared/FileUtil.java +++ b/src/shared/FileUtil.java @@ -1,13 +1,20 @@ package shared; +import client.ChunkedCompressedChecksumFileWriter; import net.jpountz.lz4.LZ4Compressor; import net.jpountz.lz4.LZ4Factory; import net.jpountz.lz4.LZ4FastDecompressor; import net.jpountz.xxhash.XXHash64; import net.jpountz.xxhash.XXHashFactory; +import server.ChunkedCompressedChecksumFileReader; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Paths; public class FileUtil { + // do not change it breaks stuff protected static final int READER_SIZE = 8192; public static final long SEED = 691; @@ -18,5 +25,79 @@ public class FileUtil { public static final XXHashFactory XX_HASH_FACTORY = XXHashFactory.fastestInstance(); public static final XXHash64 HASH_64 = XX_HASH_FACTORY.hash64(); + public enum COMMAND { + WRITE((byte) 1); + public final byte type; + + COMMAND(byte type) { + this.type = type; + } + } + + public static void write(String path, DataOutputStream dataOut) { + validatePath(path); + String relative_path = path.replace(System.getProperty("user.dir"), ""); + try { + DataInputStream fileReader = new DataInputStream(new BufferedInputStream(Files.newInputStream(Paths.get(path)))); + + dataOut.writeByte(COMMAND.WRITE.type); + dataOut.writeUTF(relative_path); + + ChunkedCompressedChecksumFileWriter writer = new ChunkedCompressedChecksumFileWriter(dataOut, fileReader, FileUtil.READER_SIZE, FileUtil.SEED); + + while (fileReader.available() > 0) + writer.processChunk(); + + writer.close(); + fileReader.close(); + } catch (Exception e) { + ExceptionLogger.log(e); + } + } + + public static void receive(DataInputStream dataIn) { + try { + String path = createPath(dataIn.readUTF()); + System.out.println("Writing to file: " + path); + + ChunkedCompressedChecksumFileReader reader = new ChunkedCompressedChecksumFileReader(dataIn, path, FileUtil.SEED); + + // ugh I want while(reader.readChunk().getUncompressed()); but it makes warnings!!! + while(true) { + if (reader.readChunk().getUncompressed() == 0) + break; + } + reader.close(); + System.out.println("Writing " + path + " complete"); + } catch (Exception e) { + ExceptionLogger.log(e); + } + } + + public static class InvalidUsageException extends RuntimeException { + public InvalidUsageException(String str) { + super(str); + } + } + + private static void validatePath(String path) { + File pf = new File(path); + if (!pf.exists()) + throw new InvalidUsageException("Unable to send a file which doesn't exist!"); + if (pf.isDirectory()) + throw new InvalidUsageException("Path is a directory unable to send! Did you mean sendDir()?"); + } + + private static String createPath(String userFile) { + String[] pathParts = userFile.split("/"); + String userDirectory = userFile.replace(pathParts[pathParts.length - 1], ""); + + File ld = new File(System.getProperty("user.dir") + "/write/" + userDirectory); + if (!ld.exists() && !ld.mkdirs()) + throw new RuntimeException("Failed to create directory"); + + return System.getProperty("user.dir") + "/write/" + userFile; + } + }