Mercurial > jhg
diff src/com/tmate/hgkit/fs/DataAccessProvider.java @ 26:71a9ba42cee8
Memory-mapped files for bigger files. Defect reading number of bytes greater than size of the buffer fixed
author | Artem Tikhomirov <tikhomirov.artem@gmail.com> |
---|---|
date | Sun, 09 Jan 2011 15:59:54 +0100 |
parents | 6f9aca1a97be |
children | b0a15cefdfd6 |
line wrap: on
line diff
--- a/src/com/tmate/hgkit/fs/DataAccessProvider.java Thu Jan 06 04:45:40 2011 +0100 +++ b/src/com/tmate/hgkit/fs/DataAccessProvider.java Sun Jan 09 15:59:54 2011 +0100 @@ -31,12 +31,14 @@ try { FileChannel fc = new FileInputStream(f).getChannel(); if (fc.size() > mapioMagicBoundary) { - return new MemoryMapFileAccess(fc, fc.size()); + // TESTS: bufLen of 1024 was used to test MemMapFileAccess + return new MemoryMapFileAccess(fc, fc.size(), mapioMagicBoundary); } else { // XXX once implementation is more or less stable, // may want to try ByteBuffer.allocateDirect() to see // if there's any performance gain. boolean useDirectBuffer = false; + // TESTS: bufferSize of 100 was used to check buffer underflow states when readBytes reads chunks bigger than bufSize return new FileAccess(fc, fc.size(), bufferSize, useDirectBuffer); } } catch (IOException ex) { @@ -50,33 +52,92 @@ private static class MemoryMapFileAccess extends DataAccess { private FileChannel fileChannel; private final long size; - private long position = 0; + private long position = 0; // always points to buffer's absolute position in the file + private final int memBufferSize; + private MappedByteBuffer buffer; - public MemoryMapFileAccess(FileChannel fc, long channelSize) { + public MemoryMapFileAccess(FileChannel fc, long channelSize, int /*long?*/ bufferSize) { fileChannel = fc; size = channelSize; + memBufferSize = bufferSize; } @Override + public boolean isEmpty() { + return position + (buffer == null ? 0 : buffer.position()) >= size; + } + + @Override public void seek(long offset) { - position = offset; + assert offset >= 0; + // offset may not necessarily be further than current position in the file (e.g. rewind) + if (buffer != null && /*offset is within buffer*/ offset >= position && (offset - position) < buffer.limit()) { + buffer.position((int) (offset - position)); + } else { + position = offset; + buffer = null; + } } @Override public void skip(int bytes) throws IOException { - position += bytes; + assert bytes >= 0; + if (buffer == null) { + position += bytes; + return; + } + if (buffer.remaining() > bytes) { + buffer.position(buffer.position() + bytes); + } else { + position += buffer.position() + bytes; + buffer = null; + } } - private boolean fill() throws IOException { - final int BUFFER_SIZE = 8 * 1024; + private void fill() throws IOException { + if (buffer != null) { + position += buffer.position(); + } long left = size - position; - MappedByteBuffer rv = fileChannel.map(FileChannel.MapMode.READ_ONLY, position, left < BUFFER_SIZE ? left : BUFFER_SIZE); - position += rv.capacity(); - return rv.hasRemaining(); + buffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, position, left < memBufferSize ? left : memBufferSize); + } + + @Override + public void readBytes(byte[] buf, int offset, int length) throws IOException { + if (buffer == null || !buffer.hasRemaining()) { + fill(); + } + // XXX in fact, we may try to create a MappedByteBuffer of exactly length size here, and read right away + while (length > 0) { + int tail = buffer.remaining(); + if (tail == 0) { + throw new IOException(); + } + if (tail >= length) { + buffer.get(buf, offset, length); + } else { + buffer.get(buf, offset, tail); + fill(); + } + offset += tail; + length -= tail; + } + } + + @Override + public byte readByte() throws IOException { + if (buffer == null || !buffer.hasRemaining()) { + fill(); + } + if (buffer.hasRemaining()) { + return buffer.get(); + } + throw new IOException(); } @Override public void done() { + buffer = null; if (fileChannel != null) { try { fileChannel.close(); @@ -152,16 +213,22 @@ @Override public void readBytes(byte[] buf, int offset, int length) throws IOException { - final int tail = buffer.remaining(); - if (tail >= length) { - buffer.get(buf, offset, length); - } else { - buffer.get(buf, offset, tail); - if (fill()) { - buffer.get(buf, offset + tail, length - tail); + if (!buffer.hasRemaining()) { + fill(); + } + while (length > 0) { + int tail = buffer.remaining(); + if (tail == 0) { + throw new IOException(); // shall not happen provided stream contains expected data and no attempts to read past isEmpty() == true are made. + } + if (tail >= length) { + buffer.get(buf, offset, length); } else { - throw new IOException(); // shall not happen provided stream contains expected data and no attempts to read past nonEmpty() == false are made. + buffer.get(buf, offset, tail); + fill(); } + offset += tail; + length -= tail; } }