Mercurial > jhg

comparison src/com/tmate/hgkit/fs/DataAccessProvider.java @ 26:71a9ba42cee8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Memory-mapped files for bigger files. Defect reading number of bytes greater than size of the buffer fixed
author Artem Tikhomirov <tikhomirov.artem@gmail.com>
date Sun, 09 Jan 2011 15:59:54 +0100
parents 6f9aca1a97be
children b0a15cefdfd6
comparison
equal deleted inserted replaced
25:da8ccbfae64d 26:71a9ba42cee8
29 return new DataAccess(); 29 return new DataAccess();
30 } 30 }
31 try { 31 try {
32 FileChannel fc = new FileInputStream(f).getChannel(); 32 FileChannel fc = new FileInputStream(f).getChannel();
33 if (fc.size() > mapioMagicBoundary) { 33 if (fc.size() > mapioMagicBoundary) {
34 return new MemoryMapFileAccess(fc, fc.size()); 34 // TESTS: bufLen of 1024 was used to test MemMapFileAccess
35 return new MemoryMapFileAccess(fc, fc.size(), mapioMagicBoundary);
35 } else { 36 } else {
36 // XXX once implementation is more or less stable, 37 // XXX once implementation is more or less stable,
37 // may want to try ByteBuffer.allocateDirect() to see 38 // may want to try ByteBuffer.allocateDirect() to see
38 // if there's any performance gain. 39 // if there's any performance gain.
39 boolean useDirectBuffer = false; 40 boolean useDirectBuffer = false;
41 // TESTS: bufferSize of 100 was used to check buffer underflow states when readBytes reads chunks bigger than bufSize
40 return new FileAccess(fc, fc.size(), bufferSize, useDirectBuffer); 42 return new FileAccess(fc, fc.size(), bufferSize, useDirectBuffer);
41 } 43 }
42 } catch (IOException ex) { 44 } catch (IOException ex) {
43 // unlikely to happen, we've made sure file exists. 45 // unlikely to happen, we've made sure file exists.
44 ex.printStackTrace(); // FIXME log error 46 ex.printStackTrace(); // FIXME log error
48 50
49 // DOESN'T WORK YET 51 // DOESN'T WORK YET
50 private static class MemoryMapFileAccess extends DataAccess { 52 private static class MemoryMapFileAccess extends DataAccess {
51 private FileChannel fileChannel; 53 private FileChannel fileChannel;
52 private final long size; 54 private final long size;
53 private long position = 0; 55 private long position = 0; // always points to buffer's absolute position in the file
54 56 private final int memBufferSize;
55 public MemoryMapFileAccess(FileChannel fc, long channelSize) { 57 private MappedByteBuffer buffer;
58
59 public MemoryMapFileAccess(FileChannel fc, long channelSize, int /*long?*/ bufferSize) {
56 fileChannel = fc; 60 fileChannel = fc;
57 size = channelSize; 61 size = channelSize;
58 } 62 memBufferSize = bufferSize;
59 63 }
64
65 @Override
66 public boolean isEmpty() {
67 return position + (buffer == null ? 0 : buffer.position()) >= size;
68 }
69
60 @Override 70 @Override
61 public void seek(long offset) { 71 public void seek(long offset) {
62 position = offset; 72 assert offset >= 0;
73 // offset may not necessarily be further than current position in the file (e.g. rewind)
74 if (buffer != null && /*offset is within buffer*/ offset >= position && (offset - position) < buffer.limit()) {
75 buffer.position((int) (offset - position));
76 } else {
77 position = offset;
78 buffer = null;
79 }
63 } 80 }
64 81
65 @Override 82 @Override
66 public void skip(int bytes) throws IOException { 83 public void skip(int bytes) throws IOException {
67 position += bytes; 84 assert bytes >= 0;
68 } 85 if (buffer == null) {
69 86 position += bytes;
70 private boolean fill() throws IOException { 87 return;
71 final int BUFFER_SIZE = 8 * 1024; 88 }
89 if (buffer.remaining() > bytes) {
90 buffer.position(buffer.position() + bytes);
91 } else {
92 position += buffer.position() + bytes;
93 buffer = null;
94 }
95 }
96
97 private void fill() throws IOException {
98 if (buffer != null) {
99 position += buffer.position();
100 }
72 long left = size - position; 101 long left = size - position;
73 MappedByteBuffer rv = fileChannel.map(FileChannel.MapMode.READ_ONLY, position, left < BUFFER_SIZE ? left : BUFFER_SIZE); 102 buffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, position, left < memBufferSize ? left : memBufferSize);
74 position += rv.capacity(); 103 }
75 return rv.hasRemaining(); 104
105 @Override
106 public void readBytes(byte[] buf, int offset, int length) throws IOException {
107 if (buffer == null || !buffer.hasRemaining()) {
108 fill();
109 }
110 // XXX in fact, we may try to create a MappedByteBuffer of exactly length size here, and read right away
111 while (length > 0) {
112 int tail = buffer.remaining();
113 if (tail == 0) {
114 throw new IOException();
115 }
116 if (tail >= length) {
117 buffer.get(buf, offset, length);
118 } else {
119 buffer.get(buf, offset, tail);
120 fill();
121 }
122 offset += tail;
123 length -= tail;
124 }
125 }
126
127 @Override
128 public byte readByte() throws IOException {
129 if (buffer == null || !buffer.hasRemaining()) {
130 fill();
131 }
132 if (buffer.hasRemaining()) {
133 return buffer.get();
134 }
135 throw new IOException();
76 } 136 }
77 137
78 @Override 138 @Override
79 public void done() { 139 public void done() {
140 buffer = null;
80 if (fileChannel != null) { 141 if (fileChannel != null) {
81 try { 142 try {
82 fileChannel.close(); 143 fileChannel.close();
83 } catch (IOException ex) { 144 } catch (IOException ex) {
84 ex.printStackTrace(); // log debug 145 ex.printStackTrace(); // log debug
150 return buffer.hasRemaining(); 211 return buffer.hasRemaining();
151 } 212 }
152 213
153 @Override 214 @Override
154 public void readBytes(byte[] buf, int offset, int length) throws IOException { 215 public void readBytes(byte[] buf, int offset, int length) throws IOException {
155 final int tail = buffer.remaining(); 216 if (!buffer.hasRemaining()) {
156 if (tail >= length) { 217 fill();
157 buffer.get(buf, offset, length); 218 }
158 } else { 219 while (length > 0) {
159 buffer.get(buf, offset, tail); 220 int tail = buffer.remaining();
160 if (fill()) { 221 if (tail == 0) {
161 buffer.get(buf, offset + tail, length - tail); 222 throw new IOException(); // shall not happen provided stream contains expected data and no attempts to read past isEmpty() == true are made.
223 }
224 if (tail >= length) {
225 buffer.get(buf, offset, length);
162 } else { 226 } else {
163 throw new IOException(); // shall not happen provided stream contains expected data and no attempts to read past nonEmpty() == false are made. 227 buffer.get(buf, offset, tail);
164 } 228 fill();
229 }
230 offset += tail;
231 length -= tail;
165 } 232 }
166 } 233 }
167 234
168 @Override 235 @Override
169 public byte readByte() throws IOException { 236 public byte readByte() throws IOException {