view src/org/tmatesoft/hg/internal/DataAccessProvider.java @ 338:3cfa4d908fc9

Add options to control DataAccessProvider, allow to turn off use of file memory mapping in particular to solve potential sharing violation (os file handle gets released on MappedByteByffer being GC'd, not on FileChannel.close())
author Artem Tikhomirov <tikhomirov.artem@gmail.com>
date Tue, 15 Nov 2011 04:47:03 +0100
parents 981f9f50bb6c
children 0ae53c32ecef 31a719b9f95e
line wrap: on
line source
/*
 * Copyright (c) 2010-2011 TMate Software Ltd
 *  
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; version 2 of the License.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * For information on how to redistribute this software under
 * the terms of a license other than GNU General Public License
 * contact TMate Software at support@hg4j.com
 */
package org.tmatesoft.hg.internal;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;

import org.tmatesoft.hg.core.HgBadStateException;
import org.tmatesoft.hg.core.SessionContext;

/**
 * 
 * @author Artem Tikhomirov
 * @author TMate Software Ltd.
 */
public class DataAccessProvider {
	/**
	 * Boundary to start using file memory mapping instead of regular file access, in bytes.  
	 * Set to 0 to indicate mapping files into memory shall not be used.
	 * If set to -1, file of any size would be mapped in memory.
	 */
	public static final String CFG_PROPERTY_MAPIO_LIMIT				= "hg4j.dap.mapio_limit";
	public static final String CFG_PROPERTY_MAPIO_BUFFER_SIZE		= "hg4j.dap.mapio_buffer";
	public static final String CFG_PROPERTY_FILE_BUFFER_SIZE		= "hg4j.dap.file_buffer";

	private final int mapioMagicBoundary;
	private final int bufferSize;
	private final SessionContext context;

	public DataAccessProvider(SessionContext ctx) {
		this(ctx, getConfigOption(ctx, CFG_PROPERTY_MAPIO_LIMIT, 100 * 1024), getConfigOption(ctx, CFG_PROPERTY_FILE_BUFFER_SIZE, 8 * 1024));
	}
	
	private static int getConfigOption(SessionContext ctx, String optName, int defaultValue) {
		Object v = ctx.getProperty(optName, defaultValue);
		if (false == v instanceof Number) {
			v = Integer.parseInt(v.toString());
		}
		return ((Number) v).intValue();
	}

	public DataAccessProvider(SessionContext ctx, int mapioBoundary, int regularBufferSize) {
		context = ctx;
		mapioMagicBoundary = mapioBoundary == 0 ? Integer.MAX_VALUE : mapioBoundary;
		bufferSize = regularBufferSize;
	}

	public DataAccess create(File f) {
		if (!f.exists()) {
			return new DataAccess();
		}
		try {
			FileChannel fc = new FileInputStream(f).getChannel();
			int flen = (int) fc.size();
			if (fc.size() - flen != 0) {
				throw new HgBadStateException("Files greater than 2Gb are not yet supported");
			}
			if (flen > mapioMagicBoundary) {
				// TESTS: bufLen of 1024 was used to test MemMapFileAccess
				return new MemoryMapFileAccess(fc, flen, getConfigOption(context, CFG_PROPERTY_MAPIO_BUFFER_SIZE, 100*1024 /*same as default boundary*/));
			} else {
				// XXX once implementation is more or less stable,
				// may want to try ByteBuffer.allocateDirect() to see
				// if there's any performance gain. 
				boolean useDirectBuffer = false; // XXX might be another config option
				// TESTS: bufferSize of 100 was used to check buffer underflow states when readBytes reads chunks bigger than bufSize
				return new FileAccess(fc, flen, bufferSize, useDirectBuffer);
			}
		} catch (IOException ex) {
			// unlikely to happen, we've made sure file exists.
			context.getLog().error(getClass(), ex, null);
		}
		return new DataAccess(); // non-null, empty.
	}

	// DOESN'T WORK YET 
	private static class MemoryMapFileAccess extends DataAccess {
		private FileChannel fileChannel;
		private final int size;
		private long position = 0; // always points to buffer's absolute position in the file
		private final int memBufferSize;
		private MappedByteBuffer buffer;

		public MemoryMapFileAccess(FileChannel fc, int channelSize, int bufferSize) {
			fileChannel = fc;
			size = channelSize;
			memBufferSize = bufferSize > channelSize ? channelSize : bufferSize; // no reason to waste memory more than there's data 
		}

		@Override
		public boolean isEmpty() {
			return position + (buffer == null ? 0 : buffer.position()) >= size;
		}
		
		@Override
		public int length() {
			return size;
		}
		
		@Override
		public DataAccess reset() throws IOException {
			seek(0);
			return this;
		}
		
		@Override
		public void seek(int offset) {
			assert offset >= 0;
			// offset may not necessarily be further than current position in the file (e.g. rewind) 
			if (buffer != null && /*offset is within buffer*/ offset >= position && (offset - position) < buffer.limit()) {
				buffer.position((int) (offset - position));
			} else {
				position = offset;
				buffer = null;
			}
		}

		@Override
		public void skip(int bytes) throws IOException {
			assert bytes >= 0;
			if (buffer == null) {
				position += bytes;
				return;
			}
			if (buffer.remaining() > bytes) {
				buffer.position(buffer.position() + bytes);
			} else {
				position += buffer.position() + bytes;
				buffer = null;
			}
		}

		private void fill() throws IOException {
			if (buffer != null) {
				position += buffer.position(); 
			}
			long left = size - position;
			buffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, position, left < memBufferSize ? left : memBufferSize);
		}

		@Override
		public void readBytes(byte[] buf, int offset, int length) throws IOException {
			if (buffer == null || !buffer.hasRemaining()) {
				fill();
			}
			// XXX in fact, we may try to create a MappedByteBuffer of exactly length size here, and read right away
			while (length > 0) {
				int tail = buffer.remaining();
				if (tail == 0) {
					throw new IOException();
				}
				if (tail >= length) {
					buffer.get(buf, offset, length);
				} else {
					buffer.get(buf, offset, tail);
					fill();
				}
				offset += tail;
				length -= tail;
			}
		}

		@Override
		public byte readByte() throws IOException {
			if (buffer == null || !buffer.hasRemaining()) {
				fill();
			}
			if (buffer.hasRemaining()) {
				return buffer.get();
			}
			throw new IOException();
		}

		@Override
		public void done() {
			buffer = null;
			if (fileChannel != null) {
				try {
					fileChannel.close();
				} catch (IOException ex) {
					StreamLogFacility.newDefault().debug(getClass(), ex, null);
				}
				fileChannel = null;
			}
		}
	}

	// (almost) regular file access - FileChannel and buffers.
	private static class FileAccess extends DataAccess {
		private FileChannel fileChannel;
		private final int size;
		private ByteBuffer buffer;
		private int bufferStartInFile = 0; // offset of this.buffer in the file.

		public FileAccess(FileChannel fc, int channelSize, int bufferSizeHint, boolean useDirect) {
			fileChannel = fc;
			size = channelSize;
			final int capacity = size < bufferSizeHint ? size : bufferSizeHint;
			buffer = useDirect ? ByteBuffer.allocateDirect(capacity) : ByteBuffer.allocate(capacity);
			buffer.flip(); // or .limit(0) to indicate it's empty
		}
		
		@Override
		public boolean isEmpty() {
			return bufferStartInFile + buffer.position() >= size;
		}
		
		@Override
		public int length() {
			return size;
		}
		
		@Override
		public DataAccess reset() throws IOException {
			seek(0);
			return this;
		}
		
		@Override
		public void seek(int offset) throws IOException {
			if (offset > size) {
				throw new IllegalArgumentException();
			}
			if (offset < bufferStartInFile + buffer.limit() && offset >= bufferStartInFile) {
				buffer.position((int) (offset - bufferStartInFile));
			} else {
				// out of current buffer, invalidate it (force re-read) 
				// XXX or ever re-read it right away?
				bufferStartInFile = offset;
				buffer.clear();
				buffer.limit(0); // or .flip() to indicate we switch to reading
				fileChannel.position(offset);
			}
		}

		@Override
		public void skip(int bytes) throws IOException {
			final int newPos = buffer.position() + bytes;
			if (newPos >= 0 && newPos < buffer.limit()) {
				// no need to move file pointer, just rewind/seek buffer 
				buffer.position(newPos);
			} else {
				//
				seek(bufferStartInFile + newPos);
			}
		}

		private boolean fill() throws IOException {
			if (!buffer.hasRemaining()) {
				bufferStartInFile += buffer.limit();
				buffer.clear();
				if (bufferStartInFile < size) { // just in case there'd be any exception on EOF, not -1 
					fileChannel.read(buffer);
					// may return -1 when EOF, but empty will reflect this, hence no explicit support here   
				}
				buffer.flip();
			}
			return buffer.hasRemaining();
		}

		@Override
		public void readBytes(byte[] buf, int offset, int length) throws IOException {
			if (!buffer.hasRemaining()) {
				fill();
			}
			while (length > 0) {
				int tail = buffer.remaining();
				if (tail == 0) {
					throw new IOException(); // shall not happen provided stream contains expected data and no attempts to read past isEmpty() == true are made.
				}
				if (tail >= length) {
					buffer.get(buf, offset, length);
				} else {
					buffer.get(buf, offset, tail);
					fill();
				}
				offset += tail;
				length -= tail;
			}
		}

		@Override
		public byte readByte() throws IOException {
			if (buffer.hasRemaining()) {
				return buffer.get();
			}
			if (fill()) {
				return buffer.get();
			}
			throw new IOException();
		}

		@Override
		public void done() {
			if (buffer != null) {
				buffer = null;
			}
			if (fileChannel != null) {
				try {
					fileChannel.close();
				} catch (IOException ex) {
					StreamLogFacility.newDefault().debug(getClass(), ex, null);
				}
				fileChannel = null;
			}
		}
	}
}