tikhomirov@113: /*
tikhomirov@113:  * Copyright (c) 2011 TMate Software Ltd
tikhomirov@113:  *  
tikhomirov@113:  * This program is free software; you can redistribute it and/or modify
tikhomirov@113:  * it under the terms of the GNU General Public License as published by
tikhomirov@113:  * the Free Software Foundation; version 2 of the License.
tikhomirov@113:  *
tikhomirov@113:  * This program is distributed in the hope that it will be useful,
tikhomirov@113:  * but WITHOUT ANY WARRANTY; without even the implied warranty of
tikhomirov@113:  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
tikhomirov@113:  * GNU General Public License for more details.
tikhomirov@113:  *
tikhomirov@113:  * For information on how to redistribute this software under
tikhomirov@113:  * the terms of a license other than GNU General Public License
tikhomirov@113:  * contact TMate Software at support@svnkit.com
tikhomirov@113:  */
tikhomirov@113: package org.tmatesoft.hg.internal;
tikhomirov@113: 
tikhomirov@113: import static org.tmatesoft.hg.internal.Filter.Direction.FromRepo;
tikhomirov@113: import static org.tmatesoft.hg.internal.Filter.Direction.ToRepo;
tikhomirov@113: import static org.tmatesoft.hg.internal.KeywordFilter.copySlice;
tikhomirov@113: 
tikhomirov@113: import java.io.File;
tikhomirov@113: import java.io.FileInputStream;
tikhomirov@113: import java.io.FileOutputStream;
tikhomirov@113: import java.nio.ByteBuffer;
tikhomirov@114: import java.util.ArrayList;
tikhomirov@114: import java.util.Map;
tikhomirov@113: 
tikhomirov@113: import org.tmatesoft.hg.core.Path;
tikhomirov@114: import org.tmatesoft.hg.repo.HgInternals;
tikhomirov@113: import org.tmatesoft.hg.repo.HgRepository;
tikhomirov@113: 
tikhomirov@113: /**
tikhomirov@113:  *
tikhomirov@113:  * @author Artem Tikhomirov
tikhomirov@113:  * @author TMate Software Ltd.
tikhomirov@113:  */
tikhomirov@113: public class NewlineFilter implements Filter {
tikhomirov@113: 
tikhomirov@113: 	// if allowInconsistent is true, filter simply pass incorrect newline characters (single \r or \r\n on *nix and single \n on Windows) as is,
tikhomirov@113: 	// i.e. doesn't try to convert them into appropriate newline characters. XXX revisit if Keyword extension behaves differently
tikhomirov@113: 	private final boolean allowInconsistent;
tikhomirov@113: 	private final boolean winToNix;
tikhomirov@113: 
tikhomirov@113: 	private NewlineFilter(boolean failIfInconsistent, int transform) {
tikhomirov@113: 		winToNix = transform == 0;
tikhomirov@113: 		allowInconsistent = !failIfInconsistent;
tikhomirov@113: 	}
tikhomirov@113: 
tikhomirov@113: 	public ByteBuffer filter(ByteBuffer src) {
tikhomirov@113: 		if (winToNix) {
tikhomirov@113: 			return win2nix(src);
tikhomirov@113: 		} else {
tikhomirov@113: 			return nix2win(src);
tikhomirov@113: 		}
tikhomirov@113: 	}
tikhomirov@113: 
tikhomirov@113: 	private ByteBuffer win2nix(ByteBuffer src) {
tikhomirov@113: 		int x = src.position(); // source index
tikhomirov@113: 		int lookupStart = x;
tikhomirov@113: 		ByteBuffer dst = null;
tikhomirov@113: 		while (x < src.limit()) {
tikhomirov@113: 			// x, lookupStart, ir and in are absolute positions within src buffer, which is never read with modifying operations
tikhomirov@113: 			int ir = indexOf('\r', src, lookupStart);
tikhomirov@113: 			int in = indexOf('\n', src, lookupStart);
tikhomirov@113: 			if (ir == -1) {
tikhomirov@113: 				if (in == -1 || allowInconsistent) {
tikhomirov@113: 					if (dst != null) {
tikhomirov@113: 						copySlice(src, x, src.limit(), dst);
tikhomirov@113: 						x = src.limit(); // consumed all
tikhomirov@113: 					}
tikhomirov@113: 					break;
tikhomirov@113: 				} else {
tikhomirov@113: 					fail(src, in);
tikhomirov@113: 				}
tikhomirov@113: 			}
tikhomirov@113: 			// in == -1 while ir != -1 may be valid case if ir is the last char of the buffer, we check below for that
tikhomirov@113: 			if (in != -1 && in != ir+1 && !allowInconsistent) {
tikhomirov@113: 				fail(src, in);
tikhomirov@113: 			}
tikhomirov@113: 			if (dst == null) {
tikhomirov@113: 				dst = ByteBuffer.allocate(src.remaining());
tikhomirov@113: 			}
tikhomirov@113: 			copySlice(src, x, ir, dst);
tikhomirov@113: 			if (ir+1 == src.limit()) {
tikhomirov@113: 				// last char of the buffer - 
tikhomirov@113: 				// consume src till that char and let next iteration work on it
tikhomirov@113: 				x = ir;
tikhomirov@113: 				break;
tikhomirov@113: 			}
tikhomirov@113: 			if (in != ir + 1) {
tikhomirov@113: 				x = ir+1; // generally in, but if allowInconsistent==true and \r is not followed by \n, then 
tikhomirov@113: 				// cases like "one \r two \r\n three" shall be processed correctly (second pair would be ignored if x==in)
tikhomirov@113: 				lookupStart = ir+1;
tikhomirov@113: 			} else {
tikhomirov@113: 				x = in;
tikhomirov@113: 				lookupStart = x+1; // skip \n for next lookup
tikhomirov@113: 			}
tikhomirov@113: 		}
tikhomirov@113: 		src.position(x); // mark we've consumed up to x
tikhomirov@113: 		return dst == null ? src : (ByteBuffer) dst.flip();
tikhomirov@113: 	}
tikhomirov@113: 
tikhomirov@113: 	private ByteBuffer nix2win(ByteBuffer src) {
tikhomirov@113: 		int x = src.position();
tikhomirov@113: 		ByteBuffer dst = null;
tikhomirov@113: 		while (x < src.limit()) {
tikhomirov@113: 			int in = indexOf('\n', src, x);
tikhomirov@113: 			int ir = indexOf('\r', src, x, in == -1 ? src.limit() : in);
tikhomirov@113: 			if (in == -1) {
tikhomirov@113: 				if (ir == -1 || allowInconsistent) {
tikhomirov@113: 					break;
tikhomirov@113: 				} else {
tikhomirov@113: 					fail(src, ir);
tikhomirov@113: 				}
tikhomirov@113: 			} else if (ir != -1 && !allowInconsistent) {
tikhomirov@113: 				fail(src, ir);
tikhomirov@113: 			}
tikhomirov@113: 			
tikhomirov@113: 			// x <= in < src.limit
tikhomirov@113: 			// allowInconsistent && x <= ir < in   || ir == -1  
tikhomirov@113: 			if (dst == null) {
tikhomirov@113: 				// buffer full of \n grows as much as twice in size
tikhomirov@113: 				dst = ByteBuffer.allocate(src.remaining() * 2);
tikhomirov@113: 			}
tikhomirov@113: 			copySlice(src, x, in, dst);
tikhomirov@113: 			if (ir == -1 || ir+1 != in) {
tikhomirov@113: 				dst.put((byte) '\r');
tikhomirov@113: 			} // otherwise (ir!=-1 && ir+1==in) we found \r\n pair, don't convert to \r\r\n
tikhomirov@113: 			// we may copy \n at src[in] on the next iteration, but would need extra lookupIndex variable then.
tikhomirov@113: 			dst.put((byte) '\n');
tikhomirov@113: 			x = in+1;
tikhomirov@113: 		}
tikhomirov@113: 		src.position(x);
tikhomirov@113: 		return dst == null ? src : (ByteBuffer) dst.flip();
tikhomirov@113: 	}
tikhomirov@113: 
tikhomirov@113: 
tikhomirov@113: 	private void fail(ByteBuffer b, int pos) {
tikhomirov@113: 		throw new RuntimeException(String.format("Inconsistent newline characters in the stream (char 0x%x, local index:%d)", b.get(pos), pos));
tikhomirov@113: 	}
tikhomirov@113: 
tikhomirov@113: 	private static int indexOf(char ch, ByteBuffer b, int from) {
tikhomirov@113: 		return indexOf(ch, b, from, b.limit());
tikhomirov@113: 	}
tikhomirov@113: 
tikhomirov@113: 	// looks up in buf[from..to)
tikhomirov@113: 	private static int indexOf(char ch, ByteBuffer b, int from, int to) {
tikhomirov@113: 		for (int i = from; i < to; i++) {
tikhomirov@113: 			byte c = b.get(i);
tikhomirov@113: 			if (ch == c) {
tikhomirov@113: 				return i;
tikhomirov@113: 			}
tikhomirov@113: 		}
tikhomirov@113: 		return -1;
tikhomirov@113: 	}
tikhomirov@113: 
tikhomirov@113: 	public static class Factory implements Filter.Factory {
tikhomirov@114: 		private boolean failIfInconsistent = true;
tikhomirov@114: 		private Path.Matcher lfMatcher;
tikhomirov@114: 		private Path.Matcher crlfMatcher;
tikhomirov@114: 		private Path.Matcher binMatcher;
tikhomirov@114: 		private Path.Matcher nativeMatcher;
tikhomirov@114: 		private String nativeRepoFormat;
tikhomirov@114: 		private String nativeOSFormat;
tikhomirov@113: 
tikhomirov@114: 		public void initialize(HgRepository hgRepo, ConfigFile cfg) {
tikhomirov@114: 			failIfInconsistent = cfg.getBoolean("eol", "only-consistent", true);
tikhomirov@114: 			File cfgFile = new File(new HgInternals(hgRepo).getRepositoryDir(), ".hgeol");
tikhomirov@114: 			if (!cfgFile.canRead()) {
tikhomirov@114: 				return;
tikhomirov@113: 			}
tikhomirov@114: 			// XXX if .hgeol is not checked out, we may get it from repository
tikhomirov@114: //			HgDataFile cfgFileNode = hgRepo.getFileNode(".hgeol");
tikhomirov@114: //			if (!cfgFileNode.exists()) {
tikhomirov@114: //				return;
tikhomirov@114: //			}
tikhomirov@114: 			// XXX perhaps, add HgDataFile.hasWorkingCopy and workingCopyContent()?
tikhomirov@114: 			ConfigFile hgeol = new ConfigFile();
tikhomirov@114: 			hgeol.addLocation(cfgFile);
tikhomirov@114: 			nativeRepoFormat = hgeol.getSection("repository").get("native");
tikhomirov@114: 			if (nativeRepoFormat == null) {
tikhomirov@114: 				nativeRepoFormat = "LF";
tikhomirov@114: 			}
tikhomirov@114: 			final String os = System.getProperty("os.name"); // XXX need centralized set of properties
tikhomirov@114: 			nativeOSFormat = os.indexOf("Windows") != -1 ? "CRLF" : "LF";
tikhomirov@114: 			// I assume pattern ordering in .hgeol is not important
tikhomirov@114: 			ArrayList<String> lfPatterns = new ArrayList<String>();
tikhomirov@114: 			ArrayList<String> crlfPatterns = new ArrayList<String>();
tikhomirov@114: 			ArrayList<String> nativePatterns = new ArrayList<String>();
tikhomirov@114: 			ArrayList<String> binPatterns = new ArrayList<String>();
tikhomirov@114: 			for (Map.Entry<String,String> e : hgeol.getSection("patterns").entrySet()) {
tikhomirov@114: 				if ("CRLF".equals(e.getValue())) {
tikhomirov@114: 					crlfPatterns.add(e.getKey());
tikhomirov@114: 				} else if ("LF".equals(e.getValue())) {
tikhomirov@114: 					lfPatterns.add(e.getKey());
tikhomirov@114: 				} else if ("native".equals(e.getValue())) {
tikhomirov@114: 					nativePatterns.add(e.getKey());
tikhomirov@114: 				} else if ("BIN".equals(e.getValue())) {
tikhomirov@114: 					binPatterns.add(e.getKey());
tikhomirov@114: 				} else {
tikhomirov@114: 					System.out.printf("Can't recognize .hgeol entry: %s for %s", e.getValue(), e.getKey()); // FIXME log warning
tikhomirov@114: 				}
tikhomirov@114: 			}
tikhomirov@114: 			if (!crlfPatterns.isEmpty()) {
tikhomirov@114: 				crlfMatcher = new PathGlobMatcher(crlfPatterns.toArray(new String[crlfPatterns.size()]));
tikhomirov@114: 			}
tikhomirov@114: 			if (!lfPatterns.isEmpty()) {
tikhomirov@114: 				lfMatcher = new PathGlobMatcher(lfPatterns.toArray(new String[lfPatterns.size()]));
tikhomirov@114: 			}
tikhomirov@114: 			if (!binPatterns.isEmpty()) {
tikhomirov@114: 				binMatcher = new PathGlobMatcher(binPatterns.toArray(new String[binPatterns.size()]));
tikhomirov@114: 			}
tikhomirov@114: 			if (!nativePatterns.isEmpty()) {
tikhomirov@114: 				nativeMatcher = new PathGlobMatcher(nativePatterns.toArray(new String[nativePatterns.size()]));
tikhomirov@114: 			}
tikhomirov@114: 		}
tikhomirov@114: 
tikhomirov@114: 		public Filter create(Path path, Options opts) {
tikhomirov@114: 			if (binMatcher == null && crlfMatcher == null && lfMatcher == null && nativeMatcher == null) {
tikhomirov@114: 				// not initialized - perhaps, no .hgeol found
tikhomirov@114: 				return null;
tikhomirov@114: 			}
tikhomirov@114: 			if (binMatcher != null && binMatcher.accept(path)) {
tikhomirov@114: 				return null;
tikhomirov@114: 			}
tikhomirov@114: 			if (crlfMatcher != null && crlfMatcher.accept(path)) {
tikhomirov@114: 				return new NewlineFilter(failIfInconsistent, 1);
tikhomirov@114: 			} else if (lfMatcher != null && lfMatcher.accept(path)) {
tikhomirov@114: 				return new NewlineFilter(failIfInconsistent, 0);
tikhomirov@114: 			} else if (nativeMatcher != null && nativeMatcher.accept(path)) {
tikhomirov@114: 				if (nativeOSFormat.equals(nativeRepoFormat)) {
tikhomirov@114: 					return null;
tikhomirov@114: 				}
tikhomirov@114: 				if (opts.getDirection() == FromRepo) {
tikhomirov@114: 					int transform = "CRLF".equals(nativeOSFormat) ? 1 : 0;
tikhomirov@114: 					return new NewlineFilter(failIfInconsistent, transform);
tikhomirov@114: 				} else if (opts.getDirection() == ToRepo) {
tikhomirov@114: 					int transform = "CRLF".equals(nativeOSFormat) ? 0 : 1;
tikhomirov@114: 					return new NewlineFilter(failIfInconsistent, transform);
tikhomirov@114: 				}
tikhomirov@114: 				return null;
tikhomirov@114: 			}
tikhomirov@114: 			return null;
tikhomirov@113: 		}
tikhomirov@113: 	}
tikhomirov@113: 
tikhomirov@113: 	public static void main(String[] args) throws Exception {
tikhomirov@113: 		FileInputStream fis = new FileInputStream(new File("/temp/design.lf.txt"));
tikhomirov@113: 		FileOutputStream fos = new FileOutputStream(new File("/temp/design.newline.out"));
tikhomirov@113: 		ByteBuffer b = ByteBuffer.allocate(12);
tikhomirov@113: 		NewlineFilter nlFilter = new NewlineFilter(true, 1);
tikhomirov@113: 		while (fis.getChannel().read(b) != -1) {
tikhomirov@113: 			b.flip(); // get ready to be read
tikhomirov@113: 			ByteBuffer f = nlFilter.filter(b);
tikhomirov@113: 			fos.getChannel().write(f); // XXX in fact, f may not be fully consumed
tikhomirov@113: 			if (b.hasRemaining()) {
tikhomirov@113: 				b.compact();
tikhomirov@113: 			} else {
tikhomirov@113: 				b.clear();
tikhomirov@113: 			}
tikhomirov@113: 		}
tikhomirov@113: 		fis.close();
tikhomirov@113: 		fos.flush();
tikhomirov@113: 		fos.close();
tikhomirov@113: 	}
tikhomirov@113: 
tikhomirov@113: }