java超快速文本去重复代码详解编程语言

import java.io.*; 
import java.util.HashSet; 
import java.util.Iterator; 
import java.util.Set; 
 
public class SpeedClear { 
 
	public static void main(String[] args) { 
		if(args.length==0){ 
			print(); 
			System.exit(1); 
		} 
		if(args.length!=2){ 
			System.out.println("Format error..."); 
			System.exit(1); 
		} 
		String pathname = args[0]; 
		String newPath = args[1]; 
		clear(pathname,newPath);     //调用去重复的方法... 
	} 
 
	/** 
	 *  
	 * @param pathname 
	 *            源文件路径 
	 * @param newPath 
	 *            新的文件路径 
	 * @throws Exception 
	 */ 
	public static void clear(String pathname, String newPath) { 
 
		System.out.println("Start... "); 
		 
		try{	//懒的写Try..直接都包围起来吧.... 
 
			File file = new File(pathname); 
		BufferedInputStream fis = new BufferedInputStream(new FileInputStream(file)); 
		 
BufferedReader buffer = new BufferedReader(new InputStreamReader(fis,"utf-8"),20*1024*1024);// 用5M的缓冲读取文本文件  
 
			//FileWriter fw  = new FileWriter(new File(newPath),true);  //去除后的文本 
 
			OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(new File(newPath)),"utf-8") ; 
 
			Set<String> set = new HashSet<String>(); 
			String temp = ""; // 临时字符串 
			int x = 0; 
			while ((temp = buffer.readLine()) != null) { // 读文件,一行读一个 
				set.add(temp); // 存储到Set集合里面 
				if(x%30000==0){ 
					System.out.print("..") ; 
				} 
				x++; 
			} 
			fis.close(); 
			buffer.close();   //关闭读取操作 
			 
			//下面开始写文件 
			 
			for (String xxser : set) { 
				out.write(xxser+"/r/n"); 
				 
			} 
			System.out.println("") ; 
			out.close();   //关闭写操作 
			System.out.println("size = " + set.size()); 
			System.out.println("End..."); 
		}catch(Exception e){ 
		 
			System.out.println("文件太大了,建议先100MB大小..") ; 
		} 
		 
		 
	} 
 
 
		public static void  print(){ 
		System.out.println("*************************************************"); 
		System.out.println("/t/tTo repeat /t/t"); 
		System.out.println(); 
		System.out.println("  format: java -Xmx1000m SpeedClear c://old.txt c://new.txt/t/t"); 
		System.out.println(); 
		System.out.println("/t/tAuthor:xxser	QQ:616100108"); 
		System.out.println("*************************************************"); 
		 
	} 
 
}