Java read big data file, deal with big data file performance comparison?
By using io provided by java, scanner class, apache api provided by processing large file data performance analysis comparison, the code is as follows:
package test;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.util.Random;
import java.util.Scanner;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.LineIterator;
import org.junit.Test;
public class TestFile {
//@Test
//Create data and test the performance of each of the following methods to read data
public void makeFile() throws IOException
{
File file = new File("D:\\phone.txt");
OutputStream os = new BufferedOutputStream(new FileOutputStream(file));
BufferedWriter bw=new BufferedWriter(new OutputStreamWriter(os));
//2 million
for(int i=0; i < 2000000; i++)
{
bw.write(bulidPhone());
bw.newLine();
}
bw.close();
os.close();
}
//generate string
private String bulidPhone()
{
Long lo = new Random().nextLong();
return String.valueOf(lo);
}
/**
* @Title: readTxt1
* @Description: Parse output file data using regular jdk io
* @throws IOException
*/
@Test
public void readTxt1() throws IOException
{
long start = System.currentTimeMillis();
File file = new File("D:\\phone.txt");
Reader in = new FileReader(file);
BufferedReader br = new BufferedReader(in);
while(br.ready())
{
//System.out.println(br.readLine());
br.readLine();
}
in.close();
br.close();
long end = System.currentTimeMillis();
System.out.println("readTxt1 method, memory usage ="+(Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory())+", time usage ms ="+(end-start));
}
/**
* @Title: readTxt2
* @Description: Parse file data using Scanner
* @throws IOException
*/
@Test
public void readTxt2() throws IOException
{
long start = System.currentTimeMillis();
File file = new File("D:\\phone.txt");
InputStream is = new FileInputStream(file);
Scanner scan = new Scanner(is,"UTF-8");
while(scan.hasNextLine())
{
//System.out.println(scan.nextLine());
scan.nextLine();
//scan.next();
}
is.close();
scan.close();
long end = System.currentTimeMillis();
System.out.println("readTxt2 method, memory usage ="+(Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory())+", time usage ms ="+(end-start));
}
/**
* @Title: readTxt3
* @Description: Parse files using org.apache.commons.io.FileUtils, apache utility classes
* @throws IOException
*/
@Test
public void readTxt3() throws IOException
{
long start = System.currentTimeMillis();
File file = new File("D:\\phone.txt");
LineIterator it = FileUtils.lineIterator(file, "UTF-8");
while(it.hasNext())
{
it.next();
}
it.close();
long end = System.currentTimeMillis();
System.out.println("readTxt3 method, memory usage ="+(Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory())+", time usage ms ="+(end-start));
}
}
The results are as follows:
Through analysis and comparison: Get [Download address]
Apache has the shortest API processing time, but consumes more memory than JDK's IO.
2. Scanner class performance is the worst, sales memory is high, time is long.
3. Traditional jdk io takes slightly longer to process and consumes less memory.
