要求就是向mysql数据库插入到数据库里大量的数据 比如10W 或者 100W 而且要求内存稳定.
首先说下代码:
数据库MySQL
CREATE TABLE `users` (
`id` int(11) NOT NULL auto_increment,
`firstname` varchar(50) NOT NULL,
`lastname` varchar(50) NOT NULL,
`age` int(11) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
数据库辅助类
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
public final class DBUtils {
private static String mysqlurl = “jdbc:mysql://localhost:3306/mytest”;
private static String accessurl = “jdbc:mysql://localhost:3306/mytest”;
private static String user = “root”;
private static String password = “root”;
// 获得连接
public static Connection getAccessConn() throws SQLException {
return DriverManager.getConnection(accessurl, user, password);
}
public static Connection getMySqlConn() throws SQLException {
return DriverManager.getConnection(mysqlurl, user, password);
}
// 释放连接
public static void free(ResultSet rs, PreparedStatement ps, Connection conn) {
try {
if (rs != null) {
rs.close();
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
try {
if (ps != null) {
ps.close();
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
try {
if (conn != null) {
conn.close();
}
} catch (SQLException e) {
e.printStackTrace();
}
}
}
}
// 加载驱动
static {
try {
Class.forName(“com.mysql.jdbc.Driver”);
} catch (ClassNotFoundException e) {
System.out.println(“驱动加载出错”);
}
}
}
测试类
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
public class BatchExample {
private static Connection mysqlConn = null;
private static ResultSet rs = null;
// 总条数
private static int allCount = 10000;
// 分批条数
private static int preCount = 1000;
// 计数器
private static int count = 0;
private static String insertSQL = “insert into users(firstname, lastname, age) values(?, ?, ?)”;
private static PreparedStatement mysqlPs = null;
public static void main(String[] args) throws SQLException {
try {
mysqlConn = DBUtils.getMySqlConn();
mysqlPs = mysqlConn.prepareStatement(insertSQL);
mysqlConn.setAutoCommit(false);
long start = System.currentTimeMillis();
for (int i = 1; i <= allCount; i++) {
mysqlPs.setString(1, “firstname” + i);
mysqlPs.setString(2, “lastname” + i);
mysqlPs.setInt(3, 23);
mysqlPs.addBatch();
if ((i % preCount) == 0){
mysqlPs.executeBatch();
System.out.println(“当前进行完毕===>” + (++count) * preCount + “条”);
}
}
long end = System.currentTimeMillis();
System.out.println(“数据导入完毕,所用时间为: ” + (end – start) + ” ms”);
} catch (Exception e) {
mysqlConn.rollback();
System.out.println(“数据出错,已进行回滚”);
throw new RuntimeException();
} finally {
mysqlConn.commit();
DBUtils.free(rs, mysqlPs, mysqlConn);
}
}
}
用的批处理.最后执行10000条的记录是
当前进行完毕===>1000条
当前进行完毕===>2000条
当前进行完毕===>3000条
当前进行完毕===>4000条
当前进行完毕===>5000条
当前进行完毕===>6000条
当前进行完毕===>7000条
当前进行完毕===>8000条
当前进行完毕===>9000条
当前进行完毕===>10000条
数据导入完毕,所用时间为: 8140 ms
主要看下测试类
BatchExample 用的批处理 addBatch每1000条打包发送一次.但结果还是不尽人意8140ms.
而换了一种方案 用executeUpdate一次提交.SQL用StringBuilder串接 效率提升很快.
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
public class BufferQuery {
private static Connection mysqlConn = null;
private static ResultSet rs = null;
// 总条数
private static int allCount = 10000;
// 分批条数
private static int preCount = 1000;
// 计数器
private static int count = 0;
private static String insertSQL = “insert into users(firstname, lastname, age) values(?, ?, ?)”;
private static PreparedStatement mysqlPs = null;
public static void main(String[] args) throws SQLException {
try {
StringBuilder sb = new StringBuilder();
sb.append(“insert into users(firstname, lastname, age) values”);
mysqlConn = DBUtils.getMySqlConn();
mysqlPs = mysqlConn.prepareStatement(insertSQL);
mysqlConn.setAutoCommit(false);
long start = System.currentTimeMillis();
for (int i = 1; i <= allCount; i++) {
if(i > 1) sb.append(“,”);
sb.append(“(‘aa”+ i +”‘,’bb’,23)”);
if(i % preCount == 0){
System.out.println(“导入进行===>” + (++count * preCount) + “条”);
}
}
mysqlPs.executeUpdate(sb.toString());
long end = System.currentTimeMillis();
System.out.println(“数据导入完毕,所用时间为: ” + (end – start) + ” ms”);
} catch (Exception e) {
mysqlConn.rollback();
System.out.println(“数据出错,已进行回滚”);
throw new RuntimeException();
} finally {
mysqlConn.commit();
DBUtils.free(rs, mysqlPs, mysqlConn);
}
}
}
//用的StringBuilder串接
long start = System.currentTimeMillis();
for (int i = 1; i <= allCount; i++) {
if(i > 1) sb.append(“,”);
sb.append(“(‘aa”+ i +”‘,’bb’,23)”);
if(i % preCount == 0){
System.out.println(“导入进行===>” + (++count * preCount) + “条”);
}
}
mysqlPs.executeUpdate(sb.toString());
运行结果为:
导入进行===>1000条
导入进行===>2000条
导入进行===>3000条
导入进行===>4000条
导入进行===>5000条
导入进行===>6000条
导入进行===>7000条
导入进行===>8000条
导入进行===>9000条
导入进行===>10000条
数据导入完毕,所用时间为: 219 ms
1W条才129ms为什么会比批处理快这么多.但是还有问题就是 如果数据量更大 如20W 那么StringBuilder就装不下 堆栈溢出….
转载请注明来源网站:blog.ytso.com谢谢！
原创文章，作者：奋斗，如若转载，请注明出处：https://blog.ytso.com/4538.html