大数据技术之日志清洗案例详解大数据

7.7 日志清洗案例

7.7.1 简单解析版

1）需求：

去除日志中字段长度小于等于11的日志。

2）输入数据

194.237.142.21 - - [18/Sep/2013:06:49:18 +0000] "GET /wp-content/uploads/2013/07/rstudio-git3.png HTTP/1.1" 304 0 "-" "Mozilla/4.0 (compatible;)" 
183.49.46.228 - - [18/Sep/2013:06:49:23 +0000] "-" 400 0 "-" "-" 
163.177.71.12 - - [18/Sep/2013:06:49:33 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
163.177.71.12 - - [18/Sep/2013:06:49:36 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
101.226.68.137 - - [18/Sep/2013:06:49:42 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
101.226.68.137 - - [18/Sep/2013:06:49:45 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
60.208.6.156 - - [18/Sep/2013:06:49:48 +0000] "GET /wp-content/uploads/2013/07/rcassandra.png HTTP/1.0" 200 185524 "http://cos.name/category/software/packages/" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.66 Safari/537.36" 
222.68.172.190 - - [18/Sep/2013:06:49:57 +0000] "GET /images/my.jpg HTTP/1.1" 200 19939 "http://www.angularjs.cn/A00n" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.66 Safari/537.36" 
222.68.172.190 - - [18/Sep/2013:06:50:08 +0000] "-" 400 0 "-" "-" 
183.195.232.138 - - [18/Sep/2013:06:50:16 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
183.195.232.138 - - [18/Sep/2013:06:50:16 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
66.249.66.84 - - [18/Sep/2013:06:50:28 +0000] "GET /page/6/ HTTP/1.1" 200 27777 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 
221.130.41.168 - - [18/Sep/2013:06:50:37 +0000] "GET /feed/ HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.66 Safari/537.36" 
157.55.35.40 - - [18/Sep/2013:06:51:13 +0000] "GET /robots.txt HTTP/1.1" 200 150 "-" "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
50.116.27.194 - - [18/Sep/2013:06:51:35 +0000] "POST /wp-cron.php?doing_wp_cron=1379487095.2510800361633300781250 HTTP/1.0" 200 0 "-" "WordPress/3.6; http://blog.fens.me" 
58.215.204.118 - - [18/Sep/2013:06:51:35 +0000] "GET /nodejs-socketio-chat/ HTTP/1.1" 200 10818 "http://www.google.com/url?sa=t&rct=j&q=nodejs%20%E5%BC%82%E6%AD%A5%E5%B9%BF%E6%92%AD&source=web&cd=1&cad=rja&ved=0CCgQFjAA&url=%68%74%74%70%3a%2f%2f%62%6c%6f%67%2e%66%65%6e%73%2e%6d%65%2f%6e%6f%64%65%6a%73%2d%73%6f%63%6b%65%74%69%6f%2d%63%68%61%74%2f&ei=rko5UrylAefOiAe7_IGQBw&usg=AFQjCNG6YWoZsJ_bSj8kTnMHcH51hYQkAA&bvm=bv.52288139,d.aGc" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:36 +0000] "GET /wp-includes/js/jquery/jquery-migrate.min.js?ver=1.2.1 HTTP/1.1" 304 0 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:35 +0000] "GET /wp-includes/js/jquery/jquery.js?ver=1.10.2 HTTP/1.1" 304 0 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:36 +0000] "GET /wp-includes/js/comment-reply.min.js?ver=3.6 HTTP/1.1" 304 0 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:36 +0000] "GET /wp-content/uploads/2013/08/chat.png HTTP/1.1" 200 48968 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:36 +0000] "GET /wp-content/uploads/2013/08/chat2.png HTTP/1.1" 200 59852 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:37 +0000] "GET /wp-content/uploads/2013/08/socketio.png HTTP/1.1" 200 80493 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.248.178.212 - - [18/Sep/2013:06:51:37 +0000] "GET /nodejs-grunt-intro/ HTTP/1.1" 200 51770 "http://blog.fens.me/series-nodejs/" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MDDR; InfoPath.2; .NET4.0C)" 
58.248.178.212 - - [18/Sep/2013:06:51:40 +0000] "GET /wp-includes/js/jquery/jquery-migrate.min.js?ver=1.2.1 HTTP/1.1" 200 7200 "http://blog.fens.me/nodejs-grunt-intro/" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MDDR; InfoPath.2; .NET4.0C)"

3）实现代码：

（1）编写LogMapper

package com.xyg.mapreduce.weblog; 
import java.io.IOException; 
import org.apache.hadoop.io.LongWritable; 
import org.apache.hadoop.io.NullWritable; 
import org.apache.hadoop.io.Text; 
import org.apache.hadoop.mapreduce.Mapper; 
public class LogMapper extends Mapper<LongWritable, Text, Text, NullWritable>{ 
Text k = new Text(); 
@Override 
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { 
// 1 获取1行数据 
String line = value.toString();         
// 2 解析日志 
boolean result = parseLog(line,context);        
// 3 日志不合法退出 
if (!result) { 
return; 
}      
// 4 设置key 
k.set(line);    
// 5 写出数据 
context.write(k, NullWritable.get()); 
} 
// 2 解析日志 
private boolean parseLog(String line, Context context) { 
// 1 截取 
String[] fields = line.split(" "); 
// 2 日志长度大于11的为合法 
if (fields.length > 11) { 
// 系统计数器 
context.getCounter("map", "true").increment(1); 
return true; 
}else { 
context.getCounter("map", "false").increment(1); 
return false; 
} 
} 
}

（2）编写LogDriver

package com.xyg.mapreduce.weblog; 
import org.apache.hadoop.conf.Configuration; 
import org.apache.hadoop.fs.Path; 
import org.apache.hadoop.io.NullWritable; 
import org.apache.hadoop.io.Text; 
import org.apache.hadoop.mapreduce.Job; 
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 
public class LogDriver { 
    public static void main(String[] args) throws Exception { 
args = new String[] { "e:/inputlog", "e:/output1" }; 
    // 1 获取job信息 
Configuration conf = new Configuration(); 
Job job = Job.getInstance(conf); 
    // 2 加载jar包 
job.setJarByClass(LogDriver.class); 
    // 3 关联map 
job.setMapperClass(LogMapper.class); 
    // 4 设置最终输出类型 
job.setOutputKeyClass(Text.class); 
job.setOutputValueClass(NullWritable.class); 
    // 5 设置输入和输出路径 
FileInputFormat.setInputPaths(job, new Path(args[0])); 
FileOutputFormat.setOutputPath(job, new Path(args[1])); 
    // 6 提交 
job.waitForCompletion(true); 
} 
}

7.7.2 复杂解析版

1）需求：

对web访问日志中的各字段识别切分

去除日志中不合法的记录

根据统计需求，生成各类访问请求过滤数据

2）输入数据

194.237.142.21 - - [18/Sep/2013:06:49:18 +0000] "GET /wp-content/uploads/2013/07/rstudio-git3.png HTTP/1.1" 304 0 "-" "Mozilla/4.0 (compatible;)" 
183.49.46.228 - - [18/Sep/2013:06:49:23 +0000] "-" 400 0 "-" "-" 
163.177.71.12 - - [18/Sep/2013:06:49:33 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
163.177.71.12 - - [18/Sep/2013:06:49:36 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
101.226.68.137 - - [18/Sep/2013:06:49:42 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
101.226.68.137 - - [18/Sep/2013:06:49:45 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
60.208.6.156 - - [18/Sep/2013:06:49:48 +0000] "GET /wp-content/uploads/2013/07/rcassandra.png HTTP/1.0" 200 185524 "http://cos.name/category/software/packages/" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.66 Safari/537.36" 
222.68.172.190 - - [18/Sep/2013:06:49:57 +0000] "GET /images/my.jpg HTTP/1.1" 200 19939 "http://www.angularjs.cn/A00n" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.66 Safari/537.36" 
222.68.172.190 - - [18/Sep/2013:06:50:08 +0000] "-" 400 0 "-" "-" 
183.195.232.138 - - [18/Sep/2013:06:50:16 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
183.195.232.138 - - [18/Sep/2013:06:50:16 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
66.249.66.84 - - [18/Sep/2013:06:50:28 +0000] "GET /page/6/ HTTP/1.1" 200 27777 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 
221.130.41.168 - - [18/Sep/2013:06:50:37 +0000] "GET /feed/ HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.66 Safari/537.36" 
157.55.35.40 - - [18/Sep/2013:06:51:13 +0000] "GET /robots.txt HTTP/1.1" 200 150 "-" "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
50.116.27.194 - - [18/Sep/2013:06:51:35 +0000] "POST /wp-cron.php?doing_wp_cron=1379487095.2510800361633300781250 HTTP/1.0" 200 0 "-" "WordPress/3.6; http://blog.fens.me" 
58.215.204.118 - - [18/Sep/2013:06:51:35 +0000] "GET /nodejs-socketio-chat/ HTTP/1.1" 200 10818 "http://www.google.com/url?sa=t&rct=j&q=nodejs%20%E5%BC%82%E6%AD%A5%E5%B9%BF%E6%92%AD&source=web&cd=1&cad=rja&ved=0CCgQFjAA&url=%68%74%74%70%3a%2f%2f%62%6c%6f%67%2e%66%65%6e%73%2e%6d%65%2f%6e%6f%64%65%6a%73%2d%73%6f%63%6b%65%74%69%6f%2d%63%68%61%74%2f&ei=rko5UrylAefOiAe7_IGQBw&usg=AFQjCNG6YWoZsJ_bSj8kTnMHcH51hYQkAA&bvm=bv.52288139,d.aGc" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:36 +0000] "GET /wp-includes/js/jquery/jquery-migrate.min.js?ver=1.2.1 HTTP/1.1" 304 0 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:35 +0000] "GET /wp-includes/js/jquery/jquery.js?ver=1.10.2 HTTP/1.1" 304 0 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:36 +0000] "GET /wp-includes/js/comment-reply.min.js?ver=3.6 HTTP/1.1" 304 0 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:36 +0000] "GET /wp-content/uploads/2013/08/chat.png HTTP/1.1" 200 48968 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:36 +0000] "GET /wp-content/uploads/2013/08/chat2.png HTTP/1.1" 200 59852 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:37 +0000] "GET /wp-content/uploads/2013/08/socketio.png HTTP/1.1" 200 80493 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.248.178.212 - - [18/Sep/2013:06:51:37 +0000] "GET /nodejs-grunt-intro/ HTTP/1.1" 200 51770 "http://blog.fens.me/series-nodejs/" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MDDR; InfoPath.2; .NET4.0C)" 
58.248.178.212 - - [18/Sep/2013:06:51:40 +0000] "GET /wp-includes/js/jquery/jquery-migrate.min.js?ver=1.2.1 HTTP/1.1" 200 7200 "http://blog.fens.me/nodejs-grunt-intro/" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MDDR; InfoPath.2; .NET4.0C)"

3）实现代码：

（1）定义一个bean，用来记录日志数据中的各数据字段

package com.xyg.mapreduce.log; 
public class LogBean { 
private String remote_addr;// 记录客户端的ip地址 
private String remote_user;// 记录客户端用户名称,忽略属性"-" 
private String time_local;// 记录访问时间与时区 
private String request;// 记录请求的url与http协议 
private String status;// 记录请求状态；成功是200 
private String body_bytes_sent;// 记录发送给客户端文件主体内容大小 
private String http_referer;// 用来记录从那个页面链接访问过来的 
private String http_user_agent;// 记录客户浏览器的相关信息 
private boolean valid = true;// 判断数据是否合法 
public String getRemote_addr() { 
return remote_addr; 
} 
public void setRemote_addr(String remote_addr) { 
this.remote_addr = remote_addr; 
} 
public String getRemote_user() { 
return remote_user; 
} 
public void setRemote_user(String remote_user) { 
this.remote_user = remote_user; 
} 
public String getTime_local() { 
return time_local; 
} 
public void setTime_local(String time_local) { 
this.time_local = time_local; 
} 
public String getRequest() { 
return request; 
} 
public void setRequest(String request) { 
this.request = request; 
} 
public String getStatus() { 
return status; 
} 
public void setStatus(String status) { 
this.status = status; 
} 
public String getBody_bytes_sent() { 
return body_bytes_sent; 
} 
public void setBody_bytes_sent(String body_bytes_sent) { 
this.body_bytes_sent = body_bytes_sent; 
} 
public String getHttp_referer() { 
return http_referer; 
} 
public void setHttp_referer(String http_referer) { 
this.http_referer = http_referer; 
} 
public String getHttp_user_agent() { 
return http_user_agent; 
} 
public void setHttp_user_agent(String http_user_agent) { 
this.http_user_agent = http_user_agent; 
} 
public boolean isValid() { 
return valid; 
} 
public void setValid(boolean valid) { 
this.valid = valid; 
} 
@Override 
public String toString() { 
StringBuilder sb = new StringBuilder(); 
sb.append(this.valid); 
sb.append("/001").append(this.remote_addr); 
sb.append("/001").append(this.remote_user); 
sb.append("/001").append(this.time_local); 
sb.append("/001").append(this.request); 
sb.append("/001").append(this.status); 
sb.append("/001").append(this.body_bytes_sent); 
sb.append("/001").append(this.http_referer); 
sb.append("/001").append(this.http_user_agent); 
return sb.toString(); 
} 
}

（2）编写LogMapper程序

package com.xyg.mapreduce.log;
 
import java.io.IOException; 
import org.apache.hadoop.io.LongWritable; 
import org.apache.hadoop.io.NullWritable; 
import org.apache.hadoop.io.Text; 
import org.apache.hadoop.mapreduce.Mapper; 
public class LogMapper extends Mapper<LongWritable, Text, Text, NullWritable>{ 
Text k = new Text(); 
@Override 
protected void map(LongWritable key, Text value, Context context) 
throws IOException, InterruptedException { 
// 1 获取1行 
String line = value.toString();         
// 2 解析日志是否合法 
LogBean bean = pressLog(line);         
if (!bean.isValid()) { 
return; 
}         
k.set(bean.toString());         
// 3 输出 
context.write(k, NullWritable.get()); 
} 
// 解析日志 
private LogBean pressLog(String line) { 
LogBean logBean = new LogBean(); 
// 1 截取 
String[] fields = line.split(" ");    
if (fields.length > 11) { 
// 2封装数据 
logBean.setRemote_addr(fields[0]); 
logBean.setRemote_user(fields[1]); 
logBean.setTime_local(fields[3].substring(1)); 
logBean.setRequest(fields[6]); 
logBean.setStatus(fields[8]); 
logBean.setBody_bytes_sent(fields[9]); 
logBean.setHttp_referer(fields[10]); 
if (fields.length > 12) { 
logBean.setHttp_user_agent(fields[11] + " "+ fields[12]); 
}else { 
logBean.setHttp_user_agent(fields[11]); 
}            
// 大于400，HTTP错误 
if (Integer.parseInt(logBean.getStatus()) >= 400) { 
logBean.setValid(false); 
} 
}else { 
logBean.setValid(false); 
} 
return logBean; 
} 
}

（3）编写LogDriver程序

package com.xyg.mapreduce.log;
 
import org.apache.hadoop.conf.Configuration; 
import org.apache.hadoop.fs.Path; 
import org.apache.hadoop.io.NullWritable; 
import org.apache.hadoop.io.Text; 
import org.apache.hadoop.mapreduce.Job; 
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 
public class LogDriver { 
public static void main(String[] args) throws Exception { 
// 1 获取job信息 
Configuration conf = new Configuration(); 
Job job = Job.getInstance(conf); 
// 2 加载jar包 
job.setJarByClass(LogDriver.class); 
// 3 关联map 
job.setMapperClass(LogMapper.class); 
// 4 设置最终输出类型 
job.setOutputKeyClass(Text.class); 
job.setOutputValueClass(NullWritable.class); 
// 5 设置输入和输出路径 
FileInputFormat.setInputPaths(job, new Path(args[0])); 
FileOutputFormat.setOutputPath(job, new Path(args[1])); 
// 6 提交 
job.waitForCompletion(true); 
} 
}

原创文章，作者：ItWorker，如若转载，请注明出处：https://blog.ytso.com/9446.html

大数据技术之日志清洗案例详解大数据

7.7 日志清洗案例

7.7.1 简单解析版

7.7.2 复杂解析版

相关推荐

发表回复