大数据技术之日志清洗案例详解大数据

7.7 日志清洗案例

7.7.1 简单解析版

1）需求：

去除日志中字段长度小于等于11的日志。

2）输入数据

194.237.142.21 - - [18/Sep/2013:06:49:18 +0000] "GET /wp-content/uploads/2013/07/rstudio-git3.png HTTP/1.1" 304 0 "-" "Mozilla/4.0 (compatible;)" 
183.49.46.228 - - [18/Sep/2013:06:49:23 +0000] "-" 400 0 "-" "-" 
163.177.71.12 - - [18/Sep/2013:06:49:33 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
163.177.71.12 - - [18/Sep/2013:06:49:36 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
101.226.68.137 - - [18/Sep/2013:06:49:42 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
101.226.68.137 - - [18/Sep/2013:06:49:45 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
60.208.6.156 - - [18/Sep/2013:06:49:48 +0000] "GET /wp-content/uploads/2013/07/rcassandra.png HTTP/1.0" 200 185524 "http://cos.name/category/software/packages/" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.66 Safari/537.36" 
222.68.172.190 - - [18/Sep/2013:06:49:57 +0000] "GET /images/my.jpg HTTP/1.1" 200 19939 "http://www.angularjs.cn/A00n" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.66 Safari/537.36" 
222.68.172.190 - - [18/Sep/2013:06:50:08 +0000] "-" 400 0 "-" "-" 
183.195.232.138 - - [18/Sep/2013:06:50:16 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
183.195.232.138 - - [18/Sep/2013:06:50:16 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
66.249.66.84 - - [18/Sep/2013:06:50:28 +0000] "GET /page/6/ HTTP/1.1" 200 27777 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 
221.130.41.168 - - [18/Sep/2013:06:50:37 +0000] "GET /feed/ HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.66 Safari/537.36" 
157.55.35.40 - - [18/Sep/2013:06:51:13 +0000] "GET /robots.txt HTTP/1.1" 200 150 "-" "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
50.116.27.194 - - [18/Sep/2013:06:51:35 +0000] "POST /wp-cron.php?doing_wp_cron=1379487095.2510800361633300781250 HTTP/1.0" 200 0 "-" "WordPress/3.6; http://blog.fens.me" 
58.215.204.118 - - [18/Sep/2013:06:51:35 +0000] "GET /nodejs-socketio-chat/ HTTP/1.1" 200 10818 "http://www.google.com/url?sa=t&rct=j&q=nodejs%20%E5%BC%82%E6%AD%A5%E5%B9%BF%E6%92%AD&source=web&cd=1&cad=rja&ved=0CCgQFjAA&url=%68%74%74%70%3a%2f%2f%62%6c%6f%67%2e%66%65%6e%73%2e%6d%65%2f%6e%6f%64%65%6a%73%2d%73%6f%63%6b%65%74%69%6f%2d%63%68%61%74%2f&ei=rko5UrylAefOiAe7_IGQBw&usg=AFQjCNG6YWoZsJ_bSj8kTnMHcH51hYQkAA&bvm=bv.52288139,d.aGc" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:36 +0000] "GET /wp-includes/js/jquery/jquery-migrate.min.js?ver=1.2.1 HTTP/1.1" 304 0 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:35 +0000] "GET /wp-includes/js/jquery/jquery.js?ver=1.10.2 HTTP/1.1" 304 0 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:36 +0000] "GET /wp-includes/js/comment-reply.min.js?ver=3.6 HTTP/1.1" 304 0 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:36 +0000] "GET /wp-content/uploads/2013/08/chat.png HTTP/1.1" 200 48968 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:36 +0000] "GET /wp-content/uploads/2013/08/chat2.png HTTP/1.1" 200 59852 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:37 +0000] "GET /wp-content/uploads/2013/08/socketio.png HTTP/1.1" 200 80493 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.248.178.212 - - [18/Sep/2013:06:51:37 +0000] "GET /nodejs-grunt-intro/ HTTP/1.1" 200 51770 "http://blog.fens.me/series-nodejs/" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MDDR; InfoPath.2; .NET4.0C)" 
58.248.178.212 - - [18/Sep/2013:06:51:40 +0000] "GET /wp-includes/js/jquery/jquery-migrate.min.js?ver=1.2.1 HTTP/1.1" 200 7200 "http://blog.fens.me/nodejs-grunt-intro/" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MDDR; InfoPath.2; .NET4.0C)"

3）实现代码：

（1）编写LogMapper

package com.xyg.mapreduce.weblog; 
import java.io.IOException; 
import org.apache.hadoop.io.LongWritable; 
import org.apache.hadoop.io.NullWritable; 
import org.apache.hadoop.io.Text; 
import org.apache.hadoop.mapreduce.Mapper; 
 
public class LogMapper extends Mapper<LongWritable, Text, Text, NullWritable>{ 
     
    Text k = new Text(); 
     
    @Override 
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { 
         
        // 1 获取1行数据 
        String line = value.toString();         
        // 2 解析日志 
        boolean result = parseLog(line,context);        
        // 3 日志不合法退出 
        if (!result) { 
            return; 
        }      
        // 4 设置key 
        k.set(line);    
        // 5 写出数据 
        context.write(k, NullWritable.get()); 
    } 
    // 2 解析日志 
    private boolean parseLog(String line, Context context) { 
        // 1 截取 
        String[] fields = line.split(" "); 
         
        // 2 日志长度大于11的为合法 
        if (fields.length > 11) { 
            // 系统计数器 
            context.getCounter("map", "true").increment(1); 
            return true; 
        }else { 
            context.getCounter("map", "false").increment(1); 
            return false; 
        } 
    } 
}

（2）编写LogDriver

package com.xyg.mapreduce.weblog; 
 
import org.apache.hadoop.conf.Configuration; 
import org.apache.hadoop.fs.Path; 
import org.apache.hadoop.io.NullWritable; 
import org.apache.hadoop.io.Text; 
import org.apache.hadoop.mapreduce.Job; 
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 
 
public class LogDriver { 
 
    public static void main(String[] args) throws Exception { 
        args = new String[] { "e:/inputlog", "e:/output1" }; 
    // 1 获取job信息 
    Configuration conf = new Configuration(); 
    Job job = Job.getInstance(conf); 
    // 2 加载jar包 
    job.setJarByClass(LogDriver.class); 
    // 3 关联map 
    job.setMapperClass(LogMapper.class); 
    // 4 设置最终输出类型 
    job.setOutputKeyClass(Text.class); 
    job.setOutputValueClass(NullWritable.class); 
    // 5 设置输入和输出路径 
    FileInputFormat.setInputPaths(job, new Path(args[0])); 
    FileOutputFormat.setOutputPath(job, new Path(args[1])); 
    // 6 提交 
    job.waitForCompletion(true); 
    } 
}

7.7.2 复杂解析版

1）需求：

对web访问日志中的各字段识别切分

去除日志中不合法的记录

根据统计需求，生成各类访问请求过滤数据

2）输入数据

194.237.142.21 - - [18/Sep/2013:06:49:18 +0000] "GET /wp-content/uploads/2013/07/rstudio-git3.png HTTP/1.1" 304 0 "-" "Mozilla/4.0 (compatible;)" 
183.49.46.228 - - [18/Sep/2013:06:49:23 +0000] "-" 400 0 "-" "-" 
163.177.71.12 - - [18/Sep/2013:06:49:33 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
163.177.71.12 - - [18/Sep/2013:06:49:36 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
101.226.68.137 - - [18/Sep/2013:06:49:42 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
101.226.68.137 - - [18/Sep/2013:06:49:45 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
60.208.6.156 - - [18/Sep/2013:06:49:48 +0000] "GET /wp-content/uploads/2013/07/rcassandra.png HTTP/1.0" 200 185524 "http://cos.name/category/software/packages/" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.66 Safari/537.36" 
222.68.172.190 - - [18/Sep/2013:06:49:57 +0000] "GET /images/my.jpg HTTP/1.1" 200 19939 "http://www.angularjs.cn/A00n" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.66 Safari/537.36" 
222.68.172.190 - - [18/Sep/2013:06:50:08 +0000] "-" 400 0 "-" "-" 
183.195.232.138 - - [18/Sep/2013:06:50:16 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
183.195.232.138 - - [18/Sep/2013:06:50:16 +0000] "HEAD / HTTP/1.1" 200 20 "-" "DNSPod-Monitor/1.0" 
66.249.66.84 - - [18/Sep/2013:06:50:28 +0000] "GET /page/6/ HTTP/1.1" 200 27777 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 
221.130.41.168 - - [18/Sep/2013:06:50:37 +0000] "GET /feed/ HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.66 Safari/537.36" 
157.55.35.40 - - [18/Sep/2013:06:51:13 +0000] "GET /robots.txt HTTP/1.1" 200 150 "-" "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
50.116.27.194 - - [18/Sep/2013:06:51:35 +0000] "POST /wp-cron.php?doing_wp_cron=1379487095.2510800361633300781250 HTTP/1.0" 200 0 "-" "WordPress/3.6; http://blog.fens.me" 
58.215.204.118 - - [18/Sep/2013:06:51:35 +0000] "GET /nodejs-socketio-chat/ HTTP/1.1" 200 10818 "http://www.google.com/url?sa=t&rct=j&q=nodejs%20%E5%BC%82%E6%AD%A5%E5%B9%BF%E6%92%AD&source=web&cd=1&cad=rja&ved=0CCgQFjAA&url=%68%74%74%70%3a%2f%2f%62%6c%6f%67%2e%66%65%6e%73%2e%6d%65%2f%6e%6f%64%65%6a%73%2d%73%6f%63%6b%65%74%69%6f%2d%63%68%61%74%2f&ei=rko5UrylAefOiAe7_IGQBw&usg=AFQjCNG6YWoZsJ_bSj8kTnMHcH51hYQkAA&bvm=bv.52288139,d.aGc" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:36 +0000] "GET /wp-includes/js/jquery/jquery-migrate.min.js?ver=1.2.1 HTTP/1.1" 304 0 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:35 +0000] "GET /wp-includes/js/jquery/jquery.js?ver=1.10.2 HTTP/1.1" 304 0 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:36 +0000] "GET /wp-includes/js/comment-reply.min.js?ver=3.6 HTTP/1.1" 304 0 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:36 +0000] "GET /wp-content/uploads/2013/08/chat.png HTTP/1.1" 200 48968 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:36 +0000] "GET /wp-content/uploads/2013/08/chat2.png HTTP/1.1" 200 59852 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.215.204.118 - - [18/Sep/2013:06:51:37 +0000] "GET /wp-content/uploads/2013/08/socketio.png HTTP/1.1" 200 80493 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0" 
58.248.178.212 - - [18/Sep/2013:06:51:37 +0000] "GET /nodejs-grunt-intro/ HTTP/1.1" 200 51770 "http://blog.fens.me/series-nodejs/" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MDDR; InfoPath.2; .NET4.0C)" 
58.248.178.212 - - [18/Sep/2013:06:51:40 +0000] "GET /wp-includes/js/jquery/jquery-migrate.min.js?ver=1.2.1 HTTP/1.1" 200 7200 "http://blog.fens.me/nodejs-grunt-intro/" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MDDR; InfoPath.2; .NET4.0C)"

3）实现代码：

（1）定义一个bean，用来记录日志数据中的各数据字段

package com.xyg.mapreduce.log; 
 
public class LogBean { 
    private String remote_addr;// 记录客户端的ip地址 
    private String remote_user;// 记录客户端用户名称,忽略属性"-" 
    private String time_local;// 记录访问时间与时区 
    private String request;// 记录请求的url与http协议 
    private String status;// 记录请求状态；成功是200 
    private String body_bytes_sent;// 记录发送给客户端文件主体内容大小 
    private String http_referer;// 用来记录从那个页面链接访问过来的 
    private String http_user_agent;// 记录客户浏览器的相关信息 
 
    private boolean valid = true;// 判断数据是否合法 
 
    public String getRemote_addr() { 
        return remote_addr; 
    } 
 
    public void setRemote_addr(String remote_addr) { 
        this.remote_addr = remote_addr; 
    } 
 
    public String getRemote_user() { 
        return remote_user; 
    } 
 
    public void setRemote_user(String remote_user) { 
        this.remote_user = remote_user; 
    } 
 
    public String getTime_local() { 
        return time_local; 
    } 
 
    public void setTime_local(String time_local) { 
        this.time_local = time_local; 
    } 
 
    public String getRequest() { 
        return request; 
    } 
 
    public void setRequest(String request) { 
        this.request = request; 
    } 
 
    public String getStatus() { 
        return status; 
    } 
 
    public void setStatus(String status) { 
        this.status = status; 
    } 
 
    public String getBody_bytes_sent() { 
        return body_bytes_sent; 
    } 
 
    public void setBody_bytes_sent(String body_bytes_sent) { 
        this.body_bytes_sent = body_bytes_sent; 
    } 
 
    public String getHttp_referer() { 
        return http_referer; 
    } 
 
    public void setHttp_referer(String http_referer) { 
        this.http_referer = http_referer; 
    } 
 
    public String getHttp_user_agent() { 
        return http_user_agent; 
    } 
 
    public void setHttp_user_agent(String http_user_agent) { 
        this.http_user_agent = http_user_agent; 
    } 
 
    public boolean isValid() { 
        return valid; 
    } 
 
    public void setValid(boolean valid) { 
        this.valid = valid; 
    } 
 
    @Override 
    public String toString() { 
        StringBuilder sb = new StringBuilder(); 
        sb.append(this.valid); 
        sb.append("/001").append(this.remote_addr); 
        sb.append("/001").append(this.remote_user); 
        sb.append("/001").append(this.time_local); 
        sb.append("/001").append(this.request); 
        sb.append("/001").append(this.status); 
        sb.append("/001").append(this.body_bytes_sent); 
        sb.append("/001").append(this.http_referer); 
        sb.append("/001").append(this.http_user_agent); 
         
        return sb.toString(); 
    } 
}

（2）编写LogMapper程序

package com.xyg.mapreduce.log;
 
import java.io.IOException; 
import org.apache.hadoop.io.LongWritable; 
import org.apache.hadoop.io.NullWritable; 
import org.apache.hadoop.io.Text; 
import org.apache.hadoop.mapreduce.Mapper; 
 
public class LogMapper extends Mapper<LongWritable, Text, Text, NullWritable>{ 
    Text k = new Text(); 
     
    @Override 
    protected void map(LongWritable key, Text value, Context context) 
            throws IOException, InterruptedException { 
        // 1 获取1行 
        String line = value.toString();         
        // 2 解析日志是否合法 
        LogBean bean = pressLog(line);         
        if (!bean.isValid()) { 
            return; 
        }         
        k.set(bean.toString());         
        // 3 输出 
        context.write(k, NullWritable.get()); 
    } 
 
    // 解析日志 
    private LogBean pressLog(String line) { 
        LogBean logBean = new LogBean(); 
        // 1 截取 
        String[] fields = line.split(" ");    
        if (fields.length > 11) { 
            // 2封装数据 
            logBean.setRemote_addr(fields[0]); 
            logBean.setRemote_user(fields[1]); 
            logBean.setTime_local(fields[3].substring(1)); 
            logBean.setRequest(fields[6]); 
            logBean.setStatus(fields[8]); 
            logBean.setBody_bytes_sent(fields[9]); 
            logBean.setHttp_referer(fields[10]); 
             
            if (fields.length > 12) { 
                logBean.setHttp_user_agent(fields[11] + " "+ fields[12]); 
            }else { 
                logBean.setHttp_user_agent(fields[11]); 
            }            
            // 大于400，HTTP错误 
            if (Integer.parseInt(logBean.getStatus()) >= 400) { 
                logBean.setValid(false); 
            } 
        }else { 
            logBean.setValid(false); 
        } 
        return logBean; 
    } 
}

（3）编写LogDriver程序

package com.xyg.mapreduce.log;
 
import org.apache.hadoop.conf.Configuration; 
import org.apache.hadoop.fs.Path; 
import org.apache.hadoop.io.NullWritable; 
import org.apache.hadoop.io.Text; 
import org.apache.hadoop.mapreduce.Job; 
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 
 
public class LogDriver { 
    public static void main(String[] args) throws Exception { 
        // 1 获取job信息 
        Configuration conf = new Configuration(); 
        Job job = Job.getInstance(conf); 
        // 2 加载jar包 
        job.setJarByClass(LogDriver.class); 
        // 3 关联map 
        job.setMapperClass(LogMapper.class); 
        // 4 设置最终输出类型 
        job.setOutputKeyClass(Text.class); 
        job.setOutputValueClass(NullWritable.class); 
        // 5 设置输入和输出路径 
        FileInputFormat.setInputPaths(job, new Path(args[0])); 
        FileOutputFormat.setOutputPath(job, new Path(args[1])); 
        // 6 提交 
        job.waitForCompletion(true); 
    } 
}

原创文章，作者：ItWorker，如若转载，请注明出处：https://blog.ytso.com/tech/bigdata/9446.html

大数据技术之日志清洗案例详解大数据

7.7 日志清洗案例

7.7.1 简单解析版

7.7.2 复杂解析版

相关推荐

发表回复