spark-submit提交方式测试Demo详解大数据

<properties> 
        <maven.compiler.source>1.7</maven.compiler.source> 
        <maven.compiler.target>1.7</maven.compiler.target> 
        <encoding>UTF-8</encoding> 
        <spark.version>1.6.1</spark.version> 
  </properties> 
 
  <dependencies> 
           <dependency> 
            <groupId>org.apache.spark</groupId> 
            <artifactId>spark-core_2.10</artifactId> 
            <version>${spark.version}</version> 
        </dependency> 
 
        <dependency> 
            <groupId>redis.clients</groupId> 
            <artifactId>jedis</artifactId> 
            <version>2.7.1</version> 
        </dependency> 
 
  </dependencies> 
   
   <build> 
        <plugins> 
            <plugin> 
                <groupId>org.apache.maven.plugins</groupId> 
                <artifactId>maven-compiler-plugin</artifactId> 
                <configuration> 
                    <source>1.7</source> 
                    <target>1.7</target> 
                </configuration> 
            </plugin> 
         
            <plugin> 
                <groupId>org.apache.maven.plugins</groupId> 
                <artifactId>maven-shade-plugin</artifactId> 
                <version>2.4.3</version> 
                <executions> 
                    <execution> 
                        <phase>package</phase> 
                        <goals> 
                            <goal>shade</goal> 
                        </goals> 
                        <configuration> 
                            <filters> 
                                <filter> 
                                    <artifact>*:*</artifact> 
                                    <excludes> 
                                        <exclude>META-INF/*.SF</exclude> 
                                        <exclude>META-INF/*.DSA</exclude> 
                                        <exclude>META-INF/*.RSA</exclude> 
                                    </excludes> 
                                </filter> 
                            </filters> 
                        </configuration> 
                    </execution> 
                </executions> 
            </plugin> 
        </plugins> 
    </build>

import java.util.ArrayList; 
import java.util.List; 
 
import org.apache.spark.SparkConf; 
import org.apache.spark.api.java.JavaRDD; 
import org.apache.spark.api.java.JavaSparkContext; 
import org.apache.spark.api.java.function.Function; 
import org.apache.spark.api.java.function.Function2; 
 
import redis.clients.jedis.Jedis; 
 
/**  
 * Computes an approximation to pi 
 * Usage: JavaSparkPi [slices] 
 */ 
public final class JavaSparkPi { 
 
  public static void main(String[] args) throws Exception { 
    SparkConf sparkConf = new SparkConf().setAppName("JavaSparkPi")/*.setMaster("local[2]")*/; 
    JavaSparkContext jsc = new JavaSparkContext(sparkConf); 
     
    Jedis jedis = new Jedis("192.168.49.151",19000); 
    int slices = (args.length == 1) ? Integer.parseInt(args[0]) : 2; 
    int n = 100000 * slices; 
    List<Integer> l = new ArrayList<Integer>(n); 
    for (int i = 0; i < n; i++) { 
      l.add(i); 
    } 
 
    JavaRDD<Integer> dataSet = jsc.parallelize(l, slices); 
 
    int count = dataSet.map(new Function<Integer, Integer>() { 
      @Override 
      public Integer call(Integer integer) { 
        double x = Math.random() * 2 - 1; 
        double y = Math.random() * 2 - 1; 
        return (x * x + y * y < 1) ? 1 : 0; 
      } 
    }).reduce(new Function2<Integer, Integer, Integer>() { 
      @Override 
      public Integer call(Integer integer, Integer integer2) { 
        return integer + integer2; 
      } 
    }); 
 
    jedis.set("Pi", String.valueOf(4.0 * count / n)); 
    System.out.println("Pi is roughly " + 4.0 * count / n); 
     
    jsc.stop(); 
  } 
}