Spark - StatsCounter

StatsCounterを使えば簡単に統計用の基本的な値を計算できる．

import org.apache.spark.{SparkContext, SparkConf}

/**
  * Created by neko32 on 2016/01/18.
  */
object Stats {

  def main(args:Array[String]) = {
    val conf = new SparkConf().setMaster("local").setAppName("stats")
    val sc = new SparkContext(conf)
    val rdd = sc.textFile("/user/neko32/spark/study7/nums.txt")
    val stats = rdd.flatMap(l => l.split(":")(1).split(" ").map(_.toInt)).stats
    println(s"mean:${stats.mean}, max:${stats.max}, min:${stats.min}, var:${stats.variance}")
  }
}