Summarizer

Tools for vectorized statistics on MLlib Vectors.

The methods in this package provide various statistics for Vectors contained inside DataFrames.

This class lets users pick the statistics they would like to extract for a given column. Here is an example in Scala:

import org.apache.spark.ml.linalg._
import org.apache.spark.sql.Row
val dataframe = ... // Some dataframe containing a feature column and a weight column
val multiStatsDF = dataframe.select(
    Summarizer.metrics("min", "max", "count").summary($"features", $"weight")
val Row(Row(minVec, maxVec, count)) = multiStatsDF.first()

If one wants to get a single metric, shortcuts are also available:

val meanDF = dataframe.select(Summarizer.mean($"features"))
val Row(meanVec) = meanDF.first()

Note: Currently, the performance of this interface is about 2x~3x slower than using the RDD interface.

Annotations: @Experimental() @Since( "2.3.0" )
Source: Summarizer.scala

Linear Supertypes

Logging, AnyRef, Any

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def count(col: Column): Column

Annotations
@Since( "2.3.0" )
def count(col: Column, weightCol: Column): Column

Annotations
@Since( "2.3.0" )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean = false): Boolean

Attributes
protected
Definition Classes
Logging
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes
protected
Definition Classes
Logging
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def max(col: Column): Column

Annotations
@Since( "2.3.0" )
def max(col: Column, weightCol: Column): Column

Annotations
@Since( "2.3.0" )
def mean(col: Column): Column

Annotations
@Since( "2.3.0" )
def mean(col: Column, weightCol: Column): Column

Annotations
@Since( "2.3.0" )
def metrics(metrics: String*): SummaryBuilder

Given a list of metrics, provides a builder that it turns computes metrics from a column.
Given a list of metrics, provides a builder that it turns computes metrics from a column.
See the documentation of Summarizer for an example.
The following metrics are accepted (case sensitive):
- mean: a vector that contains the coefficient-wise mean.
- variance: a vector tha contains the coefficient-wise variance.
- count: the count of all vectors seen.
- numNonzeros: a vector with the number of non-zeros for each coefficients
- max: the maximum for each coefficient.
- min: the minimum for each coefficient.
- normL2: the Euclidian norm for each coefficient.
- normL1: the L1 norm of each coefficient (sum of the absolute values).
metrics
metrics that can be provided.
returns
a builder.

Annotations
@Since( "2.3.0" ) @varargs()
Exceptions thrown
IllegalArgumentException if one of the metric names is not understood. Note: Currently, the performance of this interface is about 2x~3x slower then using the RDD interface.
def min(col: Column): Column

Annotations
@Since( "2.3.0" )
def min(col: Column, weightCol: Column): Column

Annotations
@Since( "2.3.0" )
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def normL1(col: Column): Column

Annotations
@Since( "2.3.0" )
def normL1(col: Column, weightCol: Column): Column

Annotations
@Since( "2.3.0" )
def normL2(col: Column): Column

Annotations
@Since( "2.3.0" )
def normL2(col: Column, weightCol: Column): Column

Annotations
@Since( "2.3.0" )
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def numNonZeros(col: Column): Column

Annotations
@Since( "2.3.0" )
def numNonZeros(col: Column, weightCol: Column): Column

Annotations
@Since( "2.3.0" )
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
def variance(col: Column): Column

Annotations
@Since( "2.3.0" )
def variance(col: Column, weightCol: Column): Column

Annotations
@Since( "2.3.0" )
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package stat

object Summarizer extends Logging

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def count(col: Column): Column

def count(col: Column, weightCol: Column): Column

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean = false): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

final def isInstanceOf[T0]: Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

def max(col: Column): Column

def max(col: Column, weightCol: Column): Column

def mean(col: Column): Column

def mean(col: Column, weightCol: Column): Column

def metrics(metrics: String*): SummaryBuilder

def min(col: Column): Column

def min(col: Column, weightCol: Column): Column

final def ne(arg0: AnyRef): Boolean

def normL1(col: Column): Column

def normL1(col: Column, weightCol: Column): Column

def normL2(col: Column): Column

def normL2(col: Column, weightCol: Column): Column

final def notify(): Unit

final def notifyAll(): Unit

def numNonZeros(col: Column): Column

def numNonZeros(col: Column, weightCol: Column): Column

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

def variance(col: Column): Column

def variance(col: Column, weightCol: Column): Column

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Members