Source code for mars.tensor.statistics.cov

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 1999-2021 Alibaba Group Holding Ltd.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.

import warnings

import numpy as np

from ..core import Tensor
from ..datasource import tensor as astensor, array
from ..base.squeeze import squeeze
from ..base.where import where
from .average import average

[docs]def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, aweights=None): """ Estimate a covariance matrix, given data and weights. Covariance indicates the level to which two variables vary together. If we examine N-dimensional samples, :math:`X = [x_1, x_2, ... x_N]^T`, then the covariance matrix element :math:`C_{ij}` is the covariance of :math:`x_i` and :math:`x_j`. The element :math:`C_{ii}` is the variance of :math:`x_i`. See the notes for an outline of the algorithm. Parameters ---------- m : array_like A 1-D or 2-D array containing multiple variables and observations. Each row of `m` represents a variable, and each column a single observation of all those variables. Also see `rowvar` below. y : array_like, optional An additional set of variables and observations. `y` has the same form as that of `m`. rowvar : bool, optional If `rowvar` is True (default), then each row represents a variable, with observations in the columns. Otherwise, the relationship is transposed: each column represents a variable, while the rows contain observations. bias : bool, optional Default normalization (False) is by ``(N - 1)``, where ``N`` is the number of observations given (unbiased estimate). If `bias` is True, then normalization is by ``N``. These values can be overridden by using the keyword ``ddof`` in numpy versions >= 1.5. ddof : int, optional If not ``None`` the default value implied by `bias` is overridden. Note that ``ddof=1`` will return the unbiased estimate, even if both `fweights` and `aweights` are specified, and ``ddof=0`` will return the simple average. See the notes for the details. The default value is ``None``. fweights : array_like, int, optional 1-D tensor of integer freguency weights; the number of times each observation vector should be repeated. aweights : array_like, optional 1-D tensor of observation vector weights. These relative weights are typically large for observations considered "important" and smaller for observations considered less "important". If ``ddof=0`` the array of weights can be used to assign probabilities to observation vectors. Returns ------- out : Tensor The covariance matrix of the variables. See Also -------- corrcoef : Normalized covariance matrix Notes ----- Assume that the observations are in the columns of the observation array `m` and let ``f = fweights`` and ``a = aweights`` for brevity. The steps to compute the weighted covariance are as follows:: >>> w = f * a >>> v1 = mt.sum(w) >>> v2 = mt.sum(w * a) >>> m -= mt.sum(m * w, axis=1, keepdims=True) / v1 >>> cov = * w, m.T) * v1 / (v1**2 - ddof * v2) Note that when ``a == 1``, the normalization factor ``v1 / (v1**2 - ddof * v2)`` goes over to ``1 / (np.sum(f) - ddof)`` as it should. Examples -------- Consider two variables, :math:`x_0` and :math:`x_1`, which correlate perfectly, but in opposite directions: >>> import mars.tensor as mt >>> x = mt.array([[0, 2], [1, 1], [2, 0]]).T >>> x.execute() array([[0, 1, 2], [2, 1, 0]]) Note how :math:`x_0` increases while :math:`x_1` decreases. The covariance matrix shows this clearly: >>> mt.cov(x).execute() array([[ 1., -1.], [-1., 1.]]) Note that element :math:`C_{0,1}`, which shows the correlation between :math:`x_0` and :math:`x_1`, is negative. Further, note how `x` and `y` are combined: >>> x = [-2.1, -1, 4.3] >>> y = [3, 1.1, 0.12] >>> X = mt.stack((x, y), axis=0) >>> print(mt.cov(X).execute()) [[ 11.71 -4.286 ] [ -4.286 2.14413333]] >>> print(mt.cov(x, y).execute()) [[ 11.71 -4.286 ] [ -4.286 2.14413333]] >>> print(mt.cov(x).execute()) 11.71 """ from ..merge import vstack from ..linalg import dot if ddof is not None and ddof != int(ddof): raise ValueError("ddof must be integer") m = astensor(m) if m.ndim > 2: raise ValueError("m has more than 2 dimensions") if y is None: dtype = np.result_type(m.dtype, np.float64) else: y = astensor(y) if y.ndim > 2: raise ValueError("y has more than 2 dimensions") dtype = np.result_type(m.dtype, y.dtype, np.float64) X = array(m, ndmin=2, dtype=dtype) if not rowvar and X.shape[0] != 1: X = X.T if y is not None: y = array(y, copy=False, ndmin=2, dtype=dtype) if not rowvar and y.shape[0] != 1: y = y.T X = vstack((X, y)) if ddof is None: if bias == 0: ddof = 1 else: ddof = 0 # Get the product of frequencies and weights w = None if fweights is not None: fweights = astensor(fweights, dtype=float) if fweights.ndim > 1: raise RuntimeError("cannot handle multidimensional fweights") if fweights.shape[0] != X.shape[1]: raise RuntimeError("incompatible numbers of samples and fweights") if any(fweights < 0): raise ValueError("fweights cannot be negative") w = fweights if aweights is not None: aweights = astensor(aweights, dtype=float) if aweights.ndim > 1: raise RuntimeError("cannot handle multidimensional aweights") if aweights.shape[0] != X.shape[1]: raise RuntimeError("incompatible numbers of samples and aweights") if any(aweights < 0): raise ValueError("aweights cannot be negative") if w is None: w = aweights else: w *= aweights avg, w_sum = average(X, axis=1, weights=w, returned=True) w_sum = w_sum[0] # Determine the normalization if w is None: fact = X.shape[1] - ddof elif ddof == 0: fact = w_sum elif aweights is None: fact = w_sum - ddof else: fact = w_sum - ddof * sum(w * aweights) / w_sum X -= avg[:, None] if w is None: X_T = X.T else: X_T = (X * w).T c = dot(X, X_T.conj()) if isinstance(fact, Tensor): fact = where(fact <= 0, 0.0, fact) fact = fact.astype(float) else: if fact <= 0: warnings.warn( "Degrees of freedom <= 0 for slice", RuntimeWarning, stacklevel=2 ) fact = 0.0 fact = np.float64(fact) c = c * (1.0 / fact) return squeeze(c)