Source code for mars.learn.metrics.pairwise.euclidean

# Copyright 1999-2021 Alibaba Group Holding Ltd.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np

from .... import opcodes as OperandDef
from .... import tensor as mt
from ....config import options
from ....core import recursive_tile
from ....serialization.serializables import KeyField, BoolField
from ....tensor.core import TensorOrder
from ....utils import has_unknown_shape
from ...utils import check_array
from ...utils.extmath import row_norms
from .core import PairwiseDistances

class EuclideanDistances(PairwiseDistances):

    _x = KeyField("X")
    _y = KeyField("Y")
    _x_norm_squared = KeyField("X_norm_squared")
    _y_norm_squared = KeyField("Y_norm_squared")
    _squared = BoolField("squared")

    def __init__(

    def x(self):
        return self._x

    def y(self):
        return self._y

    def x_norm_squared(self):
        return self._x_norm_squared

    def y_norm_squared(self):
        return self._y_norm_squared

    def squared(self):
        return self._squared

    def _set_inputs(self, inputs):
        input_iter = iter(self._inputs)
        self._x = next(input_iter)
        if self._y is not None:
            self._y = next(input_iter)
        if self._x_norm_squared is not None:
            self._x_norm_squared = next(input_iter)
        if self._y_norm_squared is not None:
            self._y_norm_squared = next(input_iter)

    def __call__(self, X, Y=None, Y_norm_squared=None, X_norm_squared=None):
        # If norms are passed as float32, they are unused. If arrays are passed as
        # float32, norms needs to be recomputed on upcast chunks.
        # TODO: use a float64 accumulator in row_norms to avoid the latter.
        if X_norm_squared is not None:
            XX = check_array(X_norm_squared)
            if XX.shape == (1, X.shape[0]):
                XX = XX.T
            elif XX.shape != (X.shape[0], 1):
                raise ValueError("Incompatible dimensions for X and X_norm_squared")
            if XX.dtype == np.float32:
                XX = self._x_norm_squared = None
            XX = None

        if X is Y and XX is not None:
            # shortcut in the common case euclidean_distances(X, X)
            YY = XX.T
        elif Y_norm_squared is not None:
            YY = mt.atleast_2d(Y_norm_squared)

            if YY.shape != (1, Y.shape[0]):
                raise ValueError("Incompatible dimensions for Y and Y_norm_squared")
            if YY.dtype == np.float32:
                YY = self._y_norm_squared = None
            YY = None

        inputs = [X, Y]
        if XX is not None:
        if YY is not None:
        return self.new_tensor(
            inputs, shape=(X.shape[0], Y.shape[0]), order=TensorOrder.C_ORDER

    def tile(cls, op):
        X, Y = op.x, op.y
        out = op.outputs[0]

        if X.dtype == np.float32:
            if has_unknown_shape(X, Y):
            # rechunk
            new_nsplit = max(max(X.nsplits[0]) // 2, 1)
            X = yield from recursive_tile(X.rechunk({0: new_nsplit}).astype(np.float64))
            if Y is not X:
                new_nsplit = max(max(Y.nsplits[0]) // 2, 1)
                Y = yield from recursive_tile(
                    Y.rechunk({0: new_nsplit}).astype(np.float64)

        XX = op.x_norm_squared
        if XX is None:
            XX = row_norms(X, squared=True)[:, np.newaxis]
        YY = op.y_norm_squared
        if YY is None:
            YY = row_norms(Y, squared=True)[np.newaxis, :]

        X, Y = yield from cls._adjust_chunk_sizes(op, X, Y, out)

        distances = -2 *
        if distances.issparse():
            distances = distances.todense()
        distances += XX
        distances += YY
        distances = mt.maximum(distances, 0)

        if X is Y or X.key == Y.key:
            mt.fill_diagonal(distances, 0)

        distances = distances if op.squared else mt.sqrt(distances)
        distances = distances.astype(out.dtype, copy=False)
        return [(yield from recursive_tile(distances))]

[docs]def euclidean_distances( X, Y=None, Y_norm_squared=None, squared=False, X_norm_squared=None ): """ Considering the rows of X (and Y=X) as vectors, compute the distance matrix between each pair of vectors. For efficiency reasons, the euclidean distance between a pair of row vector x and y is computed as:: dist(x, y) = sqrt(dot(x, x) - 2 * dot(x, y) + dot(y, y)) This formulation has two advantages over other ways of computing distances. First, it is computationally efficient when dealing with sparse data. Second, if one argument varies but the other remains unchanged, then `dot(x, x)` and/or `dot(y, y)` can be pre-computed. However, this is not the most precise way of doing this computation, and the distance matrix returned by this function may not be exactly symmetric as required by, e.g., ``scipy.spatial.distance`` functions. Read more in the :ref:`User Guide <metrics>`. Parameters ---------- X : {array-like, sparse matrix}, shape (n_samples_1, n_features) Y : {array-like, sparse matrix}, shape (n_samples_2, n_features) Y_norm_squared : array-like, shape (n_samples_2, ), optional Pre-computed dot-products of vectors in Y (e.g., ``(Y**2).sum(axis=1)``) May be ignored in some cases, see the note below. squared : boolean, optional Return squared Euclidean distances. X_norm_squared : array-like, shape = [n_samples_1], optional Pre-computed dot-products of vectors in X (e.g., ``(X**2).sum(axis=1)``) May be ignored in some cases, see the note below. Notes ----- To achieve better accuracy, `X_norm_squared` and `Y_norm_squared` may be unused if they are passed as ``float32``. Returns ------- distances : tensor, shape (n_samples_1, n_samples_2) Examples -------- >>> from mars.learn.metrics.pairwise import euclidean_distances >>> X = [[0, 1], [1, 1]] >>> # distance between rows of X >>> euclidean_distances(X, X).execute() array([[0., 1.], [1., 0.]]) >>> # get distance to origin >>> euclidean_distances(X, [[0, 0]]).execute() array([[1. ], [1.41421356]]) See also -------- paired_distances : distances betweens pairs of elements of X and Y. """ if X.dtype == np.float32: if Y is None: dtype = X.dtype elif Y.dtype == np.float32: dtype = np.float32 else: dtype = np.float64 else: dtype = np.float64 X, Y = EuclideanDistances.check_pairwise_arrays(X, Y) op = EuclideanDistances( x=X, y=Y, x_norm_squared=X_norm_squared, y_norm_squared=Y_norm_squared, squared=squared, dtype=np.dtype(dtype), chunk_store_limit=options.chunk_store_limit, ) return op(X, Y=Y, Y_norm_squared=Y_norm_squared, X_norm_squared=X_norm_squared)