多处理距离矩阵cdist Scipy

如何解决多处理距离矩阵cdist Scipy

我必须计算两个字符串向量之间的多个距离。我正在使用cdist来完成此操作，而且速度很快，但是对于大型数组，这需要时间，因此我不得不多次重复该操作。 cdist中的Scipy仅使用了我的核心之一。我如何利用所有核心来计算所有距离并更快地获得最终的3D矩阵？ 我想要实现的是要么在不同的过程中分别计算每个距离矩阵，要么利用我的所有核心来计算一个距离矩阵。

您可以在此处运行我要达到的目标的示例。

import numpy as np
first = np.array(["hello","hello","hellllo"])
second = np.array(["hlo","halo","alle"])

def diff_len(string1,string2):
    return abs(len(string1) - len(string2))

def diff_len2(string1,string2):
    return abs(len(string1) - len(string2)*2)

def diff_len_square(string1,string2):
    return abs(len(string1) - len(string2)**2)

def minimum_nw(*sequences):
    return -max(map(len,sequences)) * 1


def maximum_nw(*sequences):
    return max(map(len,sequences))


def normalized_distance(distance,*sequences):
    """Get distance from 0 to 1
    """
    minimum = minimum_nw(*sequences)
    maximum = maximum_nw(*sequences)

    if maximum == 0:
        return 0
    return (distance - minimum) / (maximum - minimum)


@njit
def NeedlemanWunschDP(dist_mat,s1,s2):
    for i in range(1,len(s1) + 1):
        for j in range(1,len(s2) + 1):
            match = dist_mat[i - 1,j - 1] + (s1[i-1] == s2[j-1])
            delete = dist_mat[i - 1,j] - gap_
            insert = dist_mat[i,j - 1] - gap_
            dist_mat[i,j] = max(match,delete,insert)


def NeedleW(s1,s2,mode='raw'):
    dist_mat_ = np.empty(
        (len(s1) + 1,len(s2) + 1),dtype=np.int64,)

    # DP initialization
    for i in range(len(s1) + 1):
        dist_mat_[i,0] = -(i * gap_)

    # DP initialization
    for j in range(len(s2) + 1):
        dist_mat_[0,j] = -(j * gap_)

    # Transform the strings to fast integer arrays
    tmp_s1 = np.array([ord(e) for e in s1],dtype=np.int64)
    tmp_s2 = np.array([ord(e) for e in s2],dtype=np.int64)
    # Needleman-Wunsch DP calculation
    NeedlemanWunschDP(dist_mat_,tmp_s1,tmp_s2)
    distance_ = -1 * dist_mat_[dist_mat_.shape[0] - 1,dist_mat_.shape[1] - 1]

    if mode == 'norm':
        return normalized_distance(distance_,s2)

    return - float(distance_)

first = np.array(["hello","alle"])
first = np.tile(first,20)
second = np.array(second,20)

d0 = cdist(first[:,np.newaxis],second[:,lambda a,b: diff_len(a[0],b[0]))
d1 = cdist(first[:,b: diff_len2(a[0],b[0]))
d2 = cdist(first[:,b: diff_len_square(a[0],b[0]))
d3 = cdist(first[:,b: NeedleW(a[0],b[0],'norm'))

mat3D = np.stack((d0,d1,d2,d3))