https://github.com/RaRe-Technologies/gensim
Tip revision: c2cae87b4792dde985f2bcd45dcda78afbb7a959 authored by Radim Řehůřek on 18 February 2019, 16:53:27 UTC
NMF tutorial fixes
NMF tutorial fixes
Tip revision: c2cae87
matutils.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2011 Radim Rehurek <radimrehurek@seznam.cz>
# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html
"""Math helper functions."""
from __future__ import with_statement
from itertools import chain
import logging
import math
from gensim import utils
from gensim.utils import deprecated
import numpy as np
import scipy.sparse
from scipy.stats import entropy
import scipy.linalg
from scipy.linalg.lapack import get_lapack_funcs
from scipy.linalg.special_matrices import triu
from scipy.special import psi # gamma function utils
from six import iteritems, itervalues, string_types
from six.moves import zip, range
logger = logging.getLogger(__name__)
def blas(name, ndarray):
"""Helper for getting the appropriate BLAS function, using :func:`scipy.linalg.get_blas_funcs`.
Parameters
----------
name : str
Name(s) of BLAS functions, without the type prefix.
ndarray : numpy.ndarray
Arrays can be given to determine optimal prefix of BLAS routines.
Returns
-------
object
BLAS function for the needed operation on the given data type.
"""
return scipy.linalg.get_blas_funcs((name,), (ndarray,))[0]
def argsort(x, topn=None, reverse=False):
"""Efficiently calculate indices of the `topn` smallest elements in array `x`.
Parameters
----------
x : array_like
Array to get the smallest element indices from.
topn : int, optional
Number of indices of the smallest (greatest) elements to be returned.
If not given, indices of all elements will be returned in ascending (descending) order.
reverse : bool, optional
Return the `topn` greatest elements in descending order,
instead of smallest elements in ascending order?
Returns
-------
numpy.ndarray
Array of `topn` indices that sort the array in the requested order.
"""
x = np.asarray(x) # unify code path for when `x` is not a np array (list, tuple...)
if topn is None:
topn = x.size
if topn <= 0:
return []
if reverse:
x = -x
if topn >= x.size or not hasattr(np, 'argpartition'):
return np.argsort(x)[:topn]
# np >= 1.8 has a fast partial argsort, use that!
most_extreme = np.argpartition(x, topn)[:topn]
return most_extreme.take(np.argsort(x.take(most_extreme))) # resort topn into order
def corpus2csc(corpus, num_terms=None, dtype=np.float64, num_docs=None, num_nnz=None, printprogress=0):
"""Convert a streamed corpus in bag-of-words format into a sparse matrix `scipy.sparse.csc_matrix`,
with documents as columns.
Notes
-----
If the number of terms, documents and non-zero elements is known, you can pass
them here as parameters and a (much) more memory efficient code path will be taken.
Parameters
----------
corpus : iterable of iterable of (int, number)
Input corpus in BoW format
num_terms : int, optional
Number of terms in `corpus`. If provided, the `corpus.num_terms` attribute (if any) will be ignored.
dtype : data-type, optional
Data type of output CSC matrix.
num_docs : int, optional
Number of documents in `corpus`. If provided, the `corpus.num_docs` attribute (in any) will be ignored.
num_nnz : int, optional
Number of non-zero elements in `corpus`. If provided, the `corpus.num_nnz` attribute (if any) will be ignored.
printprogress : int, optional
Log a progress message at INFO level once every `printprogress` documents. 0 to turn off progress logging.
Returns
-------
scipy.sparse.csc_matrix
`corpus` converted into a sparse CSC matrix.
See Also
--------
:class:`~gensim.matutils.Sparse2Corpus`
Convert sparse format to Gensim corpus format.
"""
try:
# if the input corpus has the `num_nnz`, `num_docs` and `num_terms` attributes
# (as is the case with MmCorpus for example), we can use a more efficient code path
if num_terms is None:
num_terms = corpus.num_terms
if num_docs is None:
num_docs = corpus.num_docs
if num_nnz is None:
num_nnz = corpus.num_nnz
except AttributeError:
pass # not a MmCorpus...
if printprogress:
logger.info("creating sparse matrix from corpus")
if num_terms is not None and num_docs is not None and num_nnz is not None:
# faster and much more memory-friendly version of creating the sparse csc
posnow, indptr = 0, [0]
indices = np.empty((num_nnz,), dtype=np.int32) # HACK assume feature ids fit in 32bit integer
data = np.empty((num_nnz,), dtype=dtype)
for docno, doc in enumerate(corpus):
if printprogress and docno % printprogress == 0:
logger.info("PROGRESS: at document #%i/%i", docno, num_docs)
posnext = posnow + len(doc)
indices[posnow: posnext] = [feature_id for feature_id, _ in doc]
data[posnow: posnext] = [feature_weight for _, feature_weight in doc]
indptr.append(posnext)
posnow = posnext
assert posnow == num_nnz, "mismatch between supplied and computed number of non-zeros"
result = scipy.sparse.csc_matrix((data, indices, indptr), shape=(num_terms, num_docs), dtype=dtype)
else:
# slower version; determine the sparse matrix parameters during iteration
num_nnz, data, indices, indptr = 0, [], [], [0]
for docno, doc in enumerate(corpus):
if printprogress and docno % printprogress == 0:
logger.info("PROGRESS: at document #%i", docno)
indices.extend(feature_id for feature_id, _ in doc)
data.extend(feature_weight for _, feature_weight in doc)
num_nnz += len(doc)
indptr.append(num_nnz)
if num_terms is None:
num_terms = max(indices) + 1 if indices else 0
num_docs = len(indptr) - 1
# now num_docs, num_terms and num_nnz contain the correct values
data = np.asarray(data, dtype=dtype)
indices = np.asarray(indices)
result = scipy.sparse.csc_matrix((data, indices, indptr), shape=(num_terms, num_docs), dtype=dtype)
return result
def pad(mat, padrow, padcol):
"""Add additional rows/columns to `mat`. The new rows/columns will be initialized with zeros.
Parameters
----------
mat : numpy.ndarray
Input 2D matrix
padrow : int
Number of additional rows
padcol : int
Number of additional columns
Returns
-------
numpy.matrixlib.defmatrix.matrix
Matrix with needed padding.
"""
if padrow < 0:
padrow = 0
if padcol < 0:
padcol = 0
rows, cols = mat.shape
return np.bmat([
[mat, np.matrix(np.zeros((rows, padcol)))],
[np.matrix(np.zeros((padrow, cols + padcol)))],
])
def zeros_aligned(shape, dtype, order='C', align=128):
"""Get array aligned at `align` byte boundary in memory.
Parameters
----------
shape : int or (int, int)
Shape of array.
dtype : data-type
Data type of array.
order : {'C', 'F'}, optional
Whether to store multidimensional data in C- or Fortran-contiguous (row- or column-wise) order in memory.
align : int, optional
Boundary for alignment in bytes.
Returns
-------
numpy.ndarray
Aligned array.
"""
nbytes = np.prod(shape, dtype=np.int64) * np.dtype(dtype).itemsize
buffer = np.zeros(nbytes + align, dtype=np.uint8) # problematic on win64 ("maximum allowed dimension exceeded")
start_index = -buffer.ctypes.data % align
return buffer[start_index: start_index + nbytes].view(dtype).reshape(shape, order=order)
def ismatrix(m):
"""Check whether `m` is a 2D `numpy.ndarray` or `scipy.sparse` matrix.
Parameters
----------
m : object
Object to check.
Returns
-------
bool
Is `m` a 2D `numpy.ndarray` or `scipy.sparse` matrix.
"""
return isinstance(m, np.ndarray) and m.ndim == 2 or scipy.sparse.issparse(m)
def any2sparse(vec, eps=1e-9):
"""Convert a numpy.ndarray or `scipy.sparse` vector into the Gensim bag-of-words format.
Parameters
----------
vec : {`numpy.ndarray`, `scipy.sparse`}
Input vector
eps : float, optional
Value used for threshold, all coordinates less than `eps` will not be presented in result.
Returns
-------
list of (int, float)
Vector in BoW format.
"""
if isinstance(vec, np.ndarray):
return dense2vec(vec, eps)
if scipy.sparse.issparse(vec):
return scipy2sparse(vec, eps)
return [(int(fid), float(fw)) for fid, fw in vec if np.abs(fw) > eps]
def scipy2scipy_clipped(matrix, topn, eps=1e-9):
"""Get the 'topn' elements of the greatest magnitude (absolute value) from a `scipy.sparse` vector or matrix.
Parameters
----------
matrix : `scipy.sparse`
Input vector or matrix (1D or 2D sparse array).
topn : int
Number of greatest elements, in absolute value, to return.
eps : float
Ignored.
Returns
-------
`scipy.sparse.csr.csr_matrix`
Clipped matrix.
"""
if not scipy.sparse.issparse(matrix):
raise ValueError("'%s' is not a scipy sparse vector." % matrix)
if topn <= 0:
return scipy.sparse.csr_matrix([])
# Return clipped sparse vector if input is a sparse vector.
if matrix.shape[0] == 1:
# use np.argpartition/argsort and only form tuples that are actually returned.
biggest = argsort(abs(matrix.data), topn, reverse=True)
indices, data = matrix.indices.take(biggest), matrix.data.take(biggest)
return scipy.sparse.csr_matrix((data, indices, [0, len(indices)]))
# Return clipped sparse matrix if input is a matrix, processing row by row.
else:
matrix_indices = []
matrix_data = []
matrix_indptr = [0]
# calling abs() on entire matrix once is faster than calling abs() iteratively for each row
matrix_abs = abs(matrix)
for i in range(matrix.shape[0]):
v = matrix.getrow(i)
v_abs = matrix_abs.getrow(i)
# Sort and clip each row vector first.
biggest = argsort(v_abs.data, topn, reverse=True)
indices, data = v.indices.take(biggest), v.data.take(biggest)
# Store the topn indices and values of each row vector.
matrix_data.append(data)
matrix_indices.append(indices)
matrix_indptr.append(matrix_indptr[-1] + min(len(indices), topn))
matrix_indices = np.concatenate(matrix_indices).ravel()
matrix_data = np.concatenate(matrix_data).ravel()
# Instantiate and return a sparse csr_matrix which preserves the order of indices/data.
return scipy.sparse.csr.csr_matrix(
(matrix_data, matrix_indices, matrix_indptr),
shape=(matrix.shape[0], np.max(matrix_indices) + 1)
)
def scipy2sparse(vec, eps=1e-9):
"""Convert a scipy.sparse vector into the Gensim bag-of-words format.
Parameters
----------
vec : `scipy.sparse`
Sparse vector.
eps : float, optional
Value used for threshold, all coordinates less than `eps` will not be presented in result.
Returns
-------
list of (int, float)
Vector in Gensim bag-of-words format.
"""
vec = vec.tocsr()
assert vec.shape[0] == 1
return [(int(pos), float(val)) for pos, val in zip(vec.indices, vec.data) if np.abs(val) > eps]
class Scipy2Corpus(object):
"""Convert a sequence of dense/sparse vectors into a streamed Gensim corpus object.
See Also
--------
:func:`~gensim.matutils.corpus2csc`
Convert corpus in Gensim format to `scipy.sparse.csc` matrix.
"""
def __init__(self, vecs):
"""
Parameters
----------
vecs : iterable of {`numpy.ndarray`, `scipy.sparse`}
Input vectors.
"""
self.vecs = vecs
def __iter__(self):
for vec in self.vecs:
if isinstance(vec, np.ndarray):
yield full2sparse(vec)
else:
yield scipy2sparse(vec)
def __len__(self):
return len(self.vecs)
def sparse2full(doc, length):
"""Convert a document in Gensim bag-of-words format into a dense numpy array.
Parameters
----------
doc : list of (int, number)
Document in BoW format.
length : int
Vector dimensionality. This cannot be inferred from the BoW, and you must supply it explicitly.
This is typically the vocabulary size or number of topics, depending on how you created `doc`.
Returns
-------
numpy.ndarray
Dense numpy vector for `doc`.
See Also
--------
:func:`~gensim.matutils.full2sparse`
Convert dense array to gensim bag-of-words format.
"""
result = np.zeros(length, dtype=np.float32) # fill with zeroes (default value)
# convert indices to int as numpy 1.12 no longer indexes by floats
doc = ((int(id_), float(val_)) for (id_, val_) in doc)
doc = dict(doc)
# overwrite some of the zeroes with explicit values
result[list(doc)] = list(itervalues(doc))
return result
def full2sparse(vec, eps=1e-9):
"""Convert a dense numpy array into the Gensim bag-of-words format.
Parameters
----------
vec : numpy.ndarray
Dense input vector.
eps : float
Feature weight threshold value. Features with `abs(weight) < eps` are considered sparse and
won't be included in the BOW result.
Returns
-------
list of (int, float)
BoW format of `vec`, with near-zero values omitted (sparse vector).
See Also
--------
:func:`~gensim.matutils.sparse2full`
Convert a document in Gensim bag-of-words format into a dense numpy array.
"""
vec = np.asarray(vec, dtype=float)
nnz = np.nonzero(abs(vec) > eps)[0]
return list(zip(nnz, vec.take(nnz)))
dense2vec = full2sparse
def full2sparse_clipped(vec, topn, eps=1e-9):
"""Like :func:`~gensim.matutils.full2sparse`, but only return the `topn` elements of the greatest magnitude (abs).
This is more efficient that sorting a vector and then taking the greatest values, especially
where `len(vec) >> topn`.
Parameters
----------
vec : numpy.ndarray
Input dense vector
topn : int
Number of greatest (abs) elements that will be presented in result.
eps : float
Threshold value, if coordinate in `vec` < eps, this will not be presented in result.
Returns
-------
list of (int, float)
Clipped vector in BoW format.
See Also
--------
:func:`~gensim.matutils.full2sparse`
Convert dense array to gensim bag-of-words format.
"""
# use np.argpartition/argsort and only form tuples that are actually returned.
# this is about 40x faster than explicitly forming all 2-tuples to run sort() or heapq.nlargest() on.
if topn <= 0:
return []
vec = np.asarray(vec, dtype=float)
nnz = np.nonzero(abs(vec) > eps)[0]
biggest = nnz.take(argsort(abs(vec).take(nnz), topn, reverse=True))
return list(zip(biggest, vec.take(biggest)))
def corpus2dense(corpus, num_terms, num_docs=None, dtype=np.float32):
"""Convert corpus into a dense numpy 2D array, with documents as columns.
Parameters
----------
corpus : iterable of iterable of (int, number)
Input corpus in the Gensim bag-of-words format.
num_terms : int
Number of terms in the dictionary. X-axis of the resulting matrix.
num_docs : int, optional
Number of documents in the corpus. If provided, a slightly more memory-efficient code path is taken.
Y-axis of the resulting matrix.
dtype : data-type, optional
Data type of the output matrix.
Returns
-------
numpy.ndarray
Dense 2D array that presents `corpus`.
See Also
--------
:class:`~gensim.matutils.Dense2Corpus`
Convert dense matrix to Gensim corpus format.
"""
if num_docs is not None:
# we know the number of documents => don't bother column_stacking
docno, result = -1, np.empty((num_terms, num_docs), dtype=dtype)
for docno, doc in enumerate(corpus):
result[:, docno] = sparse2full(doc, num_terms)
assert docno + 1 == num_docs
else:
# The below used to be a generator, but NumPy deprecated generator as of 1.16 with:
# """
# FutureWarning: arrays to stack must be passed as a "sequence" type such as list or tuple.
# Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.
# """
result = np.column_stack([sparse2full(doc, num_terms) for doc in corpus])
return result.astype(dtype)
class Dense2Corpus(object):
"""Treat dense numpy array as a streamed Gensim corpus in the bag-of-words format.
Notes
-----
No data copy is made (changes to the underlying matrix imply changes in the streamed corpus).
See Also
--------
:func:`~gensim.matutils.corpus2dense`
Convert Gensim corpus to dense matrix.
:class:`~gensim.matutils.Sparse2Corpus`
Convert sparse matrix to Gensim corpus format.
"""
def __init__(self, dense, documents_columns=True):
"""
Parameters
----------
dense : numpy.ndarray
Corpus in dense format.
documents_columns : bool, optional
Documents in `dense` represented as columns, as opposed to rows?
"""
if documents_columns:
self.dense = dense.T
else:
self.dense = dense
def __iter__(self):
"""Iterate over the corpus.
Yields
------
list of (int, float)
Document in BoW format.
"""
for doc in self.dense:
yield full2sparse(doc.flat)
def __len__(self):
return len(self.dense)
class Sparse2Corpus(object):
"""Convert a matrix in scipy.sparse format into a streaming Gensim corpus.
See Also
--------
:func:`~gensim.matutils.corpus2csc`
Convert gensim corpus format to `scipy.sparse.csc` matrix
:class:`~gensim.matutils.Dense2Corpus`
Convert dense matrix to gensim corpus.
"""
def __init__(self, sparse, documents_columns=True):
"""
Parameters
----------
sparse : `scipy.sparse`
Corpus scipy sparse format
documents_columns : bool, optional
Documents will be column?
"""
if documents_columns:
self.sparse = sparse.tocsc()
else:
self.sparse = sparse.tocsr().T # make sure shape[1]=number of docs (needed in len())
def __iter__(self):
"""
Yields
------
list of (int, float)
Document in BoW format.
"""
for indprev, indnow in zip(self.sparse.indptr, self.sparse.indptr[1:]):
yield list(zip(self.sparse.indices[indprev:indnow], self.sparse.data[indprev:indnow]))
def __len__(self):
return self.sparse.shape[1]
def __getitem__(self, document_index):
"""Retrieve a document vector from the corpus by its index.
Parameters
----------
document_index : int
Index of document
Returns
-------
list of (int, number)
Document in BoW format.
"""
indprev = self.sparse.indptr[document_index]
indnow = self.sparse.indptr[document_index + 1]
return list(zip(self.sparse.indices[indprev:indnow], self.sparse.data[indprev:indnow]))
def veclen(vec):
"""Calculate L2 (euclidean) length of a vector.
Parameters
----------
vec : list of (int, number)
Input vector in sparse bag-of-words format.
Returns
-------
float
Length of `vec`.
"""
if len(vec) == 0:
return 0.0
length = 1.0 * math.sqrt(sum(val**2 for _, val in vec))
assert length > 0.0, "sparse documents must not contain any explicit zero entries"
return length
def ret_normalized_vec(vec, length):
"""Normalize a vector in L2 (Euclidean unit norm).
Parameters
----------
vec : list of (int, number)
Input vector in BoW format.
length : float
Length of vector
Returns
-------
list of (int, number)
L2-normalized vector in BoW format.
"""
if length != 1.0:
return [(termid, val / length) for termid, val in vec]
else:
return list(vec)
def ret_log_normalize_vec(vec, axis=1):
log_max = 100.0
if len(vec.shape) == 1:
max_val = np.max(vec)
log_shift = log_max - np.log(len(vec) + 1.0) - max_val
tot = np.sum(np.exp(vec + log_shift))
log_norm = np.log(tot) - log_shift
vec -= log_norm
else:
if axis == 1: # independently normalize each sample
max_val = np.max(vec, 1)
log_shift = log_max - np.log(vec.shape[1] + 1.0) - max_val
tot = np.sum(np.exp(vec + log_shift[:, np.newaxis]), 1)
log_norm = np.log(tot) - log_shift
vec = vec - log_norm[:, np.newaxis]
elif axis == 0: # normalize each feature
k = ret_log_normalize_vec(vec.T)
return k[0].T, k[1]
else:
raise ValueError("'%s' is not a supported axis" % axis)
return vec, log_norm
blas_nrm2 = blas('nrm2', np.array([], dtype=float))
blas_scal = blas('scal', np.array([], dtype=float))
def unitvec(vec, norm='l2', return_norm=False):
"""Scale a vector to unit length.
Parameters
----------
vec : {numpy.ndarray, scipy.sparse, list of (int, float)}
Input vector in any format
norm : {'l1', 'l2'}, optional
Metric to normalize in.
return_norm : bool, optional
Return the length of vector `vec`, in addition to the normalized vector itself?
Returns
-------
numpy.ndarray, scipy.sparse, list of (int, float)}
Normalized vector in same format as `vec`.
float
Length of `vec` before normalization, if `return_norm` is set.
Notes
-----
Zero-vector will be unchanged.
"""
if norm not in ('l1', 'l2'):
raise ValueError("'%s' is not a supported norm. Currently supported norms are 'l1' and 'l2'." % norm)
if scipy.sparse.issparse(vec):
vec = vec.tocsr()
if norm == 'l1':
veclen = np.sum(np.abs(vec.data))
if norm == 'l2':
veclen = np.sqrt(np.sum(vec.data ** 2))
if veclen > 0.0:
if np.issubdtype(vec.dtype, np.integer):
vec = vec.astype(np.float)
vec /= veclen
if return_norm:
return vec, veclen
else:
return vec
else:
if return_norm:
return vec, 1.
else:
return vec
if isinstance(vec, np.ndarray):
if norm == 'l1':
veclen = np.sum(np.abs(vec))
if norm == 'l2':
veclen = blas_nrm2(vec)
if veclen > 0.0:
if np.issubdtype(vec.dtype, np.integer):
vec = vec.astype(np.float)
if return_norm:
return blas_scal(1.0 / veclen, vec).astype(vec.dtype), veclen
else:
return blas_scal(1.0 / veclen, vec).astype(vec.dtype)
else:
if return_norm:
return vec, 1
else:
return vec
try:
first = next(iter(vec)) # is there at least one element?
except StopIteration:
return vec
if isinstance(first, (tuple, list)) and len(first) == 2: # gensim sparse format
if norm == 'l1':
length = float(sum(abs(val) for _, val in vec))
if norm == 'l2':
length = 1.0 * math.sqrt(sum(val ** 2 for _, val in vec))
assert length > 0.0, "sparse documents must not contain any explicit zero entries"
if return_norm:
return ret_normalized_vec(vec, length), length
else:
return ret_normalized_vec(vec, length)
else:
raise ValueError("unknown input type")
def cossim(vec1, vec2):
"""Get cosine similarity between two sparse vectors.
Cosine similarity is a number between `<-1.0, 1.0>`, higher means more similar.
Parameters
----------
vec1 : list of (int, float)
Vector in BoW format.
vec2 : list of (int, float)
Vector in BoW format.
Returns
-------
float
Cosine similarity between `vec1` and `vec2`.
"""
vec1, vec2 = dict(vec1), dict(vec2)
if not vec1 or not vec2:
return 0.0
vec1len = 1.0 * math.sqrt(sum(val * val for val in itervalues(vec1)))
vec2len = 1.0 * math.sqrt(sum(val * val for val in itervalues(vec2)))
assert vec1len > 0.0 and vec2len > 0.0, "sparse documents must not contain any explicit zero entries"
if len(vec2) < len(vec1):
vec1, vec2 = vec2, vec1 # swap references so that we iterate over the shorter vector
result = sum(value * vec2.get(index, 0.0) for index, value in iteritems(vec1))
result /= vec1len * vec2len # rescale by vector lengths
return result
@deprecated(
"Function will be removed in 4.0.0, use "
"gensim.similarities.termsim.SparseTermSimilarityMatrix.inner_product instead")
def softcossim(vec1, vec2, similarity_matrix):
"""Get Soft Cosine Measure between two vectors given a term similarity matrix.
Return Soft Cosine Measure between two sparse vectors given a sparse term similarity matrix
in the :class:`scipy.sparse.csc_matrix` format. The similarity is a number between `<-1.0, 1.0>`,
higher is more similar.
Notes
-----
Soft Cosine Measure was perhaps first defined by `Grigori Sidorov et al.,
"Soft Similarity and Soft Cosine Measure: Similarity of Features in Vector Space Model"
<http://www.cys.cic.ipn.mx/ojs/index.php/CyS/article/view/2043/1921>`_.
Parameters
----------
vec1 : list of (int, float)
A query vector in the BoW format.
vec2 : list of (int, float)
A document vector in the BoW format.
similarity_matrix : {:class:`scipy.sparse.csc_matrix`, :class:`scipy.sparse.csr_matrix`}
A term similarity matrix. If the matrix is :class:`scipy.sparse.csr_matrix`, it is going
to be transposed. If you rely on the fact that there is at most a constant number of
non-zero elements in a single column, it is your responsibility to ensure that the matrix
is symmetric.
Returns
-------
`similarity_matrix.dtype`
The Soft Cosine Measure between `vec1` and `vec2`.
Raises
------
ValueError
When the term similarity matrix is in an unknown format.
See Also
--------
:meth:`gensim.models.keyedvectors.WordEmbeddingsKeyedVectors.similarity_matrix`
A term similarity matrix produced from term embeddings.
:class:`gensim.similarities.docsim.SoftCosineSimilarity`
A class for performing corpus-based similarity queries with Soft Cosine Measure.
"""
if not isinstance(similarity_matrix, scipy.sparse.csc_matrix):
if isinstance(similarity_matrix, scipy.sparse.csr_matrix):
similarity_matrix = similarity_matrix.T
else:
raise ValueError('unknown similarity matrix format')
if not vec1 or not vec2:
return 0.0
vec1 = dict(vec1)
vec2 = dict(vec2)
word_indices = sorted(set(chain(vec1, vec2)))
dtype = similarity_matrix.dtype
vec1 = np.fromiter((vec1[i] if i in vec1 else 0 for i in word_indices), dtype=dtype, count=len(word_indices))
vec2 = np.fromiter((vec2[i] if i in vec2 else 0 for i in word_indices), dtype=dtype, count=len(word_indices))
dense_matrix = similarity_matrix[[[i] for i in word_indices], word_indices].todense()
vec1len = vec1.T.dot(dense_matrix).dot(vec1)[0, 0]
vec2len = vec2.T.dot(dense_matrix).dot(vec2)[0, 0]
assert \
vec1len > 0.0 and vec2len > 0.0, \
u"sparse documents must not contain any explicit zero entries and the similarity matrix S " \
u"must satisfy x^T * S * x > 0 for any nonzero bag-of-words vector x."
result = vec1.T.dot(dense_matrix).dot(vec2)[0, 0]
result /= math.sqrt(vec1len) * math.sqrt(vec2len) # rescale by vector lengths
return np.clip(result, -1.0, 1.0)
def isbow(vec):
"""Checks if a vector is in the sparse Gensim bag-of-words format.
Parameters
----------
vec : object
Object to check.
Returns
-------
bool
Is `vec` in BoW format.
"""
if scipy.sparse.issparse(vec):
vec = vec.todense().tolist()
try:
id_, val_ = vec[0] # checking first value to see if it is in bag of words format by unpacking
int(id_), float(val_)
except IndexError:
return True # this is to handle the empty input case
except (ValueError, TypeError):
return False
return True
def _convert_vec(vec1, vec2, num_features=None):
if scipy.sparse.issparse(vec1):
vec1 = vec1.toarray()
if scipy.sparse.issparse(vec2):
vec2 = vec2.toarray() # converted both the vectors to dense in case they were in sparse matrix
if isbow(vec1) and isbow(vec2): # if they are in bag of words format we make it dense
if num_features is not None: # if not None, make as large as the documents drawing from
dense1 = sparse2full(vec1, num_features)
dense2 = sparse2full(vec2, num_features)
return dense1, dense2
else:
max_len = max(len(vec1), len(vec2))
dense1 = sparse2full(vec1, max_len)
dense2 = sparse2full(vec2, max_len)
return dense1, dense2
else:
# this conversion is made because if it is not in bow format, it might be a list within a list after conversion
# the scipy implementation of Kullback fails in such a case so we pick up only the nested list.
if len(vec1) == 1:
vec1 = vec1[0]
if len(vec2) == 1:
vec2 = vec2[0]
return vec1, vec2
def kullback_leibler(vec1, vec2, num_features=None):
"""Calculate Kullback-Leibler distance between two probability distributions using `scipy.stats.entropy`.
Parameters
----------
vec1 : {scipy.sparse, numpy.ndarray, list of (int, float)}
Distribution vector.
vec2 : {scipy.sparse, numpy.ndarray, list of (int, float)}
Distribution vector.
num_features : int, optional
Number of features in the vectors.
Returns
-------
float
Kullback-Leibler distance between `vec1` and `vec2`.
Value in range [0, +∞) where values closer to 0 mean less distance (higher similarity).
"""
vec1, vec2 = _convert_vec(vec1, vec2, num_features=num_features)
return entropy(vec1, vec2)
def jensen_shannon(vec1, vec2, num_features=None):
"""Calculate Jensen-Shannon distance between two probability distributions using `scipy.stats.entropy`.
Parameters
----------
vec1 : {scipy.sparse, numpy.ndarray, list of (int, float)}
Distribution vector.
vec2 : {scipy.sparse, numpy.ndarray, list of (int, float)}
Distribution vector.
num_features : int, optional
Number of features in the vectors.
Returns
-------
float
Jensen-Shannon distance between `vec1` and `vec2`.
Notes
-----
This is a symmetric and finite "version" of :func:`gensim.matutils.kullback_leibler`.
"""
vec1, vec2 = _convert_vec(vec1, vec2, num_features=num_features)
avg_vec = 0.5 * (vec1 + vec2)
return 0.5 * (entropy(vec1, avg_vec) + entropy(vec2, avg_vec))
def hellinger(vec1, vec2):
"""Calculate Hellinger distance between two probability distributions.
Parameters
----------
vec1 : {scipy.sparse, numpy.ndarray, list of (int, float)}
Distribution vector.
vec2 : {scipy.sparse, numpy.ndarray, list of (int, float)}
Distribution vector.
Returns
-------
float
Hellinger distance between `vec1` and `vec2`.
Value in range `[0, 1]`, where 0 is min distance (max similarity) and 1 is max distance (min similarity).
"""
if scipy.sparse.issparse(vec1):
vec1 = vec1.toarray()
if scipy.sparse.issparse(vec2):
vec2 = vec2.toarray()
if isbow(vec1) and isbow(vec2):
# if it is a BoW format, instead of converting to dense we use dictionaries to calculate appropriate distance
vec1, vec2 = dict(vec1), dict(vec2)
indices = set(list(vec1.keys()) + list(vec2.keys()))
sim = np.sqrt(
0.5 * sum((np.sqrt(vec1.get(index, 0.0)) - np.sqrt(vec2.get(index, 0.0)))**2 for index in indices)
)
return sim
else:
sim = np.sqrt(0.5 * ((np.sqrt(vec1) - np.sqrt(vec2))**2).sum())
return sim
def jaccard(vec1, vec2):
"""Calculate Jaccard distance between two vectors.
Parameters
----------
vec1 : {scipy.sparse, numpy.ndarray, list of (int, float)}
Distribution vector.
vec2 : {scipy.sparse, numpy.ndarray, list of (int, float)}
Distribution vector.
Returns
-------
float
Jaccard distance between `vec1` and `vec2`.
Value in range `[0, 1]`, where 0 is min distance (max similarity) and 1 is max distance (min similarity).
"""
# converting from sparse for easier manipulation
if scipy.sparse.issparse(vec1):
vec1 = vec1.toarray()
if scipy.sparse.issparse(vec2):
vec2 = vec2.toarray()
if isbow(vec1) and isbow(vec2):
# if it's in bow format, we use the following definitions:
# union = sum of the 'weights' of both the bags
# intersection = lowest weight for a particular id; basically the number of common words or items
union = sum(weight for id_, weight in vec1) + sum(weight for id_, weight in vec2)
vec1, vec2 = dict(vec1), dict(vec2)
intersection = 0.0
for feature_id, feature_weight in iteritems(vec1):
intersection += min(feature_weight, vec2.get(feature_id, 0.0))
return 1 - float(intersection) / float(union)
else:
# if it isn't in bag of words format, we can use sets to calculate intersection and union
if isinstance(vec1, np.ndarray):
vec1 = vec1.tolist()
if isinstance(vec2, np.ndarray):
vec2 = vec2.tolist()
vec1 = set(vec1)
vec2 = set(vec2)
intersection = vec1 & vec2
union = vec1 | vec2
return 1 - float(len(intersection)) / float(len(union))
def jaccard_distance(set1, set2):
"""Calculate Jaccard distance between two sets.
Parameters
----------
set1 : set
Input set.
set2 : set
Input set.
Returns
-------
float
Jaccard distance between `set1` and `set2`.
Value in range `[0, 1]`, where 0 is min distance (max similarity) and 1 is max distance (min similarity).
"""
union_cardinality = len(set1 | set2)
if union_cardinality == 0: # Both sets are empty
return 1.
return 1. - float(len(set1 & set2)) / float(union_cardinality)
try:
# try to load fast, cythonized code if possible
from gensim._matutils import logsumexp, mean_absolute_difference, dirichlet_expectation
except ImportError:
def logsumexp(x):
"""Log of sum of exponentials.
Parameters
----------
x : numpy.ndarray
Input 2d matrix.
Returns
-------
float
log of sum of exponentials of elements in `x`.
Warnings
--------
For performance reasons, doesn't support NaNs or 1d, 3d, etc arrays like :func:`scipy.special.logsumexp`.
"""
x_max = np.max(x)
x = np.log(np.sum(np.exp(x - x_max)))
x += x_max
return x
def mean_absolute_difference(a, b):
"""Mean absolute difference between two arrays.
Parameters
----------
a : numpy.ndarray
Input 1d array.
b : numpy.ndarray
Input 1d array.
Returns
-------
float
mean(abs(a - b)).
"""
return np.mean(np.abs(a - b))
def dirichlet_expectation(alpha):
"""Expected value of log(theta) where theta is drawn from a Dirichlet distribution.
Parameters
----------
alpha : numpy.ndarray
Dirichlet parameter 2d matrix or 1d vector, if 2d - each row is treated as a separate parameter vector.
Returns
-------
numpy.ndarray
Log of expected values, dimension same as `alpha.ndim`.
"""
if len(alpha.shape) == 1:
result = psi(alpha) - psi(np.sum(alpha))
else:
result = psi(alpha) - psi(np.sum(alpha, 1))[:, np.newaxis]
return result.astype(alpha.dtype, copy=False) # keep the same precision as input
def qr_destroy(la):
"""Get QR decomposition of `la[0]`.
Parameters
----------
la : list of numpy.ndarray
Run QR decomposition on the first elements of `la`. Must not be empty.
Returns
-------
(numpy.ndarray, numpy.ndarray)
Matrices :math:`Q` and :math:`R`.
Notes
-----
Using this function is less memory intense than calling `scipy.linalg.qr(la[0])`,
because the memory used in `la[0]` is reclaimed earlier. This makes a difference when
decomposing very large arrays, where every memory copy counts.
Warnings
--------
Content of `la` as well as `la[0]` gets destroyed in the process. Again, for memory-effiency reasons.
"""
a = np.asfortranarray(la[0])
del la[0], la # now `a` is the only reference to the input matrix
m, n = a.shape
# perform q, r = QR(a); code hacked out of scipy.linalg.qr
logger.debug("computing QR of %s dense matrix", str(a.shape))
geqrf, = get_lapack_funcs(('geqrf',), (a,))
qr, tau, work, info = geqrf(a, lwork=-1, overwrite_a=True)
qr, tau, work, info = geqrf(a, lwork=work[0], overwrite_a=True)
del a # free up mem
assert info >= 0
r = triu(qr[:n, :n])
if m < n: # rare case, #features < #topics
qr = qr[:, :m] # retains fortran order
gorgqr, = get_lapack_funcs(('orgqr',), (qr,))
q, work, info = gorgqr(qr, tau, lwork=-1, overwrite_a=True)
q, work, info = gorgqr(qr, tau, lwork=work[0], overwrite_a=True)
assert info >= 0, "qr failed"
assert q.flags.f_contiguous
return q, r
class MmWriter(object):
"""Store a corpus in `Matrix Market format <https://math.nist.gov/MatrixMarket/formats.html>`_,
using :class:`~gensim.corpora.mmcorpus.MmCorpus`.
Notes
-----
The output is written one document at a time, not the whole matrix at once (unlike e.g. `scipy.io.mmread`).
This allows you to write corpora which are larger than the available RAM.
The output file is created in a single pass through the input corpus, so that the input can be
a once-only stream (generator).
To achieve this, a fake MM header is written first, corpus statistics are collected
during the pass (shape of the matrix, number of non-zeroes), followed by a seek back to the beginning of the file,
rewriting the fake header with the final values.
"""
HEADER_LINE = b'%%MatrixMarket matrix coordinate real general\n' # the only supported MM format
def __init__(self, fname):
"""
Parameters
----------
fname : str
Path to output file.
"""
self.fname = fname
if fname.endswith(".gz") or fname.endswith('.bz2'):
raise NotImplementedError("compressed output not supported with MmWriter")
self.fout = utils.smart_open(self.fname, 'wb+') # open for both reading and writing
self.headers_written = False
def write_headers(self, num_docs, num_terms, num_nnz):
"""Write headers to file.
Parameters
----------
num_docs : int
Number of documents in corpus.
num_terms : int
Number of term in corpus.
num_nnz : int
Number of non-zero elements in corpus.
"""
self.fout.write(MmWriter.HEADER_LINE)
if num_nnz < 0:
# we don't know the matrix shape/density yet, so only log a general line
logger.info("saving sparse matrix to %s", self.fname)
self.fout.write(utils.to_utf8(' ' * 50 + '\n')) # 48 digits must be enough for everybody
else:
logger.info(
"saving sparse %sx%s matrix with %i non-zero entries to %s",
num_docs, num_terms, num_nnz, self.fname
)
self.fout.write(utils.to_utf8('%s %s %s\n' % (num_docs, num_terms, num_nnz)))
self.last_docno = -1
self.headers_written = True
def fake_headers(self, num_docs, num_terms, num_nnz):
"""Write "fake" headers to file, to be rewritten once we've scanned the entire corpus.
Parameters
----------
num_docs : int
Number of documents in corpus.
num_terms : int
Number of term in corpus.
num_nnz : int
Number of non-zero elements in corpus.
"""
stats = '%i %i %i' % (num_docs, num_terms, num_nnz)
if len(stats) > 50:
raise ValueError('Invalid stats: matrix too large!')
self.fout.seek(len(MmWriter.HEADER_LINE))
self.fout.write(utils.to_utf8(stats))
def write_vector(self, docno, vector):
"""Write a single sparse vector to the file.
Parameters
----------
docno : int
Number of document.
vector : list of (int, number)
Document in BoW format.
Returns
-------
(int, int)
Max word index in vector and len of vector. If vector is empty, return (-1, 0).
"""
assert self.headers_written, "must write Matrix Market file headers before writing data!"
assert self.last_docno < docno, "documents %i and %i not in sequential order!" % (self.last_docno, docno)
vector = sorted((i, w) for i, w in vector if abs(w) > 1e-12) # ignore near-zero entries
for termid, weight in vector: # write term ids in sorted order
# +1 because MM format starts counting from 1
self.fout.write(utils.to_utf8("%i %i %s\n" % (docno + 1, termid + 1, weight)))
self.last_docno = docno
return (vector[-1][0], len(vector)) if vector else (-1, 0)
@staticmethod
def write_corpus(fname, corpus, progress_cnt=1000, index=False, num_terms=None, metadata=False):
"""Save the corpus to disk in `Matrix Market format <https://math.nist.gov/MatrixMarket/formats.html>`_.
Parameters
----------
fname : str
Filename of the resulting file.
corpus : iterable of list of (int, number)
Corpus in streamed bag-of-words format.
progress_cnt : int, optional
Print progress for every `progress_cnt` number of documents.
index : bool, optional
Return offsets?
num_terms : int, optional
Number of terms in the corpus. If provided, the `corpus.num_terms` attribute (if any) will be ignored.
metadata : bool, optional
Generate a metadata file?
Returns
-------
offsets : {list of int, None}
List of offsets (if index=True) or nothing.
Notes
-----
Documents are processed one at a time, so the whole corpus is allowed to be larger than the available RAM.
See Also
--------
:func:`gensim.corpora.mmcorpus.MmCorpus.save_corpus`
Save corpus to disk.
"""
mw = MmWriter(fname)
# write empty headers to the file (with enough space to be overwritten later)
mw.write_headers(-1, -1, -1) # will print 50 spaces followed by newline on the stats line
# calculate necessary header info (nnz elements, num terms, num docs) while writing out vectors
_num_terms, num_nnz = 0, 0
docno, poslast = -1, -1
offsets = []
if hasattr(corpus, 'metadata'):
orig_metadata = corpus.metadata
corpus.metadata = metadata
if metadata:
docno2metadata = {}
else:
metadata = False
for docno, doc in enumerate(corpus):
if metadata:
bow, data = doc
docno2metadata[docno] = data
else:
bow = doc
if docno % progress_cnt == 0:
logger.info("PROGRESS: saving document #%i", docno)
if index:
posnow = mw.fout.tell()
if posnow == poslast:
offsets[-1] = -1
offsets.append(posnow)
poslast = posnow
max_id, veclen = mw.write_vector(docno, bow)
_num_terms = max(_num_terms, 1 + max_id)
num_nnz += veclen
if metadata:
utils.pickle(docno2metadata, fname + '.metadata.cpickle')
corpus.metadata = orig_metadata
num_docs = docno + 1
num_terms = num_terms or _num_terms
if num_docs * num_terms != 0:
logger.info(
"saved %ix%i matrix, density=%.3f%% (%i/%i)",
num_docs, num_terms, 100.0 * num_nnz / (num_docs * num_terms), num_nnz, num_docs * num_terms
)
# now write proper headers, by seeking and overwriting the spaces written earlier
mw.fake_headers(num_docs, num_terms, num_nnz)
mw.close()
if index:
return offsets
def __del__(self):
"""Close `self.fout` file. Alias for :meth:`~gensim.matutils.MmWriter.close`.
Warnings
--------
Closing the file explicitly via the close() method is preferred and safer.
"""
self.close() # does nothing if called twice (on an already closed file), so no worries
def close(self):
"""Close `self.fout` file."""
logger.debug("closing %s", self.fname)
if hasattr(self, 'fout'):
self.fout.close()
try:
# try to load fast, cythonized code if possible
from gensim.corpora._mmreader import MmReader
except ImportError:
FAST_VERSION = -1
class MmReader(object):
"""Matrix market file reader, used internally in :class:`~gensim.corpora.mmcorpus.MmCorpus`.
Wrap a term-document matrix on disk (in matrix-market format), and present it
as an object which supports iteration over the rows (~documents).
Attributes
----------
num_docs : int
Number of documents in market matrix file.
num_terms : int
Number of terms.
num_nnz : int
Number of non-zero terms.
Notes
-----
Note that the file is read into memory one document at a time, not the whole matrix at once
(unlike e.g. `scipy.io.mmread` and other implementations).
This allows us to process corpora which are larger than the available RAM.
"""
def __init__(self, input, transposed=True):
"""
Parameters
----------
input : {str, file-like object}
Path to the input file in MM format or a file-like object that supports `seek()`
(e.g. smart_open objects).
transposed : bool, optional
Do lines represent `doc_id, term_id, value`, instead of `term_id, doc_id, value`?
"""
logger.info("initializing corpus reader from %s", input)
self.input, self.transposed = input, transposed
with utils.open_file(self.input) as lines:
try:
header = utils.to_unicode(next(lines)).strip()
if not header.lower().startswith('%%matrixmarket matrix coordinate real general'):
raise ValueError(
"File %s not in Matrix Market format with coordinate real general; instead found: \n%s" %
(self.input, header)
)
except StopIteration:
pass
self.num_docs = self.num_terms = self.num_nnz = 0
for lineno, line in enumerate(lines):
line = utils.to_unicode(line)
if not line.startswith('%'):
self.num_docs, self.num_terms, self.num_nnz = (int(x) for x in line.split())
if not self.transposed:
self.num_docs, self.num_terms = self.num_terms, self.num_docs
break
logger.info(
"accepted corpus with %i documents, %i features, %i non-zero entries",
self.num_docs, self.num_terms, self.num_nnz
)
def __len__(self):
"""Get the corpus size: total number of documents."""
return self.num_docs
def __str__(self):
return ("MmCorpus(%i documents, %i features, %i non-zero entries)" %
(self.num_docs, self.num_terms, self.num_nnz))
def skip_headers(self, input_file):
"""Skip file headers that appear before the first document.
Parameters
----------
input_file : iterable of str
Iterable taken from file in MM format.
"""
for line in input_file:
if line.startswith(b'%'):
continue
break
def __iter__(self):
"""Iterate through all documents in the corpus.
Notes
------
Note that the total number of vectors returned is always equal to the number of rows specified
in the header: empty documents are inserted and yielded where appropriate, even if they are not explicitly
stored in the Matrix Market file.
Yields
------
(int, list of (int, number))
Document id and document in sparse bag-of-words format.
"""
with utils.file_or_filename(self.input) as lines:
self.skip_headers(lines)
previd = -1
for line in lines:
docid, termid, val = utils.to_unicode(line).split() # needed for python3
if not self.transposed:
termid, docid = docid, termid
# -1 because matrix market indexes are 1-based => convert to 0-based
docid, termid, val = int(docid) - 1, int(termid) - 1, float(val)
assert previd <= docid, "matrix columns must come in ascending order"
if docid != previd:
# change of document: return the document read so far (its id is prevId)
if previd >= 0:
yield previd, document # noqa:F821
# return implicit (empty) documents between previous id and new id
# too, to keep consistent document numbering and corpus length
for previd in range(previd + 1, docid):
yield previd, []
# from now on start adding fields to a new document, with a new id
previd = docid
document = []
document.append((termid, val,)) # add another field to the current document
# handle the last document, as a special case
if previd >= 0:
yield previd, document
# return empty documents between the last explicit document and the number
# of documents as specified in the header
for previd in range(previd + 1, self.num_docs):
yield previd, []
def docbyoffset(self, offset):
"""Get the document at file offset `offset` (in bytes).
Parameters
----------
offset : int
File offset, in bytes, of the desired document.
Returns
------
list of (int, str)
Document in sparse bag-of-words format.
"""
# empty documents are not stored explicitly in MM format, so the index marks
# them with a special offset, -1.
if offset == -1:
return []
if isinstance(self.input, string_types):
fin, close_fin = utils.smart_open(self.input), True
else:
fin, close_fin = self.input, False
fin.seek(offset) # works for gzip/bz2 input, too
previd, document = -1, []
for line in fin:
docid, termid, val = line.split()
if not self.transposed:
termid, docid = docid, termid
# -1 because matrix market indexes are 1-based => convert to 0-based
docid, termid, val = int(docid) - 1, int(termid) - 1, float(val)
assert previd <= docid, "matrix columns must come in ascending order"
if docid != previd:
if previd >= 0:
break
previd = docid
document.append((termid, val,)) # add another field to the current document
if close_fin:
fin.close()
return document
