python – 使用tensorflow.contrib.signal重建信号会导致放大或调制(帧,overlap_and_add,stft等)

更新：我在librosa中重新实现了这个以进行比较,结果确实与tensorflow的结果非常不同. Librosa给出了我期望的结果(但不是张量流).

我已经在张量流回购上发布了这个issue,但它在那里很安静所以我在这里尝试.此外,我不确定它是否是tensorflow中的错误,或代表我的用户错误.为了完整起见,我还将在此处包含完整的来源和结果.

A.)当我使用hann窗口(也尝试汉明)从frame_length = 1024和frame_step = 256(即25％跳跃大小,75％重叠)的信号创建帧时,然后我使用overlap_and_add重建,我期望要正确重建的信号(因为COLA等).但相反,它的幅度恰好是两倍.我需要将结果信号除以2才能正确.

B.)如果我使用STFT创建一系列重叠频谱图,然后用逆STFT重建,再次使用frame_length = 1024和frame_step = 256,再次以双幅度重建信号.

我意识到为什么会出现这种情况(hann在50％重叠时的单位增益,因此75％的重叠会使信号加倍).但重建功能考虑到这一点是不正常的吗？例如. librosa istft确实返回具有正确幅度的信号,而tensorflow返回double.

C.)
在任何其他frame_step处,存在严重的幅度调制.见下图.这似乎不对.

更新：如果我在inverse_stft中明确设置window_fn = tf.contrib.signal.inverse_stft_window_fn(frame_step),则输出正确.所以似乎inverse_stft中的frame_step没有传递给窗口函数(这也是结果提示的).

原始数据：

帧的tensorflow输出overlap_and_add：

stft istft的tensorflow输出：

来自stft istft的librosa输出：

张量流代码：

from __future__ import print_function
from __future__ import division

import numpy as np
import scipy.io.wavfile
import math
import random
import matplotlib.pyplot as plt

import tensorflow as tf
out_prefix = 'tensorflow'


def plot(data, title, do_save=True):
    plt.figure(figsize=(20,5))
    plt.plot(data[:3*frame_length])
    plt.ylim([-1, 1])
    plt.title(title)
    plt.grid()
    if do_save: plt.savefig(title + '.png')
    plt.show()


def reconstruct_from_frames(x, frame_length, frame_step):
    name = 'frame'
    frames_T = tf.contrib.signal.frame(x, frame_length=frame_length, frame_step=frame_step)
    windowed_frames_T = frames_T * tf.contrib.signal.hann_window(frame_length, periodic=True)
    output_T = tf.contrib.signal.overlap_and_add(windowed_frames_T, frame_step=frame_step)
    return name, output_T


def reconstruct_from_stft(x, frame_length, frame_step):
    name = 'stft'
    spectrograms_T = tf.contrib.signal.stft(x, frame_length, frame_step)
    output_T = tf.contrib.signal.inverse_stft(spectrograms_T, frame_length, frame_step)
    return name, output_T


def test(fn, input_data):
    print('-'*80)
    tf.reset_default_graph()
    input_T = tf.placeholder(tf.float32, [None]) 
    name, output_T = fn(input_T, frame_length, frame_step)

    title = "{}.{}.{}.l{}.s{}".format(out_prefix, sample_rate, name, frame_length, frame_step)
    print(title)

    with tf.Session():
        output_data =  output_T.eval({input_T:input_data})

#    output_data /= frame_length/frame_step/2 # tensorflow needs this to normalise amp
    plot(output_data, title)
    scipy.io.wavfile.write(title+'.wav', sample_rate, output_data)


def generate_data(duration_secs, sample_rate, num_sin, min_freq=10, max_freq=500, rnd_seed=0, max_val=0):
    '''generate signal from multiple random sin waves'''
    if rnd_seed>0: random.seed(rnd_seed)
    data = np.zeros([duration_secs*sample_rate], np.float32)
    for i in range(num_sin):
        w = np.float32(np.sin(np.linspace(0, math.pi*2*random.randrange(min_freq, max_freq), num=duration_secs*sample_rate)))
        data += random.random() * w
    if max_val>0:
        data *= max_val / np.max(np.abs(data))
    return data


frame_length = 1024
sample_rate = 22050

input_data = generate_data(duration_secs=1, sample_rate=sample_rate, num_sin=1, rnd_seed=2, max_val=0.5)

title = "{}.orig".format(sample_rate)
plot(input_data, title)
scipy.io.wavfile.write(title+'.wav', sample_rate, input_data)

for frame_step in [256, 512, 768, 1024]:
    test(reconstruct_from_frames, input_data)
    test(reconstruct_from_stft, input_data)

print('done.')

librosa代码：

from __future__ import print_function
from __future__ import division

import numpy as np
import scipy.io.wavfile
import math
import random
import matplotlib.pyplot as plt

import librosa.core as lc
out_prefix = 'librosa'


def plot(data, title, do_save=True):
    plt.figure(figsize=(20,5))
    plt.plot(data[:3*frame_length])
    plt.ylim([-1, 1])
    plt.title(title)
    plt.grid()
    if do_save: plt.savefig(title + '.png')
    plt.show()


def reconstruct_from_stft(x, frame_length, frame_step):
    name = 'stft'
    stft = lc.stft(x, n_fft=frame_length, hop_length=frame_step)
    istft = lc.istft(stft, frame_step)
    return name, istft


def test(fn, input_data):
    print('-'*80)
    name, output_data = fn(input_data, frame_length, frame_step)

    title = "{}.{}.{}.l{}.s{}".format(out_prefix, sample_rate, name, frame_length, frame_step)
    print(title)

#    output_data /= frame_length/frame_step/2 # tensorflow needs this to normalise amp
    plot(output_data, title)
    scipy.io.wavfile.write(title+'.wav', sample_rate, output_data)


def generate_data(duration_secs, sample_rate, num_sin, min_freq=10, max_freq=500, rnd_seed=0, max_val=0):
    '''generate signal from multiple random sin waves'''
    if rnd_seed>0: random.seed(rnd_seed)
    data = np.zeros([duration_secs*sample_rate], np.float32)
    for i in range(num_sin):
        w = np.float32(np.sin(np.linspace(0, math.pi*2*random.randrange(min_freq, max_freq), num=duration_secs*sample_rate)))
        data += random.random() * w
    if max_val>0:
        data *= max_val / np.max(np.abs(data))
    return data


frame_length = 1024
sample_rate = 22050

input_data = generate_data(duration_secs=1, sample_rate=sample_rate, num_sin=1, rnd_seed=2, max_val=0.5)

title = "{}.orig".format(sample_rate)
plot(input_data, title)
scipy.io.wavfile.write(title+'.wav', sample_rate, input_data)

for frame_step in [256, 512, 768, 1024]:
    test(reconstruct_from_stft, input_data)

print('done.')

> Linux Ubuntu 16.04
>从二进制v1.4.0-19-ga52c8d9,1.4.1安装的Tensorflow
> Python 2.7.14 | Anaconda定制(64位)| (默认,2017年10月16日,17：29：19). IPython 5.4.1
> Cuda版本8.0,V8.0.61,cuDNN 6
> Geforce GTX 970M,驱动程序版本：384.111

(刚试过TF1.5,Cuda9.0,cuDNN 7.0.5,结果相同).

解决方法:

无限序列的Von Hann窗口具有50％的重叠总和达到平坦的单位增益.如果重叠率为25％,则每单位时间窗口数量增加一倍,从而使增益加倍.

重叠 – 添加快速卷积滤波通常在没有重叠和没有(非矩形)窗口的情况下完成,仅对滤波器函数的脉冲响应的长度进行足够的零填充.任何重叠百分比都不应包括任何添加的零填充长度.