From d786e7812198c3d9af5ff36489f7f96d975bddf5 Mon Sep 17 00:00:00 2001 From: babysor00 Date: Thu, 3 Mar 2022 23:34:47 +0800 Subject: [PATCH] Add UI usage of PPG-vc --- .vscode/launch.json | 8 ++++ demo_toolbox.py | 2 + toolbox/__init__.py | 33 ++++++++------ toolbox/ui.py | 107 ++++++++++++++++++++++++-------------------- 4 files changed, 89 insertions(+), 61 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 449cdfd..23e5203 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -35,6 +35,14 @@ "console": "integratedTerminal", "args": ["-d","..\\audiodata"] }, + { + "name": "Python: Demo Box VC", + "type": "python", + "request": "launch", + "program": "demo_toolbox.py", + "console": "integratedTerminal", + "args": ["-d","..\\audiodata","-vc"] + }, { "name": "Python: Synth Train", "type": "python", diff --git a/demo_toolbox.py b/demo_toolbox.py index a05bab6..7030bd5 100644 --- a/demo_toolbox.py +++ b/demo_toolbox.py @@ -15,6 +15,8 @@ if __name__ == '__main__': parser.add_argument("-d", "--datasets_root", type=Path, help= \ "Path to the directory containing your datasets. See toolbox/__init__.py for a list of " "supported datasets.", default=None) + parser.add_argument("-vc", "--vc_mode", action="store_true", + help="Voice Conversion Mode(PPG based)") parser.add_argument("-e", "--enc_models_dir", type=Path, default="encoder/saved_models", help="Directory containing saved encoder models") parser.add_argument("-s", "--syn_models_dir", type=Path, default="synthesizer/saved_models", diff --git a/toolbox/__init__.py b/toolbox/__init__.py index e961be1..30f2865 100644 --- a/toolbox/__init__.py +++ b/toolbox/__init__.py @@ -50,8 +50,9 @@ recognized_datasets = [ MAX_WAVES = 15 class Toolbox: - def __init__(self, datasets_root, enc_models_dir, syn_models_dir, voc_models_dir, extractor_models_dir, convertor_models_dir, seed, no_mp3_support): + def __init__(self, datasets_root, enc_models_dir, syn_models_dir, voc_models_dir, extractor_models_dir, convertor_models_dir, seed, no_mp3_support, vc_mode): self.no_mp3_support = no_mp3_support + self.vc_mode = vc_mode sys.excepthook = self.excepthook self.datasets_root = datasets_root self.utterances = set() @@ -76,7 +77,7 @@ class Toolbox: self.trim_silences = False # Initialize the events and the interface - self.ui = UI() + self.ui = UI(vc_mode) self.style_idx = 0 self.reset_ui(enc_models_dir, syn_models_dir, voc_models_dir, extractor_models_dir, convertor_models_dir, seed) self.setup_events() @@ -102,9 +103,12 @@ class Toolbox: self.ui.encoder_box.currentIndexChanged.connect(self.init_encoder) def func(): self.synthesizer = None - self.ui.synthesizer_box.currentIndexChanged.connect(func) + if self.vc_mode: + self.ui.extractor_box.currentIndexChanged.connect(self.init_extractor) + else: + self.ui.synthesizer_box.currentIndexChanged.connect(func) + self.ui.vocoder_box.currentIndexChanged.connect(self.init_vocoder) - self.ui.extractor_box.currentIndexChanged.connect(self.init_extractor) # Utterance selection func = lambda: self.load_from_browser(self.ui.browse_file()) @@ -117,8 +121,9 @@ class Toolbox: self.ui.record_button.clicked.connect(self.record) # Source Utterance selection - func = lambda: self.load_soruce_button(self.ui.selected_utterance) - self.ui.load_soruce_button.clicked.connect(func) + if self.vc_mode: + func = lambda: self.load_soruce_button(self.ui.selected_utterance) + self.ui.load_soruce_button.clicked.connect(func) #Audio self.ui.setup_audio_devices(Synthesizer.sample_rate) @@ -131,14 +136,16 @@ class Toolbox: self.ui.waves_cb.currentIndexChanged.connect(self.set_current_wav) # Generation - func = lambda: self.synthesize() or self.vocode() - self.ui.generate_button.clicked.connect(func) - self.ui.synthesize_button.clicked.connect(self.synthesize) self.ui.vocode_button.clicked.connect(self.vocode) self.ui.random_seed_checkbox.clicked.connect(self.update_seed_textbox) - func = lambda: self.convert() or self.vocode() - self.ui.convert_button.clicked.connect(func) + if self.vc_mode: + func = lambda: self.convert() or self.vocode() + self.ui.convert_button.clicked.connect(func) + else: + func = lambda: self.synthesize() or self.vocode() + self.ui.generate_button.clicked.connect(func) + self.ui.synthesize_button.clicked.connect(self.synthesize) # UMAP legend self.ui.clear_button.clicked.connect(self.clear_utterances) @@ -154,7 +161,7 @@ class Toolbox: def reset_ui(self, encoder_models_dir, synthesizer_models_dir, vocoder_models_dir, extractor_models_dir, convertor_models_dir, seed): self.ui.populate_browser(self.datasets_root, recognized_datasets, 0, True) - self.ui.populate_models(encoder_models_dir, synthesizer_models_dir, vocoder_models_dir, extractor_models_dir, convertor_models_dir) + self.ui.populate_models(encoder_models_dir, synthesizer_models_dir, vocoder_models_dir, extractor_models_dir, convertor_models_dir, self.vc_mode) self.ui.populate_gen_options(seed, self.trim_silences) def load_from_browser(self, fpath=None): @@ -213,7 +220,7 @@ class Toolbox: # Add the utterance utterance = Utterance(name, speaker_name, wav, spec, embed, partial_embeds, False) self.utterances.add(utterance) - self.ui.register_utterance(utterance) + self.ui.register_utterance(utterance, self.vc_mode) # Plot it self.ui.draw_embed(embed, name, "current") diff --git a/toolbox/ui.py b/toolbox/ui.py index e4767ef..fe51e73 100644 --- a/toolbox/ui.py +++ b/toolbox/ui.py @@ -335,41 +335,42 @@ class UI(QDialog): return self.convertor_box.itemData(self.convertor_box.currentIndex()) def populate_models(self, encoder_models_dir: Path, synthesizer_models_dir: Path, - vocoder_models_dir: Path, extractor_models_dir: Path, convertor_models_dir: Path): + vocoder_models_dir: Path, extractor_models_dir: Path, convertor_models_dir: Path, vc_mode: bool): # Encoder encoder_fpaths = list(encoder_models_dir.glob("*.pt")) if len(encoder_fpaths) == 0: raise Exception("No encoder models found in %s" % encoder_models_dir) self.repopulate_box(self.encoder_box, [(f.stem, f) for f in encoder_fpaths]) - # Synthesizer - synthesizer_fpaths = list(synthesizer_models_dir.glob("**/*.pt")) - if len(synthesizer_fpaths) == 0: - raise Exception("No synthesizer models found in %s" % synthesizer_models_dir) - self.repopulate_box(self.synthesizer_box, [(f.stem, f) for f in synthesizer_fpaths]) + if vc_mode: + # Extractor + extractor_fpaths = list(extractor_models_dir.glob("*.pt")) + if len(extractor_fpaths) == 0: + self.log("No extractor models found in %s" % extractor_fpaths) + self.repopulate_box(self.extractor_box, [(f.stem, f) for f in extractor_fpaths]) + + # Convertor + convertor_fpaths = list(convertor_models_dir.glob("*.pth")) + if len(convertor_fpaths) == 0: + self.log("No convertor models found in %s" % convertor_fpaths) + self.repopulate_box(self.convertor_box, [(f.stem, f) for f in convertor_fpaths]) + else: + # Synthesizer + synthesizer_fpaths = list(synthesizer_models_dir.glob("**/*.pt")) + if len(synthesizer_fpaths) == 0: + raise Exception("No synthesizer models found in %s" % synthesizer_models_dir) + self.repopulate_box(self.synthesizer_box, [(f.stem, f) for f in synthesizer_fpaths]) # Vocoder vocoder_fpaths = list(vocoder_models_dir.glob("**/*.pt")) vocoder_items = [(f.stem, f) for f in vocoder_fpaths] + [("Griffin-Lim", None)] self.repopulate_box(self.vocoder_box, vocoder_items) - # Extractor - extractor_fpaths = list(extractor_models_dir.glob("*.pt")) - if len(extractor_fpaths) == 0: - self.log("No extractor models found in %s" % extractor_fpaths) - self.repopulate_box(self.extractor_box, [(f.stem, f) for f in extractor_fpaths]) - - # Convertor - convertor_fpaths = list(convertor_models_dir.glob("*.pth")) - if len(convertor_fpaths) == 0: - self.log("No convertor models found in %s" % convertor_fpaths) - self.repopulate_box(self.convertor_box, [(f.stem, f) for f in convertor_fpaths]) - @property def selected_utterance(self): return self.utterance_history.itemData(self.utterance_history.currentIndex()) - def register_utterance(self, utterance: Utterance): + def register_utterance(self, utterance: Utterance, vc_mode): self.utterance_history.blockSignals(True) self.utterance_history.insertItem(0, utterance.name, utterance) self.utterance_history.setCurrentIndex(0) @@ -379,8 +380,11 @@ class UI(QDialog): self.utterance_history.removeItem(self.max_saved_utterances) self.play_button.setDisabled(False) - self.generate_button.setDisabled(False) - self.synthesize_button.setDisabled(False) + if vc_mode: + self.convert_button.setDisabled(False) + else: + self.generate_button.setDisabled(False) + self.synthesize_button.setDisabled(False) def log(self, line, mode="newline"): if mode == "newline": @@ -422,7 +426,7 @@ class UI(QDialog): else: self.seed_textbox.setEnabled(False) - def reset_interface(self): + def reset_interface(self, vc_mode): self.draw_embed(None, None, "current") self.draw_embed(None, None, "generated") self.draw_spec(None, "current") @@ -430,14 +434,17 @@ class UI(QDialog): self.draw_umap_projections(set()) self.set_loading(0) self.play_button.setDisabled(True) - self.generate_button.setDisabled(True) - self.synthesize_button.setDisabled(True) + if vc_mode: + self.convert_button.setDisabled(True) + else: + self.generate_button.setDisabled(True) + self.synthesize_button.setDisabled(True) self.vocode_button.setDisabled(True) self.replay_wav_button.setDisabled(True) self.export_wav_button.setDisabled(True) [self.log("") for _ in range(self.max_log_lines)] - def __init__(self): + def __init__(self, vc_mode): ## Initialize the application self.app = QApplication(sys.argv) super().__init__(None) @@ -530,8 +537,9 @@ class UI(QDialog): browser_layout.addWidget(self.play_button, i, 2) self.stop_button = QPushButton("Stop(暂停)") browser_layout.addWidget(self.stop_button, i, 3) - self.load_soruce_button = QPushButton("Select(选择为被转换的语音输入)") - browser_layout.addWidget(self.load_soruce_button, i, 4) + if vc_mode: + self.load_soruce_button = QPushButton("Select(选择为被转换的语音输入)") + browser_layout.addWidget(self.load_soruce_button, i, 4) i += 1 model_groupbox = QGroupBox('Models(模型选择)') @@ -544,19 +552,20 @@ class UI(QDialog): model_layout.addWidget(QLabel("Encoder:")) model_layout.addWidget(self.encoder_box) self.synthesizer_box = QComboBox() - model_layout.addWidget(QLabel("Synthesizer:")) - model_layout.addWidget(self.synthesizer_box) + if vc_mode: + self.extractor_box = QComboBox() + model_layout.addWidget(QLabel("Extractor:")) + model_layout.addWidget(self.extractor_box) + self.convertor_box = QComboBox() + model_layout.addWidget(QLabel("Convertor:")) + model_layout.addWidget(self.convertor_box) + else: + model_layout.addWidget(QLabel("Synthesizer:")) + model_layout.addWidget(self.synthesizer_box) self.vocoder_box = QComboBox() model_layout.addWidget(QLabel("Vocoder:")) model_layout.addWidget(self.vocoder_box) - self.extractor_box = QComboBox() - model_layout.addWidget(QLabel("Extractor:")) - model_layout.addWidget(self.extractor_box) - self.convertor_box = QComboBox() - model_layout.addWidget(QLabel("Convertor:")) - model_layout.addWidget(self.convertor_box) - #Replay & Save Audio i = 0 output_layout.addWidget(QLabel("Toolbox Output:"), i, 0) @@ -578,7 +587,7 @@ class UI(QDialog): ## Embed & spectrograms vis_layout.addStretch() - + # TODO: add spectrograms for source gridspec_kw = {"width_ratios": [1, 4]} fig, self.current_ax = plt.subplots(1, 2, figsize=(10, 2.25), facecolor="#F0F0F0", gridspec_kw=gridspec_kw) @@ -599,20 +608,22 @@ class UI(QDialog): self.text_prompt = QPlainTextEdit(default_text) gen_layout.addWidget(self.text_prompt, stretch=1) - self.generate_button = QPushButton("Synthesize and vocode") - gen_layout.addWidget(self.generate_button) - - layout = QHBoxLayout() - self.synthesize_button = QPushButton("Synthesize only") - layout.addWidget(self.synthesize_button) + if vc_mode: + layout = QHBoxLayout() + self.convert_button = QPushButton("Extract and Convert") + layout.addWidget(self.convert_button) + gen_layout.addLayout(layout) + else: + self.generate_button = QPushButton("Synthesize and vocode") + gen_layout.addWidget(self.generate_button) + layout = QHBoxLayout() + self.synthesize_button = QPushButton("Synthesize only") + layout.addWidget(self.synthesize_button) + self.vocode_button = QPushButton("Vocode only") layout.addWidget(self.vocode_button) gen_layout.addLayout(layout) - layout = QHBoxLayout() - self.convert_button = QPushButton("Extract and Convert") - layout.addWidget(self.convert_button) - gen_layout.addLayout(layout) layout_seed = QGridLayout() self.random_seed_checkbox = QCheckBox("Random seed:") @@ -681,7 +692,7 @@ class UI(QDialog): self.resize(max_size) ## Finalize the display - self.reset_interface() + self.reset_interface(vc_mode) self.show() def start(self):