Merge pull request #9 from YuzukiTsuru/AudioModelV2

Introduce AudioModel V2 as AudioModel interface, use STL operation

Author: YuzukiTsuru

.clang-format

@@ -0,0 +1,66 @@
+# Generated from CLion C/C++ Code Style settings
+BasedOnStyle: LLVM
+AccessModifierOffset: -4
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: None
+AlignOperands: Align
+AllowAllArgumentsOnNextLine: false
+AllowAllConstructorInitializersOnNextLine: false
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: Always
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: Always
+AllowShortLambdasOnASingleLine: All
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterReturnType: None
+AlwaysBreakTemplateDeclarations: Yes
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterCaseLabel: false
+  AfterClass: false
+  AfterControlStatement: Never
+  AfterEnum: false
+  AfterFunction: false
+  AfterNamespace: false
+  AfterUnion: false
+  BeforeCatch: false
+  BeforeElse: false
+  IndentBraces: false
+  SplitEmptyFunction: false
+  SplitEmptyRecord: true
+BreakBeforeBinaryOperators: None
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializers: BeforeColon
+BreakInheritanceList: BeforeColon
+ColumnLimit: 0
+CompactNamespaces: false
+ContinuationIndentWidth: 8
+IndentCaseLabels: true
+IndentPPDirectives: None
+IndentWidth: 4
+KeepEmptyLinesAtTheStartOfBlocks: true
+MaxEmptyLinesToKeep: 2
+NamespaceIndentation: All
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PointerAlignment: Right
+ReflowComments: false
+SpaceAfterCStyleCast: true
+SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: false
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 0
+SpacesInAngles: false
+SpacesInCStyleCastParentheses: false
+SpacesInContainerLiterals: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+TabWidth: 4
+UseTab: Never

.gitmodules

@@ -19,6 +19,3 @@
 [submodule "lib/sndfile"]
 	path = lib/sndfile
 	url = https://github.com/libsndfile/libsndfile
-[submodule "lib/lz4"]
-	path = lib/lz4
-	url = https://github.com/lz4/lz4

CMakeLists.txt

@@ -87,7 +87,6 @@ add_subdirectory(lib/dialog EXCLUDE_FROM_ALL)
 add_subdirectory(lib/inicpp EXCLUDE_FROM_ALL)
 add_subdirectory(lib/ftxui EXCLUDE_FROM_ALL)
 add_subdirectory(lib/rapidjson EXCLUDE_FROM_ALL)
-add_subdirectory(lib/lz4/build/cmake EXCLUDE_FROM_ALL)
 add_subdirectory(lib/sndfile EXCLUDE_FROM_ALL)
 
 # include header files
@@ -101,7 +100,6 @@ include_directories(
         lib/inicpp/include
         lib/rapidjson/include
         lib/sndfile/include
-        lib/lz4/lib
         ${PROJECT_BINARY_DIR}
         ${PROJECT_BINARY_DIR}/lib/sndfile/include
 )

docs/README.md

@@ -34,3 +34,28 @@ lessampler is a Singing Voice Synthesizer. It provides complete pitch shifting,
 ***Currently lesssampler is still very unstable, there are many bugs that need to be fixed, but you are very welcome to participate in the test.***
 
 You can find the alpha version at [Release](https://github.com/YuzukiTsuru/lessampler/releases)
+
+# Contributing to lessampler
+
+This section contains articles that are relevant to anyone who wants to contribute to lessampler and help improve the project.
+
+It provides an introduction to contributing, along with a few ideas for how you can help.
+
+## lessaudio file format
+
+lessaudio is the audio source model format file used by lessampler, which contains the audio f0, spectrogram, aperiodicity and some basic parameters corresponding to them. The following table lists the index order and its function.
+
+> ! Note: the current version of lessaudio only saves the data of lessaudiomodel, if there is any modification in the future, it will be explained here
+
+| index | data | data size |
+| ----- | ---- | --------- |
+| 1     |      |           |
+| 2     |      |           |
+| 3     |      |           |
+| 4     |      |           |
+| 5     |      |           |
+| 6     |      |           |
+| 7     |      |           |
+| 8     |      |           |
+| 9     |      |           |
+

docs/shine/shine.md

@@ -0,0 +1,41 @@
+# Shine
+Shine is a data class used to describe the parameters of individual audio transforms. Contains input and output audio files, audio interception and transformation parameters, audio pitch adjustment parameters, and elongation parameters. Shine also provides a convenient interface for describing timbre transformations, etc.
+
+## Basic define
+
+```c++
+class ShinePara {
+public:
+    // Basic Transformation Parameters
+    std::string input_file_name = {};
+    std::string output_file_name = {};
+    int time_percent = 0;
+    double velocity = 0.0;
+    double offset = 0.0;
+    double required_length = 0.0;
+    int required_frame = 0;
+    double first_half_fixed_part = 0.0;
+    double last_unused_part = 0.0;
+    double volumes = 0;
+    int modulation = 0;
+    double wave_length = 0.0;
+    double pre_cross_length = 0.0;
+    double base_length = 0.0;
+    double cross_length = 0.0;
+    double stretch_length = 0.0;
+    int output_samples = 0;
+    double scale_num = 0.0;
+    int tempo_num = 0;
+
+public:
+    // Pitch sections
+    int *pitch_bend = nullptr;
+    int pitch_length = 0;
+    int pitch_step = 256;
+
+public:
+    // Options
+    bool is_custom_pitch = false;
+    bool is_gender = false;
+}
+```

lib/lz4

@@ -11,21 +11,18 @@
 //
 // Created by gloom on 2022/5/2.
 //
-#include <memory>
-
-#include "FileIO/AudioModelIO.h"
-#include "FileIO/JSONFileIO.h"
-
 #include "AudioModel.h"
 #include "WorldModule/WorldModule.h"
 
-AudioModel::AudioModel(double *x, size_t x_length, int fs, const lessConfigure &configure) : configure(configure) {
-    _lessAudioModel.x = x;
-    _lessAudioModel.x_length = static_cast<int>(x_length);
+AudioModel::AudioModel(double *x, int x_length, int fs, const lessConfigure &configure) : configure(configure) {
+    _lessAudioModel.x.reserve(x_length);
+    _lessAudioModel.x.insert(_lessAudioModel.x.end(), x, x + x_length);
+
+    _lessAudioModel.x_length = x_length;
     _lessAudioModel.fs = fs;
 
     // initialize the audio model from x, x_length, fs using World Vocoder
-    WorldModule model(_lessAudioModel.x, _lessAudioModel.x_length, _lessAudioModel.fs, configure);
+    WorldModule model(x, x_length, _lessAudioModel.fs, configure);
     worldPara = model.GetModule();
     InitAudioModel();
 }
@@ -39,23 +36,21 @@ void AudioModel::InitAudioModel() {
     _lessAudioModel.frame_period = worldPara.frame_period;
     _lessAudioModel.f0_length = worldPara.f0_length;
 
-    _lessAudioModel.f0 = new double[worldPara.f0_length];
-    std::memcpy(_lessAudioModel.f0, worldPara.f0, sizeof(double) * worldPara.f0_length);
+    _lessAudioModel.f0.reserve(worldPara.f0_length);
+    _lessAudioModel.f0.insert(_lessAudioModel.f0.end(), worldPara.f0, worldPara.f0 + worldPara.f0_length);
 
-    _lessAudioModel.time_axis = new double[worldPara.f0_length];
-    std::memcpy(_lessAudioModel.time_axis, worldPara.time_axis, sizeof(double) * worldPara.f0_length);
+    _lessAudioModel.time_axis.reserve(worldPara.f0_length);
+    _lessAudioModel.time_axis.insert(_lessAudioModel.time_axis.end(), worldPara.time_axis, worldPara.time_axis + worldPara.f0_length);
 
     _lessAudioModel.w_length = worldPara.fft_size / 2 + 1;
 
-    _lessAudioModel.spectrogram = new double *[worldPara.f0_length];
+    _lessAudioModel.spectrogram.resize(worldPara.f0_length, std::vector<double>(_lessAudioModel.w_length));
     for (int i = 0; i < worldPara.f0_length; ++i) {
-        _lessAudioModel.spectrogram[i] = new double[_lessAudioModel.w_length];
-        std::memcpy(_lessAudioModel.spectrogram[i], worldPara.spectrogram[i], sizeof(double) * _lessAudioModel.w_length);
+        _lessAudioModel.spectrogram[i].assign(&(worldPara.spectrogram[i][0]), &(worldPara.spectrogram[i][_lessAudioModel.w_length]));
     }
 
-    _lessAudioModel.aperiodicity = new double *[worldPara.f0_length];
+    _lessAudioModel.aperiodicity.resize(worldPara.f0_length, std::vector<double>(_lessAudioModel.w_length));
     for (int i = 0; i < worldPara.f0_length; ++i) {
-        _lessAudioModel.aperiodicity[i] = new double[_lessAudioModel.w_length];
-        std::memcpy(_lessAudioModel.aperiodicity[i], worldPara.aperiodicity[i], sizeof(double) * _lessAudioModel.w_length);
+        _lessAudioModel.aperiodicity[i].assign(&(worldPara.aperiodicity[i][0]), &(worldPara.aperiodicity[i][_lessAudioModel.w_length]));
     }
 }

src/AudioModel/AudioModel.cpp

@@ -22,7 +22,7 @@
 
 class AudioModel {
 public:
-    AudioModel(double *x, size_t x_length, int fs, const lessConfigure &configure);
+    AudioModel(double *x, int x_length, int fs, const lessConfigure &configure);
 
     lessAudioModel GetAudioModel();
 

src/AudioModel/AudioModel.h

@@ -14,16 +14,22 @@
 
 #include "Synthesis.h"
 
-#include <world/synthesis.h>
+#include <utility>
 
 #include "Utils/LOG.h"
-#include "world/synthesisrealtime.h"
+#include "Utils/Timer.h"
 
-Synthesis::Synthesis(lessAudioModel audioModel, int x_length) : audioModel(audioModel), x_length(x_length) {
+#include <world/synthesisrealtime.h>
+
+Synthesis::Synthesis(lessAudioModel audioModel, int x_length) : audioModel(std::move(audioModel)), x_length(x_length) {
     YALL_DEBUG_ << "Allocate Out Memory, Length: " + std::to_string(x_length);
     AllocateMemory();
-    YALL_DEBUG_ << "Synthesis Wav...";
+    YALL_DEBUG_ << "Synthesis Audio...";
+    uint64_t tmsStart = get_perf_count();
     SynthesisWav();
+    uint64_t tmsEnd = get_perf_count();
+    uint64_t usVal = (tmsEnd - tmsStart) / 10000;
+    YALL_INFO_ << "Synthesis Audio: " + std::to_string(usVal) + " ms";
 }
 
 Synthesis::~Synthesis() {
@@ -44,11 +50,22 @@ void Synthesis::SynthesisWav() const {
     InitializeSynthesizer(audioModel.fs, audioModel.frame_period,
                           audioModel.fft_size, buffer_size, 100, &synthesizer);
 
+    auto f0 = new double[audioModel.f0.size()];
+    std::copy(audioModel.f0.begin(), audioModel.f0.end(), f0);
+    auto spectrogram = new double *[audioModel.f0_length];
+    auto aperiodicity = new double *[audioModel.f0_length];
+    for (int i = 0; i < audioModel.f0_length; ++i) {
+        spectrogram[i] = new double[audioModel.w_length];
+        aperiodicity[i] = new double[audioModel.w_length];
+        std::copy(audioModel.spectrogram[i].begin(), audioModel.spectrogram[i].end(), spectrogram[i]);
+        std::copy(audioModel.aperiodicity[i].begin(), audioModel.aperiodicity[i].end(), aperiodicity[i]);
+    }
+
     int offset = 0;
-    int index = 0;
+    int index;
     for (int i = 0; i < audioModel.f0_length;) {
         // Add one frame ('i' shows the frame index that should be added)
-        if (AddParameters(&audioModel.f0[i], 1, &audioModel.spectrogram[i], &audioModel.aperiodicity[i], &synthesizer) == 1) {
+        if (AddParameters(&f0[i], 1, &spectrogram[i], &aperiodicity[i], &synthesizer) == 1) {
             ++i;
         }
 

src/AudioModel/Synthesis/Synthesis.cpp

@@ -22,16 +22,20 @@
 #include <world/harvest.h>
 #include <world/d4c.h>
 
+#include <utility>
+
 WorldModule::WorldModule(double *x, int x_length, int fs, const lessConfigure &configure) : x(x), x_length(x_length), configure(configure) {
+    // Set the para
     this->worldPara.fs = fs;
     this->worldPara.frame_period = configure.audio_model_frame_period;
+
     YALL_DEBUG_ << "Generate F0 from PCM file.";
     if (configure.f0_mode == lessConfigure::F0_MODE::F0_MODE_DIO) {
         F0EstimationDio();
     } else if (configure.f0_mode == lessConfigure::F0_MODE::F0_MODE_HARVEST) {
         F0EstimationHarvest();
     } else {
-        YALL_ERROR_ << "F0 Estimation Mode Error.";
+        throw std::runtime_error("F0 Estimation Mode Error.");
     }
     YALL_DEBUG_ << "Generate Envelope from PCM file and F0.";
     SpectralEnvelopeEstimation();