Resampler Flag Curves and Classic Voice Color Curves via N-Way Spectral Morphing

OpenUtau.Core/Classic/SpectralMorpher.cs

@@ -0,0 +1,163 @@
+using System;
+using System.Collections.Generic;
+using System.Numerics;
+using MathNet.Numerics.IntegralTransforms;
+using OpenUtau.Core.Util;
+
+namespace OpenUtau.Core.Render {
+    public static class SpectralMorpher {
+        private const int N_FFT = 2048;
+        private const int HOP_LENGTH = 256;
+
+        public static float[] MorphN(float[] baseAudio, List<float[]> colorAudios, List<float[]> colorCurves, int sampleRate = 44100) {
+            double[] window = new double[N_FFT];
+            for (int i = 0; i < N_FFT; i++) window[i] = 0.5 * (1 - Math.Cos(2 * Math.PI * i / (N_FFT - 1)));
+
+            Complex[][] stftBase = ComputeSTFT(baseAudio, window);
+            List<Complex[][]> stftColors = new List<Complex[][]>();
+            foreach (var audio in colorAudios) stftColors.Add(ComputeSTFT(audio, window));
+
+            int numFrames = stftBase.Length; 
+            Complex[][] stftMorphed = new Complex[numFrames][];
+            int numColors = colorAudios.Count;
+
+            double[] targetMags = new double[N_FFT];
+            Complex[] targetComplexes = new Complex[N_FFT];
+            double[] rawCoherence = new double[N_FFT];
+            double[] smoothCoherence = new double[N_FFT];
+
+            for (int i = 0; i < numFrames; i++) {
+                stftMorphed[i] = new Complex[N_FFT];
+                double timeMs = (i * HOP_LENGTH / (double)sampleRate) * 1000.0;
+                int curveIndex = (int)(timeMs / 5.0); 
+
+                double[] weights = new double[numColors];
+                double sumColorWeights = 0;
+                for (int c = 0; c < numColors; c++) {
+                    int idx = Math.Min(curveIndex, colorCurves[c].Length - 1);
+                    weights[c] = Math.Max(0, colorCurves[c][idx]) / 100.0; 
+                    sumColorWeights += weights[c];
+                }
+                if (sumColorWeights > 1.0) {
+                    for (int c = 0; c < numColors; c++) weights[c] /= sumColorWeights;
+                    sumColorWeights = 1.0;
+                }
+                double baseWeight = 1.0 - sumColorWeights;
+
+                bool usePhaseLocked = Preferences.Default.PhaseLocked; 
+
+                if (usePhaseLocked) {
+                    // AUTO MODE: Dynamic Per-Bin Adaptive Blend (Optimized for Organic Transitions)
+                    // raw processing vectors and basic vector coherence metrics
+                    for (int bin = 0; bin < N_FFT; bin++) {
+                        double targetMag = baseWeight * stftBase[i][bin].Magnitude;
+                        for (int c = 0; c < numColors; c++) {
+                            var frameColor = (i < stftColors[c].Length) ? stftColors[c][i] : stftBase[i];
+                            targetMag += weights[c] * frameColor[bin].Magnitude;
+                        }
+                        targetMags[bin] = targetMag;
+
+                        Complex targetComplex = stftBase[i][bin] * baseWeight;
+                        for (int c = 0; c < numColors; c++) {
+                            var frameColor = (i < stftColors[c].Length) ? stftColors[c][i] : stftBase[i];
+                            targetComplex += frameColor[bin] * weights[c];
+                        }
+                        targetComplexes[bin] = targetComplex;
+
+                        double complexMag = targetComplex.Magnitude;
+                        rawCoherence[bin] = (targetMag > 1e-8) ? Math.Clamp(complexMag / targetMag, 0.0, 1.0) : 1.0;
+                    }
+
+                    // Smooth coherence across the frequency axis via a localized 5-tap filter.
+                    for (int bin = 0; bin < N_FFT; bin++) {
+                        double sum = 0;
+                        double weightSum = 0;
+
+                        for (int k = -2; k <= 2; k++) {
+                            int neighbor = bin + k;
+                            if (neighbor >= 0 && neighbor < N_FFT) {
+                                // Smooth bell curve distribution weights
+                                double w = (k == 0) ? 0.40 : (Math.Abs(k) == 1 ? 0.25 : 0.05);
+                                sum += rawCoherence[neighbor] * w;
+                                weightSum += w;
+                            }
+                        }
+                        smoothCoherence[bin] = sum / weightSum;
+                    }
+
+                    for (int bin = 0; bin < N_FFT; bin++) {
+                        Complex purePhaseLocked = Complex.FromPolarCoordinates(targetMags[bin], stftBase[i][bin].Phase);
+                        double c = smoothCoherence[bin];
+                        double blendFactor = c * c * (3.0 - 2.0 * c);
+
+                        stftMorphed[i][bin] = (purePhaseLocked * (1.0 - blendFactor)) + (targetComplexes[bin] * blendFactor);
+                    }
+                } else {
+                    // Force Pure Phase-Locked (Stable volume, better for smooth vowels)
+                    for (int bin = 0; bin < N_FFT; bin++) {
+                        double targetMag = baseWeight * stftBase[i][bin].Magnitude;
+
+                        for (int c = 0; c < numColors; c++) {
+                            var frameColor = (i < stftColors[c].Length) ? stftColors[c][i] : stftBase[i];
+                            targetMag += weights[c] * frameColor[bin].Magnitude;
+                        }
+
+                        stftMorphed[i][bin] = Complex.FromPolarCoordinates(targetMag, stftBase[i][bin].Phase);
+                    }
+                }
+            }
+            return ComputeISTFT(stftMorphed, window, baseAudio.Length);
+        }
+
+        private static Complex[][] ComputeSTFT(float[] audio, double[] window) {
+            int numFrames = (audio.Length / HOP_LENGTH) + 3; 
+            Complex[][] frames = new Complex[numFrames][];
+
+            for (int i = 0; i < numFrames; i++) {
+                frames[i] = new Complex[N_FFT];
+                int offset = i * HOP_LENGTH - N_FFT / 2; 
+
+                for (int j = 0; j < N_FFT; j++) {
+                    int idx = offset + j;
+                    float sample = (idx >= 0 && idx < audio.Length) ? audio[idx] : 0f; 
+                    frames[i][j] = new Complex(sample * window[j], 0);
+                }
+                Fourier.Forward(frames[i], FourierOptions.Default);
+            }
+            return frames;
+        }
+
+        private static float[] ComputeISTFT(Complex[][] stft, double[] window, int expectedLength) {
+            int numFrames = stft.Length;
+            int maxOutLength = numFrames * HOP_LENGTH + N_FFT;
+            float[] output = new float[maxOutLength];
+            double[] windowSum = new double[maxOutLength];
+
+            for (int i = 0; i < numFrames; i++) {
+                Complex[] frame = new Complex[N_FFT];
+                Array.Copy(stft[i], frame, N_FFT);
+
+                Fourier.Inverse(frame, FourierOptions.Default);
+
+                int offset = i * HOP_LENGTH - N_FFT / 2; 
+                for (int j = 0; j < N_FFT; j++) {
+                    int idx = offset + j;
+                    if (idx >= 0 && idx < maxOutLength) {
+                        output[idx] += (float)(frame[j].Real * window[j]);
+                        windowSum[idx] += window[j] * window[j];
+                    }
+                }
+            }
+
+            float[] finalOutput = new float[expectedLength];
+            for (int i = 0; i < expectedLength; i++) {
+                double wSum = windowSum[i];
+                float val = 0f;
+                if (wSum > 1e-8) val = (float)(output[i] / wSum);
+                if (float.IsNaN(val) || float.IsInfinity(val)) val = 0f;
+                finalOutput[i] = val;
+            }
+            return finalOutput;
+        }
+    }
+}

OpenUtau.Core/Format/USTx.cs

@@ -10,7 +10,7 @@
 
 namespace OpenUtau.Core.Format {
     public class Ustx {
-        public static readonly Version kUstxVersion = new Version(0, 9);
+        public static readonly Version kUstxVersion = new Version(0, 10);
 
         public const string DYN = "dyn";
         public const string PITD = "pitd";

OpenUtau.Core/OpenUtau.Core.csproj

@@ -14,6 +14,7 @@
     <PackageReference Include="csharp-pinyin" Version="1.0.0" />
     <PackageReference Include="Ignore" Version="0.1.50" />
     <PackageReference Include="K4os.Hash.xxHash" Version="1.0.8" />
+    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
     <PackageReference Include="Melanchall.DryWetMidi" Version="7.2.0" />
     <PackageReference Include="NAudio.Core" Version="2.2.1" />
     <PackageReference Include="NAudio.Vorbis" Version="1.5.0" />

OpenUtau.Core/Render/RenderPhrase.cs

@@ -418,7 +418,7 @@ double Fade(double diff, int pit) {
             var curves = new List<Tuple<string, float[]>>();
 
             foreach(var descriptor in project.expressions.Values) {
-                if(descriptor.type != UExpressionType.Curve) {
+                if(descriptor.type != UExpressionType.Curve && descriptor.type != UExpressionType.MorphingCurve) {
                     continue;
                 }
                 var curve = part.curves.FirstOrDefault(c => c.abbr == descriptor.abbr);

OpenUtau.Core/Ustx/UExpression.cs

@@ -8,6 +8,7 @@ public enum UExpressionType : int {
         Numerical = 0,
         Options = 1,
         Curve = 2,
+        MorphingCurve = 3,
     }
 
     /// <summary>

OpenUtau.Core/Util/Preferences.cs

@@ -251,6 +251,8 @@ public class SerializablePreferences {
 ";
             public string RecoveryPath = string.Empty;
             public bool DetachPianoRoll = false;
+            public bool PhraseLevelMorphing = false;
+            public bool PhaseLocked = true;
 
             // ----- Mix FX (post-processing) -----
             // Per-track FX state lives in UTrack.MixFx and the project ustx.

OpenUtau/Controls/ExpressionCanvas.cs

@@ -120,7 +120,7 @@ public override void Render(DrawingContext context) {
             double optionHeight = descriptor.type == UExpressionType.Options
                 ? Bounds.Height / descriptor.options.Length
                 : 0;
-            if (descriptor.type == UExpressionType.Curve) {
+            if (descriptor.type == UExpressionType.Curve || descriptor.type == UExpressionType.MorphingCurve) {
                 var curve = Part.curves.FirstOrDefault(c => c.descriptor == descriptor);
                 double defaultHeight = Math.Round(Bounds.Height - Bounds.Height * (descriptor.defaultValue - descriptor.min) / (descriptor.max - descriptor.min));
                 var lPen = ThemeManager.AccentPen1;

OpenUtau/Controls/PianoRoll.axaml.cs

@@ -1007,7 +1007,7 @@ public void ExpCanvasPointerPressed(object sender, PointerPressedEventArgs args)
                 return;
             }
             if (point.Properties.IsLeftButtonPressed) {
-                if (descriptor.type == UExpressionType.Curve) {
+                if (descriptor.type == UExpressionType.Curve || descriptor.type == UExpressionType.MorphingCurve) {
                     switch (ViewModel.CurveViewModel.CurveTool) {
                         case CurveTools.CurveSelectTool:
                             editState = new CurveSelectionState(control, ViewModel, this, descriptor);
@@ -1029,7 +1029,7 @@ public void ExpCanvasPointerPressed(object sender, PointerPressedEventArgs args)
                 }
                 Cursor = null;
             } else if (point.Properties.IsRightButtonPressed) {
-                if (descriptor.type == UExpressionType.Curve && ViewModel.CurveViewModel.CurveTool == CurveTools.CurveSelectTool) {
+                if ((descriptor.type == UExpressionType.Curve || descriptor.type == UExpressionType.MorphingCurve) && ViewModel.CurveViewModel.CurveTool == CurveTools.CurveSelectTool) {
                     ViewModel.CurveViewModel.ClearSelect();
                 } else {
                     ViewModel.CurveViewModel.ClearSelect();

OpenUtau/Strings/Strings.axaml

@@ -221,6 +221,7 @@ Do you want to continue by splitting at the nearest position after current playh
   <system:String x:Key="exps.trackdefault">Track Default</system:String>
   <system:String x:Key="exps.type">Type</system:String>
   <system:String x:Key="exps.type.curve">Curve</system:String>
+  <system:String x:Key="exps.type.morphingcurve">Morphing Curve</system:String>
   <system:String x:Key="exps.type.numerical">Numerical</system:String>
   <system:String x:Key="exps.type.options">Options</system:String>
 
@@ -642,6 +643,8 @@ Warning: this option removes custom presets.</system:String>
   <system:String x:Key="prefs.rendering.threads.numthreads">Maximum Render Threads</system:String>
   <system:String x:Key="prefs.rendering.wavtool">Wavtool</system:String>
   <system:String x:Key="prefs.utau">UTAU</system:String>
+  <system:String x:Key="prefs.advanced.phraselevelmorphing">Phrase Level Morphing</system:String>
+  <system:String x:Key="prefs.advanced.phasemorphing">Auto Phase-locked Morphing</system:String>
 
   <system:String x:Key="progress.cachecleared">Cache cleared.</system:String>
   <system:String x:Key="progress.clearingcache">Clearing cache...</system:String>

OpenUtau/ViewModels/ExpressionsViewModel.cs

@@ -65,8 +65,8 @@ public ExpressionBuilder(string name, string abbr, float min, float max, bool is
             this.WhenAnyValue(x => x.Abbr)
                 .Select(abbr => !Core.Format.Ustx.required.Contains(abbr) || ExpressionsViewModel.isTrackOverride)
                 .ToProperty(this, x => x.IsRemovable, out isRemovable);
-            this.WhenAnyValue(x => x.ExpressionType)
-                .Select(type => type == 0) // Numerical
+           this.WhenAnyValue(x => x.ExpressionType)
+                .Select(type => type == 0 || type == 3) // 0 = Numerical, 3 = MorphingCurve
                 .ToProperty(this, x => x.IsNumerical, out isNumerical);
             this.WhenAnyValue(x => x.ExpressionType)
                 .Select(type => type == 1) // Options
@@ -83,7 +83,7 @@ public ExpressionBuilder(string name, string abbr, float min, float max, bool is
             if (string.IsNullOrWhiteSpace(Abbr)) {
                 return new string[] { "Abbreviation must be set.", "<translate:errors.expression.abbrset>" };
             }
-            if (ExpressionType == 0) { // Numerical
+            if (ExpressionType == 0 || ExpressionType == 3) { // Numerical or MorphingCurve
                 if (Abbr.Trim().Length < 1 || Abbr.Trim().Length > 4) {
                     return new string[] { "Abbreviation must be between 1 and 4 characters long.", $"<translate:errors.expression.abbrlong>: {Name}" };
                 }
@@ -110,6 +110,10 @@ public UExpressionDescriptor Build() {
                     return new UExpressionDescriptor(Name.Trim(), Abbr.Trim().ToLower(), Min, Max, DefaultValue) {
                         type = UExpressionType.Curve,
                     };
+                case UExpressionType.MorphingCurve:
+                    return new UExpressionDescriptor(Name.Trim(), Abbr.Trim().ToLower(), Min, Max, DefaultValue, Flag, CustomeDefaultValue, SkipOutputIfDefault) {
+                        type = UExpressionType.MorphingCurve,
+                    };
             }
             throw new Exception("Unexpected expression type");
         }

OpenUtau/ViewModels/NotePropertiesViewModel.cs

@@ -229,7 +229,7 @@ public void LoadPart(UPart? part) {
                 this.Part = part as UVoicePart;
                 var track = DocManager.Inst.Project.tracks[part.trackNo];
                 foreach (var descriptor in track.GetSupportedExps(DocManager.Inst.Project)) {
-                    if (descriptor.type != UExpressionType.Curve) {
+                    if (descriptor.type != UExpressionType.Curve && descriptor.type != UExpressionType.MorphingCurve) {
                         var viewModel = new NotePropertyExpViewModel(descriptor, this);
                         if (descriptor.abbr == Ustx.CLR) {
                             if (track.VoiceColorExp != null && track.VoiceColorExp.options.Length > 0) {

OpenUtau/ViewModels/NotesViewModel.cs

@@ -168,7 +168,7 @@ public NotesViewModel() {
                             ExpTrackHeight = t.Item1.Height / descriptor.options.Length;
                             ExpShadowOpacity = 0;
                         }
-                        ShowCurveToolbox = descriptor.type == UExpressionType.Curve;
+                        ShowCurveToolbox = descriptor.type == UExpressionType.Curve || descriptor.type == UExpressionType.MorphingCurve;
                     } else {
                         ExpTrackHeight = 0;
                         ExpShadowOpacity = 0.3;

OpenUtau/ViewModels/PasteParamViewModel.cs

@@ -9,7 +9,7 @@ public PasteParamViewModel() {
             Params.Add(new PasteParameter("pitch points", ""));
             Params.Add(new PasteParameter("vibrato", ""));
             foreach(var exp in DocManager.Inst.Project.expressions) {
-                if(exp.Value.type != Core.Ustx.UExpressionType.Curve) {
+                if(exp.Value.type != Core.Ustx.UExpressionType.Curve && exp.Value.type != Core.Ustx.UExpressionType.MorphingCurve) {
                     Params.Add(new PasteParameter(exp.Value.name, exp.Key));
                 }
             }

OpenUtau/ViewModels/PreferencesViewModel.cs

@@ -110,6 +110,8 @@ public int SafeMaxThreadCount {
         [Reactive] public int OtoEditor { get; set; }
         public string VLabelerPath => Preferences.Default.VLabelerPath;
         public string SetParamPath => Preferences.Default.SetParamPath;
+        [Reactive] public bool PhraseLevelMorphing { get; set; }
+        [Reactive] public bool PhaseLocked { get; set; }
 
         // Diffsinger
         public List<int> DiffSingerStepsOptions { get; } = new List<int> { 2, 5, 10, 20, 50, 100, 200, 500, 1000 };
@@ -121,7 +123,6 @@ public int SafeMaxThreadCount {
         [Reactive] public double DiffSingerDepth { get; set; }
         [Reactive] public bool DiffSingerTensorCache { get; set; }
         [Reactive] public bool DiffSingerLangCodeHide { get; set; }
-
         // Advanced
         [Reactive] public bool RememberMid { get; set; }
         [Reactive] public bool RememberUst { get; set; }
@@ -191,6 +192,8 @@ public PreferencesViewModel() {
             RememberUst = Preferences.Default.RememberUst;
             RememberVsqx = Preferences.Default.RememberVsqx;
             ClearCacheOnQuit = Preferences.Default.ClearCacheOnQuit;
+            PhraseLevelMorphing = Preferences.Default.PhraseLevelMorphing;
+            PhaseLocked = Preferences.Default.PhaseLocked;
 
             MessageBus.Current.Listen<ThemeEditorStateChangedEvent>()
                 .Subscribe(_ => this.RaisePropertyChanged(nameof(IsThemeEditorOpen)));
@@ -403,6 +406,16 @@ public PreferencesViewModel() {
                     Preferences.Default.SkipRenderingMutedTracks = skipRenderingMutedTracks;
                     Preferences.Save();
                 });
+            this.WhenAnyValue(vm => vm.PhraseLevelMorphing)
+                .Subscribe(index => {
+                    Preferences.Default.PhraseLevelMorphing = index;
+                    Preferences.Save();
+                });
+            this.WhenAnyValue(vm => vm.PhaseLocked)
+                .Subscribe(index => {
+                    Preferences.Default.PhaseLocked = index;
+                    Preferences.Save();
+                });
         }
 
         public void TestAudioOutputDevice() {

OpenUtau/Views/ExpressionsDialog.axaml

@@ -62,6 +62,7 @@
           <ComboBoxItem Content="{DynamicResource exps.type.numerical}"/>
           <ComboBoxItem Content="{DynamicResource exps.type.options}"/>
           <ComboBoxItem Content="{DynamicResource exps.type.curve}"/>
+          <ComboBoxItem Content="{DynamicResource exps.type.morphingcurve}"/>
         </ComboBox>
       </DockPanel>
       <DockPanel IsVisible="{Binding Expression.IsOptions}">

OpenUtau/Views/NoteEditStates.cs

@@ -682,7 +682,7 @@ public override void Update(IPointer pointer, Point point) {
             if (descriptor == null) {
                 return;
             }
-            if (descriptor.type != UExpressionType.Curve) {
+            if (descriptor.type != UExpressionType.Curve && descriptor.type != UExpressionType.MorphingCurve) {
                 UpdatePhonemeExp(pointer, point);
             } else {
                 UpdateCurveExp(pointer, point);
@@ -808,7 +808,7 @@ public override void Update(IPointer pointer, Point point) {
             if (descriptor == null) {
                 return;
             }
-            if (descriptor.type != UExpressionType.Curve) {
+            if (descriptor.type != UExpressionType.Curve && descriptor.type != UExpressionType.MorphingCurve) {
                 ResetPhonemeExp(pointer, point);
             } else {
                 ResetCurveExp(pointer, point);