diff --git a/src/main/kotlin/me/ksanstone/wavesync/wavesync/gui/component/visualizer/SpectrogramVisualizer.kt b/src/main/kotlin/me/ksanstone/wavesync/wavesync/gui/component/visualizer/SpectrogramVisualizer.kt index d74b086..1b13d36 100644 --- a/src/main/kotlin/me/ksanstone/wavesync/wavesync/gui/component/visualizer/SpectrogramVisualizer.kt +++ b/src/main/kotlin/me/ksanstone/wavesync/wavesync/gui/component/visualizer/SpectrogramVisualizer.kt @@ -1,5 +1,7 @@ package me.ksanstone.wavesync.wavesync.gui.component.visualizer +import com.huskerdev.openglfx.canvas.GLCanvas +import javafx.application.Platform import javafx.beans.binding.IntegerBinding import javafx.beans.property.* import javafx.fxml.FXMLLoader @@ -7,7 +9,6 @@ import javafx.geometry.Orientation import javafx.scene.canvas.GraphicsContext import javafx.scene.chart.NumberAxis import javafx.scene.chart.ValueAxis -import javafx.scene.image.WritableImage import javafx.scene.layout.Background import javafx.scene.layout.HBox import javafx.scene.paint.CycleMethod @@ -22,19 +23,27 @@ import me.ksanstone.wavesync.wavesync.ApplicationSettingDefaults.DEFAULT_SPECTRO import me.ksanstone.wavesync.wavesync.WaveSyncBootApplication import me.ksanstone.wavesync.wavesync.gui.controller.visualizer.spectrogram.SpectrogramSettingsController import me.ksanstone.wavesync.wavesync.gui.gradient.pure.GradientSerializer +import me.ksanstone.wavesync.wavesync.gui.gradient.pure.SGradient import me.ksanstone.wavesync.wavesync.gui.utility.AutoCanvas -import me.ksanstone.wavesync.wavesync.service.* +import me.ksanstone.wavesync.wavesync.gui.utility.GlUtil +import me.ksanstone.wavesync.wavesync.service.AudioCaptureService +import me.ksanstone.wavesync.wavesync.service.LocalizationService +import me.ksanstone.wavesync.wavesync.service.PreferenceService +import me.ksanstone.wavesync.wavesync.service.SupportedCaptureSource import me.ksanstone.wavesync.wavesync.service.fftScaling.DeciBelFFTScalar import me.ksanstone.wavesync.wavesync.service.fftScaling.DeciBelFFTScalarParameters -import me.ksanstone.wavesync.wavesync.utility.CachingRangeMapper +import me.ksanstone.wavesync.wavesync.service.fftScaling.FFTScalar import me.ksanstone.wavesync.wavesync.utility.FreeRangeMapper import me.ksanstone.wavesync.wavesync.utility.LogRangeMapper -import me.ksanstone.wavesync.wavesync.utility.RollingBuffer -import kotlin.math.ceil -import kotlin.math.floor -import kotlin.math.max +import me.ksanstone.wavesync.wavesync.utility.RangeMapper +import me.ksanstone.wavesync.wavesync.utility.size +import org.lwjgl.opengl.GL11 +import org.lwjgl.opengl.GL30.* +import java.nio.ByteBuffer +import java.util.* +import java.util.concurrent.ConcurrentLinkedQueue -class SpectrogramVisualizer : AutoCanvas() { +class SpectrogramVisualizer : AutoCanvas(useGL = true) { val bufferDuration: ObjectProperty = SimpleObjectProperty(Duration.seconds(20.0)) val orientation = SimpleObjectProperty(Orientation.VERTICAL) @@ -51,52 +60,33 @@ class SpectrogramVisualizer : AutoCanvas() { val rangeMax: FloatProperty = SimpleFloatProperty(DEFAULT_DB_MAX) val logarithmic: BooleanProperty = SimpleBooleanProperty(DEFAULT_LOGARITHMIC_MODE) - private var buffer: RollingBuffer = RollingBuffer(100) { FloatArray(0) } - private var stripeBuffer: FloatArray = FloatArray(0) - private val fftArraySize: IntegerBinding = - WaveSyncBootApplication.applicationContext.getBean(AudioCaptureService::class.java).fftSize.divide(2) - private val fftRate: IntegerProperty = - WaveSyncBootApplication.applicationContext.getBean(AudioCaptureService::class.java).fftRate + private val acs = WaveSyncBootApplication.applicationContext.getBean(AudioCaptureService::class.java) + private val fftArraySize: IntegerBinding = acs.fftSize.divide(2) + private val fftRate: IntegerProperty = acs.fftRate private val gradientSerializer: GradientSerializer = WaveSyncBootApplication.applicationContext.getBean(GradientSerializer::class.java) - private var imageTop = WritableImage(1, 1) - private var imageBottom = WritableImage(1, 1) - private var imageOffset = 0 - private var lastWritten = 0L - private var bufferPos = 0 - private var stripeStepAccumulator = 0.0 - private var processedStripe = FloatArray(1) - private var stripeMapper: CachingRangeMapper = CachingRangeMapper(FreeRangeMapper(0..1, 2..4)) - private var mapperLog: Boolean = false - - private var canvasWidth = 0 - private var canvasHeight = 0 - private val scalar = DeciBelFFTScalar() private var source: SupportedCaptureSource? = null init { scalar.update(DeciBelFFTScalarParameters(rangeMin.value, rangeMax.value)) - changeBufferWidth() + fftArraySize.addListener { _, _, _ -> - changeBufferWidth() + updateBuffer() } - fftRate.addListener { _, _, v -> - changeBufferDuration(bufferDuration.get(), v.toInt()) - resetBuffer() + fftRate.addListener { _, _, _ -> + updateBuffer() } bufferDuration.addListener { _, _, _ -> - changeBufferDuration(bufferDuration.get(), fftRate.get()) - resetBuffer() + updateBuffer() sizeAxis() } listOf(orientation, highPass, lowPass, effectiveLogarithmic).forEach { it.addListener { _ -> - resetBuffer() sizeAxis() } } @@ -104,15 +94,14 @@ class SpectrogramVisualizer : AutoCanvas() { listOf(effectiveRangeMax, effectiveRangeMin).forEach { it.addListener { _ -> scalar.update(DeciBelFFTScalarParameters(effectiveRangeMin.value, effectiveRangeMax.value)) - resetBuffer() sizeAxis() } } sizeAxis() gradient.addListener { _ -> - resetBuffer() sizeAxis() + if (::glData.isInitialized) glData.scheduleSettingChange(changeGradient = true) } } @@ -156,39 +145,11 @@ class SpectrogramVisualizer : AutoCanvas() { } } - private fun changeBufferDuration(time: Duration, rate: Int) { - val newSize = rate * time.toSeconds() - this.buffer = RollingBuffer(newSize.toInt()) { FloatArray(fftArraySize.value) } - } - - private fun changeBufferWidth() { - changeBufferDuration(bufferDuration.get(), fftRate.value) - stripeBuffer = FloatArray(fftArraySize.value) - } fun handleFFT(event: AudioCaptureService.FftEvent) { - if (!canDraw || buffer.size == 2 || event.data.size != fftArraySize.value) return + if (!canDraw || event.data.size != fftArraySize.value || !::glData.isInitialized) return this.source = event.source - buffer.incrementPosition() - System.arraycopy(event.data, 0, buffer.last(), 0, event.data.size) - } - - private fun resetBuffer() { - lastWritten = buffer.written - (canvasWidth + canvasHeight).toLong() - imageOffset = 0 - stripeStepAccumulator = 0.0 - bufferPos = 0 - } - - private fun createImageBuffers() { - imageTop = WritableImage(canvasWidth.coerceAtLeast(1), canvasHeight.coerceAtLeast(1)) - imageBottom = WritableImage(canvasWidth.coerceAtLeast(1), canvasHeight.coerceAtLeast(1)) - } - - private fun flipImageBuffers() { - val temp = imageTop - imageTop = imageBottom - imageBottom = temp + glData.scheduleSendRow(event.data) } private fun sizeAxis() { @@ -225,192 +186,231 @@ class SpectrogramVisualizer : AutoCanvas() { ) } - private fun drawChunk(isHorizontal: Boolean) { - val size = if (isHorizontal) canvasWidth else canvasHeight - - val chunkStep = buffer.size.toDouble() / size - val chunksWritten = (buffer.written - lastWritten).coerceIn(0, buffer.size.toLong()).toInt() - lastWritten = buffer.written - bufferPos -= chunksWritten - bufferPos = bufferPos.coerceAtLeast(0) - val chunksLeft = buffer.size - bufferPos + private lateinit var glData: GlData + + /** + * Contains all the buffer handles + */ + data class GlData( + val scalar: FFTScalar<*>, + var sampleBuffer: Int = 0, + /** + * Number of entries + */ + var sampleBufferWidth: Int = 0, + /** + * Size of each entry + */ + var sampleBufferHeight: Int = 0, + var sampleBufferPosition: Int = 0, + var program: Int = 0, + + var entryMapperBuffer: Int = 0, + var mapper: RangeMapper = FreeRangeMapper(0..1, 0..1), + + var gradientBuffer: Int = 0, + + var sendBuffer: Queue = ConcurrentLinkedQueue(), + var resizeBuffer: Boolean = false, + var changeGradient: Boolean = true, // initial setup + ) { + + companion object { + const val GRADIENT_TEXTURE_SIZE = 1024 + } - val stripesToDraw = floor(chunksLeft / chunkStep).toInt() + init { + sampleBuffer = glGenTextures() + entryMapperBuffer = glGenTextures() + gradientBuffer = glGenTextures() + val vertexShader = GlUtil.compileShader("/shaders/spectrogram.vert", GL_VERTEX_SHADER) + val fragmentShader = GlUtil.compileShader("/shaders/spectrogram.frag", GL_FRAGMENT_SHADER) + program = GlUtil.linkProgram(listOf(vertexShader, fragmentShader)) + + glBindTexture(GL_TEXTURE_1D, gradientBuffer) + glTexImage1D(GL_TEXTURE_1D, 0, GL_RGBA32F, GRADIENT_TEXTURE_SIZE, 0, GL_RGBA, GL_FLOAT, null as ByteBuffer?) + glTexParameteri(GL_TEXTURE_1D, GL_TEXTURE_MIN_FILTER, GL_LINEAR) + glTexParameteri(GL_TEXTURE_1D, GL_TEXTURE_MAG_FILTER, GL_LINEAR) + glTexParameteri(GL_TEXTURE_1D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE) + } - for (i in 0 until stripesToDraw) { - var iterationsDone = 0 + fun scheduleSettingChange(resizeBuffer: Boolean = false, changeGradient: Boolean = false) { + if (resizeBuffer) this.resizeBuffer = true + if (changeGradient) this.changeGradient = true + } - // First iteration done separately to fill the buffer. - if (bufferPos < buffer.size && stripeStepAccumulator < chunkStep) { - val currentRes = buffer[bufferPos] - System.arraycopy(currentRes, 0, stripeBuffer, 0, stripeBuffer.size) - stripeStepAccumulator++ - iterationsDone++ - bufferPos++ - } + private fun sampleBuffer(length: Int, sampleSize: Int) { + glBindTexture(GL_TEXTURE_2D, sampleBuffer) + glTexImage2D(GL_TEXTURE_2D, 0, GL_R32F, sampleSize, length, 0, GL_RED, GL_FLOAT, null as ByteBuffer?) + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR) + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR) + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE) + sampleBufferWidth = length + sampleBufferHeight = sampleSize + sampleBufferPosition = 0 + glBindTexture(GL_TEXTURE_2D, 0) + } - while (stripeStepAccumulator < chunkStep) { - if (bufferPos >= buffer.size) break - val currentRes = buffer[bufferPos] - for (j in stripeBuffer.indices) { - stripeBuffer[j] += currentRes[j] - } + fun entryMapperBuffer(mapper: RangeMapper) { + this.mapper = mapper + glBindTexture(GL_TEXTURE_1D, entryMapperBuffer) + glTexImage1D( + GL_TEXTURE_1D, + 0, + GL_R32I, + mapper.from.size(), + 0, + GL_RED_INTEGER, + GL_INT, + IntArray(mapper.from.size()) { mapper.forwards(it) }) + glTexParameteri(GL_TEXTURE_1D, GL_TEXTURE_MIN_FILTER, GL_LINEAR) + glTexParameteri(GL_TEXTURE_1D, GL_TEXTURE_MAG_FILTER, GL_LINEAR) + glTexParameteri(GL_TEXTURE_1D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE) + GlUtil.checkGLError("mapper buffer") + glBindTexture(GL_TEXTURE_1D, 0) + } - stripeStepAccumulator++ - iterationsDone++ - bufferPos++ + private fun setGradient(gradient: SGradient) { + val textureData = FloatArray(GRADIENT_TEXTURE_SIZE * 4) + for (i in 0 until GRADIENT_TEXTURE_SIZE) { + val pos = i.toFloat() / (GRADIENT_TEXTURE_SIZE - 1) + val color = gradient[pos] + textureData[i * 4 + 0] = color.red.toFloat() + textureData[i * 4 + 1] = color.green.toFloat() + textureData[i * 4 + 2] = color.blue.toFloat() + textureData[i * 4 + 3] = color.opacity.toFloat() } + glBindTexture(GL_TEXTURE_1D, gradientBuffer) + glTexSubImage1D( + GL_TEXTURE_1D, + 0, + 0, + GRADIENT_TEXTURE_SIZE, + GL_RGBA, + GL_FLOAT, + textureData, + ) + } - stripeStepAccumulator -= chunkStep - - if (iterationsDone == 0) continue + fun scheduleSendRow(row: FloatArray) { + if (sendBuffer.size < 20) + sendBuffer.add(row) + } - val avgFactor = 1.0F / iterationsDone.coerceAtLeast(1) - for (j in stripeBuffer.indices) { - stripeBuffer[j] *= avgFactor + private fun sendScheduled() { + var elem: FloatArray? + while (sendBuffer.poll().also { elem = it } != null) { + sendSampleRow(elem!!) } - - val stripeWidth = 1.0 / chunkStep - drawStripe(stripeBuffer, stripeWidth, isHorizontal, effectiveLogarithmic.value) } - } - private var justSwitched = false + fun changeScheduled(fftRate: Int, bufferDuration: Duration, fftArraySize: Int, gradient: SGradient) { + if (resizeBuffer) sampleBuffer((fftRate * bufferDuration.toSeconds()).toInt(), fftArraySize) + if (changeGradient) setGradient(gradient) - private fun drawStripe(stripe: FloatArray, width: Double, isHorizontal: Boolean, isLog: Boolean) { - if (source == null) return - val stripePixelLength: Int - val size: Int - var realWidth = width + resizeBuffer = false + changeGradient = false - if (isHorizontal) { - stripePixelLength = canvasHeight - size = canvasWidth - } else { - stripePixelLength = canvasWidth - size = canvasHeight + sendScheduled() } - if (justSwitched) { - if (imageOffset != 0) { - realWidth += imageOffset - imageOffset = 0 - } - justSwitched = false + private fun sendSampleRow(row: FloatArray) { + if (row.size != sampleBufferHeight) return + sampleBufferPosition = (sampleBufferPosition + 1) % sampleBufferWidth + glBindTexture(GL_TEXTURE_2D, sampleBuffer) + glTexSubImage2D( + GL_TEXTURE_2D, + 0, + 0, + sampleBufferPosition, + sampleBufferHeight, + 1, + GL_RED, + GL_FLOAT, + FloatArray(row.size) { scalar.scale(row[it]) }, + ) + GlUtil.checkGLError("send row") + glBindTexture(GL_TEXTURE_2D, 0) } - var effectiveStripeLength = source!!.trimResultTo(stripe.size * 2, effectiveHighPass.get()) - val frequencyBinSkip = source!!.bufferBeginningSkipFor(effectiveLowPass.get(), stripe.size * 2) - effectiveStripeLength -= frequencyBinSkip - - - val rate = source!!.rate - val fftSize = stripe.size * 2 - val startFreq = FourierMath.frequencyOfBin(frequencyBinSkip, rate, fftSize) - val endFreq = FourierMath.frequencyOfBin(frequencyBinSkip + effectiveStripeLength, rate, fftSize) - - val resizeStripe = processedStripe.size != stripePixelLength - if (resizeStripe) - processedStripe = FloatArray(stripePixelLength) - - val secRangeEqual = stripeMapper.to.first == startFreq && stripeMapper.to.last == endFreq - if (resizeStripe || !secRangeEqual || isLog != mapperLog) { - val newToRange = startFreq..endFreq - stripeMapper = CachingRangeMapper( - if (isLog) { - LogRangeMapper(processedStripe.indices, newToRange) - } else { - FreeRangeMapper(processedStripe.indices, newToRange) - }, - ) { FourierMath.binOfFrequency(rate, fftSize, it) } - mapperLog = isLog + fun dispose() { + glDeleteTextures(sampleBuffer) + GL11.glDeleteTextures(entryMapperBuffer) + glDeleteProgram(program) } + } - for (i in processedStripe.indices) { - val rMin = stripeMapper.forwards(i) - val rMax = if (i + 1 == processedStripe.size) stripe.size - 1 - else (stripeMapper.forwards(i + 1)) - 1 - var value = 0.0F - for (j in rMin..rMax.coerceAtLeast(rMin)) { - value = max(value, stripe[j]) + override fun setupGl(canvas: GLCanvas) { + canvas.addOnInitEvent { + try { + glData = GlData(scalar) + updateBuffer() + } catch (e: Throwable) { + e.printStackTrace() + Platform.exit() } - processedStripe[i] = scalar.scale(value) } - val writer = imageTop.pixelWriter - val effectiveGradient = gradient.value - val realOffset = ceil(realWidth).toInt().coerceIn(1, size - imageOffset) - - try { - for (offset in 0 until realOffset) { - if (isHorizontal) { - for (i in 0 until stripePixelLength) { - writer.setArgb( - imageOffset + offset, - stripePixelLength - i - 1, - effectiveGradient.argb(processedStripe[i]) - ) - } - } else { - for (i in 0 until stripePixelLength) { - writer.setArgb( - i, - (imageTop.height - imageOffset - 1 + offset).toInt(), - effectiveGradient.argb(processedStripe[i]) - ) - } - } + canvas.addOnRenderEvent { + if (source == null) return@addOnRenderEvent + + val displayEntrySize = if (orientation.value == Orientation.VERTICAL) it.width else it.height + var effectiveStripeLength = source!!.trimResultTo(fftArraySize.value * 2, effectiveHighPass.get()) + val frequencyBinSkip = source!!.bufferBeginningSkipFor(effectiveLowPass.get(), fftArraySize.value * 2) + effectiveStripeLength -= frequencyBinSkip + + val fromRange = 0 until displayEntrySize + val toRange = frequencyBinSkip until frequencyBinSkip + effectiveStripeLength + val mapper = if (effectiveLogarithmic.value) LogRangeMapper(fromRange, toRange) else FreeRangeMapper(fromRange, toRange) + if (mapper.from != glData.mapper.from || mapper.to != glData.mapper.to || mapper.javaClass != glData.mapper.javaClass) { + // the javaClass check checks for the LogRange vs FreeRange mapper type without storing additional booleans + glData.entryMapperBuffer(mapper) } - } catch (ignored: IndexOutOfBoundsException) { - } - imageOffset += realOffset - if (imageOffset >= size) { - imageOffset %= size - flipImageBuffers() - justSwitched = true - } - } + glData.changeScheduled(fftRate.value, bufferDuration.get(), fftArraySize.value, gradient.value) + glUseProgram(glData.program) + + glActiveTexture(GL_TEXTURE0) + glBindTexture(GL_TEXTURE_2D, glData.sampleBuffer) + glUniform1i(glGetUniformLocation(glData.program, "tex"), 0) + + glActiveTexture(GL_TEXTURE1) + glBindTexture(GL_TEXTURE_1D, glData.entryMapperBuffer) + glUniform1i(glGetUniformLocation(glData.program, "coordMapTex"), 1) + + glActiveTexture(GL_TEXTURE2) + glBindTexture(GL_TEXTURE_1D, glData.gradientBuffer) + glUniform1i(glGetUniformLocation(glData.program, "gradientTex"), 2) - override fun draw(gc: GraphicsContext, deltaT: Double, now: Long, width: Double, height: Double) { - if (width.toInt() != canvasWidth || height.toInt() != canvasHeight) { - canvasWidth = width.toInt() - canvasHeight = height.toInt() - resetBuffer() - createImageBuffers() + glUniform2i(glGetUniformLocation(glData.program, "size"), it.width, it.height) + glUniform1i(glGetUniformLocation(glData.program, "bufferSize"), glData.sampleBufferWidth) + glUniform1i(glGetUniformLocation(glData.program, "headOffset"), glData.sampleBufferPosition) + glUniform1i(glGetUniformLocation(glData.program, "isVertical"), if (orientation.value == Orientation.VERTICAL) 1 else 0) + + glBegin(GL_QUADS) + glVertex2d(-1.0, -1.0) + glVertex2d(-1.0, 1.0) + glVertex2d(1.0, 1.0) + glVertex2d(1.0, -1.0) + glEnd() } - gc.isImageSmoothing = false - gc.clearRect(0.0, 0.0, width, height) - val isHorizontal = orientation.value == Orientation.HORIZONTAL - try { - drawChunk(isHorizontal) - } catch (ignored: ArrayIndexOutOfBoundsException) {} // happens during downsizing - - if (isHorizontal) { - val rOffset = imageOffset.toDouble() - gc.drawImage(imageBottom, -rOffset, 0.0) - gc.drawImage(imageTop, width - rOffset, 0.0) - } else { - val rOffset = height - imageOffset - gc.drawImage(imageTop, 0.0, -rOffset) - gc.drawImage(imageBottom, 0.0, height - rOffset) + canvas.addOnReshapeEvent {} + + canvas.addOnDisposeEvent { + glData.dispose() } } - override fun usedState(state: Boolean) { - if (state) { - resetBuffer() - createImageBuffers() - changeBufferWidth() - } else { - this.buffer = RollingBuffer(2) { FloatArray(0) } - this.imageTop = WritableImage(1, 1) - this.imageBottom = WritableImage(1, 1) + private fun updateBuffer() { + if (::glData.isInitialized) { + glData.scheduleSettingChange(resizeBuffer = true) } } + override fun draw(gc: GraphicsContext, deltaT: Double, now: Long, width: Double, height: Double) {} + override fun registerListeners(acs: AudioCaptureService) { acs.registerFFTObserver(0, this::handleFFT) } diff --git a/src/main/kotlin/me/ksanstone/wavesync/wavesync/gui/utility/GlUtil.kt b/src/main/kotlin/me/ksanstone/wavesync/wavesync/gui/utility/GlUtil.kt index e754cf5..7ed9838 100644 --- a/src/main/kotlin/me/ksanstone/wavesync/wavesync/gui/utility/GlUtil.kt +++ b/src/main/kotlin/me/ksanstone/wavesync/wavesync/gui/utility/GlUtil.kt @@ -1,5 +1,6 @@ package me.ksanstone.wavesync.wavesync.gui.utility +import org.lwjgl.opengl.ARBFramebufferObject.GL_INVALID_FRAMEBUFFER_OPERATION import org.lwjgl.opengl.GL20.* object GlUtil { @@ -49,4 +50,22 @@ object GlUtil { return programId } + fun checkGLError(tag: String = "OpenGL") { + var errorCode: Int + while (glGetError().also { errorCode = it } != GL_NO_ERROR) { + val errorString = when (errorCode) { + GL_INVALID_ENUM -> "GL_INVALID_ENUM" + GL_INVALID_VALUE -> "GL_INVALID_VALUE" + GL_INVALID_OPERATION -> "GL_INVALID_OPERATION" + GL_STACK_OVERFLOW -> "GL_STACK_OVERFLOW" + GL_STACK_UNDERFLOW -> "GL_STACK_UNDERFLOW" + GL_OUT_OF_MEMORY -> "GL_OUT_OF_MEMORY" + GL_INVALID_FRAMEBUFFER_OPERATION -> "GL_INVALID_FRAMEBUFFER_OPERATION" + else -> "UNKNOWN_ERROR" + } + println("$tag: OpenGL Error: $errorString (Code: $errorCode)") + } + } + + } \ No newline at end of file diff --git a/src/main/resources/shaders/spectrogram.frag b/src/main/resources/shaders/spectrogram.frag new file mode 100644 index 0000000..e17cf53 --- /dev/null +++ b/src/main/resources/shaders/spectrogram.frag @@ -0,0 +1,42 @@ +#version 430 core + +out vec4 FragColor; + +uniform sampler2D tex; +uniform isampler1D coordMapTex; +uniform sampler1D gradientTex; + +uniform ivec2 size; +uniform int headOffset; +uniform int bufferSize; + +uniform bool isVertical; + +void main() +{ + // Size in px of the viewport + float relevantSize = float(isVertical ? size.y : size.x); + + // How many samples into each pixel + float perPx = float(bufferSize) / relevantSize; + float fragPosInBuffer = (isVertical ? gl_FragCoord.y : gl_FragCoord.x) / relevantSize * bufferSize; + + // Align the sample we are fetching from to the written counter + // This prevents jitering on the temporal-axis + float headAdjusted = fragPosInBuffer + headOffset; + int buffPos = int(mod(headAdjusted - mod(headAdjusted, perPx) + 2, bufferSize)); + + // Map the pixel position to the relevant sample index in the resulting buffer + int mappedPos = int(isVertical ? gl_FragCoord.x : gl_FragCoord.y); + int mappedStartIndex = texelFetch(coordMapTex, mappedPos, 0).x; + int mappedEndIndex = max(texelFetch(coordMapTex, mappedPos + 1, 0).x - 1, mappedStartIndex); + + // Fetch the max sample from all the samples lying in this pixel + float val = 0; + for (int i = mappedStartIndex; i <= mappedEndIndex; i++) { + val = max(texelFetch(tex, ivec2(i, buffPos), 0).x, val); + } + + // Map value to color from gradient + FragColor = texture(gradientTex, val); +} diff --git a/src/main/resources/shaders/spectrogram.vert b/src/main/resources/shaders/spectrogram.vert new file mode 100644 index 0000000..cd9433b --- /dev/null +++ b/src/main/resources/shaders/spectrogram.vert @@ -0,0 +1,8 @@ +#version 330 core + +layout(location = 0) in vec3 aPos; + +void main() +{ + gl_Position = vec4(aPos, 1.0); +}