From 429244d820b967fc4ff07c6fd4c2d81350300f4d Mon Sep 17 00:00:00 2001 From: tkrmagid Date: Fri, 15 May 2026 19:45:42 +0900 Subject: [PATCH] audio: route JavaCV samples through SourceDataLine with live gain setVolume/Mute previously stored gain without affecting audible output: the backend only called grabImage() and never opened an audio sink. Switch to grab() (interleaved video+audio frames), force AV_SAMPLE_FMT_S16 on the grabber so samples are always interleaved signed 16-bit PCM, open a matching JavaSound SourceDataLine and write scaled samples per-frame. gain is read on every block so /videoMute, GUI Mute and the per-tick distance attenuation now take effect immediately. SourceDataLine.write blocking provides natural A/V pacing, so the legacy 15ms sleep is dropped when an audio line is open; sleep is retained as a 60fps cap when there is no audio device. bump version to 0.3.1. --- gradle.properties | 2 +- .../client/playback/JavaCvBackend.java | 114 ++++++++++++++++-- 2 files changed, 103 insertions(+), 13 deletions(-) diff --git a/gradle.properties b/gradle.properties index 8807df3..15dab72 100644 --- a/gradle.properties +++ b/gradle.properties @@ -5,7 +5,7 @@ org.gradle.configuration-cache=false # Mod mod_id=video_player -mod_version=0.3.0 +mod_version=0.3.1 maven_group=com.ejclaw.videoplayer archives_base_name=video_player diff --git a/src/main/java/com/ejclaw/videoplayer/client/playback/JavaCvBackend.java b/src/main/java/com/ejclaw/videoplayer/client/playback/JavaCvBackend.java index cb3d7e5..e1ddee7 100644 --- a/src/main/java/com/ejclaw/videoplayer/client/playback/JavaCvBackend.java +++ b/src/main/java/com/ejclaw/videoplayer/client/playback/JavaCvBackend.java @@ -4,24 +4,34 @@ import com.ejclaw.videoplayer.VideoPlayerMod; import net.fabricmc.api.EnvType; import net.fabricmc.api.Environment; +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.SourceDataLine; +import java.lang.reflect.Field; import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.ShortBuffer; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; /** * SPEC ยง5.3 โ€” fallback mp4/http(s) backend driven by JavaCV's FFmpegFrameGrabber. * - * JavaCV is referenced entirely through reflection so that the mod jar stays loadable when the - * (large) JavaCV dependency isn't bundled โ€” the backend just reports {@code !isReady()} until the - * runtime classpath contains org.bytedeco.javacv.FFmpegFrameGrabber. + *

Video frames are decoded through {@code grab()}, audio samples are forced to interleaved + * signed 16-bit PCM ({@code AV_SAMPLE_FMT_S16}) and written to a {@link SourceDataLine} so that + * {@link #setVolume(float)} actually mutes / attenuates audible output. JavaCV is referenced + * entirely through reflection so that the mod jar stays loadable when the (large) JavaCV + * dependency isn't bundled โ€” the backend just reports {@code !isReady()} until the runtime + * classpath contains {@code org.bytedeco.javacv.FFmpegFrameGrabber}. */ @Environment(EnvType.CLIENT) public class JavaCvBackend implements VideoBackend { private static final String GRABBER_CLASS = "org.bytedeco.javacv.FFmpegFrameGrabber"; private static final String FRAME_CLASS = "org.bytedeco.javacv.Frame"; private static final String CONVERTER_CLASS = "org.bytedeco.javacv.Java2DFrameConverter"; + /** {@code AV_SAMPLE_FMT_S16} from {@code org.bytedeco.ffmpeg.global.avutil}. */ + private static final int AV_SAMPLE_FMT_S16 = 1; private final Object lock = new Object(); private Thread worker; @@ -84,28 +94,41 @@ public class JavaCvBackend implements VideoBackend { /** Pure-reflection decode loop. Silent fallback if JavaCV isn't present. */ private void runLoop(String url) { Object grabber = null; + SourceDataLine audioLine = null; try { Class grabberCls = Class.forName(GRABBER_CLASS); grabber = grabberCls.getConstructor(String.class).newInstance(url); Method start = grabberCls.getMethod("start"); Method stop = grabberCls.getMethod("stop"); - Method grab = grabberCls.getMethod("grabImage"); + Method grab = grabberCls.getMethod("grab"); Method getW = grabberCls.getMethod("getImageWidth"); Method getH = grabberCls.getMethod("getImageHeight"); + Method getSampleRate = grabberCls.getMethod("getSampleRate"); + Method getAudioChannels = grabberCls.getMethod("getAudioChannels"); Method setOpt = grabberCls.getMethod("setOption", String.class, String.class); + Method setSampleFormat = grabberCls.getMethod("setSampleFormat", int.class); // mp4/http(s) network tuning try { setOpt.invoke(grabber, "rw_timeout", "5000000"); } catch (Throwable ignored) {} try { setOpt.invoke(grabber, "stimeout", "5000000"); } catch (Throwable ignored) {} + // Force interleaved signed 16-bit PCM so the audio sink path is single-shape. + try { setSampleFormat.invoke(grabber, AV_SAMPLE_FMT_S16); } catch (Throwable ignored) {} start.invoke(grabber); this.width = (int) getW.invoke(grabber); this.height = (int) getH.invoke(grabber); this.ready = (width > 0 && height > 0); - Class convCls = Class.forName(CONVERTER_CLASS); - Object converter = convCls.getDeclaredConstructor().newInstance(); - Method toImage = convCls.getMethod("getBufferedImage", Class.forName(FRAME_CLASS)); + int sampleRate = safeInt(getSampleRate, grabber); + int audioChannels = safeInt(getAudioChannels, grabber); + audioLine = openLine(sampleRate, audioChannels); + + Class frameCls = Class.forName(FRAME_CLASS); + Field imageField = frameCls.getField("image"); + Field samplesField = frameCls.getField("samples"); + Class convCls = Class.forName(CONVERTER_CLASS); + Object converter = convCls.getDeclaredConstructor().newInstance(); + Method toImage = convCls.getMethod("getBufferedImage", frameCls); while (running.get() && !closed) { if (paused.get()) { Thread.sleep(20); continue; } @@ -118,11 +141,25 @@ public class JavaCvBackend implements VideoBackend { } break; } - java.awt.image.BufferedImage img = (java.awt.image.BufferedImage) toImage.invoke(converter, frame); - if (img == null) continue; - ByteBuffer buf = toRgba(img); - if (buf != null) latest.set(buf); - Thread.sleep(15); // ~60fps cap + + Object[] samples = (Object[]) samplesField.get(frame); + if (samples != null && samples.length > 0 && audioLine != null) { + writeAudio(audioLine, samples, this.gain); + } + + Object[] images = (Object[]) imageField.get(frame); + if (images != null && images.length > 0) { + java.awt.image.BufferedImage img = + (java.awt.image.BufferedImage) toImage.invoke(converter, frame); + if (img != null) { + ByteBuffer buf = toRgba(img); + if (buf != null) latest.set(buf); + } + } + + // If we have an open audio line, SourceDataLine.write() blocks for backpressure + // and provides natural A/V pacing; otherwise tick ~60fps so we don't busy-loop. + if (audioLine == null) Thread.sleep(15); } } catch (ClassNotFoundException cnf) { VideoPlayerMod.LOG.info("[{}] JavaCV not on classpath; backend inactive", VideoPlayerMod.MOD_ID); @@ -132,12 +169,65 @@ public class JavaCvBackend implements VideoBackend { VideoPlayerMod.LOG.warn("[{}] JavaCV decode error: {}", VideoPlayerMod.MOD_ID, t.toString()); } finally { ready = false; + if (audioLine != null) { + try { audioLine.drain(); } catch (Throwable ignored) {} + try { audioLine.stop(); } catch (Throwable ignored) {} + try { audioLine.close(); } catch (Throwable ignored) {} + } if (grabber != null) { try { grabber.getClass().getMethod("close").invoke(grabber); } catch (Throwable ignored) {} } } } + /** Open a JavaSound output line for the stream's sample rate / channel count, or null. */ + private static SourceDataLine openLine(int sampleRate, int channels) { + if (sampleRate <= 0 || channels <= 0) return null; + try { + AudioFormat fmt = new AudioFormat(sampleRate, 16, channels, true, false); // signed 16-bit LE + SourceDataLine line = AudioSystem.getSourceDataLine(fmt); + line.open(fmt); + line.start(); + return line; + } catch (Throwable t) { + VideoPlayerMod.LOG.info("[{}] no audio sink ({} Hz x{}): {}", + VideoPlayerMod.MOD_ID, sampleRate, channels, t.toString()); + return null; + } + } + + /** Scale & write interleaved S16 PCM samples to the audio line. */ + private static void writeAudio(SourceDataLine line, Object[] samples, float gain) { + if (!(samples[0] instanceof ShortBuffer sb)) return; // sample format forcing failed + int remaining = sb.remaining(); + if (remaining <= 0) return; + byte[] pcm = new byte[remaining * 2]; + int idx = 0; + if (gain >= 0.999F) { + while (sb.hasRemaining()) { + short s = sb.get(); + pcm[idx++] = (byte) (s & 0xFF); + pcm[idx++] = (byte) ((s >> 8) & 0xFF); + } + } else if (gain <= 0F) { + // Mute: consume but emit silence so the line keeps cadence. + sb.position(sb.limit()); + } else { + while (sb.hasRemaining()) { + int scaled = (int) (sb.get() * gain); + if (scaled > 32767) scaled = 32767; + if (scaled < -32768) scaled = -32768; + pcm[idx++] = (byte) (scaled & 0xFF); + pcm[idx++] = (byte) ((scaled >> 8) & 0xFF); + } + } + line.write(pcm, 0, pcm.length); + } + + private static int safeInt(Method m, Object target) { + try { return (int) m.invoke(target); } catch (Throwable t) { return 0; } + } + private static ByteBuffer toRgba(java.awt.image.BufferedImage img) { int w = img.getWidth(), h = img.getHeight(); int[] argb = img.getRGB(0, 0, w, h, null, 0, w);