audio: route JavaCV samples through SourceDataLine with live gain

setVolume/Mute previously stored gain without affecting audible output: the
backend only called grabImage() and never opened an audio sink. Switch to
grab() (interleaved video+audio frames), force AV_SAMPLE_FMT_S16 on the
grabber so samples are always interleaved signed 16-bit PCM, open a matching
JavaSound SourceDataLine and write scaled samples per-frame. gain is read
on every block so /videoMute, GUI Mute and the per-tick distance attenuation
now take effect immediately. SourceDataLine.write blocking provides natural
A/V pacing, so the legacy 15ms sleep is dropped when an audio line is open;
sleep is retained as a 60fps cap when there is no audio device.

bump version to 0.3.1.

gradle.properties

@@ -5,7 +5,7 @@ org.gradle.configuration-cache=false
 
 # Mod
 mod_id=video_player
-mod_version=0.3.0
+mod_version=0.3.1
 maven_group=com.ejclaw.videoplayer
 archives_base_name=video_player
 

src/main/java/com/ejclaw/videoplayer/client/playback/JavaCvBackend.java

@@ -4,24 +4,34 @@ import com.ejclaw.videoplayer.VideoPlayerMod;
 import net.fabricmc.api.EnvType;
 import net.fabricmc.api.Environment;
 
+import javax.sound.sampled.AudioFormat;
+import javax.sound.sampled.AudioSystem;
+import javax.sound.sampled.SourceDataLine;
+import java.lang.reflect.Field;
 import java.lang.reflect.Method;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.ShortBuffer;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
 /**
  * SPEC §5.3 — fallback mp4/http(s) backend driven by JavaCV's FFmpegFrameGrabber.
  *
- * JavaCV is referenced entirely through reflection so that the mod jar stays loadable when the
+ * <p>Video frames are decoded through {@code grab()}, audio samples are forced to interleaved
- * (large) JavaCV dependency isn't bundled — the backend just reports {@code !isReady()} until the
+ * signed 16-bit PCM ({@code AV_SAMPLE_FMT_S16}) and written to a {@link SourceDataLine} so that
- * runtime classpath contains org.bytedeco.javacv.FFmpegFrameGrabber.
+ * {@link #setVolume(float)} actually mutes / attenuates audible output. JavaCV is referenced
+ * entirely through reflection so that the mod jar stays loadable when the (large) JavaCV
+ * dependency isn't bundled — the backend just reports {@code !isReady()} until the runtime
+ * classpath contains {@code org.bytedeco.javacv.FFmpegFrameGrabber}.
  */
 @Environment(EnvType.CLIENT)
 public class JavaCvBackend implements VideoBackend {
     private static final String GRABBER_CLASS = "org.bytedeco.javacv.FFmpegFrameGrabber";
     private static final String FRAME_CLASS   = "org.bytedeco.javacv.Frame";
     private static final String CONVERTER_CLASS = "org.bytedeco.javacv.Java2DFrameConverter";
+    /** {@code AV_SAMPLE_FMT_S16} from {@code org.bytedeco.ffmpeg.global.avutil}. */
+    private static final int AV_SAMPLE_FMT_S16 = 1;
 
     private final Object lock = new Object();
     private Thread worker;
@@ -84,28 +94,41 @@ public class JavaCvBackend implements VideoBackend {
     /** Pure-reflection decode loop. Silent fallback if JavaCV isn't present. */
     private void runLoop(String url) {
         Object grabber = null;
+        SourceDataLine audioLine = null;
         try {
             Class<?> grabberCls = Class.forName(GRABBER_CLASS);
             grabber = grabberCls.getConstructor(String.class).newInstance(url);
             Method start  = grabberCls.getMethod("start");
             Method stop   = grabberCls.getMethod("stop");
-            Method grab   = grabberCls.getMethod("grabImage");
+            Method grab   = grabberCls.getMethod("grab");
             Method getW   = grabberCls.getMethod("getImageWidth");
             Method getH   = grabberCls.getMethod("getImageHeight");
+            Method getSampleRate    = grabberCls.getMethod("getSampleRate");
+            Method getAudioChannels = grabberCls.getMethod("getAudioChannels");
             Method setOpt = grabberCls.getMethod("setOption", String.class, String.class);
+            Method setSampleFormat  = grabberCls.getMethod("setSampleFormat", int.class);
 
             // mp4/http(s) network tuning
             try { setOpt.invoke(grabber, "rw_timeout", "5000000"); } catch (Throwable ignored) {}
             try { setOpt.invoke(grabber, "stimeout",   "5000000"); } catch (Throwable ignored) {}
+            // Force interleaved signed 16-bit PCM so the audio sink path is single-shape.
+            try { setSampleFormat.invoke(grabber, AV_SAMPLE_FMT_S16); } catch (Throwable ignored) {}
 
             start.invoke(grabber);
             this.width  = (int) getW.invoke(grabber);
             this.height = (int) getH.invoke(grabber);
             this.ready = (width > 0 && height > 0);
 
+            int sampleRate    = safeInt(getSampleRate, grabber);
+            int audioChannels = safeInt(getAudioChannels, grabber);
+            audioLine = openLine(sampleRate, audioChannels);
+
+            Class<?> frameCls    = Class.forName(FRAME_CLASS);
+            Field imageField     = frameCls.getField("image");
+            Field samplesField   = frameCls.getField("samples");
             Class<?> convCls     = Class.forName(CONVERTER_CLASS);
             Object converter     = convCls.getDeclaredConstructor().newInstance();
-            Method toImage   = convCls.getMethod("getBufferedImage", Class.forName(FRAME_CLASS));
+            Method toImage       = convCls.getMethod("getBufferedImage", frameCls);
 
             while (running.get() && !closed) {
                 if (paused.get()) { Thread.sleep(20); continue; }
@@ -118,11 +141,25 @@ public class JavaCvBackend implements VideoBackend {
                     }
                     break;
                 }
-                java.awt.image.BufferedImage img = (java.awt.image.BufferedImage) toImage.invoke(converter, frame);
+
-                if (img == null) continue;
+                Object[] samples = (Object[]) samplesField.get(frame);
+                if (samples != null && samples.length > 0 && audioLine != null) {
+                    writeAudio(audioLine, samples, this.gain);
+                }
+
+                Object[] images = (Object[]) imageField.get(frame);
+                if (images != null && images.length > 0) {
+                    java.awt.image.BufferedImage img =
+                        (java.awt.image.BufferedImage) toImage.invoke(converter, frame);
+                    if (img != null) {
                         ByteBuffer buf = toRgba(img);
                         if (buf != null) latest.set(buf);
-                Thread.sleep(15); // ~60fps cap
+                    }
+                }
+
+                // If we have an open audio line, SourceDataLine.write() blocks for backpressure
+                // and provides natural A/V pacing; otherwise tick ~60fps so we don't busy-loop.
+                if (audioLine == null) Thread.sleep(15);
             }
         } catch (ClassNotFoundException cnf) {
             VideoPlayerMod.LOG.info("[{}] JavaCV not on classpath; backend inactive", VideoPlayerMod.MOD_ID);
@@ -132,12 +169,65 @@ public class JavaCvBackend implements VideoBackend {
             VideoPlayerMod.LOG.warn("[{}] JavaCV decode error: {}", VideoPlayerMod.MOD_ID, t.toString());
         } finally {
             ready = false;
+            if (audioLine != null) {
+                try { audioLine.drain(); } catch (Throwable ignored) {}
+                try { audioLine.stop();  } catch (Throwable ignored) {}
+                try { audioLine.close(); } catch (Throwable ignored) {}
+            }
             if (grabber != null) {
                 try { grabber.getClass().getMethod("close").invoke(grabber); } catch (Throwable ignored) {}
             }
         }
     }
 
+    /** Open a JavaSound output line for the stream's sample rate / channel count, or null. */
+    private static SourceDataLine openLine(int sampleRate, int channels) {
+        if (sampleRate <= 0 || channels <= 0) return null;
+        try {
+            AudioFormat fmt = new AudioFormat(sampleRate, 16, channels, true, false); // signed 16-bit LE
+            SourceDataLine line = AudioSystem.getSourceDataLine(fmt);
+            line.open(fmt);
+            line.start();
+            return line;
+        } catch (Throwable t) {
+            VideoPlayerMod.LOG.info("[{}] no audio sink ({} Hz x{}): {}",
+                VideoPlayerMod.MOD_ID, sampleRate, channels, t.toString());
+            return null;
+        }
+    }
+
+    /** Scale & write interleaved S16 PCM samples to the audio line. */
+    private static void writeAudio(SourceDataLine line, Object[] samples, float gain) {
+        if (!(samples[0] instanceof ShortBuffer sb)) return; // sample format forcing failed
+        int remaining = sb.remaining();
+        if (remaining <= 0) return;
+        byte[] pcm = new byte[remaining * 2];
+        int idx = 0;
+        if (gain >= 0.999F) {
+            while (sb.hasRemaining()) {
+                short s = sb.get();
+                pcm[idx++] = (byte) (s & 0xFF);
+                pcm[idx++] = (byte) ((s >> 8) & 0xFF);
+            }
+        } else if (gain <= 0F) {
+            // Mute: consume but emit silence so the line keeps cadence.
+            sb.position(sb.limit());
+        } else {
+            while (sb.hasRemaining()) {
+                int scaled = (int) (sb.get() * gain);
+                if (scaled >  32767) scaled =  32767;
+                if (scaled < -32768) scaled = -32768;
+                pcm[idx++] = (byte) (scaled & 0xFF);
+                pcm[idx++] = (byte) ((scaled >> 8) & 0xFF);
+            }
+        }
+        line.write(pcm, 0, pcm.length);
+    }
+
+    private static int safeInt(Method m, Object target) {
+        try { return (int) m.invoke(target); } catch (Throwable t) { return 0; }
+    }
+
     private static ByteBuffer toRgba(java.awt.image.BufferedImage img) {
         int w = img.getWidth(), h = img.getHeight();
         int[] argb = img.getRGB(0, 0, w, h, null, 0, w);