Repair inherited Dolby Vision cache metadata

README.md

@@ -58,7 +58,9 @@ Cached video is H.264 High Profile, 8-bit `yuv420p`, CRF 18, capped at 15 Mbps.
 Resolution and frame rate are preserved. Existing audio, subtitles, chapters,
 and Matroska attachments are copied. Ten-bit SDR is converted with
 error-diffusion dithering; sources carrying explicit HDR10 or HLG transfer
-metadata are tone-mapped to BT.709 SDR.
+metadata are tone-mapped to BT.709 SDR. The filtered video is mapped as a new
+stream so source-only Dolby Vision configuration data cannot leak into H.264
+outputs.
 
 The fixed source blacklist is:
 
@@ -99,6 +101,22 @@ partials and orphan files. Atomic symlinks always point to either an original or
 a completed cache file. If the cache mount marker is absent, the worker refuses
 to start.
 
+## Upgrading from v1.2.0
+
+Version 1.2.1 uses a v4 cache profile. Before starting the new worker, stop the
+old worker and migrate its manifests:
+
+```bash
+docker compose stop pretranscode
+docker compose run --rm pretranscode migrate-v4
+```
+
+The migration re-probes immutable sources, updates every valid v3 manifest, and
+creates a state backup. Cached H.264 streams that inherited Dolby Vision
+configuration data are repaired with a lossless stream-copy remux; video is not
+re-encoded. Other cached media files are not rewritten. The command refuses to
+run while Jellyfin reports active playback.
+
 ## Migration
 
 Copy or recreate these resources on the destination:

pretranscode.py

@@ -4,6 +4,7 @@
 import hashlib
 import json
 import os
+import shutil
 import signal
 import subprocess
 import sys
@@ -39,7 +40,8 @@ H264_CRF = os.environ.get("H264_CRF", "18")
 H264_PRESET = os.environ.get("H264_PRESET", "medium")
 H264_MAXRATE = os.environ.get("H264_MAXRATE", "15M")
 H264_BUFSIZE = os.environ.get("H264_BUFSIZE", "30M")
-ENCODE_PROFILE = f"h264-high-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v3"
+LEGACY_ENCODE_PROFILE = f"h264-high-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v3"
+ENCODE_PROFILE = f"h264-high-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v4"
 FFMPEG = os.environ.get("FFMPEG", "/usr/bin/ffmpeg")
 FFPROBE = os.environ.get("FFPROBE", "/usr/bin/ffprobe")
 
@@ -236,11 +238,9 @@ class Manager:
     def touch_cache(self, relative):
         if not self.cache_valid(relative):
             return
-        output, manifest = self.cache_paths(relative)
+        _, manifest = self.cache_paths(relative)
-        now = time.time()
-        for path in (output, manifest):
         try:
-                os.utime(path, (now, now), follow_symlinks=False)
+            os.utime(manifest, None, follow_symlinks=False)
         except OSError:
             pass
 
@@ -296,8 +296,11 @@ class Manager:
     def probe(self, path):
         command = [
             FFPROBE, "-v", "error", "-select_streams", "v:0",
-            "-show_entries", "stream=codec_name,bits_per_raw_sample,pix_fmt,profile,color_transfer,color_primaries,color_space",
+            "-show_entries",
-            "-show_entries", "format=duration", "-of", "json", str(path),
+            "stream=codec_name,bits_per_raw_sample,pix_fmt,profile,color_transfer,color_primaries,color_space:"
+            "stream_side_data=side_data_type,dv_profile,rpu_present_flag,el_present_flag,bl_present_flag:"
+            "format=duration",
+            "-of", "json", str(path),
         ]
         result = subprocess.run(command, capture_output=True, text=True, timeout=120)
         if result.returncode:
@@ -318,6 +321,7 @@ class Manager:
                 depth = 10
             else:
                 depth = 8
+        side_data = stream.get("side_data_list") or []
         return {
             "codec": (stream.get("codec_name") or "").lower(),
             "bit_depth": depth,
@@ -325,9 +329,26 @@ class Manager:
             "color_transfer": (stream.get("color_transfer") or "").lower(),
             "color_primaries": (stream.get("color_primaries") or "").lower(),
             "color_space": (stream.get("color_space") or "").lower(),
+            "dolby_vision": any(
+                "dovi" in (value.get("side_data_type") or "").lower()
+                for value in side_data
+            ),
             "duration": float((data.get("format") or {}).get("duration") or 0),
         }
 
+    def source_info(self, relative):
+        source = SOURCE_ROOT / relative
+        details = self.probe(source)
+        previous = self.state["inventory"].get(relative) or {}
+        if previous.get("item_id"):
+            details["item_id"] = previous["item_id"]
+        details["fingerprint"] = fingerprint(source)
+        stat = source.stat()
+        details["size"] = stat.st_size
+        details["mtime"] = stat.st_mtime
+        self.state["inventory"][relative] = details
+        return details
+
     def jellyfin_items(self):
         values = {}
         start = 0
@@ -389,17 +410,9 @@ class Manager:
                     continue
                 old = inventory.get(relative) or {}
                 if old.get("fingerprint") == current_fingerprint:
-                    if relative in catalog:
+                    if relative in catalog and catalog[relative].get("item_id"):
-                        for key in ("item_id", "color_transfer", "color_primaries", "color_space"):
+                        old["item_id"] = catalog[relative]["item_id"]
-                            if catalog[relative].get(key):
-                                old[key] = catalog[relative][key]
                     continue
-                details = catalog.get(relative)
-                view = VIEW_ROOT / relative
-                view_uses_cache = view.is_symlink() and os.readlink(view) == cache_target(relative)
-                if details and not old and not view_uses_cache:
-                    details = dict(details)
-                else:
                 try:
                     details = self.probe(source)
                 except Exception as error:
@@ -498,7 +511,7 @@ class Manager:
             if not output.is_file():
                 manifest_path.unlink(missing_ok=True)
                 continue
-            entries.append((output.stat().st_mtime, relative, output, manifest_path))
+            entries.append((manifest_path.stat().st_mtime, relative, output, manifest_path))
         return sorted(entries)
 
     def remove_cache(self, relative, force=False):
@@ -595,8 +608,8 @@ class Manager:
 
         common = [
             FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning", "-i", str(source),
-            "-map", "0:v:0", "-map", "0:a?", "-map", "0:s?", "-map_metadata", "0", "-map_chapters", "0",
+            "-filter_complex", f"[0:v:0]{video_filter}[video]",
-            "-vf", video_filter,
+            "-map", "[video]", "-map", "0:a?", "-map", "0:s?", "-map_metadata", "0", "-map_chapters", "0",
             "-c:v", "libx264", "-preset", H264_PRESET, "-crf", H264_CRF,
             "-maxrate", H264_MAXRATE, "-bufsize", H264_BUFSIZE,
             "-pix_fmt", "yuv420p", "-profile:v", "high",
@@ -617,6 +630,8 @@ class Manager:
         details = self.probe(output)
         if details.get("codec") != "h264" or int(details.get("bit_depth") or 0) > 8:
             raise RuntimeError("output is not 8-bit H.264")
+        if details.get("dolby_vision"):
+            raise RuntimeError("output incorrectly retains Dolby Vision stream metadata")
         source_duration = float(expected_duration or 0)
         output_duration = float(details.get("duration") or 0)
         if source_duration and abs(source_duration - output_duration) > max(3, source_duration * 0.01):
@@ -624,6 +639,96 @@ class Manager:
         if output.stat().st_size < 1_000_000:
             raise RuntimeError("output is unexpectedly small")
 
+    def video_stream_hash(self, path):
+        command = [
+            FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "error", "-i", str(path),
+            "-map", "0:v:0", "-c:v", "copy", "-bsf:v", "h264_mp4toannexb",
+            "-f", "hash", "-hash", "sha256", "-",
+        ]
+        result = subprocess.run(command, capture_output=True, text=True, timeout=3600)
+        if result.returncode:
+            raise RuntimeError(result.stderr.strip() or "unable to hash video stream")
+        return result.stdout.strip()
+
+    def sanitize_cache_metadata(self, relative, output):
+        if output.suffix.lower() not in {".mkv", ".mp4"}:
+            raise RuntimeError(f"unsupported cache container for metadata repair: {output.suffix}")
+        stage = output.with_name(output.name + ".sanitize-video.mp4.partial")
+        repaired = output.with_name(output.name + ".sanitize.partial")
+        stage.unlink(missing_ok=True)
+        repaired.unlink(missing_ok=True)
+        original_hash = self.video_stream_hash(output)
+        log(f"Losslessly removing inherited Dolby Vision metadata from {relative}")
+        first = [
+            FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning", "-i", str(output),
+            "-map", "0:v:0", "-c:v", "copy", "-an", "-sn", "-dn",
+            "-map_metadata", "-1", "-map_chapters", "-1", "-movflags", "+faststart",
+            "-f", "mp4", str(stage),
+        ]
+        second = [
+            FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning",
+            "-i", str(stage), "-i", str(output),
+            "-map", "0:v:0", "-map", "1:a?", "-map", "1:s?",
+            "-map_metadata", "1", "-map_chapters", "1", "-c", "copy",
+        ]
+        if output.suffix.lower() == ".mkv":
+            second += ["-map", "1:t?", "-c:t", "copy", "-f", "matroska"]
+        else:
+            second += ["-movflags", "+faststart", "-f", "mp4"]
+        second.append(str(repaired))
+        log_path = STATE_ROOT / "ffmpeg.log"
+        try:
+            with log_path.open("ab", buffering=0) as ffmpeg_log:
+                for command in (first, second):
+                    result = subprocess.run(command, stdout=ffmpeg_log, stderr=ffmpeg_log)
+                    if result.returncode:
+                        raise RuntimeError(f"metadata repair ffmpeg exited with status {result.returncode}")
+            details = self.probe(repaired)
+            if details.get("dolby_vision"):
+                raise RuntimeError("metadata repair did not remove Dolby Vision stream metadata")
+            if self.video_stream_hash(repaired) != original_hash:
+                raise RuntimeError("metadata repair changed the encoded video stream")
+            os.replace(repaired, output)
+        finally:
+            stage.unlink(missing_ok=True)
+            repaired.unlink(missing_ok=True)
+
+    def migrate_v4(self):
+        if self.active():
+            raise RuntimeError("manifest migration refused while media is playing")
+        backup = STATE_FILE.with_name(f"state.json.pre-v4-{int(time.time())}")
+        if STATE_FILE.is_file():
+            shutil.copy2(STATE_FILE, backup)
+            log(f"Backed up state to {backup}")
+        migrated = 0
+        sanitized = 0
+        skipped = 0
+        for _, relative, output, manifest_path in self.cache_entries():
+            manifest = load_json(manifest_path, {})
+            if manifest.get("encode_profile") not in {LEGACY_ENCODE_PROFILE, ENCODE_PROFILE}:
+                log(f"Skipping manifest with unsupported profile: {relative}")
+                skipped += 1
+                continue
+            source = SOURCE_ROOT / relative
+            if not source.is_file() or manifest.get("source_fingerprint") != fingerprint(source):
+                log(f"Skipping stale manifest: {relative}")
+                skipped += 1
+                continue
+            source_details = self.source_info(relative)
+            output_details = self.probe(output)
+            if output_details.get("dolby_vision"):
+                self.sanitize_cache_metadata(relative, output)
+                manifest.pop("refresh_version", None)
+                sanitized += 1
+            self.validate_output(source, output, source_details.get("duration"))
+            manifest["encode_profile"] = ENCODE_PROFILE
+            manifest["migration_version"] = 4
+            manifest["migrated_at"] = time.time()
+            atomic_json(manifest_path, manifest)
+            migrated += 1
+        self.save()
+        log(f"Manifest migration complete: migrated={migrated}, sanitized={sanitized}, skipped={skipped}")
+
     def encode(self, relative, priority):
         source = SOURCE_ROOT / relative
         if not source.is_file() or self.is_locked(relative):
@@ -631,7 +736,7 @@ class Manager:
         output, manifest_path = self.cache_paths(relative)
         partial = output.with_name(output.name + ".partial")
         partial.unlink(missing_ok=True)
-        info = self.state["inventory"].get(relative) or {}
+        info = self.source_info(relative)
         estimate = min(max(int(source.stat().st_size * 2.2), 1_000_000_000), 100_000_000_000)
         if not self.ensure_space(estimate, may_evict=priority):
             return "full"
@@ -793,6 +898,8 @@ def main():
         manager.trim(int(sys.argv[2]))
     elif command == "purge":
         manager.purge(force="--force" in sys.argv[2:])
+    elif command == "migrate-v4":
+        manager.migrate_v4()
     else:
         raise SystemExit(f"unknown command: {command}")
 

tests/test_core.py

@@ -4,6 +4,8 @@ import os
 import tempfile
 import unittest
 from pathlib import Path
+from types import SimpleNamespace
+from unittest import mock
 
 
 class CoreTests(unittest.TestCase):
@@ -56,11 +58,44 @@ class CoreTests(unittest.TestCase):
         hdr = manager.ffmpeg_command(
             Path("show.HDR10.mkv"), Path("out.mkv.partial"), {"color_transfer": "smpte2084"}
         )
-        self.assertIn("zscale=dither=error_diffusion,format=yuv420p", sdr)
+        self.assertTrue(any("zscale=dither=error_diffusion,format=yuv420p" in value for value in sdr))
+        self.assertIn("-filter_complex", sdr)
+        self.assertIn("[video]", sdr)
+        self.assertNotIn("-vf", sdr)
         self.assertIn("18", sdr)
         self.assertTrue(any("tonemap=tonemap=hable" in value for value in hdr))
         self.assertIn("bt709", hdr)
 
+    def test_probe_detects_dolby_vision_stream_metadata(self):
+        manager = self.module.Manager()
+        result = SimpleNamespace(
+            returncode=0,
+            stderr="",
+            stdout=json.dumps({
+                "streams": [{
+                    "codec_name": "h264",
+                    "profile": "High",
+                    "pix_fmt": "yuv420p",
+                    "bits_per_raw_sample": "8",
+                    "color_transfer": "bt709",
+                    "side_data_list": [{"side_data_type": "DOVI configuration record"}],
+                }],
+                "format": {"duration": "10.0"},
+            }),
+        )
+        with mock.patch.object(self.module.subprocess, "run", return_value=result):
+            details = manager.probe(Path("cached.mkv"))
+        self.assertTrue(details["dolby_vision"])
+
+    def test_validation_rejects_inherited_dolby_vision_metadata(self):
+        manager = self.module.Manager()
+        output = self.root / "cache" / "invalid.mkv"
+        output.write_bytes(b"x" * 1_000_001)
+        details = {"codec": "h264", "bit_depth": 8, "dolby_vision": True, "duration": 10.0}
+        with mock.patch.object(manager, "probe", return_value=details):
+            with self.assertRaisesRegex(RuntimeError, "Dolby Vision"):
+                manager.validate_output(Path("source.mkv"), output, 10.0)
+
     def test_cache_requires_matching_source_fingerprint(self):
         m = self.module
         manager = m.Manager()
@@ -79,6 +114,26 @@ class CoreTests(unittest.TestCase):
         source.write_bytes(b"changed")
         self.assertFalse(manager.cache_valid(relative))
 
+    def test_lru_touches_only_manifest(self):
+        m = self.module
+        manager = m.Manager()
+        relative = "movies/LRU/LRU.mkv"
+        source = self.root / "source" / relative
+        source.parent.mkdir(parents=True, exist_ok=True)
+        source.write_bytes(b"original")
+        output, manifest = manager.cache_paths(relative)
+        output.write_bytes(b"cached")
+        manifest.write_text(json.dumps({
+            "relative": relative,
+            "source_fingerprint": m.fingerprint(source),
+            "encode_profile": m.ENCODE_PROFILE,
+        }))
+        os.utime(output, (100, 100))
+        os.utime(manifest, (100, 100))
+        manager.touch_cache(relative)
+        self.assertEqual(output.stat().st_mtime, 100)
+        self.assertGreater(manifest.stat().st_mtime, 100)
+
     def test_smaller_cache_limit_evicts_oldest_content(self):
         m = self.module
         manager = m.Manager()