Repair inherited Dolby Vision cache metadata

README.md

@@ -54,11 +54,13 @@ the `.env.example` values through Coolify's environment-variable UI.
 
 ## Codec policy
 
-Cached video is H.264 High Profile, Level 4.1, 8-bit `yuv420p`, CRF 18, capped
+Cached video is H.264 High Profile, 8-bit `yuv420p`, CRF 18, capped at 15 Mbps.
-at 15 Mbps. Existing audio, subtitles, chapters, and Matroska attachments are
+Resolution and frame rate are preserved. Existing audio, subtitles, chapters,
-preserved when the container supports them. Ten-bit SDR is converted with
+and Matroska attachments are copied. Ten-bit SDR is converted with
-error-diffusion dithering; HDR10, HLG, and Dolby Vision-labelled sources are
+error-diffusion dithering; sources carrying explicit HDR10 or HLG transfer
-tone-mapped to BT.709 SDR before encoding.
+metadata are tone-mapped to BT.709 SDR. The filtered video is mapped as a new
+stream so source-only Dolby Vision configuration data cannot leak into H.264
+outputs.
 
 The fixed source blacklist is:
 
@@ -99,6 +101,22 @@ partials and orphan files. Atomic symlinks always point to either an original or
 a completed cache file. If the cache mount marker is absent, the worker refuses
 to start.
 
+## Upgrading from v1.2.0
+
+Version 1.2.1 uses a v4 cache profile. Before starting the new worker, stop the
+old worker and migrate its manifests:
+
+```bash
+docker compose stop pretranscode
+docker compose run --rm pretranscode migrate-v4
+```
+
+The migration re-probes immutable sources, updates every valid v3 manifest, and
+creates a state backup. Cached H.264 streams that inherited Dolby Vision
+configuration data are repaired with a lossless stream-copy remux; video is not
+re-encoded. Other cached media files are not rewritten. The command refuses to
+run while Jellyfin reports active playback.
+
 ## Migration
 
 Copy or recreate these resources on the destination:

pretranscode.py

@@ -4,6 +4,7 @@
 import hashlib
 import json
 import os
+import shutil
 import signal
 import subprocess
 import sys
@@ -39,7 +40,8 @@ H264_CRF = os.environ.get("H264_CRF", "18")
 H264_PRESET = os.environ.get("H264_PRESET", "medium")
 H264_MAXRATE = os.environ.get("H264_MAXRATE", "15M")
 H264_BUFSIZE = os.environ.get("H264_BUFSIZE", "30M")
-ENCODE_PROFILE = f"h264-high41-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v2"
+LEGACY_ENCODE_PROFILE = f"h264-high-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v3"
+ENCODE_PROFILE = f"h264-high-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v4"
 FFMPEG = os.environ.get("FFMPEG", "/usr/bin/ffmpeg")
 FFPROBE = os.environ.get("FFPROBE", "/usr/bin/ffprobe")
 
@@ -236,11 +238,9 @@ class Manager:
     def touch_cache(self, relative):
         if not self.cache_valid(relative):
             return
-        output, manifest = self.cache_paths(relative)
+        _, manifest = self.cache_paths(relative)
-        now = time.time()
-        for path in (output, manifest):
         try:
-                os.utime(path, (now, now), follow_symlinks=False)
+            os.utime(manifest, None, follow_symlinks=False)
         except OSError:
             pass
 
@@ -296,8 +296,11 @@ class Manager:
     def probe(self, path):
         command = [
             FFPROBE, "-v", "error", "-select_streams", "v:0",
-            "-show_entries", "stream=codec_name,bits_per_raw_sample,pix_fmt,profile,color_transfer,color_primaries,color_space",
+            "-show_entries",
-            "-show_entries", "format=duration", "-of", "json", str(path),
+            "stream=codec_name,bits_per_raw_sample,pix_fmt,profile,color_transfer,color_primaries,color_space:"
+            "stream_side_data=side_data_type,dv_profile,rpu_present_flag,el_present_flag,bl_present_flag:"
+            "format=duration",
+            "-of", "json", str(path),
         ]
         result = subprocess.run(command, capture_output=True, text=True, timeout=120)
         if result.returncode:
@@ -318,6 +321,7 @@ class Manager:
                 depth = 10
             else:
                 depth = 8
+        side_data = stream.get("side_data_list") or []
         return {
             "codec": (stream.get("codec_name") or "").lower(),
             "bit_depth": depth,
@@ -325,9 +329,26 @@ class Manager:
             "color_transfer": (stream.get("color_transfer") or "").lower(),
             "color_primaries": (stream.get("color_primaries") or "").lower(),
             "color_space": (stream.get("color_space") or "").lower(),
+            "dolby_vision": any(
+                "dovi" in (value.get("side_data_type") or "").lower()
+                for value in side_data
+            ),
             "duration": float((data.get("format") or {}).get("duration") or 0),
         }
 
+    def source_info(self, relative):
+        source = SOURCE_ROOT / relative
+        details = self.probe(source)
+        previous = self.state["inventory"].get(relative) or {}
+        if previous.get("item_id"):
+            details["item_id"] = previous["item_id"]
+        details["fingerprint"] = fingerprint(source)
+        stat = source.stat()
+        details["size"] = stat.st_size
+        details["mtime"] = stat.st_mtime
+        self.state["inventory"][relative] = details
+        return details
+
     def jellyfin_items(self):
         values = {}
         start = 0
@@ -392,12 +413,6 @@ class Manager:
                     if relative in catalog and catalog[relative].get("item_id"):
                         old["item_id"] = catalog[relative]["item_id"]
                     continue
-                details = catalog.get(relative)
-                view = VIEW_ROOT / relative
-                view_uses_cache = view.is_symlink() and os.readlink(view) == cache_target(relative)
-                if details and not old and not view_uses_cache:
-                    details = dict(details)
-                else:
                 try:
                     details = self.probe(source)
                 except Exception as error:
@@ -469,13 +484,8 @@ class Manager:
         profile = (info.get("profile") or "").lower()
         return codec in BLACKLISTED_CODECS or (codec == "h264" and (depth > 8 or "10" in profile))
 
-    def is_hdr(self, info, source):
+    def is_hdr(self, info):
-        transfer = (info.get("color_transfer") or "").lower()
+        return (info.get("color_transfer") or "").lower() in {"smpte2084", "arib-std-b67"}
-        name = source.name.lower()
-        return (
-            transfer in {"smpte2084", "arib-std-b67"}
-            or any(token in name for token in ("hdr10", "hdr.", ".hdr", "dovi", "dolby.vision"))
-        )
 
     def candidate_list(self):
         now = time.time()
@@ -501,7 +511,7 @@ class Manager:
             if not output.is_file():
                 manifest_path.unlink(missing_ok=True)
                 continue
-            entries.append((output.stat().st_mtime, relative, output, manifest_path))
+            entries.append((manifest_path.stat().st_mtime, relative, output, manifest_path))
         return sorted(entries)
 
     def remove_cache(self, relative, force=False):
@@ -581,15 +591,13 @@ class Manager:
         self.process = None
 
     def ffmpeg_command(self, source, partial, info):
-        hdr = self.is_hdr(info, source)
+        hdr = self.is_hdr(info)
         if hdr:
             transfer = (info.get("color_transfer") or "").lower()
             if transfer == "arib-std-b67":
                 input_colors = "tin=arib-std-b67:pin=bt2020:min=bt2020nc:"
-            elif transfer == "smpte2084":
-                input_colors = ""
             else:
-                input_colors = "tin=smpte2084:pin=bt2020:min=bt2020nc:"
+                input_colors = ""
             video_filter = (
                 f"zscale={input_colors}t=linear:npl=100,format=gbrpf32le,"
                 "zscale=p=bt709,tonemap=tonemap=hable:desat=0,"
@@ -600,11 +608,11 @@ class Manager:
 
         common = [
             FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning", "-i", str(source),
-            "-map", "0:v:0", "-map", "0:a?", "-map", "0:s?", "-map_metadata", "0", "-map_chapters", "0",
+            "-filter_complex", f"[0:v:0]{video_filter}[video]",
-            "-vf", video_filter,
+            "-map", "[video]", "-map", "0:a?", "-map", "0:s?", "-map_metadata", "0", "-map_chapters", "0",
             "-c:v", "libx264", "-preset", H264_PRESET, "-crf", H264_CRF,
             "-maxrate", H264_MAXRATE, "-bufsize", H264_BUFSIZE,
-            "-pix_fmt", "yuv420p", "-profile:v", "high", "-level:v", "4.1",
+            "-pix_fmt", "yuv420p", "-profile:v", "high",
             "-c:a", "copy", "-c:s", "copy",
         ]
         if hdr:
@@ -622,6 +630,8 @@ class Manager:
         details = self.probe(output)
         if details.get("codec") != "h264" or int(details.get("bit_depth") or 0) > 8:
             raise RuntimeError("output is not 8-bit H.264")
+        if details.get("dolby_vision"):
+            raise RuntimeError("output incorrectly retains Dolby Vision stream metadata")
         source_duration = float(expected_duration or 0)
         output_duration = float(details.get("duration") or 0)
         if source_duration and abs(source_duration - output_duration) > max(3, source_duration * 0.01):
@@ -629,6 +639,96 @@ class Manager:
         if output.stat().st_size < 1_000_000:
             raise RuntimeError("output is unexpectedly small")
 
+    def video_stream_hash(self, path):
+        command = [
+            FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "error", "-i", str(path),
+            "-map", "0:v:0", "-c:v", "copy", "-bsf:v", "h264_mp4toannexb",
+            "-f", "hash", "-hash", "sha256", "-",
+        ]
+        result = subprocess.run(command, capture_output=True, text=True, timeout=3600)
+        if result.returncode:
+            raise RuntimeError(result.stderr.strip() or "unable to hash video stream")
+        return result.stdout.strip()
+
+    def sanitize_cache_metadata(self, relative, output):
+        if output.suffix.lower() not in {".mkv", ".mp4"}:
+            raise RuntimeError(f"unsupported cache container for metadata repair: {output.suffix}")
+        stage = output.with_name(output.name + ".sanitize-video.mp4.partial")
+        repaired = output.with_name(output.name + ".sanitize.partial")
+        stage.unlink(missing_ok=True)
+        repaired.unlink(missing_ok=True)
+        original_hash = self.video_stream_hash(output)
+        log(f"Losslessly removing inherited Dolby Vision metadata from {relative}")
+        first = [
+            FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning", "-i", str(output),
+            "-map", "0:v:0", "-c:v", "copy", "-an", "-sn", "-dn",
+            "-map_metadata", "-1", "-map_chapters", "-1", "-movflags", "+faststart",
+            "-f", "mp4", str(stage),
+        ]
+        second = [
+            FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning",
+            "-i", str(stage), "-i", str(output),
+            "-map", "0:v:0", "-map", "1:a?", "-map", "1:s?",
+            "-map_metadata", "1", "-map_chapters", "1", "-c", "copy",
+        ]
+        if output.suffix.lower() == ".mkv":
+            second += ["-map", "1:t?", "-c:t", "copy", "-f", "matroska"]
+        else:
+            second += ["-movflags", "+faststart", "-f", "mp4"]
+        second.append(str(repaired))
+        log_path = STATE_ROOT / "ffmpeg.log"
+        try:
+            with log_path.open("ab", buffering=0) as ffmpeg_log:
+                for command in (first, second):
+                    result = subprocess.run(command, stdout=ffmpeg_log, stderr=ffmpeg_log)
+                    if result.returncode:
+                        raise RuntimeError(f"metadata repair ffmpeg exited with status {result.returncode}")
+            details = self.probe(repaired)
+            if details.get("dolby_vision"):
+                raise RuntimeError("metadata repair did not remove Dolby Vision stream metadata")
+            if self.video_stream_hash(repaired) != original_hash:
+                raise RuntimeError("metadata repair changed the encoded video stream")
+            os.replace(repaired, output)
+        finally:
+            stage.unlink(missing_ok=True)
+            repaired.unlink(missing_ok=True)
+
+    def migrate_v4(self):
+        if self.active():
+            raise RuntimeError("manifest migration refused while media is playing")
+        backup = STATE_FILE.with_name(f"state.json.pre-v4-{int(time.time())}")
+        if STATE_FILE.is_file():
+            shutil.copy2(STATE_FILE, backup)
+            log(f"Backed up state to {backup}")
+        migrated = 0
+        sanitized = 0
+        skipped = 0
+        for _, relative, output, manifest_path in self.cache_entries():
+            manifest = load_json(manifest_path, {})
+            if manifest.get("encode_profile") not in {LEGACY_ENCODE_PROFILE, ENCODE_PROFILE}:
+                log(f"Skipping manifest with unsupported profile: {relative}")
+                skipped += 1
+                continue
+            source = SOURCE_ROOT / relative
+            if not source.is_file() or manifest.get("source_fingerprint") != fingerprint(source):
+                log(f"Skipping stale manifest: {relative}")
+                skipped += 1
+                continue
+            source_details = self.source_info(relative)
+            output_details = self.probe(output)
+            if output_details.get("dolby_vision"):
+                self.sanitize_cache_metadata(relative, output)
+                manifest.pop("refresh_version", None)
+                sanitized += 1
+            self.validate_output(source, output, source_details.get("duration"))
+            manifest["encode_profile"] = ENCODE_PROFILE
+            manifest["migration_version"] = 4
+            manifest["migrated_at"] = time.time()
+            atomic_json(manifest_path, manifest)
+            migrated += 1
+        self.save()
+        log(f"Manifest migration complete: migrated={migrated}, sanitized={sanitized}, skipped={skipped}")
+
     def encode(self, relative, priority):
         source = SOURCE_ROOT / relative
         if not source.is_file() or self.is_locked(relative):
@@ -636,7 +736,7 @@ class Manager:
         output, manifest_path = self.cache_paths(relative)
         partial = output.with_name(output.name + ".partial")
         partial.unlink(missing_ok=True)
-        info = self.state["inventory"].get(relative) or {}
+        info = self.source_info(relative)
         estimate = min(max(int(source.stat().st_size * 2.2), 1_000_000_000), 100_000_000_000)
         if not self.ensure_space(estimate, may_evict=priority):
             return "full"
@@ -798,6 +898,8 @@ def main():
         manager.trim(int(sys.argv[2]))
     elif command == "purge":
         manager.purge(force="--force" in sys.argv[2:])
+    elif command == "migrate-v4":
+        manager.migrate_v4()
     else:
         raise SystemExit(f"unknown command: {command}")
 

tests/test_core.py

@@ -4,6 +4,8 @@ import os
 import tempfile
 import unittest
 from pathlib import Path
+from types import SimpleNamespace
+from unittest import mock
 
 
 class CoreTests(unittest.TestCase):
@@ -56,11 +58,44 @@ class CoreTests(unittest.TestCase):
         hdr = manager.ffmpeg_command(
             Path("show.HDR10.mkv"), Path("out.mkv.partial"), {"color_transfer": "smpte2084"}
         )
-        self.assertIn("zscale=dither=error_diffusion,format=yuv420p", sdr)
+        self.assertTrue(any("zscale=dither=error_diffusion,format=yuv420p" in value for value in sdr))
+        self.assertIn("-filter_complex", sdr)
+        self.assertIn("[video]", sdr)
+        self.assertNotIn("-vf", sdr)
         self.assertIn("18", sdr)
         self.assertTrue(any("tonemap=tonemap=hable" in value for value in hdr))
         self.assertIn("bt709", hdr)
 
+    def test_probe_detects_dolby_vision_stream_metadata(self):
+        manager = self.module.Manager()
+        result = SimpleNamespace(
+            returncode=0,
+            stderr="",
+            stdout=json.dumps({
+                "streams": [{
+                    "codec_name": "h264",
+                    "profile": "High",
+                    "pix_fmt": "yuv420p",
+                    "bits_per_raw_sample": "8",
+                    "color_transfer": "bt709",
+                    "side_data_list": [{"side_data_type": "DOVI configuration record"}],
+                }],
+                "format": {"duration": "10.0"},
+            }),
+        )
+        with mock.patch.object(self.module.subprocess, "run", return_value=result):
+            details = manager.probe(Path("cached.mkv"))
+        self.assertTrue(details["dolby_vision"])
+
+    def test_validation_rejects_inherited_dolby_vision_metadata(self):
+        manager = self.module.Manager()
+        output = self.root / "cache" / "invalid.mkv"
+        output.write_bytes(b"x" * 1_000_001)
+        details = {"codec": "h264", "bit_depth": 8, "dolby_vision": True, "duration": 10.0}
+        with mock.patch.object(manager, "probe", return_value=details):
+            with self.assertRaisesRegex(RuntimeError, "Dolby Vision"):
+                manager.validate_output(Path("source.mkv"), output, 10.0)
+
     def test_cache_requires_matching_source_fingerprint(self):
         m = self.module
         manager = m.Manager()
@@ -79,6 +114,26 @@ class CoreTests(unittest.TestCase):
         source.write_bytes(b"changed")
         self.assertFalse(manager.cache_valid(relative))
 
+    def test_lru_touches_only_manifest(self):
+        m = self.module
+        manager = m.Manager()
+        relative = "movies/LRU/LRU.mkv"
+        source = self.root / "source" / relative
+        source.parent.mkdir(parents=True, exist_ok=True)
+        source.write_bytes(b"original")
+        output, manifest = manager.cache_paths(relative)
+        output.write_bytes(b"cached")
+        manifest.write_text(json.dumps({
+            "relative": relative,
+            "source_fingerprint": m.fingerprint(source),
+            "encode_profile": m.ENCODE_PROFILE,
+        }))
+        os.utime(output, (100, 100))
+        os.utime(manifest, (100, 100))
+        manager.touch_cache(relative)
+        self.assertEqual(output.stat().st_mtime, 100)
+        self.assertGreater(manifest.stat().st_mtime, 100)
+
     def test_smaller_cache_limit_evicts_oldest_content(self):
         m = self.module
         manager = m.Manager()