diff --git a/README.md b/README.md index 2ca38f7..e4f8c20 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,9 @@ Cached video is H.264 High Profile, 8-bit `yuv420p`, CRF 18, capped at 15 Mbps. Resolution and frame rate are preserved. Existing audio, subtitles, chapters, and Matroska attachments are copied. Ten-bit SDR is converted with error-diffusion dithering; sources carrying explicit HDR10 or HLG transfer -metadata are tone-mapped to BT.709 SDR. +metadata are tone-mapped to BT.709 SDR. The filtered video is mapped as a new +stream so source-only Dolby Vision configuration data cannot leak into H.264 +outputs. The fixed source blacklist is: @@ -99,6 +101,22 @@ partials and orphan files. Atomic symlinks always point to either an original or a completed cache file. If the cache mount marker is absent, the worker refuses to start. +## Upgrading from v1.2.0 + +Version 1.2.1 uses a v4 cache profile. Before starting the new worker, stop the +old worker and migrate its manifests: + +```bash +docker compose stop pretranscode +docker compose run --rm pretranscode migrate-v4 +``` + +The migration re-probes immutable sources, updates every valid v3 manifest, and +creates a state backup. Cached H.264 streams that inherited Dolby Vision +configuration data are repaired with a lossless stream-copy remux; video is not +re-encoded. Other cached media files are not rewritten. The command refuses to +run while Jellyfin reports active playback. + ## Migration Copy or recreate these resources on the destination: diff --git a/pretranscode.py b/pretranscode.py index bbb2dbb..1896693 100755 --- a/pretranscode.py +++ b/pretranscode.py @@ -4,6 +4,7 @@ import hashlib import json import os +import shutil import signal import subprocess import sys @@ -39,7 +40,8 @@ H264_CRF = os.environ.get("H264_CRF", "18") H264_PRESET = os.environ.get("H264_PRESET", "medium") H264_MAXRATE = os.environ.get("H264_MAXRATE", "15M") H264_BUFSIZE = os.environ.get("H264_BUFSIZE", "30M") -ENCODE_PROFILE = f"h264-high-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v3" +LEGACY_ENCODE_PROFILE = f"h264-high-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v3" +ENCODE_PROFILE = f"h264-high-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v4" FFMPEG = os.environ.get("FFMPEG", "/usr/bin/ffmpeg") FFPROBE = os.environ.get("FFPROBE", "/usr/bin/ffprobe") @@ -236,13 +238,11 @@ class Manager: def touch_cache(self, relative): if not self.cache_valid(relative): return - output, manifest = self.cache_paths(relative) - now = time.time() - for path in (output, manifest): - try: - os.utime(path, (now, now), follow_symlinks=False) - except OSError: - pass + _, manifest = self.cache_paths(relative) + try: + os.utime(manifest, None, follow_symlinks=False) + except OSError: + pass def ensure_link(self, relative, target, locked=False): link = VIEW_ROOT / relative @@ -296,8 +296,11 @@ class Manager: def probe(self, path): command = [ FFPROBE, "-v", "error", "-select_streams", "v:0", - "-show_entries", "stream=codec_name,bits_per_raw_sample,pix_fmt,profile,color_transfer,color_primaries,color_space", - "-show_entries", "format=duration", "-of", "json", str(path), + "-show_entries", + "stream=codec_name,bits_per_raw_sample,pix_fmt,profile,color_transfer,color_primaries,color_space:" + "stream_side_data=side_data_type,dv_profile,rpu_present_flag,el_present_flag,bl_present_flag:" + "format=duration", + "-of", "json", str(path), ] result = subprocess.run(command, capture_output=True, text=True, timeout=120) if result.returncode: @@ -318,6 +321,7 @@ class Manager: depth = 10 else: depth = 8 + side_data = stream.get("side_data_list") or [] return { "codec": (stream.get("codec_name") or "").lower(), "bit_depth": depth, @@ -325,9 +329,26 @@ class Manager: "color_transfer": (stream.get("color_transfer") or "").lower(), "color_primaries": (stream.get("color_primaries") or "").lower(), "color_space": (stream.get("color_space") or "").lower(), + "dolby_vision": any( + "dovi" in (value.get("side_data_type") or "").lower() + for value in side_data + ), "duration": float((data.get("format") or {}).get("duration") or 0), } + def source_info(self, relative): + source = SOURCE_ROOT / relative + details = self.probe(source) + previous = self.state["inventory"].get(relative) or {} + if previous.get("item_id"): + details["item_id"] = previous["item_id"] + details["fingerprint"] = fingerprint(source) + stat = source.stat() + details["size"] = stat.st_size + details["mtime"] = stat.st_mtime + self.state["inventory"][relative] = details + return details + def jellyfin_items(self): values = {} start = 0 @@ -389,25 +410,17 @@ class Manager: continue old = inventory.get(relative) or {} if old.get("fingerprint") == current_fingerprint: - if relative in catalog: - for key in ("item_id", "color_transfer", "color_primaries", "color_space"): - if catalog[relative].get(key): - old[key] = catalog[relative][key] + if relative in catalog and catalog[relative].get("item_id"): + old["item_id"] = catalog[relative]["item_id"] continue - details = catalog.get(relative) - view = VIEW_ROOT / relative - view_uses_cache = view.is_symlink() and os.readlink(view) == cache_target(relative) - if details and not old and not view_uses_cache: - details = dict(details) - else: - try: - details = self.probe(source) - except Exception as error: - log(f"Probe failed for {relative}: {error}") - self.state["retry_after"][relative] = time.time() + 21600 - continue - if relative in catalog: - details["item_id"] = catalog[relative].get("item_id") + try: + details = self.probe(source) + except Exception as error: + log(f"Probe failed for {relative}: {error}") + self.state["retry_after"][relative] = time.time() + 21600 + continue + if relative in catalog: + details["item_id"] = catalog[relative].get("item_id") details["fingerprint"] = current_fingerprint details["size"] = source.stat().st_size details["mtime"] = source.stat().st_mtime @@ -498,7 +511,7 @@ class Manager: if not output.is_file(): manifest_path.unlink(missing_ok=True) continue - entries.append((output.stat().st_mtime, relative, output, manifest_path)) + entries.append((manifest_path.stat().st_mtime, relative, output, manifest_path)) return sorted(entries) def remove_cache(self, relative, force=False): @@ -595,8 +608,8 @@ class Manager: common = [ FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning", "-i", str(source), - "-map", "0:v:0", "-map", "0:a?", "-map", "0:s?", "-map_metadata", "0", "-map_chapters", "0", - "-vf", video_filter, + "-filter_complex", f"[0:v:0]{video_filter}[video]", + "-map", "[video]", "-map", "0:a?", "-map", "0:s?", "-map_metadata", "0", "-map_chapters", "0", "-c:v", "libx264", "-preset", H264_PRESET, "-crf", H264_CRF, "-maxrate", H264_MAXRATE, "-bufsize", H264_BUFSIZE, "-pix_fmt", "yuv420p", "-profile:v", "high", @@ -617,6 +630,8 @@ class Manager: details = self.probe(output) if details.get("codec") != "h264" or int(details.get("bit_depth") or 0) > 8: raise RuntimeError("output is not 8-bit H.264") + if details.get("dolby_vision"): + raise RuntimeError("output incorrectly retains Dolby Vision stream metadata") source_duration = float(expected_duration or 0) output_duration = float(details.get("duration") or 0) if source_duration and abs(source_duration - output_duration) > max(3, source_duration * 0.01): @@ -624,6 +639,96 @@ class Manager: if output.stat().st_size < 1_000_000: raise RuntimeError("output is unexpectedly small") + def video_stream_hash(self, path): + command = [ + FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "error", "-i", str(path), + "-map", "0:v:0", "-c:v", "copy", "-bsf:v", "h264_mp4toannexb", + "-f", "hash", "-hash", "sha256", "-", + ] + result = subprocess.run(command, capture_output=True, text=True, timeout=3600) + if result.returncode: + raise RuntimeError(result.stderr.strip() or "unable to hash video stream") + return result.stdout.strip() + + def sanitize_cache_metadata(self, relative, output): + if output.suffix.lower() not in {".mkv", ".mp4"}: + raise RuntimeError(f"unsupported cache container for metadata repair: {output.suffix}") + stage = output.with_name(output.name + ".sanitize-video.mp4.partial") + repaired = output.with_name(output.name + ".sanitize.partial") + stage.unlink(missing_ok=True) + repaired.unlink(missing_ok=True) + original_hash = self.video_stream_hash(output) + log(f"Losslessly removing inherited Dolby Vision metadata from {relative}") + first = [ + FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning", "-i", str(output), + "-map", "0:v:0", "-c:v", "copy", "-an", "-sn", "-dn", + "-map_metadata", "-1", "-map_chapters", "-1", "-movflags", "+faststart", + "-f", "mp4", str(stage), + ] + second = [ + FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning", + "-i", str(stage), "-i", str(output), + "-map", "0:v:0", "-map", "1:a?", "-map", "1:s?", + "-map_metadata", "1", "-map_chapters", "1", "-c", "copy", + ] + if output.suffix.lower() == ".mkv": + second += ["-map", "1:t?", "-c:t", "copy", "-f", "matroska"] + else: + second += ["-movflags", "+faststart", "-f", "mp4"] + second.append(str(repaired)) + log_path = STATE_ROOT / "ffmpeg.log" + try: + with log_path.open("ab", buffering=0) as ffmpeg_log: + for command in (first, second): + result = subprocess.run(command, stdout=ffmpeg_log, stderr=ffmpeg_log) + if result.returncode: + raise RuntimeError(f"metadata repair ffmpeg exited with status {result.returncode}") + details = self.probe(repaired) + if details.get("dolby_vision"): + raise RuntimeError("metadata repair did not remove Dolby Vision stream metadata") + if self.video_stream_hash(repaired) != original_hash: + raise RuntimeError("metadata repair changed the encoded video stream") + os.replace(repaired, output) + finally: + stage.unlink(missing_ok=True) + repaired.unlink(missing_ok=True) + + def migrate_v4(self): + if self.active(): + raise RuntimeError("manifest migration refused while media is playing") + backup = STATE_FILE.with_name(f"state.json.pre-v4-{int(time.time())}") + if STATE_FILE.is_file(): + shutil.copy2(STATE_FILE, backup) + log(f"Backed up state to {backup}") + migrated = 0 + sanitized = 0 + skipped = 0 + for _, relative, output, manifest_path in self.cache_entries(): + manifest = load_json(manifest_path, {}) + if manifest.get("encode_profile") not in {LEGACY_ENCODE_PROFILE, ENCODE_PROFILE}: + log(f"Skipping manifest with unsupported profile: {relative}") + skipped += 1 + continue + source = SOURCE_ROOT / relative + if not source.is_file() or manifest.get("source_fingerprint") != fingerprint(source): + log(f"Skipping stale manifest: {relative}") + skipped += 1 + continue + source_details = self.source_info(relative) + output_details = self.probe(output) + if output_details.get("dolby_vision"): + self.sanitize_cache_metadata(relative, output) + manifest.pop("refresh_version", None) + sanitized += 1 + self.validate_output(source, output, source_details.get("duration")) + manifest["encode_profile"] = ENCODE_PROFILE + manifest["migration_version"] = 4 + manifest["migrated_at"] = time.time() + atomic_json(manifest_path, manifest) + migrated += 1 + self.save() + log(f"Manifest migration complete: migrated={migrated}, sanitized={sanitized}, skipped={skipped}") + def encode(self, relative, priority): source = SOURCE_ROOT / relative if not source.is_file() or self.is_locked(relative): @@ -631,7 +736,7 @@ class Manager: output, manifest_path = self.cache_paths(relative) partial = output.with_name(output.name + ".partial") partial.unlink(missing_ok=True) - info = self.state["inventory"].get(relative) or {} + info = self.source_info(relative) estimate = min(max(int(source.stat().st_size * 2.2), 1_000_000_000), 100_000_000_000) if not self.ensure_space(estimate, may_evict=priority): return "full" @@ -793,6 +898,8 @@ def main(): manager.trim(int(sys.argv[2])) elif command == "purge": manager.purge(force="--force" in sys.argv[2:]) + elif command == "migrate-v4": + manager.migrate_v4() else: raise SystemExit(f"unknown command: {command}") diff --git a/tests/test_core.py b/tests/test_core.py index 3a77fa9..3e9463e 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -4,6 +4,8 @@ import os import tempfile import unittest from pathlib import Path +from types import SimpleNamespace +from unittest import mock class CoreTests(unittest.TestCase): @@ -56,11 +58,44 @@ class CoreTests(unittest.TestCase): hdr = manager.ffmpeg_command( Path("show.HDR10.mkv"), Path("out.mkv.partial"), {"color_transfer": "smpte2084"} ) - self.assertIn("zscale=dither=error_diffusion,format=yuv420p", sdr) + self.assertTrue(any("zscale=dither=error_diffusion,format=yuv420p" in value for value in sdr)) + self.assertIn("-filter_complex", sdr) + self.assertIn("[video]", sdr) + self.assertNotIn("-vf", sdr) self.assertIn("18", sdr) self.assertTrue(any("tonemap=tonemap=hable" in value for value in hdr)) self.assertIn("bt709", hdr) + def test_probe_detects_dolby_vision_stream_metadata(self): + manager = self.module.Manager() + result = SimpleNamespace( + returncode=0, + stderr="", + stdout=json.dumps({ + "streams": [{ + "codec_name": "h264", + "profile": "High", + "pix_fmt": "yuv420p", + "bits_per_raw_sample": "8", + "color_transfer": "bt709", + "side_data_list": [{"side_data_type": "DOVI configuration record"}], + }], + "format": {"duration": "10.0"}, + }), + ) + with mock.patch.object(self.module.subprocess, "run", return_value=result): + details = manager.probe(Path("cached.mkv")) + self.assertTrue(details["dolby_vision"]) + + def test_validation_rejects_inherited_dolby_vision_metadata(self): + manager = self.module.Manager() + output = self.root / "cache" / "invalid.mkv" + output.write_bytes(b"x" * 1_000_001) + details = {"codec": "h264", "bit_depth": 8, "dolby_vision": True, "duration": 10.0} + with mock.patch.object(manager, "probe", return_value=details): + with self.assertRaisesRegex(RuntimeError, "Dolby Vision"): + manager.validate_output(Path("source.mkv"), output, 10.0) + def test_cache_requires_matching_source_fingerprint(self): m = self.module manager = m.Manager() @@ -79,6 +114,26 @@ class CoreTests(unittest.TestCase): source.write_bytes(b"changed") self.assertFalse(manager.cache_valid(relative)) + def test_lru_touches_only_manifest(self): + m = self.module + manager = m.Manager() + relative = "movies/LRU/LRU.mkv" + source = self.root / "source" / relative + source.parent.mkdir(parents=True, exist_ok=True) + source.write_bytes(b"original") + output, manifest = manager.cache_paths(relative) + output.write_bytes(b"cached") + manifest.write_text(json.dumps({ + "relative": relative, + "source_fingerprint": m.fingerprint(source), + "encode_profile": m.ENCODE_PROFILE, + })) + os.utime(output, (100, 100)) + os.utime(manifest, (100, 100)) + manager.touch_cache(relative) + self.assertEqual(output.stat().st_mtime, 100) + self.assertGreater(manifest.stat().st_mtime, 100) + def test_smaller_cache_limit_evicts_oldest_content(self): m = self.module manager = m.Manager()