MrDevBot/pretranscode
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Repair inherited Dolby Vision cache metadata

Browse Source
This commit is contained in:
SpiderFish
2026-08-29 01:12:18 +10:00
parent 3a2d79609b
commit 512f70d5c1
3 changed files with 214 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+19 -1
View File
@@ -58,7 +58,9 @@ Cached video is H.264 High Profile, 8-bit `yuv420p`, CRF 18, capped at 15 Mbps.
Resolution and frame rate are preserved. Existing audio, subtitles, chapters,
and Matroska attachments are copied. Ten-bit SDR is converted with
error-diffusion dithering; sources carrying explicit HDR10 or HLG transfer
metadata are tone-mapped to BT.709 SDR.
metadata are tone-mapped to BT.709 SDR. The filtered video is mapped as a new
stream so source-only Dolby Vision configuration data cannot leak into H.264
outputs.
The fixed source blacklist is:
@@ -99,6 +101,22 @@ partials and orphan files. Atomic symlinks always point to either an original or
a completed cache file. If the cache mount marker is absent, the worker refuses
to start.
## Upgrading from v1.2.0
Version 1.2.1 uses a v4 cache profile. Before starting the new worker, stop the
old worker and migrate its manifests:
```bash
docker compose stop pretranscode
docker compose run --rm pretranscode migrate-v4
```
The migration re-probes immutable sources, updates every valid v3 manifest, and
creates a state backup. Cached H.264 streams that inherited Dolby Vision
configuration data are repaired with a lossless stream-copy remux; video is not
re-encoded. Other cached media files are not rewritten. The command refuses to
run while Jellyfin reports active playback.
## Migration
Copy or recreate these resources on the destination:
pretranscode.py
+128 -21
View File
@@ -4,6 +4,7 @@
import hashlib
import json
import os
import shutil
import signal
import subprocess
import sys
@@ -39,7 +40,8 @@ H264_CRF = os.environ.get("H264_CRF", "18")
H264_PRESET = os.environ.get("H264_PRESET", "medium")
H264_MAXRATE = os.environ.get("H264_MAXRATE", "15M")
H264_BUFSIZE = os.environ.get("H264_BUFSIZE", "30M")
ENCODE_PROFILE = f"h264-high-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v3"
LEGACY_ENCODE_PROFILE = f"h264-high-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v3"
ENCODE_PROFILE = f"h264-high-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v4"
FFMPEG = os.environ.get("FFMPEG", "/usr/bin/ffmpeg")
FFPROBE = os.environ.get("FFPROBE", "/usr/bin/ffprobe")
@@ -236,11 +238,9 @@ class Manager:
def touch_cache(self, relative):
if not self.cache_valid(relative):
return
output, manifest = self.cache_paths(relative)
now = time.time()
for path in (output, manifest):
_, manifest = self.cache_paths(relative)
try:
os.utime(path, (now, now), follow_symlinks=False)
os.utime(manifest, None, follow_symlinks=False)
except OSError:
pass
@@ -296,8 +296,11 @@ class Manager:
def probe(self, path):
command = [
FFPROBE, "-v", "error", "-select_streams", "v:0",
"-show_entries", "stream=codec_name,bits_per_raw_sample,pix_fmt,profile,color_transfer,color_primaries,color_space",
"-show_entries", "format=duration", "-of", "json", str(path),
"-show_entries",
"stream=codec_name,bits_per_raw_sample,pix_fmt,profile,color_transfer,color_primaries,color_space:"
"stream_side_data=side_data_type,dv_profile,rpu_present_flag,el_present_flag,bl_present_flag:"
"format=duration",
"-of", "json", str(path),
]
result = subprocess.run(command, capture_output=True, text=True, timeout=120)
if result.returncode:
@@ -318,6 +321,7 @@ class Manager:
depth = 10
else:
depth = 8
side_data = stream.get("side_data_list") or []
return {
"codec": (stream.get("codec_name") or "").lower(),
"bit_depth": depth,
@@ -325,9 +329,26 @@ class Manager:
"color_transfer": (stream.get("color_transfer") or "").lower(),
"color_primaries": (stream.get("color_primaries") or "").lower(),
"color_space": (stream.get("color_space") or "").lower(),
"dolby_vision": any(
"dovi" in (value.get("side_data_type") or "").lower()
for value in side_data
),
"duration": float((data.get("format") or {}).get("duration") or 0),
}
def source_info(self, relative):
source = SOURCE_ROOT / relative
details = self.probe(source)
previous = self.state["inventory"].get(relative) or {}
if previous.get("item_id"):
details["item_id"] = previous["item_id"]
details["fingerprint"] = fingerprint(source)
stat = source.stat()
details["size"] = stat.st_size
details["mtime"] = stat.st_mtime
self.state["inventory"][relative] = details
return details
def jellyfin_items(self):
values = {}
start = 0
@@ -389,17 +410,9 @@ class Manager:
continue
old = inventory.get(relative) or {}
if old.get("fingerprint") == current_fingerprint:
if relative in catalog:
for key in ("item_id", "color_transfer", "color_primaries", "color_space"):
if catalog[relative].get(key):
old[key] = catalog[relative][key]
if relative in catalog and catalog[relative].get("item_id"):
old["item_id"] = catalog[relative]["item_id"]
continue
details = catalog.get(relative)
view = VIEW_ROOT / relative
view_uses_cache = view.is_symlink() and os.readlink(view) == cache_target(relative)
if details and not old and not view_uses_cache:
details = dict(details)
else:
try:
details = self.probe(source)
except Exception as error:
@@ -498,7 +511,7 @@ class Manager:
if not output.is_file():
manifest_path.unlink(missing_ok=True)
continue
entries.append((output.stat().st_mtime, relative, output, manifest_path))
entries.append((manifest_path.stat().st_mtime, relative, output, manifest_path))
return sorted(entries)
def remove_cache(self, relative, force=False):
@@ -595,8 +608,8 @@ class Manager:
common = [
FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning", "-i", str(source),
"-map", "0:v:0", "-map", "0:a?", "-map", "0:s?", "-map_metadata", "0", "-map_chapters", "0",
"-vf", video_filter,
"-filter_complex", f"[0:v:0]{video_filter}[video]",
"-map", "[video]", "-map", "0:a?", "-map", "0:s?", "-map_metadata", "0", "-map_chapters", "0",
"-c:v", "libx264", "-preset", H264_PRESET, "-crf", H264_CRF,
"-maxrate", H264_MAXRATE, "-bufsize", H264_BUFSIZE,
"-pix_fmt", "yuv420p", "-profile:v", "high",
@@ -617,6 +630,8 @@ class Manager:
details = self.probe(output)
if details.get("codec") != "h264" or int(details.get("bit_depth") or 0) > 8:
raise RuntimeError("output is not 8-bit H.264")
if details.get("dolby_vision"):
raise RuntimeError("output incorrectly retains Dolby Vision stream metadata")
source_duration = float(expected_duration or 0)
output_duration = float(details.get("duration") or 0)
if source_duration and abs(source_duration - output_duration) > max(3, source_duration * 0.01):
@@ -624,6 +639,96 @@ class Manager:
if output.stat().st_size < 1_000_000:
raise RuntimeError("output is unexpectedly small")
def video_stream_hash(self, path):
command = [
FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "error", "-i", str(path),
"-map", "0:v:0", "-c:v", "copy", "-bsf:v", "h264_mp4toannexb",
"-f", "hash", "-hash", "sha256", "-",
]
result = subprocess.run(command, capture_output=True, text=True, timeout=3600)
if result.returncode:
raise RuntimeError(result.stderr.strip() or "unable to hash video stream")
return result.stdout.strip()
def sanitize_cache_metadata(self, relative, output):
if output.suffix.lower() not in {".mkv", ".mp4"}:
raise RuntimeError(f"unsupported cache container for metadata repair: {output.suffix}")
stage = output.with_name(output.name + ".sanitize-video.mp4.partial")
repaired = output.with_name(output.name + ".sanitize.partial")
stage.unlink(missing_ok=True)
repaired.unlink(missing_ok=True)
original_hash = self.video_stream_hash(output)
log(f"Losslessly removing inherited Dolby Vision metadata from {relative}")
first = [
FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning", "-i", str(output),
"-map", "0:v:0", "-c:v", "copy", "-an", "-sn", "-dn",
"-map_metadata", "-1", "-map_chapters", "-1", "-movflags", "+faststart",
"-f", "mp4", str(stage),
]
second = [
FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning",
"-i", str(stage), "-i", str(output),
"-map", "0:v:0", "-map", "1:a?", "-map", "1:s?",
"-map_metadata", "1", "-map_chapters", "1", "-c", "copy",
]
if output.suffix.lower() == ".mkv":
second += ["-map", "1:t?", "-c:t", "copy", "-f", "matroska"]
else:
second += ["-movflags", "+faststart", "-f", "mp4"]
second.append(str(repaired))
log_path = STATE_ROOT / "ffmpeg.log"
try:
with log_path.open("ab", buffering=0) as ffmpeg_log:
for command in (first, second):
result = subprocess.run(command, stdout=ffmpeg_log, stderr=ffmpeg_log)
if result.returncode:
raise RuntimeError(f"metadata repair ffmpeg exited with status {result.returncode}")
details = self.probe(repaired)
if details.get("dolby_vision"):
raise RuntimeError("metadata repair did not remove Dolby Vision stream metadata")
if self.video_stream_hash(repaired) != original_hash:
raise RuntimeError("metadata repair changed the encoded video stream")
os.replace(repaired, output)
finally:
stage.unlink(missing_ok=True)
repaired.unlink(missing_ok=True)
def migrate_v4(self):
if self.active():
raise RuntimeError("manifest migration refused while media is playing")
backup = STATE_FILE.with_name(f"state.json.pre-v4-{int(time.time())}")
if STATE_FILE.is_file():
shutil.copy2(STATE_FILE, backup)
log(f"Backed up state to {backup}")
migrated = 0
sanitized = 0
skipped = 0
for _, relative, output, manifest_path in self.cache_entries():
manifest = load_json(manifest_path, {})
if manifest.get("encode_profile") not in {LEGACY_ENCODE_PROFILE, ENCODE_PROFILE}:
log(f"Skipping manifest with unsupported profile: {relative}")
skipped += 1
continue
source = SOURCE_ROOT / relative
if not source.is_file() or manifest.get("source_fingerprint") != fingerprint(source):
log(f"Skipping stale manifest: {relative}")
skipped += 1
continue
source_details = self.source_info(relative)
output_details = self.probe(output)
if output_details.get("dolby_vision"):
self.sanitize_cache_metadata(relative, output)
manifest.pop("refresh_version", None)
sanitized += 1
self.validate_output(source, output, source_details.get("duration"))
manifest["encode_profile"] = ENCODE_PROFILE
manifest["migration_version"] = 4
manifest["migrated_at"] = time.time()
atomic_json(manifest_path, manifest)
migrated += 1
self.save()
log(f"Manifest migration complete: migrated={migrated}, sanitized={sanitized}, skipped={skipped}")
def encode(self, relative, priority):
source = SOURCE_ROOT / relative
if not source.is_file() or self.is_locked(relative):
@@ -631,7 +736,7 @@ class Manager:
output, manifest_path = self.cache_paths(relative)
partial = output.with_name(output.name + ".partial")
partial.unlink(missing_ok=True)
info = self.state["inventory"].get(relative) or {}
info = self.source_info(relative)
estimate = min(max(int(source.stat().st_size * 2.2), 1_000_000_000), 100_000_000_000)
if not self.ensure_space(estimate, may_evict=priority):
return "full"
@@ -793,6 +898,8 @@ def main():
manager.trim(int(sys.argv[2]))
elif command == "purge":
manager.purge(force="--force" in sys.argv[2:])
elif command == "migrate-v4":
manager.migrate_v4()
else:
raise SystemExit(f"unknown command: {command}")
tests/test_core.py
+56 -1
View File
@@ -4,6 +4,8 @@ import os
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest import mock
class CoreTests(unittest.TestCase):
@@ -56,11 +58,44 @@ class CoreTests(unittest.TestCase):
hdr = manager.ffmpeg_command(
Path("show.HDR10.mkv"), Path("out.mkv.partial"), {"color_transfer": "smpte2084"}
)
self.assertIn("zscale=dither=error_diffusion,format=yuv420p", sdr)
self.assertTrue(any("zscale=dither=error_diffusion,format=yuv420p" in value for value in sdr))
self.assertIn("-filter_complex", sdr)
self.assertIn("[video]", sdr)
self.assertNotIn("-vf", sdr)
self.assertIn("18", sdr)
self.assertTrue(any("tonemap=tonemap=hable" in value for value in hdr))
self.assertIn("bt709", hdr)
def test_probe_detects_dolby_vision_stream_metadata(self):
manager = self.module.Manager()
result = SimpleNamespace(
returncode=0,
stderr="",
stdout=json.dumps({
"streams": [{
"codec_name": "h264",
"profile": "High",
"pix_fmt": "yuv420p",
"bits_per_raw_sample": "8",
"color_transfer": "bt709",
"side_data_list": [{"side_data_type": "DOVI configuration record"}],
}],
"format": {"duration": "10.0"},
}),
)
with mock.patch.object(self.module.subprocess, "run", return_value=result):
details = manager.probe(Path("cached.mkv"))
self.assertTrue(details["dolby_vision"])
def test_validation_rejects_inherited_dolby_vision_metadata(self):
manager = self.module.Manager()
output = self.root / "cache" / "invalid.mkv"
output.write_bytes(b"x" * 1_000_001)
details = {"codec": "h264", "bit_depth": 8, "dolby_vision": True, "duration": 10.0}
with mock.patch.object(manager, "probe", return_value=details):
with self.assertRaisesRegex(RuntimeError, "Dolby Vision"):
manager.validate_output(Path("source.mkv"), output, 10.0)
def test_cache_requires_matching_source_fingerprint(self):
m = self.module
manager = m.Manager()
@@ -79,6 +114,26 @@ class CoreTests(unittest.TestCase):
source.write_bytes(b"changed")
self.assertFalse(manager.cache_valid(relative))
def test_lru_touches_only_manifest(self):
m = self.module
manager = m.Manager()
relative = "movies/LRU/LRU.mkv"
source = self.root / "source" / relative
source.parent.mkdir(parents=True, exist_ok=True)
source.write_bytes(b"original")
output, manifest = manager.cache_paths(relative)
output.write_bytes(b"cached")
manifest.write_text(json.dumps({
"relative": relative,
"source_fingerprint": m.fingerprint(source),
"encode_profile": m.ENCODE_PROFILE,
}))
os.utime(output, (100, 100))
os.utime(manifest, (100, 100))
manager.touch_cache(relative)
self.assertEqual(output.stat().st_mtime, 100)
self.assertGreater(manifest.stat().st_mtime, 100)
def test_smaller_cache_limit_evicts_oldest_content(self):
m = self.module
manager = m.Manager()