diff --git a/.env.example b/.env.example index ec5971c..cca3523 100644 --- a/.env.example +++ b/.env.example @@ -27,7 +27,7 @@ JELLYFIN_CACHE_ROOT=/data/cache SERIES_COOLDOWN_SECONDS=3600 POLL_SECONDS=5 RESCAN_SECONDS=300 -H264_CRF=20 +H264_CRF=18 H264_PRESET=medium -H264_MAXRATE=12M -H264_BUFSIZE=24M +H264_MAXRATE=15M +H264_BUFSIZE=30M diff --git a/README.md b/README.md index 0410a76..241f79b 100644 --- a/README.md +++ b/README.md @@ -54,9 +54,11 @@ the `.env.example` values through Coolify's environment-variable UI. ## Codec policy -Cached video is H.264 High Profile, Level 4.1, 8-bit `yuv420p`, CRF 20, capped -at 12 Mbps. Existing audio, subtitles, chapters, and Matroska attachments are -preserved when the container supports them. +Cached video is H.264 High Profile, Level 4.1, 8-bit `yuv420p`, CRF 18, capped +at 15 Mbps. Existing audio, subtitles, chapters, and Matroska attachments are +preserved when the container supports them. Ten-bit SDR is converted with +error-diffusion dithering; HDR10, HLG, and Dolby Vision-labelled sources are +tone-mapped to BT.709 SDR before encoding. The fixed source blacklist is: @@ -65,7 +67,9 @@ The fixed source blacklist is: - MPEG-2 - VC-1 -Client error reports never modify this list. +Client error reports never modify this list. Encoding settings form part of +each cache manifest, so a global quality-policy change automatically invalidates +and replaces older renditions without touching originals. ## Cache resizing diff --git a/compose.yaml b/compose.yaml index 7a92baa..d0d35c7 100644 --- a/compose.yaml +++ b/compose.yaml @@ -20,10 +20,10 @@ services: SERIES_COOLDOWN_SECONDS: ${SERIES_COOLDOWN_SECONDS:-3600} POLL_SECONDS: ${POLL_SECONDS:-5} RESCAN_SECONDS: ${RESCAN_SECONDS:-300} - H264_CRF: ${H264_CRF:-20} + H264_CRF: ${H264_CRF:-18} H264_PRESET: ${H264_PRESET:-medium} - H264_MAXRATE: ${H264_MAXRATE:-12M} - H264_BUFSIZE: ${H264_BUFSIZE:-24M} + H264_MAXRATE: ${H264_MAXRATE:-15M} + H264_BUFSIZE: ${H264_BUFSIZE:-30M} volumes: - ${MEDIA_LIBRARY_PATH:?}:/source:ro - ${PRETRANSCODE_VIEW_PATH:?}:/view diff --git a/pretranscode.py b/pretranscode.py index 7e4fb7d..22df88b 100755 --- a/pretranscode.py +++ b/pretranscode.py @@ -35,10 +35,11 @@ CACHE_MAX_BYTES = env_int("CACHE_MAX_BYTES", 1_000_000_000_000) SERIES_COOLDOWN_SECONDS = env_int("SERIES_COOLDOWN_SECONDS", 3600) POLL_SECONDS = env_int("POLL_SECONDS", 5) RESCAN_SECONDS = env_int("RESCAN_SECONDS", 300) -H264_CRF = os.environ.get("H264_CRF", "20") +H264_CRF = os.environ.get("H264_CRF", "18") H264_PRESET = os.environ.get("H264_PRESET", "medium") -H264_MAXRATE = os.environ.get("H264_MAXRATE", "12M") -H264_BUFSIZE = os.environ.get("H264_BUFSIZE", "24M") +H264_MAXRATE = os.environ.get("H264_MAXRATE", "15M") +H264_BUFSIZE = os.environ.get("H264_BUFSIZE", "30M") +ENCODE_PROFILE = f"h264-high41-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v2" FFMPEG = os.environ.get("FFMPEG", "/usr/bin/ffmpeg") FFPROBE = os.environ.get("FFPROBE", "/usr/bin/ffprobe") @@ -225,7 +226,10 @@ class Manager: return False manifest = load_json(manifest_path, {}) try: - return manifest.get("source_fingerprint") == fingerprint(source) + return ( + manifest.get("source_fingerprint") == fingerprint(source) + and manifest.get("encode_profile") == ENCODE_PROFILE + ) except OSError: return False @@ -292,7 +296,7 @@ class Manager: def probe(self, path): command = [ FFPROBE, "-v", "error", "-select_streams", "v:0", - "-show_entries", "stream=codec_name,bits_per_raw_sample,pix_fmt,profile", + "-show_entries", "stream=codec_name,bits_per_raw_sample,pix_fmt,profile,color_transfer,color_primaries,color_space", "-show_entries", "format=duration", "-of", "json", str(path), ] result = subprocess.run(command, capture_output=True, text=True, timeout=120) @@ -318,6 +322,9 @@ class Manager: "codec": (stream.get("codec_name") or "").lower(), "bit_depth": depth, "profile": profile, + "color_transfer": (stream.get("color_transfer") or "").lower(), + "color_primaries": (stream.get("color_primaries") or "").lower(), + "color_space": (stream.get("color_space") or "").lower(), "duration": float((data.get("format") or {}).get("duration") or 0), } @@ -350,6 +357,9 @@ class Manager: "codec": (video.get("Codec") or "").lower(), "bit_depth": int(video.get("BitDepth") or 8), "profile": (video.get("Profile") or "").lower(), + "color_transfer": (video.get("ColorTransfer") or "").lower(), + "color_primaries": (video.get("ColorPrimaries") or "").lower(), + "color_space": (video.get("ColorSpace") or "").lower(), "duration": float(item.get("RunTimeTicks") or 0) / 10_000_000, } start += len(items) @@ -459,6 +469,14 @@ class Manager: profile = (info.get("profile") or "").lower() return codec in BLACKLISTED_CODECS or (codec == "h264" and (depth > 8 or "10" in profile)) + def is_hdr(self, info, source): + transfer = (info.get("color_transfer") or "").lower() + name = source.name.lower() + return ( + transfer in {"smpte2084", "arib-std-b67"} + or any(token in name for token in ("hdr10", "hdr.", ".hdr", "dovi", "dolby.vision")) + ) + def candidate_list(self): now = time.time() values = [] @@ -486,8 +504,8 @@ class Manager: entries.append((output.stat().st_mtime, relative, output, manifest_path)) return sorted(entries) - def remove_cache(self, relative): - if self.is_locked(relative): + def remove_cache(self, relative, force=False): + if not force and self.is_locked(relative): return False output, manifest = self.cache_paths(relative) link = VIEW_ROOT / relative @@ -515,6 +533,13 @@ class Manager: configured_free = max(0, CACHE_MAX_BYTES - self.cache_usage()) return min(filesystem_free, configured_free) + def purge_incompatible_cache_profiles(self): + for _, relative, _, _ in self.cache_entries(): + if self.cache_valid(relative) or self.is_locked(relative): + continue + log(f"Dropping obsolete cache rendition for {relative}") + self.remove_cache(relative) + def enforce_cache_limit(self): usage = self.cache_usage() for _, relative, output, manifest in self.cache_entries(): @@ -555,15 +580,38 @@ class Manager: pass self.process = None - def ffmpeg_command(self, source, partial): + def ffmpeg_command(self, source, partial, info): + hdr = self.is_hdr(info, source) + if hdr: + transfer = (info.get("color_transfer") or "").lower() + if transfer == "arib-std-b67": + input_colors = "tin=arib-std-b67:pin=bt2020:min=bt2020nc:" + elif transfer == "smpte2084": + input_colors = "" + else: + input_colors = "tin=smpte2084:pin=bt2020:min=bt2020nc:" + video_filter = ( + f"zscale={input_colors}t=linear:npl=100,format=gbrpf32le," + "zscale=p=bt709,tonemap=tonemap=hable:desat=0," + "zscale=t=bt709:m=bt709:r=tv:dither=error_diffusion,format=yuv420p" + ) + else: + video_filter = "zscale=dither=error_diffusion,format=yuv420p" + common = [ FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning", "-i", str(source), "-map", "0:v:0", "-map", "0:a?", "-map", "0:s?", "-map_metadata", "0", "-map_chapters", "0", + "-vf", video_filter, "-c:v", "libx264", "-preset", H264_PRESET, "-crf", H264_CRF, "-maxrate", H264_MAXRATE, "-bufsize", H264_BUFSIZE, "-pix_fmt", "yuv420p", "-profile:v", "high", "-level:v", "4.1", "-c:a", "copy", "-c:s", "copy", ] + if hdr: + common += [ + "-color_primaries", "bt709", "-color_trc", "bt709", + "-colorspace", "bt709", "-color_range", "tv", + ] if partial.name.endswith(".mp4.partial"): common += ["-movflags", "+faststart", "-f", "mp4"] else: @@ -594,7 +642,7 @@ class Manager: return "full" log(f"Encoding {relative}") - command = self.ffmpeg_command(source, partial) + command = self.ffmpeg_command(source, partial, info) log_path = STATE_ROOT / "ffmpeg.log" with log_path.open("ab", buffering=0) as ffmpeg_log: self.process = subprocess.Popen( @@ -633,6 +681,7 @@ class Manager: "relative": relative, "source_fingerprint": fingerprint(source), "created_at": time.time(), + "encode_profile": ENCODE_PROFILE, }) if self.is_locked(relative): output.unlink(missing_ok=True) @@ -649,6 +698,7 @@ class Manager: if not JELLYFIN_API_KEY: raise RuntimeError("JELLYFIN_API_KEY is required") self.read_sessions() + self.purge_incompatible_cache_profiles() self.enforce_cache_limit() self.scan_inventory() self.sync_view() @@ -657,6 +707,7 @@ class Manager: if not self.read_sessions(): time.sleep(POLL_SECONDS) continue + self.purge_incompatible_cache_profiles() self.enforce_cache_limit() now = time.time() if now - self.last_scan >= RESCAN_SECONDS: @@ -706,6 +757,15 @@ class Manager: return True return bool(self.active_items) + def purge(self, force=False): + if not force and self.active(): + raise RuntimeError("cache purge refused while media is playing") + for _, relative, _, _ in self.cache_entries(): + log(f"Purging {relative}") + self.remove_cache(relative, force=force) + self.sync_view() + self.save() + def trim(self, target_bytes): if self.active(): raise RuntimeError("cache resize refused while media is playing") @@ -736,6 +796,8 @@ def main(): raise SystemExit(0 if not manager.active() else 1) elif command == "trim" and len(sys.argv) == 3: manager.trim(int(sys.argv[2])) + elif command == "purge": + manager.purge(force="--force" in sys.argv[2:]) else: raise SystemExit(f"unknown command: {command}") diff --git a/tests/integration.sh b/tests/integration.sh index 0eee6eb..364827a 100755 --- a/tests/integration.sh +++ b/tests/integration.sh @@ -14,17 +14,27 @@ docker run --rm --entrypoint ffmpeg -v "$ROOT/source:/source" "$IMAGE" \ -hide_banner -loglevel error -f lavfi -i testsrc2=size=1280x720:rate=24 \ -t 6 -c:v libx265 -pix_fmt yuv420p10le -an /source/movies/Test/Test.mkv +docker run --rm --entrypoint ffmpeg -v "$ROOT/source:/source" "$IMAGE" \ + -hide_banner -loglevel error -f lavfi -i testsrc2=size=1280x720:rate=24 \ + -t 6 -c:v libx265 -pix_fmt yuv420p10le \ + -color_primaries bt2020 -color_trc smpte2084 -colorspace bt2020nc \ + -x265-params 'colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc' \ + -an /source/movies/Test/Test.HDR10.mkv + docker run --rm --network "$NETWORK" --entrypoint python3 \ -e SOURCE_ROOT=/source -e VIEW_ROOT=/view -e CACHE_ROOT=/cache -e STATE_ROOT=/state \ -e JELLYFIN_URL="$JELLYFIN_URL" -e JELLYFIN_API_KEY \ -e JELLYFIN_LIBRARY_ROOT=/data/library -e JELLYFIN_ORIGINAL_ROOT=/data/original \ -e JELLYFIN_CACHE_ROOT=/data/cache \ -v "$ROOT/source:/source:ro" -v "$ROOT/view:/view" -v "$ROOT/cache:/cache" -v "$ROOT/state:/state" \ - "$IMAGE" -c 'import importlib.machinery; m=importlib.machinery.SourceFileLoader("p","/app/pretranscode").load_module(); x=m.Manager(); assert x.read_sessions(); x.scan_inventory(); x.sync_view(); assert x.encode("movies/Test/Test.mkv",False)=="completed"' + "$IMAGE" -c 'import importlib.machinery; m=importlib.machinery.SourceFileLoader("p","/app/pretranscode").load_module(); x=m.Manager(); assert x.read_sessions(); x.scan_inventory(); x.sync_view(); assert x.encode("movies/Test/Test.mkv",False)=="completed"; assert x.encode("movies/Test/Test.HDR10.mkv",False)=="completed"' -OUTPUT=$(find "$ROOT/cache/files" -name '*.mkv' ! -name '*.partial' | head -1) +SDR_TARGET=$(readlink "$ROOT/view/movies/Test/Test.mkv") +HDR_TARGET=$(readlink "$ROOT/view/movies/Test/Test.HDR10.mkv") docker run --rm --entrypoint ffprobe -v "$ROOT/cache:/cache:ro" "$IMAGE" \ -v error -select_streams v:0 -show_entries stream=codec_name,pix_fmt \ - -of default=nw=1 "/cache/files/$(basename "$OUTPUT")" | grep -qx 'codec_name=h264' -test "$(readlink "$ROOT/view/movies/Test/Test.mkv")" = "/data/cache/files/$(basename "$OUTPUT")" + -of default=nw=1 "${SDR_TARGET/\/data\/cache/\/cache}" | grep -qx 'codec_name=h264' +docker run --rm --entrypoint ffprobe -v "$ROOT/cache:/cache:ro" "$IMAGE" \ + -v error -select_streams v:0 -show_entries stream=color_transfer,color_primaries,color_space \ + -of default=nw=1 "${HDR_TARGET/\/data\/cache/\/cache}" | grep -qx 'color_transfer=bt709' echo "integration test passed" diff --git a/tests/test_core.py b/tests/test_core.py index cc59ec2..3a77fa9 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -50,6 +50,17 @@ class CoreTests(unittest.TestCase): self.assertFalse(manager.incompatible({"codec": "h264", "bit_depth": 8})) self.assertFalse(manager.incompatible({"codec": "av1", "bit_depth": 10})) + def test_quality_filters_are_global(self): + manager = self.module.Manager() + sdr = manager.ffmpeg_command(Path("show.mkv"), Path("out.mkv.partial"), {}) + hdr = manager.ffmpeg_command( + Path("show.HDR10.mkv"), Path("out.mkv.partial"), {"color_transfer": "smpte2084"} + ) + self.assertIn("zscale=dither=error_diffusion,format=yuv420p", sdr) + self.assertIn("18", sdr) + self.assertTrue(any("tonemap=tonemap=hable" in value for value in hdr)) + self.assertIn("bt709", hdr) + def test_cache_requires_matching_source_fingerprint(self): m = self.module manager = m.Manager() @@ -59,7 +70,11 @@ class CoreTests(unittest.TestCase): source.write_bytes(b"original") output, manifest = manager.cache_paths(relative) output.write_bytes(b"cached") - manifest.write_text(json.dumps({"relative": relative, "source_fingerprint": m.fingerprint(source)})) + manifest.write_text(json.dumps({ + "relative": relative, + "source_fingerprint": m.fingerprint(source), + "encode_profile": m.ENCODE_PROFILE, + })) self.assertTrue(manager.cache_valid(relative)) source.write_bytes(b"changed") self.assertFalse(manager.cache_valid(relative)) @@ -73,7 +88,11 @@ class CoreTests(unittest.TestCase): source.write_bytes(b"original") output, manifest = manager.cache_paths(relative) output.write_bytes(b"cached rendition") - manifest.write_text(json.dumps({"relative": relative, "source_fingerprint": m.fingerprint(source)})) + manifest.write_text(json.dumps({ + "relative": relative, + "source_fingerprint": m.fingerprint(source), + "encode_profile": m.ENCODE_PROFILE, + })) manager.ensure_link(relative, m.cache_target(relative)) old_limit = m.CACHE_MAX_BYTES try: