Raise quality and apply global dithering and HDR tone mapping

.env.example

@@ -27,7 +27,7 @@ JELLYFIN_CACHE_ROOT=/data/cache
 SERIES_COOLDOWN_SECONDS=3600
 POLL_SECONDS=5
 RESCAN_SECONDS=300
-H264_CRF=20
+H264_CRF=18
 H264_PRESET=medium
-H264_MAXRATE=12M
-H264_BUFSIZE=24M
+H264_MAXRATE=15M
+H264_BUFSIZE=30M

README.md

@@ -54,9 +54,11 @@ the `.env.example` values through Coolify's environment-variable UI.
 
 ## Codec policy
 
-Cached video is H.264 High Profile, Level 4.1, 8-bit `yuv420p`, CRF 20, capped
-at 12 Mbps. Existing audio, subtitles, chapters, and Matroska attachments are
-preserved when the container supports them.
+Cached video is H.264 High Profile, Level 4.1, 8-bit `yuv420p`, CRF 18, capped
+at 15 Mbps. Existing audio, subtitles, chapters, and Matroska attachments are
+preserved when the container supports them. Ten-bit SDR is converted with
+error-diffusion dithering; HDR10, HLG, and Dolby Vision-labelled sources are
+tone-mapped to BT.709 SDR before encoding.
 
 The fixed source blacklist is:
 
@@ -65,7 +67,22 @@ The fixed source blacklist is:
 - MPEG-2
 - VC-1
 
-Client error reports never modify this list.
+Client error reports never modify this list. Encoding settings form part of
+each cache manifest, so a global quality-policy change automatically invalidates
+and replaces older renditions without touching originals.
+
+## Cache resizing
+
+`CACHE_MAX_BYTES` is the worker's configurable LRU limit. After changing it and
+restarting the container:
+
+- A smaller value evicts the oldest unlocked entries until usage fits.
+- A larger value immediately makes the additional capacity available for
+  background population, up to the size of the mounted cache filesystem.
+
+The host filesystem is deliberately separate from application logic. Enlarge or
+shrink its preallocated image during infrastructure maintenance, then set the
+matching `CACHE_MAX_BYTES` value in Coolify.
 
 ## Playback locking
 

compose.yaml

@@ -20,10 +20,10 @@ services:
       SERIES_COOLDOWN_SECONDS: ${SERIES_COOLDOWN_SECONDS:-3600}
       POLL_SECONDS: ${POLL_SECONDS:-5}
       RESCAN_SECONDS: ${RESCAN_SECONDS:-300}
-      H264_CRF: ${H264_CRF:-20}
+      H264_CRF: ${H264_CRF:-18}
       H264_PRESET: ${H264_PRESET:-medium}
-      H264_MAXRATE: ${H264_MAXRATE:-12M}
-      H264_BUFSIZE: ${H264_BUFSIZE:-24M}
+      H264_MAXRATE: ${H264_MAXRATE:-15M}
+      H264_BUFSIZE: ${H264_BUFSIZE:-30M}
     volumes:
       - ${MEDIA_LIBRARY_PATH:?}:/source:ro
       - ${PRETRANSCODE_VIEW_PATH:?}:/view

pretranscode.py

@@ -31,13 +31,15 @@ JELLYFIN_LIBRARY_ROOT = os.environ.get("JELLYFIN_LIBRARY_ROOT", "/data/library")
 JELLYFIN_ORIGINAL_ROOT = os.environ.get("JELLYFIN_ORIGINAL_ROOT", "/data/original").rstrip("/")
 JELLYFIN_CACHE_ROOT = os.environ.get("JELLYFIN_CACHE_ROOT", "/data/cache").rstrip("/")
 JELLYFIN_API_KEY = os.environ.get("JELLYFIN_API_KEY", "")
+CACHE_MAX_BYTES = env_int("CACHE_MAX_BYTES", 1_000_000_000_000)
 SERIES_COOLDOWN_SECONDS = env_int("SERIES_COOLDOWN_SECONDS", 3600)
 POLL_SECONDS = env_int("POLL_SECONDS", 5)
 RESCAN_SECONDS = env_int("RESCAN_SECONDS", 300)
-H264_CRF = os.environ.get("H264_CRF", "20")
+H264_CRF = os.environ.get("H264_CRF", "18")
 H264_PRESET = os.environ.get("H264_PRESET", "medium")
-H264_MAXRATE = os.environ.get("H264_MAXRATE", "12M")
-H264_BUFSIZE = os.environ.get("H264_BUFSIZE", "24M")
+H264_MAXRATE = os.environ.get("H264_MAXRATE", "15M")
+H264_BUFSIZE = os.environ.get("H264_BUFSIZE", "30M")
+ENCODE_PROFILE = f"h264-high41-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v2"
 FFMPEG = os.environ.get("FFMPEG", "/usr/bin/ffmpeg")
 FFPROBE = os.environ.get("FFPROBE", "/usr/bin/ffprobe")
 
@@ -224,7 +226,10 @@ class Manager:
             return False
         manifest = load_json(manifest_path, {})
         try:
-            return manifest.get("source_fingerprint") == fingerprint(source)
+            return (
+                manifest.get("source_fingerprint") == fingerprint(source)
+                and manifest.get("encode_profile") == ENCODE_PROFILE
+            )
         except OSError:
             return False
 
@@ -291,7 +296,7 @@ class Manager:
     def probe(self, path):
         command = [
             FFPROBE, "-v", "error", "-select_streams", "v:0",
-            "-show_entries", "stream=codec_name,bits_per_raw_sample,pix_fmt,profile",
+            "-show_entries", "stream=codec_name,bits_per_raw_sample,pix_fmt,profile,color_transfer,color_primaries,color_space",
             "-show_entries", "format=duration", "-of", "json", str(path),
         ]
         result = subprocess.run(command, capture_output=True, text=True, timeout=120)
@@ -317,6 +322,9 @@ class Manager:
             "codec": (stream.get("codec_name") or "").lower(),
             "bit_depth": depth,
             "profile": profile,
+            "color_transfer": (stream.get("color_transfer") or "").lower(),
+            "color_primaries": (stream.get("color_primaries") or "").lower(),
+            "color_space": (stream.get("color_space") or "").lower(),
             "duration": float((data.get("format") or {}).get("duration") or 0),
         }
 
@@ -349,6 +357,9 @@ class Manager:
                         "codec": (video.get("Codec") or "").lower(),
                         "bit_depth": int(video.get("BitDepth") or 8),
                         "profile": (video.get("Profile") or "").lower(),
+                        "color_transfer": (video.get("ColorTransfer") or "").lower(),
+                        "color_primaries": (video.get("ColorPrimaries") or "").lower(),
+                        "color_space": (video.get("ColorSpace") or "").lower(),
                         "duration": float(item.get("RunTimeTicks") or 0) / 10_000_000,
                     }
                 start += len(items)
@@ -458,6 +469,14 @@ class Manager:
         profile = (info.get("profile") or "").lower()
         return codec in BLACKLISTED_CODECS or (codec == "h264" and (depth > 8 or "10" in profile))
 
+    def is_hdr(self, info, source):
+        transfer = (info.get("color_transfer") or "").lower()
+        name = source.name.lower()
+        return (
+            transfer in {"smpte2084", "arib-std-b67"}
+            or any(token in name for token in ("hdr10", "hdr.", ".hdr", "dovi", "dolby.vision"))
+        )
+
     def candidate_list(self):
         now = time.time()
         values = []
@@ -485,8 +504,8 @@ class Manager:
             entries.append((output.stat().st_mtime, relative, output, manifest_path))
         return sorted(entries)
 
-    def remove_cache(self, relative):
-        if self.is_locked(relative):
+    def remove_cache(self, relative, force=False):
+        if not force and self.is_locked(relative):
             return False
         output, manifest = self.cache_paths(relative)
         link = VIEW_ROOT / relative
@@ -501,9 +520,40 @@ class Manager:
         manifest.unlink(missing_ok=True)
         return True
 
+    def cache_usage(self):
+        return sum(
+            path.stat().st_size
+            for path in (CACHE_ROOT / "files").iterdir()
+            if path.is_file()
+        )
+
+    def cache_available(self):
+        stat = os.statvfs(CACHE_ROOT)
+        filesystem_free = stat.f_bavail * stat.f_frsize
+        configured_free = max(0, CACHE_MAX_BYTES - self.cache_usage())
+        return min(filesystem_free, configured_free)
+
+    def purge_incompatible_cache_profiles(self):
+        for _, relative, _, _ in self.cache_entries():
+            if self.cache_valid(relative) or self.is_locked(relative):
+                continue
+            log(f"Dropping obsolete cache rendition for {relative}")
+            self.remove_cache(relative)
+
+    def enforce_cache_limit(self):
+        usage = self.cache_usage()
+        for _, relative, output, manifest in self.cache_entries():
+            if usage <= CACHE_MAX_BYTES:
+                break
+            if self.is_locked(relative):
+                continue
+            size = output.stat().st_size + manifest.stat().st_size
+            log(f"Evicting {relative} after cache resize")
+            if self.remove_cache(relative):
+                usage -= size
+
     def ensure_space(self, required, may_evict):
-        free = os.statvfs(CACHE_ROOT).f_bavail * os.statvfs(CACHE_ROOT).f_frsize
-        if free >= required:
+        if self.cache_available() >= required:
             return True
         if not may_evict:
             return False
@@ -512,8 +562,7 @@ class Manager:
                 continue
             log(f"Evicting {relative}")
             self.remove_cache(relative)
-            free = os.statvfs(CACHE_ROOT).f_bavail * os.statvfs(CACHE_ROOT).f_frsize
-            if free >= required:
+            if self.cache_available() >= required:
                 return True
         return False
 
@@ -531,15 +580,38 @@ class Manager:
                 pass
         self.process = None
 
-    def ffmpeg_command(self, source, partial):
+    def ffmpeg_command(self, source, partial, info):
+        hdr = self.is_hdr(info, source)
+        if hdr:
+            transfer = (info.get("color_transfer") or "").lower()
+            if transfer == "arib-std-b67":
+                input_colors = "tin=arib-std-b67:pin=bt2020:min=bt2020nc:"
+            elif transfer == "smpte2084":
+                input_colors = ""
+            else:
+                input_colors = "tin=smpte2084:pin=bt2020:min=bt2020nc:"
+            video_filter = (
+                f"zscale={input_colors}t=linear:npl=100,format=gbrpf32le,"
+                "zscale=p=bt709,tonemap=tonemap=hable:desat=0,"
+                "zscale=t=bt709:m=bt709:r=tv:dither=error_diffusion,format=yuv420p"
+            )
+        else:
+            video_filter = "zscale=dither=error_diffusion,format=yuv420p"
+
         common = [
             FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning", "-i", str(source),
             "-map", "0:v:0", "-map", "0:a?", "-map", "0:s?", "-map_metadata", "0", "-map_chapters", "0",
+            "-vf", video_filter,
             "-c:v", "libx264", "-preset", H264_PRESET, "-crf", H264_CRF,
             "-maxrate", H264_MAXRATE, "-bufsize", H264_BUFSIZE,
             "-pix_fmt", "yuv420p", "-profile:v", "high", "-level:v", "4.1",
             "-c:a", "copy", "-c:s", "copy",
         ]
+        if hdr:
+            common += [
+                "-color_primaries", "bt709", "-color_trc", "bt709",
+                "-colorspace", "bt709", "-color_range", "tv",
+            ]
         if partial.name.endswith(".mp4.partial"):
             common += ["-movflags", "+faststart", "-f", "mp4"]
         else:
@@ -570,7 +642,7 @@ class Manager:
             return "full"
 
         log(f"Encoding {relative}")
-        command = self.ffmpeg_command(source, partial)
+        command = self.ffmpeg_command(source, partial, info)
         log_path = STATE_ROOT / "ffmpeg.log"
         with log_path.open("ab", buffering=0) as ffmpeg_log:
             self.process = subprocess.Popen(
@@ -609,6 +681,7 @@ class Manager:
             "relative": relative,
             "source_fingerprint": fingerprint(source),
             "created_at": time.time(),
+            "encode_profile": ENCODE_PROFILE,
         })
         if self.is_locked(relative):
             output.unlink(missing_ok=True)
@@ -625,6 +698,8 @@ class Manager:
         if not JELLYFIN_API_KEY:
             raise RuntimeError("JELLYFIN_API_KEY is required")
         self.read_sessions()
+        self.purge_incompatible_cache_profiles()
+        self.enforce_cache_limit()
         self.scan_inventory()
         self.sync_view()
         self.save()
@@ -632,6 +707,8 @@ class Manager:
             if not self.read_sessions():
                 time.sleep(POLL_SECONDS)
                 continue
+            self.purge_incompatible_cache_profiles()
+            self.enforce_cache_limit()
             now = time.time()
             if now - self.last_scan >= RESCAN_SECONDS:
                 changed = self.scan_inventory()
@@ -651,8 +728,7 @@ class Manager:
             for rank, _, relative in candidates:
                 source = SOURCE_ROOT / relative
                 estimate = min(max(int(source.stat().st_size * 2.2), 1_000_000_000), 100_000_000_000)
-                free = os.statvfs(CACHE_ROOT).f_bavail * os.statvfs(CACHE_ROOT).f_frsize
-                if rank == 0 or free >= estimate:
+                if rank == 0 or self.cache_available() >= estimate:
                     selected = (rank == 0, relative)
                     break
             if not selected:
@@ -681,6 +757,15 @@ class Manager:
             return True
         return bool(self.active_items)
 
+    def purge(self, force=False):
+        if not force and self.active():
+            raise RuntimeError("cache purge refused while media is playing")
+        for _, relative, _, _ in self.cache_entries():
+            log(f"Purging {relative}")
+            self.remove_cache(relative, force=force)
+        self.sync_view()
+        self.save()
+
     def trim(self, target_bytes):
         if self.active():
             raise RuntimeError("cache resize refused while media is playing")
@@ -711,6 +796,8 @@ def main():
         raise SystemExit(0 if not manager.active() else 1)
     elif command == "trim" and len(sys.argv) == 3:
         manager.trim(int(sys.argv[2]))
+    elif command == "purge":
+        manager.purge(force="--force" in sys.argv[2:])
     else:
         raise SystemExit(f"unknown command: {command}")
 

tests/integration.sh

@@ -14,17 +14,27 @@ docker run --rm --entrypoint ffmpeg -v "$ROOT/source:/source" "$IMAGE" \
   -hide_banner -loglevel error -f lavfi -i testsrc2=size=1280x720:rate=24 \
   -t 6 -c:v libx265 -pix_fmt yuv420p10le -an /source/movies/Test/Test.mkv
 
+docker run --rm --entrypoint ffmpeg -v "$ROOT/source:/source" "$IMAGE" \
+  -hide_banner -loglevel error -f lavfi -i testsrc2=size=1280x720:rate=24 \
+  -t 6 -c:v libx265 -pix_fmt yuv420p10le \
+  -color_primaries bt2020 -color_trc smpte2084 -colorspace bt2020nc \
+  -x265-params 'colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc' \
+  -an /source/movies/Test/Test.HDR10.mkv
+
 docker run --rm --network "$NETWORK" --entrypoint python3 \
   -e SOURCE_ROOT=/source -e VIEW_ROOT=/view -e CACHE_ROOT=/cache -e STATE_ROOT=/state \
   -e JELLYFIN_URL="$JELLYFIN_URL" -e JELLYFIN_API_KEY \
   -e JELLYFIN_LIBRARY_ROOT=/data/library -e JELLYFIN_ORIGINAL_ROOT=/data/original \
   -e JELLYFIN_CACHE_ROOT=/data/cache \
   -v "$ROOT/source:/source:ro" -v "$ROOT/view:/view" -v "$ROOT/cache:/cache" -v "$ROOT/state:/state" \
-  "$IMAGE" -c 'import importlib.machinery; m=importlib.machinery.SourceFileLoader("p","/app/pretranscode").load_module(); x=m.Manager(); assert x.read_sessions(); x.scan_inventory(); x.sync_view(); assert x.encode("movies/Test/Test.mkv",False)=="completed"'
+  "$IMAGE" -c 'import importlib.machinery; m=importlib.machinery.SourceFileLoader("p","/app/pretranscode").load_module(); x=m.Manager(); assert x.read_sessions(); x.scan_inventory(); x.sync_view(); assert x.encode("movies/Test/Test.mkv",False)=="completed"; assert x.encode("movies/Test/Test.HDR10.mkv",False)=="completed"'
 
-OUTPUT=$(find "$ROOT/cache/files" -name '*.mkv' ! -name '*.partial' | head -1)
+SDR_TARGET=$(readlink "$ROOT/view/movies/Test/Test.mkv")
+HDR_TARGET=$(readlink "$ROOT/view/movies/Test/Test.HDR10.mkv")
 docker run --rm --entrypoint ffprobe -v "$ROOT/cache:/cache:ro" "$IMAGE" \
   -v error -select_streams v:0 -show_entries stream=codec_name,pix_fmt \
-  -of default=nw=1 "/cache/files/$(basename "$OUTPUT")" | grep -qx 'codec_name=h264'
-test "$(readlink "$ROOT/view/movies/Test/Test.mkv")" = "/data/cache/files/$(basename "$OUTPUT")"
+  -of default=nw=1 "${SDR_TARGET/\/data\/cache/\/cache}" | grep -qx 'codec_name=h264'
+docker run --rm --entrypoint ffprobe -v "$ROOT/cache:/cache:ro" "$IMAGE" \
+  -v error -select_streams v:0 -show_entries stream=color_transfer,color_primaries,color_space \
+  -of default=nw=1 "${HDR_TARGET/\/data\/cache/\/cache}" | grep -qx 'color_transfer=bt709'
 echo "integration test passed"

tests/test_core.py

@@ -50,6 +50,17 @@ class CoreTests(unittest.TestCase):
         self.assertFalse(manager.incompatible({"codec": "h264", "bit_depth": 8}))
         self.assertFalse(manager.incompatible({"codec": "av1", "bit_depth": 10}))
 
+    def test_quality_filters_are_global(self):
+        manager = self.module.Manager()
+        sdr = manager.ffmpeg_command(Path("show.mkv"), Path("out.mkv.partial"), {})
+        hdr = manager.ffmpeg_command(
+            Path("show.HDR10.mkv"), Path("out.mkv.partial"), {"color_transfer": "smpte2084"}
+        )
+        self.assertIn("zscale=dither=error_diffusion,format=yuv420p", sdr)
+        self.assertIn("18", sdr)
+        self.assertTrue(any("tonemap=tonemap=hable" in value for value in hdr))
+        self.assertIn("bt709", hdr)
+
     def test_cache_requires_matching_source_fingerprint(self):
         m = self.module
         manager = m.Manager()
@@ -59,11 +70,39 @@ class CoreTests(unittest.TestCase):
         source.write_bytes(b"original")
         output, manifest = manager.cache_paths(relative)
         output.write_bytes(b"cached")
-        manifest.write_text(json.dumps({"relative": relative, "source_fingerprint": m.fingerprint(source)}))
+        manifest.write_text(json.dumps({
+            "relative": relative,
+            "source_fingerprint": m.fingerprint(source),
+            "encode_profile": m.ENCODE_PROFILE,
+        }))
         self.assertTrue(manager.cache_valid(relative))
         source.write_bytes(b"changed")
         self.assertFalse(manager.cache_valid(relative))
 
+    def test_smaller_cache_limit_evicts_oldest_content(self):
+        m = self.module
+        manager = m.Manager()
+        relative = "movies/Resize/Resize.mkv"
+        source = self.root / "source" / relative
+        source.parent.mkdir(parents=True, exist_ok=True)
+        source.write_bytes(b"original")
+        output, manifest = manager.cache_paths(relative)
+        output.write_bytes(b"cached rendition")
+        manifest.write_text(json.dumps({
+            "relative": relative,
+            "source_fingerprint": m.fingerprint(source),
+            "encode_profile": m.ENCODE_PROFILE,
+        }))
+        manager.ensure_link(relative, m.cache_target(relative))
+        old_limit = m.CACHE_MAX_BYTES
+        try:
+            m.CACHE_MAX_BYTES = 1
+            manager.enforce_cache_limit()
+        finally:
+            m.CACHE_MAX_BYTES = old_limit
+        self.assertFalse(output.exists())
+        self.assertEqual(os.readlink(self.root / "view" / relative), m.source_target(relative))
+
 
 if __name__ == "__main__":
     unittest.main()