3 Commits
6 changed files with 321 additions and 51 deletions
+3 -3
View File
@@ -27,7 +27,7 @@ JELLYFIN_CACHE_ROOT=/data/cache
SERIES_COOLDOWN_SECONDS=3600 SERIES_COOLDOWN_SECONDS=3600
POLL_SECONDS=5 POLL_SECONDS=5
RESCAN_SECONDS=300 RESCAN_SECONDS=300
H264_CRF=20 H264_CRF=18
H264_PRESET=medium H264_PRESET=medium
H264_MAXRATE=12M H264_MAXRATE=15M
H264_BUFSIZE=24M H264_BUFSIZE=30M
+26 -4
View File
@@ -54,9 +54,13 @@ the `.env.example` values through Coolify's environment-variable UI.
## Codec policy ## Codec policy
Cached video is H.264 High Profile, Level 4.1, 8-bit `yuv420p`, CRF 20, capped Cached video is H.264 High Profile, 8-bit `yuv420p`, CRF 18, capped at 15 Mbps.
at 12 Mbps. Existing audio, subtitles, chapters, and Matroska attachments are Resolution and frame rate are preserved. Existing audio, subtitles, chapters,
preserved when the container supports them. and Matroska attachments are copied. Ten-bit SDR is converted with
error-diffusion dithering; sources carrying explicit HDR10 or HLG transfer
metadata are tone-mapped to BT.709 SDR. The filtered video is mapped as a new
stream so source-only Dolby Vision configuration data cannot leak into H.264
outputs.
The fixed source blacklist is: The fixed source blacklist is:
@@ -65,7 +69,9 @@ The fixed source blacklist is:
- MPEG-2 - MPEG-2
- VC-1 - VC-1
Client error reports never modify this list. Client error reports never modify this list. Encoding settings form part of
each cache manifest, so a global quality-policy change automatically invalidates
and replaces older renditions without touching originals.
## Cache resizing ## Cache resizing
@@ -95,6 +101,22 @@ partials and orphan files. Atomic symlinks always point to either an original or
a completed cache file. If the cache mount marker is absent, the worker refuses a completed cache file. If the cache mount marker is absent, the worker refuses
to start. to start.
## Upgrading from v1.2.0
Version 1.2.1 uses a v4 cache profile. Before starting the new worker, stop the
old worker and migrate its manifests:
```bash
docker compose stop pretranscode
docker compose run --rm pretranscode migrate-v4
```
The migration re-probes immutable sources, updates every valid v3 manifest, and
creates a state backup. Cached H.264 streams that inherited Dolby Vision
configuration data are repaired with a lossless stream-copy remux; video is not
re-encoded. Other cached media files are not rewritten. The command refuses to
run while Jellyfin reports active playback.
## Migration ## Migration
Copy or recreate these resources on the destination: Copy or recreate these resources on the destination:
+3 -3
View File
@@ -20,10 +20,10 @@ services:
SERIES_COOLDOWN_SECONDS: ${SERIES_COOLDOWN_SECONDS:-3600} SERIES_COOLDOWN_SECONDS: ${SERIES_COOLDOWN_SECONDS:-3600}
POLL_SECONDS: ${POLL_SECONDS:-5} POLL_SECONDS: ${POLL_SECONDS:-5}
RESCAN_SECONDS: ${RESCAN_SECONDS:-300} RESCAN_SECONDS: ${RESCAN_SECONDS:-300}
H264_CRF: ${H264_CRF:-20} H264_CRF: ${H264_CRF:-18}
H264_PRESET: ${H264_PRESET:-medium} H264_PRESET: ${H264_PRESET:-medium}
H264_MAXRATE: ${H264_MAXRATE:-12M} H264_MAXRATE: ${H264_MAXRATE:-15M}
H264_BUFSIZE: ${H264_BUFSIZE:-24M} H264_BUFSIZE: ${H264_BUFSIZE:-30M}
volumes: volumes:
- ${MEDIA_LIBRARY_PATH:?}:/source:ro - ${MEDIA_LIBRARY_PATH:?}:/source:ro
- ${PRETRANSCODE_VIEW_PATH:?}:/view - ${PRETRANSCODE_VIEW_PATH:?}:/view
+188 -24
View File
@@ -4,6 +4,7 @@
import hashlib import hashlib
import json import json
import os import os
import shutil
import signal import signal
import subprocess import subprocess
import sys import sys
@@ -35,10 +36,12 @@ CACHE_MAX_BYTES = env_int("CACHE_MAX_BYTES", 1_000_000_000_000)
SERIES_COOLDOWN_SECONDS = env_int("SERIES_COOLDOWN_SECONDS", 3600) SERIES_COOLDOWN_SECONDS = env_int("SERIES_COOLDOWN_SECONDS", 3600)
POLL_SECONDS = env_int("POLL_SECONDS", 5) POLL_SECONDS = env_int("POLL_SECONDS", 5)
RESCAN_SECONDS = env_int("RESCAN_SECONDS", 300) RESCAN_SECONDS = env_int("RESCAN_SECONDS", 300)
H264_CRF = os.environ.get("H264_CRF", "20") H264_CRF = os.environ.get("H264_CRF", "18")
H264_PRESET = os.environ.get("H264_PRESET", "medium") H264_PRESET = os.environ.get("H264_PRESET", "medium")
H264_MAXRATE = os.environ.get("H264_MAXRATE", "12M") H264_MAXRATE = os.environ.get("H264_MAXRATE", "15M")
H264_BUFSIZE = os.environ.get("H264_BUFSIZE", "24M") H264_BUFSIZE = os.environ.get("H264_BUFSIZE", "30M")
LEGACY_ENCODE_PROFILE = f"h264-high-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v3"
ENCODE_PROFILE = f"h264-high-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v4"
FFMPEG = os.environ.get("FFMPEG", "/usr/bin/ffmpeg") FFMPEG = os.environ.get("FFMPEG", "/usr/bin/ffmpeg")
FFPROBE = os.environ.get("FFPROBE", "/usr/bin/ffprobe") FFPROBE = os.environ.get("FFPROBE", "/usr/bin/ffprobe")
@@ -225,18 +228,19 @@ class Manager:
return False return False
manifest = load_json(manifest_path, {}) manifest = load_json(manifest_path, {})
try: try:
return manifest.get("source_fingerprint") == fingerprint(source) return (
manifest.get("source_fingerprint") == fingerprint(source)
and manifest.get("encode_profile") == ENCODE_PROFILE
)
except OSError: except OSError:
return False return False
def touch_cache(self, relative): def touch_cache(self, relative):
if not self.cache_valid(relative): if not self.cache_valid(relative):
return return
output, manifest = self.cache_paths(relative) _, manifest = self.cache_paths(relative)
now = time.time()
for path in (output, manifest):
try: try:
os.utime(path, (now, now), follow_symlinks=False) os.utime(manifest, None, follow_symlinks=False)
except OSError: except OSError:
pass pass
@@ -292,8 +296,11 @@ class Manager:
def probe(self, path): def probe(self, path):
command = [ command = [
FFPROBE, "-v", "error", "-select_streams", "v:0", FFPROBE, "-v", "error", "-select_streams", "v:0",
"-show_entries", "stream=codec_name,bits_per_raw_sample,pix_fmt,profile", "-show_entries",
"-show_entries", "format=duration", "-of", "json", str(path), "stream=codec_name,bits_per_raw_sample,pix_fmt,profile,color_transfer,color_primaries,color_space:"
"stream_side_data=side_data_type,dv_profile,rpu_present_flag,el_present_flag,bl_present_flag:"
"format=duration",
"-of", "json", str(path),
] ]
result = subprocess.run(command, capture_output=True, text=True, timeout=120) result = subprocess.run(command, capture_output=True, text=True, timeout=120)
if result.returncode: if result.returncode:
@@ -314,13 +321,34 @@ class Manager:
depth = 10 depth = 10
else: else:
depth = 8 depth = 8
side_data = stream.get("side_data_list") or []
return { return {
"codec": (stream.get("codec_name") or "").lower(), "codec": (stream.get("codec_name") or "").lower(),
"bit_depth": depth, "bit_depth": depth,
"profile": profile, "profile": profile,
"color_transfer": (stream.get("color_transfer") or "").lower(),
"color_primaries": (stream.get("color_primaries") or "").lower(),
"color_space": (stream.get("color_space") or "").lower(),
"dolby_vision": any(
"dovi" in (value.get("side_data_type") or "").lower()
for value in side_data
),
"duration": float((data.get("format") or {}).get("duration") or 0), "duration": float((data.get("format") or {}).get("duration") or 0),
} }
def source_info(self, relative):
source = SOURCE_ROOT / relative
details = self.probe(source)
previous = self.state["inventory"].get(relative) or {}
if previous.get("item_id"):
details["item_id"] = previous["item_id"]
details["fingerprint"] = fingerprint(source)
stat = source.stat()
details["size"] = stat.st_size
details["mtime"] = stat.st_mtime
self.state["inventory"][relative] = details
return details
def jellyfin_items(self): def jellyfin_items(self):
values = {} values = {}
start = 0 start = 0
@@ -350,6 +378,9 @@ class Manager:
"codec": (video.get("Codec") or "").lower(), "codec": (video.get("Codec") or "").lower(),
"bit_depth": int(video.get("BitDepth") or 8), "bit_depth": int(video.get("BitDepth") or 8),
"profile": (video.get("Profile") or "").lower(), "profile": (video.get("Profile") or "").lower(),
"color_transfer": (video.get("ColorTransfer") or "").lower(),
"color_primaries": (video.get("ColorPrimaries") or "").lower(),
"color_space": (video.get("ColorSpace") or "").lower(),
"duration": float(item.get("RunTimeTicks") or 0) / 10_000_000, "duration": float(item.get("RunTimeTicks") or 0) / 10_000_000,
} }
start += len(items) start += len(items)
@@ -382,12 +413,6 @@ class Manager:
if relative in catalog and catalog[relative].get("item_id"): if relative in catalog and catalog[relative].get("item_id"):
old["item_id"] = catalog[relative]["item_id"] old["item_id"] = catalog[relative]["item_id"]
continue continue
details = catalog.get(relative)
view = VIEW_ROOT / relative
view_uses_cache = view.is_symlink() and os.readlink(view) == cache_target(relative)
if details and not old and not view_uses_cache:
details = dict(details)
else:
try: try:
details = self.probe(source) details = self.probe(source)
except Exception as error: except Exception as error:
@@ -459,6 +484,9 @@ class Manager:
profile = (info.get("profile") or "").lower() profile = (info.get("profile") or "").lower()
return codec in BLACKLISTED_CODECS or (codec == "h264" and (depth > 8 or "10" in profile)) return codec in BLACKLISTED_CODECS or (codec == "h264" and (depth > 8 or "10" in profile))
def is_hdr(self, info):
return (info.get("color_transfer") or "").lower() in {"smpte2084", "arib-std-b67"}
def candidate_list(self): def candidate_list(self):
now = time.time() now = time.time()
values = [] values = []
@@ -483,11 +511,11 @@ class Manager:
if not output.is_file(): if not output.is_file():
manifest_path.unlink(missing_ok=True) manifest_path.unlink(missing_ok=True)
continue continue
entries.append((output.stat().st_mtime, relative, output, manifest_path)) entries.append((manifest_path.stat().st_mtime, relative, output, manifest_path))
return sorted(entries) return sorted(entries)
def remove_cache(self, relative): def remove_cache(self, relative, force=False):
if self.is_locked(relative): if not force and self.is_locked(relative):
return False return False
output, manifest = self.cache_paths(relative) output, manifest = self.cache_paths(relative)
link = VIEW_ROOT / relative link = VIEW_ROOT / relative
@@ -515,6 +543,13 @@ class Manager:
configured_free = max(0, CACHE_MAX_BYTES - self.cache_usage()) configured_free = max(0, CACHE_MAX_BYTES - self.cache_usage())
return min(filesystem_free, configured_free) return min(filesystem_free, configured_free)
def purge_incompatible_cache_profiles(self):
for _, relative, _, _ in self.cache_entries():
if self.cache_valid(relative) or self.is_locked(relative):
continue
log(f"Dropping obsolete cache rendition for {relative}")
self.remove_cache(relative)
def enforce_cache_limit(self): def enforce_cache_limit(self):
usage = self.cache_usage() usage = self.cache_usage()
for _, relative, output, manifest in self.cache_entries(): for _, relative, output, manifest in self.cache_entries():
@@ -555,15 +590,36 @@ class Manager:
pass pass
self.process = None self.process = None
def ffmpeg_command(self, source, partial): def ffmpeg_command(self, source, partial, info):
hdr = self.is_hdr(info)
if hdr:
transfer = (info.get("color_transfer") or "").lower()
if transfer == "arib-std-b67":
input_colors = "tin=arib-std-b67:pin=bt2020:min=bt2020nc:"
else:
input_colors = ""
video_filter = (
f"zscale={input_colors}t=linear:npl=100,format=gbrpf32le,"
"zscale=p=bt709,tonemap=tonemap=hable:desat=0,"
"zscale=t=bt709:m=bt709:r=tv:dither=error_diffusion,format=yuv420p"
)
else:
video_filter = "zscale=dither=error_diffusion,format=yuv420p"
common = [ common = [
FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning", "-i", str(source), FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning", "-i", str(source),
"-map", "0:v:0", "-map", "0:a?", "-map", "0:s?", "-map_metadata", "0", "-map_chapters", "0", "-filter_complex", f"[0:v:0]{video_filter}[video]",
"-map", "[video]", "-map", "0:a?", "-map", "0:s?", "-map_metadata", "0", "-map_chapters", "0",
"-c:v", "libx264", "-preset", H264_PRESET, "-crf", H264_CRF, "-c:v", "libx264", "-preset", H264_PRESET, "-crf", H264_CRF,
"-maxrate", H264_MAXRATE, "-bufsize", H264_BUFSIZE, "-maxrate", H264_MAXRATE, "-bufsize", H264_BUFSIZE,
"-pix_fmt", "yuv420p", "-profile:v", "high", "-level:v", "4.1", "-pix_fmt", "yuv420p", "-profile:v", "high",
"-c:a", "copy", "-c:s", "copy", "-c:a", "copy", "-c:s", "copy",
] ]
if hdr:
common += [
"-color_primaries", "bt709", "-color_trc", "bt709",
"-colorspace", "bt709", "-color_range", "tv",
]
if partial.name.endswith(".mp4.partial"): if partial.name.endswith(".mp4.partial"):
common += ["-movflags", "+faststart", "-f", "mp4"] common += ["-movflags", "+faststart", "-f", "mp4"]
else: else:
@@ -574,6 +630,8 @@ class Manager:
details = self.probe(output) details = self.probe(output)
if details.get("codec") != "h264" or int(details.get("bit_depth") or 0) > 8: if details.get("codec") != "h264" or int(details.get("bit_depth") or 0) > 8:
raise RuntimeError("output is not 8-bit H.264") raise RuntimeError("output is not 8-bit H.264")
if details.get("dolby_vision"):
raise RuntimeError("output incorrectly retains Dolby Vision stream metadata")
source_duration = float(expected_duration or 0) source_duration = float(expected_duration or 0)
output_duration = float(details.get("duration") or 0) output_duration = float(details.get("duration") or 0)
if source_duration and abs(source_duration - output_duration) > max(3, source_duration * 0.01): if source_duration and abs(source_duration - output_duration) > max(3, source_duration * 0.01):
@@ -581,6 +639,96 @@ class Manager:
if output.stat().st_size < 1_000_000: if output.stat().st_size < 1_000_000:
raise RuntimeError("output is unexpectedly small") raise RuntimeError("output is unexpectedly small")
def video_stream_hash(self, path):
command = [
FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "error", "-i", str(path),
"-map", "0:v:0", "-c:v", "copy", "-bsf:v", "h264_mp4toannexb",
"-f", "hash", "-hash", "sha256", "-",
]
result = subprocess.run(command, capture_output=True, text=True, timeout=3600)
if result.returncode:
raise RuntimeError(result.stderr.strip() or "unable to hash video stream")
return result.stdout.strip()
def sanitize_cache_metadata(self, relative, output):
if output.suffix.lower() not in {".mkv", ".mp4"}:
raise RuntimeError(f"unsupported cache container for metadata repair: {output.suffix}")
stage = output.with_name(output.name + ".sanitize-video.mp4.partial")
repaired = output.with_name(output.name + ".sanitize.partial")
stage.unlink(missing_ok=True)
repaired.unlink(missing_ok=True)
original_hash = self.video_stream_hash(output)
log(f"Losslessly removing inherited Dolby Vision metadata from {relative}")
first = [
FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning", "-i", str(output),
"-map", "0:v:0", "-c:v", "copy", "-an", "-sn", "-dn",
"-map_metadata", "-1", "-map_chapters", "-1", "-movflags", "+faststart",
"-f", "mp4", str(stage),
]
second = [
FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning",
"-i", str(stage), "-i", str(output),
"-map", "0:v:0", "-map", "1:a?", "-map", "1:s?",
"-map_metadata", "1", "-map_chapters", "1", "-c", "copy",
]
if output.suffix.lower() == ".mkv":
second += ["-map", "1:t?", "-c:t", "copy", "-f", "matroska"]
else:
second += ["-movflags", "+faststart", "-f", "mp4"]
second.append(str(repaired))
log_path = STATE_ROOT / "ffmpeg.log"
try:
with log_path.open("ab", buffering=0) as ffmpeg_log:
for command in (first, second):
result = subprocess.run(command, stdout=ffmpeg_log, stderr=ffmpeg_log)
if result.returncode:
raise RuntimeError(f"metadata repair ffmpeg exited with status {result.returncode}")
details = self.probe(repaired)
if details.get("dolby_vision"):
raise RuntimeError("metadata repair did not remove Dolby Vision stream metadata")
if self.video_stream_hash(repaired) != original_hash:
raise RuntimeError("metadata repair changed the encoded video stream")
os.replace(repaired, output)
finally:
stage.unlink(missing_ok=True)
repaired.unlink(missing_ok=True)
def migrate_v4(self):
if self.active():
raise RuntimeError("manifest migration refused while media is playing")
backup = STATE_FILE.with_name(f"state.json.pre-v4-{int(time.time())}")
if STATE_FILE.is_file():
shutil.copy2(STATE_FILE, backup)
log(f"Backed up state to {backup}")
migrated = 0
sanitized = 0
skipped = 0
for _, relative, output, manifest_path in self.cache_entries():
manifest = load_json(manifest_path, {})
if manifest.get("encode_profile") not in {LEGACY_ENCODE_PROFILE, ENCODE_PROFILE}:
log(f"Skipping manifest with unsupported profile: {relative}")
skipped += 1
continue
source = SOURCE_ROOT / relative
if not source.is_file() or manifest.get("source_fingerprint") != fingerprint(source):
log(f"Skipping stale manifest: {relative}")
skipped += 1
continue
source_details = self.source_info(relative)
output_details = self.probe(output)
if output_details.get("dolby_vision"):
self.sanitize_cache_metadata(relative, output)
manifest.pop("refresh_version", None)
sanitized += 1
self.validate_output(source, output, source_details.get("duration"))
manifest["encode_profile"] = ENCODE_PROFILE
manifest["migration_version"] = 4
manifest["migrated_at"] = time.time()
atomic_json(manifest_path, manifest)
migrated += 1
self.save()
log(f"Manifest migration complete: migrated={migrated}, sanitized={sanitized}, skipped={skipped}")
def encode(self, relative, priority): def encode(self, relative, priority):
source = SOURCE_ROOT / relative source = SOURCE_ROOT / relative
if not source.is_file() or self.is_locked(relative): if not source.is_file() or self.is_locked(relative):
@@ -588,13 +736,13 @@ class Manager:
output, manifest_path = self.cache_paths(relative) output, manifest_path = self.cache_paths(relative)
partial = output.with_name(output.name + ".partial") partial = output.with_name(output.name + ".partial")
partial.unlink(missing_ok=True) partial.unlink(missing_ok=True)
info = self.state["inventory"].get(relative) or {} info = self.source_info(relative)
estimate = min(max(int(source.stat().st_size * 2.2), 1_000_000_000), 100_000_000_000) estimate = min(max(int(source.stat().st_size * 2.2), 1_000_000_000), 100_000_000_000)
if not self.ensure_space(estimate, may_evict=priority): if not self.ensure_space(estimate, may_evict=priority):
return "full" return "full"
log(f"Encoding {relative}") log(f"Encoding {relative}")
command = self.ffmpeg_command(source, partial) command = self.ffmpeg_command(source, partial, info)
log_path = STATE_ROOT / "ffmpeg.log" log_path = STATE_ROOT / "ffmpeg.log"
with log_path.open("ab", buffering=0) as ffmpeg_log: with log_path.open("ab", buffering=0) as ffmpeg_log:
self.process = subprocess.Popen( self.process = subprocess.Popen(
@@ -633,6 +781,7 @@ class Manager:
"relative": relative, "relative": relative,
"source_fingerprint": fingerprint(source), "source_fingerprint": fingerprint(source),
"created_at": time.time(), "created_at": time.time(),
"encode_profile": ENCODE_PROFILE,
}) })
if self.is_locked(relative): if self.is_locked(relative):
output.unlink(missing_ok=True) output.unlink(missing_ok=True)
@@ -649,6 +798,7 @@ class Manager:
if not JELLYFIN_API_KEY: if not JELLYFIN_API_KEY:
raise RuntimeError("JELLYFIN_API_KEY is required") raise RuntimeError("JELLYFIN_API_KEY is required")
self.read_sessions() self.read_sessions()
self.purge_incompatible_cache_profiles()
self.enforce_cache_limit() self.enforce_cache_limit()
self.scan_inventory() self.scan_inventory()
self.sync_view() self.sync_view()
@@ -657,6 +807,7 @@ class Manager:
if not self.read_sessions(): if not self.read_sessions():
time.sleep(POLL_SECONDS) time.sleep(POLL_SECONDS)
continue continue
self.purge_incompatible_cache_profiles()
self.enforce_cache_limit() self.enforce_cache_limit()
now = time.time() now = time.time()
if now - self.last_scan >= RESCAN_SECONDS: if now - self.last_scan >= RESCAN_SECONDS:
@@ -706,6 +857,15 @@ class Manager:
return True return True
return bool(self.active_items) return bool(self.active_items)
def purge(self, force=False):
if not force and self.active():
raise RuntimeError("cache purge refused while media is playing")
for _, relative, _, _ in self.cache_entries():
log(f"Purging {relative}")
self.remove_cache(relative, force=force)
self.sync_view()
self.save()
def trim(self, target_bytes): def trim(self, target_bytes):
if self.active(): if self.active():
raise RuntimeError("cache resize refused while media is playing") raise RuntimeError("cache resize refused while media is playing")
@@ -736,6 +896,10 @@ def main():
raise SystemExit(0 if not manager.active() else 1) raise SystemExit(0 if not manager.active() else 1)
elif command == "trim" and len(sys.argv) == 3: elif command == "trim" and len(sys.argv) == 3:
manager.trim(int(sys.argv[2])) manager.trim(int(sys.argv[2]))
elif command == "purge":
manager.purge(force="--force" in sys.argv[2:])
elif command == "migrate-v4":
manager.migrate_v4()
else: else:
raise SystemExit(f"unknown command: {command}") raise SystemExit(f"unknown command: {command}")
+14 -4
View File
@@ -14,17 +14,27 @@ docker run --rm --entrypoint ffmpeg -v "$ROOT/source:/source" "$IMAGE" \
-hide_banner -loglevel error -f lavfi -i testsrc2=size=1280x720:rate=24 \ -hide_banner -loglevel error -f lavfi -i testsrc2=size=1280x720:rate=24 \
-t 6 -c:v libx265 -pix_fmt yuv420p10le -an /source/movies/Test/Test.mkv -t 6 -c:v libx265 -pix_fmt yuv420p10le -an /source/movies/Test/Test.mkv
docker run --rm --entrypoint ffmpeg -v "$ROOT/source:/source" "$IMAGE" \
-hide_banner -loglevel error -f lavfi -i testsrc2=size=1280x720:rate=24 \
-t 6 -c:v libx265 -pix_fmt yuv420p10le \
-color_primaries bt2020 -color_trc smpte2084 -colorspace bt2020nc \
-x265-params 'colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc' \
-an /source/movies/Test/Test.HDR10.mkv
docker run --rm --network "$NETWORK" --entrypoint python3 \ docker run --rm --network "$NETWORK" --entrypoint python3 \
-e SOURCE_ROOT=/source -e VIEW_ROOT=/view -e CACHE_ROOT=/cache -e STATE_ROOT=/state \ -e SOURCE_ROOT=/source -e VIEW_ROOT=/view -e CACHE_ROOT=/cache -e STATE_ROOT=/state \
-e JELLYFIN_URL="$JELLYFIN_URL" -e JELLYFIN_API_KEY \ -e JELLYFIN_URL="$JELLYFIN_URL" -e JELLYFIN_API_KEY \
-e JELLYFIN_LIBRARY_ROOT=/data/library -e JELLYFIN_ORIGINAL_ROOT=/data/original \ -e JELLYFIN_LIBRARY_ROOT=/data/library -e JELLYFIN_ORIGINAL_ROOT=/data/original \
-e JELLYFIN_CACHE_ROOT=/data/cache \ -e JELLYFIN_CACHE_ROOT=/data/cache \
-v "$ROOT/source:/source:ro" -v "$ROOT/view:/view" -v "$ROOT/cache:/cache" -v "$ROOT/state:/state" \ -v "$ROOT/source:/source:ro" -v "$ROOT/view:/view" -v "$ROOT/cache:/cache" -v "$ROOT/state:/state" \
"$IMAGE" -c 'import importlib.machinery; m=importlib.machinery.SourceFileLoader("p","/app/pretranscode").load_module(); x=m.Manager(); assert x.read_sessions(); x.scan_inventory(); x.sync_view(); assert x.encode("movies/Test/Test.mkv",False)=="completed"' "$IMAGE" -c 'import importlib.machinery; m=importlib.machinery.SourceFileLoader("p","/app/pretranscode").load_module(); x=m.Manager(); assert x.read_sessions(); x.scan_inventory(); x.sync_view(); assert x.encode("movies/Test/Test.mkv",False)=="completed"; assert x.encode("movies/Test/Test.HDR10.mkv",False)=="completed"'
OUTPUT=$(find "$ROOT/cache/files" -name '*.mkv' ! -name '*.partial' | head -1) SDR_TARGET=$(readlink "$ROOT/view/movies/Test/Test.mkv")
HDR_TARGET=$(readlink "$ROOT/view/movies/Test/Test.HDR10.mkv")
docker run --rm --entrypoint ffprobe -v "$ROOT/cache:/cache:ro" "$IMAGE" \ docker run --rm --entrypoint ffprobe -v "$ROOT/cache:/cache:ro" "$IMAGE" \
-v error -select_streams v:0 -show_entries stream=codec_name,pix_fmt \ -v error -select_streams v:0 -show_entries stream=codec_name,pix_fmt \
-of default=nw=1 "/cache/files/$(basename "$OUTPUT")" | grep -qx 'codec_name=h264' -of default=nw=1 "${SDR_TARGET/\/data\/cache/\/cache}" | grep -qx 'codec_name=h264'
test "$(readlink "$ROOT/view/movies/Test/Test.mkv")" = "/data/cache/files/$(basename "$OUTPUT")" docker run --rm --entrypoint ffprobe -v "$ROOT/cache:/cache:ro" "$IMAGE" \
-v error -select_streams v:0 -show_entries stream=color_transfer,color_primaries,color_space \
-of default=nw=1 "${HDR_TARGET/\/data\/cache/\/cache}" | grep -qx 'color_transfer=bt709'
echo "integration test passed" echo "integration test passed"
+76 -2
View File
@@ -4,6 +4,8 @@ import os
import tempfile import tempfile
import unittest import unittest
from pathlib import Path from pathlib import Path
from types import SimpleNamespace
from unittest import mock
class CoreTests(unittest.TestCase): class CoreTests(unittest.TestCase):
@@ -50,6 +52,50 @@ class CoreTests(unittest.TestCase):
self.assertFalse(manager.incompatible({"codec": "h264", "bit_depth": 8})) self.assertFalse(manager.incompatible({"codec": "h264", "bit_depth": 8}))
self.assertFalse(manager.incompatible({"codec": "av1", "bit_depth": 10})) self.assertFalse(manager.incompatible({"codec": "av1", "bit_depth": 10}))
def test_quality_filters_are_global(self):
manager = self.module.Manager()
sdr = manager.ffmpeg_command(Path("show.mkv"), Path("out.mkv.partial"), {})
hdr = manager.ffmpeg_command(
Path("show.HDR10.mkv"), Path("out.mkv.partial"), {"color_transfer": "smpte2084"}
)
self.assertTrue(any("zscale=dither=error_diffusion,format=yuv420p" in value for value in sdr))
self.assertIn("-filter_complex", sdr)
self.assertIn("[video]", sdr)
self.assertNotIn("-vf", sdr)
self.assertIn("18", sdr)
self.assertTrue(any("tonemap=tonemap=hable" in value for value in hdr))
self.assertIn("bt709", hdr)
def test_probe_detects_dolby_vision_stream_metadata(self):
manager = self.module.Manager()
result = SimpleNamespace(
returncode=0,
stderr="",
stdout=json.dumps({
"streams": [{
"codec_name": "h264",
"profile": "High",
"pix_fmt": "yuv420p",
"bits_per_raw_sample": "8",
"color_transfer": "bt709",
"side_data_list": [{"side_data_type": "DOVI configuration record"}],
}],
"format": {"duration": "10.0"},
}),
)
with mock.patch.object(self.module.subprocess, "run", return_value=result):
details = manager.probe(Path("cached.mkv"))
self.assertTrue(details["dolby_vision"])
def test_validation_rejects_inherited_dolby_vision_metadata(self):
manager = self.module.Manager()
output = self.root / "cache" / "invalid.mkv"
output.write_bytes(b"x" * 1_000_001)
details = {"codec": "h264", "bit_depth": 8, "dolby_vision": True, "duration": 10.0}
with mock.patch.object(manager, "probe", return_value=details):
with self.assertRaisesRegex(RuntimeError, "Dolby Vision"):
manager.validate_output(Path("source.mkv"), output, 10.0)
def test_cache_requires_matching_source_fingerprint(self): def test_cache_requires_matching_source_fingerprint(self):
m = self.module m = self.module
manager = m.Manager() manager = m.Manager()
@@ -59,11 +105,35 @@ class CoreTests(unittest.TestCase):
source.write_bytes(b"original") source.write_bytes(b"original")
output, manifest = manager.cache_paths(relative) output, manifest = manager.cache_paths(relative)
output.write_bytes(b"cached") output.write_bytes(b"cached")
manifest.write_text(json.dumps({"relative": relative, "source_fingerprint": m.fingerprint(source)})) manifest.write_text(json.dumps({
"relative": relative,
"source_fingerprint": m.fingerprint(source),
"encode_profile": m.ENCODE_PROFILE,
}))
self.assertTrue(manager.cache_valid(relative)) self.assertTrue(manager.cache_valid(relative))
source.write_bytes(b"changed") source.write_bytes(b"changed")
self.assertFalse(manager.cache_valid(relative)) self.assertFalse(manager.cache_valid(relative))
def test_lru_touches_only_manifest(self):
m = self.module
manager = m.Manager()
relative = "movies/LRU/LRU.mkv"
source = self.root / "source" / relative
source.parent.mkdir(parents=True, exist_ok=True)
source.write_bytes(b"original")
output, manifest = manager.cache_paths(relative)
output.write_bytes(b"cached")
manifest.write_text(json.dumps({
"relative": relative,
"source_fingerprint": m.fingerprint(source),
"encode_profile": m.ENCODE_PROFILE,
}))
os.utime(output, (100, 100))
os.utime(manifest, (100, 100))
manager.touch_cache(relative)
self.assertEqual(output.stat().st_mtime, 100)
self.assertGreater(manifest.stat().st_mtime, 100)
def test_smaller_cache_limit_evicts_oldest_content(self): def test_smaller_cache_limit_evicts_oldest_content(self):
m = self.module m = self.module
manager = m.Manager() manager = m.Manager()
@@ -73,7 +143,11 @@ class CoreTests(unittest.TestCase):
source.write_bytes(b"original") source.write_bytes(b"original")
output, manifest = manager.cache_paths(relative) output, manifest = manager.cache_paths(relative)
output.write_bytes(b"cached rendition") output.write_bytes(b"cached rendition")
manifest.write_text(json.dumps({"relative": relative, "source_fingerprint": m.fingerprint(source)})) manifest.write_text(json.dumps({
"relative": relative,
"source_fingerprint": m.fingerprint(source),
"encode_profile": m.ENCODE_PROFILE,
}))
manager.ensure_link(relative, m.cache_target(relative)) manager.ensure_link(relative, m.cache_target(relative))
old_limit = m.CACHE_MAX_BYTES old_limit = m.CACHE_MAX_BYTES
try: try: