Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a66945c245 | ||
|
|
3a2d79609b | ||
|
|
b9b31abf2e |
+3
-3
@@ -27,7 +27,7 @@ JELLYFIN_CACHE_ROOT=/data/cache
|
|||||||
SERIES_COOLDOWN_SECONDS=3600
|
SERIES_COOLDOWN_SECONDS=3600
|
||||||
POLL_SECONDS=5
|
POLL_SECONDS=5
|
||||||
RESCAN_SECONDS=300
|
RESCAN_SECONDS=300
|
||||||
H264_CRF=20
|
H264_CRF=18
|
||||||
H264_PRESET=medium
|
H264_PRESET=medium
|
||||||
H264_MAXRATE=12M
|
H264_MAXRATE=15M
|
||||||
H264_BUFSIZE=24M
|
H264_BUFSIZE=30M
|
||||||
|
|||||||
@@ -54,9 +54,13 @@ the `.env.example` values through Coolify's environment-variable UI.
|
|||||||
|
|
||||||
## Codec policy
|
## Codec policy
|
||||||
|
|
||||||
Cached video is H.264 High Profile, Level 4.1, 8-bit `yuv420p`, CRF 20, capped
|
Cached video is H.264 High Profile, 8-bit `yuv420p`, CRF 18, capped at 15 Mbps.
|
||||||
at 12 Mbps. Existing audio, subtitles, chapters, and Matroska attachments are
|
Resolution and frame rate are preserved. Existing audio, subtitles, chapters,
|
||||||
preserved when the container supports them.
|
and Matroska attachments are copied. Ten-bit SDR is converted with
|
||||||
|
error-diffusion dithering; sources carrying explicit HDR10 or HLG transfer
|
||||||
|
metadata are tone-mapped to BT.709 SDR. The filtered video is mapped as a new
|
||||||
|
stream so source-only Dolby Vision configuration data cannot leak into H.264
|
||||||
|
outputs.
|
||||||
|
|
||||||
The fixed source blacklist is:
|
The fixed source blacklist is:
|
||||||
|
|
||||||
@@ -65,7 +69,9 @@ The fixed source blacklist is:
|
|||||||
- MPEG-2
|
- MPEG-2
|
||||||
- VC-1
|
- VC-1
|
||||||
|
|
||||||
Client error reports never modify this list.
|
Client error reports never modify this list. Encoding settings form part of
|
||||||
|
each cache manifest, so a global quality-policy change automatically invalidates
|
||||||
|
and replaces older renditions without touching originals.
|
||||||
|
|
||||||
## Cache resizing
|
## Cache resizing
|
||||||
|
|
||||||
@@ -95,6 +101,22 @@ partials and orphan files. Atomic symlinks always point to either an original or
|
|||||||
a completed cache file. If the cache mount marker is absent, the worker refuses
|
a completed cache file. If the cache mount marker is absent, the worker refuses
|
||||||
to start.
|
to start.
|
||||||
|
|
||||||
|
## Upgrading from v1.2.0
|
||||||
|
|
||||||
|
Version 1.2.1 uses a v4 cache profile. Before starting the new worker, stop the
|
||||||
|
old worker and migrate its manifests:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose stop pretranscode
|
||||||
|
docker compose run --rm pretranscode migrate-v4
|
||||||
|
```
|
||||||
|
|
||||||
|
The migration re-probes immutable sources, updates every valid v3 manifest, and
|
||||||
|
creates a state backup. Cached H.264 streams that inherited Dolby Vision
|
||||||
|
configuration data are repaired with a lossless stream-copy remux; video is not
|
||||||
|
re-encoded. Other cached media files are not rewritten. The command refuses to
|
||||||
|
run while Jellyfin reports active playback.
|
||||||
|
|
||||||
## Migration
|
## Migration
|
||||||
|
|
||||||
Copy or recreate these resources on the destination:
|
Copy or recreate these resources on the destination:
|
||||||
|
|||||||
+3
-3
@@ -20,10 +20,10 @@ services:
|
|||||||
SERIES_COOLDOWN_SECONDS: ${SERIES_COOLDOWN_SECONDS:-3600}
|
SERIES_COOLDOWN_SECONDS: ${SERIES_COOLDOWN_SECONDS:-3600}
|
||||||
POLL_SECONDS: ${POLL_SECONDS:-5}
|
POLL_SECONDS: ${POLL_SECONDS:-5}
|
||||||
RESCAN_SECONDS: ${RESCAN_SECONDS:-300}
|
RESCAN_SECONDS: ${RESCAN_SECONDS:-300}
|
||||||
H264_CRF: ${H264_CRF:-20}
|
H264_CRF: ${H264_CRF:-18}
|
||||||
H264_PRESET: ${H264_PRESET:-medium}
|
H264_PRESET: ${H264_PRESET:-medium}
|
||||||
H264_MAXRATE: ${H264_MAXRATE:-12M}
|
H264_MAXRATE: ${H264_MAXRATE:-15M}
|
||||||
H264_BUFSIZE: ${H264_BUFSIZE:-24M}
|
H264_BUFSIZE: ${H264_BUFSIZE:-30M}
|
||||||
volumes:
|
volumes:
|
||||||
- ${MEDIA_LIBRARY_PATH:?}:/source:ro
|
- ${MEDIA_LIBRARY_PATH:?}:/source:ro
|
||||||
- ${PRETRANSCODE_VIEW_PATH:?}:/view
|
- ${PRETRANSCODE_VIEW_PATH:?}:/view
|
||||||
|
|||||||
+199
-35
@@ -4,6 +4,7 @@
|
|||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import shutil
|
||||||
import signal
|
import signal
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
@@ -35,10 +36,12 @@ CACHE_MAX_BYTES = env_int("CACHE_MAX_BYTES", 1_000_000_000_000)
|
|||||||
SERIES_COOLDOWN_SECONDS = env_int("SERIES_COOLDOWN_SECONDS", 3600)
|
SERIES_COOLDOWN_SECONDS = env_int("SERIES_COOLDOWN_SECONDS", 3600)
|
||||||
POLL_SECONDS = env_int("POLL_SECONDS", 5)
|
POLL_SECONDS = env_int("POLL_SECONDS", 5)
|
||||||
RESCAN_SECONDS = env_int("RESCAN_SECONDS", 300)
|
RESCAN_SECONDS = env_int("RESCAN_SECONDS", 300)
|
||||||
H264_CRF = os.environ.get("H264_CRF", "20")
|
H264_CRF = os.environ.get("H264_CRF", "18")
|
||||||
H264_PRESET = os.environ.get("H264_PRESET", "medium")
|
H264_PRESET = os.environ.get("H264_PRESET", "medium")
|
||||||
H264_MAXRATE = os.environ.get("H264_MAXRATE", "12M")
|
H264_MAXRATE = os.environ.get("H264_MAXRATE", "15M")
|
||||||
H264_BUFSIZE = os.environ.get("H264_BUFSIZE", "24M")
|
H264_BUFSIZE = os.environ.get("H264_BUFSIZE", "30M")
|
||||||
|
LEGACY_ENCODE_PROFILE = f"h264-high-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v3"
|
||||||
|
ENCODE_PROFILE = f"h264-high-8bit-crf{H264_CRF}-{H264_PRESET}-{H264_MAXRATE}-quality-v4"
|
||||||
FFMPEG = os.environ.get("FFMPEG", "/usr/bin/ffmpeg")
|
FFMPEG = os.environ.get("FFMPEG", "/usr/bin/ffmpeg")
|
||||||
FFPROBE = os.environ.get("FFPROBE", "/usr/bin/ffprobe")
|
FFPROBE = os.environ.get("FFPROBE", "/usr/bin/ffprobe")
|
||||||
|
|
||||||
@@ -225,20 +228,21 @@ class Manager:
|
|||||||
return False
|
return False
|
||||||
manifest = load_json(manifest_path, {})
|
manifest = load_json(manifest_path, {})
|
||||||
try:
|
try:
|
||||||
return manifest.get("source_fingerprint") == fingerprint(source)
|
return (
|
||||||
|
manifest.get("source_fingerprint") == fingerprint(source)
|
||||||
|
and manifest.get("encode_profile") == ENCODE_PROFILE
|
||||||
|
)
|
||||||
except OSError:
|
except OSError:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def touch_cache(self, relative):
|
def touch_cache(self, relative):
|
||||||
if not self.cache_valid(relative):
|
if not self.cache_valid(relative):
|
||||||
return
|
return
|
||||||
output, manifest = self.cache_paths(relative)
|
_, manifest = self.cache_paths(relative)
|
||||||
now = time.time()
|
try:
|
||||||
for path in (output, manifest):
|
os.utime(manifest, None, follow_symlinks=False)
|
||||||
try:
|
except OSError:
|
||||||
os.utime(path, (now, now), follow_symlinks=False)
|
pass
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def ensure_link(self, relative, target, locked=False):
|
def ensure_link(self, relative, target, locked=False):
|
||||||
link = VIEW_ROOT / relative
|
link = VIEW_ROOT / relative
|
||||||
@@ -292,8 +296,11 @@ class Manager:
|
|||||||
def probe(self, path):
|
def probe(self, path):
|
||||||
command = [
|
command = [
|
||||||
FFPROBE, "-v", "error", "-select_streams", "v:0",
|
FFPROBE, "-v", "error", "-select_streams", "v:0",
|
||||||
"-show_entries", "stream=codec_name,bits_per_raw_sample,pix_fmt,profile",
|
"-show_entries",
|
||||||
"-show_entries", "format=duration", "-of", "json", str(path),
|
"stream=codec_name,bits_per_raw_sample,pix_fmt,profile,color_transfer,color_primaries,color_space:"
|
||||||
|
"stream_side_data=side_data_type,dv_profile,rpu_present_flag,el_present_flag,bl_present_flag:"
|
||||||
|
"format=duration",
|
||||||
|
"-of", "json", str(path),
|
||||||
]
|
]
|
||||||
result = subprocess.run(command, capture_output=True, text=True, timeout=120)
|
result = subprocess.run(command, capture_output=True, text=True, timeout=120)
|
||||||
if result.returncode:
|
if result.returncode:
|
||||||
@@ -314,13 +321,34 @@ class Manager:
|
|||||||
depth = 10
|
depth = 10
|
||||||
else:
|
else:
|
||||||
depth = 8
|
depth = 8
|
||||||
|
side_data = stream.get("side_data_list") or []
|
||||||
return {
|
return {
|
||||||
"codec": (stream.get("codec_name") or "").lower(),
|
"codec": (stream.get("codec_name") or "").lower(),
|
||||||
"bit_depth": depth,
|
"bit_depth": depth,
|
||||||
"profile": profile,
|
"profile": profile,
|
||||||
|
"color_transfer": (stream.get("color_transfer") or "").lower(),
|
||||||
|
"color_primaries": (stream.get("color_primaries") or "").lower(),
|
||||||
|
"color_space": (stream.get("color_space") or "").lower(),
|
||||||
|
"dolby_vision": any(
|
||||||
|
"dovi" in (value.get("side_data_type") or "").lower()
|
||||||
|
for value in side_data
|
||||||
|
),
|
||||||
"duration": float((data.get("format") or {}).get("duration") or 0),
|
"duration": float((data.get("format") or {}).get("duration") or 0),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def source_info(self, relative):
|
||||||
|
source = SOURCE_ROOT / relative
|
||||||
|
details = self.probe(source)
|
||||||
|
previous = self.state["inventory"].get(relative) or {}
|
||||||
|
if previous.get("item_id"):
|
||||||
|
details["item_id"] = previous["item_id"]
|
||||||
|
details["fingerprint"] = fingerprint(source)
|
||||||
|
stat = source.stat()
|
||||||
|
details["size"] = stat.st_size
|
||||||
|
details["mtime"] = stat.st_mtime
|
||||||
|
self.state["inventory"][relative] = details
|
||||||
|
return details
|
||||||
|
|
||||||
def jellyfin_items(self):
|
def jellyfin_items(self):
|
||||||
values = {}
|
values = {}
|
||||||
start = 0
|
start = 0
|
||||||
@@ -350,6 +378,9 @@ class Manager:
|
|||||||
"codec": (video.get("Codec") or "").lower(),
|
"codec": (video.get("Codec") or "").lower(),
|
||||||
"bit_depth": int(video.get("BitDepth") or 8),
|
"bit_depth": int(video.get("BitDepth") or 8),
|
||||||
"profile": (video.get("Profile") or "").lower(),
|
"profile": (video.get("Profile") or "").lower(),
|
||||||
|
"color_transfer": (video.get("ColorTransfer") or "").lower(),
|
||||||
|
"color_primaries": (video.get("ColorPrimaries") or "").lower(),
|
||||||
|
"color_space": (video.get("ColorSpace") or "").lower(),
|
||||||
"duration": float(item.get("RunTimeTicks") or 0) / 10_000_000,
|
"duration": float(item.get("RunTimeTicks") or 0) / 10_000_000,
|
||||||
}
|
}
|
||||||
start += len(items)
|
start += len(items)
|
||||||
@@ -382,20 +413,14 @@ class Manager:
|
|||||||
if relative in catalog and catalog[relative].get("item_id"):
|
if relative in catalog and catalog[relative].get("item_id"):
|
||||||
old["item_id"] = catalog[relative]["item_id"]
|
old["item_id"] = catalog[relative]["item_id"]
|
||||||
continue
|
continue
|
||||||
details = catalog.get(relative)
|
try:
|
||||||
view = VIEW_ROOT / relative
|
details = self.probe(source)
|
||||||
view_uses_cache = view.is_symlink() and os.readlink(view) == cache_target(relative)
|
except Exception as error:
|
||||||
if details and not old and not view_uses_cache:
|
log(f"Probe failed for {relative}: {error}")
|
||||||
details = dict(details)
|
self.state["retry_after"][relative] = time.time() + 21600
|
||||||
else:
|
continue
|
||||||
try:
|
if relative in catalog:
|
||||||
details = self.probe(source)
|
details["item_id"] = catalog[relative].get("item_id")
|
||||||
except Exception as error:
|
|
||||||
log(f"Probe failed for {relative}: {error}")
|
|
||||||
self.state["retry_after"][relative] = time.time() + 21600
|
|
||||||
continue
|
|
||||||
if relative in catalog:
|
|
||||||
details["item_id"] = catalog[relative].get("item_id")
|
|
||||||
details["fingerprint"] = current_fingerprint
|
details["fingerprint"] = current_fingerprint
|
||||||
details["size"] = source.stat().st_size
|
details["size"] = source.stat().st_size
|
||||||
details["mtime"] = source.stat().st_mtime
|
details["mtime"] = source.stat().st_mtime
|
||||||
@@ -459,6 +484,9 @@ class Manager:
|
|||||||
profile = (info.get("profile") or "").lower()
|
profile = (info.get("profile") or "").lower()
|
||||||
return codec in BLACKLISTED_CODECS or (codec == "h264" and (depth > 8 or "10" in profile))
|
return codec in BLACKLISTED_CODECS or (codec == "h264" and (depth > 8 or "10" in profile))
|
||||||
|
|
||||||
|
def is_hdr(self, info):
|
||||||
|
return (info.get("color_transfer") or "").lower() in {"smpte2084", "arib-std-b67"}
|
||||||
|
|
||||||
def candidate_list(self):
|
def candidate_list(self):
|
||||||
now = time.time()
|
now = time.time()
|
||||||
values = []
|
values = []
|
||||||
@@ -483,11 +511,11 @@ class Manager:
|
|||||||
if not output.is_file():
|
if not output.is_file():
|
||||||
manifest_path.unlink(missing_ok=True)
|
manifest_path.unlink(missing_ok=True)
|
||||||
continue
|
continue
|
||||||
entries.append((output.stat().st_mtime, relative, output, manifest_path))
|
entries.append((manifest_path.stat().st_mtime, relative, output, manifest_path))
|
||||||
return sorted(entries)
|
return sorted(entries)
|
||||||
|
|
||||||
def remove_cache(self, relative):
|
def remove_cache(self, relative, force=False):
|
||||||
if self.is_locked(relative):
|
if not force and self.is_locked(relative):
|
||||||
return False
|
return False
|
||||||
output, manifest = self.cache_paths(relative)
|
output, manifest = self.cache_paths(relative)
|
||||||
link = VIEW_ROOT / relative
|
link = VIEW_ROOT / relative
|
||||||
@@ -515,6 +543,13 @@ class Manager:
|
|||||||
configured_free = max(0, CACHE_MAX_BYTES - self.cache_usage())
|
configured_free = max(0, CACHE_MAX_BYTES - self.cache_usage())
|
||||||
return min(filesystem_free, configured_free)
|
return min(filesystem_free, configured_free)
|
||||||
|
|
||||||
|
def purge_incompatible_cache_profiles(self):
|
||||||
|
for _, relative, _, _ in self.cache_entries():
|
||||||
|
if self.cache_valid(relative) or self.is_locked(relative):
|
||||||
|
continue
|
||||||
|
log(f"Dropping obsolete cache rendition for {relative}")
|
||||||
|
self.remove_cache(relative)
|
||||||
|
|
||||||
def enforce_cache_limit(self):
|
def enforce_cache_limit(self):
|
||||||
usage = self.cache_usage()
|
usage = self.cache_usage()
|
||||||
for _, relative, output, manifest in self.cache_entries():
|
for _, relative, output, manifest in self.cache_entries():
|
||||||
@@ -555,15 +590,36 @@ class Manager:
|
|||||||
pass
|
pass
|
||||||
self.process = None
|
self.process = None
|
||||||
|
|
||||||
def ffmpeg_command(self, source, partial):
|
def ffmpeg_command(self, source, partial, info):
|
||||||
|
hdr = self.is_hdr(info)
|
||||||
|
if hdr:
|
||||||
|
transfer = (info.get("color_transfer") or "").lower()
|
||||||
|
if transfer == "arib-std-b67":
|
||||||
|
input_colors = "tin=arib-std-b67:pin=bt2020:min=bt2020nc:"
|
||||||
|
else:
|
||||||
|
input_colors = ""
|
||||||
|
video_filter = (
|
||||||
|
f"zscale={input_colors}t=linear:npl=100,format=gbrpf32le,"
|
||||||
|
"zscale=p=bt709,tonemap=tonemap=hable:desat=0,"
|
||||||
|
"zscale=t=bt709:m=bt709:r=tv:dither=error_diffusion,format=yuv420p"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
video_filter = "zscale=dither=error_diffusion,format=yuv420p"
|
||||||
|
|
||||||
common = [
|
common = [
|
||||||
FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning", "-i", str(source),
|
FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning", "-i", str(source),
|
||||||
"-map", "0:v:0", "-map", "0:a?", "-map", "0:s?", "-map_metadata", "0", "-map_chapters", "0",
|
"-filter_complex", f"[0:v:0]{video_filter}[video]",
|
||||||
|
"-map", "[video]", "-map", "0:a?", "-map", "0:s?", "-map_metadata", "0", "-map_chapters", "0",
|
||||||
"-c:v", "libx264", "-preset", H264_PRESET, "-crf", H264_CRF,
|
"-c:v", "libx264", "-preset", H264_PRESET, "-crf", H264_CRF,
|
||||||
"-maxrate", H264_MAXRATE, "-bufsize", H264_BUFSIZE,
|
"-maxrate", H264_MAXRATE, "-bufsize", H264_BUFSIZE,
|
||||||
"-pix_fmt", "yuv420p", "-profile:v", "high", "-level:v", "4.1",
|
"-pix_fmt", "yuv420p", "-profile:v", "high",
|
||||||
"-c:a", "copy", "-c:s", "copy",
|
"-c:a", "copy", "-c:s", "copy",
|
||||||
]
|
]
|
||||||
|
if hdr:
|
||||||
|
common += [
|
||||||
|
"-color_primaries", "bt709", "-color_trc", "bt709",
|
||||||
|
"-colorspace", "bt709", "-color_range", "tv",
|
||||||
|
]
|
||||||
if partial.name.endswith(".mp4.partial"):
|
if partial.name.endswith(".mp4.partial"):
|
||||||
common += ["-movflags", "+faststart", "-f", "mp4"]
|
common += ["-movflags", "+faststart", "-f", "mp4"]
|
||||||
else:
|
else:
|
||||||
@@ -574,6 +630,8 @@ class Manager:
|
|||||||
details = self.probe(output)
|
details = self.probe(output)
|
||||||
if details.get("codec") != "h264" or int(details.get("bit_depth") or 0) > 8:
|
if details.get("codec") != "h264" or int(details.get("bit_depth") or 0) > 8:
|
||||||
raise RuntimeError("output is not 8-bit H.264")
|
raise RuntimeError("output is not 8-bit H.264")
|
||||||
|
if details.get("dolby_vision"):
|
||||||
|
raise RuntimeError("output incorrectly retains Dolby Vision stream metadata")
|
||||||
source_duration = float(expected_duration or 0)
|
source_duration = float(expected_duration or 0)
|
||||||
output_duration = float(details.get("duration") or 0)
|
output_duration = float(details.get("duration") or 0)
|
||||||
if source_duration and abs(source_duration - output_duration) > max(3, source_duration * 0.01):
|
if source_duration and abs(source_duration - output_duration) > max(3, source_duration * 0.01):
|
||||||
@@ -581,6 +639,96 @@ class Manager:
|
|||||||
if output.stat().st_size < 1_000_000:
|
if output.stat().st_size < 1_000_000:
|
||||||
raise RuntimeError("output is unexpectedly small")
|
raise RuntimeError("output is unexpectedly small")
|
||||||
|
|
||||||
|
def video_stream_hash(self, path):
|
||||||
|
command = [
|
||||||
|
FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "error", "-i", str(path),
|
||||||
|
"-map", "0:v:0", "-c:v", "copy", "-bsf:v", "h264_mp4toannexb",
|
||||||
|
"-f", "hash", "-hash", "sha256", "-",
|
||||||
|
]
|
||||||
|
result = subprocess.run(command, capture_output=True, text=True, timeout=3600)
|
||||||
|
if result.returncode:
|
||||||
|
raise RuntimeError(result.stderr.strip() or "unable to hash video stream")
|
||||||
|
return result.stdout.strip()
|
||||||
|
|
||||||
|
def sanitize_cache_metadata(self, relative, output):
|
||||||
|
if output.suffix.lower() not in {".mkv", ".mp4"}:
|
||||||
|
raise RuntimeError(f"unsupported cache container for metadata repair: {output.suffix}")
|
||||||
|
stage = output.with_name(output.name + ".sanitize-video.mp4.partial")
|
||||||
|
repaired = output.with_name(output.name + ".sanitize.partial")
|
||||||
|
stage.unlink(missing_ok=True)
|
||||||
|
repaired.unlink(missing_ok=True)
|
||||||
|
original_hash = self.video_stream_hash(output)
|
||||||
|
log(f"Losslessly removing inherited Dolby Vision metadata from {relative}")
|
||||||
|
first = [
|
||||||
|
FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning", "-i", str(output),
|
||||||
|
"-map", "0:v:0", "-c:v", "copy", "-an", "-sn", "-dn",
|
||||||
|
"-map_metadata", "-1", "-map_chapters", "-1", "-movflags", "+faststart",
|
||||||
|
"-f", "mp4", str(stage),
|
||||||
|
]
|
||||||
|
second = [
|
||||||
|
FFMPEG, "-nostdin", "-hide_banner", "-loglevel", "warning",
|
||||||
|
"-i", str(stage), "-i", str(output),
|
||||||
|
"-map", "0:v:0", "-map", "1:a?", "-map", "1:s?",
|
||||||
|
"-map_metadata", "1", "-map_chapters", "1", "-c", "copy",
|
||||||
|
]
|
||||||
|
if output.suffix.lower() == ".mkv":
|
||||||
|
second += ["-map", "1:t?", "-c:t", "copy", "-f", "matroska"]
|
||||||
|
else:
|
||||||
|
second += ["-movflags", "+faststart", "-f", "mp4"]
|
||||||
|
second.append(str(repaired))
|
||||||
|
log_path = STATE_ROOT / "ffmpeg.log"
|
||||||
|
try:
|
||||||
|
with log_path.open("ab", buffering=0) as ffmpeg_log:
|
||||||
|
for command in (first, second):
|
||||||
|
result = subprocess.run(command, stdout=ffmpeg_log, stderr=ffmpeg_log)
|
||||||
|
if result.returncode:
|
||||||
|
raise RuntimeError(f"metadata repair ffmpeg exited with status {result.returncode}")
|
||||||
|
details = self.probe(repaired)
|
||||||
|
if details.get("dolby_vision"):
|
||||||
|
raise RuntimeError("metadata repair did not remove Dolby Vision stream metadata")
|
||||||
|
if self.video_stream_hash(repaired) != original_hash:
|
||||||
|
raise RuntimeError("metadata repair changed the encoded video stream")
|
||||||
|
os.replace(repaired, output)
|
||||||
|
finally:
|
||||||
|
stage.unlink(missing_ok=True)
|
||||||
|
repaired.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
def migrate_v4(self):
|
||||||
|
if self.active():
|
||||||
|
raise RuntimeError("manifest migration refused while media is playing")
|
||||||
|
backup = STATE_FILE.with_name(f"state.json.pre-v4-{int(time.time())}")
|
||||||
|
if STATE_FILE.is_file():
|
||||||
|
shutil.copy2(STATE_FILE, backup)
|
||||||
|
log(f"Backed up state to {backup}")
|
||||||
|
migrated = 0
|
||||||
|
sanitized = 0
|
||||||
|
skipped = 0
|
||||||
|
for _, relative, output, manifest_path in self.cache_entries():
|
||||||
|
manifest = load_json(manifest_path, {})
|
||||||
|
if manifest.get("encode_profile") not in {LEGACY_ENCODE_PROFILE, ENCODE_PROFILE}:
|
||||||
|
log(f"Skipping manifest with unsupported profile: {relative}")
|
||||||
|
skipped += 1
|
||||||
|
continue
|
||||||
|
source = SOURCE_ROOT / relative
|
||||||
|
if not source.is_file() or manifest.get("source_fingerprint") != fingerprint(source):
|
||||||
|
log(f"Skipping stale manifest: {relative}")
|
||||||
|
skipped += 1
|
||||||
|
continue
|
||||||
|
source_details = self.source_info(relative)
|
||||||
|
output_details = self.probe(output)
|
||||||
|
if output_details.get("dolby_vision"):
|
||||||
|
self.sanitize_cache_metadata(relative, output)
|
||||||
|
manifest.pop("refresh_version", None)
|
||||||
|
sanitized += 1
|
||||||
|
self.validate_output(source, output, source_details.get("duration"))
|
||||||
|
manifest["encode_profile"] = ENCODE_PROFILE
|
||||||
|
manifest["migration_version"] = 4
|
||||||
|
manifest["migrated_at"] = time.time()
|
||||||
|
atomic_json(manifest_path, manifest)
|
||||||
|
migrated += 1
|
||||||
|
self.save()
|
||||||
|
log(f"Manifest migration complete: migrated={migrated}, sanitized={sanitized}, skipped={skipped}")
|
||||||
|
|
||||||
def encode(self, relative, priority):
|
def encode(self, relative, priority):
|
||||||
source = SOURCE_ROOT / relative
|
source = SOURCE_ROOT / relative
|
||||||
if not source.is_file() or self.is_locked(relative):
|
if not source.is_file() or self.is_locked(relative):
|
||||||
@@ -588,13 +736,13 @@ class Manager:
|
|||||||
output, manifest_path = self.cache_paths(relative)
|
output, manifest_path = self.cache_paths(relative)
|
||||||
partial = output.with_name(output.name + ".partial")
|
partial = output.with_name(output.name + ".partial")
|
||||||
partial.unlink(missing_ok=True)
|
partial.unlink(missing_ok=True)
|
||||||
info = self.state["inventory"].get(relative) or {}
|
info = self.source_info(relative)
|
||||||
estimate = min(max(int(source.stat().st_size * 2.2), 1_000_000_000), 100_000_000_000)
|
estimate = min(max(int(source.stat().st_size * 2.2), 1_000_000_000), 100_000_000_000)
|
||||||
if not self.ensure_space(estimate, may_evict=priority):
|
if not self.ensure_space(estimate, may_evict=priority):
|
||||||
return "full"
|
return "full"
|
||||||
|
|
||||||
log(f"Encoding {relative}")
|
log(f"Encoding {relative}")
|
||||||
command = self.ffmpeg_command(source, partial)
|
command = self.ffmpeg_command(source, partial, info)
|
||||||
log_path = STATE_ROOT / "ffmpeg.log"
|
log_path = STATE_ROOT / "ffmpeg.log"
|
||||||
with log_path.open("ab", buffering=0) as ffmpeg_log:
|
with log_path.open("ab", buffering=0) as ffmpeg_log:
|
||||||
self.process = subprocess.Popen(
|
self.process = subprocess.Popen(
|
||||||
@@ -633,6 +781,7 @@ class Manager:
|
|||||||
"relative": relative,
|
"relative": relative,
|
||||||
"source_fingerprint": fingerprint(source),
|
"source_fingerprint": fingerprint(source),
|
||||||
"created_at": time.time(),
|
"created_at": time.time(),
|
||||||
|
"encode_profile": ENCODE_PROFILE,
|
||||||
})
|
})
|
||||||
if self.is_locked(relative):
|
if self.is_locked(relative):
|
||||||
output.unlink(missing_ok=True)
|
output.unlink(missing_ok=True)
|
||||||
@@ -649,6 +798,7 @@ class Manager:
|
|||||||
if not JELLYFIN_API_KEY:
|
if not JELLYFIN_API_KEY:
|
||||||
raise RuntimeError("JELLYFIN_API_KEY is required")
|
raise RuntimeError("JELLYFIN_API_KEY is required")
|
||||||
self.read_sessions()
|
self.read_sessions()
|
||||||
|
self.purge_incompatible_cache_profiles()
|
||||||
self.enforce_cache_limit()
|
self.enforce_cache_limit()
|
||||||
self.scan_inventory()
|
self.scan_inventory()
|
||||||
self.sync_view()
|
self.sync_view()
|
||||||
@@ -657,6 +807,7 @@ class Manager:
|
|||||||
if not self.read_sessions():
|
if not self.read_sessions():
|
||||||
time.sleep(POLL_SECONDS)
|
time.sleep(POLL_SECONDS)
|
||||||
continue
|
continue
|
||||||
|
self.purge_incompatible_cache_profiles()
|
||||||
self.enforce_cache_limit()
|
self.enforce_cache_limit()
|
||||||
now = time.time()
|
now = time.time()
|
||||||
if now - self.last_scan >= RESCAN_SECONDS:
|
if now - self.last_scan >= RESCAN_SECONDS:
|
||||||
@@ -706,6 +857,15 @@ class Manager:
|
|||||||
return True
|
return True
|
||||||
return bool(self.active_items)
|
return bool(self.active_items)
|
||||||
|
|
||||||
|
def purge(self, force=False):
|
||||||
|
if not force and self.active():
|
||||||
|
raise RuntimeError("cache purge refused while media is playing")
|
||||||
|
for _, relative, _, _ in self.cache_entries():
|
||||||
|
log(f"Purging {relative}")
|
||||||
|
self.remove_cache(relative, force=force)
|
||||||
|
self.sync_view()
|
||||||
|
self.save()
|
||||||
|
|
||||||
def trim(self, target_bytes):
|
def trim(self, target_bytes):
|
||||||
if self.active():
|
if self.active():
|
||||||
raise RuntimeError("cache resize refused while media is playing")
|
raise RuntimeError("cache resize refused while media is playing")
|
||||||
@@ -736,6 +896,10 @@ def main():
|
|||||||
raise SystemExit(0 if not manager.active() else 1)
|
raise SystemExit(0 if not manager.active() else 1)
|
||||||
elif command == "trim" and len(sys.argv) == 3:
|
elif command == "trim" and len(sys.argv) == 3:
|
||||||
manager.trim(int(sys.argv[2]))
|
manager.trim(int(sys.argv[2]))
|
||||||
|
elif command == "purge":
|
||||||
|
manager.purge(force="--force" in sys.argv[2:])
|
||||||
|
elif command == "migrate-v4":
|
||||||
|
manager.migrate_v4()
|
||||||
else:
|
else:
|
||||||
raise SystemExit(f"unknown command: {command}")
|
raise SystemExit(f"unknown command: {command}")
|
||||||
|
|
||||||
|
|||||||
+14
-4
@@ -14,17 +14,27 @@ docker run --rm --entrypoint ffmpeg -v "$ROOT/source:/source" "$IMAGE" \
|
|||||||
-hide_banner -loglevel error -f lavfi -i testsrc2=size=1280x720:rate=24 \
|
-hide_banner -loglevel error -f lavfi -i testsrc2=size=1280x720:rate=24 \
|
||||||
-t 6 -c:v libx265 -pix_fmt yuv420p10le -an /source/movies/Test/Test.mkv
|
-t 6 -c:v libx265 -pix_fmt yuv420p10le -an /source/movies/Test/Test.mkv
|
||||||
|
|
||||||
|
docker run --rm --entrypoint ffmpeg -v "$ROOT/source:/source" "$IMAGE" \
|
||||||
|
-hide_banner -loglevel error -f lavfi -i testsrc2=size=1280x720:rate=24 \
|
||||||
|
-t 6 -c:v libx265 -pix_fmt yuv420p10le \
|
||||||
|
-color_primaries bt2020 -color_trc smpte2084 -colorspace bt2020nc \
|
||||||
|
-x265-params 'colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc' \
|
||||||
|
-an /source/movies/Test/Test.HDR10.mkv
|
||||||
|
|
||||||
docker run --rm --network "$NETWORK" --entrypoint python3 \
|
docker run --rm --network "$NETWORK" --entrypoint python3 \
|
||||||
-e SOURCE_ROOT=/source -e VIEW_ROOT=/view -e CACHE_ROOT=/cache -e STATE_ROOT=/state \
|
-e SOURCE_ROOT=/source -e VIEW_ROOT=/view -e CACHE_ROOT=/cache -e STATE_ROOT=/state \
|
||||||
-e JELLYFIN_URL="$JELLYFIN_URL" -e JELLYFIN_API_KEY \
|
-e JELLYFIN_URL="$JELLYFIN_URL" -e JELLYFIN_API_KEY \
|
||||||
-e JELLYFIN_LIBRARY_ROOT=/data/library -e JELLYFIN_ORIGINAL_ROOT=/data/original \
|
-e JELLYFIN_LIBRARY_ROOT=/data/library -e JELLYFIN_ORIGINAL_ROOT=/data/original \
|
||||||
-e JELLYFIN_CACHE_ROOT=/data/cache \
|
-e JELLYFIN_CACHE_ROOT=/data/cache \
|
||||||
-v "$ROOT/source:/source:ro" -v "$ROOT/view:/view" -v "$ROOT/cache:/cache" -v "$ROOT/state:/state" \
|
-v "$ROOT/source:/source:ro" -v "$ROOT/view:/view" -v "$ROOT/cache:/cache" -v "$ROOT/state:/state" \
|
||||||
"$IMAGE" -c 'import importlib.machinery; m=importlib.machinery.SourceFileLoader("p","/app/pretranscode").load_module(); x=m.Manager(); assert x.read_sessions(); x.scan_inventory(); x.sync_view(); assert x.encode("movies/Test/Test.mkv",False)=="completed"'
|
"$IMAGE" -c 'import importlib.machinery; m=importlib.machinery.SourceFileLoader("p","/app/pretranscode").load_module(); x=m.Manager(); assert x.read_sessions(); x.scan_inventory(); x.sync_view(); assert x.encode("movies/Test/Test.mkv",False)=="completed"; assert x.encode("movies/Test/Test.HDR10.mkv",False)=="completed"'
|
||||||
|
|
||||||
OUTPUT=$(find "$ROOT/cache/files" -name '*.mkv' ! -name '*.partial' | head -1)
|
SDR_TARGET=$(readlink "$ROOT/view/movies/Test/Test.mkv")
|
||||||
|
HDR_TARGET=$(readlink "$ROOT/view/movies/Test/Test.HDR10.mkv")
|
||||||
docker run --rm --entrypoint ffprobe -v "$ROOT/cache:/cache:ro" "$IMAGE" \
|
docker run --rm --entrypoint ffprobe -v "$ROOT/cache:/cache:ro" "$IMAGE" \
|
||||||
-v error -select_streams v:0 -show_entries stream=codec_name,pix_fmt \
|
-v error -select_streams v:0 -show_entries stream=codec_name,pix_fmt \
|
||||||
-of default=nw=1 "/cache/files/$(basename "$OUTPUT")" | grep -qx 'codec_name=h264'
|
-of default=nw=1 "${SDR_TARGET/\/data\/cache/\/cache}" | grep -qx 'codec_name=h264'
|
||||||
test "$(readlink "$ROOT/view/movies/Test/Test.mkv")" = "/data/cache/files/$(basename "$OUTPUT")"
|
docker run --rm --entrypoint ffprobe -v "$ROOT/cache:/cache:ro" "$IMAGE" \
|
||||||
|
-v error -select_streams v:0 -show_entries stream=color_transfer,color_primaries,color_space \
|
||||||
|
-of default=nw=1 "${HDR_TARGET/\/data\/cache/\/cache}" | grep -qx 'color_transfer=bt709'
|
||||||
echo "integration test passed"
|
echo "integration test passed"
|
||||||
|
|||||||
+76
-2
@@ -4,6 +4,8 @@ import os
|
|||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
|
||||||
class CoreTests(unittest.TestCase):
|
class CoreTests(unittest.TestCase):
|
||||||
@@ -50,6 +52,50 @@ class CoreTests(unittest.TestCase):
|
|||||||
self.assertFalse(manager.incompatible({"codec": "h264", "bit_depth": 8}))
|
self.assertFalse(manager.incompatible({"codec": "h264", "bit_depth": 8}))
|
||||||
self.assertFalse(manager.incompatible({"codec": "av1", "bit_depth": 10}))
|
self.assertFalse(manager.incompatible({"codec": "av1", "bit_depth": 10}))
|
||||||
|
|
||||||
|
def test_quality_filters_are_global(self):
|
||||||
|
manager = self.module.Manager()
|
||||||
|
sdr = manager.ffmpeg_command(Path("show.mkv"), Path("out.mkv.partial"), {})
|
||||||
|
hdr = manager.ffmpeg_command(
|
||||||
|
Path("show.HDR10.mkv"), Path("out.mkv.partial"), {"color_transfer": "smpte2084"}
|
||||||
|
)
|
||||||
|
self.assertTrue(any("zscale=dither=error_diffusion,format=yuv420p" in value for value in sdr))
|
||||||
|
self.assertIn("-filter_complex", sdr)
|
||||||
|
self.assertIn("[video]", sdr)
|
||||||
|
self.assertNotIn("-vf", sdr)
|
||||||
|
self.assertIn("18", sdr)
|
||||||
|
self.assertTrue(any("tonemap=tonemap=hable" in value for value in hdr))
|
||||||
|
self.assertIn("bt709", hdr)
|
||||||
|
|
||||||
|
def test_probe_detects_dolby_vision_stream_metadata(self):
|
||||||
|
manager = self.module.Manager()
|
||||||
|
result = SimpleNamespace(
|
||||||
|
returncode=0,
|
||||||
|
stderr="",
|
||||||
|
stdout=json.dumps({
|
||||||
|
"streams": [{
|
||||||
|
"codec_name": "h264",
|
||||||
|
"profile": "High",
|
||||||
|
"pix_fmt": "yuv420p",
|
||||||
|
"bits_per_raw_sample": "8",
|
||||||
|
"color_transfer": "bt709",
|
||||||
|
"side_data_list": [{"side_data_type": "DOVI configuration record"}],
|
||||||
|
}],
|
||||||
|
"format": {"duration": "10.0"},
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
with mock.patch.object(self.module.subprocess, "run", return_value=result):
|
||||||
|
details = manager.probe(Path("cached.mkv"))
|
||||||
|
self.assertTrue(details["dolby_vision"])
|
||||||
|
|
||||||
|
def test_validation_rejects_inherited_dolby_vision_metadata(self):
|
||||||
|
manager = self.module.Manager()
|
||||||
|
output = self.root / "cache" / "invalid.mkv"
|
||||||
|
output.write_bytes(b"x" * 1_000_001)
|
||||||
|
details = {"codec": "h264", "bit_depth": 8, "dolby_vision": True, "duration": 10.0}
|
||||||
|
with mock.patch.object(manager, "probe", return_value=details):
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "Dolby Vision"):
|
||||||
|
manager.validate_output(Path("source.mkv"), output, 10.0)
|
||||||
|
|
||||||
def test_cache_requires_matching_source_fingerprint(self):
|
def test_cache_requires_matching_source_fingerprint(self):
|
||||||
m = self.module
|
m = self.module
|
||||||
manager = m.Manager()
|
manager = m.Manager()
|
||||||
@@ -59,11 +105,35 @@ class CoreTests(unittest.TestCase):
|
|||||||
source.write_bytes(b"original")
|
source.write_bytes(b"original")
|
||||||
output, manifest = manager.cache_paths(relative)
|
output, manifest = manager.cache_paths(relative)
|
||||||
output.write_bytes(b"cached")
|
output.write_bytes(b"cached")
|
||||||
manifest.write_text(json.dumps({"relative": relative, "source_fingerprint": m.fingerprint(source)}))
|
manifest.write_text(json.dumps({
|
||||||
|
"relative": relative,
|
||||||
|
"source_fingerprint": m.fingerprint(source),
|
||||||
|
"encode_profile": m.ENCODE_PROFILE,
|
||||||
|
}))
|
||||||
self.assertTrue(manager.cache_valid(relative))
|
self.assertTrue(manager.cache_valid(relative))
|
||||||
source.write_bytes(b"changed")
|
source.write_bytes(b"changed")
|
||||||
self.assertFalse(manager.cache_valid(relative))
|
self.assertFalse(manager.cache_valid(relative))
|
||||||
|
|
||||||
|
def test_lru_touches_only_manifest(self):
|
||||||
|
m = self.module
|
||||||
|
manager = m.Manager()
|
||||||
|
relative = "movies/LRU/LRU.mkv"
|
||||||
|
source = self.root / "source" / relative
|
||||||
|
source.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
source.write_bytes(b"original")
|
||||||
|
output, manifest = manager.cache_paths(relative)
|
||||||
|
output.write_bytes(b"cached")
|
||||||
|
manifest.write_text(json.dumps({
|
||||||
|
"relative": relative,
|
||||||
|
"source_fingerprint": m.fingerprint(source),
|
||||||
|
"encode_profile": m.ENCODE_PROFILE,
|
||||||
|
}))
|
||||||
|
os.utime(output, (100, 100))
|
||||||
|
os.utime(manifest, (100, 100))
|
||||||
|
manager.touch_cache(relative)
|
||||||
|
self.assertEqual(output.stat().st_mtime, 100)
|
||||||
|
self.assertGreater(manifest.stat().st_mtime, 100)
|
||||||
|
|
||||||
def test_smaller_cache_limit_evicts_oldest_content(self):
|
def test_smaller_cache_limit_evicts_oldest_content(self):
|
||||||
m = self.module
|
m = self.module
|
||||||
manager = m.Manager()
|
manager = m.Manager()
|
||||||
@@ -73,7 +143,11 @@ class CoreTests(unittest.TestCase):
|
|||||||
source.write_bytes(b"original")
|
source.write_bytes(b"original")
|
||||||
output, manifest = manager.cache_paths(relative)
|
output, manifest = manager.cache_paths(relative)
|
||||||
output.write_bytes(b"cached rendition")
|
output.write_bytes(b"cached rendition")
|
||||||
manifest.write_text(json.dumps({"relative": relative, "source_fingerprint": m.fingerprint(source)}))
|
manifest.write_text(json.dumps({
|
||||||
|
"relative": relative,
|
||||||
|
"source_fingerprint": m.fingerprint(source),
|
||||||
|
"encode_profile": m.ENCODE_PROFILE,
|
||||||
|
}))
|
||||||
manager.ensure_link(relative, m.cache_target(relative))
|
manager.ensure_link(relative, m.cache_target(relative))
|
||||||
old_limit = m.CACHE_MAX_BYTES
|
old_limit = m.CACHE_MAX_BYTES
|
||||||
try:
|
try:
|
||||||
|
|||||||
Reference in New Issue
Block a user