Replace stateful cache with atomic C controller
This commit is contained in:
SpiderFish
+128
-29
@@ -1,40 +1,139 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
: "${JELLYFIN_API_KEY:?}"
|
||||
IMAGE=${IMAGE:-pretranscode:test}
|
||||
NETWORK=${MEDIA_NETWORK:-media-internal}
|
||||
JELLYFIN_URL=${JELLYFIN_URL:-http://jellyfin:8096}
|
||||
IMAGE=${IMAGE:-localhost/pretranscode-c-review}
|
||||
ROOT=$(mktemp -d)
|
||||
trap 'rm -rf "$ROOT"' EXIT
|
||||
mkdir -p "$ROOT"/{source/movies/Test,view,cache,state}
|
||||
PORT=$(python3 - <<'PY'
|
||||
import socket
|
||||
s=socket.socket(); s.bind(('127.0.0.1', 0)); print(s.getsockname()[1]); s.close()
|
||||
PY
|
||||
)
|
||||
PID=
|
||||
CONTAINER=
|
||||
cleanup() {
|
||||
[ -z "$CONTAINER" ] || podman rm -f "$CONTAINER" >/dev/null 2>&1 || true
|
||||
[ -z "$PID" ] || kill "$PID" >/dev/null 2>&1 || true
|
||||
rm -rf "$ROOT"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
mkdir -p "$ROOT"/{source/movies/Test,view,cache/files,state}
|
||||
touch "$ROOT/cache/.pretranscode-cache"
|
||||
printf partial >"$ROOT/cache/files/failed.partial"
|
||||
printf orphan >"$ROOT/cache/files/orphan.bin"
|
||||
|
||||
docker build -q -t "$IMAGE" . >/dev/null
|
||||
docker run --rm --entrypoint ffmpeg -v "$ROOT/source:/source" "$IMAGE" \
|
||||
cat >"$ROOT/source/movies/Test/embedded.srt" <<'EOF'
|
||||
1
|
||||
00:00:00,000 --> 00:00:05,000
|
||||
Compatibility test
|
||||
EOF
|
||||
|
||||
podman run --rm --userns=keep-id --entrypoint ffmpeg -v "$ROOT/source:/source:Z" "$IMAGE" \
|
||||
-hide_banner -loglevel error -f lavfi -i testsrc2=size=1280x720:rate=24 \
|
||||
-t 6 -c:v libx265 -pix_fmt yuv420p10le -an /source/movies/Test/Test.mkv
|
||||
-f lavfi -i sine=frequency=440:sample_rate=48000 \
|
||||
-f srt -i /source/movies/Test/embedded.srt -t 6 \
|
||||
-map 0:v:0 -map 1:a:0 -map 2:s:0 -c:v libx265 -pix_fmt yuv420p10le \
|
||||
-c:a eac3 -c:s srt /source/movies/Test/Test.mkv
|
||||
|
||||
docker run --rm --entrypoint ffmpeg -v "$ROOT/source:/source" "$IMAGE" \
|
||||
-hide_banner -loglevel error -f lavfi -i testsrc2=size=1280x720:rate=24 \
|
||||
-t 6 -c:v libx265 -pix_fmt yuv420p10le \
|
||||
-color_primaries bt2020 -color_trc smpte2084 -colorspace bt2020nc \
|
||||
-x265-params 'colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc' \
|
||||
-an /source/movies/Test/Test.HDR10.mkv
|
||||
|
||||
docker run --rm --network "$NETWORK" --entrypoint python3 \
|
||||
TEST_VIEW="$ROOT/view" TEST_STATE="$ROOT/jellyfin-state.json" \
|
||||
python3 "$(dirname "$0")/fake_jellyfin.py" "$PORT" &
|
||||
PID=$!
|
||||
sleep 0.2
|
||||
CONTAINER=pretranscode-c-integration-$RANDOM
|
||||
podman run -d --userns=keep-id --name "$CONTAINER" \
|
||||
--add-host host.containers.internal:host-gateway \
|
||||
-e SOURCE_ROOT=/source -e VIEW_ROOT=/view -e CACHE_ROOT=/cache -e STATE_ROOT=/state \
|
||||
-e JELLYFIN_URL="$JELLYFIN_URL" -e JELLYFIN_API_KEY \
|
||||
-e JELLYFIN_URL="http://host.containers.internal:$PORT" -e JELLYFIN_API_KEY=test \
|
||||
-e JELLYFIN_LIBRARY_ROOT=/data/library -e JELLYFIN_ORIGINAL_ROOT=/data/original \
|
||||
-e JELLYFIN_CACHE_ROOT=/data/cache \
|
||||
-v "$ROOT/source:/source:ro" -v "$ROOT/view:/view" -v "$ROOT/cache:/cache" -v "$ROOT/state:/state" \
|
||||
"$IMAGE" -c 'import importlib.machinery; m=importlib.machinery.SourceFileLoader("p","/app/pretranscode").load_module(); x=m.Manager(); assert x.read_sessions(); x.scan_inventory(); x.sync_view(); assert x.encode("movies/Test/Test.mkv",False)=="completed"; assert x.encode("movies/Test/Test.HDR10.mkv",False)=="completed"'
|
||||
-e JELLYFIN_CACHE_ROOT=/data/cache -e POLL_SECONDS=1 -e RESCAN_SECONDS=1 \
|
||||
-e H264_PRESET=ultrafast \
|
||||
-v "$ROOT/source:/source:ro,Z" -v "$ROOT/view:/view:Z" \
|
||||
-v "$ROOT/cache:/cache:Z" -v "$ROOT/state:/state:Z" "$IMAGE" >/dev/null
|
||||
|
||||
SDR_TARGET=$(readlink "$ROOT/view/movies/Test/Test.mkv")
|
||||
HDR_TARGET=$(readlink "$ROOT/view/movies/Test/Test.HDR10.mkv")
|
||||
docker run --rm --entrypoint ffprobe -v "$ROOT/cache:/cache:ro" "$IMAGE" \
|
||||
-v error -select_streams v:0 -show_entries stream=codec_name,pix_fmt \
|
||||
-of default=nw=1 "${SDR_TARGET/\/data\/cache/\/cache}" | grep -qx 'codec_name=h264'
|
||||
docker run --rm --entrypoint ffprobe -v "$ROOT/cache:/cache:ro" "$IMAGE" \
|
||||
-v error -select_streams v:0 -show_entries stream=color_transfer,color_primaries,color_space \
|
||||
-of default=nw=1 "${HDR_TARGET/\/data\/cache/\/cache}" | grep -qx 'color_transfer=bt709'
|
||||
for _ in $(seq 1 90); do
|
||||
LINK=$(readlink "$ROOT/view/movies/Test/Test.mkv" 2>/dev/null || true)
|
||||
case "$LINK" in
|
||||
/data/cache/files/*) break ;;
|
||||
esac
|
||||
if [ "$(podman inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != true ]; then
|
||||
podman logs "$CONTAINER" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
LINK=$(readlink "$ROOT/view/movies/Test/Test.mkv" 2>/dev/null || true)
|
||||
if [[ "$LINK" != /data/cache/files/* ]]; then
|
||||
podman logs "$CONTAINER" >&2 || true
|
||||
cat "$ROOT/state/ffmpeg.log" >&2 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
[[ "$LINK" == /data/cache/files/* ]]
|
||||
BASENAME=${LINK##*/}
|
||||
[[ "$BASENAME" =~ ^[0-9a-f]{32}\.[0-9a-f]{32}\.[0-9a-f]{32}\.[0-9a-f]{32}\.(mkv|mp4)$ ]]
|
||||
OUTPUT="$ROOT/cache/${LINK#/data/cache/}"
|
||||
test -s "$OUTPUT"
|
||||
! find "$ROOT/cache/files" -type f \( -name '*.json' -o -name '*.partial' \) | grep -q .
|
||||
test ! -e "$ROOT/cache/files/orphan.bin"
|
||||
podman run --rm --userns=keep-id --entrypoint ffprobe -v "$ROOT/cache:/cache:ro,Z" "$IMAGE" \
|
||||
-v error -show_entries stream=codec_type,codec_name,pix_fmt,channels:stream_side_data=side_data_type \
|
||||
-show_entries chapter=id -of json "/cache/${LINK#/data/cache/}" | python3 -c '
|
||||
import json,sys
|
||||
d=json.load(sys.stdin); streams=d.get("streams",[])
|
||||
video=[s for s in streams if s.get("codec_type")=="video"]
|
||||
audio=[s for s in streams if s.get("codec_type")=="audio"]
|
||||
subs=[s for s in streams if s.get("codec_type")=="subtitle"]
|
||||
assert len(video)==1 and video[0].get("codec_name")=="h264" and video[0].get("pix_fmt")=="yuv420p"
|
||||
assert len(audio)==1 and audio[0].get("codec_name")=="aac" and audio[0].get("channels")==2
|
||||
assert not subs and not d.get("chapters")
|
||||
assert not any("dovi" in x.get("side_data_type","").lower() for x in video[0].get("side_data_list",[]))
|
||||
'
|
||||
python3 - "$ROOT/jellyfin-state.json" <<'PY'
|
||||
import json, sys
|
||||
state=json.load(open(sys.argv[1]))
|
||||
assert state["codec"] == "h264" and state["depth"] == 8
|
||||
assert state["attempts"] >= 2
|
||||
PY
|
||||
|
||||
# A profile change, including a published 1.x object, keeps the old object live
|
||||
# until its replacement exists.
|
||||
podman stop -t 10 "$CONTAINER" >/dev/null
|
||||
podman rm "$CONTAINER" >/dev/null
|
||||
CONTAINER=
|
||||
mv "$OUTPUT" "$ROOT/cache/files/legacy.mkv"
|
||||
ln -sfn /data/cache/files/legacy.mkv "$ROOT/view/movies/Test/Test.mkv"
|
||||
OLD_LINK=/data/cache/files/legacy.mkv
|
||||
podman run --rm --userns=keep-id \
|
||||
--add-host host.containers.internal:host-gateway \
|
||||
-e SOURCE_ROOT=/source -e VIEW_ROOT=/view -e CACHE_ROOT=/cache -e STATE_ROOT=/state \
|
||||
-e JELLYFIN_URL="http://host.containers.internal:$PORT" -e JELLYFIN_API_KEY=test \
|
||||
-e JELLYFIN_LIBRARY_ROOT=/data/library -e JELLYFIN_ORIGINAL_ROOT=/data/original \
|
||||
-e JELLYFIN_CACHE_ROOT=/data/cache -e POLL_SECONDS=1 -e RESCAN_SECONDS=1 \
|
||||
-e H264_PRESET=ultrafast -e H264_CRF=19 \
|
||||
-v "$ROOT/source:/source:ro,Z" -v "$ROOT/view:/view:Z" \
|
||||
-v "$ROOT/cache:/cache:Z" -v "$ROOT/state:/state:Z" "$IMAGE" sync
|
||||
[[ "$(readlink "$ROOT/view/movies/Test/Test.mkv")" == "$OLD_LINK" ]]
|
||||
test -s "$ROOT/cache/${OLD_LINK#/data/cache/}"
|
||||
|
||||
CONTAINER=pretranscode-c-integration-$RANDOM
|
||||
podman run -d --userns=keep-id --name "$CONTAINER" \
|
||||
--add-host host.containers.internal:host-gateway \
|
||||
-e SOURCE_ROOT=/source -e VIEW_ROOT=/view -e CACHE_ROOT=/cache -e STATE_ROOT=/state \
|
||||
-e JELLYFIN_URL="http://host.containers.internal:$PORT" -e JELLYFIN_API_KEY=test \
|
||||
-e JELLYFIN_LIBRARY_ROOT=/data/library -e JELLYFIN_ORIGINAL_ROOT=/data/original \
|
||||
-e JELLYFIN_CACHE_ROOT=/data/cache -e POLL_SECONDS=1 -e RESCAN_SECONDS=1 \
|
||||
-e H264_PRESET=ultrafast -e H264_CRF=19 \
|
||||
-v "$ROOT/source:/source:ro,Z" -v "$ROOT/view:/view:Z" \
|
||||
-v "$ROOT/cache:/cache:Z" -v "$ROOT/state:/state:Z" "$IMAGE" >/dev/null
|
||||
for _ in $(seq 1 90); do
|
||||
LINK=$(readlink "$ROOT/view/movies/Test/Test.mkv" 2>/dev/null || true)
|
||||
[[ "$LINK" == /data/cache/files/* && "$LINK" != "$OLD_LINK" ]] && break
|
||||
sleep 1
|
||||
done
|
||||
[[ "$LINK" == /data/cache/files/* && "$LINK" != "$OLD_LINK" ]]
|
||||
for _ in $(seq 1 10); do
|
||||
test ! -e "$ROOT/cache/${OLD_LINK#/data/cache/}" && break
|
||||
sleep 1
|
||||
done
|
||||
test ! -e "$ROOT/cache/${OLD_LINK#/data/cache/}"
|
||||
podman stop -t 10 "$CONTAINER" >/dev/null
|
||||
podman rm "$CONTAINER" >/dev/null
|
||||
CONTAINER=
|
||||
echo "integration test passed"
|
||||
|
||||
Reference in New Issue
Block a user