music-assistant-server

1"""Smart Fades - Audio fade implementations."""
2
3from __future__ import annotations
4
5import asyncio
6import logging
7from abc import ABC, abstractmethod
8from collections.abc import AsyncGenerator
9from contextlib import suppress
10from typing import TYPE_CHECKING
11
12import aiofiles
13import shortuuid
14
15from music_assistant.constants import VERBOSE_LOG_LEVEL
16from music_assistant.controllers.streams.smart_fades.filters import CrossfadeFilter, Filter
17from music_assistant.controllers.streams.smart_fades.helpers import SMART_CROSSFADE_DURATION
18from music_assistant.controllers.streams.smart_fades.models import (
19    CrossfadeTimingInfo,
20    SmartFadeNotApplicable,
21    TransitionPlan,
22)
23from music_assistant.controllers.streams.smart_fades.planner import SmartCrossFadePlanner
24from music_assistant.controllers.streams.smart_fades.renderer import TransitionRenderer
25from music_assistant.helpers.audio import iter_pcm_slices
26from music_assistant.helpers.ffmpeg import get_ffmpeg_channel_args
27from music_assistant.helpers.process import AsyncProcess
28from music_assistant.helpers.util import remove_file
29
30if TYPE_CHECKING:
31    from music_assistant_models.media_items import AudioFormat
32
33    from music_assistant.models.audio_analysis import AudioAnalysisData
34
35__all__ = [
36    "CrossfadeTimingInfo",
37    "SmartCrossFade",
38    "SmartFade",
39    "SmartFadeNotApplicable",
40    "StandardCrossFade",
41]
42
43
44class SmartFade(ABC):
45    """Abstract base class for Smart Fades."""
46
47    filters: list[Filter]
48    timing_info: CrossfadeTimingInfo
49
50    def __init__(self, logger: logging.Logger) -> None:
51        """Initialize SmartFade base class."""
52        self.filters = []
53        self.logger = logger
54
55    @abstractmethod
56    def build(
57        self,
58        fade_out_bytes_len: int,
59        fade_in_bytes_len: int,
60        pcm_format: AudioFormat,
61    ) -> None:
62        """
63        Build the filter chain and assign ``self.timing_info``.
64
65        Must be called once before ``apply()``.
66
67        :param fade_out_bytes_len: Length in bytes of the outgoing track's tail buffer.
68        :param fade_in_bytes_len: Length in bytes of the incoming track's head buffer.
69        :param pcm_format: Audio format of both input buffers.
70        """
71        ...
72
73    async def apply(
74        self,
75        fade_out_part: bytes,
76        fade_in_part: bytes | AsyncGenerator[bytes],
77        pcm_format: AudioFormat,
78    ) -> AsyncGenerator[bytes]:
79        """
80        Apply the smart fade, yielding PCM audio chunks as they become available.
81
82        :param fade_out_part: Raw PCM bytes for the outgoing track's tail.
83        :param fade_in_part: Raw PCM bytes or async generator for the incoming track's head.
84        :param pcm_format: Audio format of both input parts and the output.
85        """
86        # Write the fade_out_part to a temporary file
87        fadeout_filename = f"/tmp/{shortuuid.random(20)}.pcm"  # noqa: S108
88        async with aiofiles.open(fadeout_filename, "wb") as outfile:
89            await outfile.write(fade_out_part)
90
91        args = [
92            "ffmpeg",
93            "-hide_banner",
94            "-loglevel",
95            "error",
96            # Input 1: fadeout part (as file)
97            "-acodec",
98            pcm_format.content_type.name.lower(),  # e.g., "pcm_f32le" not just "f32le"
99            *get_ffmpeg_channel_args(pcm_format),
100            "-ar",
101            str(pcm_format.sample_rate),
102            "-f",
103            pcm_format.content_type.value,
104            "-i",
105            fadeout_filename,
106            # Input 2: fade_in part (stdin)
107            "-acodec",
108            pcm_format.content_type.name.lower(),
109            *get_ffmpeg_channel_args(pcm_format),
110            "-ar",
111            str(pcm_format.sample_rate),
112            "-f",
113            pcm_format.content_type.value,
114            "-i",
115            "-",
116        ]
117        smart_fade_filters = self._get_ffmpeg_filters()
118        self.logger.debug(
119            "Applying smartfade: %s",
120            self,
121        )
122        args.extend(
123            [
124                "-filter_complex",
125                ";".join(smart_fade_filters),
126                # Output format specification - must match input codec format
127                "-acodec",
128                pcm_format.content_type.name.lower(),
129                *get_ffmpeg_channel_args(pcm_format),
130                "-ar",
131                str(pcm_format.sample_rate),
132                "-f",
133                pcm_format.content_type.value,
134                "-",
135            ]
136        )
137        self.logger.log(VERBOSE_LOG_LEVEL, "FFmpeg command args: %s", " ".join(args))
138
139        got_output = False
140        stderr_lines: list[str] = []
141        try:
142            proc = AsyncProcess(args, stdin=True, stdout=True, stderr=True, name="smartfade")
143            async with proc:
144
145                async def _feed_stdin() -> None:
146                    if isinstance(fade_in_part, bytes):
147                        await proc.write(fade_in_part)
148                    else:
149                        async for fade_chunk in fade_in_part:
150                            await proc.write(fade_chunk)
151                    await proc.write_eof()
152
153                async def _drain_stderr() -> None:
154                    """Read stderr to prevent pipe deadlock."""
155                    async for line in proc.iter_stderr():
156                        stderr_lines.append(line)
157
158                feed_task = asyncio.create_task(_feed_stdin())
159                stderr_task = asyncio.create_task(_drain_stderr())
160                try:
161                    async for chunk in proc.iter_any():
162                        got_output = True
163                        yield chunk
164                finally:
165                    if not feed_task.done():
166                        feed_task.cancel()
167                    with suppress(asyncio.CancelledError):
168                        await feed_task
169                    # Bounded wait on stderr_task so its output is still captured
170                    # for error reporting on the happy/error paths, but we don't
171                    # hang on consumer abort — ffmpeg is still alive then and
172                    # stderr won't EOF until proc.close() closes stdin, which
173                    # only runs via the async-with __aexit__ *after* this finally.
174                    # wait_for cancels stderr_task on timeout so cleanup proceeds.
175                    with suppress(TimeoutError, asyncio.CancelledError):
176                        await asyncio.wait_for(stderr_task, timeout=2)
177
178            if proc.returncode != 0:
179                stderr_msg = "; ".join(stderr_lines) if stderr_lines else "(no stderr)"
180                raise RuntimeError(f"Crossfade FFmpeg failed (rc={proc.returncode}): {stderr_msg}")
181            if not got_output:
182                msg = "Crossfade FFmpeg produced no output"
183                if stderr_lines:
184                    msg += f": {'; '.join(stderr_lines)}"
185                raise RuntimeError(msg)
186        finally:
187            # Always cleanup temp file, even if ffmpeg fails
188            await remove_file(fadeout_filename)
189
190    def __repr__(self) -> str:
191        """Return string representation of SmartFade showing the filter chain."""
192        if not self.filters:
193            return f"<{self.__class__.__name__}: 0 filters>"
194
195        chain = " → ".join(repr(f) for f in self.filters)
196        return f"<{self.__class__.__name__}: {len(self.filters)} filters> {chain}"
197
198    def _get_ffmpeg_filters(
199        self,
200        input_fadein_label: str = "[1]",
201        input_fadeout_label: str = "[0]",
202    ) -> list[str]:
203        """Get FFmpeg filters for smart fades."""
204        if not self.filters:
205            raise RuntimeError("SmartFade not built — call Mixer.build() first")
206        filters = []
207        _cur_fadein_label = input_fadein_label
208        _cur_fadeout_label = input_fadeout_label
209        for audio_filter in self.filters:
210            filter_strings = audio_filter.apply(_cur_fadein_label, _cur_fadeout_label)
211            filters.extend(filter_strings)
212            _cur_fadein_label = f"[{audio_filter.output_fadein_label}]"
213            _cur_fadeout_label = f"[{audio_filter.output_fadeout_label}]"
214        return filters
215
216
217class SmartCrossFade(SmartFade):
218    """
219    Smart fades class that implements a Smart Fade mode.
220
221    Delegates the decision-making to a ``SmartCrossFadePlanner`` (pure, over the
222    stored analysis) and the filter/timing construction to a ``TransitionRenderer``.
223    Alternative transition strategies are siblings that swap in their own planner.
224    """
225
226    def __init__(
227        self,
228        logger: logging.Logger,
229        fade_out_analysis: AudioAnalysisData,
230        fade_in_analysis: AudioAnalysisData,
231    ) -> None:
232        """
233        Initialize SmartCrossFade with analysis data.
234
235        :param logger: Logger for debug output.
236        :param fade_out_analysis: Analysis data for the outgoing track.
237        :param fade_in_analysis: Analysis data for the incoming track.
238        """
239        super().__init__(logger)
240        self.fade_out_analysis = fade_out_analysis
241        self.fade_in_analysis = fade_in_analysis
242        self.planner = SmartCrossFadePlanner(logger)
243        self.renderer = TransitionRenderer(logger)
244        self.plan: TransitionPlan | None = None
245        # populated by build(); read by the timing/lyrics-sync tests
246        self.effective_end: float = float(SMART_CROSSFADE_DURATION)
247        self.tempo_steps: list[tuple[float, float]] = []
248
249    def build(
250        self,
251        fade_out_bytes_len: int,
252        fade_in_bytes_len: int,
253        pcm_format: AudioFormat,
254    ) -> None:
255        """Plan the transition, then render its filter chain and ``timing_info``."""
256        buffer_duration = min(
257            float(SMART_CROSSFADE_DURATION),
258            fade_out_bytes_len / pcm_format.pcm_sample_size,
259        )
260        self.plan = self.planner.plan(
261            self.fade_out_analysis, self.fade_in_analysis, buffer_duration
262        )
263        self.filters, self.timing_info = self.renderer.render(
264            self.plan, pcm_format, fade_in_bytes_len
265        )
266        # convenience copies for the timing/lyrics-sync tests
267        self.effective_end = self.plan.fade_out_window
268        self.tempo_steps = self.plan.tempo_plan.steps
269        self.fade_out_beats = self.planner.outgoing.beats
270
271
272class StandardCrossFade(SmartFade):
273    """Standard crossfade class that implements a standard crossfade mode."""
274
275    def __init__(
276        self,
277        logger: logging.Logger,
278        crossfade_duration: float = 10.0,
279        trailing_silence_bytes: int = 0,
280    ) -> None:
281        """
282        Initialize StandardCrossFade.
283
284        :param logger: Logger for debug output.
285        :param crossfade_duration: Length of the crossfade overlap in seconds.
286        :param trailing_silence_bytes: Trailing silence in the outgoing tail that
287            ``apply()`` slices off before crossfading.
288        """
289        super().__init__(logger)
290        self.crossfade_duration = crossfade_duration
291        self.trailing_silence_bytes = trailing_silence_bytes
292        self.crossfade_size: int = 0
293
294    def build(
295        self,
296        fade_out_bytes_len: int,
297        fade_in_bytes_len: int,
298        pcm_format: AudioFormat,
299    ) -> None:
300        """Build the standard crossfade filter chain and assign ``self.timing_info``."""
301        fade_out_seconds = fade_out_bytes_len / pcm_format.pcm_sample_size
302        fade_in_seconds = fade_in_bytes_len / pcm_format.pcm_sample_size
303        # clamp CF to fit shorter inputs (defensive — normally full buffers)
304        effective_cf = min(self.crossfade_duration, fade_out_seconds, fade_in_seconds)
305        # Quantize the overlap to a whole number of PCM frames and drive both the
306        # byte slice (in apply) and the acrossfade length from this one integer.
307        # apply slices the buffers on frame boundaries, so a fractional effective_cf
308        # leaves the rendered buffer a fraction of a sample short of the acrossfade
309        # duration — and acrossfade then silently produces no output at all.
310        frame_size = (pcm_format.bit_depth // 8) * pcm_format.channels
311        crossfade_bytes = int(pcm_format.pcm_sample_size * effective_cf)
312        self.crossfade_size = crossfade_bytes // frame_size * frame_size
313        crossfade_samples = self.crossfade_size // frame_size
314        effective_cf = self.crossfade_size / pcm_format.pcm_sample_size
315        self.timing_info = CrossfadeTimingInfo(
316            pre_crossfade_duration=max(0.0, fade_out_seconds - effective_cf),
317            crossfade_duration=effective_cf,
318            fadein_trimmed_duration=0.0,
319            post_crossfade_duration=max(0.0, fade_in_seconds - effective_cf),
320        )
321        self.filters = [
322            CrossfadeFilter(logger=self.logger, crossfade_samples=crossfade_samples),
323        ]
324
325    async def apply(
326        self,
327        fade_out_part: bytes,
328        fade_in_part: bytes | AsyncGenerator[bytes],
329        pcm_format: AudioFormat,
330    ) -> AsyncGenerator[bytes]:
331        """
332        Apply standard crossfade, yielding PCM audio chunks.
333
334        Only the overlapping portions are crossfaded, not the full buffers.
335        """
336        # crossfade_size legitimately ends up 0 for a silent/tiny buffer, so guard on
337        # the filter chain (set in build) to still fail fast on apply-before-build,
338        # consistent with SmartFade._get_ffmpeg_filters()
339        if not self.filters:
340            raise RuntimeError("SmartFade not built — call Mixer.build() first")
341        if self.trailing_silence_bytes:
342            fade_out_part = fade_out_part[: len(fade_out_part) - self.trailing_silence_bytes]
343        # frame-aligned overlap computed once in build, so it exactly matches the
344        # acrossfade `ns=` length the filter was built with
345        crossfade_size = self.crossfade_size
346        if crossfade_size == 0:
347            # nothing to blend — concatenate without spawning ffmpeg
348            for pcm_slice in iter_pcm_slices(fade_out_part, pcm_format, 1000):
349                yield pcm_slice
350            if isinstance(fade_in_part, bytes):
351                for pcm_slice in iter_pcm_slices(fade_in_part, pcm_format, 1000):
352                    yield pcm_slice
353            else:
354                async for chunk in fade_in_part:
355                    for pcm_slice in iter_pcm_slices(chunk, pcm_format, 1000):
356                        yield pcm_slice
357            return
358        # Pre-crossfade: outgoing track minus the crossfaded portion
359        split = len(fade_out_part) - crossfade_size
360        pre_crossfade = fade_out_part[:split]
361        adjusted_fade_out_part = fade_out_part[split:]
362
363        # Collect only the crossfade portion from fade_in, keep the rest as a generator
364        if isinstance(fade_in_part, bytes):
365            adjusted_fade_in_part = fade_in_part[:crossfade_size]
366            post_crossfade: bytes | AsyncGenerator[bytes] = fade_in_part[crossfade_size:]
367        else:
368            # read exactly crossfade_size bytes from the generator
369            buf = bytearray()
370            async for chunk in fade_in_part:
371                buf.extend(chunk)
372                if len(buf) >= crossfade_size:
373                    break
374            adjusted_fade_in_part = bytes(buf[:crossfade_size])
375            # anything beyond crossfade_size plus the remaining generator is post_crossfade
376            leftover = bytes(buf[crossfade_size:])
377
378            async def _post_crossfade() -> AsyncGenerator[bytes]:
379                if leftover:
380                    for pcm_slice in iter_pcm_slices(leftover, pcm_format, 1000):
381                        yield pcm_slice
382                async for remaining_chunk in fade_in_part:
383                    for pcm_slice in iter_pcm_slices(remaining_chunk, pcm_format, 1000):
384                        yield pcm_slice
385
386            post_crossfade = _post_crossfade()
387
388        # Yield pre-crossfade, crossfaded section, and post-crossfade
389        for pcm_slice in iter_pcm_slices(pre_crossfade, pcm_format, 1000):
390            yield pcm_slice
391        async for chunk in super().apply(adjusted_fade_out_part, adjusted_fade_in_part, pcm_format):
392            yield chunk
393        if isinstance(post_crossfade, bytes):
394            for pcm_slice in iter_pcm_slices(post_crossfade, pcm_format, 1000):
395                yield pcm_slice
396        else:
397            async for chunk in post_crossfade:
398                yield chunk
399