music-assistant-server
20.6 KB•PY
podcast_parsers.py
20.6 KB • 541 lines • python
1"""Podcastfeed -> Mass."""
2
3import logging
4from datetime import UTC, datetime
5from io import BytesIO
6from math import isfinite
7from typing import TYPE_CHECKING, Any
8
9import podcastparser
10from aiohttp.client import ClientError, ClientTimeout
11from music_assistant_models.enums import ContentType, ImageType, LinkType, MediaType
12from music_assistant_models.errors import MediaNotFoundError
13from music_assistant_models.media_items import (
14 AudioFormat,
15 ItemMapping,
16 MediaItemChapter,
17 MediaItemImage,
18 MediaItemLink,
19 Podcast,
20 PodcastEpisode,
21 ProviderMapping,
22 UniqueList,
23)
24
25if TYPE_CHECKING:
26 import aiohttp
27
28 from music_assistant.mass import MusicAssistant
29
30LOGGER = logging.getLogger(__name__)
31
32# best-effort enrichment must never stall episode resolution, so cap the chapter fetch
33_CHAPTERS_FETCH_TIMEOUT = ClientTimeout(total=10)
34
35# defaults for the parsed-feed cache shared by the podcast providers
36CACHE_CATEGORY_PODCAST_FEED = 0
37PODCAST_FEED_CACHE_EXPIRATION = 24 * 3600
38
39
40async def get_podcastparser_dict(
41 *, session: aiohttp.ClientSession, feed_url: str, max_episodes: int = 0
42) -> dict[str, Any]:
43 """
44 Get feed parsed by podcastparser by providing the url.
45
46 max_episodes = 0 does not limit the returned episodes.
47 """
48 feed_data: bytes | None = None
49 # without user agent, some feeds can not be retrieved
50 # https://github.com/music-assistant/support/issues/3596
51 # but, reports on discord show, that also the opposite may be true
52 for headers in [{"User-Agent": "Mozilla/5.0"}, {}]:
53 # raises ClientError on status failure
54 # ClientError is the base class of all possible Error, i.e. not authorized,
55 # url doesn't exist etc. The body is read inside the context manager, so a
56 # connection is never left open when a feed fails midway.
57 try:
58 async with session.get(feed_url, headers=headers, raise_for_status=True) as response:
59 feed_data = await response.read()
60 except ClientError:
61 continue
62 break
63 if feed_data is None:
64 # we did not get a single acceptable response
65 raise MediaNotFoundError(
66 f"Did not get acceptable response while trying to access {feed_url}."
67 )
68 feed_stream = BytesIO(feed_data)
69 try:
70 return podcastparser.parse(feed_url, feed_stream, max_episodes=max_episodes) # type: ignore[no-any-return]
71 except podcastparser.FeedParseError:
72 raise MediaNotFoundError(f"The url at {feed_url} returns invalid RSS data.")
73
74
75async def get_cached_podcast(
76 *,
77 mass: MusicAssistant,
78 provider_instance_id: str,
79 feed_url: str,
80 max_episodes: int = 0,
81 cache_category: int = CACHE_CATEGORY_PODCAST_FEED,
82 cache_expiration: int = PODCAST_FEED_CACHE_EXPIRATION,
83) -> dict[str, Any]:
84 """
85 Return a podcast's parsed feed, retrieving and caching it when not cached yet.
86
87 :param mass: The MusicAssistant instance holding the cache.
88 :param provider_instance_id: Provider instance the cache entry belongs to.
89 :param feed_url: The podcast's feed url, also used as the cache key.
90 :param max_episodes: Maximum number of episodes to parse, 0 for unlimited.
91 :param cache_category: Cache category to store the parsed feed under.
92 :param cache_expiration: Time in seconds the cached feed stays valid.
93 :raises MediaNotFoundError: If the feed could not be retrieved or parsed.
94 """
95 parsed_feed = await mass.cache.get(
96 key=feed_url,
97 provider=provider_instance_id,
98 category=cache_category,
99 default=None,
100 )
101 if parsed_feed is None:
102 return await refresh_cached_podcast(
103 mass=mass,
104 provider_instance_id=provider_instance_id,
105 feed_url=feed_url,
106 max_episodes=max_episodes,
107 cache_category=cache_category,
108 cache_expiration=cache_expiration,
109 )
110 # this is a dictionary from podcastparser
111 return parsed_feed # type: ignore[no-any-return]
112
113
114async def refresh_cached_podcast(
115 *,
116 mass: MusicAssistant,
117 provider_instance_id: str,
118 feed_url: str,
119 max_episodes: int = 0,
120 cache_category: int = CACHE_CATEGORY_PODCAST_FEED,
121 cache_expiration: int = PODCAST_FEED_CACHE_EXPIRATION,
122) -> dict[str, Any]:
123 """
124 Retrieve a podcast's feed and store it in the cache, replacing any cached copy.
125
126 Use this on the library sync path: the sync must always refresh the cached feed,
127 regardless of whether a (still valid) cache entry exists.
128
129 :param mass: The MusicAssistant instance holding the cache.
130 :param provider_instance_id: Provider instance the cache entry belongs to.
131 :param feed_url: The podcast's feed url, also used as the cache key.
132 :param max_episodes: Maximum number of episodes to parse, 0 for unlimited.
133 :param cache_category: Cache category to store the parsed feed under.
134 :param cache_expiration: Time in seconds the cached feed stays valid.
135 :raises MediaNotFoundError: If the feed could not be retrieved or parsed.
136 """
137 parsed_feed = await get_podcastparser_dict(
138 session=mass.http_session, feed_url=feed_url, max_episodes=max_episodes
139 )
140 await mass.cache.set(
141 key=feed_url,
142 provider=provider_instance_id,
143 category=cache_category,
144 data=parsed_feed,
145 expiration=cache_expiration,
146 )
147 return parsed_feed
148
149
150def parse_podcast(
151 *,
152 feed_url: str,
153 parsed_feed: dict[str, Any],
154 instance_id: str,
155 domain: str,
156 mass_item_id: str | None = None,
157) -> Podcast:
158 """
159 Podcast -> Mass Podcast.
160
161 The item_id is the feed url by default, or the optional mass_item_id instead.
162 """
163 publisher = parsed_feed.get("author") or parsed_feed.get("itunes_author", "NO_AUTHOR")
164 item_id = feed_url if mass_item_id is None else mass_item_id
165 mass_podcast = Podcast(
166 item_id=item_id,
167 name=parsed_feed.get("title", "NO_TITLE"),
168 publisher=publisher,
169 provider=instance_id,
170 uri=parsed_feed.get("link"),
171 provider_mappings={
172 ProviderMapping(
173 item_id=item_id,
174 provider_domain=domain,
175 provider_instance=instance_id,
176 )
177 },
178 )
179 genres: list[str] = []
180 if _genres := parsed_feed.get("itunes_categories"):
181 for _sub_genre in _genres:
182 if isinstance(_sub_genre, list):
183 genres.extend(x for x in _sub_genre if isinstance(x, str))
184 elif isinstance(_sub_genre, str):
185 genres.append(_sub_genre)
186
187 mass_podcast.metadata.genres = set(genres)
188 mass_podcast.metadata.description = parsed_feed.get("description", "")
189 mass_podcast.metadata.explicit = parsed_feed.get("explicit", False)
190 language = parsed_feed.get("language")
191 if language is not None:
192 mass_podcast.metadata.languages = UniqueList([language])
193 episodes = parsed_feed.get("episodes", [])
194 mass_podcast.total_episodes = len(episodes)
195 podcast_cover = parsed_feed.get("cover_url")
196 if podcast_cover is not None:
197 mass_podcast.metadata.images = UniqueList(
198 [
199 MediaItemImage(
200 type=ImageType.THUMB,
201 path=podcast_cover,
202 provider=instance_id,
203 remotely_accessible=True,
204 )
205 ]
206 )
207 return mass_podcast
208
209
210def get_stream_url_from_episode(*, episode: dict[str, Any]) -> str | None:
211 """
212 Give the url of the episode's playable enclosure, or None if the episode has none.
213
214 Prefers the first audio or video enclosure, falling back to any non-image one.
215
216 :param episode: A single episode dict as returned by podcastparser.
217 """
218 fallback: str | None = None
219 for enclosure in episode.get("enclosures", []):
220 url = enclosure.get("url")
221 if not url:
222 continue
223 mime_type = str(enclosure.get("mime_type") or "")
224 if mime_type.startswith(("audio/", "video/")):
225 return str(url)
226 # feeds do declare bogus mime types (e.g. type="file") for real audio, so anything
227 # that is not an image stays a candidate in case no audio enclosure is declared
228 if fallback is None and not mime_type.startswith("image/"):
229 fallback = str(url)
230 return fallback
231
232
233def get_stream_url_and_guid_from_episode(*, episode: dict[str, Any]) -> tuple[str, str | None]:
234 """Give episode's stream url and guid, if it exists."""
235 stream_url = get_stream_url_from_episode(episode=episode)
236 if stream_url is None:
237 raise ValueError("Episode has no playable enclosure")
238 guid = episode.get("guid")
239 if guid is not None:
240 # The media's item_id is {prov_podcast_id} {guid_or_stream_url}
241 # see parse_podcast_episode.
242 # However, the guid must not contain a space, otherwise it is invalid.
243 # We cannot check, if it is a proper guid (uuid.UUID4(...)), as some podcast feeds
244 # do not follow the standard.
245 guid = None if len(guid.split(" ")) > 1 else guid
246 return stream_url, guid
247
248
249def find_episode_stream_url(*, parsed_feed: dict[str, Any], guid_or_stream_url: str) -> str | None:
250 """
251 Return the stream url of the episode identified by the item_id's episode part.
252
253 :param parsed_feed: The podcastparser dict of the feed holding the episode.
254 :param guid_or_stream_url: Episode part of the item_id, see parse_podcast_episode.
255 """
256 for episode in parsed_feed.get("episodes", []):
257 try:
258 stream_url, guid = get_stream_url_and_guid_from_episode(episode=episode)
259 except ValueError:
260 # episode without a playable enclosure carries no stream; skip it instead of
261 # aborting the lookup for the (potentially later) requested episode
262 continue
263 # only a guid rejected as unusable (None) falls back to the stream url, so an
264 # empty guid resolves the same way it was turned into an item_id
265 if guid_or_stream_url == (stream_url if guid is None else guid):
266 return stream_url
267 return None
268
269
270def rank_episodes_by_date(dates: list[Any]) -> list[int]:
271 """
272 Return the position of every episode, in the order the provider lists them.
273
274 Positions run oldest to newest, so the newest episode has the highest one. Episodes
275 without a date rank as the oldest, and when none of them has one the provider is
276 assumed to list its episodes newest-first.
277
278 :param dates: The publication dates in listing order, None where an episode has none.
279 Any comparable type will do as long as the provider reports them all the same way.
280 """
281 total = len(dates)
282 dated = [idx for idx, date in enumerate(dates) if date is not None]
283 if not dated:
284 return [total - idx for idx in range(total)]
285 undated = [idx for idx, date in enumerate(dates) if date is None]
286 positions = [0] * total
287 for position, idx in enumerate(undated + sorted(dated, key=lambda idx: dates[idx]), 1):
288 positions[idx] = position
289 return positions
290
291
292def get_episode_positions(episodes: list[dict[str, Any]]) -> list[int]:
293 """
294 Return the position of every episode, in the order the feed lists them.
295
296 Positions run oldest to newest, so the newest episode has the highest one.
297
298 :param episodes: The episodes of a single feed, as parsed by podcastparser.
299 """
300 # a feed that numbers only part of its episodes, or restarts its numbering every
301 # season, mixes incompatible scales, so its numbers are only used when unique
302 numbered = all(isinstance(ep.get("number"), int) and ep["number"] > 0 for ep in episodes)
303 if numbered and len({ep.get("season") or 0 for ep in episodes}) == 1:
304 return [ep["number"] for ep in episodes]
305 # podcastparser lists serial feeds oldest-first and all others newest-first,
306 # so rank on the publication date instead of the feed order
307 return rank_episodes_by_date([ep.get("published") or None for ep in episodes])
308
309
310def parse_podcast_episode(
311 *,
312 episode: dict[str, Any],
313 prov_podcast_id: str,
314 position: int,
315 podcast_cover: str | None = None,
316 podcast_name: str | None = None,
317 instance_id: str,
318 domain: str,
319 mass_item_id: str | None = None,
320) -> PodcastEpisode | None:
321 """
322 Podcast Episode -> Mass Podcast Episode.
323
324 The item_id is {prov_podcast_id} {guid_or_stream_url} by default, or the optional mass_item_id
325 instead. The podcast_cover is used, if the episode should not have its own cover. The
326 podcast_name names the parent podcast reference (falls back to the episode title if unset).
327
328 The function returns None, if the episode enclosure is missing, i.e. there is no stream
329 information present.
330
331 :param position: The episode's listing position, oldest to newest.
332 """
333 episode_duration = episode.get("total_time", 0.0)
334 episode_title = episode.get("title", "NO_EPISODE_TITLE")
335 episode_cover = episode.get("episode_art_url", podcast_cover)
336
337 # this is unix epoch in s, and 0 if unknown
338 episode_published: int | None = episode.get("published")
339 if episode_published == 0:
340 episode_published = None
341
342 try:
343 stream_url, guid = get_stream_url_and_guid_from_episode(episode=episode)
344 except ValueError:
345 # we are missing the episode enclosure or stream information
346 return None
347 # We treat a guid as invalid if contains a space.
348 guid_or_stream_url = guid if guid is not None and len(guid.split(" ")) == 1 else stream_url
349
350 # Default episode id. A guid is preferred as identification.
351 episode_id = f"{prov_podcast_id} {guid_or_stream_url}" if mass_item_id is None else mass_item_id
352 mass_episode = PodcastEpisode(
353 item_id=episode_id,
354 provider=instance_id,
355 name=episode_title,
356 duration=int(episode_duration),
357 position=position,
358 podcast=ItemMapping(
359 item_id=prov_podcast_id,
360 provider=instance_id,
361 name=podcast_name or episode_title,
362 media_type=MediaType.PODCAST,
363 ),
364 provider_mappings={
365 ProviderMapping(
366 item_id=episode_id,
367 provider_domain=domain,
368 provider_instance=instance_id,
369 audio_format=AudioFormat(
370 content_type=ContentType.try_parse(stream_url),
371 ),
372 url=stream_url,
373 )
374 },
375 )
376 if episode_published is not None:
377 mass_episode.metadata.release_date = datetime.fromtimestamp(episode_published, tz=UTC)
378
379 # description (podcastparser normalizes this to plain text, defaulting to "")
380 if description := episode.get("description"):
381 mass_episode.metadata.description = description
382
383 # explicit flag (itunes:explicit); only set when the feed actually declared it
384 explicit = episode.get("explicit")
385 if explicit is not None:
386 mass_episode.metadata.explicit = bool(explicit)
387
388 # episode webpage (the item <link>)
389 if link := episode.get("link"):
390 mass_episode.metadata.links = {MediaItemLink(type=LinkType.WEBSITE, url=link)}
391
392 # hosts/guests (podcast:person), mapped to performer names
393 if performers := parse_podcast_persons(episode.get("persons")):
394 mass_episode.metadata.performers = set(performers)
395
396 # inline chapters (Podlove Simple Chapters, parsed by podcastparser)
397 if chapters := episode.get("chapters"):
398 _chapters: list[MediaItemChapter] = []
399 for chapter in chapters:
400 if not isinstance(chapter, dict):
401 continue
402 title = chapter.get("title")
403 start = chapter.get("start")
404 # start may legitimately be 0 (opening chapter), so test against None
405 if title and start is not None:
406 _chapters.append(
407 MediaItemChapter(position=len(_chapters) + 1, name=title, start=start)
408 )
409 if _chapters:
410 mass_episode.metadata.chapters = _chapters
411
412 # cover image
413 if episode_cover is not None:
414 mass_episode.metadata.images = UniqueList(
415 [
416 MediaItemImage(
417 type=ImageType.THUMB,
418 path=episode_cover,
419 provider=instance_id,
420 remotely_accessible=True,
421 )
422 ]
423 )
424
425 return mass_episode
426
427
428def parse_podcast_persons(persons: Any) -> list[str]:
429 """
430 Extract performer names from a Podcasting 2.0 ``podcast:person`` collection.
431
432 Accepts the persons list as produced by podcastparser (feeds) or by the Podcast
433 Index API. Returns the display names de-duplicated (case-insensitive) in feed
434 order; any non-list input yields no names, so callers can pass a raw
435 ``.get("persons")`` without guarding.
436
437 :param persons: The raw persons collection, or any value (non-lists yield []).
438 """
439 names: list[str] = []
440 seen: set[str] = set()
441 if not isinstance(persons, list):
442 return names
443 for person in persons:
444 name = person.get("name") if isinstance(person, dict) else person
445 if not isinstance(name, str) or not (name := name.strip()):
446 continue
447 key = name.casefold()
448 if key in seen:
449 continue
450 seen.add(key)
451 names.append(name)
452 return names
453
454
455def _coerce_seconds(value: Any) -> float | None:
456 """Coerce a chapter time value to float seconds, or None if not parseable."""
457 if value is None:
458 return None
459 try:
460 seconds = float(value)
461 except TypeError, ValueError:
462 return None
463 return seconds if isfinite(seconds) else None
464
465
466def parse_chapters_from_json(data: dict[str, Any]) -> list[MediaItemChapter]:
467 """
468 Parse a Podcasting 2.0 ``podcast:chapters`` JSON document into chapters.
469
470 Spec: https://github.com/Podcastindex-org/podcast-namespace/blob/main/docs/1.0.md#chapters
471 Entries without a usable ``startTime``/``title`` are skipped, as are entries
472 explicitly hidden from the table of contents (``toc: false``).
473 """
474 chapters: list[MediaItemChapter] = []
475 raw_chapters = data.get("chapters")
476 if not isinstance(raw_chapters, list):
477 return chapters
478 for raw_chapter in raw_chapters:
479 if not isinstance(raw_chapter, dict):
480 continue
481 if raw_chapter.get("toc") is False:
482 continue
483 title = raw_chapter.get("title")
484 start = _coerce_seconds(raw_chapter.get("startTime"))
485 if not isinstance(title, str) or not title or start is None:
486 continue
487 chapters.append(
488 MediaItemChapter(
489 position=len(chapters) + 1,
490 name=title,
491 start=start,
492 end=_coerce_seconds(raw_chapter.get("endTime")),
493 )
494 )
495 return chapters
496
497
498async def enrich_episode_chapters(
499 *,
500 session: aiohttp.ClientSession,
501 chapters_json_url: str | None,
502 mass_episode: PodcastEpisode,
503) -> None:
504 """
505 Attach ``podcast:chapters`` (external JSON) to an episode lacking inline chapters.
506
507 Transport-agnostic: the caller supplies the chapters JSON URL directly (e.g.
508 podcastparser's ``chapters_json_url`` or the Podcast Index API ``chaptersUrl``),
509 so any podcast provider can reuse it. Chapter data is supplementary: any
510 fetch/parse failure is logged and ignored so it can never break episode
511 resolution or playback. Intended for the single-episode path only, to avoid a
512 network request per episode when listing a whole podcast.
513
514 :param session: The aiohttp session used for the best-effort fetch.
515 :param chapters_json_url: The Podcasting 2.0 chapters JSON URL, or None to skip.
516 :param mass_episode: The episode to enrich; untouched if it already has chapters.
517 """
518 if mass_episode.metadata.chapters:
519 return
520 url = chapters_json_url
521 if not url:
522 return
523 # send a browser UA: some podcast hosts/CDNs reject non-browser agents
524 # (music-assistant/support#3596). Chapter data is supplementary, so unlike
525 # get_podcastparser_dict we make a single best-effort attempt with no UA-less retry.
526 # TimeoutError (raised on total-timeout) is not a ClientError, so catch it explicitly
527 # to keep this enrichment best-effort. The parse stays inside the try so a malformed
528 # document can never break episode resolution either.
529 try:
530 async with session.get(
531 url,
532 headers={"User-Agent": "Mozilla/5.0"},
533 raise_for_status=True,
534 timeout=_CHAPTERS_FETCH_TIMEOUT,
535 ) as response:
536 data = await response.json(content_type=None)
537 if isinstance(data, dict) and (chapters := parse_chapters_from_json(data)):
538 mass_episode.metadata.chapters = chapters
539 except (ClientError, TimeoutError, ValueError, TypeError) as err:
540 LOGGER.warning("Failed to fetch podcast chapters from %s: %s", url, err)
541