/
/
/
This repo is destined for my server automations and setup.
1#!/usr/bin/env python3
2"""
3Fetch sound effects from Freesound into the D&D media library.
4
5Uses the sanctioned OAuth2 download endpoint rather than the public CDN
6preview URLs. The previews are technically reachable without authentication,
7but routing around a deliberate access control is what the API terms call
8"other technology to access or use Content", and it bypasses the download
9counts and attribution the platform relies on. One browser authorisation buys
10a refresh token; everything after that is automatic.
11
12Most Freesound content is CC-BY, which requires crediting the author. This
13script writes credits.md from the licence metadata as it downloads - that is
14a licence obligation, not a nicety.
15
16Setup
17-----
18Get credentials at https://freesound.org/apiv2/apply/ (log in FIRST - the page
19silently redirects to a login form otherwise). You need the Client id and the
20Client secret.
21
22Leave the Callback URL at its default:
23
24 https://freesound.org/home/app_permissions/permission_granted/
25
26That is Freesound's out-of-band flow - it uses Freesound itself as the
27redirect target and displays the authorisation code on screen for you to copy,
28which is what --authorise prompts for. No change to the credential is needed.
29
30 export FREESOUND_CLIENT_ID=...
31 export FREESOUND_CLIENT_SECRET=...
32 ./fetch-dnd-effects.py --authorise # once, prints a URL to visit
33 ./fetch-dnd-effects.py --starter # then this
34
35Usage
36-----
37 ./fetch-dnd-effects.py --search "door creak" # preview a search
38 ./fetch-dnd-effects.py --starter # curated effect set
39 ./fetch-dnd-effects.py --cue sword_clash --query "sword clash metal"
40"""
41
42from __future__ import annotations
43
44import argparse
45import json
46import os
47import re
48import sys
49import time
50import urllib.error
51import urllib.parse
52import urllib.request
53from pathlib import Path
54
55API = "https://freesound.org/apiv2"
56AUTHORIZE = f"{API}/oauth2/authorize/"
57TOKEN = f"{API}/oauth2/access_token/"
58OOB = "urn:ietf:wg:oauth:2.0:oob"
59
60DEFAULT_DEST = Path("/mnt/rstorage/media/dnd")
61TOKEN_FILE = Path.home() / ".config" / "dnd-agent" / "freesound-token.json"
62
63# Licences we accept. Freesound also carries "Sampling+" and NonCommercial
64# variants; all three permitted here allow private use with attribution, and
65# CC0 requires none. Anything else is skipped rather than silently used.
66ACCEPTED_LICENCES = ("creativecommons.org/publicdomain/zero",
67 "creativecommons.org/licenses/by/",
68 "creativecommons.org/licenses/by-nc/")
69
70# A deliberately small opening set: the effects a table actually reaches for.
71# Each entry is (cue name, search query). Duration filtering keeps out
72# field recordings and ambient loops that happen to match the words.
73STARTER_EFFECTS = [
74 ("sword_clash", "sword clash metal hit"),
75 ("door_creak", "wooden door creak open"),
76 ("door_slam", "door slam shut heavy"),
77 ("thunder", "thunder clap close"),
78 ("wolf_howl", "wolf howl"),
79 ("horse_gallop", "horse hooves galloping"),
80 ("crowd_gasp", "crowd gasp reaction"),
81 ("coins", "coins purse drop"),
82 ("bow_shot", "bow arrow shot"),
83 ("magic_whoosh", "magic spell whoosh"),
84 ("footsteps_stone", "footsteps stone corridor"),
85 ("chest_open", "chest open wooden lid"),
86]
87
88# The wider set. Grouped by what actually happens at a table rather than by
89# sound category, because that is how the agent will reason about them.
90FULL_EFFECTS = STARTER_EFFECTS + [
91 # Combat
92 ("sword_unsheath", "sword unsheathe draw metal"),
93 ("shield_block", "shield metal impact"),
94 ("arrow_impact", "arrow hit thud impact"),
95 ("punch_impact", "punch body impact fight"),
96 ("crossbow", "crossbow shot fire"),
97 ("armour_move", "chainmail armor movement"),
98 ("battle_cry", "battle cry shout man"),
99 ("body_fall", "body fall ground thud"),
100 # Magic
101 ("fireball", "fireball explosion magic"),
102 ("spell_cast", "magic spell cast sparkle"),
103 ("lightning_bolt", "electric lightning zap"),
104 ("healing", "magic heal chime shimmer"),
105 ("teleport", "teleport warp magic"),
106 ("summon", "dark magic summon drone"),
107 # Creatures
108 ("dragon_roar", "dragon roar monster"),
109 ("goblin_shriek", "creature screech shriek"),
110 ("growl", "monster growl low"),
111 ("bat_wings", "bat wings flapping"),
112 ("rat_squeak", "rat squeaking"),
113 ("horse_neigh", "horse neigh whinny"),
114 ("crow", "crow raven caw"),
115 ("owl", "owl hoot night"),
116 # Doors, locks, mechanisms
117 ("door_knock", "knock knock door"),
118 ("lock_pick", "lock picking mechanism"),
119 ("key_lock", "key turning lock door"),
120 ("portcullis", "heavy gate chain mechanism"),
121 ("lever", "metal lever switch"),
122 ("stone_grind", "stone sliding heavy"),
123 # Environment and weather
124 ("rain_start", "rain beginning downpour"),
125 ("wind_gust", "wind gust howling"),
126 ("fire_crackle", "fireplace fire crackling"),
127 ("water_splash", "water splash body"),
128 ("bell_toll", "church bell toll single"),
129 ("earthquake", "rumble earthquake low"),
130 # Table and social
131 ("tavern_cheer", "crowd cheer applause"),
132 ("crowd_murmur", "crowd talking murmur indoor"),
133 ("glass_break", "glass bottle breaking"),
134 ("pour_drink", "pouring liquid into cup"),
135 ("dice_roll", "dice roll table"),
136 ("page_turn", "paper page turning book"),
137 # Stings and transitions
138 ("suspense_sting", "suspense sting tension hit"),
139 ("dramatic_boom", "cinematic impact boom"),
140 ("mystery_chime", "mysterious chime bell"),
141 ("footsteps_wood", "footsteps wooden floor"),
142
143 # --- Beats a soundboard needs that a category list does not suggest ---
144 # These came from what GMs actually reach for mid-session rather than
145 # from enumerating sound types: the punctuation of play, not its scenery.
146 ("victory_sting", "victory fanfare success"),
147 ("failure_sting", "fail sad trombone"),
148 ("level_up", "level up achievement chime"),
149 ("wilhelm_scream", "wilhelm scream"),
150 ("scream_male", "man scream pain"),
151 ("scream_female", "woman scream terror"),
152 ("heartbeat", "heartbeat slow pulse"),
153 ("breathing_heavy", "heavy breathing exhausted"),
154 ("crowd_boo", "crowd booing disapproval"),
155 ("laughter", "group laughter tavern"),
156 ("baby_cry", "baby crying"),
157 ("church_choir", "choir voices holy"),
158 ("bone_crack", "bone break crack"),
159 ("blood_splatter", "blood splatter gore"),
160 ("torch_ignite", "torch fire ignite whoosh"),
161 ("potion_drink", "drinking gulping liquid"),
162 ("scroll_unfurl", "parchment paper unfurl"),
163 ("writing_quill", "writing quill pen paper"),
164 ("cart_wheels", "wooden cart wheels rolling"),
165 ("ship_creak", "ship wood creaking sea"),
166 ("seagulls", "seagulls harbour"),
167 ("market_crowd", "market crowd bustle"),
168 ("forge_hammer", "blacksmith hammer anvil"),
169 ("chains_rattle", "chains rattling metal"),
170 ("cave_drip", "water dripping cave echo"),
171 ("wind_chimes", "wind chimes gentle"),
172 ("clock_tick", "clock ticking"),
173 ("swarm_insects", "insect swarm buzzing"),
174]
175
176# Queries matter more than they look. "treasure chest lid open" returned
177# nothing acceptably licensed while "chest open wooden lid" found a clean CC0
178# one-shot; "heavy wooden door slam" surfaced a body-impact recording. When a
179# cue sounds wrong, re-run --search with different words before blaming the
180# licence filter.
181
182
183class FreesoundError(RuntimeError):
184 pass
185
186
187# Freesound throttles at 60 requests/minute. Every cue costs one search plus
188# one download per variant, so a 3-variant run makes 4 calls per cue and hits
189# the ceiling well before finishing. This paces every call - search included -
190# rather than only sleeping between downloads.
191_MIN_INTERVAL = 1.1
192_last_call = 0.0
193
194
195def _throttle() -> None:
196 global _last_call
197 wait = _MIN_INTERVAL - (time.time() - _last_call)
198 if wait > 0:
199 time.sleep(wait)
200 _last_call = time.time()
201
202
203def _post(url: str, data: dict) -> dict:
204 req = urllib.request.Request(
205 url, data=urllib.parse.urlencode(data).encode(),
206 headers={"Content-Type": "application/x-www-form-urlencoded"})
207 try:
208 with urllib.request.urlopen(req, timeout=60) as r:
209 return json.load(r)
210 except urllib.error.HTTPError as exc:
211 raise FreesoundError(f"{exc.code}: {exc.read()[:300]!r}") from exc
212
213
214def _request(req: urllib.request.Request, timeout: int, binary: bool,
215 retries: int = 3):
216 """Issue a throttled request, backing off if we are throttled anyway."""
217 for attempt in range(retries):
218 _throttle()
219 try:
220 with urllib.request.urlopen(req, timeout=timeout) as r:
221 return r.read() if binary else json.load(r)
222 except urllib.error.HTTPError as exc:
223 if exc.code == 429 and attempt < retries - 1:
224 # Rate limits are per minute, so waiting out the window is
225 # the only useful response.
226 print(" rate limited, waiting 60s")
227 time.sleep(60)
228 continue
229 raise FreesoundError(f"{exc.code}: {exc.read()[:300]!r}") from exc
230 raise FreesoundError("exhausted retries")
231
232
233def _get(url: str, token: str, binary: bool = False):
234 return _request(
235 urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}),
236 timeout=180, binary=binary)
237
238
239def _search(url: str, api_key: str) -> dict:
240 """Search uses simple token auth; only downloads need OAuth2."""
241 return _request(
242 urllib.request.Request(url, headers={"Authorization": f"Token {api_key}"}),
243 timeout=60, binary=False)
244
245
246# ---------------------------------------------------------------- auth
247
248def authorise(client_id: str, client_secret: str) -> None:
249 """Walk the one-time browser authorisation and store the tokens."""
250 url = f"{AUTHORIZE}?{urllib.parse.urlencode({'client_id': client_id, 'response_type': 'code'})}"
251 print("\n1. Open this URL in a browser (log in to Freesound first):\n")
252 print(f" {url}\n")
253 print("2. Grant access. Freesound redirects to its own permission_granted")
254 print(" page, which displays the authorisation code on screen.")
255 print(" The code expires after 10 minutes and is single-use.\n")
256 code = input("3. Paste the code here: ").strip()
257 if not code:
258 sys.exit("no code entered")
259
260 tok = _post(TOKEN, {
261 "client_id": client_id,
262 "client_secret": client_secret,
263 "grant_type": "authorization_code",
264 "code": code,
265 })
266 tok["obtained"] = time.time()
267 TOKEN_FILE.parent.mkdir(parents=True, exist_ok=True)
268 TOKEN_FILE.write_text(json.dumps(tok, indent=2))
269 TOKEN_FILE.chmod(0o600)
270 print(f"\n stored in {TOKEN_FILE}")
271 print(" access tokens last 24h and refresh automatically from here on.")
272
273
274def access_token(client_id: str, client_secret: str) -> str:
275 """Return a valid access token, refreshing if it has expired."""
276 if not TOKEN_FILE.exists():
277 sys.exit("not authorised yet - run with --authorise first")
278 tok = json.loads(TOKEN_FILE.read_text())
279
280 age = time.time() - tok.get("obtained", 0)
281 # Refresh a little early rather than discovering expiry mid-download.
282 if age < tok.get("expires_in", 86400) - 600:
283 return tok["access_token"]
284
285 print(" access token expired, refreshing")
286 new = _post(TOKEN, {
287 "client_id": client_id,
288 "client_secret": client_secret,
289 "grant_type": "refresh_token",
290 "refresh_token": tok["refresh_token"],
291 })
292 new["obtained"] = time.time()
293 TOKEN_FILE.write_text(json.dumps(new, indent=2))
294 TOKEN_FILE.chmod(0o600)
295 return new["access_token"]
296
297
298# ---------------------------------------------------------------- search
299
300def search(api_key: str, query: str, max_duration: float = 12.0,
301 page_size: int = 8) -> list[dict]:
302 """Find candidate one-shots for a cue.
303
304 Sorted by downloads: on Freesound that correlates well with clean,
305 usable recordings, which matters more here than exact relevance.
306 """
307 params = {
308 "query": query,
309 "filter": f"duration:[0.1 TO {max_duration}]",
310 "sort": "downloads_desc",
311 "fields": "id,name,license,duration,username,previews,download",
312 "page_size": page_size,
313 }
314 data = _search(f"{API}/search/text/?{urllib.parse.urlencode(params)}", api_key)
315 return [r for r in data.get("results", [])
316 if any(l in r.get("license", "") for l in ACCEPTED_LICENCES)]
317
318
319def licence_label(url: str) -> str:
320 if "publicdomain/zero" in url:
321 return "CC0"
322 if "licenses/by-nc/" in url:
323 return "CC-BY-NC"
324 if "licenses/by/" in url:
325 return "CC-BY"
326 return url
327
328
329# ---------------------------------------------------------------- download
330
331# The download endpoint returns the original upload, so the extension is not
332# known until the bytes arrive. Any of these may already be on disk from an
333# earlier run.
334KNOWN_EXTS = (".mp3", ".wav", ".flac", ".aiff", ".ogg")
335
336
337def existing_file(dest: Path, cue: str) -> Path | None:
338 return next((p for ext in KNOWN_EXTS if (p := dest / f"{cue}{ext}").exists()), None)
339
340
341def download(token: str, sound: dict, dest: Path, cue: str, force: bool) -> str:
342 if (found := existing_file(dest, cue)) and not force:
343 print(f" skip {found.name} (exists)")
344 return "skipped"
345
346 try:
347 blob = _get(f"{API}/sounds/{sound['id']}/download/", token, binary=True)
348 except FreesoundError as exc:
349 print(f" FAILED {cue}: {exc}")
350 return "failed"
351
352 if len(blob) < 2000:
353 print(f" FAILED {cue}: suspiciously small ({len(blob)} bytes)")
354 return "failed"
355
356 # The download endpoint returns the original upload, whose format varies
357 # (wav/aiff/flac/mp3). Keep the bytes as-is under a .mp3 name only if it
358 # really is mp3; otherwise preserve the true extension so ffmpeg in the
359 # mixer is never misled by the filename.
360 ext = ".mp3"
361 if blob[:4] == b"RIFF":
362 ext = ".wav"
363 elif blob[:4] == b"fLaC":
364 ext = ".flac"
365 elif blob[:4] == b"FORM":
366 ext = ".aiff"
367 elif blob[:4] == b"OggS":
368 ext = ".ogg"
369 target = dest / f"{cue}{ext}"
370
371 tmp = target.with_suffix(target.suffix + ".part")
372 tmp.write_bytes(blob)
373 tmp.rename(target)
374 print(f" ok {cue}{ext} ({len(blob)/1024:.0f} KB, "
375 f"{licence_label(sound['license'])}) <- {sound['name'][:38]}")
376 return "downloaded"
377
378
379CREDIT_ROW = re.compile(
380 r"^\| `(?P<cue>[^`]+)` \| \[(?P<name>.*?)\]\("
381 r"https://freesound\.org/s/(?P<id>\d+)/\) \| (?P<username>.*?) \| (?P<licence>\S+) \|$"
382)
383
384
385def read_credits(path: Path) -> list[dict]:
386 """Parse an existing credits.md so a partial run does not discard it."""
387 if not path.exists():
388 return []
389 out = []
390 for line in path.read_text(encoding="utf-8").splitlines():
391 if m := CREDIT_ROW.match(line.strip()):
392 out.append(m.groupdict())
393 return out
394
395
396def write_credits(dest: Path, entries: list[dict]) -> Path:
397 """Attribution file. CC-BY requires this; CC0 does not but costs nothing.
398
399 Merged with whatever is already recorded, so a run that covers only part
400 of the library never drops attribution for the rest of it.
401 """
402 path = dest / "credits.md"
403 merged = {e["cue"]: e for e in read_credits(path)}
404 merged.update({e["cue"]: e for e in entries})
405 # Only credit files that are actually present.
406 entries = [e for e in merged.values() if existing_file(dest, e["cue"])]
407 lines = [
408 "# Sound effect credits",
409 "",
410 "Sourced from [Freesound](https://freesound.org) via its API.",
411 "CC-BY and CC-BY-NC sounds require attribution to their authors.",
412 "",
413 "| Cue | Sound | Author | Licence |",
414 "|---|---|---|---|",
415 ]
416 for e in sorted(entries, key=lambda x: x["cue"]):
417 lines.append(
418 f"| `{e['cue']}` | [{e['name']}](https://freesound.org/s/{e['id']}/) "
419 f"| {e['username']} | {e['licence']} |"
420 )
421 lines.append("")
422 path.write_text("\n".join(lines))
423 return path
424
425
426def main() -> int:
427 ap = argparse.ArgumentParser(
428 description="Fetch sound effects from Freesound (OAuth2).")
429 ap.add_argument("--dest", type=Path, default=DEFAULT_DEST)
430 ap.add_argument("--authorise", action="store_true",
431 help="run the one-time browser authorisation")
432 ap.add_argument("--starter", action="store_true",
433 help="download the curated starter effect set (12 cues)")
434 ap.add_argument("--full", action="store_true",
435 help="download the full effect set (~50 cues)")
436 ap.add_argument("--variants", type=int, default=1, metavar="N",
437 help="distinct recordings per cue, named <cue>_1..<cue>_N "
438 "so repeated effects do not sound canned (default: 1)")
439 ap.add_argument("--search", metavar="QUERY",
440 help="show search results without downloading")
441 ap.add_argument("--cue", help="cue name for --query")
442 ap.add_argument("--query", help="search query to download as --cue")
443 ap.add_argument("--max-duration", type=float, default=12.0)
444 ap.add_argument("--force", action="store_true")
445 args = ap.parse_args()
446
447 client_id = os.environ.get("FREESOUND_CLIENT_ID", "")
448 client_secret = os.environ.get("FREESOUND_CLIENT_SECRET", "")
449 if not client_id or not client_secret:
450 sys.exit("set FREESOUND_CLIENT_ID and FREESOUND_CLIENT_SECRET")
451
452 if args.authorise:
453 authorise(client_id, client_secret)
454 return 0
455
456 # The client secret doubles as the token-auth key for search.
457 api_key = client_secret
458
459 if args.search:
460 results = search(api_key, args.search, args.max_duration)
461 if not results:
462 print("no results with an accepted licence")
463 return 1
464 for r in results:
465 print(f" {r['id']:>9} {r['duration']:5.1f}s "
466 f"{licence_label(r['license']):<9} {r['name'][:44]} ({r['username']})")
467 return 0
468
469 jobs: list[tuple[str, str]] = []
470 if args.full:
471 jobs = list(FULL_EFFECTS)
472 elif args.starter:
473 jobs = list(STARTER_EFFECTS)
474 if args.cue and args.query:
475 jobs.append((args.cue, args.query))
476 if not jobs:
477 ap.error("nothing to do - use --starter, --full, --cue/--query, or --search")
478 if args.variants < 1:
479 ap.error("--variants must be at least 1")
480
481 token = access_token(client_id, client_secret)
482 effects = args.dest / "effects"
483 effects.mkdir(parents=True, exist_ok=True)
484
485 stats = {"downloaded": 0, "skipped": 0, "failed": 0}
486 credits: list[dict] = []
487
488 for i, (cue, query) in enumerate(jobs, 1):
489 print(f"[{i}/{len(jobs)}] {cue}: \"{query}\"")
490 # An API failure part-way through must not discard the credits for
491 # everything already on disk - that is a licence obligation, not a
492 # convenience. Abort the run but still write what we have.
493 try:
494 results = search(api_key, query, args.max_duration)
495 except FreesoundError as exc:
496 print(f" ABORTED at {cue}: {exc}")
497 print(" re-run to resume; existing files are skipped")
498 break
499 if not results:
500 print(f" FAILED {cue}: no acceptably licensed result")
501 stats["failed"] += 1
502 continue
503
504 # Take several distinct recordings per cue where available. Hearing
505 # the identical door creak on every door is the fastest way to break
506 # the illusion; the agent picks a variant at random at play time.
507 picks = results[:args.variants]
508 for n, sound in enumerate(picks, 1):
509 name = cue if args.variants == 1 else f"{cue}_{n}"
510 outcome = download(token, sound, effects, name, args.force)
511 stats[outcome] += 1
512 if outcome != "failed":
513 credits.append({
514 "cue": name, "id": sound["id"], "name": sound["name"],
515 "username": sound["username"],
516 "licence": licence_label(sound["license"]),
517 })
518 # Pacing is handled centrally by _throttle(); no sleep needed here.
519
520 path = write_credits(effects, credits)
521 print(f"\n {stats['downloaded']} downloaded, {stats['skipped']} skipped, "
522 f"{stats['failed']} failed")
523 if path:
524 print(f" credits: {path}")
525 return 1 if stats["failed"] else 0
526
527
528if __name__ == "__main__":
529 try:
530 sys.exit(main())
531 except KeyboardInterrupt:
532 print("\ninterrupted")
533 sys.exit(130)
534 except FreesoundError as exc:
535 sys.exit(f"error: {exc}")
536