From bcd72b264867d972129f955aade69c7c1526cdbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lauri=20V=C3=B5sandi?= Date: Tue, 28 May 2019 00:42:56 +0300 Subject: [PATCH] Initial commit --- README.md | 41 ++++++ main.py | 369 ++++++++++++++++++++++++++++++++++++++++++++++ static/index.html | 79 ++++++++++ static/main.css | 30 ++++ static/main.js | 229 ++++++++++++++++++++++++++++ 5 files changed, 748 insertions(+) create mode 100644 README.md create mode 100644 main.py create mode 100644 static/index.html create mode 100644 static/main.css create mode 100644 static/main.js diff --git a/README.md b/README.md new file mode 100644 index 0000000..d7bf5f1 --- /dev/null +++ b/README.md @@ -0,0 +1,41 @@ + +# Features + +Music streaming server for my quirky needs: + +* No damn apps +* No damn force fed indexing +* No damn overengineering +* Yes works pretty much like grep for searching tracks +* Yes damn scalable, tested to work with 56k+ tracks +* Yes just damn web application, use your browser +* Optional metadata caching system +* Optional transcode support + +# Dependencies + +On FreeBSD: + +```bash +pkg install py36-mutagen py36-Flask py36-click-7.0 ffmpeg nginx madplay gmake vorbis-tools +cd /usr/ports/audio/lame/ && make install +``` + +# Running + +To run self-contained web server: + +```bash +python3 main.py serve --root /path/to/library +``` + +To cache metainformation in index.xspf playlist files: + +```bash +python3 main.py metagen --root /path/to/library +``` + +# Known bugs + +FLAC playback stops abruptly in Firefox. Seems to be a common problem on the web. +Use Chrome or Chromium, you need the performance of Webkit to handle lots of search results anyway. diff --git a/main.py b/main.py new file mode 100644 index 0000000..1f362e4 --- /dev/null +++ b/main.py @@ -0,0 +1,369 @@ +import os +from flask import Flask, Response, request +import re +import json +import click +from time import time +from flask import send_from_directory + +def which(*names): + for d in os.environ.get("PATH").split(":"): + for name in names: + path = os.path.join(d, name) + if os.path.exists(path): + return path + raise ValueError("Couldn't find %s anywhere in $PATH" % names) + +MADPLAY = which("madplay") +CURL = which("curl") +FLAC = which("flac") +LAME = which("lame") +FFMPEG = which("avconv", "ffmpeg") +OGGDEC = which("oggdec") +OGGENC = which("oggenc") + +TYPES = { + "mp3": "audio/mpeg", + "flac": "audio/flac", + "oga": "audio/ogg", + "ogg": "audio/ogg", + "wma": "audio/x-ms-wma", + "wav": "audio/wave", +} + +RE_URL = re.compile( + r'^https?://' # http:// or https:// + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... + r'localhost|' # localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4 + r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6 + r'(?::\d+)?' # optional port + r'(?:/?|[/?]\S+)$', re.IGNORECASE) + +XML_ENTITIES = ( + ("&", "&"), + ('"', """), + ("'", "'"), + (">", ">"), + ("<", "<"), +) + +def escape_xml_entities(j): + for letter, replacement in XML_ENTITIES: + j = j.replace(letter, replacement) + return j + +def generate_xspf(root, names): + names.sort() + yield "" + yield "" + yield "%d" % time() + yield "" + + for name in names: + path = os.path.join(root, name) + basename, extension = os.path.splitext(name) + + import mutagen + import mutagen.mp3 + + try: + if name.lower().endswith(".mp3"): + tags = mutagen.mp3.EasyMP3(path) + else: + tags = mutagen.File(path) + except mutagen.MutagenError: + click.echo("Failed to parse metainformation for %s" % repr(path)) + + # Fetch stream attributes + sample_rate = getattr(tags.info, "sample_rate", None) + duration = getattr(tags.info, "length", None) + channels = getattr(tags.info, "channels", None) + bitrate = getattr(tags.info, "bitrate", None) + bit_depth = getattr(tags.info, "bits_per_sample", None) + + + # Workaround for calculating bitrate + if not bitrate and duration: + bitrate = int(item.size / duration) / 1000 * 8000 + + # Artist + artist_name = tags.pop("artist", (None,))[0] + if not artist_name: + artist_name = tags.pop("----:com.apple.iTunes:Artist Credit", (None,))[0] + + # Track album + album_title = tags.pop("album", (None,))[0] + if not album_title: + album_title = tags.pop("----:com.apple.iTunes:Album Credit", (None,))[0] + + # Album artist + album_artist_name = None + if not album_artist_name: + album_artist_name = tags.pop("----:com.apple.iTunes:Album Artist Credit", (None,))[0] + + # Track title + track_title = tags.pop("title", (None,))[0] + + # Track year + year, = tags.pop("date", (None,)) + if not year: + year, = tags.pop("----:com.apple.iTunes:ORIGINAL YEAR", (None,)) + + def parse_x_of_y(mixed): + if not mixed: + return None, None + elif "\x00" in mixed: # Probably bug in mutagen + return None, None + elif re.match("\s*\d+\s*$", mixed): + return int(mixed), None + elif re.match("\s*\d+\s*/\s*\d+\s*$", mixed): + return [int(j) for j in mixed.split("/", 1)] + elif re.match("\s*/\s*\d+\s*$", mixed): + return None, int(mixed.split("/", 1)[1]) + return None, None + + # Disc numbers + disc_number, disc_count = None, None + disc, = tags.pop("discnumber", (None,)) + if disc: + disc_number, disc_count = parse_x_of_y(disc) + + # Release country + country_code, = tags.pop("releasecountry", (None,)) + if not country_code: + country_code, = tags.pop("----:com.apple.iTunes:MusicBrainz Album Release Country", (None,)) + + # Track number could be in various formats: '1', '1/12', 'B2' + track_number = tags.pop("tracknumber", (None,))[0] + track_number, track_count = parse_x_of_y(track_number) + if not track_number: + (track_number, track_count), = tags.pop("trkn", ((None,None),)) + + # Normalize country code + if country_code: + country_code = country_code.lower() + if country_code in ("xe", "xw"): + country_code = None + + yield "" + yield "%s" % escape_xml_entities(name) + + if duration: + yield "%d" % (duration * 1000) + + if track_title: + yield "%s" % escape_xml_entities(track_title) + if artist_name: + yield "%s" % escape_xml_entities(artist_name) + if album_title: + yield "%s" % escape_xml_entities(album_title) + else: + yield "%s" % escape_xml_entities(basename) + + + if track_number: + yield "%d" % track_number + + + if track_count: + yield "%d" % track_count + if disc_number: + yield "%d" % disc_number + if disc_count: + yield "%d" % disc_count + + if year: + yield "%s" % year + + if album_artist_name: + yield "%s" % escape_xml_entities(album_artist_name) + yield "%s" % TYPES[extension[1:].lower()] + yield "%s" % bitrate + yield "%s" % bit_depth + yield "%s" % sample_rate + + yield "" + yield "" + yield "" + + +@click.command("metagen") +@click.option('--root', '-r', prompt='Path to media directory', help='Root directory to index') +@click.option('--force', '-f', is_flag=True, help="Reindex even if up to date according to timestamps") +def metagen(root, force): + for abspath, dirs, files in os.walk(root): + playlist = os.path.join(abspath, "index.xspf") + needed = False + relevant_files = [] + for name in files: + for extension, mimetype in TYPES.items(): + if name.lower().endswith("." + extension): + needed = True + relevant_files.append(name) + + if "index.xspf" in files: + # TODO proper handling here, microsecond differences + if not force and int(os.stat(abspath).st_mtime) <= int(os.stat(playlist).st_mtime): + click.echo("Folder %s is up to date" % repr(abspath)) + needed = False + if not relevant_files: + click.echo("No more relevant files, removing %s" % playlist) + os.remove(playlist) + + if "index.xspf.part" in files: + click.echo("Cleaning up %s.part" % playlist) + os.remove(playlist + ".part") + if needed: + with open(playlist + ".part", "w") as fh: + for chunk in generate_xspf(abspath, relevant_files): + fh.write(chunk) + os.rename(playlist + ".part", playlist) + click.echo("Generated %s" % repr(playlist)) + + + +@click.command("serve") +@click.option('--root', prompt='Path to media directory', help='Root directory to serve') +def serve(root): + app = Flask(__name__) + + @app.route("/api/transcode/") + def transcode(): + def build_pipeline(source, output_mimetype): + sys.stderr.write("Transcoding from: %s\n" % source) + basename, extension = os.path.splitext(source) + extension = extension.lower() + + try: + input_mimetype = TYPES.get(extension) + except: + raise ValueError("Unknown input file extension:", extension) + + sys.stderr.write("Input mimetype: %s\n" % input_mimetype) + sys.stderr.write("Output mimetype: %s\n" % output_mimetype) + + pipeline = [] + if source.startswith("http://") or source.startswith("https://"): + # Escape path component + url = urlparse.urlparse(source) + source = url.scheme + "://" + url.hostname.encode("idna") + urllib.quote(url.path) + yield CURL, "--insecure", "--silent", "--limit-rate", "300k", source # 1Mbps is pretty max + else: + # Perhaps we could read file directly here if transcoder and web server are in the same machine? + raise ValueError("Won't transcode:" + source) + + if input_mimetype == "audio/wave": + pass # Do nothing + elif input_mimetype == "audio/mpeg": + yield MADPLAY, "-", "-o", "wave:-" + elif input_mimetype == "audio/ogg": + yield OGGDEC, "-", "-o", "-" + elif input_mimetype == "audio/flac": + if output_mimetype != "audio/ogg": # oggenc handles FLAC input aswell + yield FLAC, "--decode", "-", "--stdout", "--silent" + else: + # Use FFMPEG/avconv as fallback for decoding wma + yield FFMPEG, "-i", "-", "-f", "wav", "-", "-loglevel", "quiet" + + if output_mimetype == "audio/wave": + pass + elif output_mimetype == "audio/mpeg": + yield LAME, "--quiet", "-aq", "2", "-", "-" + elif output_mimetype == "audio/ogg": + yield OGGENC, "-", "-o", "-" + else: + raise ValueError("Don't know how to encode mimetype: %s" % output_mimetype) + + + mimetype = request.args.get("mimetype") + + pipeline = tuple(build_pipeline(request.args.get("url"), mimetype)) + + first = True + for element in pipeline: + if not first: + print("|",) + first = False + print(" ".join(repr(j) if " " in j else j for j in element)) + print() + + process = None + for element in pipeline: + process = Popen(element, stdout=PIPE, stdin=process.stdout if process else PIPE, close_fds=True) + + return send_file(process.stdout, "stream", mimetype) + + + @app.route("/api/search/") + def search(): + expr = re.compile(request.args.get('expr', 'metallica'), re.IGNORECASE) + limit = request.args.get('limit', 100) + + def generate(): + yield 'retry:999999999999\n' # Tell browser to never reconnect + total = 0 + hits = 0 + counter = 0 + + for abspath, dirs, files in os.walk(root): + dirs.sort() + relpath = os.path.relpath(abspath, root) + hit = False + filtered_filenames = [] + for name in files: + lower_name = name.lower() + for extension, mimetype in TYPES.items(): + if lower_name.endswith("." + extension): + filtered_filenames.append(name) + total += 1 + if expr.search(name): + hit = True + hits += 1 + + if expr.search(relpath): + hit = True + + if hit and filtered_filenames: + yield "event:chdir\ndata:" + yield relpath + yield "\n\n" + yield "event:xspf\n" + yield "data:" + if "index.xspf" in files: + with open(os.path.join(abspath, "index.xspf")) as fh: + yield fh.read() + else: + yield from generate_xspf(abspath, filtered_filenames) + yield "\n" + yield "\n" + counter += 1 + if counter > limit: + break + if counter > limit: + break + + yield "event:done\n" + yield "data:%s\n" % json.dumps({"relevant_files":total, "hits":hits}) + yield "\n" + return Response(generate(), mimetype='text/event-stream') + + @app.route('/api/stream/') + def send_stream(path): + return send_from_directory(root, path) + + + @app.route('/') + def send_js(path): + return send_from_directory('static', path) + + app.run(host= '0.0.0.0', debug=True, threaded=True) + +@click.group() +def entry_point(): pass + +entry_point.add_command(serve) +entry_point.add_command(metagen) + +if __name__ == '__main__': + entry_point() diff --git a/static/index.html b/static/index.html new file mode 100644 index 0000000..d479db3 --- /dev/null +++ b/static/index.html @@ -0,0 +1,79 @@ + + + + + + + + + + µgrep + + + +
+ +
+ + + + + + + + + + + + + + + + +
URLArtistAlbum#TitleDurationMIMEBitrateQuality
+
+
+
+ + +
+ + + + + +
+
+
+ + + + + + + + + + diff --git a/static/main.css b/static/main.css new file mode 100644 index 0000000..99a3fd0 --- /dev/null +++ b/static/main.css @@ -0,0 +1,30 @@ +.maximize { + position:absolute; + left:0; + top:0; + bottom:0; + right: 0; +} + +body { + + font-family: sans; +} + + + +table td, table th { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + font-size: 11pt; + padding: 3pt 7pt !important; +} + +div.dataTables_length { + display: none; +} + +table { + cursor: pointer; +} diff --git a/static/main.js b/static/main.js new file mode 100644 index 0000000..a326c46 --- /dev/null +++ b/static/main.js @@ -0,0 +1,229 @@ + +// Probe for supported codecs +AUDIO_CODECS = [ + 'audio/ogg', + 'audio/mpeg', + 'audio/flac', +]; + +timeout = null; +es = null; +prev = null; + +function humanize_duration(d) { + var seconds = Math.floor(d % 60); + var minutes = Math.floor(d / 60) % 60; + var hours = Math.floor(d / 3600); + if (seconds < 10) seconds = '0' + seconds; + if (minutes < 10) minutes = '0' + minutes; + if (hours < 10) hours = '0' + hours; + return hours + ":" + minutes + ":" + seconds; +} + +function onDelayedSearch() { + var expr = $("#search").val(); + if (expr == window.expr) return; + console.info("Searching", expr); + results.rows().clear().draw(false); + if (es) { + es.close(); + $("#results tbody").empty(); + } + + es = new EventSource("/api/search/?expr=" + expr); + + // Close socket event listener + es.addEventListener("close", function(e) { + console.info("Closing search socket"); + }); + + // Open socket event listener + es.addEventListener("open", function(e) { + console.info("Opened event stream for search expression", expr); + }) + + // Search end result marker + es.addEventListener("done", function(e) { + var totals = JSON.parse(e.data) + console.info("Finished searching, total", totals.relevant_files, "relevant files and", totals.hits, "hits"); + es.close(); + }) + + // Keep track of relative path of the XSPF blobs + es.addEventListener("chdir", function(e) { + console.info("Moving to directory:", e.data); + window.relpath = e.data; + + }) + + // If XSPF blob is received append it to search results + es.addEventListener("xspf", function(e) { + var $trackMetadatas = $("playlist trackList track", $.parseXML(e.data)); + $trackMetadatas.each(function(index, element) { + + var location = element.getElementsByTagName("location")[0]; + var filename = location.innerHTML; + + // Decode escaped XML + var filename = decodeURIComponent(location.innerHTML); + if (filename.indexOf("/") >= 0) { + filename = filename.substring(filename.lastIndexOf("/") + 1); + } + + // Escape quotes for jQuery + filename = filename.replace("\"", "\\\""); + results.row.add([ + window.relpath + "/" + filename, + $("creator", element).html() || "-", + $("album", element).html() || "-", + $("trackNum", element).html() || "-", + $("title", element).html() || "-", + humanize_duration(parseInt($("duration", element).html())/1000), + $("meta[rel='mimetype']", element).html(), + Math.floor(parseInt($("meta[rel='bitrate']", element).html())/1000)+"kbps", +// $("meta[rel='bit_depth']", element).html() + "bit @ " + $("meta[rel='sample_rate']", element).html(), + "bla" + ]).on('click', 'tr', function () { + $(this).toggleClass('selected'); + }); + clearTimeout(window.timeoutDraw); + window.timeoutDraw = setTimeout(results.draw, 100); + }); + }); +} + +function playNextAudioTrack() { + queue.row(queue.row({selected:true}).index()+1).select(); +} + + +function playPrevAudioTrack() { + queue.row(queue.row({selected:true}).index()-1).select(); +} + + +function playTrack(url, mimetype) { + var url = "/api/stream/" + url; + console.info("Playing:", url, mimetype); + + $("#seek").val(0); + $("#offset").html("00:00:00"); + $("#player").unbind("ended").bind("ended", function () { + playNextAudioTrack(); + }); + $("#player").empty(); + + if (window.AUDIO_CODECS_SUPPORTED.indexOf(mimetype) >= 0) { + $("#player").append(""); + } else { + console.info("Codec", mimetype, "not supported by browser"); + } + + for (var j = 0; j < window.AUDIO_CODECS_SUPPORTED.length; j++) { + var alternative = window.AUDIO_CODECS_SUPPORTED[j]; + if (mimetype != alternative) { + $("#player").append(""); + } + } + $("#player").trigger("load"); + $("#player").trigger("play"); +} + +$(document).ready(function() { + $('#results tbody').on('dblclick', 'tr', function () { + console.info("Got double click"); + queue.rows.add(results.rows({ selected: true }).data()); + queue.rows.add([results.row(this).data()]); + results.rows().deselect(); + results.draw(false); + queue.draw(false); + }); + + $("#playback-next").on("click", function() { + playNextAudioTrack(); + }); + $("#playback-prev").on("click", function() { + playPrevAudioTrack(); + }); + $("#queue-selection").on("click", function() { + queue.rows.add(results.rows({ selected: true }).data()).draw(false); + }); + $("#queue-all").on("click", function() { + queue.rows.add(results.data()).draw(false); + }); + $("#queue thead").html($("#results thead").html()); + $("#queue-clear").on("click", function() { + queue.rows().clear().draw(false); + }); + + window.results = $('#results').DataTable({ + autoWidth: false, + columns: [ + null, + {width:"20%"}, + {width:"20%"}, + {width:"1em"}, + {width:"20%"}, + ], + + paging: true, + ordering: false, + searching: false, + select: { + style: 'multi' + }, + columnDefs: [ + { + "targets": [ 0, 6, 7, 8 ], + "visible": false, + "searchable": false + }, + ] + }); + window.queue = $('#queue').DataTable({ + autoWidth: false, + columns: [ + null, + {width:"20%"}, + {width:"20%"}, + {width:"1em"}, + {width:"20%"}, + ], + paging: true, + ordering: false, + searching: false, + select: { + style: 'single' + }, + columnDefs: [ + { + "targets": [ 0, 6, 7, 8 ], + "visible": false, + "searchable": false + }, + ], + }).on("select", function(e, dt, type, indexes) { + var item = queue.row(indexes).data(); + playTrack(item[0], item[6]); + }); + + window.AUDIO_CODECS_SUPPORTED = []; + + var audioPlayer = document.getElementById("player"); + for (var j = 0; j < window.AUDIO_CODECS.length; j++) { + if (audioPlayer.canPlayType(window.AUDIO_CODECS[j])) { + window.AUDIO_CODECS_SUPPORTED.push(AUDIO_CODECS[j]); + } + } + + console.info("This browser supports following audio codecs:", window.AUDIO_CODECS_SUPPORTED); + $("#search").keyup(function() { + if (prev == $("#search").val()) + return; + prev = $("#search").val(); + clearTimeout(window.timeout); + window.timeout = setTimeout(function() { + onDelayedSearch(); + }, 200); + }); +});