lauri/mjugrep
1
0
Fork 0
Code Issues Pull Requests Activity

Initial commit

Browse Source
This commit is contained in:
Lauri Võsandi 2019-05-28 00:42:56 +03:00
commit bcd72b2648
5 changed files with 748 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
README.md Normal file
View File

@ -0,0 +1,41 @@
# Features
Music streaming server for my quirky needs:
* No damn apps
* No damn force fed indexing
* No damn overengineering
* Yes works pretty much like grep for searching tracks
* Yes damn scalable, tested to work with 56k+ tracks
* Yes just damn web application, use your browser
* Optional metadata caching system
* Optional transcode support
# Dependencies
On FreeBSD:
```bash
pkg install py36-mutagen py36-Flask py36-click-7.0 ffmpeg nginx madplay gmake vorbis-tools
cd /usr/ports/audio/lame/ && make install
```
# Running
To run self-contained web server:
```bash
python3 main.py serve --root /path/to/library
```
To cache metainformation in index.xspf playlist files:
```bash
python3 main.py metagen --root /path/to/library
```
# Known bugs
FLAC playback stops abruptly in Firefox. Seems to be a common problem on the web.
Use Chrome or Chromium, you need the performance of Webkit to handle lots of search results anyway.

369
main.py Normal file
View File

@ -0,0 +1,369 @@
import os
from flask import Flask, Response, request
import re
import json
import click
from time import time
from flask import send_from_directory
def which(*names):
for d in os.environ.get("PATH").split(":"):
for name in names:
path = os.path.join(d, name)
if os.path.exists(path):
return path
raise ValueError("Couldn't find %s anywhere in $PATH" % names)
MADPLAY = which("madplay")
CURL = which("curl")
FLAC = which("flac")
LAME = which("lame")
FFMPEG = which("avconv", "ffmpeg")
OGGDEC = which("oggdec")
OGGENC = which("oggenc")
TYPES = {
"mp3": "audio/mpeg",
"flac": "audio/flac",
"oga": "audio/ogg",
"ogg": "audio/ogg",
"wma": "audio/x-ms-wma",
"wav": "audio/wave",
}
RE_URL = re.compile(
r'^https?://' # http:// or https://
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
r'localhost|' # localhost...
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
r'(?::\d+)?' # optional port
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
XML_ENTITIES = (
("&", "&"),
('"', """),
("'", "'"),
(">", ">"),
("<", "&lt;"),
)
def escape_xml_entities(j):
for letter, replacement in XML_ENTITIES:
j = j.replace(letter, replacement)
return j
def generate_xspf(root, names):
names.sort()
yield "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
yield "<playlist xmlns=\"http://xspf.org/ns/0/\" version=\"1\">"
yield "<meta rel=\"timestamp\">%d</meta>" % time()
yield "<trackList>"
for name in names:
path = os.path.join(root, name)
basename, extension = os.path.splitext(name)
import mutagen
import mutagen.mp3
try:
if name.lower().endswith(".mp3"):
tags = mutagen.mp3.EasyMP3(path)
else:
tags = mutagen.File(path)
except mutagen.MutagenError:
click.echo("Failed to parse metainformation for %s" % repr(path))
# Fetch stream attributes
sample_rate = getattr(tags.info, "sample_rate", None)
duration = getattr(tags.info, "length", None)
channels = getattr(tags.info, "channels", None)
bitrate = getattr(tags.info, "bitrate", None)
bit_depth = getattr(tags.info, "bits_per_sample", None)
# Workaround for calculating bitrate
if not bitrate and duration:
bitrate = int(item.size / duration) / 1000 * 8000
# Artist
artist_name = tags.pop("artist", (None,))[0]
if not artist_name:
artist_name = tags.pop("----:com.apple.iTunes:Artist Credit", (None,))[0]
# Track album
album_title = tags.pop("album", (None,))[0]
if not album_title:
album_title = tags.pop("----:com.apple.iTunes:Album Credit", (None,))[0]
# Album artist
album_artist_name = None
if not album_artist_name:
album_artist_name = tags.pop("----:com.apple.iTunes:Album Artist Credit", (None,))[0]
# Track title
track_title = tags.pop("title", (None,))[0]
# Track year
year, = tags.pop("date", (None,))
if not year:
year, = tags.pop("----:com.apple.iTunes:ORIGINAL YEAR", (None,))
def parse_x_of_y(mixed):
if not mixed:
return None, None
elif "\x00" in mixed: # Probably bug in mutagen
return None, None
elif re.match("\s*\d+\s*$", mixed):
return int(mixed), None
elif re.match("\s*\d+\s*/\s*\d+\s*$", mixed):
return [int(j) for j in mixed.split("/", 1)]
elif re.match("\s*/\s*\d+\s*$", mixed):
return None, int(mixed.split("/", 1)[1])
return None, None
# Disc numbers
disc_number, disc_count = None, None
disc, = tags.pop("discnumber", (None,))
if disc:
disc_number, disc_count = parse_x_of_y(disc)
# Release country
country_code, = tags.pop("releasecountry", (None,))
if not country_code:
country_code, = tags.pop("----:com.apple.iTunes:MusicBrainz Album Release Country", (None,))
# Track number could be in various formats: '1', '1/12', 'B2'
track_number = tags.pop("tracknumber", (None,))[0]
track_number, track_count = parse_x_of_y(track_number)
if not track_number:
(track_number, track_count), = tags.pop("trkn", ((None,None),))
# Normalize country code
if country_code:
country_code = country_code.lower()
if country_code in ("xe", "xw"):
country_code = None
yield "<track>"
yield "<location>%s</location>" % escape_xml_entities(name)
if duration:
yield "<duration>%d</duration>" % (duration * 1000)
if track_title:
yield "<title>%s</title>" % escape_xml_entities(track_title)
if artist_name:
yield "<creator>%s</creator>" % escape_xml_entities(artist_name)
if album_title:
yield "<album>%s</album>" % escape_xml_entities(album_title)
else:
yield "<title>%s</title>" % escape_xml_entities(basename)
if track_number:
yield "<trackNum>%d</trackNum>" % track_number
if track_count:
yield "<meta rel=\"track-count\">%d</meta>" % track_count
if disc_number:
yield "<meta rel=\"disc-number\">%d</meta>" % disc_number
if disc_count:
yield "<meta rel=\"disc-count\">%d</meta>" % disc_count
if year:
yield "<meta rel=\"year\">%s</meta>" % year
if album_artist_name:
yield "<meta rel=\"album-artist\">%s</meta>" % escape_xml_entities(album_artist_name)
yield "<meta rel=\"mimetype\">%s</meta>" % TYPES[extension[1:].lower()]
yield "<meta rel=\"bitrate\">%s</meta>" % bitrate
yield "<meta rel=\"bit_depth\">%s</meta>" % bit_depth
yield "<meta rel=\"sample_rate\">%s</meta>" % sample_rate
yield "</track>"
yield "</trackList>"
yield "</playlist>"
@click.command("metagen")
@click.option('--root', '-r', prompt='Path to media directory', help='Root directory to index')
@click.option('--force', '-f', is_flag=True, help="Reindex even if up to date according to timestamps")
def metagen(root, force):
for abspath, dirs, files in os.walk(root):
playlist = os.path.join(abspath, "index.xspf")
needed = False
relevant_files = []
for name in files:
for extension, mimetype in TYPES.items():
if name.lower().endswith("." + extension):
needed = True
relevant_files.append(name)
if "index.xspf" in files:
# TODO proper handling here, microsecond differences
if not force and int(os.stat(abspath).st_mtime) <= int(os.stat(playlist).st_mtime):
click.echo("Folder %s is up to date" % repr(abspath))
needed = False
if not relevant_files:
click.echo("No more relevant files, removing %s" % playlist)
os.remove(playlist)
if "index.xspf.part" in files:
click.echo("Cleaning up %s.part" % playlist)
os.remove(playlist + ".part")
if needed:
with open(playlist + ".part", "w") as fh:
for chunk in generate_xspf(abspath, relevant_files):
fh.write(chunk)
os.rename(playlist + ".part", playlist)
click.echo("Generated %s" % repr(playlist))
@click.command("serve")
@click.option('--root', prompt='Path to media directory', help='Root directory to serve')
def serve(root):
app = Flask(__name__)
@app.route("/api/transcode/")
def transcode():
def build_pipeline(source, output_mimetype):
sys.stderr.write("Transcoding from: %s\n" % source)
basename, extension = os.path.splitext(source)
extension = extension.lower()
try:
input_mimetype = TYPES.get(extension)
except:
raise ValueError("Unknown input file extension:", extension)
sys.stderr.write("Input mimetype: %s\n" % input_mimetype)
sys.stderr.write("Output mimetype: %s\n" % output_mimetype)
pipeline = []
if source.startswith("http://") or source.startswith("https://"):
# Escape path component
url = urlparse.urlparse(source)
source = url.scheme + "://" + url.hostname.encode("idna") + urllib.quote(url.path)
yield CURL, "--insecure", "--silent", "--limit-rate", "300k", source # 1Mbps is pretty max
else:
# Perhaps we could read file directly here if transcoder and web server are in the same machine?
raise ValueError("Won't transcode:" + source)
if input_mimetype == "audio/wave":
pass # Do nothing
elif input_mimetype == "audio/mpeg":
yield MADPLAY, "-", "-o", "wave:-"
elif input_mimetype == "audio/ogg":
yield OGGDEC, "-", "-o", "-"
elif input_mimetype == "audio/flac":
if output_mimetype != "audio/ogg": # oggenc handles FLAC input aswell
yield FLAC, "--decode", "-", "--stdout", "--silent"
else:
# Use FFMPEG/avconv as fallback for decoding wma
yield FFMPEG, "-i", "-", "-f", "wav", "-", "-loglevel", "quiet"
if output_mimetype == "audio/wave":
pass
elif output_mimetype == "audio/mpeg":
yield LAME, "--quiet", "-aq", "2", "-", "-"
elif output_mimetype == "audio/ogg":
yield OGGENC, "-", "-o", "-"
else:
raise ValueError("Don't know how to encode mimetype: %s" % output_mimetype)
mimetype = request.args.get("mimetype")
pipeline = tuple(build_pipeline(request.args.get("url"), mimetype))
first = True
for element in pipeline:
if not first:
print("|",)
first = False
print(" ".join(repr(j) if " " in j else j for j in element))
print()
process = None
for element in pipeline:
process = Popen(element, stdout=PIPE, stdin=process.stdout if process else PIPE, close_fds=True)
return send_file(process.stdout, "stream", mimetype)
@app.route("/api/search/")
def search():
expr = re.compile(request.args.get('expr', 'metallica'), re.IGNORECASE)
limit = request.args.get('limit', 100)
def generate():
yield 'retry:999999999999\n' # Tell browser to never reconnect
total = 0
hits = 0
counter = 0
for abspath, dirs, files in os.walk(root):
dirs.sort()
relpath = os.path.relpath(abspath, root)
hit = False
filtered_filenames = []
for name in files:
lower_name = name.lower()
for extension, mimetype in TYPES.items():
if lower_name.endswith("." + extension):
filtered_filenames.append(name)
total += 1
if expr.search(name):
hit = True
hits += 1
if expr.search(relpath):
hit = True
if hit and filtered_filenames:
yield "event:chdir\ndata:"
yield relpath
yield "\n\n"
yield "event:xspf\n"
yield "data:"
if "index.xspf" in files:
with open(os.path.join(abspath, "index.xspf")) as fh:
yield fh.read()
else:
yield from generate_xspf(abspath, filtered_filenames)
yield "\n"
yield "\n"
counter += 1
if counter > limit:
break
if counter > limit:
break
yield "event:done\n"
yield "data:%s\n" % json.dumps({"relevant_files":total, "hits":hits})
yield "\n"
return Response(generate(), mimetype='text/event-stream')
@app.route('/api/stream/<path:path>')
def send_stream(path):
return send_from_directory(root, path)
@app.route('/<path:path>')
def send_js(path):
return send_from_directory('static', path)
app.run(host= '0.0.0.0', debug=True, threaded=True)
@click.group()
def entry_point(): pass
entry_point.add_command(serve)
entry_point.add_command(metagen)
if __name__ == '__main__':
entry_point()

79
static/index.html Normal file
View File

@ -0,0 +1,79 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.8.2/css/all.css" integrity="sha384-oS3vJWv+0UjzBfQzYUhtDYW+Pj2yciDJxpsK1OYPAYjqT085Qq/1cq5FLXAZQ7Ay" crossorigin="anonymous">
<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/dt/dt-1.10.18/datatables.min.css"/>
<link rel="stylesheet" type="text/css" href="main.css"/>
<title>µgrep</title>
</head>
<body>
<audio id="player" style="display:none;"></audio>
<div class="overflow-auto" style="display: block; padding: 4px; position: absolute; bottom: 50%; height: 50%; top: 0; width: 100%;">
<div class="btn-toolbar" role="toolbar" aria-label="Toolbar with button groups">
<div class="btn-group mr-2" role="group" aria-label="First group">
<input class="form-control" type="text" placeholder="Search" aria-label="Search" id="search">
<button id="queue-selection" type="button" class="btn btn-primary" title="Queue selected items"><i class="fa fa-plus"></i></button>
<button id="queue-all" type="button" class="btn btn-primary" title="Queue all search results"><i class="fa fa-cart-plus"></i></button>
</div>
</div>
<div style="top:3em; bottom:0; left:0; right:0; position: absolute; padding:4px;">
<table id="results" class="table table-striped table-bordered pageResize display" style="width:100%;">
<thead>
<tr>
<th>URL</th>
<th>Artist</th>
<th>Album</th>
<th>#</th>
<th>Title</th>
<th>Duration</th>
<th>MIME</th>
<th>Bitrate</th>
<th>Quality</th>
</tr>
</thead>
<tbody>
</tbody>
</table>
</div>
</div>
<div class="overflow-auto" style="display: block; padding: 4px; position: absolute; top: 50%; bottom: 0; width: 100%;">
<div class="btn-toolbar" role="toolbar" aria-label="Toolbar with button groups">
<div class="btn-group mr-2" role="group" aria-label="First group">
<button id="playback-prev" type="button" class="btn btn-primary"><i class="fa fa-step-backward"></i></button>
<button type="button" class="btn btn-primary"><i class="fa fa-stop"></i></button>
<button type="button" class="btn btn-primary"><i class="fa fa-pause"></i></button>
<button type="button" class="btn btn-primary"><i class="fa fa-play"></i></button>
<button id="playback-next" type="button" class="btn btn-primary"><i class="fa fa-step-forward"></i></button>
</div>
<div class="btn-group mr-2" role="group" aria-label="Second group">
<button type="button" class="btn btn-secondary" data-toggle="button" title="Repeat"><i class="fa fa-redo"></i></button>
<button type="button" class="btn btn-secondary" data-toggle="button" title="Play in random order"><i class="fa fa-random"></i></button>
</div>
<div class="btn-group" role="group" aria-label="Third group">
<button type="button" class="btn btn-secondary" title="Clear queue" id="queue-clear"><i class="fa fa-trash"></i></button>
<button type="button" class="btn btn-secondary" data-toggle="button" title="Always transcode to save bandwidth"><i class="fa fa-piggy-bank"></i></button>
</div>
</div>
<input type="range" min="1" max="100" value="50" class="slider" id="seek" style="width:100%;">
<div style="top:6em; bottom:0; left:0; right:0; position: absolute; padding:0;">
<table id="queue" class="table table-striped table-bordered pageResize" style="width:100%;">
<thead>
</thead>
<tbody>
</tbody>
</table>
</div>
</div>
<script src="https://code.jquery.com/jquery-3.4.1.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js" integrity="sha384-UO2eT0CpHqdSJQ6hJty5KVphtPhzWj9WO1clHTMGa3JDZwrnQq4sF86dIHNDz0W1" crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js" integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM" crossorigin="anonymous"></script>
<script type="text/javascript" src="https://cdn.datatables.net/v/dt/dt-1.10.18/datatables.min.js"></script>
<script type="text/javascript" src="https://cdn.datatables.net/buttons/1.5.6/js/dataTables.buttons.min.js"></script>
<script type="text/javascript" src="https://cdn.datatables.net/select/1.3.0/js/dataTables.select.min.js"></script>
<script src="//cdn.datatables.net/plug-ins/1.10.19/features/pageResize/dataTables.pageResize.min.js" type="text/javascript"></script>
<script src="main.js"></script>
</body>
</html>

30
static/main.css Normal file
View File

@ -0,0 +1,30 @@
.maximize {
position:absolute;
left:0;
top:0;
bottom:0;
right: 0;
}
body {
font-family: sans;
}
table td, table th {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
font-size: 11pt;
padding: 3pt 7pt !important;
}
div.dataTables_length {
display: none;
}
table {
cursor: pointer;
}

229
static/main.js Normal file
View File

@ -0,0 +1,229 @@
// Probe for supported codecs
AUDIO_CODECS = [
'audio/ogg',
'audio/mpeg',
'audio/flac',
];
timeout = null;
es = null;
prev = null;
function humanize_duration(d) {
var seconds = Math.floor(d % 60);
var minutes = Math.floor(d / 60) % 60;
var hours = Math.floor(d / 3600);
if (seconds < 10) seconds = '0' + seconds;
if (minutes < 10) minutes = '0' + minutes;
if (hours < 10) hours = '0' + hours;
return hours + ":" + minutes + ":" + seconds;
}
function onDelayedSearch() {
var expr = $("#search").val();
if (expr == window.expr) return;
console.info("Searching", expr);
results.rows().clear().draw(false);
if (es) {
es.close();
$("#results tbody").empty();
}
es = new EventSource("/api/search/?expr=" + expr);
// Close socket event listener
es.addEventListener("close", function(e) {
console.info("Closing search socket");
});
// Open socket event listener
es.addEventListener("open", function(e) {
console.info("Opened event stream for search expression", expr);
})
// Search end result marker
es.addEventListener("done", function(e) {
var totals = JSON.parse(e.data)
console.info("Finished searching, total", totals.relevant_files, "relevant files and", totals.hits, "hits");
es.close();
})
// Keep track of relative path of the XSPF blobs
es.addEventListener("chdir", function(e) {
console.info("Moving to directory:", e.data);
window.relpath = e.data;
})
// If XSPF blob is received append it to search results
es.addEventListener("xspf", function(e) {
var $trackMetadatas = $("playlist trackList track", $.parseXML(e.data));
$trackMetadatas.each(function(index, element) {
var location = element.getElementsByTagName("location")[0];
var filename = location.innerHTML;
// Decode escaped XML
var filename = decodeURIComponent(location.innerHTML);
if (filename.indexOf("/") >= 0) {
filename = filename.substring(filename.lastIndexOf("/") + 1);
}
// Escape quotes for jQuery
filename = filename.replace("\"", "\\\"");
results.row.add([
window.relpath + "/" + filename,
$("creator", element).html() || "-",
$("album", element).html() || "-",
$("trackNum", element).html() || "-",
$("title", element).html() || "-",
humanize_duration(parseInt($("duration", element).html())/1000),
$("meta[rel='mimetype']", element).html(),
Math.floor(parseInt($("meta[rel='bitrate']", element).html())/1000)+"kbps",
// $("meta[rel='bit_depth']", element).html() + "bit @ " + $("meta[rel='sample_rate']", element).html(),
"bla"
]).on('click', 'tr', function () {
$(this).toggleClass('selected');
});
clearTimeout(window.timeoutDraw);
window.timeoutDraw = setTimeout(results.draw, 100);
});
});
}
function playNextAudioTrack() {
queue.row(queue.row({selected:true}).index()+1).select();
}
function playPrevAudioTrack() {
queue.row(queue.row({selected:true}).index()-1).select();
}
function playTrack(url, mimetype) {
var url = "/api/stream/" + url;
console.info("Playing:", url, mimetype);
$("#seek").val(0);
$("#offset").html("00:00:00");
$("#player").unbind("ended").bind("ended", function () {
playNextAudioTrack();
});
$("#player").empty();
if (window.AUDIO_CODECS_SUPPORTED.indexOf(mimetype) >= 0) {
$("#player").append("<source src=\"" + url + "\" type=\"" + mimetype + "\"/>");
} else {
console.info("Codec", mimetype, "not supported by browser");
}
for (var j = 0; j < window.AUDIO_CODECS_SUPPORTED.length; j++) {
var alternative = window.AUDIO_CODECS_SUPPORTED[j];
if (mimetype != alternative) {
$("#player").append("<source src=\"/transcode/?mimetype=" + alternative + "&url=" + url.replace("&", "%26") + "\" type=\"" + alternative + "\"/>");
}
}
$("#player").trigger("load");
$("#player").trigger("play");
}
$(document).ready(function() {
$('#results tbody').on('dblclick', 'tr', function () {
console.info("Got double click");
queue.rows.add(results.rows({ selected: true }).data());
queue.rows.add([results.row(this).data()]);
results.rows().deselect();
results.draw(false);
queue.draw(false);
});
$("#playback-next").on("click", function() {
playNextAudioTrack();
});
$("#playback-prev").on("click", function() {
playPrevAudioTrack();
});
$("#queue-selection").on("click", function() {
queue.rows.add(results.rows({ selected: true }).data()).draw(false);
});
$("#queue-all").on("click", function() {
queue.rows.add(results.data()).draw(false);
});
$("#queue thead").html($("#results thead").html());
$("#queue-clear").on("click", function() {
queue.rows().clear().draw(false);
});
window.results = $('#results').DataTable({
autoWidth: false,
columns: [
null,
{width:"20%"},
{width:"20%"},
{width:"1em"},
{width:"20%"},
],
paging: true,
ordering: false,
searching: false,
select: {
style: 'multi'
},
columnDefs: [
{
"targets": [ 0, 6, 7, 8 ],
"visible": false,
"searchable": false
},
]
});
window.queue = $('#queue').DataTable({
autoWidth: false,
columns: [
null,
{width:"20%"},
{width:"20%"},
{width:"1em"},
{width:"20%"},
],
paging: true,
ordering: false,
searching: false,
select: {
style: 'single'
},
columnDefs: [
{
"targets": [ 0, 6, 7, 8 ],
"visible": false,
"searchable": false
},
],
}).on("select", function(e, dt, type, indexes) {
var item = queue.row(indexes).data();
playTrack(item[0], item[6]);
});
window.AUDIO_CODECS_SUPPORTED = [];
var audioPlayer = document.getElementById("player");
for (var j = 0; j < window.AUDIO_CODECS.length; j++) {
if (audioPlayer.canPlayType(window.AUDIO_CODECS[j])) {
window.AUDIO_CODECS_SUPPORTED.push(AUDIO_CODECS[j]);
}
}
console.info("This browser supports following audio codecs:", window.AUDIO_CODECS_SUPPORTED);
$("#search").keyup(function() {
if (prev == $("#search").val())
return;
prev = $("#search").val();
clearTimeout(window.timeout);
window.timeout = setTimeout(function() {
onDelayedSearch();
}, 200);
});
});