diff --git a/INSTALLATION.md b/INSTALLATION.md index 34161ac2..6c82933b 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -21,6 +21,7 @@ Install the dependencies: sudo yum install numpy scipy python-matplotlib ffmpeg portaudio-devel pip install PyAudio pip install pydub + pip install audioread # https://github.com/sampsyo/audioread Now setup virtualenv ([howto?](http://www.pythoncentral.io/how-to-install-virtualenv-python/)): diff --git a/dejavu/__init__.py b/dejavu/__init__.py index 6194cecb..09872d8b 100755 --- a/dejavu/__init__.py +++ b/dejavu/__init__.py @@ -6,6 +6,24 @@ import traceback import sys +import shutil +import subprocess +import os.path +from dejavu.decoder import get_duration + +def assure_path_exists(path): + if not os.path.isdir(path): + os.makedirs(path) + +class SplitError(Exception): + def __init__(self, file_path, output_file, error_code): + Exception.__init__(self) + self.file_path = file_path + self.error_code = error_code + self.output_file = output_file + + def __str__(self): + return "Spliting of file({0}) failed to ({1}). ffmpeg returned error code: {2}".format(self.file_path, self.output_file, self.error_code) class Dejavu(object): @@ -16,6 +34,11 @@ class Dejavu(object): OFFSET = 'offset' OFFSET_SECS = 'offset_seconds' + SPLIT_DIR = "split_dir" + SLICE_LIMIT_WHEN_SPLITTING = 3 # in minutes + LIMIT_CPU_CORES_FOR_SPLITS = 3 + OVERWRITE_TEMP_FILES_WHEN_SPLITING = 1 + def __init__(self, config): super(Dejavu, self).__init__() @@ -43,7 +66,7 @@ def get_fingerprinted_songs(self): song_name = song[self.db.FIELD_SONGNAME] self.songnames_set.add(song_name) - def fingerprint_directory(self, path, extensions, nprocesses=None): + def fingerprint_directory(self, path, extensions, nprocesses=None, treat_as_split=False, song_name_for_the_split=""): # Try to use the maximum amount of processes if not given. try: nprocesses = nprocesses or multiprocessing.cpu_count() @@ -71,6 +94,8 @@ def fingerprint_directory(self, path, extensions, nprocesses=None): # Send off our tasks iterator = pool.imap_unordered(_fingerprint_worker, worker_input) + if treat_as_split and song_name_for_the_split: + sid = self.db.insert_song(song_name_for_the_split) # Loop till we have all of them while True: @@ -85,8 +110,8 @@ def fingerprint_directory(self, path, extensions, nprocesses=None): # Print traceback because we can't reraise it here traceback.print_exc(file=sys.stdout) else: - sid = self.db.insert_song(song_name) - + if not treat_as_split: + sid = self.db.insert_song(song_name) self.db.insert_hashes(sid, hashes) self.db.set_song_fingerprinted(sid) self.get_fingerprinted_songs() @@ -111,6 +136,48 @@ def fingerprint_file(self, filepath, song_name=None): self.db.set_song_fingerprinted(sid) self.get_fingerprinted_songs() + def fingerprint_with_duration_check(self, input_file, song_name=None): + duration = get_duration(input_file) + split_length = self.SLICE_LIMIT_WHEN_SPLITTING * 60 + if duration < split_length: + return self.fingerprint_file(input_file) + songname, extension = os.path.splitext(os.path.basename(input_file)) + song_name = song_name or songname + # don't refingerprint already fingerprinted files + if song_name in self.songnames_set: + print "%s already fingerprinted, continuing..." % song_name + return + file_directory = os.path.dirname(input_file) + output_path = os.path.join(file_directory, self.SPLIT_DIR, song_name) + assure_path_exists(output_path) + start_offset = 0 + end_offset = split_length + retcode = 0 + sid = self.db.insert_song(song_name) + while start_offset < duration: + output_file = os.path.join(output_path, "start_sec{0}_end_sec{1}{2}".format(start_offset, end_offset, extension)) + convertion_command = [ 'ffmpeg', + '-i', input_file, + "-acodec", "copy", #fastest convertion possible 1:1 copy + ["-n","-y"][self.OVERWRITE_TEMP_FILES_WHEN_SPLITING], # always overwrite existing files + "-vn", # Drop any video streams if there are any + '-ss', str(start_offset), + '-t', str(split_length), + output_file] + retcode = subprocess.call(convertion_command, stderr=open(os.devnull)) + if retcode != 0: + raise SplitError(input_file, output_file, retcode) + start_offset += split_length + end_offset += split_length + end_offset = min(end_offset, duration) + + self.db.set_song_fingerprinted(sid) + self.get_fingerprinted_songs() + self.fingerprint_directory(output_path, [extension], + nprocesses=self.LIMIT_CPU_CORES_FOR_SPLITS, + treat_as_split=True, song_name_for_the_split=song_name) + shutil.rmtree(output_path) + def find_matches(self, samples, Fs=fingerprint.DEFAULT_FS): hashes = fingerprint.fingerprint(samples, Fs=Fs) return self.db.return_matches(hashes) diff --git a/dejavu/decoder.py b/dejavu/decoder.py index 830b8f7b..3e845777 100755 --- a/dejavu/decoder.py +++ b/dejavu/decoder.py @@ -5,6 +5,19 @@ from pydub.utils import audioop import wavio +# pip install audioread +# https://github.com/sampsyo/audioread +import audioread + +def get_duration(file_path): + duration = 0 + with audioread.audio_open(file_path) as f: + duration = f.duration + f.close() + return duration + + + def find_files(path, extensions): # Allow both with ".mp3" and without "mp3" to be used for extensions extensions = [e.replace(".", "") for e in extensions] diff --git a/example.py b/example.py index 991c6a96..22c651c9 100755 --- a/example.py +++ b/example.py @@ -6,29 +6,30 @@ # load config from a JSON file (or anything outputting a python dictionary) with open("dejavu.cnf.SAMPLE") as f: config = json.load(f) +if __name__ == '__main__': -# create a Dejavu instance -djv = Dejavu(config) + # create a Dejavu instance + djv = Dejavu(config) -# Fingerprint all the mp3's in the directory we give it -djv.fingerprint_directory("mp3", [".mp3"]) + # Fingerprint all the mp3's in the directory we give it + djv.fingerprint_directory("mp3", [".mp3"]) -# Recognize audio from a file -from dejavu.recognize import FileRecognizer -song = djv.recognize(FileRecognizer, "mp3/Sean-Fournier--Falling-For-You.mp3") -print "From file we recognized: %s\n" % song + # Recognize audio from a file + from dejavu.recognize import FileRecognizer + song = djv.recognize(FileRecognizer, "mp3/Sean-Fournier--Falling-For-You.mp3") + print "From file we recognized: %s\n" % song -# Or recognize audio from your microphone for `secs` seconds -from dejavu.recognize import MicrophoneRecognizer -secs = 5 -song = djv.recognize(MicrophoneRecognizer, seconds=secs) -if song is None: - print "Nothing recognized -- did you play the song out loud so your mic could hear it? :)" -else: - print "From mic with %d seconds we recognized: %s\n" % (secs, song) + # Or recognize audio from your microphone for `secs` seconds + from dejavu.recognize import MicrophoneRecognizer + secs = 5 + song = djv.recognize(MicrophoneRecognizer, seconds=secs) + if song is None: + print "Nothing recognized -- did you play the song out loud so your mic could hear it? :)" + else: + print "From mic with %d seconds we recognized: %s\n" % (secs, song) -# Or use a recognizer without the shortcut, in anyway you would like -from dejavu.recognize import FileRecognizer -recognizer = FileRecognizer(djv) -song = recognizer.recognize_file("mp3/Josh-Woodward--I-Want-To-Destroy-Something-Beautiful.mp3") -print "No shortcut, we recognized: %s\n" % song \ No newline at end of file + # Or use a recognizer without the shortcut, in anyway you would like + from dejavu.recognize import FileRecognizer + recognizer = FileRecognizer(djv) + song = recognizer.recognize_file("mp3/Josh-Woodward--I-Want-To-Destroy-Something-Beautiful.mp3") + print "No shortcut, we recognized: %s\n" % song \ No newline at end of file diff --git a/mp3/concatenation_list.txt b/mp3/concatenation_list.txt new file mode 100644 index 00000000..4651ee4c --- /dev/null +++ b/mp3/concatenation_list.txt @@ -0,0 +1,23 @@ +# as described in https://trac.ffmpeg.org/wiki/Concatenate +# you can create a huge length file by concatenating many small ones: +# by calling: +# ffmpeg -f concat -i mp3/concatenation_list.txt -c copy mp3/concatenated.mp3 +# concatenated song will be about 56 minutes long +# +file Brad-Sucks--Total-Breakdown.mp3 +file Choc--Eigenvalue-Subspace-Decomposition.mp3 +file Josh-Woodward--I-Want-To-Destroy-Something-Beautiful.mp3 +file Sean-Fournier--Falling-For-You.mp3 +file The-Lights-Galaxia--While-She-Sleeps.mp3 +# +file Brad-Sucks--Total-Breakdown.mp3 +file Choc--Eigenvalue-Subspace-Decomposition.mp3 +file Josh-Woodward--I-Want-To-Destroy-Something-Beautiful.mp3 +file Sean-Fournier--Falling-For-You.mp3 +file The-Lights-Galaxia--While-She-Sleeps.mp3 +# +file Brad-Sucks--Total-Breakdown.mp3 +file Choc--Eigenvalue-Subspace-Decomposition.mp3 +file Josh-Woodward--I-Want-To-Destroy-Something-Beautiful.mp3 +file Sean-Fournier--Falling-For-You.mp3 +file The-Lights-Galaxia--While-She-Sleeps.mp3 diff --git a/test_fingerprint_by_splitting.py b/test_fingerprint_by_splitting.py new file mode 100644 index 00000000..f825364d --- /dev/null +++ b/test_fingerprint_by_splitting.py @@ -0,0 +1,48 @@ +from dejavu import Dejavu +import warnings +import json +import os, subprocess +warnings.filterwarnings("ignore") + +# load config from a JSON file (or anything outputting a python dictionary) +with open("dejavu.cnf.SAMPLE") as f: + config = json.load(f) + +class ConcatError(Exception): + def __init__(self, list_file, output_file, error_code): + Exception.__init__(self) + self.list_file = list_file + self.error_code = error_code + self.output_file = output_file + + def __str__(self): + return "Problem with list file({0}). Failed to create({1}). ffmpeg returned error code: {2}".format(self.list_file, self.output_file, self.error_code) + + +if __name__ == '__main__': + ''' + Concatenates ./mp3/*.mp3 + Test fingerprinting the long concatenated file + ''' + list_file = "mp3/concatenation_list.txt" + long_song = "mp3/concatenated.mp3" + + concat_mp3_file_for_test = "ffmpeg -f concat -i {0} -y -c copy {1}".format(list_file, long_song) + retcode = subprocess.call(concat_mp3_file_for_test, stderr=open(os.devnull)) + if retcode != 0: + raise ConcatError(list_file, long_song, retcode) + + # create a Dejavu instance + djv = Dejavu(config) + + try: + djv.fingerprint_file(long_song) + except Exception as err: + err = str(err) or "Memory Error" # Memory Errors does not have a string representation (as tested in Windows) + print "Exception raised during common fingerprint_file():({0}) so will split the file".format(err) + else: + raise "This file was successfully ingerprinted and splitting was not needed" + + djv.fingerprint_with_duration_check(long_song, song_name="Concatenates12345") + +