Enhancement: compute the correlation between two matches instead of only counting the matches (#1)

* tmp: add correlation

* cli: add a record command

* add missing apply window of the Hamming window function

* enhancement: Add a scoring function that uses the correlation + the count of matches for better results when matching against the database

* cli: listen cmd should remove the temporary file created

* spectrogram: make the windowing function optional to allow for easy testing

Author: sfluor

.gitignore

@@ -15,3 +15,5 @@ bin/*
 
 # Ignore dataset by default
 assets/dataset/*
+
+.idea/

cmd/musig/cmd/listen.go

@@ -1,12 +1,11 @@
 package cmd
 
 import (
-	"io/ioutil"
+	log "github.com/sirupsen/logrus"
 	"os"
-	"os/signal"
+	"path"
 	"time"
 
-	"github.com/sfluor/musig/internal/pkg/sound"
 	"github.com/spf13/cobra"
 )
 
@@ -20,33 +19,17 @@ var listenCmd = &cobra.Command{
 	Use:   "listen",
 	Short: "listen will record the microphone input and try to find a matching song from the database (Ctrl-C will stop the recording)",
 	Run: func(cmd *cobra.Command, args []string) {
-		tmpFile, err := ioutil.TempFile(os.TempDir(), "musig_record.wav")
-		failIff(err, "error creating temporary file for recording in %s", tmpFile.Name())
-		defer os.Remove(tmpFile.Name())
-
-		stopCh := make(chan struct{}, 1)
-		sig := make(chan os.Signal, 1)
-		signal.Notify(sig, os.Interrupt, os.Kill)
-
+		name := path.Join(os.TempDir(), "musig_record.wav")
 		dur, err := cmd.Flags().GetDuration("duration")
 		failIff(err, "could not get duration, got: %v", dur)
 
-		go func() {
-			defer func() { stopCh <- struct{}{} }()
-			for {
-				select {
-				case <-time.After(dur):
-					return
-				case <-sig:
-					return
-				}
+		defer func() {
+			if err := os.Remove(name); err != nil {
+				log.Errorf("Failed to remove temporary file stored at %s used to record the sample: %s", name , err)
 			}
 		}()
 
-		err = sound.RecordWAV(tmpFile, stopCh)
-		failIff(err, "an error occured recording WAV file")
-		failIff(tmpFile.Sync(), "error syncing temp file")
-
-		cmdRead(tmpFile.Name())
+		recordAudioToFile(name, dur)
+		cmdRead(name)
 	},
 }

cmd/musig/cmd/load.go

@@ -12,6 +12,7 @@ import (
 func init() {
 	rootCmd.AddCommand(loadCmd)
 	loadCmd.Flags().BoolP("dry-run", "d", false, "disable saving to the database")
+	loadCmd.Flags().BoolP("reset", "r", false, "reset the database if it already exists")
 	loadCmd.Flags().BoolP("verbose", "v", false, "enable verbose output")
 }
 
@@ -29,6 +30,14 @@ var loadCmd = &cobra.Command{
 			os.Exit(0)
 		}
 
+		resetDB, err := cmd.Flags().GetBool("reset")
+		if resetDB && err == nil {
+			log.Info("removing the existing database...")
+			if err := os.Remove(dbFile); err != nil {
+				log.Errorf("Error removing the database at %s: %s", dbFile, err)
+			}
+		}
+
 		p, err := pipeline.NewDefaultPipeline(dbFile)
 		failIff(err, "error creating pipeline")
 		defer p.Close()

cmd/musig/cmd/read.go

@@ -2,9 +2,9 @@ package cmd
 
 import (
 	"fmt"
-
 	"github.com/sfluor/musig/internal/pkg/model"
 	"github.com/sfluor/musig/internal/pkg/pipeline"
+	"github.com/sfluor/musig/pkg/dsp"
 	"github.com/spf13/cobra"
 )
 
@@ -29,34 +29,47 @@ func cmdRead(file string) {
 	res, err := p.Process(file)
 	failIff(err, "error processing file %s", file)
 
-	// Will hold a count of songID => occurences
-	counts := map[uint32]int{}
+	// Will hold a count of songID => occurrences
 	keys := make([]model.EncodedKey, 0, len(res.Fingerprint))
+	sample := map[model.EncodedKey]model.TableValue{}
+	// songID => points that matched
+	matches := map[uint32]map[model.EncodedKey]model.TableValue{}
 
-	for k := range res.Fingerprint {
+	for k, v := range res.Fingerprint {
 		keys = append(keys, k)
+		sample[k] = v
 	}
 
 	m, err := p.DB.Get(keys)
-	for _, values := range m {
+	for key, values := range m {
 		for _, val := range values {
-			counts[val.SongID] += 1
+
+			if _, ok := matches[val.SongID]; !ok {
+				matches[val.SongID] = map[model.EncodedKey]model.TableValue{}
+			}
+
+			matches[val.SongID][key] = val
 		}
 	}
 
+	// songID => correlation
+	scores := map[uint32]float64{}
+	for songID, points := range matches {
+		scores[songID] = dsp.MatchScore(sample, points)
+	}
+
 	var song string
-	var max, total int
+	var max float64
 	fmt.Println("Matches:")
-	for id, count := range counts {
+	for id, score := range scores {
 		name, err := p.DB.GetSong(id)
 		failIff(err, "error getting song id: %d", id)
-		fmt.Printf("\t- %s, count: %d\n", name, count)
-		if count > max {
-			song, max = name, count
+		fmt.Printf("\t- %s, score: %f\n", name, score)
+		if score > max {
+			song, max = name, score
 		}
-		total += count
 	}
 
 	fmt.Println("---")
-	fmt.Printf("Song is: %s (count: %d, pct: %.2f %%)\n", song, max, 100*float64(max)/float64(total))
+	fmt.Printf("Song is: %s (score: %f)\n", song, max)
 }

cmd/musig/cmd/record.go

@@ -0,0 +1,55 @@
+package cmd
+
+import (
+	"os"
+	"os/signal"
+	"time"
+
+	"github.com/sfluor/musig/internal/pkg/sound"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(recordCmd)
+	recordCmd.Flags().DurationP("duration", "d", 10*time.Second, "duration of the listening")
+}
+
+// recordCmd represents the listen command
+var recordCmd = &cobra.Command{
+	Use:   "record",
+	Short: "record will record the microphone input and save the signal to the given file",
+	Args:  cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		dur, err := cmd.Flags().GetDuration("duration")
+		failIff(err, "could not get duration, got: %v", dur)
+
+		recordAudioToFile(args[0], dur)
+	},
+}
+
+func recordAudioToFile(name string, duration time.Duration) {
+	file, err := os.Create(name)
+	failIff(err, "error creating file for recording in %s", name)
+
+	stopCh := make(chan struct{}, 1)
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, os.Interrupt, os.Kill)
+
+	go func() {
+		defer func() {
+			stopCh <- struct{}{}
+		}()
+		for {
+			select {
+			case <-time.After(duration):
+				return
+			case <-sig:
+				return
+			}
+		}
+	}()
+
+	err = sound.RecordWAV(file, stopCh)
+	failIff(err, "an error occurred recording WAV file")
+	failIff(file.Sync(), "error syncing temp file")
+}

cmd/musig/cmd/spec.go

@@ -36,7 +36,7 @@ func genSpectrogram(path string, imgPath string) error {
 	}
 	defer file.Close()
 
-	s := dsp.NewSpectrogrammer(model.DOWNSAMPLERATIO, model.MAXFREQ, model.SAMPLESIZE)
+	s := dsp.NewSpectrogrammer(model.DownsampleRatio, model.MaxFreq, model.SampleSize, true)
 
 	spec, _, err := s.Spectrogram(file)
 	if err != nil {

internal/pkg/db/bolt.go

@@ -68,7 +68,7 @@ func (db *BoltDB) Get(keys []model.EncodedKey) (map[model.EncodedKey][]model.Tab
 		return nil
 	})
 
-	return res, errors.Wrap(err, "an error occured when reading from bolt")
+	return res, errors.Wrap(err, "an error occurred when reading from bolt")
 }
 
 // Set stores the list of (key, value) into the bolt file
@@ -90,7 +90,7 @@ func (db *BoltDB) Set(batch map[model.EncodedKey]model.TableValue) error {
 		return nil
 	})
 
-	return errors.Wrap(err, "an error occured when writing to bolt")
+	return errors.Wrap(err, "an error occurred when writing to bolt")
 }
 
 // GetSongID does a song name => songID lookup in the database
@@ -113,7 +113,7 @@ func (db *BoltDB) GetSongID(name string) (uint32, error) {
 		return fmt.Errorf("could not find id for song name: %s", name)
 	})
 
-	return id, errors.Wrap(err, "an error occured when reading from bolt")
+	return id, errors.Wrap(err, "an error occurred when reading from bolt")
 }
 
 // GetSong does a songID => song name lookup in the database
@@ -133,7 +133,7 @@ func (db *BoltDB) GetSong(songID uint32) (string, error) {
 		return nil
 	})
 
-	return name, errors.Wrap(err, "an error occured when reading from bolt")
+	return name, errors.Wrap(err, "an error occurred when reading from bolt")
 }
 
 // SetSong stores a song name in the database and returns it's song ID
@@ -163,7 +163,7 @@ func (db *BoltDB) SetSong(song string) (uint32, error) {
 		return errors.Wrap(b.Put(rawKey, []byte(song)), "error setting song")
 	})
 
-	return songID, errors.Wrap(err, "an error occured when writing to bolt")
+	return songID, errors.Wrap(err, "an error occurred when writing to bolt")
 }
 
 func itob(s uint32) []byte {

internal/pkg/fingerprint/fingerprint_test.go

@@ -22,12 +22,13 @@ func TestFingerprinting440And880(t *testing.T) {
 }
 
 func testFingerprintingOnFile(t *testing.T, path string) {
-	sampleSize := model.SAMPLESIZE
+	sampleSize := model.SampleSize
 
 	s := dsp.NewSpectrogrammer(
-		model.DOWNSAMPLERATIO,
-		model.MAXFREQ,
+		model.DownsampleRatio,
+		model.MaxFreq,
 		sampleSize,
+		true,
 	)
 
 	file, err := os.Open(path)

internal/pkg/fingerprint/simple.go

@@ -34,7 +34,7 @@ func (sf *SimpleFingerprinter) Fingerprint(songID uint32, cMap []model.Constella
 	for i := 0; i+sf.lastOffset < length; i++ {
 		anchor := cMap[i]
 		for _, p := range cMap[i+sf.anchorOffset : i+sf.lastOffset] {
-			res[model.NewTableKey(anchor, p).Encode()] = *model.NewTableValue(songID, anchor)
+			res[model.NewAnchorKey(anchor, p).Encode()] = *model.NewTableValue(songID, anchor)
 		}
 	}
 

internal/pkg/model/model.go

@@ -6,14 +6,14 @@ import (
 	"strings"
 )
 
-// DOWNSAMPLERATIO is the default down sample ratio (4)
-const DOWNSAMPLERATIO = 4
+// DownsampleRatio is the default down sample ratio (4)
+const DownsampleRatio = 4
 
-// SAMPLESIZE is the default sample size (1024)
-const SAMPLESIZE = 1024.0
+// SampleSize is the default sample size (1024)
+const SampleSize = 1024.0
 
-// MAXFREQ is 5kHz
-const MAXFREQ = 5000.0
+// MaxFreq is 5kHz
+const MaxFreq = 5000.0
 
 // ConstellationPoint represents a point in the constellation map (time + frequency)
 type ConstellationPoint struct {

internal/pkg/model/table.go

@@ -8,16 +8,16 @@ import (
 
 // Used to down size the frequences to a 9 bit ints
 // XXX: we could also use 10.7Hz directly
-const freqStep = MAXFREQ / float64(1<<9)
+const freqStep = MaxFreq / float64(1<<9)
 
 // Used to down size the delta times to 14 bit ints (we use 16s as the max duration)
 const deltaTimeStep = 16 / float64(1<<14)
 
 // TableValueSize represents the TableValueSize when encoded in bytes
 const TableValueSize = 8
 
-// TableKey represents a table key
-type TableKey struct {
+// AnchorKey represents a anchor key
+type AnchorKey struct {
 	// Frequency of the anchor point for the given point's target zone
 	AnchorFreq float64
 	// Frequency of the given point
@@ -29,9 +29,9 @@ type TableKey struct {
 // EncodedKey represents an encoded key
 type EncodedKey uint32
 
-// NewTableKey creates a new table key from the given anchor and the given point
-func NewTableKey(anchor, point ConstellationPoint) *TableKey {
-	return &TableKey{
+// NewAnchorKey creates a new anchor key from the given anchor and the given point
+func NewAnchorKey(anchor, point ConstellationPoint) *AnchorKey {
+	return &AnchorKey{
 		AnchorFreq: anchor.Freq,
 		PointFreq:  point.Freq,
 		// Use absolute just in case anchor is after the target zone
@@ -47,13 +47,13 @@ func (ek EncodedKey) Bytes() []byte {
 	return bk
 }
 
-// Encode encodes the table key using:
+// Encode encodes the anchor key using:
 // 9 bits for the “frequency of the anchor”: fa
 // 9 bits for the ” frequency of the point”: fp
 // 14 bits for the ”delta time between the anchor and the point”: dt
 // The result is then dt | fa | fp
 // XXX: this only works if frequencies are coded in 9 bits or less (if we used a 1024 samples FFT, it will be the case)
-func (tk *TableKey) Encode() EncodedKey {
+func (tk *AnchorKey) Encode() EncodedKey {
 	// down size params
 	fp := uint32(tk.PointFreq / freqStep)
 	fa := uint32(tk.AnchorFreq / freqStep)

internal/pkg/pipeline/pipeline.go

@@ -24,7 +24,7 @@ func NewDefaultPipeline(dbFile string) (*Pipeline, error) {
 	if err != nil {
 		return nil, errors.Wrapf(err, "error connection to database at: %s", dbFile)
 	}
-	s := dsp.NewSpectrogrammer(model.DOWNSAMPLERATIO, model.MAXFREQ, model.SAMPLESIZE)
+	s := dsp.NewSpectrogrammer(model.DownsampleRatio, model.MaxFreq, model.SampleSize, true)
 	fpr := fingerprint.NewDefaultFingerprinter()
 
 	return &Pipeline{

internal/pkg/sound/wav.go

@@ -5,8 +5,8 @@ import (
 
 	"github.com/gordonklaus/portaudio"
 	"github.com/pkg/errors"
-	riff "github.com/youpy/go-riff"
-	wav "github.com/youpy/go-wav"
+	"github.com/youpy/go-riff"
+	"github.com/youpy/go-wav"
 )
 
 var _ Reader = &WAVReader{}