Index: Mirage/MirageDbMigrator.cs
===================================================================
--- Mirage/MirageDbMigrator.cs	(revision 85)
+++ Mirage/MirageDbMigrator.cs	(working copy)
@@ -26,7 +26,7 @@
 using System.Data;
 using System.Reflection;
 using Mono.Data;
-using Mono.Data.SqliteClient;
+using Mono.Data.Sqlite;
 
 namespace Mirage
 {
Index: Mirage/Db.cs
===================================================================
--- Mirage/Db.cs	(revision 85)
+++ Mirage/Db.cs	(working copy)
@@ -23,7 +23,7 @@
 
 using System.Data;
 using Mono.Data;
-using Mono.Data.SqliteClient;
+using Mono.Data.Sqlite;
 using System.IO;
 using System;
 using System.Collections;
@@ -113,7 +113,7 @@
             
             try {
                 dbcmd.ExecuteNonQuery();
-            } catch (SqliteExecutionException) {
+            } catch (SqliteException) {
                 throw new DbFailureException();
             } finally {
                 // unlock db mutex
@@ -131,7 +131,7 @@
             
             try {
                 dbcmd.ExecuteNonQuery();
-            } catch (SqliteExecutionException) {
+            } catch (SqliteException) {
                 throw new DbFailureException();
             } finally {
                 // unlock db mutex
@@ -155,7 +155,7 @@
             
             try {
                 dbcmd.ExecuteNonQuery();
-            } catch (SqliteExecutionException) {
+            } catch (SqliteException) {
                 throw new DbFailureException();
             } finally {
                 // unlock db mutex
@@ -274,7 +274,7 @@
             try {
                 dbcmd.ExecuteNonQuery();
                 was_reset = true;
-            } catch (SqliteExecutionException) {
+            } catch (SqliteException) {
                 throw new DbFailureException();
             } finally {
                 // unlock db mutex
Index: Mirage/Mir.cs
===================================================================
--- Mirage/Mir.cs	(revision 85)
+++ Mirage/Mir.cs	(working copy)
@@ -39,6 +39,8 @@
         private const int melcoefficients = 36;
         private const int mfcccoefficients = 20;
         private const int secondstoanalyze = 120;
+ 
+        private static Dictionary<int, float> ignoredlist = new Dictionary<int, float>();
         
         private static Mfcc mfcc = new Mfcc(windowsize, samplingrate, melcoefficients, mfcccoefficients);
 
@@ -49,6 +51,21 @@
             ad.CancelDecode();
         }
 
+        public static void ClearIgnore()
+        {
+            ignoredlist.Clear();
+        }
+
+        public static Dictionary<int, float> IgnoredList
+        {
+            // tracks that are below the specified distance ceiling will be added
+            // to this Dictionary so it can be retreived and cleared after processing by
+            // the Banshee extension.
+            get {
+                return ignoredlist;
+            }
+        }
+
         public static Scms Analyze(string file)
         {
             
@@ -73,6 +90,11 @@
 
         public static int[] SimilarTracks(int[] id, int[] exclude, Db db, int length)
         {
+            return SimilarTracks(id, exclude, db, length, 0);
+        }
+
+        public static int[] SimilarTracks(int[] id, int[] exclude, Db db, int length, float distceiling)
+        {
             // Get Seed-Song SCMS
             Scms[] seedScms = new Scms[id.Length];
             for (int i = 0; i < seedScms.Length; i++) {
@@ -115,10 +137,14 @@
                             break;
                         }
                     }
-                    
-                    if (d >= 0) {
+       
+
+                    if (d >= distceiling) {
                         ht.Add(mapping[i], d/count);
-                    }
+                    } else {
+                        // distance below distance ceiling, add to ignore list for reference
+                        ignoredlist.Add(mapping[i], d);
+                    }  
                 }
                 
             }
@@ -136,7 +162,7 @@
             long stop = 0;
             t.Stop(ref stop);
             Dbg.WriteLine("Mirage - playlist in: {0}ms", stop);
-            
+
             return keys;
         }
     }
Index: Banshee/SimilarityCalculator.cs
===================================================================
--- Banshee/SimilarityCalculator.cs	(revision 85)
+++ Banshee/SimilarityCalculator.cs	(working copy)
@@ -35,6 +35,17 @@
         PlaylistGeneratorSource.UpdatePlaylistDelegate play;
         int length;
         Db db;
+
+        // Mir.SimilarTracks() returns a sorted list of tracks sorted by
+        // a "distance" value -- how closely the seed track and another
+        // track is related. legitimate library duplicates such as a track
+        // being on a studio album and a greatest hits album should have a
+        // distance value of <1 with the current scms model. (the highest
+        // difference in my library was 0.74 with no false positives.
+        // A larger value will catch things such as instrumental/karaoke
+        // versions (the highest difference in my library was 3.92), though
+        // it also will false positive non duplicates. 
+        float distceiling = 0.75f;
         
         public SimilarityCalculator(int[] trackId, int[] excludeTrackId,
                 Db db, PlaylistGeneratorSource.UpdatePlaylistDelegate playlist, int length)
@@ -50,7 +61,7 @@
         {
             int[] playlist;
             try {
-                playlist = Mir.SimilarTracks(trackId, excludeTrackId, db, length*4);
+                playlist = Mir.SimilarTracks(trackId, excludeTrackId, db, length*4, distceiling);
                 play(playlist, length);
             } catch (DbTrackNotFoundException) {
                 Log.Error ("Mirage: ERROR. Track not found in Mirage DB");
Index: Banshee/DuplicateFilter.cs
===================================================================
--- Banshee/DuplicateFilter.cs	(revision 0)
+++ Banshee/DuplicateFilter.cs	(revision 0)
@@ -0,0 +1,79 @@
+/*
+ * Mirage - High Performance Music Similarity and Automatic Playlist Generator
+ * http://hop.at/mirage
+ * 
+ * Copyright (C) 2007 Dominik Schnitzer <dominik@schnitzer.at>
+ *           (C) 2008 Bertrand Lorentz <bertrand.lorentz@gmail.com>
+ * 
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ * 
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+
+using Hyena;
+using Mirage;
+
+namespace Banshee.Mirage
+{
+    public class DuplicateFilter
+    {
+        int Id;
+        PlaylistGeneratorSource.DuplicateCheckDelegate handler;
+        Db db;
+
+        // Mir.SimilarTracks() returns a sorted list of tracks sorted by
+        // a "distance" value -- how closely the seed track and another
+        // track is related. legitimate library duplicates such as a track
+        // being on a studio album and a greatest hits album should have a
+        // distance value of <1 with the current scms model. (the highest
+        // difference in my library was 0.74 with no false positives.
+        // A larger value will catch things such as instrumental/karaoke
+        // versions (the highest difference in my library was 3.92), though
+        // it also will false positive non duplicates. 
+        float distceiling = 0.75f;
+        
+        public DuplicateFilter (int id, Db db, PlaylistGeneratorSource.DuplicateCheckDelegate update)
+        {
+            this.Id = id;
+            this.handler = update;
+            this.db = db;
+        }
+
+        public void ComputeDuplicates()
+        {
+            int i = 0;
+            int[] trackId = new int[1];
+            trackId[0] = Id;
+            lock (db) {
+                try {
+                    Mir.SimilarTracks(trackId, trackId, db, 20, distceiling);
+
+                    Dictionary<int, float> ship = new Dictionary<int, float>();
+                    foreach (KeyValuePair<int, float> pair in Mir.IgnoredList) {
+                        ship.Add(pair.Key, pair.Value);
+                    }
+                    handler(trackId[0], ship);
+                    Mir.ClearIgnore();
+                } catch {
+                }
+            }            
+            i++;
+        }        
+
+    }
+}
Index: Banshee/res/GlobalUI.xml
===================================================================
--- Banshee/res/GlobalUI.xml	(revision 85)
+++ Banshee/res/GlobalUI.xml	(working copy)
@@ -3,4 +3,4 @@
         <menuitem action="ClearMiragePlaylistAction"></menuitem>
         <menuitem action="ClearMiragePlaylistOnQuitAction"></menuitem>
     </popup>
-</ui>
\ No newline at end of file
+</ui>
Index: Banshee/Makefile.am
===================================================================
--- Banshee/Makefile.am	(revision 85)
+++ Banshee/Makefile.am	(working copy)
@@ -4,7 +4,8 @@
 SOURCES = \
 	MiragePlugin.cs \
 	PlaylistGeneratorSource.cs \
-	SimilarityCalculator.cs
+	SimilarityCalculator.cs \
+	DuplicateFilter.cs
 
 RESOURCES = \
 	res/cp.png \
Index: Banshee/PlaylistGeneratorSource.cs
===================================================================
--- Banshee/PlaylistGeneratorSource.cs	(revision 85)
+++ Banshee/PlaylistGeneratorSource.cs	(working copy)
@@ -31,6 +31,8 @@
 
 using Hyena;
 using Hyena.Data.Sqlite;
+using Hyena.Collections;
+
 using Banshee.Collection;
 using Banshee.Collection.Database;
 using Banshee.Gui;
@@ -50,6 +52,9 @@
         public static List<DatabaseTrackInfo> seeds = new List<DatabaseTrackInfo>();
         protected List<DatabaseTrackInfo> tracksOverride = new List<DatabaseTrackInfo>();
         protected List<DatabaseTrackInfo> skipped = new List<DatabaseTrackInfo>();
+
+        List<int> dupefilterqueue = new List<int>();
+
         protected DatabaseTrackInfo processed;
         protected Db db;
         InterfaceActionService action_service;
@@ -111,6 +116,8 @@
                     
             action_service.PlaybackActions["NextAction"].Activated += OnNextPrevAction;
             action_service.PlaybackActions["PreviousAction"].Activated += OnNextPrevAction;
+        
+            action_service.TrackActions["RemoveTracksAction"].Activated += OnRemoveTracks;
             
             ServiceManager.PlayerEngine.ConnectEvent (OnPlayerEvent, 
                                                       PlayerEvent.StartOfStream | 
@@ -168,6 +175,7 @@
 
         protected void UpdatePlaylist (int[] playlist, int length)
         {
+
             Gtk.Application.Invoke(delegate {
                 if (playlist == null) {
                     SetStatus (AddinManager.CurrentLocalizer.GetString ("Error building playlist. You might need to rescan your music collection."), true);
@@ -175,6 +183,7 @@
                 }
                 
                 lock (DatabaseTrackModel) {
+
                     RemoveTrackRange (DatabaseTrackModel, new Hyena.Collections.RangeCollection.Range (current_track + 1, Count - 1));
                     int sameArtistCount = 0;
                     int i = 0;
@@ -193,7 +202,18 @@
                             i++;
                         
                         AddTrack(track);
+                        dupefilterqueue.Add(track.TrackId);
+                    }              
+
+                    if (Mir.IgnoredList.Count > 0) {
+                        UpdateIgnoreList(-1, IgnoreListDeepCopy());
+                        Mir.ClearIgnore();
+                        
                     }
+                    Log.DebugFormat("Mirage - Checking for duplicates of tracks in new playlist");
+                    RunDuplicateFilter();
+         
+                                   
                     SetStatus (AddinManager.CurrentLocalizer.GetString ("Playlist ready."), false, false, null);
                 }
                 
@@ -247,7 +267,9 @@
         {
             current_track = 0;
             RemoveTrackRange ((DatabaseTrackListModel)TrackModel, new Hyena.Collections.RangeCollection.Range (0, Count));
+            skipped.Clear();
             tracksOverride.Clear ();
+            Mir.ClearIgnore();
             Reload ();
         }
 
@@ -271,7 +293,33 @@
         {
             skipped.Add(ServiceManager.PlayerEngine.CurrentTrack as DatabaseTrackInfo);
         }
-        
+
+        public void OnRemoveTracks(object o, EventArgs e)
+        {
+            if (ServiceManager.SourceManager.ActiveSource != this) {
+                return;
+            }
+
+            // add removed tracks to exclude list
+
+            ITrackModelSource source = ServiceManager.SourceManager.ActiveSource as ITrackModelSource;
+
+            IEnumerator sE = source.TrackModel.Selection.GetEnumerator();
+            while (sE.MoveNext())
+            {
+                DatabaseTrackInfo dT = (DatabaseTrackInfo)DatabaseTrackModel[(int)sE.Current];
+
+                try {
+                    if (!skipped.Exists(delegate(DatabaseTrackInfo t) { return t.TrackId == dT.TrackId; })) {
+                        Log.DebugFormat ("Mirage - Adding \"{0}\" on \"{1}\" to exclude list (not permanent)", dT.TrackTitle,
+                            dT.AlbumTitle);
+			            skipped.Add(dT);
+                    }
+                } catch {
+                }
+            }
+        }
+       
         private void Refresh ()
         {
             processed = ServiceManager.PlayerEngine.CurrentTrack as DatabaseTrackInfo;
@@ -304,13 +352,16 @@
         {
             int[] trackIds = new int[tracks.Count];
             for (int i = 0; i < tracks.Count; i++) {
-                Log.DebugFormat ("Mirage - Looking for similars to {0}-{1}", tracks[i].TrackId, tracks[i].TrackTitle);
+                Log.DebugFormat ("Mirage - Looking for similars to [{0}] \"{1}\"", tracks[i].TrackId, tracks[i].TrackTitle);
                 trackIds[i] = tracks[i].TrackId;
             }
 
+            if (exclude.Count > 2) {
+                exclude.Sort(delegate(DatabaseTrackInfo t1, DatabaseTrackInfo t2) { return t1.TrackTitle.CompareTo(t2.TrackTitle); });
+            }
             int[] excludeTrackIds = new int[exclude.Count];
             for (int i = 0; i < exclude.Count; i++) {
-                Log.DebugFormat ("Mirage - Excluding {0}-{1}", exclude[i].TrackId, exclude[i].TrackTitle);
+                Log.DebugFormat ("Mirage - Excluding [{0}] \"{1}\" on \"{2}\"", exclude[i].TrackId, exclude[i].TrackTitle, exclude[i].AlbumTitle);
                 excludeTrackIds[i] = exclude[i].TrackId;
             }
 
@@ -322,6 +373,80 @@
             similarityCalculatorThread.Start();
         }
 
+        private void RunDuplicateFilter()
+        {
+            if (dupefilterqueue.Count == 0) {
+                Log.DebugFormat ("Mirage - done scanning new playlist for duplicates");
+                return;
+            }
+
+            if (Mir.IgnoredList.Count > 0) {
+                UpdateIgnoreList(-1, IgnoreListDeepCopy());
+                Mir.ClearIgnore();
+            }
+
+            if (!skipped.Exists(delegate(DatabaseTrackInfo t) { return t.TrackId == dupefilterqueue[0]; })) {
+                DuplicateFilter df = new DuplicateFilter(dupefilterqueue[0], db, DuplicateCheck);
+                Thread duplicateFilterThread = new Thread(new ThreadStart(df.ComputeDuplicates));
+                duplicateFilterThread.Start();
+                dupefilterqueue.Remove(dupefilterqueue[0]);
+            } else {
+                dupefilterqueue.Remove(dupefilterqueue[0]);
+                if (dupefilterqueue.Count == 0) {
+                    Log.DebugFormat ("Mirage - done scanning new playlist for duplicates");
+                    return;
+                }
+            }
+
+            
+        }
+
+        public delegate void DuplicateCheckDelegate (int s, Dictionary<int, float> list);
+
+        protected void DuplicateCheck(int s, Dictionary<int, float> list)
+        {
+            Gtk.Application.Invoke(delegate {
+                UpdateIgnoreList(s, list);
+                RunDuplicateFilter();
+            });
+        }
+
+        protected void UpdateIgnoreList(int s, Dictionary<int, float> list)
+        {
+                // import any ignored tracks from Mir to add to our exclude list for future iterations
+                if (list.Count > 0) {
+                    int seed;
+                    int id;
+                    float distance;
+
+                    if (s < 0) {
+                        seed = seeds[seeds.Count-1].TrackId;
+                    } else {
+                        seed = s;
+                    }
+
+                    DatabaseTrackInfo sTrack = DatabaseTrackInfo.Provider.FetchSingle(seed);
+              
+                    foreach (KeyValuePair<int, float> pair in list) {
+                        id = pair.Key;
+                        distance = pair.Value;
+             
+                        DatabaseTrackInfo dTrack = DatabaseTrackInfo.Provider.FetchSingle(id);
+
+                        // check to see if it's in our skipped/exclude list. if not add.
+                        if (!skipped.Exists(delegate(DatabaseTrackInfo t) { return t.TrackId == id; })) {
+                            Log.DebugFormat("Mirage - Considering [{0}] \"{1}\" on \"{2}\" a duplicate of [{3}] \"{4}\" on \"{5}\" (distance: {6})",
+                                dTrack.TrackId, dTrack.TrackTitle, dTrack.AlbumTitle, sTrack.TrackId, sTrack.TrackTitle,
+                                sTrack.AlbumTitle, distance);
+						    skipped.Add(dTrack);
+                        }
+
+                        // check to see if the duplicate is in our current playlist, if so remove
+                        // ... or not. This may require changes in Banshee
+                    }
+                }
+        }
+
         private void OnPlayerEvent(PlayerEventArgs args)
         {
             if (ServiceManager.SourceManager.ActiveSource != this) {
@@ -395,5 +520,14 @@
         public override bool ShowBrowser {
             get { return false; }
         }
+
+        public Dictionary<int, float> IgnoreListDeepCopy() {
+            Dictionary<int, float> ship = new Dictionary<int, float>();
+            foreach (KeyValuePair<int, float> pair in Mir.IgnoredList) {
+                ship.Add(pair.Key, pair.Value);
+            }
+            return ship;
+        }
+                
     }
 }
