first thing not entirely tested

ext_open/searcher.py

@@ -4,6 +4,7 @@ import re
 import update as _update
 from config import config
 import time
+import threading
 # https://stackoverflow.com/a/5365533
 osp = os.path
 DB_PATH = osp.join(osp.dirname(__file__),'ext.db')
@@ -11,6 +12,7 @@ ext_database = sqlite3.connect(DB_PATH)
 def init_db():
     cur = ext_database.cursor()
     cur.execute('CREATE TABLE IF NOT EXISTS updates ( `ext` TEXT, `update_time` INTEGER, `file_count` INTEGER);')
+    cur.execute('CREATE TABLE IF NOT EXISTS `update_status` ( `ext` TEXT UNIQUE, `status` INTEGER, PRIMARY KEY(`ext`);')
 
 def regexp(expr, item,case):
     if case:
@@ -31,6 +33,8 @@ class Searcher:
     def __init__(self,ext,case_sensitive=True):
         self.cur = ext_database.cursor()
         self.ext = ext
+        self.updating = False
+        self.update_thread = None
         self.case = case_sensitive
         self.index = '{}_index'.format(ext)
         self.__init_table__()
@@ -38,25 +42,57 @@ class Searcher:
     def __init_table__(self):
         self.cur.execute('CREATE TABLE IF NOT EXISTS {} (`name` TEXT,`fullpath` TEXT, `update_time` INTEGER,UNIQUE(name,fullpath));'.format(self.ext))
         self.cur.execute('CREATE INDEX IF NOT EXISTS {} ON {} ( `name`, `fullpath` )'.format(self.index,self.ext))
+        self.cur.execute('INSERT OR IGNORE INTO update_status VALUES (?, 0)',[self.ext])
+);
         self.cur.execute(
             'INSERT INTO updates (ext,update_time,file_count) SELECT :ext, 0, 0 WHERE NOT EXISTS(SELECT 1 FROM updates WHERE ext = :ext AND update_time = 0 AND file_count = 0);',
             {'ext':self.ext}
             )
         self.commit()
-    def update(self,update_mtimes):
-        self.cur.execute('SELECT MAX(update_time) FROM updates WHERE ext=?;',[self.ext])
-        last_update = self.cur.fetchone()[0]
-        # input(str(last_update))
-        _time = int(time.time())
-        files = _update.update(
-            self.ext,
-            config.getlist(self.ext,'paths'),
-            last_update,
-            update_mtimes
-            )
-        self.cur.executemany('INSERT OR IGNORE INTO {} VALUES (?,?,?)'.format(self.ext),(row + [_time] for row in files))
-        self.cur.execute('INSERT INTO updates VALUES (?, ?, ?)',[self.ext,_time,len(files)])
-        self.commit()
+    def update(self,update_mtimes=False):
+        if not self.get_updating(ext_database):
+            self.update_thread = threading.Thread(target = self._update_,args = (update_mtimes,))
+            self.update_thread.start()
+    def set_updating(self,con,updating = True):
+        cur = con.cursor()
+        if updating:
+            status = 1
+        else:
+            status = 0
+        cur.execute('UPDATE update_status SET status=? WHERE ext=?;'[self.ext,self.status])
+        con.commit()
+    def get_updating(self,con):
+        cur = con.cursor()
+        cur.execute('SELECT status FROM update_status WHERE ext=?;',[self.ext])
+        return cur.fetchone()[0][0] == 1
+
+    def _update_(self,update_mtimes):
+        con = sqlite3.connect(DB_PATH)
+        try:
+            self.set_updating(con)
+            thread_cur = con.cursor()
+            thread_cur.execute('SELECT MAX(update_time) FROM updates WHERE ext=?;',[self.ext])
+            last_update = thread_cur.fetchone()[0]
+            # input(str(last_update))
+            _time = int(time.time())
+            params = (
+                '.'+self.ext,
+                config.getlist(self.ext,'paths'),
+                last_update,
+                update_mtimes
+                )
+            files = list(_update.update(*params))
+            count = 0
+            for count,row in enumerate(files):
+                row.append(_time)
+                thread_cur.execute('INSERT OR IGNORE INTO {} VALUES (?,?,?);'.format(self.ext),row)
+                if (count+1) % 30 == 0:
+                    con.commit()
+            thread_cur.execute('INSERT INTO updates VALUES (?, ?, ?);',[self.ext,_time,count+1])
+            self.set_updating(con,False)
+        except Exception as e:
+            self.set_updating(con,False)
+            raise e
 
     def commit(self):
         ext_database.commit()

ext_open/update.py

@@ -7,7 +7,9 @@ from config import config
 import subprocess
 
 FORCE_ADMIN = not config.getboolean('global','no_admin',fallback=False)
-
+TEST = True
+if TEST:
+    import time
 
 def is_admin():
     return ctypes.windll.shell32.IsUserAnAdmin() != 0
@@ -26,32 +28,29 @@ def update_mtimes(starts):
 def build(after,starts):
     if isinstance(after,datetime.datetime):
         after = after.timestamp()
-    var_unsubbed =  map(osp.expandvars,starts)
-    abs_paths = map(osp.abspath,var_unsubbed)
-    starts = list(no_parents(abs_paths))
-    ret = []
+    starts =  map(osp.expandvars,starts)
+    starts = map(lambda item: item.replace('"','').replace("'",''),starts)
     for path in starts:
         for root,dirs,files in os.walk(path,topdown=True):
             try:
+                for file in files:
+                    yield os.path.join(root,file)
                 dirs[:] = list(filter(
                     lambda p: osp.getmtime(osp.join(root,p)) > after,
                     dirs
                     ))
-                ret += [osp.join(root,file) for file in files]
             except:
                 print('Error',root,sep= ': ')
-    return ret
 
 def build_ext(ext,paths):
-    ret = []
     for path in paths:
         if path.endswith(ext):
             name = osp.basename(path)
-            ret.append([name,path])
-    return ret
+            yield [name,path]
 
 def update(ext,starts,after,do_update_mtimes = False):
     if do_update_mtimes:
         update_mtimes(starts)
     files = build(after,starts)
-    return build_ext(ext,files)
+    for file in build_ext(ext,files):
+        yield file

ext_open/util.py

@@ -1,5 +1,6 @@
 from ctypes import windll
 import os
+import re
 # https://stackoverflow.com/a/827397
 def get_drives():
     uppercase = map(chr,range(ord('A'),ord('A')+26))
@@ -11,7 +12,6 @@ def get_drives():
         bitmask >>= 1
 
     return drives
-
 def no_parents(paths):
     paths = sorted(paths,key=len)
     ret = []