diff --git a/ext_open/ext.db-journal b/ext_open/ext.db-journal new file mode 100644 index 0000000..a73675e Binary files /dev/null and b/ext_open/ext.db-journal differ diff --git a/ext_open/searcher.py b/ext_open/searcher.py index 4d6619f..9c26c3b 100644 --- a/ext_open/searcher.py +++ b/ext_open/searcher.py @@ -4,6 +4,7 @@ import re import update as _update from config import config import time +import threading # https://stackoverflow.com/a/5365533 osp = os.path DB_PATH = osp.join(osp.dirname(__file__),'ext.db') @@ -11,6 +12,7 @@ ext_database = sqlite3.connect(DB_PATH) def init_db(): cur = ext_database.cursor() cur.execute('CREATE TABLE IF NOT EXISTS updates ( `ext` TEXT, `update_time` INTEGER, `file_count` INTEGER);') + cur.execute('CREATE TABLE IF NOT EXISTS `update_status` ( `ext` TEXT UNIQUE, `status` INTEGER, PRIMARY KEY(`ext`);') def regexp(expr, item,case): if case: @@ -31,6 +33,8 @@ class Searcher: def __init__(self,ext,case_sensitive=True): self.cur = ext_database.cursor() self.ext = ext + self.updating = False + self.update_thread = None self.case = case_sensitive self.index = '{}_index'.format(ext) self.__init_table__() @@ -38,25 +42,57 @@ class Searcher: def __init_table__(self): self.cur.execute('CREATE TABLE IF NOT EXISTS {} (`name` TEXT,`fullpath` TEXT, `update_time` INTEGER,UNIQUE(name,fullpath));'.format(self.ext)) self.cur.execute('CREATE INDEX IF NOT EXISTS {} ON {} ( `name`, `fullpath` )'.format(self.index,self.ext)) + self.cur.execute('INSERT OR IGNORE INTO update_status VALUES (?, 0)',[self.ext]) +); self.cur.execute( 'INSERT INTO updates (ext,update_time,file_count) SELECT :ext, 0, 0 WHERE NOT EXISTS(SELECT 1 FROM updates WHERE ext = :ext AND update_time = 0 AND file_count = 0);', {'ext':self.ext} ) self.commit() - def update(self,update_mtimes): - self.cur.execute('SELECT MAX(update_time) FROM updates WHERE ext=?;',[self.ext]) - last_update = self.cur.fetchone()[0] - # input(str(last_update)) - _time = int(time.time()) - files = _update.update( - self.ext, - config.getlist(self.ext,'paths'), - last_update, - update_mtimes - ) - self.cur.executemany('INSERT OR IGNORE INTO {} VALUES (?,?,?)'.format(self.ext),(row + [_time] for row in files)) - self.cur.execute('INSERT INTO updates VALUES (?, ?, ?)',[self.ext,_time,len(files)]) - self.commit() + def update(self,update_mtimes=False): + if not self.get_updating(ext_database): + self.update_thread = threading.Thread(target = self._update_,args = (update_mtimes,)) + self.update_thread.start() + def set_updating(self,con,updating = True): + cur = con.cursor() + if updating: + status = 1 + else: + status = 0 + cur.execute('UPDATE update_status SET status=? WHERE ext=?;'[self.ext,self.status]) + con.commit() + def get_updating(self,con): + cur = con.cursor() + cur.execute('SELECT status FROM update_status WHERE ext=?;',[self.ext]) + return cur.fetchone()[0][0] == 1 + + def _update_(self,update_mtimes): + con = sqlite3.connect(DB_PATH) + try: + self.set_updating(con) + thread_cur = con.cursor() + thread_cur.execute('SELECT MAX(update_time) FROM updates WHERE ext=?;',[self.ext]) + last_update = thread_cur.fetchone()[0] + # input(str(last_update)) + _time = int(time.time()) + params = ( + '.'+self.ext, + config.getlist(self.ext,'paths'), + last_update, + update_mtimes + ) + files = list(_update.update(*params)) + count = 0 + for count,row in enumerate(files): + row.append(_time) + thread_cur.execute('INSERT OR IGNORE INTO {} VALUES (?,?,?);'.format(self.ext),row) + if (count+1) % 30 == 0: + con.commit() + thread_cur.execute('INSERT INTO updates VALUES (?, ?, ?);',[self.ext,_time,count+1]) + self.set_updating(con,False) + except Exception as e: + self.set_updating(con,False) + raise e def commit(self): ext_database.commit() diff --git a/ext_open/update.py b/ext_open/update.py index d2368fa..46f9053 100644 --- a/ext_open/update.py +++ b/ext_open/update.py @@ -7,7 +7,9 @@ from config import config import subprocess FORCE_ADMIN = not config.getboolean('global','no_admin',fallback=False) - +TEST = True +if TEST: + import time def is_admin(): return ctypes.windll.shell32.IsUserAnAdmin() != 0 @@ -26,32 +28,29 @@ def update_mtimes(starts): def build(after,starts): if isinstance(after,datetime.datetime): after = after.timestamp() - var_unsubbed = map(osp.expandvars,starts) - abs_paths = map(osp.abspath,var_unsubbed) - starts = list(no_parents(abs_paths)) - ret = [] + starts = map(osp.expandvars,starts) + starts = map(lambda item: item.replace('"','').replace("'",''),starts) for path in starts: for root,dirs,files in os.walk(path,topdown=True): try: + for file in files: + yield os.path.join(root,file) dirs[:] = list(filter( lambda p: osp.getmtime(osp.join(root,p)) > after, dirs )) - ret += [osp.join(root,file) for file in files] except: print('Error',root,sep= ': ') - return ret def build_ext(ext,paths): - ret = [] for path in paths: if path.endswith(ext): name = osp.basename(path) - ret.append([name,path]) - return ret + yield [name,path] def update(ext,starts,after,do_update_mtimes = False): if do_update_mtimes: update_mtimes(starts) files = build(after,starts) - return build_ext(ext,files) \ No newline at end of file + for file in build_ext(ext,files): + yield file \ No newline at end of file diff --git a/ext_open/util.py b/ext_open/util.py index 9334bff..c748202 100644 --- a/ext_open/util.py +++ b/ext_open/util.py @@ -1,5 +1,6 @@ from ctypes import windll import os +import re # https://stackoverflow.com/a/827397 def get_drives(): uppercase = map(chr,range(ord('A'),ord('A')+26)) @@ -11,7 +12,6 @@ def get_drives(): bitmask >>= 1 return drives - def no_parents(paths): paths = sorted(paths,key=len) ret = []