From 29d6c9a7466061abffc9822fffdd66cfd715d6ef Mon Sep 17 00:00:00 2001
From: Avery Pennarun <apenwarr@gmail.com>
Date: Thu, 9 Dec 2010 02:44:33 -0800
Subject: [PATCH] Don't db.commit() so frequently.

Just commit when we're about to do something blocking.  sqlite goes a lot
faster with bigger transactions.  This change does show a small percentage
speedup in tests, but not as much as I'd like.
---
 builder.py |  4 ++++
 state.py   | 53 +++++++++++++++++++++++++++++++----------------------
 2 files changed, 35 insertions(+), 22 deletions(-)

diff --git a/builder.py b/builder.py
index c36b13a..b63fdb0 100644
--- a/builder.py
+++ b/builder.py
@@ -104,6 +104,7 @@ class BuildJob:
         dof = state.File(name=dofile)
         dof.set_static()
         dof.save()
+        state.commit()
         jwack.start_job(t, self._do_subproc, self._after)
 
     def _do_subproc(self):
@@ -184,6 +185,7 @@ class BuildJob:
         try:
             self.donefunc(self.t, rv)
             assert(self.lock.owned)
+            state.commit()
         finally:
             self.lock.unlock()
 
@@ -205,6 +207,7 @@ def main(targets, shouldbuildfunc):
     # In the first cycle, we just build as much as we can without worrying
     # about any lock contention.  If someone else has it locked, we move on.
     for t in targets:
+        state.commit()
         jwack.get_token(t)
         if retcode[0] and not vars.KEEP_GOING:
             break
@@ -246,4 +249,5 @@ def main(targets, shouldbuildfunc):
                 lock.unlock()
             else:
                 BuildJob(t, lock, shouldbuildfunc, done).start()
+    state.commit()
     return retcode[0]
diff --git a/state.py b/state.py
index a449271..661ee3d 100644
--- a/state.py
+++ b/state.py
@@ -1,6 +1,7 @@
 import sys, os, errno, glob, stat, sqlite3
 import vars
 from helpers import unlink, err, debug2, debug3, mkdirp, close_on_exec
+import helpers
 
 SCHEMA_VER=1
 TIMEOUT=60
@@ -47,16 +48,15 @@ def db():
                     "     primary key (target,source))")
         _db.execute("insert into Schema (version) values (?)", [SCHEMA_VER])
         _db.execute("insert into Runid default values")  # eat the '0' runid
-        _db.commit()
 
     if not vars.RUNID:
         _db.execute("insert into Runid default values")
-        _db.commit()
         vars.RUNID = _db.execute("select last_insert_rowid()").fetchone()[0]
         os.environ['REDO_RUNID'] = str(vars.RUNID)
     
     _db.execute("pragma journal_mode = PERSIST")
     _db.execute("pragma synchronous = off")
+    _db.commit()
     return _db
     
 
@@ -70,6 +70,22 @@ def init():
         os.unlink(f)
 
 
+_wrote = 0
+def _write(q, l):
+    global _wrote
+    _wrote += 1
+    #helpers.log_('W: %r %r\n' % (q,l))
+    db().execute(q, l)
+
+
+def commit():
+    global _wrote
+    if _wrote:
+        #helpers.log_("COMMIT (%d)\n" % _wrote)
+        db().commit()
+        _wrote = 0
+
+
 _insane = None
 def is_sane():
     global _insane
@@ -139,8 +155,7 @@ class File(object):
             if not name:
                 raise Exception('File with id=%r not found and '
                                 'name not given' % id)
-            d.execute('insert into Files (name) values (?)', [name])
-            d.commit()
+            _write('insert into Files (name) values (?)', [name])
             row = d.execute(q, l).fetchone()
             assert(row)
         self._init_from_cols(row)
@@ -149,15 +164,14 @@ class File(object):
         if not os.path.exists('%s/.redo' % vars.BASE):
             # this might happen if 'make clean' removes the .redo dir
             return
-        d = db()
-        d.execute('update Files set '
-                  '    is_generated=?, checked_runid=?, changed_runid=?, '
-                  '    stamp=?, csum=? '
-                  '    where rowid=?',
-                  [self.is_generated, self.checked_runid, self.changed_runid,
-                   self.stamp, self.csum,
-                   self.id])
-        d.commit()
+        _write('update Files set '
+               '    is_generated=?, checked_runid=?, changed_runid=?, '
+               '    stamp=?, csum=? '
+               '    where rowid=?',
+               [self.is_generated,
+                self.checked_runid, self.changed_runid,
+                self.stamp, self.csum,
+                self.id])
 
     def set_checked(self):
         self.checked_runid = vars.RUNID
@@ -199,21 +213,16 @@ class File(object):
 
     def zap_deps(self):
         debug2('zap-deps: %r\n' % self.name)
-        db().execute('delete from Deps where target=?', [self.id])
-        db().commit()
+        _write('delete from Deps where target=?', [self.id])
 
     def add_dep(self, mode, dep):
         src = File(name=dep)
         reldep = relpath(dep, vars.BASE)
         debug2('add-dep: %r < %s %r\n' % (self.name, mode, reldep))
         assert(src.name == reldep)
-        d = db()
-        d.execute("delete from Deps where target=? and source=?",
-                  [self.id, src.id])
-        d.execute("insert into Deps "
-                  "    (target, mode, source) values (?,?,?)",
-                  [self.id, mode, src.id])
-        d.commit()
+        _write("insert or replace into Deps "
+               "    (target, mode, source) values (?,?,?)",
+               [self.id, mode, src.id])
 
     def read_stamp(self):
         try: