Add a fetchAll method to statement objects.

Due to not having to bounce in and out of JavaScript, this should make
fetching results from SELECTs significantly faster than manually using step to
iterate over the results.

build.sh

+cd deps/mpool-2.1.0/
+make
+cd ../..
+node-waf clean
+node-waf configure build

deps/mpool-2.1.0/ChangeLog.1

+2006-05-31  Gray Watson  <>
+
+	* Version 2.1.0 released.
+
+	* Added MPOOL_ERROR_PNT_OVER to distinguish between pointer
+	overwrite and mpool structure overwrite.
+
+2005-05-20  Gray Watson  <>
+
+	* Version 2.0.0 released.
+
+	* First external publication of library.
+
+Thu Mar  4 10:14:21 1999  Gray Watson  <>
+
+	* Reworked the way the blocks were split up.
+
+	* Removed the round arguments.  Not used.
+
+Wed Mar  3 19:29:38 1999  Gray Watson  <>
+
+	* Moved to random().  Fucking rand() was hiding a lot of problems
+ 	from me.
+
+	* Added some additional sanity checks in free().
+
+	* Added mpool_set_max_pages to the library.
+
+Thu Feb 25 12:41:51 1999  Gray Watson  <>
+
+	* Added log_function transaction callback.
+
+Thu Feb 25 09:53:33 1999  Gray Watson  <>
+
+	* Changed the default page size to 16 * getpagesize.
+
+Wed Feb 24 17:52:52 1999  Gray Watson  <>
+
+	* Major reworking of internals to simplify structures. 
+
+Fri Feb 19 12:52:55 1999  Gray Watson  <>
+
+	* Made a number of changes to the internals which removed the
+ 	addr_to_block as a performance pig.
+
+Tue Feb 16 21:11:23 1999  Gray Watson  <>
+
+	* Added ability for free to look up in the free bit lists for
+ 	memory to use.
+
+	* Added mpool_clear.  Good idea.
+
+Thu Feb 11 02:53:45 1999  Gray Watson  <>
+
+	* Finally a working version.  Looks much better.
+
+	* Added rounding sizes so it will allocate aligned memory.
+
+	* Added minimum size to the mpool_free function to speed it up.
+
+Wed Feb 10 23:30:48 1999  Gray Watson  <>
+
+	* Version 1 with new fine grained memory resolution almost
+ 	working.
+
+Fri May  2 02:26:28 1997  Gray Watson  <>
+
+	* Moved to MAP_PRIVATE from MAP_SHARED.
+
+	* Fixed the min/max handling.
+
+	* Added additional info to mpool_stat.
+
+Thu May  1 16:51:06 1997  Gray Watson  <>
+
+	* Added page-size information request.
+
+	* Added better no-memory errors.
+
+Thu Apr 24 01:58:41 1997  Gray Watson  <>
+
+	* Added handling of null for debugging purposes.
+
+Mon Apr 14 03:31:26 1997  Gray Watson  <>
+
+	* Started the mpool routines.

deps/mpool-2.1.0/Makefile

+#
+# $Id: Makefile.all,v 1.1.1.1 2005/05/20 19:58:29 gray Exp $
+#
+
+HFLS	= mpool.h
+OBJS	= mpool.o
+
+CC	= gcc
+
+CFLAGS	= -g -I. $(DEFINES) -fPIC
+#CFLAGS	= -g -I.
+LDFLAGS	=
+RANLIB	= ranlib
+
+DESTDIR	= /usr/local
+TEST	= mpool_t
+LIBRARY	= libmpool.a
+
+all : $(LIBRARY) $(UTIL)
+
+clean :
+	rm -f a.out core *.o *.t
+	rm -f $(LIBRARY) $(TEST)
+
+install : $(HFLS) $(LIBRARY)
+	install -c -m 444 $(HFLS) $(DESTDIR)/include
+	install -c -m 444 $(LIBRARY) $(DESTDIR)/lib
+	$(RANLIB) $(DESTDIR)/libo/$(LIBRARY)
+
+$(LIBRARY) : $(OBJS)
+	ar cr $(LIBRARY) $?
+	$(RANLIB) $@
+
+tests : $(TEST)
+
+$(TEST) : $(TEST).o $(LIBRARY)
+	rm -f $@
+	$(CC) $(LDFLAGS) $(TEST).o $(LIBRARY)
+	mv a.out $@
+
+$(UTIL) : $(UTIL).o $(LIBRARY)
+	rm -f $@
+	$(CC) $(LDFLAGS) $(UTIL).o $(LIBRARY)
+	mv a.out $@
+
+.c.o :
+	rm -f $@
+	$(CC) $(CFLAGS) -c $< -o $@
+
+#
+# Below are dependencies that are automatically generated by make
+# depend.  Please do not edit by hand.
+#
+
+mpool.o: mpool.c mpool.h mpool_loc.h
+mpool_t.o: mpool_t.c mpool.h

deps/mpool-2.1.0/NEWS

+-------------------------------------------------------------------------------
+$Id: NEWS,v 1.2 2006/05/31 20:28:31 gray Exp $
+-------------------------------------------------------------------------------
+
+Version 2.1.0:
+
+	* Added MPOOL_ERROR_PNT_OVER to show pointer overwrites.
+
+Version 2.0.0:
+
+	* Initial external release of library after use since 1996.

deps/mpool-2.1.0/README

+-------------------------------------------------------------------------------
+$Id: README,v 1.2 2005/05/22 19:49:30 gray Exp $
+-------------------------------------------------------------------------------
+
+BACKGROUND:
+
+This is a memory pool library which was written to allow a program to
+have heaps that it could destroy without fragmenting memory.  You can
+have multiple heaps and reset them easily completely reclaiming the
+memory (as opposed to standard heaps).
+
+With it you can mpool_open() a new heap, then mpool_alloc(),
+mpool_calloc(), mpool_realloc(), mpool_free() to your heart's content.
+Once you are done with the memory-pool you can run mpool_clear() or
+mpool_close() and completely remove the memory associated with the
+pools.  This is very handy if you are working with some large blocks
+of memory and want to reset back to a clean state.
+
+Check out the mpool.h file for more information.  Sorry for minimal
+docs.
+
+-------------------------------------------------------------------------------
+
+INSTALLATION:
+
+1) Typing 'make' should be enough to build libskip.a.
+2) Typing 'make tests' should make the mpool_t test program.
+
+-------------------------------------------------------------------------------
+
+REPOSITORY:
+
+The newest versions of the library are available from:
+
+	http://256.com/sources/mpool/
+
+-------------------------------------------------------------------------------
+
+AUTHOR:
+
+If you have any questions or problems feel free to send me mail.
+
+Gray Watson
+http://256.com/gray/
+
+-------------------------------------------------------------------------------

deps/mpool-2.1.0/mpool_loc.h

+/*
+ * Memory pool local defines.
+ *
+ * Copyright 1996 by Gray Watson.
+ *
+ * This file is part of the mpool package.
+ *
+ * Permission to use, copy, modify, and distribute this software for
+ * any purpose and without fee is hereby granted, provided that the
+ * above copyright notice and this permission notice appear in all
+ * copies, and that the name of Gray Watson not be used in advertising
+ * or publicity pertaining to distribution of the document or software
+ * without specific, written prior permission.
+ *
+ * Gray Watson makes no representations about the suitability of the
+ * software described herein for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * The author may be reached via http://256.com/gray/
+ *
+ * $Id: mpool_loc.h,v 1.2 2005/05/20 20:08:54 gray Exp $
+ */
+
+#ifndef __MPOOL_LOC_H__
+#define __MPOOL_LOC_H__
+
+#define MPOOL_MAGIC	0xABACABA		/* magic for struct */
+#define BLOCK_MAGIC	0xB1B1007		/* magic for blocks */
+#define FENCE_MAGIC0	(unsigned char)(0xFAU)	/* 1st magic mem byte */
+#define FENCE_MAGIC1	(unsigned char)(0xD3U)	/* 2nd magic mem byte */
+
+#define FENCE_SIZE		2		/* fence space */
+#define MIN_ALLOCATION		(sizeof(mpool_free_t))	/* min alloc */
+#define MAX_FREE_SEARCH		10240		/* max size to search */
+#define MAX_FREE_LIST_SEARCH	100		/* max looking for free mem */ 
+
+/*
+ * bitflag tools for Variable and a Flag
+ */
+#define BIT_FLAG(x)		(1 << (x))
+#define BIT_SET(v,f)		(v) |= (f)
+#define BIT_CLEAR(v,f)		(v) &= ~(f)
+#define BIT_IS_SET(v,f)		((v) & (f))
+#define BIT_TOGGLE(v,f)		(v) ^= (f)
+
+#define SET_POINTER(pnt, val) \
+	do { \
+	  if ((pnt) != NULL) { \
+	    (*(pnt)) = (val); \
+          } \
+        } while(0)
+
+#define BLOCK_FLAG_USED		BIT_FLAG(0)		/* block is used */
+#define BLOCK_FLAG_FREE		BIT_FLAG(1)		/* block is free */
+
+#define DEFAULT_PAGE_MULT		16   /* pagesize = this * getpagesize*/
+
+/* How many pages SIZE bytes resides in.  We add in the block header. */
+#define PAGES_IN_SIZE(mp_p, size)	(((size) + sizeof(mpool_block_t) + \
+					  (mp_p)->mp_page_size - 1) / \
+					 (mp_p)->mp_page_size)
+#define SIZE_OF_PAGES(mp_p, page_n)	((page_n) * (mp_p)->mp_page_size)
+#define MAX_BITS	30		/* we only can allocate 1gb chunks */
+
+#define MAX_BLOCK_USER_MEMORY(mp_p)	((mp_p)->mp_page_size - \
+					 sizeof(mpool_block_t))
+#define FIRST_ADDR_IN_BLOCK(block_p)	(void *)((char *)(block_p) + \
+						 sizeof(mpool_block_t))
+#define MEMORY_IN_BLOCK(block_p)	((char *)(block_p)->mb_bounds_p - \
+					 ((char *)(block_p) + \
+					  sizeof(mpool_block_t)))
+
+typedef struct {
+  unsigned int		mp_magic;	/* magic number for struct */
+  unsigned int		mp_flags;	/* flags for the struct */
+  unsigned long		mp_alloc_c;	/* number of allocations */
+  unsigned long		mp_user_alloc;	/* user bytes allocated */
+  unsigned long		mp_max_alloc;	/* maximum user bytes allocated */
+  unsigned int		mp_page_c;	/* number of pages allocated */
+  unsigned int		mp_max_pages;	/* maximum number of pages to use */
+  unsigned int		mp_page_size;	/* page-size of our system */
+  int			mp_fd;		/* fd for /dev/zero if mmap-ing */
+  off_t			mp_top;		/* top of our allocations in fd */ 
+  mpool_log_func_t	mp_log_func;	/* log callback function */
+  void			*mp_addr;	/* current address for mmaping */
+  void			*mp_min_p;	/* min address in pool for checks */
+  void			*mp_bounds_p;	/* max address in pool for checks */
+  struct mpool_block_st	*mp_first_p;	/* first memory block we are using */
+  struct mpool_block_st	*mp_last_p;	/* last memory block we are using */
+  struct mpool_block_st	*mp_free[MAX_BITS + 1]; /* free lists based on size */
+  unsigned int		mp_magic2;	/* upper magic for overwrite sanity */
+} mpool_t;
+
+/* for debuggers to be able to interrogate the generic type in the .h file */
+typedef mpool_t	mpool_ext_t;
+
+/*
+ * Block header structure.  This structure *MUST* be long-word
+ * aligned.
+ */
+typedef struct mpool_block_st {
+  unsigned int		mb_magic;	/* magic number for block header */
+  void			*mb_bounds_p;	/* block boundary location */
+  struct mpool_block_st	*mb_next_p;	/* linked list next pointer */
+  unsigned int		mb_magic2;	/* upper magic for overwrite sanity */
+} mpool_block_t;
+
+/*
+ * Free list structure.
+ */
+typedef struct {
+  void			*mf_next_p;	/* pointer to the next free address */
+  unsigned long		mf_size;	/* size of the free block */
+} mpool_free_t;
+
+#endif /* ! __MPOOL_LOC_H__ */

src/database.cc

@@ -40,8 +40,6 @@ void Database::Init(v8::Handle<Object> target) {
   NODE_SET_PROTOTYPE_METHOD(constructor_template, "close", Close);
   NODE_SET_PROTOTYPE_METHOD(constructor_template, "prepare", Prepare);
   NODE_SET_PROTOTYPE_METHOD(constructor_template, "prepareAndStep", PrepareAndStep);
-//   NODE_SET_PROTOTYPE_METHOD(constructor_template, "changes", Changes);
-//   NODE_SET_PROTOTYPE_METHOD(constructor_template, "lastInsertRowid", LastInsertRowid);
 
   target->Set(v8::String::NewSymbol("Database"),
           constructor_template->GetFunction());
@@ -140,14 +138,6 @@ Handle<Value> Database::Open(const Arguments& args) {
   return Undefined();
 }
 
-// // TODO: libeio'fy
-// Handle<Value> Changes(const Arguments& args) {
-//   HandleScope scope;
-//   Database* dbo = ObjectWrap::Unwrap<Database>(args.This());
-//   Local<Number> result = Integer::New(sqlite3_changes(dbo->db_));
-//   return scope.Close(result);
-// }
-
 int Database::EIO_AfterClose(eio_req *req) {
   ev_unref(EV_DEFAULT_UC);
 
@@ -214,15 +204,8 @@ Handle<Value> Database::Close(const Arguments& args) {
 }
 
 // // TODO: libeio'fy
-// Handle<Value> LastInsertRowid(const Arguments& args) {
-//   HandleScope scope;
-//   Database* dbo = ObjectWrap::Unwrap<Database>(args.This());
-//   Local<Number> result = Integer::New(sqlite3_last_insert_rowid(dbo->db_));
-//   return scope.Close(result);
-// };
-
-  // Hooks
 
+// Hooks
 //   static int CommitHook(void* v_this) {
 //     HandleScope scope;
 //     Database* db = static_cast<Database*>(v_this);

src/statement.h

@@ -51,22 +51,28 @@ class Statement : public EventEmitter {
       if (column_data_) FreeColumnData();
     }
 
-    static Handle<Value> Bind(const Arguments& args);
-    static Handle<Value> BindObject(const Arguments& args);
-    static Handle<Value> BindArray(const Arguments& args);
+    static Handle<Value> Bind(const Arguments &args);
+    static Handle<Value> BindObject(const Arguments &args);
+    static Handle<Value> BindArray(const Arguments &args);
     static int EIO_BindArray(eio_req *req);
     static int EIO_AfterBindArray(eio_req *req);
 
     static int EIO_AfterFinalize(eio_req *req);
     static int EIO_Finalize(eio_req *req);
-    static Handle<Value> Finalize(const Arguments& args);
+    static Handle<Value> Finalize(const Arguments &args);
 
-    static Handle<Value> Reset(const Arguments& args);
-    static Handle<Value> ClearBindings(const Arguments& args);
+    static Handle<Value> Reset(const Arguments &args);
+    static Handle<Value> ClearBindings(const Arguments &args);
 
     static int EIO_AfterStep(eio_req *req);
     static int EIO_Step(eio_req *req);
-    static Handle<Value> Step(const Arguments& args);
+    static Handle<Value> Step(const Arguments &args);
+
+    static int EIO_AfterFetchAll(eio_req *req);
+    static int EIO_FetchAll(eio_req *req);
+    static Handle<Value> FetchAll(const Arguments &args);
+
+    void InitializeColumns(void);
     void FreeColumnData(void);
 
     bool HasCallback();

tests/old/speedtest-low-parallel.js

@@ -24,6 +24,7 @@ function getRows() {
         statement.finalize(function () { db.close(function () {}); });
         d = ((new Date)-t0)/1000;
         puts("**** " + d + "s to fetch " + rows + " rows (" + (rows/d) + "/s)");
+        getRowsFetchAll();
         return;
       }
       rows++;
@@ -34,6 +35,20 @@ function getRows() {
   });
 }
 
+function getRowsFetchAll() {
+  db.prepare("SELECT * FROM t1", function (error, statement) {
+    if (error) throw error;
+    t0 = new Date();
+
+    statement.fetchAll(function (error, rows) {
+      d = ((new Date)-t0)/1000;
+      puts("**** " + d + "s to fetchAll " + rows.length + " rows (" + (rows.length/d) + "/s)");
+      statement.finalize(function () { db.close(function () {}); });
+      return;
+    });
+  });
+}
+
 function createTable(db, callback) {
   db.prepareAndStep("CREATE TABLE t1 (id INTEGER PRIMARY KEY, alpha INTEGER)", function (error, statement) {
     if (error) throw error;

tests/test-statement-fetchall.js

+sys = require('sys');
+fs = require('fs');
+path = require('path');
+
+TestSuite = require('async-testing/async_testing').TestSuite;
+sqlite = require('sqlite3_bindings');
+
+puts = sys.puts;
+inspect = sys.inspect;
+
+var name = "Fetching all results";
+var suite = exports[name] = new TestSuite(name);
+
+function createTestTable(db, callback) {
+  db.prepare('CREATE TABLE table1 (id INTEGER, name TEXT, age FLOAT)',
+    function (error, createStatement) {
+      if (error) throw error;
+      createStatement.step(function (error, row) {
+        if (error) throw error;
+        callback();
+      });
+    });
+}
+
+var testRows = [ [ 1, "foo",  9 ]
+               , [ 2, "bar",  8 ]
+               , [ 3, "baz",  7 ]
+               , [ 4, "quux", 6 ]
+               , [ 5, "juju", 5 ]
+               ];
+
+var testRowsExpected = [ { id: 5, name: 'juju', age: 5 }
+                       , { id: 4, name: 'quux', age: 6 }
+                       , { id: 3, name: 'baz', age: 7 }
+                       , { id: 2, name: 'bar', age: 8 }
+                       , { id: 1, name: 'foo', age: 9 }
+                       ];
+
+var tests = [
+  { 'insert a row with lastinsertedid':
+    function (assert, finished) {
+      var self = this;
+
+      self.db.open(':memory:', function (error) {
+        function selectStatementPrepared(error, statement) {
+          if (error) throw error;
+          statement.fetchAll(function (error, rows) {
+            if (error) throw error;
+
+            assert.deepEqual(testRowsExpected, rows);
+
+            self.db.close(function () {
+              finished();
+            });
+          });
+        }
+
+        createTestTable(self.db,
+          function () {
+            function insertRows(db, rows, callback) {
+              var i = rows.length;
+              db.prepare('INSERT INTO table1 (id, name, age) VALUES (?, ?, ?)',
+                function (error, statement) {
+                  function doStep(i) {
+                    statement.bindArray(rows[i], function () {
+                      statement.step(function (error, row) {
+                        if (error) throw error;
+                        assert.ok(!row, "Row should be unset");
+                        statement.reset();
+                        if (i) {
+                          doStep(--i);
+                        }
+                        else {
+                          statement.finalize(function () {
+                            callback();
+                          });
+                        }
+                      });
+                    });
+                  }
+
+                  doStep(--i);
+                });
+            }
+
+            var selectSQL
+                = 'SELECT * from table1';
+
+            insertRows(self.db, testRows, function () {
+              self.db.prepare(selectSQL
+                            , selectStatementPrepared);
+            });
+          });
+      });
+    }
+  }
+];
+
+// order matters in our tests
+for (var i=0,il=tests.length; i < il; i++) {
+  suite.addTests(tests[i]);
+}
+
+var currentTest = 0;
+var testCount = tests.length;
+
+suite.setup(function(finished, test) {
+  this.db = new sqlite.Database();
+  finished();
+});
+suite.teardown(function(finished) {
+  if (this.db) this.db.close(function (error) {
+                               finished();
+                             });
+  ++currentTest == testCount;
+});
+
+if (module == require.main) {
+  suite.runTests();
+}

wscript

 import Options
 from os import unlink, symlink
-from os.path import exists
+from os.path import exists, abspath
 
 srcdir = "."
 blddir = "build"
@@ -20,12 +20,17 @@ def configure(conf):
 #   conf.env.append_value('LIBPATH_PROFILER', '/usr/local/lib')
 #   conf.env.append_value('LIB_PROFILER', 'profiler')
 
+  conf.env.append_value("LIBPATH_MPOOL", abspath("./deps/mpool-2.1.0/"))
+  conf.env.append_value("LIB_MPOOL",     "mpool")
+  conf.env.append_value("CPPPATH_MPOOL", abspath("./deps/mpool-2.1.0/"))
+
+
 def build(bld):
   obj = bld.new_task_gen("cxx", "shlib", "node_addon")
   obj.cxxflags = ["-g", "-D_FILE_OFFSET_BITS=64", "-D_LARGEFILE_SOURCE", "-Wall"]
   obj.target = "sqlite3_bindings"
   obj.source = "src/sqlite3_bindings.cc src/database.cc src/statement.cc"
-  obj.uselib = "SQLITE3 PROFILER"
+  obj.uselib = "SQLITE3 PROFILER MPOOL"
 
 t = 'sqlite3_bindings.node'
 def shutdown():