Merge pull request #192 from developmentseed/binary-deployment

Support for binary deployment

.gitignore

@@ -5,6 +5,8 @@
 *.lo
 *.Makefile
 *.target.gyp.mk
+stage
+lib/binding
 build
 out
 Release

.travis.yml

 language: node_js
+
 node_js:
-  - "0.10"
-  - "0.8"
-  - "0.6"
-before_script: "npm install mocha"
+ - "0.10"
+ - "0.8"
+ - "0.6"
+
+install:
+ - npm install
+ - npm install mocha
+ - npm test
+
+before_script: "make clean"
+
+script:
+ - npm install --stage
+ - npm test

Makefile

 build:
-	node-gyp build
+	npm install --build-from-source
 
 clean:
-	rm -f test/support/big.db*
-	rm -f test/tmp/*
+	rm -f ./lib/node_sqlite3.node
+	rm -rf ./lib/binding/
+	#rm -f ./test/support/big.db*
+	rm -f ./test/tmp/*
 	rm -rf ./deps/sqlite-autoconf-*/
 	rm -rf ./build
 	rm -rf ./out

binding.gyp

@@ -3,17 +3,6 @@
   'variables': {
       'sqlite%':'internal',
   },
-  'conditions': [
-      ['OS=="win"', {
-        'variables': {
-          'copy_command%': 'copy',
-        },
-      },{
-        'variables': {
-          'copy_command%': 'cp',
-        },
-      }]
-  ],
   'targets': [
     {
       'target_name': 'node_sqlite3',
@@ -37,23 +26,6 @@
         'src/node_sqlite3.cc',
         'src/statement.cc'
       ],
-    },
-    {
-      'target_name': 'action_after_build',
-      'type': 'none',
-      'dependencies': [ 'node_sqlite3' ],
-      'actions': [
-        {
-          'action_name': 'move_node_module',
-          'inputs': [
-            '<@(PRODUCT_DIR)/node_sqlite3.node'
-          ],
-          'outputs': [
-            'lib/node_sqlite3.node'
-          ],
-          'action': ['<@(copy_command)', '<@(PRODUCT_DIR)/node_sqlite3.node', 'lib/node_sqlite3.node']
-        }
-      ]
     }
   ]
 }

build-util/remake.sh

+export ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+export UNAME=$(uname -s);
+
+cd $ROOTDIR
+cd ../
+
+if [ ${UNAME} = 'Darwin' ]; then
+    # note: requires FAT (duel-arch) node installed via .pkg
+    npm install --stage --target_arch=ia32
+    npm install --stage --target_arch=ia32 --debug
+    npm install --stage --target_arch=x64
+    npm install --stage --target_arch=x64 --debug
+
+elif [ ${UNAME} = 'Linux' ]; then
+    rm -rf ./bin/linux-*
+    apt-get -y update
+    apt-get -y install git make build-essential
+    git clone https://github.com/creationix/nvm.git ~/.nvm
+    source ~/.nvm/nvm.sh
+    nvm install 0.10
+    npm install -g node-gyp
+    node ./build.js --target_arch=x64
+    # now do 32 bit
+    NVER=`node -v`
+    wget http://nodejs.org/dist/${NVER}/node-${NVER}-linux-x86.tar.gz
+    tar xf node-${NVER}-linux-x86.tar.gz
+    export PATH=$(pwd)/node-${NVER}-linux-x86/bin:$PATH
+    # ignore: 
+    # dependency problems - leaving unconfigure  gcc-4.6:i386 g++-4.6:i386 libstdc++6-4.6-dev:i386
+    # E: Sub-process /usr/bin/dpkg returned an error code (1)
+    apt-get -y install binutils:i386 cpp:i386 gcc-4.6:i386 g++-4.6:i386 libstdc++6-4.6-dev:i386 | true
+    CC=gcc-4.6 CXX=g++-4.6 node ./build.js --target_arch=ia32    
+fi
\ No newline at end of file

build-util/tools.js

+var ProgressBar = require('progress');
+var http = require('http');
+var url = require('url');
+var fs = require('fs');
+
+function download(from,to,callback) {
+    var uri = url.parse(from);
+    var req = http.request(uri);
+    req.on('response', function(res){
+        // needed for end to be called
+        res.resume();
+        if (res.statusCode !== 200) {
+            return callback(new Error('Server returned '+ res.statusCode),false);
+        }
+        var len = parseInt(res.headers['content-length'], 10);
+        console.log();
+        var bar = new ProgressBar('  downloading [:bar] :percent :etas', {
+          complete: '='
+        , incomplete: ' '
+        , width: 40
+        , total: len
+        });
+        function returnBuffer() {
+            for (var length = 0, i = 0; i < out.length; i++) {
+                length += out[i].length;
+            }
+            var result = new Buffer(length);
+            for (var pos = 0, j = 0; j < out.length; j++) {
+                out[j].copy(result, pos);
+                pos += out[j].length;
+            }
+            fs.writeFile(to,result,function(err) {
+                if (err) return callback(err,true);
+                return callback(null,true);
+            });
+        }
+        var out = [];
+        res.on('data', function(chunk) {
+            bar.tick(chunk.length);
+            out.push(chunk);
+            found_remote = true;
+        });
+        res.on('end', function(){
+            console.log('\n');
+            returnBuffer();
+        });
+        res.on('close', function(){
+            returnBuffer();
+        });
+    });
+    req.on('error', function(err){
+        callback(err,false);
+    });
+    req.end();
+}
+
+
+function parse_args(_args, opts) {
+    // first split them like npm returns
+    var args = [];
+    _args.forEach(function(a) {
+        var parts = a.split('=');
+        parts.forEach(function(p) {
+            args.push(p);
+        })
+    })
+    // respect flags passed to npm install
+    if (process.env.npm_config_argv) {
+        var argv_obj = JSON.parse(process.env.npm_config_argv);
+        args = args.concat(argv_obj.cooked.slice(1))
+    }
+    var debug = (args.indexOf('--debug') > -1);
+    if (debug) opts.configuration = 'Debug';
+
+    opts.stage = (args.indexOf('--stage') > -1);
+    if (opts.stage) {
+        opts.force = true;
+    } else {
+        var from_source = args.indexOf('--build-from-source');
+        if ( from_source > -1) {
+            // no specific module name passed
+            var next_arg = args[from_source+1];
+            if (!next_arg || next_arg.indexOf('--') <= 0) {
+                opts.force = true;
+            } else if (next_arg == 'sqlite3'){
+                opts.force = true; 
+            }
+        }
+    }
+    var target_arch = args.indexOf('--target_arch');
+    if (target_arch > -1) {
+        var next_arg = args[target_arch+1];
+        if (next_arg && next_arg.indexOf('--') < 0) {
+            opts.target_arch = next_arg;
+        }
+    }
+    opts.args = args;
+    return opts;
+}
+
+module.exports.parse_args = parse_args;
+module.exports.download = download;
\ No newline at end of file

build-util/upload.sh

+export ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+export DRY_RUN="--dry-run"
+
+cd ${ROOTDIR}/../stage/
+if [ -d Debug ]; then
+cd Debug
+../../../s3cmd/s3cmd sync --no-check-md5 --acl-public ./*.tar.gz s3://node-sqlite3/Debug/ ${DRY_RUN}
+cd ../
+fi
+
+if [ -d Release ]; then
+cd Release
+../../../s3cmd/s3cmd sync --no-check-md5 --acl-public ./*.tar.gz s3://node-sqlite3/Release/ ${DRY_RUN}
+cd ../
+fi
+
+#../../s3cmd/s3cmd ls s3://node-sqlite3/

build.js

+#!/usr/bin/env node
+
+/*
+
+TODO
+
+Really should do:
+ - checksumming
+
+Future:
+ - travis/nvm/32bit auto-build and post to s3 for linux
+ - script to check for acl-public
+ - use require() to support node_modules location of binary?
+*/
+
+var package_json = require('./package.json');
+var Binary = require('./lib/binary_name.js').Binary;
+var util = require('./build-util/tools.js');
+var mkdirp = require('mkdirp');
+// https://github.com/isaacs/node-tar/issues/11
+//var tar = require('tar');
+var targz = require('tar.gz');
+var cp = require('child_process');
+var fs = require('fs');
+var path = require('path');
+var os = require('os');
+
+var opts = {
+    name: 'node_sqlite3',
+    force: false,
+    stage: false,
+    configuration: 'Release',
+    target_arch: process.arch,
+    platform: process.platform,
+    uri: 'http://dei9kzb8scfgo.cloudfront.net/',
+    paths: {}
+}
+
+function log(msg) {
+    console.log('['+package_json.name+']: ' + msg);
+}
+
+// only for dev
+function log_debug(msg) {
+    //log(msg);
+}
+
+function done(err) {
+    if (err) {
+        log(err);
+        process.exit(1);
+    }
+    process.exit(0);
+}
+
+function test(opts,try_build,callback) {
+    fs.statSync(opts.paths.runtime_module_path);
+    var args = [];
+    var shell_cmd;
+    var arch_names = {
+        'ia32':'-i386',
+        'x64':'-x86_64'
+    }
+    if (process.platform === 'darwin' && arch_names[opts.target_arch]) {
+        shell_cmd = 'arch';
+        args.push(arch_names[opts.target_arch]);
+        args.push(process.execPath);
+    } else if (process.arch == opts.target_arch) {
+        shell_cmd = process.execPath;
+    }
+    if (!shell_cmd) {
+        // system we cannot test on - likely since we are cross compiling
+        log("Skipping testing binary for " + process.target_arch);
+        return callback();
+    }
+    args.push('lib/sqlite3');
+    cp.execFile(shell_cmd, args, function(err, stdout, stderr) {
+        if (err || stderr) {
+            var output = err.message || stderr;
+            log('Testing the binary failed: "' + output + '"');
+            if (try_build) {
+                log('Attempting source compile...');
+                build(opts,callback);
+            }
+        } else {
+            log('Sweet: "' + opts.binary.filename() + '" is valid, node-sqlite3 is now installed!');
+            return callback();
+        }
+    });
+}
+
+function build(opts,callback) {
+    var shell_cmd = process.platform === 'win32' ? 'node-gyp.cmd' : 'node-gyp';
+    var shell_args = ['rebuild'].concat(opts.args);
+    var cmd = cp.spawn(shell_cmd,shell_args);
+    cmd.on('error', function(err) {
+        if (err) {
+            return callback(new Error("Failed to execute '" + shell_cmd + ' ' + shell_args.join(' ') + "' (" + err + ")"));
+        }
+    });
+    cmd.stdout.on('data',function(data) {
+        console.log(data.slice(0,data.length-1).toString());
+    })
+    // TODO - this node-gyp output comes through formatted poorly, hence disabled
+    /*
+    cmd.stderr.on('data',function(data) {
+        console.error(data.slice(0,data.length-1).toString());
+    })
+    */
+    cmd.on('exit', function(err) {
+        if (err) {
+            if (err === 127) {
+                console.error(
+                    'node-gyp not found! Please upgrade your install of npm! You need at least 1.1.5 (I think) '+
+                    'and preferably 1.1.30.'
+                );
+            } else {
+                console.error('Build failed');
+            }
+            return callback(err);
+        }
+        move(opts,callback);
+    });
+}
+
+function tarball(opts,callback) {
+    var source = path.dirname(opts.paths.staged_module_file_name);
+    log('compressing: ' + source + ' to ' + opts.paths.tarball_path);
+    new targz(9).compress(source, opts.paths.tarball_path, function(err) {
+        if (err) return callback(err);
+        log('Versioned binary staged for upload at ' + opts.paths.tarball_path);
+        return callback();
+    });
+}
+
+function move(opts,callback) {
+    try {
+        fs.statSync(opts.paths.build_module_path);
+    } catch (ex) {
+        return callback(new Error('Build succeeded but target not found at ' + opts.paths.build_module_path));
+    }
+    try {
+        mkdirp.sync(path.dirname(opts.paths.runtime_module_path));
+        log('Created: ' + path.dirname(opts.paths.runtime_module_path));
+    } catch (err) {
+        log_debug(err);
+    }
+    fs.renameSync(opts.paths.build_module_path,opts.paths.runtime_module_path);
+    if (opts.stage) {
+        try {
+            mkdirp.sync(path.dirname(opts.paths.staged_module_file_name));
+            log('Created: ' + path.dirname(opts.paths.staged_module_file_name))
+        } catch (err) {
+            log_debug(err);
+        }
+        fs.writeFileSync(opts.paths.staged_module_file_name,fs.readFileSync(opts.paths.runtime_module_path));
+        // drop build metadata into build folder
+        var metapath = path.join(path.dirname(opts.paths.staged_module_file_name),'build-info.json');
+        // more build info
+        opts.date = new Date();
+        opts.node_features = process.features;
+        opts.versions = process.versions;
+        opts.config = process.config;
+        opts.execPath = process.execPath;
+        fs.writeFileSync(metapath,JSON.stringify(opts,null,2));
+        tarball(opts,callback);
+    } else {
+        log('Installed in ' + opts.paths.runtime_module_path + '');
+        test(opts,false,callback);
+    }
+}
+
+function rel(p) {
+    return path.relative(process.cwd(),p);
+}
+
+var opts = util.parse_args(process.argv.slice(2),opts);
+opts.binary = new Binary(opts);
+var versioned = opts.binary.getRequirePath();
+opts.paths.runtime_module_path = rel(path.join(__dirname, 'lib', versioned));
+opts.paths.runtime_folder = rel(path.join(__dirname, 'lib', 'binding',opts.binary.configuration));
+var staged_module_path = path.join(__dirname, 'stage', opts.binary.getModuleAbi(), opts.binary.getBasePath());
+opts.paths.staged_module_file_name = rel(path.join(staged_module_path,opts.binary.filename()));
+opts.paths.build_module_path = rel(path.join(__dirname, 'build', opts.binary.configuration, opts.binary.filename()));
+opts.paths.tarball_path = rel(path.join(__dirname, 'stage', opts.binary.configuration, opts.binary.getArchivePath()));
+
+if (!{ia32: true, x64: true, arm: true}.hasOwnProperty(opts.target_arch)) {
+    return done(new Error('Unsupported (?) architecture: '+ opts.target_arch+ ''));
+}
+
+if (opts.force) {
+    build(opts,done);
+} else {
+    try {
+        test(opts,true,done);
+    } catch (ex) {
+        var from = opts.binary.getRemotePath();
+        var tmpdir;
+        if (os.tmpdir) {
+            tmpdir = os.tmpdir();
+        } else {
+            var tmpdir = '/tmp/node-sqlite3-' + opts.binary.configuration;
+            try {
+                mkdirp.sync(tmpdir);
+            } catch (err) {
+                log_debug(err);
+            }
+        }
+        var tmpfile = path.join(tmpdir,path.basename(from));
+        util.download(from,tmpfile,function(err,found_remote) {
+            if (err) {
+                if (!found_remote) {
+                    log(from + ' not found, falling back to source compile (' + err + ')');
+                    build(opts,done);
+                } else {
+                    return done(err);
+                }
+            } else {
+                log('downloaded to temp location: '+ tmpfile);
+                new targz().extract(tmpfile, opts.paths.runtime_folder, function(err) {
+                    if (err) return done(err);
+                    try {
+                        test(opts,true,done);
+                    } catch (ex) {
+                        // Stat failed
+                        log(opts.paths.runtime_folder + ' not found, falling back to source compile');
+                        build(opts,done);
+                    }
+                });
+            }        
+        });
+    }
+}

lib/binary_name.js

+
+var path = require('path');
+
+var Binary = function(options) {
+  var options = options || {};
+  var package_json = options.package_json || require('../package.json');
+  this.name = options.name || 'binding';
+  this.configuration = options.configuration || 'Release';
+  this.uri = options.uri || 'http://'+this.name+'.s3.amazonaws.com/';
+  this.module_maj_min = package_json.version.split('.').slice(0,2).join('.');
+  this.module_abi = package_json.abi;
+  this.platform = options.platform || process.platform;
+  this.target_arch = options.target_arch || process.arch;
+  if (process.versions.modules) {
+    this.node_abi = 'node-v' + process.versions.modules
+  } else {
+    this.node_abi = 'v8-' + process.versions.v8.split('.').slice(0,2).join('.');
+  }
+}
+
+Binary.prototype.filename = function() {
+    return this.name + '.node';
+}
+
+Binary.prototype.compression = function() {
+    return '.tar.gz';
+}
+
+Binary.prototype.getBasePath = function() {
+    return this.node_abi
+           + '-' + this.platform
+           + '-' + this.target_arch;
+}
+
+Binary.prototype.getRequirePath = function(configuration) {
+    return './' + path.join('binding',
+        configuration || this.configuration,
+        this.getBasePath(),
+        this.filename());
+}
+
+Binary.prototype.getModuleAbi = function() {
+    return this.name + '-v' + this.module_maj_min + '.' + this.module_abi;
+}
+
+Binary.prototype.getArchivePath = function() {
+    return this.getModuleAbi()
+           + '-'
+           + this.getBasePath()
+           + this.compression();
+}
+
+Binary.prototype.getRemotePath = function() {
+    return this.uri+this.configuration+'/'+this.getArchivePath();
+}
+
+module.exports.Binary = Binary;
\ No newline at end of file

lib/sqlite3.js

-var sqlite3 = module.exports = exports = require('./node_sqlite3.node');
+var Binary = require('./binary_name.js').Binary;
+var binary = new Binary({name:'node_sqlite3'});
+var binding;
+try {
+    binding = require(binary.getRequirePath('Debug'));
+} catch (err) {
+    binding = require(binary.getRequirePath('Release'));
+}
+var sqlite3 = module.exports = exports = binding;
 var path = require('path');
 var util = require('util');
 var EventEmitter = require('events').EventEmitter;

package.json

@@ -2,6 +2,7 @@
     "name": "sqlite3",
     "description": "Asynchronous, non-blocking SQLite3 bindings",
     "version": "2.1.15",
+    "abi":"a",
     "homepage": "http://github.com/developmentseed/node-sqlite3",
     "author": {
         "name": "Development Seed",
@@ -29,10 +30,17 @@
         "type": "git",
         "url": "git://github.com/developmentseed/node-sqlite3.git"
     },
+    "dependencies": {
+        "progress":"~1.0.1",
+        "mkdirp":"~0.3.5",
+        "tar.gz": "~0.1.1"
+    },
+    "bundledDependencies":["mkdirp","tar.gz","progress"],
     "engines": {
         "node": ">= 0.6.13 < 0.11.0"
     },
     "scripts": {
+        "install": "node build.js",
         "pretest": "node test/support/createdb.js",
         "test": "mocha -R spec --timeout 200000"
     },