update

resume video download

version bump

4 files changed, 394 insertions, 10 deletions

diff --git a/.resume/deploy.js b/.resume/deploy.js
new file mode 100644
index 0000000..5374c77
--- /dev/null
+++ b/.resume/deploy.js
@@ -0,0 +1,36 @@
+/*
+ * mongo-edu
+ *
+ * Copyright (c) 2014-2016 Przemyslaw Pluta
+ * Licensed under the MIT license.
+ * https://github.com/przemyslawpluta/mongo-edu/blob/master/LICENSE
+ */
+
+var path = require('path'),
+    colors = require('colors'),
+    fs = require('fs');
+
+var rs = fs.createReadStream(__dirname + '/resume.js'),
+    ws = fs.createWriteStream(path.resolve(__dirname, '../node_modules/youtube-dl/lib/youtube-dl.js'));
+
+function failed() {
+    'use strict';
+    console.log('Unable to deploy youtube-dl resume option\n');
+}
+
+function success() {
+    'use strict';
+    console.log('Resume option for youtube-dl deployed\n');
+}
+
+rs.pipe(ws);
+
+rs.on('error', function error(err) {
+    'use strict';
+    if (err) { return failed(); }
+});
+
+rs.on('end', function end() {
+    'use strict';
+    success();
+});
diff --git a/.resume/resume.js b/.resume/resume.js
new file mode 100644
index 0000000..5d92f51
--- /dev/null
+++ b/.resume/resume.js
@@ -0,0 +1,337 @@
+var execFile  = require('child_process').execFile;
+var fs        = require('fs');
+var path      = require('path');
+var url       = require('url');
+var http      = require('http');
+var streamify = require('streamify');
+var request   = require('request');
+var util      = require('./util');
+
+
+// Check that youtube-dl file exists.
+var ytdlBinary = path.join(__dirname, '..', 'bin', 'youtube-dl');
+fs.exists(ytdlBinary, function(exists) {
+  if (!exists) {
+    throw new Error('youtube-dl file does not exist.');
+  }
+});
+
+var isDebug = /^\[debug\] /;
+var isWarning = /^WARNING: /;
+var isYouTubeRegex = /^(https?:\/\/)?(www\.)?(youtube\.com|youtu\.be)\//;
+var isNoSubsRegex =
+  /WARNING: video doesn't have subtitles|no closed captions found/;
+var subsRegex = /--write-sub|--write-srt|--srt-lang|--all-subs/;
+
+/**
+ * Downloads a video.
+ *
+ * @param {String} videoUrl
+ * @param {!Array.<String>} args
+ * @param {!Object} options
+ */
+var ytdl = module.exports = function(videoUrl, args, options) {
+  var stream = streamify({
+    superCtor: http.ClientResponse,
+    readable: true,
+    writable: false
+  });
+
+  ytdl.getInfo(videoUrl, args, options, function(err, data) {
+    if (err) {
+      stream.emit('error', err);
+      return;
+    }
+
+    var item = (!data.length) ? data : data.shift();
+
+    // fix for pause/resume downloads
+    var headers = {
+      'Host': url.parse(item.url).hostname
+    };
+
+    if (options && options.start > 0) {
+      headers.Range = 'bytes=' + options.start + '-';
+    }
+
+    var req = request({
+      url: item.url,
+      headers: headers
+    });
+
+    req.on('response', function(res) {
+      if (options && options.start > 0 && res.statusCode == 416) {
+        // the file that is being resumed is complete.
+        stream.emit('end');
+        return;
+      }
+
+      if (res.statusCode !== 200 && res.statusCode !== 206) {
+        stream.emit('error', new Error('status code ' + res.statusCode));
+        return;
+      }
+
+      var size = parseInt(res.headers['content-length'], 10);
+      if (size) {
+        item.size = size;
+      }
+      stream.emit('info', item);
+    });
+    stream.resolve(req);
+  });
+
+  return stream;
+};
+
+/**
+ * Calls youtube-dl with some arguments and the `callback`
+ * gets called with the output.
+ *
+ * @param {String} url
+ * @param {Array.<String>} args
+ * @param {Object} options
+ * @param {Function(!Error, String)} callback
+ */
+ytdl.exec = function (url, args, options, callback) {
+  return call(url, [], args, options, callback);
+};
+
+
+/**
+ * Calls youtube-dl with some arguments and the `callback`
+ * gets called with the output.
+ *
+ * @param {String|Array.<String>}
+ * @param {Array.<String>} args
+ * @param {Array.<String>} args2
+ * @param {Object} options
+ * @param {Function(!Error, String)} callback
+ */
+function call(urls, args1, args2, options, callback) {
+  var args = args1;
+  if (args2) {
+    args = args.concat(util.parseOpts(args2));
+  }
+  options = options || {};
+
+  if (urls != null) {
+    if (typeof urls === 'string') {
+      urls = [urls];
+    }
+
+    for (var i = 0; i < urls.length; i++) {
+      var video = urls[i];
+      if (isYouTubeRegex.test(video)) {
+        // Get possible IDs.
+        var details = url.parse(video, true);
+        var id = details.query.v || '';
+        if (id) {
+          args.push('http://www.youtube.com/watch?v=' + id);
+        } else {
+          // Get possible IDs for youtu.be from urladdr.
+          id = details.pathname.slice(1).replace(/^v\//, '');
+          if (id || id === 'playlist') {
+            args.push(video);
+          }
+        }
+      } else {
+        args.push(video);
+      }
+    }
+  }
+
+  var file = process.env.PYTHON || 'python';
+  args = [ytdlBinary].concat(args);
+
+  // Call youtube-dl.
+  execFile(file, args, options, function(err, stdout, stderr) {
+    if (err) return callback(err);
+
+    if (stderr) {
+      // Try once to download video if no subtitles available
+      if (!options.nosubs && isNoSubsRegex.test(stderr)) {
+        var i;
+        var cleanupOpt = args2;
+
+        for (i = cleanupOpt.length - 1; i >= 0; i--) {
+          if (subsRegex.test(cleanupOpt[i])) { cleanupOpt.splice(i, 1); }
+        }
+
+        options.nosubs = true;
+
+        return call(video, args1, cleanupOpt, options, callback);
+
+      }
+
+      if (isDebug.test(stderr) && args.indexOf('--verbose') > -1) {
+        console.log('\n' + stderr);
+      } else if (isWarning.test(stderr)) {
+        console.warn(stderr);
+      } else {
+        return callback(new Error(stderr.slice(7)));
+      }
+
+    }
+
+    var data = stdout.trim().split(/\r?\n/);
+    callback(null, data);
+  });
+
+}
+
+
+/**
+ * @param {Object} data
+ * @returns {Object}
+ */
+function parseInfo(data) {
+  var info = JSON.parse(data);
+
+  // Add and process some entries to keep backwards compatibility
+  Object.defineProperty(info, 'filename', {
+    get: function() {
+      console.warn('`info.filename` is deprecated, use `info._filename`');
+      return info._filename;
+    }
+  });
+  Object.defineProperty(info, 'itag', {
+    get: function() {
+      console.warn('`info.itag` is deprecated, use `info.format_id`');
+      return info.format_id;
+    }
+  });
+  Object.defineProperty(info, 'resolution', {
+    get: function() {
+      console.warn('`info.resolution` is deprecated, use `info.format`');
+      return info.format.split(' - ')[1];
+    }
+  });
+  info.duration = util.formatDuration(info.duration);
+  return info;
+}
+
+
+/**
+ * Gets info from a video.
+ *
+ * @param {String} url
+ * @param {Array.<String>} args
+ * @param {Object} options
+ * @param {Function(!Error, Object)} callback
+ */
+ytdl.getInfo = function(url, args, options, callback) {
+  if (typeof options === 'function') {
+    callback = options;
+    options = {};
+  } else if (typeof args === 'function') {
+    callback = args;
+    options = {};
+    args = [];
+  }
+  var defaultArgs = ['--dump-json'];
+  if (!args || args.indexOf('-f') < 0 && args.indexOf('--format') < 0 &&
+      args.every(function(a) {
+        return a.indexOf('--format=') !== 0;
+      })) {
+    defaultArgs.push('-f');
+    defaultArgs.push('best');
+  }
+  call(url, defaultArgs, args, options, function(err, data) {
+    if (err) return callback(err);
+
+    var info;
+    try {
+      info = data.map(parseInfo);
+    } catch (err) {
+      return callback(err);
+    }
+
+    callback(null, info.length === 1 ? info[0] : info);
+  });
+};
+
+
+/**
+ * @param {String} url
+ * @param {!Array.<String>} args
+ * @param {Function(!Error, Object)} callback
+ */
+ytdl.getFormats = function(url, args, callback) {
+  console.warn('`getFormats()` is deprecated. Please use `getInfo()`');
+  if (typeof args === 'function') {
+    callback = args;
+    args = [];
+  }
+  ytdl.getInfo(url, args, {}, function (err, video_info) {
+    if (err) return callback(err);
+
+    var formats_info = video_info.formats || [video_info];
+    var formats = formats_info.map(function(format) {
+        return {
+          id: video_info.id,
+          itag: format.format_id,
+          filetype: format.ext,
+          resolution: format.format.split(' - ')[1].split(' (')[0],
+        };
+    });
+
+    callback(null, formats);
+  });
+};
+
+/**
+ * @param {String} url
+ * @param {Object} options
+ *   {Boolean} auto
+ *   {Boolean} all
+ *   {String} lang
+ *   {String} cwd
+ * @param {Function(!Error, Object)} callback
+ */
+ytdl.getSubs = function(url, options, callback) {
+  if (typeof options === 'function') {
+    callback = options;
+    options = {};
+  }
+
+  var args = ['--skip-download'];
+  args.push('--write' + (options.auto ? '-auto' : '') + '-sub');
+  if (options.all) {
+    args.push('--all-subs');
+  }
+  if (options.lang) {
+    args.push('--sub-lang=' + options.lang);
+  }
+  call(url, args, [], { cwd: options.cwd }, function(err, data) {
+    if (err) return callback(err);
+
+    var files = [];
+    for (var i = 0, len = data.length; i < len; i++) {
+      var line = data[i];
+      if (line.indexOf('[info] Writing video subtitles to: ') === 0) {
+        files.push(line.slice(35));
+      }
+    }
+    callback(null, files);
+  });
+};
+
+/**
+ * @param {!Boolean} descriptions
+ * @param {!Object} options
+ * @param {Function(!Error, Object)} callback
+ */
+ytdl.getExtractors = function(descriptions, options, callback) {
+  if (typeof options === 'function') {
+    callback = options;
+    options = {};
+  } else if (typeof descriptions === 'function') {
+    callback = descriptions;
+    options = {};
+    descriptions = false;
+  }
+
+  var args = descriptions ?
+    ['--extractor-descriptions'] : ['--list-extractors'];
+  call(null, args, null, options, callback);
+};
diff --git a/lib/videos.js b/lib/videos.js
index b7480fe..0f58523 100644
--- a/lib/videos.js
+++ b/lib/videos.js
@@ -19,7 +19,7 @@ var path = require('path'),
     mv = require('mv'),
     _ = require('lodash');
 
-var isDebug = /[debug]/, downloadPath = '', proxy = '', downloadList = [],
+var isDebug = /[debug]/, downloadPath = '', proxy = '', downloadList = [], hash = {},
     co = false, ncc = false, handout = false, cc = false, uz = false, hq = false, verbose = false;
 
 function setOptions(argv) {
@@ -164,17 +164,23 @@ var handleList = function handleList(list, tags) {
 
         if (handout) { return getHandouts(item); }
 
-        var dl = youtubedl(item, nocc || opt, { cwd: downloadPath }), size = 0, stash = {}, bar;
+        var downloaded = 0, size = 0, stash = {}, bar;
+
+        if (fs.existsSync(downloadPath + hash[item])) {
+            downloaded = fs.statSync(downloadPath + hash[item]).size;
+        }
+
+        var dl = youtubedl(item, nocc || opt, {start: downloaded, cwd: downloadPath});
 
         dl.on('info', function(info) {
-            size = info.size;
+            size = info.size + downloaded;
             stash = info;
             if (co) { downloadList.push({id: item, name: path.basename(info._filename)}); }
             if (notAvailable) { console.log('i'.magenta + ' No HQ video available for ' + info.fulltitle.white.bold + ' trying default quality ...'); }
-            console.log('i'.magenta + ' Downloading: ' + info._filename.cyan + ' > ' + item);
-            bar = new ProgressBar('>'.green + ' ' + filesize(size) + ' [:bar] :percent :etas', { complete: '=', incomplete: ' ', width: 20, total: parseInt(size, 10) });
+            console.log('i'.magenta + ((downloaded > 0) ? ' Resuming download: ' : ' Downloading: ') + info._filename.cyan + ' > ' + item);
+            bar = new ProgressBar('>'.green + ' ' + filesize(info.size) + ' [:bar] :percent :etas', { complete: '=', incomplete: ' ', width: 20, total: parseInt(info.size, 10) });
             console.time('i'.magenta + ' ' + info._filename + '. Done in');
-            dl.pipe(fs.createWriteStream(downloadPath + info._filename));
+            dl.pipe(fs.createWriteStream(downloadPath + info._filename, {flags: 'a'}));
         });
 
         dl.on('data', function(data) {
@@ -201,7 +207,9 @@ var handleList = function handleList(list, tags) {
         });
     };
 
-    if (currentList.length) { return getVideos(currentList.shift()); }
+    if (currentList.length) {
+        return getVideos(currentList.shift());
+    }
 
     if (co) {
 
@@ -259,6 +267,8 @@ module.exports = {
 
                     if (verbose && isDebug.test(err)) { console.log(err); }
 
+                    hash[item] = info._filename;
+
                     items.push((!err)?{name: info.fulltitle + ' - ' + info.width + 'x' + info.height, value: item, id: i}:{name: 'No info: ' + item, value: item, id: i});
 
                     count = count - 1;
diff --git a/package.json b/package.json
index 690e2a0..ba2ae7b 100644
--- a/package.json
+++ b/package.json
@@ -1,12 +1,13 @@
 {
   "name": "mongo-edu",
   "preferGlobal": true,
-  "version": "0.2.31",
+  "version": "0.2.4",
   "author": "Przemyslaw Pluta <przemyslawplutadev@gmail.com> (http://przemyslawpluta.com)",
   "description": "Select and download videos and handouts from university.mongodb.com courses",
   "main": "./mongo-edu",
   "scripts": {
-    "start": "node ./bin/mongo-edu"
+    "start": "node ./bin/mongo-edu",
+    "postinstall": "node ./.resume/deploy"
   },
   "bin": {
     "mongo-edu": "./bin/mongo-edu"
@@ -44,7 +45,7 @@
     "pretty-error": "~2.0.0",
     "progress": "~1.1.8",
     "request": "~2.67.0",
-    "request-progress": "~0.4.0",
+    "request-progress": "~1.0.2",
     "rimraf": "~2.5.0",
     "which": "~1.2.1",
     "yargs": "~3.31.0",