diff options
| -rw-r--r-- | .resume/deploy.js | 36 | ||||
| -rw-r--r-- | .resume/resume.js | 337 | ||||
| -rw-r--r-- | lib/videos.js | 24 | ||||
| -rw-r--r-- | package.json | 7 |
4 files changed, 394 insertions, 10 deletions
diff --git a/.resume/deploy.js b/.resume/deploy.js new file mode 100644 index 0000000..5374c77 --- /dev/null +++ b/.resume/deploy.js @@ -0,0 +1,36 @@ +/* + * mongo-edu + * + * Copyright (c) 2014-2016 Przemyslaw Pluta + * Licensed under the MIT license. + * https://github.com/przemyslawpluta/mongo-edu/blob/master/LICENSE + */ + +var path = require('path'), + colors = require('colors'), + fs = require('fs'); + +var rs = fs.createReadStream(__dirname + '/resume.js'), + ws = fs.createWriteStream(path.resolve(__dirname, '../node_modules/youtube-dl/lib/youtube-dl.js')); + +function failed() { + 'use strict'; + console.log('Unable to deploy youtube-dl resume option\n'); +} + +function success() { + 'use strict'; + console.log('Resume option for youtube-dl deployed\n'); +} + +rs.pipe(ws); + +rs.on('error', function error(err) { + 'use strict'; + if (err) { return failed(); } +}); + +rs.on('end', function end() { + 'use strict'; + success(); +}); diff --git a/.resume/resume.js b/.resume/resume.js new file mode 100644 index 0000000..5d92f51 --- /dev/null +++ b/.resume/resume.js @@ -0,0 +1,337 @@ +var execFile = require('child_process').execFile; +var fs = require('fs'); +var path = require('path'); +var url = require('url'); +var http = require('http'); +var streamify = require('streamify'); +var request = require('request'); +var util = require('./util'); + + +// Check that youtube-dl file exists. +var ytdlBinary = path.join(__dirname, '..', 'bin', 'youtube-dl'); +fs.exists(ytdlBinary, function(exists) { + if (!exists) { + throw new Error('youtube-dl file does not exist.'); + } +}); + +var isDebug = /^\[debug\] /; +var isWarning = /^WARNING: /; +var isYouTubeRegex = /^(https?:\/\/)?(www\.)?(youtube\.com|youtu\.be)\//; +var isNoSubsRegex = + /WARNING: video doesn't have subtitles|no closed captions found/; +var subsRegex = /--write-sub|--write-srt|--srt-lang|--all-subs/; + +/** + * Downloads a video. + * + * @param {String} videoUrl + * @param {!Array.<String>} args + * @param {!Object} options + */ +var ytdl = module.exports = function(videoUrl, args, options) { + var stream = streamify({ + superCtor: http.ClientResponse, + readable: true, + writable: false + }); + + ytdl.getInfo(videoUrl, args, options, function(err, data) { + if (err) { + stream.emit('error', err); + return; + } + + var item = (!data.length) ? data : data.shift(); + + // fix for pause/resume downloads + var headers = { + 'Host': url.parse(item.url).hostname + }; + + if (options && options.start > 0) { + headers.Range = 'bytes=' + options.start + '-'; + } + + var req = request({ + url: item.url, + headers: headers + }); + + req.on('response', function(res) { + if (options && options.start > 0 && res.statusCode == 416) { + // the file that is being resumed is complete. + stream.emit('end'); + return; + } + + if (res.statusCode !== 200 && res.statusCode !== 206) { + stream.emit('error', new Error('status code ' + res.statusCode)); + return; + } + + var size = parseInt(res.headers['content-length'], 10); + if (size) { + item.size = size; + } + stream.emit('info', item); + }); + stream.resolve(req); + }); + + return stream; +}; + +/** + * Calls youtube-dl with some arguments and the `callback` + * gets called with the output. + * + * @param {String} url + * @param {Array.<String>} args + * @param {Object} options + * @param {Function(!Error, String)} callback + */ +ytdl.exec = function (url, args, options, callback) { + return call(url, [], args, options, callback); +}; + + +/** + * Calls youtube-dl with some arguments and the `callback` + * gets called with the output. + * + * @param {String|Array.<String>} + * @param {Array.<String>} args + * @param {Array.<String>} args2 + * @param {Object} options + * @param {Function(!Error, String)} callback + */ +function call(urls, args1, args2, options, callback) { + var args = args1; + if (args2) { + args = args.concat(util.parseOpts(args2)); + } + options = options || {}; + + if (urls != null) { + if (typeof urls === 'string') { + urls = [urls]; + } + + for (var i = 0; i < urls.length; i++) { + var video = urls[i]; + if (isYouTubeRegex.test(video)) { + // Get possible IDs. + var details = url.parse(video, true); + var id = details.query.v || ''; + if (id) { + args.push('http://www.youtube.com/watch?v=' + id); + } else { + // Get possible IDs for youtu.be from urladdr. + id = details.pathname.slice(1).replace(/^v\//, ''); + if (id || id === 'playlist') { + args.push(video); + } + } + } else { + args.push(video); + } + } + } + + var file = process.env.PYTHON || 'python'; + args = [ytdlBinary].concat(args); + + // Call youtube-dl. + execFile(file, args, options, function(err, stdout, stderr) { + if (err) return callback(err); + + if (stderr) { + // Try once to download video if no subtitles available + if (!options.nosubs && isNoSubsRegex.test(stderr)) { + var i; + var cleanupOpt = args2; + + for (i = cleanupOpt.length - 1; i >= 0; i--) { + if (subsRegex.test(cleanupOpt[i])) { cleanupOpt.splice(i, 1); } + } + + options.nosubs = true; + + return call(video, args1, cleanupOpt, options, callback); + + } + + if (isDebug.test(stderr) && args.indexOf('--verbose') > -1) { + console.log('\n' + stderr); + } else if (isWarning.test(stderr)) { + console.warn(stderr); + } else { + return callback(new Error(stderr.slice(7))); + } + + } + + var data = stdout.trim().split(/\r?\n/); + callback(null, data); + }); + +} + + +/** + * @param {Object} data + * @returns {Object} + */ +function parseInfo(data) { + var info = JSON.parse(data); + + // Add and process some entries to keep backwards compatibility + Object.defineProperty(info, 'filename', { + get: function() { + console.warn('`info.filename` is deprecated, use `info._filename`'); + return info._filename; + } + }); + Object.defineProperty(info, 'itag', { + get: function() { + console.warn('`info.itag` is deprecated, use `info.format_id`'); + return info.format_id; + } + }); + Object.defineProperty(info, 'resolution', { + get: function() { + console.warn('`info.resolution` is deprecated, use `info.format`'); + return info.format.split(' - ')[1]; + } + }); + info.duration = util.formatDuration(info.duration); + return info; +} + + +/** + * Gets info from a video. + * + * @param {String} url + * @param {Array.<String>} args + * @param {Object} options + * @param {Function(!Error, Object)} callback + */ +ytdl.getInfo = function(url, args, options, callback) { + if (typeof options === 'function') { + callback = options; + options = {}; + } else if (typeof args === 'function') { + callback = args; + options = {}; + args = []; + } + var defaultArgs = ['--dump-json']; + if (!args || args.indexOf('-f') < 0 && args.indexOf('--format') < 0 && + args.every(function(a) { + return a.indexOf('--format=') !== 0; + })) { + defaultArgs.push('-f'); + defaultArgs.push('best'); + } + call(url, defaultArgs, args, options, function(err, data) { + if (err) return callback(err); + + var info; + try { + info = data.map(parseInfo); + } catch (err) { + return callback(err); + } + + callback(null, info.length === 1 ? info[0] : info); + }); +}; + + +/** + * @param {String} url + * @param {!Array.<String>} args + * @param {Function(!Error, Object)} callback + */ +ytdl.getFormats = function(url, args, callback) { + console.warn('`getFormats()` is deprecated. Please use `getInfo()`'); + if (typeof args === 'function') { + callback = args; + args = []; + } + ytdl.getInfo(url, args, {}, function (err, video_info) { + if (err) return callback(err); + + var formats_info = video_info.formats || [video_info]; + var formats = formats_info.map(function(format) { + return { + id: video_info.id, + itag: format.format_id, + filetype: format.ext, + resolution: format.format.split(' - ')[1].split(' (')[0], + }; + }); + + callback(null, formats); + }); +}; + +/** + * @param {String} url + * @param {Object} options + * {Boolean} auto + * {Boolean} all + * {String} lang + * {String} cwd + * @param {Function(!Error, Object)} callback + */ +ytdl.getSubs = function(url, options, callback) { + if (typeof options === 'function') { + callback = options; + options = {}; + } + + var args = ['--skip-download']; + args.push('--write' + (options.auto ? '-auto' : '') + '-sub'); + if (options.all) { + args.push('--all-subs'); + } + if (options.lang) { + args.push('--sub-lang=' + options.lang); + } + call(url, args, [], { cwd: options.cwd }, function(err, data) { + if (err) return callback(err); + + var files = []; + for (var i = 0, len = data.length; i < len; i++) { + var line = data[i]; + if (line.indexOf('[info] Writing video subtitles to: ') === 0) { + files.push(line.slice(35)); + } + } + callback(null, files); + }); +}; + +/** + * @param {!Boolean} descriptions + * @param {!Object} options + * @param {Function(!Error, Object)} callback + */ +ytdl.getExtractors = function(descriptions, options, callback) { + if (typeof options === 'function') { + callback = options; + options = {}; + } else if (typeof descriptions === 'function') { + callback = descriptions; + options = {}; + descriptions = false; + } + + var args = descriptions ? + ['--extractor-descriptions'] : ['--list-extractors']; + call(null, args, null, options, callback); +}; diff --git a/lib/videos.js b/lib/videos.js index b7480fe..0f58523 100644 --- a/lib/videos.js +++ b/lib/videos.js @@ -19,7 +19,7 @@ var path = require('path'), mv = require('mv'), _ = require('lodash'); -var isDebug = /[debug]/, downloadPath = '', proxy = '', downloadList = [], +var isDebug = /[debug]/, downloadPath = '', proxy = '', downloadList = [], hash = {}, co = false, ncc = false, handout = false, cc = false, uz = false, hq = false, verbose = false; function setOptions(argv) { @@ -164,17 +164,23 @@ var handleList = function handleList(list, tags) { if (handout) { return getHandouts(item); } - var dl = youtubedl(item, nocc || opt, { cwd: downloadPath }), size = 0, stash = {}, bar; + var downloaded = 0, size = 0, stash = {}, bar; + + if (fs.existsSync(downloadPath + hash[item])) { + downloaded = fs.statSync(downloadPath + hash[item]).size; + } + + var dl = youtubedl(item, nocc || opt, {start: downloaded, cwd: downloadPath}); dl.on('info', function(info) { - size = info.size; + size = info.size + downloaded; stash = info; if (co) { downloadList.push({id: item, name: path.basename(info._filename)}); } if (notAvailable) { console.log('i'.magenta + ' No HQ video available for ' + info.fulltitle.white.bold + ' trying default quality ...'); } - console.log('i'.magenta + ' Downloading: ' + info._filename.cyan + ' > ' + item); - bar = new ProgressBar('>'.green + ' ' + filesize(size) + ' [:bar] :percent :etas', { complete: '=', incomplete: ' ', width: 20, total: parseInt(size, 10) }); + console.log('i'.magenta + ((downloaded > 0) ? ' Resuming download: ' : ' Downloading: ') + info._filename.cyan + ' > ' + item); + bar = new ProgressBar('>'.green + ' ' + filesize(info.size) + ' [:bar] :percent :etas', { complete: '=', incomplete: ' ', width: 20, total: parseInt(info.size, 10) }); console.time('i'.magenta + ' ' + info._filename + '. Done in'); - dl.pipe(fs.createWriteStream(downloadPath + info._filename)); + dl.pipe(fs.createWriteStream(downloadPath + info._filename, {flags: 'a'})); }); dl.on('data', function(data) { @@ -201,7 +207,9 @@ var handleList = function handleList(list, tags) { }); }; - if (currentList.length) { return getVideos(currentList.shift()); } + if (currentList.length) { + return getVideos(currentList.shift()); + } if (co) { @@ -259,6 +267,8 @@ module.exports = { if (verbose && isDebug.test(err)) { console.log(err); } + hash[item] = info._filename; + items.push((!err)?{name: info.fulltitle + ' - ' + info.width + 'x' + info.height, value: item, id: i}:{name: 'No info: ' + item, value: item, id: i}); count = count - 1; diff --git a/package.json b/package.json index 690e2a0..ba2ae7b 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,13 @@ { "name": "mongo-edu", "preferGlobal": true, - "version": "0.2.31", + "version": "0.2.4", "author": "Przemyslaw Pluta <przemyslawplutadev@gmail.com> (http://przemyslawpluta.com)", "description": "Select and download videos and handouts from university.mongodb.com courses", "main": "./mongo-edu", "scripts": { - "start": "node ./bin/mongo-edu" + "start": "node ./bin/mongo-edu", + "postinstall": "node ./.resume/deploy" }, "bin": { "mongo-edu": "./bin/mongo-edu" @@ -44,7 +45,7 @@ "pretty-error": "~2.0.0", "progress": "~1.1.8", "request": "~2.67.0", - "request-progress": "~0.4.0", + "request-progress": "~1.0.2", "rimraf": "~2.5.0", "which": "~1.2.1", "yargs": "~3.31.0", |