diff options
| -rw-r--r-- | .gitignore | 19 | ||||
| -rw-r--r-- | LICENSE | 24 | ||||
| -rw-r--r-- | README.md | 0 | ||||
| -rw-r--r-- | bin/mongo-edu | 3 | ||||
| -rw-r--r-- | lib/login.js | 160 | ||||
| -rw-r--r-- | lib/validate.js | 31 | ||||
| -rw-r--r-- | lib/videos.js | 120 | ||||
| -rw-r--r-- | mongo-edu.js | 103 | ||||
| -rw-r--r-- | package.json | 48 |
9 files changed, 508 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..33d7898 --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +lib-cov +lcov.info +*.seed +*.log +*.csv +*.dat +*.out +*.pid +*.gz + +pids +logs +results +build +livegit +.grunt +.DS_Store + +node_modules @@ -0,0 +1,24 @@ +(The MIT License) + +Copyright (c) 2014 Przemyslaw Pluta <przemyslawplutadev@gmail.com> + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/README.md diff --git a/bin/mongo-edu b/bin/mongo-edu new file mode 100644 index 0000000..e1558e3 --- /dev/null +++ b/bin/mongo-edu @@ -0,0 +1,3 @@ +#!/usr/bin/env node + +require('../mongo-edu').create();
\ No newline at end of file diff --git a/lib/login.js b/lib/login.js new file mode 100644 index 0000000..c28c334 --- /dev/null +++ b/lib/login.js @@ -0,0 +1,160 @@ +/* + * mongo-edu + * + * Copyright (c) 2014 Przemyslaw Pluta + * Licensed under the MIT license. + * https://github.com/przemyslawpluta/mongo-edu/blob/master/LICENSE + */ + +var request = require('request'), + cheerio = require('cheerio'), + ProgressBar = require('progress'), + _ = require('lodash'); + +var jar = request.jar(); + +function addCookies(cookies, url) { + _.each(cookies, function cookies(cookie) { jar.setCookie(cookie, url + '/login'); }); +} + +module.exports = { + + init: function init(opt, callback) { + + var url = 'https://education.mongodb.com', + bar = new ProgressBar('[' + '>'.magenta + '] Searching [:bar] :percent', { complete: '=', incomplete: ' ', width: 20, total: 5 }), + CSRFTokenCookie = '', + + login = function login(token) { + + request({ + uri: url + '/login', + method: 'POST', + jar: jar, + headers: { 'X-CSRFToken': token[0].split('=')[1] }, + form: { 'email': opt.user, 'password': opt.password } + }, function post(err, res, body) { + + if (err !== null) { return callback(err, null); } + + if (res.statusCode === 200) { + var status = JSON.parse(body); + if (status.success !== true) { return callback(new Error(status.value)); } + + bar.tick(); + + return checkCourses(bar); + + } + + callback(new Error(res.statusCode)); + + }); + }, + + checkCourses = function checkCourses(bar) { + + bar.tick(); + + request({ url: url + '/dashboard', jar: jar }, function get(err, res, body) { + if (err !== null) { return callback(err, null); } + bar.tick(); + if (res.statusCode === 200) { + + var list = [], getCourses = [], $ = cheerio.load(body), + + current = $('article.my-course.panel').children(), + + link = $(current).children().filter('.span-7.offset-1').children().find('h3').children(); + + bar.tick(); + + $(link).each(function each(i, item) { + list.push({ name: $(item).text(), value: url + $(item).attr('href').replace('about', '').replace('info', '') + 'course_wiki' }); + }); + + bar.tick(); + + callback(null, list); + } + + }); + }; + + request(url, function (err, res, body) { + + if (err !== null) { return callback(err, null); } + + if (res.statusCode === 200) { + + addCookies(res.headers['set-cookie'], url); + + CSRFTokenCookie = res.headers['set-cookie'][0].split(';'); + + return login(CSRFTokenCookie); + + } + + callback(new Error(res.statusCode)); + + }); + }, + + getList: function getList(opt, callback) { + + request({ url: opt.url, jar: jar }, function get(err, res, body) { + if (err !== null) { return callback(err, null); } + if (res.statusCode === 200) { + + var list = [], getCourses = [], $ = cheerio.load(body), tag = opt.url.replace('course_', ''); + + $('div.wiki-article p').children().filter('a').map(function map(i, item) { + var current = $(item); + if (current.text().indexOf('Video') !== -1) { list.push(current.attr('href')); } + }); + + getCourses = _.filter(list, function map(item) { return (item.indexOf('wiki/M101') !== -1); }); + + callback(null, _.map(getCourses, function map(item) { return { name: item.replace(tag, '').slice(0, -1).slice(1).replace(/-/g, ' ').replace(/\//g, ': '), value: item }; })); + } + + }); + + }, + + listVideos: function listVideos(opt, callback) { + + var videos = [], state = false, list, $, current; + + request({ url: opt.url, jar: jar }, function get(err, res, body) { + if (err !== null) { return callback(err, null); } + if (res.statusCode === 200) { + $ = cheerio.load(body); + list = $('div.wiki-article'); + + current = list.find('code').text(); + + if ((current.indexOf('http://') !== -1) || (current.indexOf('https://') !== -1)) { + videos = _.compact(current.split('\n')); + } + + if (!videos.length) { + + list.children('p').children().each(function each(i, item) { + var current = $(item); + if (current.attr('href')) { + videos.push({ name: current.text().split('-')[0], value: current.attr('href') }); + } + }); + + if (videos.length) { videos.unshift({name: 'All', value: 'all', checked: true}, {name: 'Cancel', value: 'cancel'}); } + state = true; + } + + return callback(null, videos, state); + } + + callback(new Error(res.statusCode)); + }); + } +};
\ No newline at end of file diff --git a/lib/validate.js b/lib/validate.js new file mode 100644 index 0000000..1480b11 --- /dev/null +++ b/lib/validate.js @@ -0,0 +1,31 @@ +/* + * mongo-edu + * + * Copyright (c) 2014 Przemyslaw Pluta + * Licensed under the MIT license. + * https://github.com/przemyslawpluta/mongo-edu/blob/master/LICENSE + */ + +var mkdirp = require('mkdirp'); + +module.exports.init = function init(opt, callback) { + + var profile = [ + { type: 'input', name: 'user', message: 'MongoDB Uni User Name', default: '', validate: function(value) { + if (value !== '') { return true; } + return "Please enter your MongoDB Uni user name - email address"; + }}, + { type: 'password', message: 'MongoDB Uni Password', name: 'password', validate: function(value) { + if (value !== '') { return true; } + return "Please enter your MongoDB Uni password"; + }} + ]; + + if (opt.u !== '') { profile[0].default = opt.u; } + + mkdirp(opt.d, function mkdirp(err) { + if (err !== null) { return callback(err); } + callback(null, profile); + }); + +}; diff --git a/lib/videos.js b/lib/videos.js new file mode 100644 index 0000000..44bfd7e --- /dev/null +++ b/lib/videos.js @@ -0,0 +1,120 @@ +/* + * mongo-edu + * + * Copyright (c) 2014 Przemyslaw Pluta + * Licensed under the MIT license. + * https://github.com/przemyslawpluta/mongo-edu/blob/master/LICENSE + */ + +var youtubedl = require('youtube-dl'), + ProgressBar = require('progress'), + colors = require('colors'), + _ = require('lodash'); + +var downloadPath = ''; + +var handleList = function handleList(list) { + + var currentList = list, + + getVideos = function getVideos(item) { + + var dl = youtubedl.download(item, downloadPath, ['--max-quality=18']), bar; + + dl.on('download', function(data) { + console.log('[' + 'i'.magenta + '] Downloading: ' + data.filename.cyan + ' > ' + item); + bar = new ProgressBar('[' + '>'.green + '] ' + data.size + ' [:bar] :percent :etas', { complete: '=', incomplete: ' ', width: 20, total: 100.0 }); + }); + + dl.on('progress', function(data) { + if (!bar.complete) { bar.tick(parseInt(data.percent, 10)); } + }); + + dl.on('error', function(err) { + return console.log(err.stack); + }); + + dl.on('end', function(data) { + var left = (currentList.length)? ' ' + currentList.length + ' left ...' : ''; + if (data.filename) { + if (data.filename.indexOf('has already been downloaded') !== -1) { + console.log('[' + '>'.magenta + '] ' + data.filename + '.' + left); + } else { + console.log('[' + 'i'.green + '] Done in ' + data.timeTakenms + 'ms.' + left); + + } + } else { + console.log('[' + 'i'.red + '] Download Issues'); + } + handleList(currentList); + }); + }; + + if (currentList.length) { return getVideos(currentList.shift()); } + console.log('[ Finished ]'.green); + +}; + +module.exports = { + + details: function details(opt, callback) { + + var bar = new ProgressBar('[' + '>'.magenta + '] Collecting [:bar] :percent', { complete: '=', incomplete: ' ', width: 20, total: opt.length }); + + var getDetails = function getDetails(item, i) { + + function getInfo() { + youtubedl.getInfo(item, function(err, info) { + + items.push((!err)?{name: info.title + ' - ' + info.resolution, value: item, id: i}:{name: 'No info: ' + item, value: item, id: i}); + + count = count - 1; + + bar.tick(); + + if (count === 0) { + + var sortedList = _.map(_.sortBy(items, 'id')); + + sortedList.unshift({name: 'All', value: 'all', checked: true}, {name: 'Cancel', value: 'cancel'}); + callback(null, sortedList); + } + }); + } + + setTimeout(function timeout(){ getInfo(); }, 100 * i); + + }, items = [], list = [], i, count; + + list = _.filter(opt, function filter(item) { return (item.indexOf('http://') !== -1) || (item.indexOf('https://') !== -1); }); + + if (!list.length) { return callback(null, list); } + + count = list.length; + + for (i = 0; i < list.length; i++) { + getDetails(list[i], i); + } + }, + + download: function download(opt, list, argv) { + + var options = opt.videos, fullList = [], selected = []; + + downloadPath = argv.d; + + if (options.length === 1 && options[0] === 'cancel') { return console.log('Cancel'); } + + if (options.length === 1 && options[0] === 'all') { + fullList = _.pull(_.map(list, function map(item) { return item.value; }), 'all', 'cancel'); + return handleList(fullList); + } + + if (options.length >= 1) { + selected = _.pull(options, 'all', 'cancel'); + + if (selected.length) { return handleList(selected); } + } + + } +};
\ No newline at end of file diff --git a/mongo-edu.js b/mongo-edu.js new file mode 100644 index 0000000..71652f0 --- /dev/null +++ b/mongo-edu.js @@ -0,0 +1,103 @@ +/* + * mongo-edu + * + * Copyright (c) 2014 Przemyslaw Pluta + * Licensed under the MIT license. + * https://github.com/przemyslawpluta/mongo-edu/blob/master/LICENSE + */ + +var mdbvideos = require('./lib/login'), + videoHandler = require('./lib/videos'), + validate = require('./lib/validate'), + colors = require('colors'), + inquirer = require('inquirer'), + argv = require('optimist') + .usage('Usage: $0 -u [user name] -d [download path]') + .describe('d', 'download path').describe('u', 'email address') + .demand('d').argv; + +exports.create = function start() { + + validate.init(argv, function (err, profile) { + if (err !== null) { throw err; } + run(profile); + }); + + function run(profile) { + + inquirer.prompt(profile, function promt(answers) { + + var list = [{ type: 'list', name: 'url', message: '', choices: [] }], classes = list, + + check = [{ type: 'checkbox', message: '', name: 'videos', choices: [], + validate: function validate(answer) { + if ( answer.length < 1 ) { return 'You must choose at least one option.'; } + return true; + } + }]; + + mdbvideos.init(answers, function get(err, data) { + if (err !== null) { throw err; } + if (data.length) { + + classes[0].message = 'Found ' + data.length + ' Course'+ ((data.length > 1)? 's' : '') + '. Select:'; + classes[0].choices = data; + return currentList(); + + } + + }); + + function currentList() { + inquirer.prompt(classes, function prompt(answers) { + + mdbvideos.getList(answers, function get(err, data) { + if (err !== null) { throw err; } + + if (data.length) { + + list[0].message = 'Found ' + data.length + ' List'+ ((data.length > 1)? 's' : '') + '. Select:'; + list[0].choices = data; + + return currentVideos(); + + } + + return console.log('[' + 'i'.red + '] Unable to locate any lists in the wiki. Is course available?'); + }); + + }); + } + + function currentVideos() { + inquirer.prompt(list, function prompt(answers) { + + mdbvideos.listVideos(answers, function get(err, data, pass) { + if (err !== null) { throw err; } + if (!pass) { return videoHandler.details(data, showDetails); } + showDetails(err, data); + }); + + }); + } + + function showDetails(err, data) { + if (err !== null) { throw err; } + if (data.length) { + check[0].message = 'Select From ' + (data.length - 2) + ' Videos. Download:'; + check[0].choices = data; + + return inquirer.prompt(check, function prompt(answers) { + videoHandler.download(answers, data, argv); + }); + + } + + console.log('[' + 'i'.red + '] Could not locate any videos.'); process.exit(0); + } + + }); + + } + +};
\ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..0c82b36 --- /dev/null +++ b/package.json @@ -0,0 +1,48 @@ +{ + "name": "mongo-edu", + "preferGlobal": true, + "version": "0.1.0", + "author": "Przemyslaw Pluta <przemyslawplutadev@gmail.com> (http://przemyslawpluta.com)", + "description": "Select and download video courses from education.mongodb.com", + "main": "./mongo-edu", + "scripts": { + "start": "node ./bin/mongo-edu" + }, + "bin": { + "mongo-edu": "./bin/mongo-edu" + }, + "repository": { + "type": "git", + "url": "https://github.com/przemyslawpluta/mongo-edu.git" + }, + "keywords": [ + "mongodb", + "download", + "videos", + "youtube", + "course", + "education", + "edu" + ], + "analyze": false, + "license": "MIT", + "dependencies": { + "request": "~2.33.0", + "youtube-dl": "~1.3.2", + "colors": "~0.6.2", + "lodash": "~2.4.1", + "cheerio": "~0.13.1", + "inquirer": "~0.4.0", + "progress": "~1.1.3", + "mkdirp": "~0.3.5", + "optimist": "~0.6.1" + }, + "engines": { + "node": ">= 0.8.x" + }, + "bugs": { + "url": "https://github.com/przemyslawpluta/mongo-edu/issues" + }, + "homepage": "https://github.com/przemyslawpluta/mongo-edu", + "devDependencies": {} +} |