API Docs for: 2.0.0

src/node-utils.js

  1. // SAGE2 is available for use under the SAGE2 Software License
  2. //
  3. // University of Illinois at Chicago's Electronic Visualization Laboratory (EVL)
  4. // and University of Hawai'i at Manoa's Laboratory for Advanced Visualization and
  5. // Applications (LAVA)
  6. //
  7. // See full text, terms and conditions in the LICENSE.txt included file
  8. //
  9. // Copyright (c) 2014
  10.  
  11. /**
  12. * Provides utility functions for the SAGE2 server
  13. *
  14. * @class node-utils
  15. * @module server
  16. * @submodule node-utils
  17. * @requires package.json, request, semver, chalk, strip-ansi
  18. */
  19.  
  20. // require variables to be declared
  21. "use strict";
  22.  
  23. var SAGE2_version = require('../package.json');
  24. try {
  25. var SAGE2_buildVersion = require('../VERSION.json');
  26. } catch (e) {
  27. // nothing yet
  28. }
  29.  
  30. var crypto = require('crypto'); // https encryption
  31. var exec = require('child_process').exec; // execute external application
  32. var fs = require('fs'); // filesystem access
  33. var path = require('path'); // resolve directory paths
  34. var tls = require('tls'); // https encryption
  35.  
  36. var querystring = require('querystring'); // utilities for dealing with URL
  37.  
  38. // npm external modules
  39. var request = require('request'); // http requests
  40. var semver = require('semver'); // parse version numbers
  41. var fsmonitor = require('fsmonitor'); // file system monitoring
  42. var sanitizer = require('sanitizer'); // Caja's HTML Sanitizer as a Node.js module
  43. var chalk = require('chalk'); // colorize console output
  44. var stripansi = require('strip-ansi'); // remove ANSI color codes (dep. of chalk)
  45. var rimraf = require('rimraf'); // command rm -rf for node
  46.  
  47. /**
  48. * Parse and store NodeJS version number: detect version 0.10.x or newer
  49. *
  50. * @property _NODE_VERSION
  51. * @type {Number}
  52. */
  53. var _NODE_VERSION = 0;
  54. if (semver.gte(process.versions.node, '0.10.0')) {
  55. _NODE_VERSION = 10;
  56. if (semver.gte(process.versions.node, '0.11.0')) {
  57. _NODE_VERSION = 11;
  58. }
  59. if (semver.gte(process.versions.node, '0.12.0')) {
  60. _NODE_VERSION = 12;
  61. }
  62. if (semver.gte(process.versions.node, '1.0.0')) {
  63. _NODE_VERSION = 1;
  64. }
  65. } else {
  66. throw new Error(" SAGE2>\tOld version of Node.js. Please update");
  67. }
  68.  
  69. /**
  70. * Test if file is exists
  71. *
  72. * @method fileExists
  73. * @param filename {String} name of the file to be tested
  74. * @return {Bool} true if exists
  75. */
  76. function fileExists(filename) {
  77. if (_NODE_VERSION === 10 || _NODE_VERSION === 11) {
  78. return fs.existsSync(filename);
  79. }
  80. // Versions 1.x or above
  81. try {
  82. var res = fs.statSync(filename);
  83. return res.isFile();
  84. } catch (err) {
  85. return false;
  86. }
  87. }
  88.  
  89. /**
  90. * Test if folder is exists
  91. *
  92. * @method folderExists
  93. * @param directory {String} name of the folder to be tested
  94. * @return {Bool} true if exists
  95. */
  96. function folderExists(directory) {
  97. if (_NODE_VERSION === 10 || _NODE_VERSION === 11) {
  98. return fs.existsSync(directory);
  99. }
  100. // Versions 1.x or above
  101. try {
  102. var res = fs.statSync(directory);
  103. return res.isDirectory();
  104. } catch (err) {
  105. return false;
  106. }
  107. }
  108.  
  109. /**
  110. * Create a SSL context / credentials
  111. *
  112. * @method secureContext
  113. * @param key {String} public key
  114. * @param crt {String} private key
  115. * @param ca {String} CA key
  116. * @return {Object} secure context
  117. */
  118. function secureContext(key, crt, ca) {
  119. var ctx;
  120. if (_NODE_VERSION === 10) {
  121. ctx = crypto.createCredentials({key: key, cert: crt, ca: ca});
  122. } else {
  123. // Versions 11 or 1.x or above
  124. ctx = tls.createSecureContext({key: key, cert: crt, ca: ca});
  125. }
  126. return ctx.context;
  127. }
  128.  
  129. /**
  130. * Load a CA bundle file and return an array of certificates
  131. *
  132. * @method loadCABundle
  133. * @param filename {String} name of the file to parse
  134. * @return {Array} array of certificates data
  135. */
  136. function loadCABundle(filename) {
  137. // Initialize the array of certs
  138. var certs_array = [];
  139. var certs_idx = -1;
  140. // Read the file
  141. if (fileExists(filename)) {
  142. var rawdata = fs.readFileSync(filename, {encoding: 'utf8'});
  143. var lines = rawdata.split('\n');
  144. lines.forEach(function(line) {
  145. if (line === "-----BEGIN CERTIFICATE-----") {
  146. certs_idx = certs_idx + 1;
  147. certs_array[certs_idx] = line + '\n';
  148. } else if (line === "-----END CERTIFICATE-----") {
  149. certs_array[certs_idx] += line;
  150. } else {
  151. certs_array[certs_idx] += line + '\n';
  152. }
  153. });
  154. } else {
  155. log('loadCABundle', 'Could not find CA file:', filename);
  156. }
  157. return certs_array;
  158. }
  159.  
  160.  
  161. /**
  162. * Base version comes from evaluating the package.json file
  163. *
  164. * @method getShortVersion
  165. * @return {String} version number as x.x.x
  166. */
  167. function getShortVersion() {
  168. // try to get the version from the VERSION.json file
  169. if (SAGE2_buildVersion && SAGE2_buildVersion.version) {
  170. SAGE2_version.version = SAGE2_buildVersion.version;
  171. }
  172. return SAGE2_version.version;
  173. }
  174.  
  175.  
  176. /**
  177. * Node.js version
  178. *
  179. * @method getNodeVersion
  180. * @return {String} version number
  181. */
  182. function getNodeVersion() {
  183. // return _NODE_VERSION.toString() + " (v" + process.versions.node + ")";
  184. return process.versions.node;
  185. }
  186.  
  187. /**
  188. * Full version is processed from git information
  189. *
  190. * @method getFullVersion
  191. * @param callback {Function} function to be run when finished, parameter is an object containing base, branch,
  192. * commit and date fields
  193. */
  194. function getFullVersion(callback) {
  195. var fullVersion = {base: "", branch: "", commit: "", date: ""};
  196. // get the base version from package.json file
  197. fullVersion.base = SAGE2_version.version;
  198. // Pick up the date from package.json, if any
  199. fullVersion.date = SAGE2_version.date || "";
  200.  
  201. // get to the root folder of the sources
  202. var dirroot = path.resolve(__dirname, '..');
  203. var cmd1 = "git rev-parse --abbrev-ref HEAD";
  204. exec(cmd1, { cwd: dirroot, timeout: 3000}, function(err1, stdout1, stderr1) {
  205. if (err1) {
  206. callback(fullVersion);
  207. return;
  208. }
  209.  
  210. var branch = stdout1.substring(0, stdout1.length - 1);
  211. var cmd2 = "git log --date=\"short\" --format=\"%h|%ad\" -n 1";
  212. exec(cmd2, { cwd: dirroot, timeout: 3000}, function(err2, stdout2, stderr2) {
  213. if (err2) {
  214. callback(fullVersion);
  215. return;
  216. }
  217.  
  218. // parsing the results
  219. var result = stdout2.replace(/\r?\n|\r/g, "");
  220. var parse = result.split("|");
  221.  
  222. // filling up the object
  223. fullVersion.branch = branch;
  224. fullVersion.commit = parse[0];
  225. fullVersion.date = parse[1].replace(/-/g, "/");
  226.  
  227. // return the object in the callback paramter
  228. callback(fullVersion);
  229. });
  230. });
  231. }
  232.  
  233. /**
  234. * Upate the source code using git
  235. *
  236. * @method updateWithGIT
  237. * @param branch {String} name of the remote branch
  238. * @param callback {Function} function to be run when finished
  239. */
  240. function updateWithGIT(branch, callback) {
  241. // get to the root folder of the sources
  242. var dirroot = path.resolve(__dirname, '..');
  243. var cmd1 = "git pull origin " + branch;
  244. exec(cmd1, { cwd: dirroot, timeout: 5000}, function(err, stdout, stderr) {
  245. // return the messages in the callback paramter
  246. if (err) {
  247. callback(stdout + ' : ' + stderr, null);
  248. } else {
  249. callback(null, stdout);
  250. }
  251. });
  252. }
  253.  
  254. /**
  255. * Cleanup URL from XSS attempts
  256. *
  257. * @method sanitizedURL
  258. * @param aURL {String} a URL we received from a request
  259. * @return {String} cleanup string
  260. */
  261. function sanitizedURL(aURL) {
  262. // replace several consecutive forward slashes into one
  263. var cleaner = aURL.replace(/\/+/g, "/");
  264. // var cleaner = aURL;
  265. // convert HTML encoded content
  266. // Node doc: It will try to use decodeURIComponent in the first place, but if that fails it falls back
  267. // to a safer equivalent that doesn't throw on malformed URLs.
  268. var decode = querystring.unescape(cleaner);
  269. // Then, remove the bad parts
  270. return sanitizer.sanitize(decode);
  271. }
  272.  
  273. /**
  274. * Utility function to create a header for console messages
  275. *
  276. * @method header
  277. * @param h {String} header text
  278. * @return header {String} formatted text
  279. */
  280. function header(h) {
  281. if (h.length <= 6) {
  282. return chalk.green.bold.dim(h + ">\t\t");
  283. }
  284. return chalk.green.bold.dim(h + ">\t");
  285. }
  286.  
  287. /**
  288. * Log function for SAGE2, adds a header with color
  289. *
  290. * @method log
  291. * @param {String} head The header
  292. * @param {Array} params The parameters
  293. */
  294. function log(head, ...params) {
  295. // Adds the header strings in a new argument array
  296. if (!global.quiet) {
  297. console.log.apply(console, [header(head)].concat(params));
  298. }
  299. if (global.emitLog) {
  300. global.emitLog(stripansi(head + "> " + params + "\n"));
  301. }
  302. if (global.logger) {
  303. global.logger.log(head, stripansi(params.toString()));
  304. }
  305. }
  306.  
  307. /**
  308. * Utility function to compare two strings independently of case.
  309. * Used for sorting
  310. *
  311. * @method compareString
  312. * @param a {String} first string
  313. * @param b {String} second string
  314. */
  315. function compareString(a, b) {
  316. var nA = a.toLowerCase();
  317. var nB = b.toLowerCase();
  318. var res = 0;
  319. if (nA < nB) {
  320. res = -1;
  321. } else if (nA > nB) {
  322. res = 1;
  323. }
  324. return res;
  325. }
  326.  
  327. /**
  328. * Utility function, used while sorting, to compare two objects based on filename independently of case.
  329. * Needs a .exif.FileName field
  330. *
  331. * @method compareFilename
  332. * @param a {Object} first object
  333. * @param b {Object} second object
  334. */
  335. function compareFilename(a, b) {
  336. var nA = a.exif.FileName.toLowerCase();
  337. var nB = b.exif.FileName.toLowerCase();
  338. var res = 0;
  339. if (nA < nB) {
  340. res = -1;
  341. } else if (nA > nB) {
  342. res = 1;
  343. }
  344. return res;
  345. }
  346.  
  347. /**
  348. * Utility function to compare two objects based on title independently of case.
  349. * Needs a .exif.metadata.title field
  350. * Used for sorting
  351. *
  352. * @method compareTitle
  353. * @param a {Object} first object
  354. * @param b {Object} second object
  355. */
  356. function compareTitle(a, b) {
  357. var nA = a.exif.metadata.title.toLowerCase();
  358. var nB = b.exif.metadata.title.toLowerCase();
  359. var res = 0;
  360. if (nA < nB) {
  361. res = -1;
  362. } else if (nA > nB) {
  363. res = 1;
  364. }
  365. return res;
  366. }
  367.  
  368. /**
  369. * Utility function to test if a string or number represents a true value.
  370. * Used for parsing JSON values
  371. *
  372. * @method isTrue
  373. * @param value {Object} value to test
  374. */
  375. function isTrue(value) {
  376. if (typeof value === 'string') {
  377. value = value.toLowerCase();
  378. }
  379. switch (value) {
  380. case true:
  381. case "true":
  382. case 1:
  383. case "1":
  384. case "on":
  385. case "yes": {
  386. return true;
  387. }
  388. default: {
  389. return false;
  390. }
  391. }
  392. }
  393.  
  394. /**
  395. * Compare the installed pacakges versus the specified ones in packages.json
  396. * warms the user of outdated packages
  397. *
  398. * @method checkPackages
  399. * @param inDevelopement {Bool} whether or not to check in production mode (no devel packages)
  400. */
  401. function checkPackages(inDevelopement) {
  402. var packages = {missing: [], outdated: []};
  403. // check the commonly used NODE_ENV variable (development or production)
  404. var indevel = (process.env.NODE_ENV === 'development') || isTrue(inDevelopement);
  405. var command = "npm outdated --depth 0 --json --production";
  406. if (indevel) {
  407. command = "npm outdated --depth 0 --json";
  408. }
  409. exec(command, {cwd: path.normalize(path.join(__dirname, "..")), timeout: 30000},
  410. function(error, stdout, stderr) {
  411. // returns error code 1 if found outdated packages
  412. if (error && error.code !== 1) {
  413. log("Packages", "Warning, error running update [ " + error.cmd + '] ',
  414. 'code: ' + error.code + ' signal: ' + error.signal);
  415. return;
  416. }
  417.  
  418. var key;
  419. var output = stdout ? JSON.parse(stdout) : {};
  420. for (key in output) {
  421. // if it is not a git repository
  422. if (output[key].wanted != "git") {
  423. // if not a valid version number
  424. if (!semver.valid(output[key].current)) {
  425. packages.missing.push(key);
  426. } else if (semver.lt(output[key].current, output[key].wanted)) {
  427. // if the version is strictly lower than requested
  428. packages.outdated.push(key);
  429. }
  430. }
  431. }
  432.  
  433. if (packages.missing.length > 0 || packages.outdated.length > 0) {
  434. log("Packages", chalk.yellow.bold("Warning") + " - Packages not up to date");
  435. if (packages.missing.length > 0) {
  436. log("Packages", chalk.red.bold("Missing:"), chalk.red.bold(packages.missing));
  437. }
  438. if (packages.outdated.length > 0) {
  439. log("Packages", chalk.yellow.bold("Outdated:"), chalk.yellow.bold(packages.outdated));
  440. }
  441. log("Packages", "To update, execute: " + chalk.yellow.bold("npm run in"));
  442. } else {
  443. log("Packages", chalk.green.bold("All packages up to date"));
  444. }
  445. }
  446. );
  447. }
  448.  
  449.  
  450. /**
  451. * Register SAGE2 with EVL server
  452. *
  453. * @method registerSAGE2
  454. * @param config {Object} local SAGE2 configuration
  455. */
  456. function registerSAGE2(config) {
  457. request({
  458. rejectUnauthorized: false,
  459. url: 'https://sage.evl.uic.edu/register',
  460. // url: 'https://131.193.183.150/register',
  461. form: config,
  462. method: "POST"},
  463. function(err, response, body) {
  464. log("SAGE2", "Registration with EVL site:",
  465. (err === null) ? chalk.green.bold("success") : chalk.red.bold(err.code));
  466. });
  467. }
  468.  
  469. /**
  470. * Unregister from EVL server
  471. *
  472. * @method deregisterSAGE2
  473. * @param config {Object} local SAGE2 configuration
  474. * @param callback {Function} to be called when done
  475. */
  476. function deregisterSAGE2(config, callback) {
  477. request({
  478. rejectUnauthorized: false,
  479. url: 'https://sage.evl.uic.edu/unregister',
  480. // url: 'https://131.193.183.150/unregister',
  481. form: config,
  482. method: "POST"},
  483. function(err, response, body) {
  484. log("SAGE2", "Deregistration with EVL site:",
  485. (err === null) ? chalk.green.bold("success") : chalk.red.bold(err.code));
  486. if (callback) {
  487. callback();
  488. }
  489. });
  490. }
  491.  
  492. /**
  493. * Return a safe URL string: convert odd characters to HTML representations
  494. *
  495. * @method encodeReservedURL
  496. * @param aUrl {String} URL to be sanitized
  497. * @return {String} cleanup version of the URL
  498. */
  499. function encodeReservedURL(aUrl) {
  500. return encodeURI(aUrl).replace(/\$/g, "%24").replace(/&/g, "%26").replace(/\+/g, "%2B")
  501. .replace(/,/g, "%2C").replace(/:/g, "%3A").replace(/;/g, "%3B").replace(/=/g, "%3D")
  502. .replace(/\?/g, "%3F").replace(/@/g, "%40");
  503. }
  504.  
  505. /**
  506. * Return a safe URL string: make Windows path to URL
  507. *
  508. * @method encodeReservedPath
  509. * @param aUrl {String} URL to be sanitized
  510. * @return {String} cleanup version of the URL
  511. */
  512. function encodeReservedPath(aPath) {
  513. return encodeReservedURL(aPath.replace(/\\/g, "/"));
  514. }
  515.  
  516.  
  517. /**
  518. * Return a home directory on every platform
  519. *
  520. * @method getHomeDirectory
  521. * @return {String} string representing a folder path
  522. */
  523. function getHomeDirectory() {
  524. return process.env[ (process.platform === 'win32') ? 'USERPROFILE' : 'HOME'];
  525. }
  526.  
  527.  
  528. /**
  529. * Creates recursively a series of folders if needed (synchronous function and throws error)
  530. *
  531. * @method mkdirParent
  532. * @param dirPath {String} path to be created
  533. * @return {String} null or directory created
  534. */
  535. function mkdirParent(dirPath) {
  536. var made = null;
  537. dirPath = path.resolve(dirPath);
  538. try {
  539. fs.mkdirSync(dirPath);
  540. made = dirPath;
  541. } catch (err0) {
  542. switch (err0.code) {
  543. case 'ENOENT' : {
  544. made = mkdirParent(path.dirname(dirPath));
  545. made = mkdirParent(dirPath);
  546. break;
  547. }
  548. default: {
  549. var stat;
  550. try {
  551. stat = fs.statSync(dirPath);
  552. } catch (err1) {
  553. throw err0;
  554. }
  555. if (!stat.isDirectory()) {
  556. throw err0;
  557. }
  558. made = dirPath;
  559. break;
  560. }
  561. }
  562. }
  563. return made;
  564. }
  565.  
  566.  
  567. /**
  568. * Place a callback on a list of folders to monitor
  569. * callback triggered when a change is detected:
  570. * this.root contains the monitored folder
  571. * parameter contains the following list:
  572. * addedFiles, modifiedFiles, removedFiles,
  573. * addedFolders, modifiedFolders, removedFolders
  574. *
  575. * @method monitorFolders
  576. * @param {Array} folders list of folders to monitor
  577. * @param {Array} excludesFiles The excludes files
  578. * @param {Array} excludesFolders The excludes folders
  579. * @param {Function} callback to be called when a change is detected
  580. */
  581. function monitorFolders(folders, excludesFiles, excludesFolders, callback) {
  582. // for each folder
  583. for (var folder in folders) {
  584. // get a full path
  585. var folderpath = path.resolve(folders[folder]);
  586. // get information on the folder
  587. var stat = fs.lstatSync(folderpath);
  588. // making sure it is a folder
  589. if (stat.isDirectory()) {
  590. log("Monitor", "watching folder " + chalk.yellow.bold(folderpath));
  591. var monitor = fsmonitor.watch(folderpath, {
  592. // excludes non-valid filenames
  593. matches: function(relpath) {
  594. var condition = excludesFiles.every(function(e, i, a) {
  595. return !relpath.endsWith(e);
  596. });
  597. return condition;
  598. },
  599. // and ignores folders
  600. excludes: function(relpath) {
  601. var condition = excludesFolders.every(function(e, i, a) {
  602. return !relpath.startsWith(e);
  603. });
  604. return !condition;
  605. }
  606. });
  607. // place the callback the change event
  608. monitor.on('change', callback);
  609. }
  610. }
  611. }
  612.  
  613. /**
  614. * Merges object a and b into b
  615. *
  616. * @method mergeObjects
  617. * @param {Object} a source object
  618. * @param {Object} b destination
  619. * @param {Array} ignore object fields to ignore during merge
  620. * @return {Boolean} return true if b was modified
  621. */
  622. function mergeObjects(a, b, ignore) {
  623. var ig = ignore || [];
  624. var modified = false;
  625. // test in case of old sessions
  626. if (a === undefined || b === undefined) {
  627. return modified;
  628. }
  629. for (var key in b) {
  630. if (a[key] !== undefined && ig.indexOf(key) < 0) {
  631. var aRecurse = (a[key] === null || a[key] instanceof Array || typeof a[key] !== "object") ? false : true;
  632. var bRecurse = (b[key] === null || b[key] instanceof Array || typeof b[key] !== "object") ? false : true;
  633. if (aRecurse && bRecurse) {
  634. modified = mergeObjects(a[key], b[key]) || modified;
  635. } else if (!aRecurse && !bRecurse && a[key] !== b[key]) {
  636. b[key] = a[key];
  637. modified = true;
  638. }
  639. }
  640. }
  641. return modified;
  642. }
  643.  
  644. /**
  645. * Delete files, with glob, and a callback when done
  646. *
  647. * @method deleteFiles
  648. * @param {String} pattern string
  649. * @param {Function} cb callback when done
  650. */
  651. function deleteFiles(pattern, cb) {
  652. // use the rimraf module
  653. if (cb) {
  654. rimraf(pattern, {glob: true}, cb);
  655. } else {
  656. rimraf(pattern, {glob: true}, function(err) {
  657. if (err) {
  658. log('Files', 'error deleting files ' + pattern);
  659. }
  660. });
  661. }
  662. }
  663.  
  664.  
  665. module.exports.nodeVersion = _NODE_VERSION;
  666. module.exports.getNodeVersion = getNodeVersion;
  667. module.exports.getShortVersion = getShortVersion;
  668. module.exports.getFullVersion = getFullVersion;
  669. module.exports.secureContext = secureContext;
  670. module.exports.fileExists = fileExists;
  671. module.exports.folderExists = folderExists;
  672. module.exports.header = header;
  673. module.exports.log = log;
  674. module.exports.compareString = compareString;
  675. module.exports.compareFilename = compareFilename;
  676. module.exports.compareTitle = compareTitle;
  677. module.exports.isTrue = isTrue;
  678. module.exports.updateWithGIT = updateWithGIT;
  679. module.exports.checkPackages = checkPackages;
  680. module.exports.registerSAGE2 = registerSAGE2;
  681. module.exports.deregisterSAGE2 = deregisterSAGE2;
  682. module.exports.loadCABundle = loadCABundle;
  683. module.exports.monitorFolders = monitorFolders;
  684. module.exports.getHomeDirectory = getHomeDirectory;
  685. module.exports.mkdirParent = mkdirParent;
  686. module.exports.deleteFiles = deleteFiles;
  687. module.exports.sanitizedURL = sanitizedURL;
  688. module.exports.mergeObjects = mergeObjects;
  689.  
  690. module.exports.encodeReservedURL = encodeReservedURL;
  691. module.exports.encodeReservedPath = encodeReservedPath;
  692.