From fadd7158ed4cd79c1a3800ff1a4973bee8a4f4c7 Mon Sep 17 00:00:00 2001 From: Dale Myers Date: Mon, 23 Jan 2017 21:39:08 +0000 Subject: [PATCH 01/78] Add string escape/unescape operations These operations are useful for taking the contents of a string, and making it suitable for use as a stand alone string. For example, in an IDE you might see a string which is represented as: "Say \"Hello\"". The escaped double quotes are shown to make it clear that they do not end the string, despite the fact that they are not truly part of the string. In order to get the raw string, you would need to copy this, then manually remove the backslashes. The new String_.run_unescape operation does this automatically. The String_.run_escape is the inverse. It allows you to take a string like the one above, and paste it between two quotes without having to manually escape it. --- src/js/config/Categories.js | 2 + src/js/config/OperationConfig.js | 16 +++++++- src/js/operations/String.js | 67 ++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100755 src/js/operations/String.js diff --git a/src/js/config/Categories.js b/src/js/config/Categories.js index db43721cb..d7ad14074 100755 --- a/src/js/config/Categories.js +++ b/src/js/config/Categories.js @@ -167,6 +167,8 @@ var Categories = [ "Parse UNIX file permissions", "Swap endianness", "Parse colour code", + "Escape String", + "Unescape String", ] }, { diff --git a/src/js/config/OperationConfig.js b/src/js/config/OperationConfig.js index b5ea04278..cbed494f0 100755 --- a/src/js/config/OperationConfig.js +++ b/src/js/config/OperationConfig.js @@ -2952,5 +2952,19 @@ var OperationConfig = { value: Cipher.SUBS_CIPHERTEXT } ] - } + }, + "Escape String": { + description: "Escapes a string so that it can be embedded in another. For example, Don't stop me now becomes Don\\'t stop me now.", + run: String_.run_escape, + input_type: "string", + output_type: "string", + args: [] + }, + "Unescape String": { + description: "Unescapes a string that was embedded inside another so that it can be used in it's own right. For example, Don\\'t stop me now becomes Don't stop me now.", + run: String_.run_unescape, + input_type: "string", + output_type: "string", + args: [] + }, }; diff --git a/src/js/operations/String.js b/src/js/operations/String.js new file mode 100755 index 000000000..a85b385b9 --- /dev/null +++ b/src/js/operations/String.js @@ -0,0 +1,67 @@ +/** + * String operations. + * Namespace is appended with an underscore to prevent overwriting the global String object. + * + * @author Vel0x + * @namespace + */ +var String_ = { + + /** + * @constant + * @default + */ + REPLACEMENTS: [ + {"escaped": "\\\\", "unescaped":"\\"}, // Must be first + {"escaped": "\\'", "unescaped":"'"}, + {"escaped": "\\\"", "unescaped":"\""}, + {"escaped": "\\n", "unescaped":"\n"}, + {"escaped": "\\r", "unescaped":"\r"}, + ], + + /** + * Escapes a string for embedding in another string. + * + * Example: "Don't do that" -> "Don\'t do that" + * + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run_escape: function(input, args) { + return String_._replace_by_keys(input, "unescaped", "escaped") + }, + + /** + * Unescapes a string that was part of another string + * + * Example: "Don\'t do that" -> "Don't do that" + * + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run_unescape: function(input, args) { + return String_._replace_by_keys(input, "escaped", "unescaped") + }, + + /** + * Replaces all matching tokens in REPLACEMENTS with the correction. The + * ordering is determined by the pattern_key and the replacement_key. + * + * @param {string} input + * @param {string} pattern_key + * @param {string} replacement_key + * @returns {string} + */ + _replace_by_keys: function(input, pattern_key, replacement_key) { + var output = input; + var replacementsLength = String_.REPLACEMENTS.length; + for (var i = 0; i < replacementsLength; i++) { + var replacement = String_.REPLACEMENTS[i]; + output = output.split(replacement[pattern_key]).join(replacement[replacement_key]); + } + return output + }, + +}; From 90ed62add26c7dd0d184298b2fe11795e4245da9 Mon Sep 17 00:00:00 2001 From: Matt C Date: Tue, 25 Jul 2017 11:49:23 +0100 Subject: [PATCH 02/78] Fixes gchq/CyberChef#137 Changes data-trigger to focus so scrolling works and sets max height. --- src/web/HTMLOperation.js | 2 +- src/web/stylesheets/utils/_overrides.css | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/web/HTMLOperation.js b/src/web/HTMLOperation.js index ea0d43970..499faab55 100755 --- a/src/web/HTMLOperation.js +++ b/src/web/HTMLOperation.js @@ -51,7 +51,7 @@ HTMLOperation.prototype.toStubHtml = function(removeIcon) { if (this.description) { html += " data-container='body' data-toggle='popover' data-placement='auto right'\ - data-content=\"" + this.description + "\" data-html='true' data-trigger='hover'"; + data-content=\"" + this.description + "\" data-html='true' data-trigger='focus' tabindex='0'"; } html += ">" + this.name; diff --git a/src/web/stylesheets/utils/_overrides.css b/src/web/stylesheets/utils/_overrides.css index 8957071a3..e060db7d5 100755 --- a/src/web/stylesheets/utils/_overrides.css +++ b/src/web/stylesheets/utils/_overrides.css @@ -140,6 +140,8 @@ optgroup { .popover { background-color: var(--popover-background); border-color: var(--popover-border-colour); + max-height: 100%; + overflow-y: auto; } From de80db73f26251ea28b72de2e32ed82cb2239726 Mon Sep 17 00:00:00 2001 From: Matt C Date: Tue, 25 Jul 2017 16:27:59 +0100 Subject: [PATCH 03/78] Adds initial JPath functionality --- package.json | 1 + src/core/config/Categories.js | 2 ++ src/core/config/OperationConfig.js | 18 +++++++++++++++ src/core/operations/Code.js | 36 ++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+) diff --git a/package.json b/package.json index 208789bd2..f1de5a5f8 100644 --- a/package.json +++ b/package.json @@ -80,6 +80,7 @@ "lodash": "^4.17.4", "moment": "^2.17.1", "moment-timezone": "^0.5.11", + "node-jpath": "^2.1.0", "sladex-blowfish": "^0.8.1", "sortablejs": "^1.5.1", "split.js": "^1.2.0", diff --git a/src/core/config/Categories.js b/src/core/config/Categories.js index ce46d2211..b3e40b33e 100755 --- a/src/core/config/Categories.js +++ b/src/core/config/Categories.js @@ -215,6 +215,7 @@ const Categories = [ "Extract dates", "Regular expression", "XPath expression", + "JPath expression", "CSS selector", "Extract EXIF", ] @@ -278,6 +279,7 @@ const Categories = [ "CSS Beautify", "CSS Minify", "XPath expression", + "JPath expression", "CSS selector", "Strip HTML tags", "Diff", diff --git a/src/core/config/OperationConfig.js b/src/core/config/OperationConfig.js index b0c005b38..c058d9e27 100755 --- a/src/core/config/OperationConfig.js +++ b/src/core/config/OperationConfig.js @@ -2243,6 +2243,24 @@ const OperationConfig = { } ] }, + "JPath expression": { + description: "Extract information from a JSON object with an JPath query", + run: Code.runJpath, + inputType: "string", + outputType: "string", + args: [ + { + name: "JPath", + type: "string", + value: Code.JPATH_INITIAL + }, + { + name: "Result delimiter", + type: "binaryShortString", + value: Code.JPATH_DELIMITER + } + ] + }, "CSS selector": { description: "Extract information from an HTML document with a CSS selector", run: Code.runCSSQuery, diff --git a/src/core/operations/Code.js b/src/core/operations/Code.js index 9840797d7..d3f5c0f05 100755 --- a/src/core/operations/Code.js +++ b/src/core/operations/Code.js @@ -4,6 +4,7 @@ import Utils from "../Utils.js"; import vkbeautify from "vkbeautify"; import {DOMParser as dom} from "xmldom"; import xpath from "xpath"; +import jpath from "node-jpath"; import prettyPrintOne from "imports-loader?window=>global!exports-loader?prettyPrintOne!google-code-prettify/bin/prettify.min.js"; @@ -354,6 +355,41 @@ const Code = { return nodes.map(nodeToString).join(delimiter); }, + /** + * @constant + * @default + */ + JPATH_INITIAL: "", + + /** + * @constant + * @default + */ + JPATH_DELIMITER: "\\n", + + /** + * XPath expression operation. + * + * @author Matt C (matt@artemisbot.uk) + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + runJpath: function(input, args) { + let query = args[0], + delimiter = args[1]; + + let obj; + try { + obj = JSON.parse(input); + } catch (err) { + return "Invalid input JSON."; + } + + let results = jpath.filter(obj, query); + return results.map(result => JSON.stringify(result)).join(delimiter); + }, + /** * @constant From e0905255bae2c2c032e1afa430108b383649fe77 Mon Sep 17 00:00:00 2001 From: Matt C Date: Tue, 25 Jul 2017 16:36:01 +0100 Subject: [PATCH 04/78] Forgot about package-lock.json oops --- package-lock.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/package-lock.json b/package-lock.json index c8e873200..42a33aa4f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4687,6 +4687,11 @@ "lower-case": "1.1.4" } }, + "node-jpath": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/node-jpath/-/node-jpath-2.1.0.tgz", + "integrity": "sha1-IYckZgObw5adyzhjq0JnH2H/UbI=" + }, "node-libs-browser": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.0.0.tgz", From 33ecbfa95b96dbce1e815c08ad0db9a9354f7c5f Mon Sep 17 00:00:00 2001 From: Matt C Date: Fri, 28 Jul 2017 21:47:47 +0100 Subject: [PATCH 05/78] Fixed arrow issue --- src/web/stylesheets/utils/_overrides.css | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/web/stylesheets/utils/_overrides.css b/src/web/stylesheets/utils/_overrides.css index e060db7d5..36069cd5e 100755 --- a/src/web/stylesheets/utils/_overrides.css +++ b/src/web/stylesheets/utils/_overrides.css @@ -140,10 +140,12 @@ optgroup { .popover { background-color: var(--popover-background); border-color: var(--popover-border-colour); - max-height: 100%; - overflow-y: auto; } +.popover-content { + max-height: 90vh; + overflow-y: auto; +} .popover.right>.arrow { border-right-color: var(--popover-border-colour); From 9ee0964d0e223689bb462844b5a244bbab20f8b8 Mon Sep 17 00:00:00 2001 From: Matt C Date: Sat, 29 Jul 2017 00:45:41 +0100 Subject: [PATCH 06/78] Fixed hover issue - now allows scrolling --- src/web/HTMLOperation.js | 4 ++-- src/web/OperationsWaiter.js | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/web/HTMLOperation.js b/src/web/HTMLOperation.js index 499faab55..aae9f282f 100755 --- a/src/web/HTMLOperation.js +++ b/src/web/HTMLOperation.js @@ -50,8 +50,8 @@ HTMLOperation.prototype.toStubHtml = function(removeIcon) { let html = "
  • Date: Tue, 1 Aug 2017 14:42:09 +0000 Subject: [PATCH 07/78] Dependencies in the node version are now kept external in the webpack build --- Gruntfile.js | 3 +++ package-lock.json | 14 ++++++++------ package.json | 5 +++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index 42cd59233..a025e940b 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -1,6 +1,7 @@ const webpack = require("webpack"); const ExtractTextPlugin = require("extract-text-webpack-plugin"); const HtmlWebpackPlugin = require("html-webpack-plugin"); +const NodeExternals = require("webpack-node-externals"); const Inliner = require("web-resource-inliner"); module.exports = function (grunt) { @@ -294,6 +295,7 @@ module.exports = function (grunt) { tests: { target: "node", entry: "./test/index.js", + externals: [NodeExternals()], output: { filename: "index.js", path: __dirname + "/build/test" @@ -302,6 +304,7 @@ module.exports = function (grunt) { node: { target: "node", entry: "./src/node/index.js", + externals: [NodeExternals()], output: { filename: "CyberChef.js", path: __dirname + "/build/node", diff --git a/package-lock.json b/package-lock.json index c8e873200..8533827a2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -865,7 +865,6 @@ "version": "6.23.0", "resolved": "https://registry.npmjs.org/babel-polyfill/-/babel-polyfill-6.23.0.tgz", "integrity": "sha1-g2TKYt+Or7gwSZ9pkXdGbDsDSZ0=", - "dev": true, "requires": { "babel-runtime": "6.23.0", "core-js": "2.4.1", @@ -929,7 +928,6 @@ "version": "6.23.0", "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.23.0.tgz", "integrity": "sha1-CpSJ8UTecO+zzkMArM2zKeL8VDs=", - "dev": true, "requires": { "core-js": "2.4.1", "regenerator-runtime": "0.10.5" @@ -1557,8 +1555,7 @@ "core-js": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.4.1.tgz", - "integrity": "sha1-TekR5mew6ukSTjQlS1OupvxhjT4=", - "dev": true + "integrity": "sha1-TekR5mew6ukSTjQlS1OupvxhjT4=" }, "core-util-is": { "version": "1.0.2", @@ -6208,8 +6205,7 @@ "regenerator-runtime": { "version": "0.10.5", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.10.5.tgz", - "integrity": "sha1-M2w+/BIgrc7dosn6tntaeVWjNlg=", - "dev": true + "integrity": "sha1-M2w+/BIgrc7dosn6tntaeVWjNlg=" }, "regenerator-transform": { "version": "0.9.11", @@ -7438,6 +7434,12 @@ } } }, + "webpack-node-externals": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.6.0.tgz", + "integrity": "sha1-Iyxi7GCSsQBjWj0p2DwXRxKN+b0=", + "dev": true + }, "webpack-sources": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.0.1.tgz", diff --git a/package.json b/package.json index 208789bd2..89729ed2b 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,6 @@ "devDependencies": { "babel-core": "^6.24.0", "babel-loader": "^7.1.1", - "babel-polyfill": "^6.23.0", "babel-preset-env": "^1.6.0", "css-loader": "^0.28.4", "exports-loader": "^0.6.4", @@ -60,11 +59,13 @@ "style-loader": "^0.18.2", "url-loader": "^0.5.8", "web-resource-inliner": "^4.1.0", - "webpack": "^3.3.0" + "webpack": "^3.3.0", + "webpack-node-externals": "^1.6.0" }, "dependencies": { "bootstrap": "^3.3.7", "bootstrap-colorpicker": "^2.5.1", + "babel-polyfill": "^6.23.0", "bootstrap-switch": "^3.3.4", "crypto-api": "^0.6.2", "crypto-js": "^3.1.9-1", From 1435acdc28be451b175117cb8b2034806db635f8 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 1 Aug 2017 14:42:52 +0000 Subject: [PATCH 08/78] 5.12.3 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7fe6de2ae..d059c100e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "5.12.2", + "version": "5.12.3", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 90f815313..fd08e668b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "5.12.2", + "version": "5.12.3", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From e8e5eb9c53993521e886b56ad31bfeac7eda3f87 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 1 Aug 2017 19:23:30 +0000 Subject: [PATCH 09/78] Fixed some edge cases for popover triggering --- src/web/HTMLOperation.js | 4 ++-- src/web/OperationsWaiter.js | 23 ++++++++++++++++++----- src/web/RecipeWaiter.js | 5 ++--- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/web/HTMLOperation.js b/src/web/HTMLOperation.js index aae9f282f..ea0d43970 100755 --- a/src/web/HTMLOperation.js +++ b/src/web/HTMLOperation.js @@ -50,8 +50,8 @@ HTMLOperation.prototype.toStubHtml = function(removeIcon) { let html = "
  • Date: Tue, 1 Aug 2017 19:27:24 +0000 Subject: [PATCH 10/78] 5.12.4 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index d059c100e..66bbd0c9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "5.12.3", + "version": "5.12.4", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index fd08e668b..0c2c2c601 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "5.12.3", + "version": "5.12.4", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From ab1c9e27dc1f646be04810eb447bdd14265d0705 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 3 Aug 2017 10:57:54 +0000 Subject: [PATCH 11/78] Added more loading messages --- src/web/html/index.html | 15 ++++++++++----- src/web/stylesheets/preloader.css | 4 ++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/web/html/index.html b/src/web/html/index.html index 34c0958ad..294b41cc0 100755 --- a/src/web/html/index.html +++ b/src/web/html/index.html @@ -48,10 +48,15 @@ "Initialising Skynet...", "[REDACTED]", "Downloading more RAM...", - "Loading more loading messages...", "Ordering 1s and 0s...", "Navigating neural network...", - "Importing machine learning..." + "Importing machine learning...", + "Issuing Alice and Bob one-time pads...", + "Mining bitcoin cash...", + "Generating key material by trying to escape vim...", + "for i in range(additional): Pylon()", + "(creating unresolved tension...", + "Symlinking emacs and vim to ed...", ]; // Shuffle array using Durstenfeld algorithm @@ -62,19 +67,19 @@ loadingMsgs[j] = temp; } - // Show next loading message then move it to the end of the array + // Show next loading message and move it to the end of the array function changeLoadingMsg() { const msg = loadingMsgs.shift(); + loadingMsgs.push(msg); try { const el = document.getElementById("preloader-msg"); el.className = "loading"; // Causes CSS transition on first message el.innerHTML = msg; } catch (err) {} // Ignore errors if DOM not yet ready - loadingMsgs.push(msg); } changeLoadingMsg(); - window.loadingMsgsInt = setInterval(changeLoadingMsg, (Math.random() * 1000) + 1000); + window.loadingMsgsInt = setInterval(changeLoadingMsg, (Math.random() * 2000) + 1500); <% if (!htmlWebpackPlugin.options.inline) { %>