8.9.0

CHANGELOG.md

@@ -1,6 +1,9 @@
 # Changelog
 All notable changes to CyberChef will be documented in this file.
 
+### [8.9.0] - 2018-11-07
+- 'Defang URL' operation added [@arnydo] | [#394]
+
 ### [8.8.0] - 2018-10-10
 - 'Parse TLV' operation added [@GCHQ77703] | [#351]
 
@@ -76,6 +79,7 @@ All notable changes to CyberChef will be documented in this file.
 [@JustAnotherMark]: https://github.com/JustAnotherMark
 [@sevzero]: https://github.com/sevzero
 [@PenguinGeorge]: https://github.com/PenguinGeorge
+[@arnydo]: https://github.com/arnydo
 
 [#95]: https://github.com/gchq/CyberChef/pull/299
 [#173]: https://github.com/gchq/CyberChef/pull/173
@@ -95,3 +99,4 @@ All notable changes to CyberChef will be documented in this file.
 [#344]: https://github.com/gchq/CyberChef/pull/344
 [#348]: https://github.com/gchq/CyberChef/pull/348
 [#351]: https://github.com/gchq/CyberChef/pull/351
+[#394]: https://github.com/gchq/CyberChef/pull/394

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.8.7",
+  "version": "8.9.0",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",
@@ -30,14 +30,14 @@
   "main": "build/node/CyberChef.js",
   "bugs": "https://github.com/gchq/CyberChef/issues",
   "devDependencies": {
-    "@babel/core": "^7.1.2",
-    "@babel/preset-env": "^7.1.0",
-    "autoprefixer": "^9.1.5",
+    "@babel/core": "^7.1.5",
+    "@babel/preset-env": "^7.1.5",
+    "autoprefixer": "^9.3.1",
     "babel-loader": "^8.0.4",
     "bootstrap": "^4.1.3",
     "colors": "^1.3.2",
-    "css-loader": "^1.0.0",
-    "eslint": "^5.6.1",
+    "css-loader": "^1.0.1",
+    "eslint": "^5.8.0",
     "exports-loader": "^0.7.0",
     "extract-text-webpack-plugin": "^4.0.0-alpha0",
     "file-loader": "^2.0.0",
@@ -55,20 +55,19 @@
     "html-webpack-plugin": "^3.2.0",
     "imports-loader": "^0.8.0",
     "ink-docstrap": "^1.3.2",
-    "js-to-mjs": "^0.2.0",
     "jsdoc-babel": "^0.5.0",
-    "node-sass": "^4.9.3",
+    "node-sass": "^4.10.0",
     "postcss-css-variables": "^0.11.0",
-    "postcss-import": "^12.0.0",
+    "postcss-import": "^12.0.1",
     "postcss-loader": "^3.0.0",
     "prompt": "^1.0.0",
     "sass-loader": "^7.1.0",
-    "sitemap": "^2.0.1",
+    "sitemap": "^2.1.0",
     "style-loader": "^0.23.1",
     "url-loader": "^1.1.2",
     "web-resource-inliner": "^4.2.1",
-    "webpack": "^4.20.2",
-    "webpack-dev-server": "^3.1.9",
+    "webpack": "^4.25.1",
+    "webpack-dev-server": "^3.1.10",
     "webpack-node-externals": "^1.7.2",
     "worker-loader": "^2.0.0"
   },
@@ -77,22 +76,22 @@
     "babel-plugin-transform-builtin-extend": "1.1.2",
     "babel-polyfill": "^6.26.0",
     "bcryptjs": "^2.4.3",
-    "bignumber.js": "^7.2.1",
+    "bignumber.js": "^8.0.1",
     "bootstrap-colorpicker": "^2.5.3",
     "bootstrap-material-design": "^4.1.1",
     "bson": "^3.0.2",
     "chi-squared": "^1.1.0",
-    "crypto-api": "^0.8.0",
+    "crypto-api": "^0.8.3",
     "crypto-js": "^3.1.9-1",
     "ctph.js": "0.0.5",
     "diff": "^3.5.0",
-    "es6-promisify": "^6.0.0",
+    "es6-promisify": "^6.0.1",
     "escodegen": "^1.11.0",
     "esmangle": "^1.0.1",
     "esprima": "^4.0.1",
     "exif-parser": "^0.1.12",
-    "file-saver": "^2.0.0-rc.3",
-    "highlight.js": "^9.12.0",
+    "file-saver": "^2.0.0-rc.4",
+    "highlight.js": "^9.13.1",
     "jquery": "^3.3.1",
     "js-crc": "^0.2.0",
     "js-sha3": "^0.8.0",
@@ -101,12 +100,12 @@
     "jsonpath": "^1.0.0",
     "jsonwebtoken": "^8.3.0",
     "jsrsasign": "8.0.12",
-    "kbpgp": "^2.0.80",
+    "kbpgp": "^2.0.82",
     "lodash": "^4.17.11",
     "loglevel": "^1.6.1",
     "loglevel-message-prefix": "^3.0.0",
     "moment": "^2.22.2",
-    "moment-timezone": "^0.5.21",
+    "moment-timezone": "^0.5.23",
     "ngeohash": "^0.6.0",
     "node-forge": "^0.7.6",
     "node-md6": "^0.1.0",
@@ -117,9 +116,9 @@
     "scryptsy": "^2.0.0",
     "snackbarjs": "^1.1.0",
     "sortablejs": "^1.7.0",
-    "split.js": "^1.5.2",
+    "split.js": "^1.5.9",
     "ssdeep.js": "0.0.2",
-    "ua-parser-js": "^0.7.18",
+    "ua-parser-js": "^0.7.19",
     "utf8": "^3.0.0",
     "vkbeautify": "^0.99.3",
     "xmldom": "^0.1.27",
@@ -133,7 +132,6 @@
     "test": "grunt test",
     "docs": "grunt docs",
     "lint": "grunt lint",
-    "newop": "node --experimental-modules src/core/config/scripts/newOperation.mjs",
-    "postinstall": "[ -f node_modules/crypto-api/src/crypto-api.mjs ] || npx j2m node_modules/crypto-api/src/crypto-api.js"
+    "newop": "node --experimental-modules src/core/config/scripts/newOperation.mjs"
   }
 }

src/core/config/Categories.json

@@ -159,7 +159,8 @@
             "Change IP format",
             "Group IP addresses",
             "Encode NetBIOS Name",
-            "Decode NetBIOS Name"
+            "Decode NetBIOS Name",
+            "Defang URL"
         ]
     },
     {

src/core/lib/Extract.mjs

@@ -39,3 +39,21 @@ export function search (input, searchRegex, removeRegex, includeTotal) {
 
     return output;
 }
+
+
+/**
+ * URL regular expression
+ */
+const protocol = "[A-Z]+://",
+    hostname = "[-\\w]+(?:\\.\\w[-\\w]*)+",
+    port = ":\\d+",
+    path = "/[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]*" +
+        "(?:[.!,?]+[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]+)*";
+
+export const URL_REGEX = new RegExp(protocol + hostname + "(?:" + port + ")?(?:" + path + ")?", "ig");
+
+
+/**
+ * Domain name regular expression
+ */
+export const DOMAIN_REGEX = /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/ig;

src/core/operations/DefangURL.mjs

@@ -0,0 +1,102 @@
+/**
+ * @author arnydo [arnydo@protonmail.com]
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import {URL_REGEX, DOMAIN_REGEX} from "../lib/Extract";
+
+/**
+ * DefangURL operation
+ */
+class DefangURL extends Operation {
+
+    /**
+     * DefangURL constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Defang URL";
+        this.module = "Default";
+        this.description = "Takes a Universal Resource Locator (URL) and 'Defangs' it; meaning the URL becomes invalid, neutralising the risk of accidentally clicking on a malicious link.<br><br>This is often used when dealing with malicious links or IOCs.<br><br>Works well when combined with the 'Extract URLs' operation.";
+        this.infoURL = "https://isc.sans.edu/forums/diary/Defang+all+the+things/22744/";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [
+            {
+                name: "Escape dots",
+                type: "boolean",
+                value: true
+            },
+            {
+                name: "Escape http",
+                type: "boolean",
+                value: true
+            },
+            {
+                name: "Escape ://",
+                type: "boolean",
+                value: true
+            },
+            {
+                name: "Process",
+                type: "option",
+                value: ["Valid domains and full URLs", "Only full URLs", "Everything"]
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        const [dots, http, slashes, process] = args;
+
+        switch (process) {
+            case "Valid domains and full URLs":
+                input = input.replace(URL_REGEX, x => {
+                    return defangURL(x, dots, http, slashes);
+                });
+                input = input.replace(DOMAIN_REGEX, x => {
+                    return defangURL(x, dots, http, slashes);
+                });
+                break;
+            case "Only full URLs":
+                input = input.replace(URL_REGEX, x => {
+                    return defangURL(x, dots, http, slashes);
+                });
+                break;
+            case "Everything":
+                input = defangURL(input, dots, http, slashes);
+                break;
+        }
+
+        return input;
+    }
+
+}
+
+
+/**
+ * Defangs a given URL
+ *
+ * @param {string} url
+ * @param {boolean} dots
+ * @param {boolean} http
+ * @param {boolean} slashes
+ * @returns {string}
+ */
+function defangURL(url, dots, http, slashes) {
+    if (dots) url = url.replace(/\./g, "[.]");
+    if (http) url = url.replace(/http/gi, "hxxp");
+    if (slashes) url = url.replace(/:\/\//g, "[://]");
+
+    return url;
+}
+
+export default DefangURL;

src/core/operations/ExtractDomains.mjs

@@ -5,7 +5,7 @@
  */
 
 import Operation from "../Operation";
-import { search } from "../lib/Extract";
+import { search, DOMAIN_REGEX } from "../lib/Extract";
 
 /**
  * Extract domains operation
@@ -38,10 +38,8 @@ class ExtractDomains extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const displayTotal = args[0],
-            regex = /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/ig;
-
-        return search(input, regex, null, displayTotal);
+        const displayTotal = args[0];
+        return search(input, DOMAIN_REGEX, null, displayTotal);
     }
 
 }

src/core/operations/ExtractURLs.mjs

@@ -5,7 +5,7 @@
  */
 
 import Operation from "../Operation";
-import { search } from "../lib/Extract";
+import { search, URL_REGEX } from "../lib/Extract";
 
 /**
  * Extract URLs operation
@@ -38,16 +38,8 @@ class ExtractURLs extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const displayTotal = args[0],
-            protocol = "[A-Z]+://",
-            hostname = "[-\\w]+(?:\\.\\w[-\\w]*)+",
-            port = ":\\d+";
-        let path = "/[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]*";
-
-        path += "(?:[.!,?]+[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]+)*";
-        const regex = new RegExp(protocol + hostname + "(?:" + port +
-            ")?(?:" + path + ")?", "ig");
-        return search(input, regex, null, displayTotal);
+        const displayTotal = args[0];
+        return search(input, URL_REGEX, null, displayTotal);
     }
 
 }

src/core/operations/HMAC.mjs

@@ -71,13 +71,6 @@ class HMAC extends Operation {
             msg = Utils.arrayBufferToStr(input, false),
             hasher = CryptoApi.getHasher(hashFunc);
 
-        // Horrible shim to fix constructor bug. Reported in nf404/crypto-api#8
-        hasher.reset = () => {
-            hasher.state = {};
-            const tmp = new hasher.constructor();
-            hasher.state = tmp.state;
-        };
-
         const mac = CryptoApi.getHmac(CryptoApi.encoder.fromUtf(key), hasher);
         mac.update(msg);
         return CryptoApi.encoder.toHex(mac.finalize());