8.10.0

CHANGELOG.md

@@ -1,6 +1,12 @@
 # Changelog
 All notable changes to CyberChef will be documented in this file.
 
+### [8.10.0] - 2018-11-07
+- 'Remove Diacritics' operation added [@klaxon1] | [#387]
+
+### [8.9.0] - 2018-11-07
+- 'Defang URL' operation added [@arnydo] | [#394]
+
 ### [8.8.0] - 2018-10-10
 - 'Parse TLV' operation added [@GCHQ77703] | [#351]
 
@@ -76,6 +82,8 @@ All notable changes to CyberChef will be documented in this file.
 [@JustAnotherMark]: https://github.com/JustAnotherMark
 [@sevzero]: https://github.com/sevzero
 [@PenguinGeorge]: https://github.com/PenguinGeorge
+[@arnydo]: https://github.com/arnydo
+[@klaxon1]: https://github.com/klaxon1
 
 [#95]: https://github.com/gchq/CyberChef/pull/299
 [#173]: https://github.com/gchq/CyberChef/pull/173
@@ -95,3 +103,5 @@ All notable changes to CyberChef will be documented in this file.
 [#344]: https://github.com/gchq/CyberChef/pull/344
 [#348]: https://github.com/gchq/CyberChef/pull/348
 [#351]: https://github.com/gchq/CyberChef/pull/351
+[#387]: https://github.com/gchq/CyberChef/pull/387
+[#394]: https://github.com/gchq/CyberChef/pull/394

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.8.7",
+  "version": "8.10.0",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",
@@ -30,14 +30,14 @@
   "main": "build/node/CyberChef.js",
   "bugs": "https://github.com/gchq/CyberChef/issues",
   "devDependencies": {
-    "@babel/core": "^7.1.2",
-    "@babel/preset-env": "^7.1.0",
-    "autoprefixer": "^9.1.5",
+    "@babel/core": "^7.1.5",
+    "@babel/preset-env": "^7.1.5",
+    "autoprefixer": "^9.3.1",
     "babel-loader": "^8.0.4",
     "bootstrap": "^4.1.3",
     "colors": "^1.3.2",
-    "css-loader": "^1.0.0",
-    "eslint": "^5.6.1",
+    "css-loader": "^1.0.1",
+    "eslint": "^5.8.0",
     "exports-loader": "^0.7.0",
     "extract-text-webpack-plugin": "^4.0.0-alpha0",
     "file-loader": "^2.0.0",
@@ -55,20 +55,19 @@
     "html-webpack-plugin": "^3.2.0",
     "imports-loader": "^0.8.0",
     "ink-docstrap": "^1.3.2",
-    "js-to-mjs": "^0.2.0",
     "jsdoc-babel": "^0.5.0",
-    "node-sass": "^4.9.3",
+    "node-sass": "^4.10.0",
     "postcss-css-variables": "^0.11.0",
-    "postcss-import": "^12.0.0",
+    "postcss-import": "^12.0.1",
     "postcss-loader": "^3.0.0",
     "prompt": "^1.0.0",
     "sass-loader": "^7.1.0",
-    "sitemap": "^2.0.1",
+    "sitemap": "^2.1.0",
     "style-loader": "^0.23.1",
     "url-loader": "^1.1.2",
     "web-resource-inliner": "^4.2.1",
-    "webpack": "^4.20.2",
-    "webpack-dev-server": "^3.1.9",
+    "webpack": "^4.25.1",
+    "webpack-dev-server": "^3.1.10",
     "webpack-node-externals": "^1.7.2",
     "worker-loader": "^2.0.0"
   },
@@ -77,22 +76,22 @@
     "babel-plugin-transform-builtin-extend": "1.1.2",
     "babel-polyfill": "^6.26.0",
     "bcryptjs": "^2.4.3",
-    "bignumber.js": "^7.2.1",
+    "bignumber.js": "^8.0.1",
     "bootstrap-colorpicker": "^2.5.3",
     "bootstrap-material-design": "^4.1.1",
     "bson": "^3.0.2",
     "chi-squared": "^1.1.0",
-    "crypto-api": "^0.8.0",
+    "crypto-api": "^0.8.3",
     "crypto-js": "^3.1.9-1",
     "ctph.js": "0.0.5",
     "diff": "^3.5.0",
-    "es6-promisify": "^6.0.0",
+    "es6-promisify": "^6.0.1",
     "escodegen": "^1.11.0",
     "esmangle": "^1.0.1",
     "esprima": "^4.0.1",
     "exif-parser": "^0.1.12",
-    "file-saver": "^2.0.0-rc.3",
-    "highlight.js": "^9.12.0",
+    "file-saver": "^2.0.0-rc.4",
+    "highlight.js": "^9.13.1",
     "jquery": "^3.3.1",
     "js-crc": "^0.2.0",
     "js-sha3": "^0.8.0",
@@ -101,12 +100,12 @@
     "jsonpath": "^1.0.0",
     "jsonwebtoken": "^8.3.0",
     "jsrsasign": "8.0.12",
-    "kbpgp": "^2.0.80",
+    "kbpgp": "^2.0.82",
     "lodash": "^4.17.11",
     "loglevel": "^1.6.1",
     "loglevel-message-prefix": "^3.0.0",
     "moment": "^2.22.2",
-    "moment-timezone": "^0.5.21",
+    "moment-timezone": "^0.5.23",
     "ngeohash": "^0.6.0",
     "node-forge": "^0.7.6",
     "node-md6": "^0.1.0",
@@ -117,9 +116,9 @@
     "scryptsy": "^2.0.0",
     "snackbarjs": "^1.1.0",
     "sortablejs": "^1.7.0",
-    "split.js": "^1.5.2",
+    "split.js": "^1.5.9",
     "ssdeep.js": "0.0.2",
-    "ua-parser-js": "^0.7.18",
+    "ua-parser-js": "^0.7.19",
     "utf8": "^3.0.0",
     "vkbeautify": "^0.99.3",
     "xmldom": "^0.1.27",
@@ -133,7 +132,6 @@
     "test": "grunt test",
     "docs": "grunt docs",
     "lint": "grunt lint",
-    "newop": "node --experimental-modules src/core/config/scripts/newOperation.mjs",
-    "postinstall": "[ -f node_modules/crypto-api/src/crypto-api.mjs ] || npx j2m node_modules/crypto-api/src/crypto-api.js"
+    "newop": "node --experimental-modules src/core/config/scripts/newOperation.mjs"
   }
 }

src/core/config/Categories.json

@@ -159,7 +159,8 @@
             "Change IP format",
             "Group IP addresses",
             "Encode NetBIOS Name",
-            "Decode NetBIOS Name"
+            "Decode NetBIOS Name",
+            "Defang URL"
         ]
     },
     {
@@ -167,6 +168,7 @@
         "ops": [
             "Encode text",
             "Decode text",
+            "Remove Diacritics",
             "Unescape Unicode Characters"
         ]
     },

src/core/lib/Extract.mjs

@@ -39,3 +39,21 @@ export function search (input, searchRegex, removeRegex, includeTotal) {
 
     return output;
 }
+
+
+/**
+ * URL regular expression
+ */
+const protocol = "[A-Z]+://",
+    hostname = "[-\\w]+(?:\\.\\w[-\\w]*)+",
+    port = ":\\d+",
+    path = "/[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]*" +
+        "(?:[.!,?]+[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]+)*";
+
+export const URL_REGEX = new RegExp(protocol + hostname + "(?:" + port + ")?(?:" + path + ")?", "ig");
+
+
+/**
+ * Domain name regular expression
+ */
+export const DOMAIN_REGEX = /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/ig;

src/core/operations/DefangURL.mjs

@@ -0,0 +1,102 @@
+/**
+ * @author arnydo [arnydo@protonmail.com]
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import {URL_REGEX, DOMAIN_REGEX} from "../lib/Extract";
+
+/**
+ * DefangURL operation
+ */
+class DefangURL extends Operation {
+
+    /**
+     * DefangURL constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Defang URL";
+        this.module = "Default";
+        this.description = "Takes a Universal Resource Locator (URL) and 'Defangs' it; meaning the URL becomes invalid, neutralising the risk of accidentally clicking on a malicious link.<br><br>This is often used when dealing with malicious links or IOCs.<br><br>Works well when combined with the 'Extract URLs' operation.";
+        this.infoURL = "https://isc.sans.edu/forums/diary/Defang+all+the+things/22744/";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [
+            {
+                name: "Escape dots",
+                type: "boolean",
+                value: true
+            },
+            {
+                name: "Escape http",
+                type: "boolean",
+                value: true
+            },
+            {
+                name: "Escape ://",
+                type: "boolean",
+                value: true
+            },
+            {
+                name: "Process",
+                type: "option",
+                value: ["Valid domains and full URLs", "Only full URLs", "Everything"]
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        const [dots, http, slashes, process] = args;
+
+        switch (process) {
+            case "Valid domains and full URLs":
+                input = input.replace(URL_REGEX, x => {
+                    return defangURL(x, dots, http, slashes);
+                });
+                input = input.replace(DOMAIN_REGEX, x => {
+                    return defangURL(x, dots, http, slashes);
+                });
+                break;
+            case "Only full URLs":
+                input = input.replace(URL_REGEX, x => {
+                    return defangURL(x, dots, http, slashes);
+                });
+                break;
+            case "Everything":
+                input = defangURL(input, dots, http, slashes);
+                break;
+        }
+
+        return input;
+    }
+
+}
+
+
+/**
+ * Defangs a given URL
+ *
+ * @param {string} url
+ * @param {boolean} dots
+ * @param {boolean} http
+ * @param {boolean} slashes
+ * @returns {string}
+ */
+function defangURL(url, dots, http, slashes) {
+    if (dots) url = url.replace(/\./g, "[.]");
+    if (http) url = url.replace(/http/gi, "hxxp");
+    if (slashes) url = url.replace(/:\/\//g, "[://]");
+
+    return url;
+}
+
+export default DefangURL;

src/core/operations/ExtractDomains.mjs

@@ -5,7 +5,7 @@
  */
 
 import Operation from "../Operation";
-import { search } from "../lib/Extract";
+import { search, DOMAIN_REGEX } from "../lib/Extract";
 
 /**
  * Extract domains operation
@@ -38,10 +38,8 @@ class ExtractDomains extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const displayTotal = args[0],
-            regex = /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/ig;
-
-        return search(input, regex, null, displayTotal);
+        const displayTotal = args[0];
+        return search(input, DOMAIN_REGEX, null, displayTotal);
     }
 
 }

src/core/operations/ExtractURLs.mjs

@@ -5,7 +5,7 @@
  */
 
 import Operation from "../Operation";
-import { search } from "../lib/Extract";
+import { search, URL_REGEX } from "../lib/Extract";
 
 /**
  * Extract URLs operation
@@ -38,16 +38,8 @@ class ExtractURLs extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const displayTotal = args[0],
-            protocol = "[A-Z]+://",
-            hostname = "[-\\w]+(?:\\.\\w[-\\w]*)+",
-            port = ":\\d+";
-        let path = "/[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]*";
-
-        path += "(?:[.!,?]+[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]+)*";
-        const regex = new RegExp(protocol + hostname + "(?:" + port +
-            ")?(?:" + path + ")?", "ig");
-        return search(input, regex, null, displayTotal);
+        const displayTotal = args[0];
+        return search(input, URL_REGEX, null, displayTotal);
     }
 
 }

src/core/operations/FromDecimal.mjs

@@ -29,38 +29,43 @@ class FromDecimal extends Operation {
                 "name": "Delimiter",
                 "type": "option",
                 "value": DELIM_OPTIONS
+            },
+            {
+                "name": "Support signed values",
+                "type": "boolean",
+                "value": false
             }
         ];
         this.patterns = [
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?: (?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["Space"]
+                args: ["Space", false]
             },
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:,(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["Comma"]
+                args: ["Comma", false]
             },
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:;(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["Semi-colon"]
+                args: ["Semi-colon", false]
             },
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?::(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["Colon"]
+                args: ["Colon", false]
             },
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["Line feed"]
+                args: ["Line feed", false]
             },
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\r\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["CRLF"]
+                args: ["CRLF", false]
             },
         ];
     }
@@ -71,7 +76,11 @@ class FromDecimal extends Operation {
      * @returns {byteArray}
      */
     run(input, args) {
-        return fromDecimal(input, args[0]);
+        let data = fromDecimal(input, args[0]);
+        if (args[1]) { // Convert negatives
+            data = data.map(v => v < 0 ? 0xFF + v + 1 : v);
+        }
+        return data;
     }
 
 }

src/core/operations/HMAC.mjs

@@ -28,8 +28,9 @@ class HMAC extends Operation {
         this.args = [
             {
                 "name": "Key",
-                "type": "binaryString",
-                "value": ""
+                "type": "toggleString",
+                "value": "",
+                "toggleValues": ["Hex", "Decimal", "Base64", "UTF8", "Latin1"]
             },
             {
                 "name": "Hashing function",
@@ -66,18 +67,11 @@ class HMAC extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const key = args[0],
+        const key = Utils.convertToByteString(args[0].string || "", args[0].option),
             hashFunc = args[1].toLowerCase(),
             msg = Utils.arrayBufferToStr(input, false),
             hasher = CryptoApi.getHasher(hashFunc);
 
-        // Horrible shim to fix constructor bug. Reported in nf404/crypto-api#8
-        hasher.reset = () => {
-            hasher.state = {};
-            const tmp = new hasher.constructor();
-            hasher.state = tmp.state;
-        };
-
         const mac = CryptoApi.getHmac(CryptoApi.encoder.fromUtf(key), hasher);
         mac.update(msg);
         return CryptoApi.encoder.toHex(mac.finalize());

src/core/operations/RemoveDiacritics.mjs

@@ -0,0 +1,41 @@
+/**
+ * @author Klaxon [klaxon@veyr.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+
+/**
+ * Remove Diacritics operation
+ */
+class RemoveDiacritics extends Operation {
+
+    /**
+     * RemoveDiacritics constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Remove Diacritics";
+        this.module = "Default";
+        this.description = "Replaces accented characters with their latin character equivalent.";
+        this.infoURL = "https://wikipedia.org/wiki/Diacritic";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        // reference: https://stackoverflow.com/questions/990904/remove-accents-diacritics-in-a-string-in-javascript/37511463
+        return input.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
+    }
+
+}
+
+export default RemoveDiacritics;

src/core/operations/ToDecimal.mjs

@@ -30,6 +30,11 @@ class ToDecimal extends Operation {
                 "name": "Delimiter",
                 "type": "option",
                 "value": DELIM_OPTIONS
+            },
+            {
+                "name": "Support signed values",
+                "type": "boolean",
+                "value": false
             }
         ];
     }
@@ -40,7 +45,11 @@ class ToDecimal extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const delim = Utils.charRep(args[0]);
+        const delim = Utils.charRep(args[0]),
+            signed = args[1];
+        if (signed) {
+            input = input.map(v => v > 0x7F ? v - 0xFF - 1 : v);
+        }
         return input.join(delim);
     }
 

test/index.mjs

@@ -42,7 +42,8 @@ import "./tests/operations/Crypt";
 import "./tests/operations/DateTime";
 import "./tests/operations/ExtractEmailAddresses";
 import "./tests/operations/Fork";
-import "./tests/operations/FromGeohash.mjs";
+import "./tests/operations/FromDecimal";
+import "./tests/operations/FromGeohash";
 import "./tests/operations/Hash";
 import "./tests/operations/HaversineDistance";
 import "./tests/operations/Hexdump";
@@ -62,6 +63,7 @@ import "./tests/operations/ParseIPRange";
 import "./tests/operations/PowerSet";
 import "./tests/operations/Regex";
 import "./tests/operations/Register";
+import "./tests/operations/RemoveDiacritics";
 import "./tests/operations/Rotate";
 import "./tests/operations/SeqUtils";
 import "./tests/operations/SetDifference";

test/tests/operations/FromDecimal.mjs

@@ -0,0 +1,33 @@
+/**
+ * From Decimal tests
+ *
+ * @author qistoph
+ * @copyright Crown Copyright 2018
+ * @licence Apache-2.0
+ */
+import TestRegister from "../../TestRegister";
+
+TestRegister.addTests([
+    {
+        name: "From Decimal",
+        input: "83 97 109 112 108 101 32 84 101 120 116",
+        expectedOutput: "Sample Text",
+        recipeConfig: [
+            {
+                op: "From Decimal",
+                args: ["Space", false]
+            },
+        ],
+    },
+    {
+        name: "From Decimal with negatives",
+        input: "-130,-140,-152,-151,115,33,0,-1",
+        expectedOutput: "~this!\u0000\u00ff",
+        recipeConfig: [
+            {
+                op: "From Decimal",
+                args: ["Comma", true]
+            },
+        ],
+    },
+]);

test/tests/operations/Hash.mjs

@@ -411,7 +411,7 @@ TestRegister.addTests([
         recipeConfig: [
             {
                 "op": "HMAC",
-                "args": ["test", "SHA256"]
+                "args": [{"option": "Latin1", "string": "test"}, "SHA256"]
             }
         ]
     },

test/tests/operations/RemoveDiacritics.mjs

@@ -0,0 +1,23 @@
+
+/**
+ * Remove Diacritics tests.
+ *
+ * @author Klaxon [klaxon@veyr.com]
+ * @copyright Crown Copyright 2017
+ * @license Apache-2.0
+ */
+import TestRegister from "../../TestRegister";
+
+TestRegister.addTests([
+    {
+        name: "Remove Diacritics",
+        input: "\xe0, \xe8, \xec, \xf2, \xf9  \xc0, \xc8, \xcc, \xd2, \xd9\n\xe1, \xe9, \xed, \xf3, \xfa, \xfd \xc1, \xc9, \xcd, \xd3, \xda, \xdd\n\xe2, \xea, \xee, \xf4, \xfb \xc2, \xca, \xce, \xd4, \xdb\n\xe3, \xf1, \xf5 \xc3, \xd1, \xd5\n\xe4, \xeb, \xef, \xf6, \xfc, \xff \xc4, \xcb, \xcf, \xd6, \xdc, \u0178\n\xe5, \xc5",
+        expectedOutput: "a, e, i, o, u  A, E, I, O, U\na, e, i, o, u, y A, E, I, O, U, Y\na, e, i, o, u A, E, I, O, U\na, n, o A, N, O\na, e, i, o, u, y A, E, I, O, U, Y\na, A",
+        recipeConfig: [
+            {
+                "op": "Remove Diacritics",
+                "args": []
+            },
+        ],
+    },
+]);