9.22.4

.github/workflows/pull_requests.yml

@@ -0,0 +1,37 @@
+name: PRs
+
+on:
+  pull_request:
+    types: [synchronize, opened, reopened]
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set node version
+      uses: actions/setup-node@v1
+      with:
+        node-version: '10.x'
+
+    - name: Install
+      run: |
+        npm install
+        export NODE_OPTIONS=--max_old_space_size=2048
+
+    - name: Lint
+      run: npx grunt lint
+
+    - name: Unit Tests
+      run: |
+        npm test
+        npx grunt testnodeconsumer
+
+    - name: Production Build
+      if: success()
+      run: npx grunt prod
+
+    - name: UI Tests
+      if: success()
+      run: xvfb-run --server-args="-screen 0 1200x800x24" npx grunt testui

Gruntfile.js

@@ -78,7 +78,6 @@ module.exports = function (grunt) {
     grunt.loadNpmTasks("grunt-contrib-watch");
     grunt.loadNpmTasks("grunt-chmod");
     grunt.loadNpmTasks("grunt-exec");
-    grunt.loadNpmTasks("grunt-accessibility");
     grunt.loadNpmTasks("grunt-concurrent");
     grunt.loadNpmTasks("grunt-contrib-connect");
     grunt.loadNpmTasks("grunt-zip");
@@ -197,18 +196,6 @@ module.exports = function (grunt) {
             node: ["src/node/**/*.{js,mjs}"],
             tests: ["tests/**/*.{js,mjs}"],
         },
-        accessibility: {
-            options: {
-                accessibilityLevel: "WCAG2A",
-                verbose: false,
-                ignore: [
-                    "WCAG2A.Principle1.Guideline1_3.1_3_1.H42.2"
-                ]
-            },
-            test: {
-                src: ["build/**/*.html"]
-            }
-        },
         webpack: {
             options: webpackConfig,
             web: webpackProdConf(),

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "9.21.6",
+  "version": "9.22.4",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",
@@ -38,21 +38,20 @@
   "devDependencies": {
     "@babel/core": "^7.12.10",
     "@babel/plugin-transform-runtime": "^7.12.10",
-    "@babel/preset-env": "^7.12.10",
-    "autoprefixer": "^10.1.0",
+    "@babel/preset-env": "^7.12.11",
+    "autoprefixer": "^10.2.4",
     "babel-eslint": "^10.1.0",
     "babel-loader": "^8.2.2",
     "babel-plugin-dynamic-import-node": "^2.3.3",
-    "chromedriver": "^87.0.4",
-    "cli-progress": "^3.8.2",
+    "chromedriver": "^88.0.0",
+    "cli-progress": "^3.9.0",
     "colors": "^1.4.0",
     "copy-webpack-plugin": "^7.0.0",
     "css-loader": "^5.0.1",
-    "eslint": "^7.15.0",
-    "exports-loader": "^1.1.1",
+    "eslint": "^7.19.0",
+    "exports-loader": "^2.0.0",
     "file-loader": "^6.2.0",
     "grunt": "^1.3.0",
-    "grunt-accessibility": "~6.0.0",
     "grunt-chmod": "~1.1.1",
     "grunt-concurrent": "^3.0.0",
     "grunt-contrib-clean": "~2.0.0",
@@ -63,26 +62,26 @@
     "grunt-exec": "~3.0.0",
     "grunt-webpack": "^4.0.2",
     "grunt-zip": "^0.18.2",
-    "html-webpack-plugin": "^4.5.0",
-    "imports-loader": "^1.2.0",
-    "mini-css-extract-plugin": "^1.3.3",
+    "html-webpack-plugin": "^4.5.1",
+    "imports-loader": "^2.0.0",
+    "mini-css-extract-plugin": "^1.3.5",
     "nightwatch": "^1.5.1",
     "node-sass": "^5.0.0",
-    "postcss": "^8.2.1",
+    "postcss": "^8.2.4",
     "postcss-css-variables": "^0.17.0",
-    "postcss-import": "^13.0.0",
-    "postcss-loader": "^4.1.0",
-    "prompt": "^1.0.0",
-    "sass-loader": "^10.1.0",
-    "sitemap": "^6.3.3",
+    "postcss-import": "^14.0.0",
+    "postcss-loader": "^4.2.0",
+    "prompt": "^1.1.0",
+    "sass-loader": "^10.1.1",
+    "sitemap": "^6.3.5",
     "style-loader": "^2.0.0",
     "svg-url-loader": "^7.1.1",
     "url-loader": "^4.1.1",
-    "webpack": "^5.10.1",
-    "webpack-bundle-analyzer": "^4.2.0",
-    "webpack-dev-server": "^3.11.0",
+    "webpack": "^5.19.0",
+    "webpack-bundle-analyzer": "^4.4.0",
+    "webpack-dev-server": "^3.11.2",
     "webpack-node-externals": "^2.5.2",
-    "worker-loader": "^3.0.6"
+    "worker-loader": "^3.0.7"
   },
   "dependencies": {
     "@babel/polyfill": "^7.12.1",
@@ -93,7 +92,7 @@
     "bcryptjs": "^2.4.3",
     "bignumber.js": "^9.0.1",
     "blakejs": "^1.1.0",
-    "bootstrap": "4.5.3",
+    "bootstrap": "4.6.0",
     "bootstrap-colorpicker": "^3.2.0",
     "bootstrap-material-design": "^4.1.3",
     "browserify-zlib": "^0.2.0",
@@ -101,12 +100,12 @@
     "buffer": "^6.0.3",
     "chi-squared": "^1.1.0",
     "codepage": "^1.14.0",
-    "core-js": "^3.8.1",
+    "core-js": "^3.8.3",
     "crypto-api": "^0.8.5",
     "crypto-browserify": "^3.12.0",
     "crypto-js": "^4.0.0",
     "ctph.js": "0.0.5",
-    "d3": "^6.3.1",
+    "d3": "^6.5.0",
     "d3-hexbin": "^0.2.2",
     "diff": "^5.0.0",
     "es6-promisify": "^6.1.1",
@@ -115,24 +114,25 @@
     "esprima": "^4.0.1",
     "exif-parser": "^0.1.12",
     "file-saver": "^2.0.5",
+    "flat": "^5.0.2",
     "geodesy": "^1.1.3",
-    "highlight.js": "^10.4.1",
+    "highlight.js": "^10.5.0",
     "jimp": "^0.16.1",
     "jquery": "3.5.1",
     "js-crc": "^0.2.0",
     "js-sha3": "^0.8.0",
     "jsesc": "^3.0.2",
-    "jsonpath": "^1.0.2",
+    "jsonpath": "^1.1.0",
     "jsonwebtoken": "^8.5.1",
     "jsqr": "^1.3.1",
-    "jsrsasign": "10.1.4",
+    "jsrsasign": "10.1.5",
     "kbpgp": "2.1.15",
     "libbzip2-wasm": "0.0.4",
     "libyara-wasm": "^1.1.0",
     "lodash": "^4.17.20",
     "loglevel": "^1.7.1",
     "loglevel-message-prefix": "^3.0.0",
-    "markdown-it": "^12.0.3",
+    "markdown-it": "^12.0.4",
     "moment": "^2.29.1",
     "moment-timezone": "^0.5.32",
     "ngeohash": "^0.6.3",
@@ -148,7 +148,7 @@
     "qr-image": "^3.2.0",
     "scryptsy": "^2.1.0",
     "snackbarjs": "^1.1.0",
-    "sortablejs": "^1.12.0",
+    "sortablejs": "^1.13.0",
     "split.js": "^1.6.2",
     "ssdeep.js": "0.0.2",
     "stream-browserify": "^3.0.0",

src/core/config/Categories.json

@@ -200,6 +200,7 @@
         "ops": [
             "Encode text",
             "Decode text",
+            "Unicode Text Format",
             "Remove Diacritics",
             "Unescape Unicode Characters",
             "Convert to NATO alphabet"

src/core/operations/JSONToCSV.mjs

@@ -6,6 +6,8 @@
 
 import Operation from "../Operation.mjs";
 import OperationError from "../errors/OperationError.mjs";
+import * as flat from "flat";
+const flatten = flat.default ? flat.default.flatten : flat.flatten;
 
 /**
  * JSON to CSV operation
@@ -38,6 +40,40 @@ class JSONToCSV extends Operation {
         ];
     }
 
+    /**
+     * Converts JSON to a CSV equivalent.
+     *
+     * @returns {string}
+     */
+    toCSV() {
+        const self = this;
+        // If the JSON is an array of arrays, this is easy
+        if (this.flattened[0] instanceof Array) {
+            return this.flattened
+                .map(row => row
+                    .map(self.escapeCellContents.bind(self))
+                    .join(this.cellDelim)
+                )
+                .join(this.rowDelim) +
+                this.rowDelim;
+        }
+
+        // If it's an array of dictionaries...
+        const header = Object.keys(this.flattened[0]);
+        return header
+            .map(self.escapeCellContents.bind(self))
+            .join(this.cellDelim) +
+            this.rowDelim +
+            this.flattened
+                .map(row => header
+                    .map(h => row[h])
+                    .map(self.escapeCellContents.bind(self))
+                    .join(this.cellDelim)
+                )
+                .join(this.rowDelim) +
+                this.rowDelim;
+    }
+
     /**
      * @param {JSON} input
      * @param {Object[]} args
@@ -49,40 +85,23 @@ class JSONToCSV extends Operation {
         // Record values so they don't have to be passed to other functions explicitly
         this.cellDelim = cellDelim;
         this.rowDelim = rowDelim;
-        const self = this;
-
-        if (!(input instanceof Array)) {
-            input = [input];
+        this.flattened = input;
+        if (!(this.flattened instanceof Array)) {
+            this.flattened = [input];
         }
 
         try {
-            // If the JSON is an array of arrays, this is easy
-            if (input[0] instanceof Array) {
-                return input
-                    .map(row => row
-                        .map(self.escapeCellContents.bind(self))
-                        .join(cellDelim)
-                    )
-                    .join(rowDelim) +
-                    rowDelim;
-            }
-
-            // If it's an array of dictionaries...
-            const header = Object.keys(input[0]);
-            return header
-                .map(self.escapeCellContents.bind(self))
-                .join(cellDelim) +
-                rowDelim +
-                input
-                    .map(row => header
-                        .map(h => row[h])
-                        .map(self.escapeCellContents.bind(self))
-                        .join(cellDelim)
-                    )
-                    .join(rowDelim) +
-                rowDelim;
+            return this.toCSV();
         } catch (err) {
-            throw new OperationError("Unable to parse JSON to CSV: " + err.toString());
+            try {
+                this.flattened = flatten(input);
+                if (!(this.flattened instanceof Array)) {
+                    this.flattened = [this.flattened];
+                }
+                return this.toCSV();
+            } catch (err) {
+                throw new OperationError("Unable to parse JSON to CSV: " + err.toString());
+            }
         }
     }
 

src/core/operations/ParseDateTime.mjs

@@ -72,7 +72,7 @@ class ParseDateTime extends Operation {
             "\nLeap year: " + date.isLeapYear() +
             "\nDays in this month: " + date.daysInMonth() +
             "\n\nDay of year: " + date.dayOfYear() +
-            "\nWeek number: " + date.weekYear() +
+            "\nWeek number: " + date.week() +
             "\nQuarter: " + date.quarter();
 
         return output;

src/core/operations/RemoveDiacritics.mjs

@@ -19,7 +19,7 @@ class RemoveDiacritics extends Operation {
 
         this.name = "Remove Diacritics";
         this.module = "Default";
-        this.description = "Replaces accented characters with their latin character equivalent.";
+        this.description = "Replaces accented characters with their latin character equivalent. Accented characters are made up of Unicode combining characters, so unicode text formatting such as strikethroughs and underlines will also be removed.";
         this.infoURL = "https://wikipedia.org/wiki/Diacritic";
         this.inputType = "string";
         this.outputType = "string";

src/core/operations/UNIXTimestampToWindowsFiletime.mjs

@@ -34,7 +34,7 @@ class UNIXTimestampToWindowsFiletime extends Operation {
             {
                 "name": "Output format",
                 "type": "option",
-                "value": ["Decimal", "Hex"]
+                "value": ["Decimal", "Hex (big endian)", "Hex (little endian)"]
             }
         ];
     }
@@ -65,11 +65,24 @@ class UNIXTimestampToWindowsFiletime extends Operation {
 
         input = input.plus(new BigNumber("116444736000000000"));
 
-        if (format === "Hex") {
-            return input.toString(16);
+        let result;
+        if (format.startsWith("Hex")) {
+            result = input.toString(16);
         } else {
-            return input.toFixed();
+            result = input.toFixed();
         }
+
+        if (format === "Hex (little endian)") {
+            // Swap endianness
+            let flipped = "";
+            for (let i = result.length - 2; i >= 0; i -= 2) {
+                flipped += result.charAt(i);
+                flipped += result.charAt(i + 1);
+            }
+            result = flipped;
+        }
+
+        return result;
     }
 
 }

src/core/operations/UnicodeTextFormat.mjs

@@ -0,0 +1,67 @@
+/**
+ * @author Matt C [me@mitt.dev]
+ * @copyright Crown Copyright 2020
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation.mjs";
+import Utils from "../Utils.mjs";
+
+/**
+ * Unicode Text Format operation
+ */
+class UnicodeTextFormat extends Operation {
+
+    /**
+     * UnicodeTextFormat constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Unicode Text Format";
+        this.module = "Default";
+        this.description = "Adds Unicode combining characters to change formatting of plaintext.";
+        this.infoURL = "https://wikipedia.org/wiki/Combining_character";
+        this.inputType = "byteArray";
+        this.outputType = "byteArray";
+        this.args = [
+            {
+                name: "Underline",
+                type: "boolean",
+                value: "false"
+            },
+            {
+                name: "Strikethrough",
+                type: "boolean",
+                value: "false"
+            }
+        ];
+    }
+
+    /**
+     * @param {byteArray} input
+     * @param {Object[]} args
+     * @returns {byteArray}
+     */
+    run(input, args) {
+        const [underline, strikethrough] = args;
+        let output = input.map(char => [char]);
+        if (strikethrough) {
+            output = output.map(charFormat => {
+                charFormat.push(...Utils.strToUtf8ByteArray("\u0336"));
+                return charFormat;
+            });
+        }
+        if (underline) {
+            output = output.map(charFormat => {
+                charFormat.push(...Utils.strToUtf8ByteArray("\u0332"));
+                return charFormat;
+            });
+        }
+        // return output.flat(); - Not supported in Node 10, polyfilled
+        return [].concat(...output);
+    }
+
+}
+
+export default UnicodeTextFormat;

src/core/operations/WindowsFiletimeToUNIXTimestamp.mjs

@@ -34,7 +34,7 @@ class WindowsFiletimeToUNIXTimestamp extends Operation {
             {
                 "name": "Input format",
                 "type": "option",
-                "value": ["Decimal", "Hex"]
+                "value": ["Decimal", "Hex (big endian)", "Hex (little endian)"]
             }
         ];
     }
@@ -49,7 +49,17 @@ class WindowsFiletimeToUNIXTimestamp extends Operation {
 
         if (!input) return "";
 
-        if (format === "Hex") {
+        if (format === "Hex (little endian)") {
+            // Swap endianness
+            let result = "";
+            for (let i = input.length - 2; i >= 0; i -= 2) {
+                result += input.charAt(i);
+                result += input.charAt(i + 1);
+            }
+            input = result;
+        }
+
+        if (format.startsWith("Hex")) {
             input = new BigNumber(input, 16);
         } else {
             input = new BigNumber(input);

tests/node/tests/operations.mjs

@@ -588,7 +588,7 @@ Password: 034148`;
         const result = await chef.generatePGPKeyPair("Back To the Drawing Board", {
             keyType: "ECC-256",
         });
-        assert.strictEqual(result.toString().length, 2560);
+        assert.strictEqual(result.toString().substr(0, 37), "-----BEGIN PGP PRIVATE KEY BLOCK-----");
     }),
 
     it("Generate UUID", () => {
@@ -656,7 +656,7 @@ Leap year: false
 Days in this month: 31
 
 Day of year: 187
-Week number: 2001
+Week number: 27
 Quarter: 3`;
         assert.strictEqual(result.toString(), expected);
     }),

tests/operations/index.mjs

@@ -69,7 +69,6 @@ import "./tests/ParseQRCode.mjs";
 import "./tests/PowerSet.mjs";
 import "./tests/Regex.mjs";
 import "./tests/Register.mjs";
-import "./tests/RemoveDiacritics.mjs";
 import "./tests/Rotate.mjs";
 import "./tests/SeqUtils.mjs";
 import "./tests/SetDifference.mjs";
@@ -101,6 +100,7 @@ import "./tests/LuhnChecksum.mjs";
 import "./tests/CipherSaber2.mjs";
 import "./tests/Colossus.mjs";
 import "./tests/ParseObjectIDTimestamp.mjs";
+import "./tests/Unicode.mjs";
 
 
 // Cannot test operations that use the File type yet

tests/operations/tests/JSONtoCSV.mjs

@@ -89,5 +89,71 @@ TestRegister.addTests([
                 args: [",", "\\r\\n"]
             },
         ],
+    },
+    {
+        name: "JSON to CSV: nested JSON",
+        input: JSON.stringify({a: 1, b: {c: 2, d: 3}}),
+        expectedOutput: "a,b.c,b.d\r\n1,2,3\r\n",
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    },
+    {
+        name: "JSON to CSV: nested array",
+        input: JSON.stringify({a: 1, b: [2, 3]}),
+        expectedOutput: "a,b.0,b.1\r\n1,2,3\r\n",
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    },
+    {
+        name: "JSON to CSV: nested JSON, nested array",
+        input: JSON.stringify({a: 1, b: {c: [2, 3], d: 4}}),
+        expectedOutput: "a,b.c.0,b.c.1,b.d\r\n1,2,3,4\r\n",
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    },
+    {
+        name: "JSON to CSV: nested array, nested JSON",
+        input: JSON.stringify({a: 1, b: [{c: 3, d: 4}]}),
+        expectedOutput: "a,b.0.c,b.0.d\r\n1,3,4\r\n",
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    },
+    {
+        name: "JSON to CSV: nested array, nested array",
+        input: JSON.stringify({a: 1, b: [[2, 3]]}),
+        expectedOutput: "a,b.0.0,b.0.1\r\n1,2,3\r\n",
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    },
+    {
+        name: "JSON to CSV: nested JSON, nested JSON",
+        input: JSON.stringify({a: 1, b: { c: { d: 2, e: 3}}}),
+        expectedOutput: "a,b.c.d,b.c.e\r\n1,2,3\r\n",
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
     }
 ]);

tests/operations/tests/RemoveDiacritics.mjs

@@ -1,22 +0,0 @@
-/**
- * Remove Diacritics tests.
- *
- * @author Klaxon [klaxon@veyr.com]
- * @copyright Crown Copyright 2017
- * @license Apache-2.0
- */
-import TestRegister from "../../lib/TestRegister.mjs";
-
-TestRegister.addTests([
-    {
-        name: "Remove Diacritics",
-        input: "\xe0, \xe8, \xec, \xf2, \xf9  \xc0, \xc8, \xcc, \xd2, \xd9\n\xe1, \xe9, \xed, \xf3, \xfa, \xfd \xc1, \xc9, \xcd, \xd3, \xda, \xdd\n\xe2, \xea, \xee, \xf4, \xfb \xc2, \xca, \xce, \xd4, \xdb\n\xe3, \xf1, \xf5 \xc3, \xd1, \xd5\n\xe4, \xeb, \xef, \xf6, \xfc, \xff \xc4, \xcb, \xcf, \xd6, \xdc, \u0178\n\xe5, \xc5",
-        expectedOutput: "a, e, i, o, u  A, E, I, O, U\na, e, i, o, u, y A, E, I, O, U, Y\na, e, i, o, u A, E, I, O, U\na, n, o A, N, O\na, e, i, o, u, y A, E, I, O, U, Y\na, A",
-        recipeConfig: [
-            {
-                "op": "Remove Diacritics",
-                "args": []
-            },
-        ],
-    },
-]);

tests/operations/tests/Unicode.mjs

@@ -0,0 +1,83 @@
+/**
+ * Unicode operation tests.
+ *
+ * @author Matt C [me@mitt.dev]
+ * @author Klaxon [klaxon@veyr.com]
+ *
+ * @copyright Crown Copyright 2020
+ * @license Apache-2.0
+ */
+import TestRegister from "../../lib/TestRegister.mjs";
+
+TestRegister.addTests([
+    {
+        name: "Unicode Text Format: underline",
+        input: "a",
+        expectedOutput: "a\u0332",
+        recipeConfig: [
+            {
+                "op": "Unicode Text Format",
+                "args": [true, false],
+            }
+        ],
+    },
+    {
+        name: "Unicode Text Format: strikethrough",
+        input: "a",
+        expectedOutput: "a\u0336",
+        recipeConfig: [
+            {
+                "op": "Unicode Text Format",
+                "args": [false, true],
+            }
+        ],
+    },
+    {
+        name: "Unicode Text Format: both",
+        input: "a",
+        expectedOutput: "a\u0336\u0332",
+        recipeConfig: [
+            {
+                "op": "Unicode Text Format",
+                "args": [true, true],
+            }
+        ],
+    },
+    {
+        name: "Remove Diacritics: text formatting",
+        input: "a",
+        expectedOutput: "a",
+        recipeConfig: [
+            {
+                "op": "Unicode Text Format",
+                "args": [true, true],
+            },
+            {
+                "op": "Remove Diacritics",
+                "args": []
+            }
+        ],
+    },
+    {
+        name: "Remove Diacritics: all diacritical marks one char",
+        input: "à̴̵̶̷̸̡̢̧̨̛̖̗̘̙̜̝̞̟̠̣̤̥̦̩̪̫̬̭̮̯̰̱̲̳̹̺̻̼́̂̃̄̅̆̇̈̉̊̋̌̍̎̏̐̑̒̓̔̽̾̿̀́͂̓̈́̕̚͠͡ͅ", // sorry about this line lol
+        expectedOutput: "a",
+        recipeConfig: [
+            {
+                "op": "Remove Diacritics",
+                "args": []
+            }
+        ],
+    },
+    {
+        name: "Remove Diacritics: default",
+        input: "\xe0, \xe8, \xec, \xf2, \xf9  \xc0, \xc8, \xcc, \xd2, \xd9\n\xe1, \xe9, \xed, \xf3, \xfa, \xfd \xc1, \xc9, \xcd, \xd3, \xda, \xdd\n\xe2, \xea, \xee, \xf4, \xfb \xc2, \xca, \xce, \xd4, \xdb\n\xe3, \xf1, \xf5 \xc3, \xd1, \xd5\n\xe4, \xeb, \xef, \xf6, \xfc, \xff \xc4, \xcb, \xcf, \xd6, \xdc, \u0178\n\xe5, \xc5",
+        expectedOutput: "a, e, i, o, u  A, E, I, O, U\na, e, i, o, u, y A, E, I, O, U, Y\na, e, i, o, u A, E, I, O, U\na, n, o A, N, O\na, e, i, o, u, y A, E, I, O, U, Y\na, A",
+        recipeConfig: [
+            {
+                "op": "Remove Diacritics",
+                "args": []
+            },
+        ],
+    },
+]);