9.38.5

This adds support for UTF8-encoded characters in the input and the parameters.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "9.38.1",
+  "version": "9.38.5",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",
@@ -39,26 +39,27 @@
     "node >= 16"
   ],
   "devDependencies": {
-    "@babel/core": "^7.17.8",
-    "@babel/eslint-parser": "^7.17.0",
-    "@babel/plugin-syntax-import-assertions": "^7.16.7",
-    "@babel/plugin-transform-runtime": "^7.17.0",
-    "@babel/preset-env": "^7.16.11",
-    "@babel/runtime": "^7.17.8",
-    "autoprefixer": "^10.4.4",
-    "babel-loader": "^8.2.4",
+    "@babel/core": "^7.18.2",
+    "@babel/eslint-parser": "^7.18.2",
+    "@babel/plugin-syntax-import-assertions": "^7.17.12",
+    "@babel/plugin-transform-runtime": "^7.18.2",
+    "@babel/preset-env": "^7.18.2",
+    "@babel/runtime": "^7.18.3",
+    "autoprefixer": "^10.4.7",
+    "babel-loader": "^8.2.5",
     "babel-plugin-dynamic-import-node": "^2.3.3",
+    "babel-plugin-transform-builtin-extend": "1.1.2",
     "chromedriver": "^101.0.0",
-    "cli-progress": "^3.10.0",
+    "cli-progress": "^3.11.1",
     "colors": "^1.4.0",
-    "copy-webpack-plugin": "^10.2.4",
-    "core-js": "^3.21.1",
+    "copy-webpack-plugin": "^11.0.0",
+    "core-js": "^3.22.8",
     "css-loader": "6.7.1",
-    "eslint": "^8.12.0",
-    "grunt": "^1.4.1",
+    "eslint": "^8.16.0",
+    "grunt": "^1.5.3",
     "grunt-chmod": "~1.1.1",
     "grunt-concurrent": "^3.0.0",
-    "grunt-contrib-clean": "~2.0.0",
+    "grunt-contrib-clean": "~2.0.1",
     "grunt-contrib-connect": "^3.0.0",
     "grunt-contrib-copy": "~1.0.0",
     "grunt-contrib-watch": "^1.1.0",
@@ -67,29 +68,28 @@
     "grunt-webpack": "^5.0.0",
     "grunt-zip": "^0.18.2",
     "html-webpack-plugin": "^5.5.0",
-    "imports-loader": "^3.1.1",
+    "imports-loader": "^4.0.0",
     "mini-css-extract-plugin": "2.6.0",
     "modify-source-webpack-plugin": "^3.0.0",
-    "nightwatch": "^2.0.10",
-    "postcss": "^8.4.12",
+    "nightwatch": "^2.1.7",
+    "postcss": "^8.4.14",
     "postcss-css-variables": "^0.18.0",
     "postcss-import": "^14.1.0",
-    "postcss-loader": "^6.2.1",
-    "prompt": "^1.2.2",
-    "sass-loader": "^12.6.0",
+    "postcss-loader": "^7.0.0",
+    "prompt": "^1.3.0",
+    "sass-loader": "^13.0.0",
     "sitemap": "^7.1.1",
-    "terser": "^5.12.1",
-    "webpack": "^5.70.0",
+    "terser": "^5.14.0",
+    "webpack": "^5.73.0",
     "webpack-bundle-analyzer": "^4.5.0",
-    "webpack-dev-server": "4.7.4",
+    "webpack-dev-server": "4.9.1",
     "webpack-node-externals": "^3.0.0",
     "worker-loader": "^3.0.8"
   },
   "dependencies": {
     "@babel/polyfill": "^7.12.1",
     "arrive": "^2.4.1",
-    "avsc": "^5.7.3",
-    "babel-plugin-transform-builtin-extend": "1.1.2",
+    "avsc": "^5.7.4",
     "bcryptjs": "^2.4.3",
     "bignumber.js": "^9.0.2",
     "blakejs": "^1.2.1",
@@ -97,7 +97,7 @@
     "bootstrap-colorpicker": "^3.4.0",
     "bootstrap-material-design": "^4.1.3",
     "browserify-zlib": "^0.2.0",
-    "bson": "^4.6.2",
+    "bson": "^4.6.4",
     "buffer": "^6.0.3",
     "cbor": "8.1.0",
     "chi-squared": "^1.1.0",
@@ -106,9 +106,9 @@
     "crypto-browserify": "^3.12.0",
     "crypto-js": "^4.1.1",
     "ctph.js": "0.0.5",
-    "d3": "7.3.0",
+    "d3": "7.4.4",
     "d3-hexbin": "^0.2.2",
-    "diff": "^5.0.0",
+    "diff": "^5.1.0",
     "es6-promisify": "^7.0.0",
     "escodegen": "^2.0.0",
     "esprima": "^4.0.1",
@@ -116,7 +116,7 @@
     "file-saver": "^2.0.5",
     "flat": "^5.0.2",
     "geodesy": "1.1.3",
-    "highlight.js": "^11.5.0",
+    "highlight.js": "^11.5.1",
     "jimp": "^0.16.1",
     "jquery": "3.6.0",
     "js-crc": "^0.2.0",
@@ -125,28 +125,28 @@
     "jsonpath": "^1.1.1",
     "jsonwebtoken": "^8.5.1",
     "jsqr": "^1.4.0",
-    "jsrsasign": "^10.5.14",
+    "jsrsasign": "^10.5.23",
     "kbpgp": "2.1.15",
     "libbzip2-wasm": "0.0.4",
     "libyara-wasm": "^1.1.0",
     "lodash": "^4.17.21",
     "loglevel": "^1.8.0",
     "loglevel-message-prefix": "^3.0.0",
-    "markdown-it": "^12.3.2",
-    "moment": "^2.29.1",
+    "markdown-it": "^13.0.1",
+    "moment": "^2.29.3",
     "moment-timezone": "^0.5.34",
     "ngeohash": "^0.6.3",
-    "node-forge": "^1.3.0",
+    "node-forge": "^1.3.1",
     "node-md6": "^0.1.0",
     "node-sass": "^7.0.1",
     "nodom": "^2.4.0",
-    "notepack.io": "^2.3.0",
+    "notepack.io": "^3.0.1",
     "nwmatcher": "^1.4.4",
     "otp": "0.1.3",
     "path": "^0.12.7",
     "popper.js": "^1.16.1",
     "process": "^0.11.10",
-    "protobufjs": "^6.11.2",
+    "protobufjs": "^6.11.3",
     "qr-image": "^3.2.0",
     "scryptsy": "^2.1.0",
     "snackbarjs": "^1.1.0",

src/core/lib/Base64.mjs

@@ -82,15 +82,46 @@ export function toBase64(data, alphabet="A-Za-z0-9+/=") {
  * // returns [72, 101, 108, 108, 111]
  * fromBase64("SGVsbG8=", null, "byteArray");
  */
-export function fromBase64(data, alphabet="A-Za-z0-9+/=", returnType="string", removeNonAlphChars=true) {
+export function fromBase64(data, alphabet="A-Za-z0-9+/=", returnType="string", removeNonAlphChars=true, strictMode=false) {
     if (!data) {
         return returnType === "string" ? "" : [];
     }
 
     alphabet = alphabet || "A-Za-z0-9+/=";
     alphabet = Utils.expandAlphRange(alphabet).join("");
+
+    // Confirm alphabet is a valid length
     if (alphabet.length !== 64 && alphabet.length !== 65) { // Allow for padding
-        throw new OperationError(`Invalid Base64 alphabet length (${alphabet.length}): ${alphabet}`);
+        throw new OperationError(`Error: Base64 alphabet should be 64 characters long, or 65 with a padding character. Found ${alphabet.length}: ${alphabet}`);
+    }
+
+    // Remove non-alphabet characters
+    if (removeNonAlphChars) {
+        const re = new RegExp("[^" + alphabet.replace(/[[\]\\\-^$]/g, "\\$&") + "]", "g");
+        data = data.replace(re, "");
+    }
+
+    if (strictMode) {
+        // Check for incorrect lengths (even without padding)
+        if (data.length % 4 === 1) {
+            throw new OperationError(`Error: Invalid Base64 input length (${data.length}). Cannot be 4n+1, even without padding chars.`);
+        }
+
+        if (alphabet.length === 65) { // Padding character included
+            const pad = alphabet.charAt(64);
+            const padPos = data.indexOf(pad);
+            if (padPos >= 0) {
+                // Check that the padding character is only used at the end and maximum of twice
+                if (padPos < data.length - 2 || data.charAt(data.length - 1) !== pad) {
+                    throw new OperationError(`Error: Base64 padding character (${pad}) not used in the correct place.`);
+                }
+
+                // Check that input is padded to the correct length
+                if (data.length % 4 !== 0) {
+                    throw new OperationError("Error: Base64 not padded to a multiple of 4.");
+                }
+            }
+        }
     }
 
     const output = [];
@@ -98,31 +129,27 @@ export function fromBase64(data, alphabet="A-Za-z0-9+/=", returnType="string", r
         enc1, enc2, enc3, enc4,
         i = 0;
 
-    if (removeNonAlphChars) {
-        const re = new RegExp("[^" + alphabet.replace(/[[\]\\\-^$]/g, "\\$&") + "]", "g");
-        data = data.replace(re, "");
-    }
-
     while (i < data.length) {
         enc1 = alphabet.indexOf(data.charAt(i++));
-        enc2 = alphabet.indexOf(data.charAt(i++) || "=");
-        enc3 = alphabet.indexOf(data.charAt(i++) || "=");
-        enc4 = alphabet.indexOf(data.charAt(i++) || "=");
+        enc2 = alphabet.indexOf(data.charAt(i++));
+        enc3 = alphabet.indexOf(data.charAt(i++));
+        enc4 = alphabet.indexOf(data.charAt(i++));
 
-        enc2 = enc2 === -1 ? 64 : enc2;
-        enc3 = enc3 === -1 ? 64 : enc3;
-        enc4 = enc4 === -1 ? 64 : enc4;
+        if (strictMode && (enc1 < 0 || enc2 < 0 || enc3 < 0 || enc4 < 0)) {
+            throw new OperationError("Error: Base64 input contains non-alphabet char(s)");
+        }
 
         chr1 = (enc1 << 2) | (enc2 >> 4);
         chr2 = ((enc2 & 15) << 4) | (enc3 >> 2);
         chr3 = ((enc3 & 3) << 6) | enc4;
 
-        output.push(chr1);
-
-        if (enc3 !== 64) {
+        if (chr1 < 256) {
+            output.push(chr1);
+        }
+        if (chr2 < 256 && enc3 !== 64) {
             output.push(chr2);
         }
-        if (enc4 !== 64) {
+        if (chr3 < 256 && enc4 !== 64) {
             output.push(chr3);
         }
     }

src/core/lib/Binary.mjs

@@ -19,31 +19,33 @@ import OperationError from "../errors/OperationError.mjs";
  * @returns {string}
  *
  * @example
- * // returns "00010000 00100000 00110000"
+ * // returns "00001010 00010100 00011110"
  * toBinary([10,20,30]);
  *
- * // returns "00010000 00100000 00110000"
- * toBinary([10,20,30], ":");
+ * // returns "00001010:00010100:00011110"
+ * toBinary([10,20,30], "Colon");
+ *
+ * // returns "1010:10100:11110"
+ * toBinary([10,20,30], "Colon", 0);
  */
 export function toBinary(data, delim="Space", padding=8) {
+    if (data === undefined || data === null)
+        throw new OperationError("Unable to convert to binary: Empty input data enocuntered");
+
     delim = Utils.charRep(delim);
     let output = "";
 
     if (data.length) { // array
         for (let i = 0; i < data.length; i++) {
-            output += data[i].toString(2).padStart(padding, "0") + delim;
+            output += data[i].toString(2).padStart(padding, "0");
+            if (i !== data.length - 1) output += delim;
         }
     } else if (typeof data === "number") { // Single value
         return data.toString(2).padStart(padding, "0");
     } else {
         return "";
     }
-
-    if (delim.length) {
-        return output.slice(0, -delim.length);
-    } else {
-        return output;
-    }
+    return output;
 }
 
 
@@ -57,10 +59,10 @@ export function toBinary(data, delim="Space", padding=8) {
  *
  * @example
  * // returns [10,20,30]
- * fromBinary("00010000 00100000 00110000");
+ * fromBinary("00001010 00010100 00011110");
  *
  * // returns [10,20,30]
- * fromBinary("00010000:00100000:00110000", "Colon");
+ * fromBinary("00001010:00010100:00011110", "Colon");
  */
 export function fromBinary(data, delim="Space", byteLen=8) {
     if (byteLen < 1 || Math.round(byteLen) !== byteLen)

src/core/operations/FromBase64.mjs

@@ -34,93 +34,98 @@ class FromBase64 extends Operation {
                 name: "Remove non-alphabet chars",
                 type: "boolean",
                 value: true
+            },
+            {
+                name: "Strict mode",
+                type: "boolean",
+                value: false
             }
         ];
         this.checks = [
             {
                 pattern: "^\\s*(?:[A-Z\\d+/]{4})+(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
                 flags: "i",
-                args: ["A-Za-z0-9+/=", true]
+                args: ["A-Za-z0-9+/=", true, false]
             },
             {
                 pattern: "^\\s*[A-Z\\d\\-_]{20,}\\s*$",
                 flags: "i",
-                args: ["A-Za-z0-9-_", true]
+                args: ["A-Za-z0-9-_", true, false]
             },
             {
                 pattern: "^\\s*(?:[A-Z\\d+\\-]{4}){5,}(?:[A-Z\\d+\\-]{2}==|[A-Z\\d+\\-]{3}=)?\\s*$",
                 flags: "i",
-                args: ["A-Za-z0-9+\\-=", true]
+                args: ["A-Za-z0-9+\\-=", true, false]
             },
             {
                 pattern: "^\\s*(?:[A-Z\\d./]{4}){5,}(?:[A-Z\\d./]{2}==|[A-Z\\d./]{3}=)?\\s*$",
                 flags: "i",
-                args: ["./0-9A-Za-z=", true]
+                args: ["./0-9A-Za-z=", true, false]
             },
             {
                 pattern: "^\\s*[A-Z\\d_.]{20,}\\s*$",
                 flags: "i",
-                args: ["A-Za-z0-9_.", true]
+                args: ["A-Za-z0-9_.", true, false]
             },
             {
                 pattern: "^\\s*(?:[A-Z\\d._]{4}){5,}(?:[A-Z\\d._]{2}--|[A-Z\\d._]{3}-)?\\s*$",
                 flags: "i",
-                args: ["A-Za-z0-9._-", true]
+                args: ["A-Za-z0-9._-", true, false]
             },
             {
                 pattern: "^\\s*(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
                 flags: "i",
-                args: ["0-9a-zA-Z+/=", true]
+                args: ["0-9a-zA-Z+/=", true, false]
             },
             {
                 pattern: "^\\s*(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
                 flags: "i",
-                args: ["0-9A-Za-z+/=", true]
+                args: ["0-9A-Za-z+/=", true, false]
             },
             {
                 pattern: "^[ !\"#$%&'()*+,\\-./\\d:;<=>?@A-Z[\\\\\\]^_]{20,}$",
                 flags: "",
-                args: [" -_", false]
+                args: [" -_", false, false]
             },
             {
                 pattern: "^\\s*[A-Z\\d+\\-]{20,}\\s*$",
                 flags: "i",
-                args: ["+\\-0-9A-Za-z", true]
+                args: ["+\\-0-9A-Za-z", true, false]
             },
             {
                 pattern: "^\\s*[!\"#$%&'()*+,\\-0-689@A-NP-VX-Z[`a-fh-mp-r]{20,}\\s*$",
                 flags: "",
-                args: ["!-,-0-689@A-NP-VX-Z[`a-fh-mp-r", true]
+                args: ["!-,-0-689@A-NP-VX-Z[`a-fh-mp-r", true, false]
             },
             {
                 pattern: "^\\s*(?:[N-ZA-M\\d+/]{4}){5,}(?:[N-ZA-M\\d+/]{2}==|[N-ZA-M\\d+/]{3}=)?\\s*$",
                 flags: "i",
-                args: ["N-ZA-Mn-za-m0-9+/=", true]
+                args: ["N-ZA-Mn-za-m0-9+/=", true, false]
             },
             {
                 pattern: "^\\s*[A-Z\\d./]{20,}\\s*$",
                 flags: "i",
-                args: ["./0-9A-Za-z", true]
+                args: ["./0-9A-Za-z", true, false]
             },
             {
                 pattern: "^\\s*(?:[A-Z=\\d\\+/]{4}){5,}(?:[A-Z=\\d\\+/]{2}CC|[A-Z=\\d\\+/]{3}C)?\\s*$",
                 flags: "i",
-                args: ["/128GhIoPQROSTeUbADfgHijKLM+n0pFWXY456xyzB7=39VaqrstJklmNuZvwcdEC", true]
+                args: ["/128GhIoPQROSTeUbADfgHijKLM+n0pFWXY456xyzB7=39VaqrstJklmNuZvwcdEC", true, false]
             },
             {
                 pattern: "^\\s*(?:[A-Z=\\d\\+/]{4}){5,}(?:[A-Z=\\d\\+/]{2}55|[A-Z=\\d\\+/]{3}5)?\\s*$",
                 flags: "i",
-                args: ["3GHIJKLMNOPQRSTUb=cdefghijklmnopWXYZ/12+406789VaqrstuvwxyzABCDEF5", true]
+                args: ["3GHIJKLMNOPQRSTUb=cdefghijklmnopWXYZ/12+406789VaqrstuvwxyzABCDEF5", true, false]
             },
             {
                 pattern: "^\\s*(?:[A-Z=\\d\\+/]{4}){5,}(?:[A-Z=\\d\\+/]{2}22|[A-Z=\\d\\+/]{3}2)?\\s*$",
                 flags: "i",
-                args: ["ZKj9n+yf0wDVX1s/5YbdxSo=ILaUpPBCHg8uvNO4klm6iJGhQ7eFrWczAMEq3RTt2", true]
+                args: ["ZKj9n+yf0wDVX1s/5YbdxSo=ILaUpPBCHg8uvNO4klm6iJGhQ7eFrWczAMEq3RTt2", true, false]
             },
             {
                 pattern: "^\\s*(?:[A-Z=\\d\\+/]{4}){5,}(?:[A-Z=\\d\\+/]{2}55|[A-Z=\\d\\+/]{3}5)?\\s*$",
                 flags: "i",
-                args: ["HNO4klm6ij9n+J2hyf0gzA8uvwDEq3X1Q7ZKeFrWcVTts/MRGYbdxSo=ILaUpPBC5", true]
+                args: ["HNO4klm6ij9n+J2hyf0gzA8uvwDEq3X1Q7ZKeFrWcVTts/MRGYbdxSo=ILaUpPBC5", true, false]
             }
         ];
     }
@@ -131,9 +136,9 @@ class FromBase64 extends Operation {
      * @returns {byteArray}
      */
     run(input, args) {
-        const [alphabet, removeNonAlphChars] = args;
+        const [alphabet, removeNonAlphChars, strictMode] = args;
 
-        return fromBase64(input, alphabet, "byteArray", removeNonAlphChars);
+        return fromBase64(input, alphabet, "byteArray", removeNonAlphChars, strictMode);
     }
 
     /**

src/core/operations/Substitute.mjs

@@ -44,8 +44,8 @@ class Substitute extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const plaintext = Utils.expandAlphRange(args[0]).join(""),
-            ciphertext = Utils.expandAlphRange(args[1]).join("");
+        const plaintext = Utils.expandAlphRange([...args[0]]),
+            ciphertext = Utils.expandAlphRange([...args[1]]);
         let output = "",
             index = -1;
 
@@ -53,9 +53,9 @@ class Substitute extends Operation {
             output = "Warning: Plaintext and Ciphertext lengths differ\n\n";
         }
 
-        for (let i = 0; i < input.length; i++) {
-            index = plaintext.indexOf(input[i]);
-            output += index > -1 && index < ciphertext.length ? ciphertext[index] : input[i];
+        for (const character of input) {
+            index = plaintext.indexOf(character);
+            output += index > -1 && index < ciphertext.length ? ciphertext[index] : character;
         }
 
         return output;

src/core/operations/ToUpperCase.mjs

@@ -5,6 +5,7 @@
  */
 
 import Operation from "../Operation.mjs";
+import OperationError from "../errors/OperationError.mjs";
 
 /**
  * To Upper case operation
@@ -37,25 +38,30 @@ class ToUpperCase extends Operation {
      * @returns {string}
      */
     run(input, args) {
+        if (!args || args.length === 0) {
+            throw new OperationError("No capitalization scope was provided.");
+        }
+
         const scope = args[0];
 
-        switch (scope) {
-            case "Word":
-                return input.replace(/(\b\w)/gi, function(m) {
-                    return m.toUpperCase();
-                });
-            case "Sentence":
-                return input.replace(/(?:\.|^)\s*(\b\w)/gi, function(m) {
-                    return m.toUpperCase();
-                });
-            case "Paragraph":
-                return input.replace(/(?:\n|^)\s*(\b\w)/gi, function(m) {
-                    return m.toUpperCase();
-                });
-            case "All": /* falls through */
-            default:
-                return input.toUpperCase();
+        if (scope === "All") {
+            return input.toUpperCase();
         }
+
+        const scopeRegex = {
+            "Word": /(\b\w)/gi,
+            "Sentence": /(?:\.|^)\s*(\b\w)/gi,
+            "Paragraph": /(?:\n|^)\s*(\b\w)/gi
+        }[scope];
+
+        if (scopeRegex === undefined) {
+            throw new OperationError("Unrecognized capitalization scope");
+        }
+
+        // Use the regex to capitalize the input
+        return input.replace(scopeRegex, function(m) {
+            return m.toUpperCase();
+        });
     }
 
     /**

tests/operations/tests/Magic.mjs

@@ -68,7 +68,7 @@ TestRegister.addTests([
     {
         name: "Magic Chain: Base64",
         input: "WkVkV2VtUkRRbnBrU0Vwd1ltMWpQUT09",
-        expectedMatch: /From_Base64\('A-Za-z0-9\+\/=',true\)\nFrom_Base64\('A-Za-z0-9\+\/=',true\)\nFrom_Base64\('A-Za-z0-9\+\/=',true\)/,
+        expectedMatch: /From_Base64\('A-Za-z0-9\+\/=',true,false\)\nFrom_Base64\('A-Za-z0-9\+\/=',true,false\)\nFrom_Base64\('A-Za-z0-9\+\/=',true,false\)/,
         recipeConfig: [
             {
                 op: "Magic",
@@ -79,7 +79,7 @@ TestRegister.addTests([
     {
         name: "Magic Chain: Hex -> Hexdump -> Base64",
         input: "MDAwMDAwMDAgIDM3IDM0IDIwIDM2IDM1IDIwIDM3IDMzIDIwIDM3IDM0IDIwIDMyIDMwIDIwIDM3ICB8NzQgNjUgNzMgNzQgMjAgN3wKMDAwMDAwMTAgIDMzIDIwIDM3IDM0IDIwIDM3IDMyIDIwIDM2IDM5IDIwIDM2IDY1IDIwIDM2IDM3ICB8MyA3NCA3MiA2OSA2ZSA2N3w=",
-        expectedMatch: /From_Base64\('A-Za-z0-9\+\/=',true\)\nFrom_Hexdump\(\)\nFrom_Hex\('Space'\)/,
+        expectedMatch: /From_Base64\('A-Za-z0-9\+\/=',true,false\)\nFrom_Hexdump\(\)\nFrom_Hex\('Space'\)/,
         recipeConfig: [
             {
                 op: "Magic",