More test tweaking

Add Fernet encryption/decryption operation

.devcontainer/devcontainer.json

@@ -0,0 +1,41 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/javascript-node
+{
+	"name": "CyberChef",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/javascript-node:1-18-bookworm",
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	"features": {
+		"ghcr.io/devcontainers/features/github-cli": "latest"
+	},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [8080],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": {
+		"npm": "bash -c \"sudo chown node node_modules && npm install\""
+	},
+
+	"containerEnv": {
+		"DISPLAY": ":99"
+	},
+
+	"mounts": [
+		"source=${localWorkspaceFolderBasename}-node_modules,target=${containerWorkspaceFolder}/node_modules,type=volume"
+	],
+
+	// Configure tool-specific properties.
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"dbaeumer.vscode-eslint",
+				"GitHub.vscode-github-actions"
+			]
+		}
+	}
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.dockerignore

@@ -0,0 +1,2 @@
+node_modules
+build

.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

.github/workflows/pull_requests.yml

@@ -33,6 +33,20 @@ jobs:
       if: success()
       run: npx grunt prod
 
+    - name: Production Image Build
+      if: success()
+      id: build-image
+      uses: redhat-actions/buildah-build@v2
+      with:
+        # Not being uploaded to any registry, use a simple name to allow Buildah to build correctly.
+        image: cyberchef
+        containerfiles: ./Dockerfile
+        platforms: linux/amd64
+        oci: true
+        # Webpack seems to use a lot of open files, increase the max open file limit to accomodate.
+        extra-args: |
+          --ulimit nofile=10000
+
     - name: UI Tests
       if: success()
       run: |

.github/workflows/releases.yml

@@ -6,6 +6,12 @@ on:
     tags:
     - 'v*'
 
+env:
+  REGISTRY: ghcr.io
+  REGISTRY_USER: ${{ github.actor }}
+  REGISTRY_PASSWORD: ${{ github.token }}
+  IMAGE_NAME: ${{ github.repository }}
+
 jobs:
   main:
     runs-on: ubuntu-latest
@@ -19,7 +25,7 @@ jobs:
 
     - name: Install
       run: |
-        npm install
+        npm ci
         npm run setheapsize
 
     - name: Lint
@@ -31,17 +37,38 @@ jobs:
         npm run testnodeconsumer
 
     - name: Production Build
-      if: success()
       run: npx grunt prod
 
     - name: UI Tests
-      if: success()
       run: |
         sudo apt-get install xvfb
         xvfb-run --server-args="-screen 0 1200x800x24" npx grunt testui
 
+    - name: Image Metadata
+      id: image-metadata
+      uses: docker/metadata-action@v4
+      with:
+        images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+        tags: |
+          type=semver,pattern={{major}}
+          type=semver,pattern={{major}}.{{minor}}
+          type=semver,pattern={{version}}
+
+    - name: Production Image Build
+      id: build-image
+      uses: redhat-actions/buildah-build@v2
+      with:
+        tags: ${{ steps.image-metadata.outputs.tags }}
+        labels: ${{ steps.image-metadata.outputs.labels }}
+        containerfiles: ./Dockerfile
+        platforms: linux/amd64
+        oci: true
+        # Webpack seems to use a lot of open files, increase the max open file limit to accomodate.
+        extra-args: |
+          --ulimit nofile=10000
+
+
     - name: Upload Release Assets
-      if: success()
       id: upload-release-assets
       uses: svenstaro/upload-release-action@v2
       with:
@@ -53,7 +80,14 @@ jobs:
         body: "See the [CHANGELOG](https://github.com/gchq/CyberChef/blob/master/CHANGELOG.md) and [commit messages](https://github.com/gchq/CyberChef/commits/master) for details."
 
     - name: Publish to NPM
-      if: success()
       uses: JS-DevTools/npm-publish@v1
       with:
         token: ${{ secrets.NPM_TOKEN }}
+
+    - name: Publish to GHCR
+      uses: redhat-actions/push-to-registry@v2
+      with:
+        tags: ${{ steps.build-image.outputs.tags }}
+        registry: ${{ env.REGISTRY }}
+        username: ${{ env.REGISTRY_USER }}
+        password: ${{ env.REGISTRY_PASSWORD }}

.gitignore

@@ -3,6 +3,7 @@ npm-debug.log
 travis.log
 build
 .vscode
+.idea
 .*.swp
 src/core/config/modules/*
 src/core/config/OperationConfig.json

CHANGELOG.md

@@ -13,6 +13,23 @@ All major and minor version changes will be documented in this file. Details of
 
 ## Details
 
+### [10.9.0] - 2024-03-26
+- Line ending sequences and UTF-8 character encoding are now detected automatically [@n1474335] | [65ffd8d]
+
+### [10.8.0] - 2024-02-13
+- Add official Docker images [@AshCorr] | [#1699]
+
+### [10.7.0] - 2024-02-09
+- Added 'File Tree' operation [@sw5678] | [#1667]
+- Added 'RISON' operation [@sg5506844] | [#1555]
+- Added 'MurmurHash3' operation [@AliceGrey] | [#1694]
+
+### [10.6.0] -  2024-02-03
+- Updated 'Forensics Wiki' URLs to new domain [@a3957273] | [#1703]
+- Added 'LZNT1 Decompress' operation [@0xThiebaut] | [#1675]
+- Updated 'Regex Expression' UUID matcher [@cnotin] | [#1678]
+- Removed duplicate 'hover' message within baking info [@KevinSJ] | [#1541]
+
 ### [10.5.0] - 2023-07-14
 - Added GOST Encrypt, Decrypt, Sign, Verify, Key Wrap, and Key Unwrap operations [@n1474335] | [#592]
 
@@ -372,8 +389,10 @@ All major and minor version changes will be documented in this file. Details of
 ## [4.0.0] - 2016-11-28
 -  Initial open source commit [@n1474335] | [b1d73a72](https://github.com/gchq/CyberChef/commit/b1d73a725dc7ab9fb7eb789296efd2b7e4b08306)
 
-
-
+[10.9.0]: https://github.com/gchq/CyberChef/releases/tag/v10.9.0
+[10.8.0]: https://github.com/gchq/CyberChef/releases/tag/v10.7.0
+[10.7.0]: https://github.com/gchq/CyberChef/releases/tag/v10.7.0
+[10.6.0]: https://github.com/gchq/CyberChef/releases/tag/v10.6.0
 [10.5.0]: https://github.com/gchq/CyberChef/releases/tag/v10.5.0
 [10.4.0]: https://github.com/gchq/CyberChef/releases/tag/v10.4.0
 [10.3.0]: https://github.com/gchq/CyberChef/releases/tag/v10.3.0
@@ -528,6 +547,15 @@ All major and minor version changes will be documented in this file. Details of
 [@joostrijneveld]: https://github.com/joostrijneveld
 [@Xenonym]: https://github.com/Xenonym
 [@gchq77703]: https://github.com/gchq77703
+[@a3957273]: https://github.com/a3957273
+[@0xThiebaut]: https://github.com/0xThiebaut
+[@cnotin]: https://github.com/cnotin
+[@KevinSJ]: https://github.com/KevinSJ
+[@sw5678]: https://github.com/sw5678
+[@sg5506844]: https://github.com/sg5506844
+[@AliceGrey]: https://github.com/AliceGrey
+[@AshCorr]: https://github.com/AshCorr
+
 
 [8ad18b]: https://github.com/gchq/CyberChef/commit/8ad18bc7db6d9ff184ba3518686293a7685bf7b7
 [9a33498]: https://github.com/gchq/CyberChef/commit/9a33498fed26a8df9c9f35f39a78a174bf50a513
@@ -537,6 +565,7 @@ All major and minor version changes will be documented in this file. Details of
 [a895d1d]: https://github.com/gchq/CyberChef/commit/a895d1d82a2f92d440a0c5eca2bc7c898107b737
 [31a7f83]: https://github.com/gchq/CyberChef/commit/31a7f83b82e78927f89689f323fcb9185144d6ff
 [760eff4]: https://github.com/gchq/CyberChef/commit/760eff49b5307aaa3104c5e5b437ffe62299acd1
+[65ffd8d]: https://github.com/gchq/CyberChef/commit/65ffd8d65d88eb369f6f61a5d1d0f807179bffb7
 
 [#95]: https://github.com/gchq/CyberChef/pull/299
 [#173]: https://github.com/gchq/CyberChef/pull/173
@@ -646,4 +675,11 @@ All major and minor version changes will be documented in this file. Details of
 [#661]: https://github.com/gchq/CyberChef/pull/661
 [#493]: https://github.com/gchq/CyberChef/pull/493
 [#592]: https://github.com/gchq/CyberChef/issues/592
-
+[#1703]: https://github.com/gchq/CyberChef/issues/1703
+[#1675]: https://github.com/gchq/CyberChef/issues/1675
+[#1678]: https://github.com/gchq/CyberChef/issues/1678
+[#1541]: https://github.com/gchq/CyberChef/issues/1541
+[#1667]: https://github.com/gchq/CyberChef/issues/1667
+[#1555]: https://github.com/gchq/CyberChef/issues/1555
+[#1694]: https://github.com/gchq/CyberChef/issues/1694
+[#1699]: https://github.com/gchq/CyberChef/issues/1694

Dockerfile

@@ -0,0 +1,9 @@
+FROM node:18-alpine AS build
+
+COPY . .
+RUN npm ci
+RUN npm run build
+
+FROM nginx:1.25-alpine3.18 AS cyberchef
+
+COPY --from=build ./build/prod /usr/share/nginx/html/

README.md

@@ -20,6 +20,22 @@ Cryptographic operations in CyberChef should not be relied upon to provide secur
 
 [A live demo can be found here][1] - have fun!
 
+## Containers
+
+If you would like to try out CyberChef locally you can either build it yourself:
+
+```bash
+docker build --tag cyberchef --ulimit nofile=10000 .
+docker run -it -p 8080:80 cyberchef
+```
+
+Or you can use our image directly:
+
+```bash
+docker run -it -p 8080:80 ghcr.io/gchq/cyberchef:latest
+```
+
+This image is built and published through our [GitHub Workflows](.github/workflows/releases.yml)
 
 ## How it works
 
@@ -89,14 +105,14 @@ CyberChef is built to support
 
 ## Node.js support
 
-CyberChef is built to fully support Node.js `v16`. For more information, see the Node API page in the project [wiki pages](https://github.com/gchq/CyberChef/wiki)
+CyberChef is built to fully support Node.js `v16`. For more information, see the ["Node API" wiki page](https://github.com/gchq/CyberChef/wiki/Node-API)
 
 
 ## Contributing
 
 Contributing a new operation to CyberChef is super easy! The quickstart script will walk you through the process. If you can write basic JavaScript, you can write a CyberChef operation.
 
-An installation walkthrough, how-to guides for adding new operations and themes, descriptions of the repository structure, available data types and coding conventions can all be found in the project [wiki pages](https://github.com/gchq/CyberChef/wiki).
+An installation walkthrough, how-to guides for adding new operations and themes, descriptions of the repository structure, available data types and coding conventions can all be found in the ["Contributing" wiki page](https://github.com/gchq/CyberChef/wiki/Contributing).
 
  - Push your changes to your fork.
  - Submit a pull request. If you are doing this for the first time, you will be prompted to sign the [GCHQ Contributor Licence Agreement](https://cla-assistant.io/gchq/CyberChef) via the CLA assistant on the pull request. This will also ask whether you are happy for GCHQ to contact you about a token of thanks for your contribution, or about job opportunities at GCHQ.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "10.5.1",
+  "version": "10.9.0",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",
@@ -39,55 +39,55 @@
     "node >= 16"
   ],
   "devDependencies": {
-    "@babel/core": "^7.21.0",
-    "@babel/eslint-parser": "^7.19.1",
-    "@babel/plugin-syntax-import-assertions": "^7.20.0",
-    "@babel/plugin-transform-runtime": "^7.21.0",
-    "@babel/preset-env": "^7.20.2",
-    "@babel/runtime": "^7.21.0",
-    "@codemirror/commands": "^6.2.1",
-    "@codemirror/language": "^6.6.0",
-    "@codemirror/search": "^6.2.3",
-    "@codemirror/state": "^6.2.0",
-    "@codemirror/view": "^6.9.2",
-    "autoprefixer": "^10.4.13",
-    "babel-loader": "^9.1.2",
+    "@babel/core": "^7.23.9",
+    "@babel/eslint-parser": "^7.23.10",
+    "@babel/plugin-syntax-import-assertions": "^7.23.3",
+    "@babel/plugin-transform-runtime": "^7.23.9",
+    "@babel/preset-env": "^7.23.9",
+    "@babel/runtime": "^7.23.9",
+    "@codemirror/commands": "^6.3.3",
+    "@codemirror/language": "^6.10.1",
+    "@codemirror/search": "^6.5.5",
+    "@codemirror/state": "^6.4.0",
+    "@codemirror/view": "^6.23.1",
+    "autoprefixer": "^10.4.17",
+    "babel-loader": "^9.1.3",
     "babel-plugin-dynamic-import-node": "^2.3.3",
     "babel-plugin-transform-builtin-extend": "1.1.2",
     "base64-loader": "^1.0.0",
-    "chromedriver": "^110.0.0",
+    "chromedriver": "^122.0.0",
     "cli-progress": "^3.12.0",
     "colors": "^1.4.0",
-    "copy-webpack-plugin": "^11.0.0",
-    "core-js": "^3.29.0",
-    "css-loader": "6.7.3",
-    "eslint": "^8.35.0",
+    "copy-webpack-plugin": "^12.0.2",
+    "core-js": "^3.35.1",
+    "css-loader": "6.10.0",
+    "eslint": "^8.56.0",
     "grunt": "^1.6.1",
     "grunt-chmod": "~1.1.1",
     "grunt-concurrent": "^3.0.0",
     "grunt-contrib-clean": "~2.0.1",
-    "grunt-contrib-connect": "^3.0.0",
+    "grunt-contrib-connect": "^4.0.0",
     "grunt-contrib-copy": "~1.0.0",
     "grunt-contrib-watch": "^1.1.0",
-    "grunt-eslint": "^24.0.1",
+    "grunt-eslint": "^24.3.0",
     "grunt-exec": "~3.0.0",
-    "grunt-webpack": "^5.0.0",
-    "grunt-zip": "^0.20.0",
-    "html-webpack-plugin": "^5.5.0",
-    "imports-loader": "^4.0.1",
-    "mini-css-extract-plugin": "2.7.3",
+    "grunt-webpack": "^6.0.0",
+    "grunt-zip": "^1.0.0",
+    "html-webpack-plugin": "^5.6.0",
+    "imports-loader": "^5.0.0",
+    "mini-css-extract-plugin": "2.8.0",
     "modify-source-webpack-plugin": "^3.0.0",
-    "nightwatch": "^2.6.16",
-    "postcss": "^8.4.21",
-    "postcss-css-variables": "^0.18.0",
-    "postcss-import": "^15.1.0",
-    "postcss-loader": "^7.0.2",
+    "nightwatch": "^3.4.0",
+    "postcss": "^8.4.33",
+    "postcss-css-variables": "^0.19.0",
+    "postcss-import": "^16.0.0",
+    "postcss-loader": "^8.1.0",
     "prompt": "^1.3.0",
     "sitemap": "^7.1.1",
-    "terser": "^5.16.6",
-    "webpack": "^5.76.0",
-    "webpack-bundle-analyzer": "^4.8.0",
-    "webpack-dev-server": "4.11.1",
+    "terser": "^5.27.0",
+    "webpack": "^5.90.1",
+    "webpack-bundle-analyzer": "^4.10.1",
+    "webpack-dev-server": "4.15.1",
     "webpack-node-externals": "^3.0.0",
     "worker-loader": "^3.0.8"
   },
@@ -100,7 +100,7 @@
     "arrive": "^2.4.1",
     "avsc": "^5.7.7",
     "bcryptjs": "^2.4.3",
-    "bignumber.js": "^9.1.1",
+    "bignumber.js": "^9.1.2",
     "blakejs": "^1.2.1",
     "bootstrap": "4.6.2",
     "bootstrap-colorpicker": "^3.4.0",
@@ -108,45 +108,46 @@
     "browserify-zlib": "^0.2.0",
     "bson": "^4.7.2",
     "buffer": "^6.0.3",
-    "cbor": "8.1.0",
+    "cbor": "9.0.2",
     "chi-squared": "^1.1.0",
     "codepage": "^1.15.0",
     "crypto-api": "^0.8.5",
     "crypto-browserify": "^3.12.0",
-    "crypto-js": "^4.1.1",
+    "crypto-js": "^4.2.0",
     "ctph.js": "0.0.5",
-    "d3": "7.8.2",
+    "d3": "7.8.5",
     "d3-hexbin": "^0.2.2",
     "diff": "^5.1.0",
     "es6-promisify": "^7.0.0",
-    "escodegen": "^2.0.0",
+    "escodegen": "^2.1.0",
     "esprima": "^4.0.1",
     "exif-parser": "^0.1.12",
+    "fernet": "^0.3.2",
     "file-saver": "^2.0.5",
-    "flat": "^5.0.2",
+    "flat": "^6.0.1",
     "geodesy": "1.1.3",
-    "highlight.js": "^11.7.0",
+    "highlight.js": "^11.9.0",
     "jimp": "^0.16.13",
-    "jquery": "3.6.4",
+    "jquery": "3.7.1",
     "js-crc": "^0.2.0",
-    "js-sha3": "^0.8.0",
+    "js-sha3": "^0.9.3",
     "jsesc": "^3.0.2",
     "json5": "^2.2.3",
-    "jsonpath-plus": "^7.2.0",
+    "jsonpath-plus": "^8.0.0",
     "jsonwebtoken": "8.5.1",
     "jsqr": "^1.4.0",
-    "jsrsasign": "^10.6.1",
+    "jsrsasign": "^11.1.0",
     "kbpgp": "2.1.15",
     "libbzip2-wasm": "0.0.4",
     "libyara-wasm": "^1.2.1",
     "lodash": "^4.17.21",
-    "loglevel": "^1.8.1",
+    "loglevel": "^1.9.1",
     "loglevel-message-prefix": "^3.0.0",
     "lz-string": "^1.5.0",
     "lz4js": "^0.2.0",
-    "markdown-it": "^13.0.1",
-    "moment": "^2.29.4",
-    "moment-timezone": "^0.5.41",
+    "markdown-it": "^14.0.0",
+    "moment": "^2.30.1",
+    "moment-timezone": "^0.5.44",
     "ngeohash": "^0.6.3",
     "node-forge": "^1.3.1",
     "node-md6": "^0.1.0",
@@ -158,22 +159,23 @@
     "path": "^0.12.7",
     "popper.js": "^1.16.1",
     "process": "^0.11.10",
-    "protobufjs": "^7.2.2",
+    "protobufjs": "^7.2.6",
     "qr-image": "^3.2.0",
-    "reflect-metadata": "^0.1.13",
+    "reflect-metadata": "^0.2.1",
+    "rison": "^0.1.1",
     "scryptsy": "^2.1.0",
     "snackbarjs": "^1.1.0",
-    "sortablejs": "^1.15.0",
+    "sortablejs": "^1.15.2",
     "split.js": "^1.6.5",
     "ssdeep.js": "0.0.3",
     "stream-browserify": "^3.0.0",
-    "tesseract.js": "3.0.3",
-    "ua-parser-js": "^1.0.34",
+    "tesseract.js": "5.0.4",
+    "ua-parser-js": "^1.0.37",
     "unorm": "^1.6.0",
     "utf8": "^3.0.0",
     "vkbeautify": "^0.99.3",
     "xmldom": "^0.6.0",
-    "xpath": "0.0.32",
+    "xpath": "0.0.34",
     "xregexp": "^5.1.1",
     "zlibjs": "^0.3.1"
   },

src/core/Utils.mjs

@@ -892,6 +892,23 @@ class Utils {
     }
 
 
+    /**
+     * Converts a string to it's title case equivalent.
+     *
+     * @param {string} str
+     * @returns string
+     *
+     * @example
+     * // return "A Tiny String"
+     * Utils.toTitleCase("a tIny String");
+     */
+    static toTitleCase(str) {
+        return str.replace(/\w\S*/g, function(txt) {
+            return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase();
+        });
+    }
+
+
     /**
      * Encodes a URI fragment (#) or query (?) using a minimal amount of percent-encoding.
      *

src/core/config/Categories.json

@@ -29,6 +29,8 @@
             "To Base64",
             "From Base64",
             "Show Base64 offsets",
+            "To Base92",
+            "From Base92",
             "To Base85",
             "From Base85",
             "To Base",
@@ -67,7 +69,10 @@
             "JSON to CSV",
             "Avro to JSON",
             "CBOR Encode",
-            "CBOR Decode"
+            "CBOR Decode",
+            "Caret/M-decode",
+            "Rison Encode",
+            "Rison Decode"
         ]
     },
     {
@@ -81,6 +86,8 @@
             "DES Decrypt",
             "Triple DES Encrypt",
             "Triple DES Decrypt",
+            "Fernet Encrypt",
+            "Fernet Decrypt",
             "LS47 Encrypt",
             "LS47 Decrypt",
             "RC2 Encrypt",
@@ -294,7 +301,8 @@
             "Escape string",
             "Unescape string",
             "Pseudo-Random Number Generator",
-            "Sleep"
+            "Sleep",
+            "File Tree"
         ]
     },
     {
@@ -328,7 +336,8 @@
             "CSS selector",
             "Extract EXIF",
             "Extract ID3",
-            "Extract Files"
+            "Extract Files",
+            "RAKE"
         ]
     },
     {
@@ -351,7 +360,8 @@
             "LZMA Decompress",
             "LZMA Compress",
             "LZ4 Decompress",
-            "LZ4 Compress"
+            "LZ4 Compress",
+            "LZNT1 Decompress"
         ]
     },
     {
@@ -368,6 +378,7 @@
             "SHA2",
             "SHA3",
             "SM3",
+            "MurmurHash3",
             "Keccak",
             "Shake",
             "RIPEMD",

src/core/lib/Base92.mjs

@@ -0,0 +1,44 @@
+/**
+ * Base92 resources.
+ *
+ * @author sg5506844 [sg5506844@gmail.com]
+ * @copyright Crown Copyright 2021
+ * @license Apache-2.0
+ */
+
+import OperationError from "../errors/OperationError.mjs";
+
+/**
+ * Base92 alphabet char
+ *
+ * @param {number} val
+ * @returns {number}
+ */
+export function base92Chr(val) {
+    if (val < 0 || val >= 91) {
+        throw new OperationError("Invalid value");
+    }
+    if (val === 0)
+        return "!".charCodeAt(0);
+    else if (val <= 61)
+        return "#".charCodeAt(0) + val - 1;
+    else
+        return "a".charCodeAt(0) + val - 62;
+}
+
+/**
+ * Base92 alphabet ord
+ *
+ * @param {string} val
+ * @returns {number}
+ */
+export function base92Ord(val) {
+    if (val === "!")
+        return 0;
+    else if ("#" <= val && val <= "_")
+        return val.charCodeAt(0) - "#".charCodeAt(0) + 1;
+    else if ("a" <= val && val <= "}")
+        return val.charCodeAt(0) - "a".charCodeAt(0) + 62;
+    throw new OperationError(`${val} is not a base92 character`);
+}
+

src/core/lib/ChrEnc.mjs

@@ -224,8 +224,85 @@ export function chrEncWidth(page) {
  * @copyright Crown Copyright 2019
  * @license Apache-2.0
  */
+export const UNICODE_NORMALISATION_FORMS = ["NFD", "NFC", "NFKD", "NFKC"];
+
 
 /**
- * Character encoding format mappings.
+ * Detects whether the input buffer is valid UTF8.
+ *
+ * @param {ArrayBuffer} data
+ * @returns {number} - 0 = not UTF8, 1 = ASCII, 2 = UTF8
  */
-export const UNICODE_NORMALISATION_FORMS = ["NFD", "NFC", "NFKD", "NFKC"];
+export function isUTF8(data) {
+    const bytes = new Uint8Array(data);
+    let i = 0;
+    let onlyASCII = true;
+    while (i < bytes.length) {
+        if (( // ASCII
+            bytes[i] === 0x09 ||
+            bytes[i] === 0x0A ||
+            bytes[i] === 0x0D ||
+            (0x20 <= bytes[i] && bytes[i] <= 0x7E)
+        )) {
+            i += 1;
+            continue;
+        }
+
+        onlyASCII = false;
+
+        if (( // non-overlong 2-byte
+            (0xC2 <= bytes[i] && bytes[i] <= 0xDF) &&
+            (0x80 <= bytes[i+1] && bytes[i+1] <= 0xBF)
+        )) {
+            i += 2;
+            continue;
+        }
+
+        if (( // excluding overlongs
+            bytes[i] === 0xE0 &&
+            (0xA0 <= bytes[i + 1] && bytes[i + 1] <= 0xBF) &&
+            (0x80 <= bytes[i + 2] && bytes[i + 2] <= 0xBF)
+        ) ||
+        ( // straight 3-byte
+            ((0xE1 <= bytes[i] && bytes[i] <= 0xEC) ||
+            bytes[i] === 0xEE ||
+            bytes[i] === 0xEF) &&
+            (0x80 <= bytes[i + 1] && bytes[i+1] <= 0xBF) &&
+            (0x80 <= bytes[i+2] && bytes[i+2] <= 0xBF)
+        ) ||
+        ( // excluding surrogates
+            bytes[i] === 0xED &&
+            (0x80 <= bytes[i+1] && bytes[i+1] <= 0x9F) &&
+            (0x80 <= bytes[i+2] && bytes[i+2] <= 0xBF)
+        )) {
+            i += 3;
+            continue;
+        }
+
+        if (( // planes 1-3
+            bytes[i] === 0xF0 &&
+            (0x90 <= bytes[i + 1] && bytes[i + 1] <= 0xBF) &&
+            (0x80 <= bytes[i + 2] && bytes[i + 2] <= 0xBF) &&
+            (0x80 <= bytes[i + 3] && bytes[i + 3] <= 0xBF)
+        ) ||
+        ( // planes 4-15
+            (0xF1 <= bytes[i] && bytes[i] <= 0xF3) &&
+            (0x80 <= bytes[i + 1] && bytes[i + 1] <= 0xBF) &&
+            (0x80 <= bytes[i + 2] && bytes[i + 2] <= 0xBF) &&
+            (0x80 <= bytes[i + 3] && bytes[i + 3] <= 0xBF)
+        ) ||
+        ( // plane 16
+            bytes[i] === 0xF4 &&
+            (0x80 <= bytes[i + 1] && bytes[i + 1] <= 0x8F) &&
+            (0x80 <= bytes[i + 2] && bytes[i + 2] <= 0xBF) &&
+            (0x80 <= bytes[i + 3] && bytes[i + 3] <= 0xBF)
+        )) {
+            i += 4;
+            continue;
+        }
+
+        return 0;
+    }
+
+    return onlyASCII ? 1 : 2;
+}

src/core/lib/LZNT1.mjs

@@ -0,0 +1,88 @@
+/**
+ *
+ * LZNT1 Decompress.
+ *
+ * @author 0xThiebaut [thiebaut.dev]
+ * @copyright Crown Copyright 2023
+ * @license Apache-2.0
+ *
+ * https://github.com/Velocidex/go-ntfs/blob/master/parser%2Flznt1.go
+ */
+
+import Utils from "../Utils.mjs";
+import OperationError from "../errors/OperationError.mjs";
+
+const COMPRESSED_MASK = 1 << 15,
+    SIZE_MASK = (1 << 12) - 1;
+
+/**
+ * @param {number} offset
+ * @returns {number}
+ */
+function getDisplacement(offset) {
+    let result = 0;
+    while (offset >= 0x10) {
+        offset >>= 1;
+        result += 1;
+    }
+    return result;
+}
+
+/**
+ * @param {byteArray} compressed
+ * @returns {byteArray}
+ */
+export function decompress(compressed) {
+    const decompressed = Array();
+    let coffset = 0;
+
+    while (coffset + 2 <= compressed.length) {
+        const doffset = decompressed.length;
+
+        const blockHeader = Utils.byteArrayToInt(compressed.slice(coffset, coffset + 2), "little");
+        coffset += 2;
+
+        const size = blockHeader & SIZE_MASK;
+        const blockEnd = coffset + size + 1;
+
+        if (size === 0) {
+            break;
+        } else if (compressed.length < coffset + size) {
+            throw new OperationError("Malformed LZNT1 stream: Block too small! Has the stream been truncated?");
+        }
+
+        if ((blockHeader & COMPRESSED_MASK) !== 0) {
+            while (coffset < blockEnd) {
+                let header = compressed[coffset++];
+
+                for (let i = 0; i < 8 && coffset < blockEnd; i++) {
+                    if ((header & 1) === 0) {
+                        decompressed.push(compressed[coffset++]);
+                    } else {
+                        const pointer = Utils.byteArrayToInt(compressed.slice(coffset, coffset + 2), "little");
+                        coffset += 2;
+
+                        const displacement = getDisplacement(decompressed.length - doffset - 1);
+                        const symbolOffset = (pointer >> (12 - displacement)) + 1;
+                        const symbolLength = (pointer & (0xFFF >> displacement)) + 2;
+                        const shiftOffset = decompressed.length - symbolOffset;
+
+                        for (let shiftDelta = 0; shiftDelta < symbolLength + 1; shiftDelta++) {
+                            const shift = shiftOffset + shiftDelta;
+                            if (shift < 0 || decompressed.length <= shift) {
+                                throw new OperationError("Malformed LZNT1 stream: Invalid shift!");
+                            }
+                            decompressed.push(decompressed[shift]);
+                        }
+                    }
+                    header >>= 1;
+                }
+            }
+        } else {
+            decompressed.push(...compressed.slice(coffset, coffset + size + 1));
+            coffset += size + 1;
+        }
+    }
+
+    return decompressed;
+}

src/core/lib/Magic.mjs

@@ -3,6 +3,7 @@ import Utils, { isWorkerEnvironment } from "../Utils.mjs";
 import Recipe from "../Recipe.mjs";
 import Dish from "../Dish.mjs";
 import {detectFileType, isType} from "./FileType.mjs";
+import {isUTF8} from "./ChrEnc.mjs";
 import chiSquared from "chi-squared";
 
 /**
@@ -111,82 +112,6 @@ class Magic {
         };
     }
 
-    /**
-     * Detects whether the input buffer is valid UTF8.
-     *
-     * @returns {boolean}
-     */
-    isUTF8() {
-        const bytes = new Uint8Array(this.inputBuffer);
-        let i = 0;
-        while (i < bytes.length) {
-            if (( // ASCII
-                bytes[i] === 0x09 ||
-                bytes[i] === 0x0A ||
-                bytes[i] === 0x0D ||
-                (0x20 <= bytes[i] && bytes[i] <= 0x7E)
-            )) {
-                i += 1;
-                continue;
-            }
-
-            if (( // non-overlong 2-byte
-                (0xC2 <= bytes[i] && bytes[i] <= 0xDF) &&
-                (0x80 <= bytes[i+1] && bytes[i+1] <= 0xBF)
-            )) {
-                i += 2;
-                continue;
-            }
-
-            if (( // excluding overlongs
-                bytes[i] === 0xE0 &&
-                (0xA0 <= bytes[i + 1] && bytes[i + 1] <= 0xBF) &&
-                (0x80 <= bytes[i + 2] && bytes[i + 2] <= 0xBF)
-            ) ||
-            ( // straight 3-byte
-                ((0xE1 <= bytes[i] && bytes[i] <= 0xEC) ||
-                bytes[i] === 0xEE ||
-                bytes[i] === 0xEF) &&
-                (0x80 <= bytes[i + 1] && bytes[i+1] <= 0xBF) &&
-                (0x80 <= bytes[i+2] && bytes[i+2] <= 0xBF)
-            ) ||
-            ( // excluding surrogates
-                bytes[i] === 0xED &&
-                (0x80 <= bytes[i+1] && bytes[i+1] <= 0x9F) &&
-                (0x80 <= bytes[i+2] && bytes[i+2] <= 0xBF)
-            )) {
-                i += 3;
-                continue;
-            }
-
-            if (( // planes 1-3
-                bytes[i] === 0xF0 &&
-                (0x90 <= bytes[i + 1] && bytes[i + 1] <= 0xBF) &&
-                (0x80 <= bytes[i + 2] && bytes[i + 2] <= 0xBF) &&
-                (0x80 <= bytes[i + 3] && bytes[i + 3] <= 0xBF)
-            ) ||
-            ( // planes 4-15
-                (0xF1 <= bytes[i] && bytes[i] <= 0xF3) &&
-                (0x80 <= bytes[i + 1] && bytes[i + 1] <= 0xBF) &&
-                (0x80 <= bytes[i + 2] && bytes[i + 2] <= 0xBF) &&
-                (0x80 <= bytes[i + 3] && bytes[i + 3] <= 0xBF)
-            ) ||
-            ( // plane 16
-                bytes[i] === 0xF4 &&
-                (0x80 <= bytes[i + 1] && bytes[i + 1] <= 0x8F) &&
-                (0x80 <= bytes[i + 2] && bytes[i + 2] <= 0xBF) &&
-                (0x80 <= bytes[i + 3] && bytes[i + 3] <= 0xBF)
-            )) {
-                i += 4;
-                continue;
-            }
-
-            return false;
-        }
-
-        return true;
-    }
-
     /**
      * Calculates the Shannon entropy of the input data.
      *
@@ -336,7 +261,7 @@ class Magic {
             data: this.inputStr.slice(0, 100),
             languageScores: this.detectLanguage(extLang),
             fileType: this.detectFileType(),
-            isUTF8: this.isUTF8(),
+            isUTF8: !!isUTF8(this.inputBuffer),
             entropy: this.calcEntropy(),
             matchingOps: matchingOps,
             useful: useful,

src/core/operations/CTPH.mjs

@@ -21,7 +21,7 @@ class CTPH extends Operation {
         this.name = "CTPH";
         this.module = "Crypto";
         this.description = "Context Triggered Piecewise Hashing, also called Fuzzy Hashing, can match inputs that have homologies. Such inputs have sequences of identical bytes in the same order, although bytes in between these sequences may be different in both content and length.<br><br>CTPH was originally based on the work of Dr. Andrew Tridgell and a spam email detector called SpamSum. This method was adapted by Jesse Kornblum and published at the DFRWS conference in 2006 in a paper 'Identifying Almost Identical Files Using Context Triggered Piecewise Hashing'.";
-        this.infoURL = "https://forensicswiki.xyz/wiki/index.php?title=Context_Triggered_Piecewise_Hashing";
+        this.infoURL = "https://forensics.wiki/context_triggered_piecewise_hashing/";
         this.inputType = "string";
         this.outputType = "string";
         this.args = [];

src/core/operations/CaretMdecode.mjs

@@ -0,0 +1,98 @@
+/**
+ * @author tedk [tedk@ted.do]
+ * @copyright Crown Copyright 2024
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation.mjs";
+
+/**
+ * Caret/M-decode operation
+ *
+ * https://gist.githubusercontent.com/JaHIY/3c91bbf7bea5661e6abfbd1349ee81a2/raw/c7b480e9ff24bcb8f5287a8a8a2dcb9bf5628506/decode_m_notation.cpp
+ */
+class CaretMdecode extends Operation {
+
+    /**
+     * CaretMdecode constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Caret/M-decode";
+        this.module = "Default";
+        this.description = "Decodes caret or M-encoded strings, i.e. ^M turns into a newline, M-^] turns into 0x9d. Sources such as `cat -v`.\n\nPlease be aware that when using `cat -v` ^_ (caret-underscore) will not be encoded, but represents a valid encoding (namely that of 0x1f).";
+        this.infoURL = "https://en.wikipedia.org/wiki/Caret_notation";
+        this.inputType = "string";
+        this.outputType = "byteArray";
+        this.args = [];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {byteArray}
+     */
+    run(input, args) {
+
+        const bytes = [];
+
+        let prev = "";
+
+        for (let i = 0; i < input.length; i++) {
+
+            const charCode = input.charCodeAt(i);
+            const curChar = input.charAt(i);
+
+            if (prev === "M-^") {
+                if (charCode > 63 && charCode <= 95) {
+                    bytes.push(charCode + 64);
+                } else if (charCode === 63) {
+                    bytes.push(255);
+                } else {
+                    bytes.push(77, 45, 94, charCode);
+                }
+                prev = "";
+            } else if (prev === "M-") {
+                if (curChar === "^") {
+                    prev = prev + "^";
+                } else if (charCode >= 32 && charCode <= 126) {
+                    bytes.push(charCode + 128);
+                    prev = "";
+                } else {
+                    bytes.push(77, 45, charCode);
+                    prev = "";
+                }
+            } else if (prev === "M") {
+                if (curChar === "-") {
+                    prev = prev + "-";
+                } else {
+                    bytes.push(77, charCode);
+                    prev = "";
+                }
+            } else if (prev === "^") {
+                if (charCode > 63 && charCode <= 126) {
+                    bytes.push(charCode - 64);
+                } else if (charCode === 63) {
+                    bytes.push(127);
+                } else {
+                    bytes.push(94, charCode);
+                }
+                prev = "";
+            } else {
+                if (curChar === "M") {
+                    prev = "M";
+                } else if (curChar === "^") {
+                    prev = "^";
+                } else {
+                    bytes.push(charCode);
+                }
+            }
+
+        }
+        return bytes;
+    }
+
+}
+
+export default CaretMdecode;

src/core/operations/CompareCTPHHashes.mjs

@@ -24,7 +24,7 @@ class CompareCTPHHashes extends Operation {
         this.name = "Compare CTPH hashes";
         this.module = "Crypto";
         this.description = "Compares two Context Triggered Piecewise Hashing (CTPH) fuzzy hashes to determine the similarity between them on a scale of 0 to 100.";
-        this.infoURL = "https://forensicswiki.xyz/wiki/index.php?title=Context_Triggered_Piecewise_Hashing";
+        this.infoURL = "https://forensics.wiki/context_triggered_piecewise_hashing/";
         this.inputType = "string";
         this.outputType = "Number";
         this.args = [

src/core/operations/CompareSSDEEPHashes.mjs

@@ -24,7 +24,7 @@ class CompareSSDEEPHashes extends Operation {
         this.name = "Compare SSDEEP hashes";
         this.module = "Crypto";
         this.description = "Compares two SSDEEP fuzzy hashes to determine the similarity between them on a scale of 0 to 100.";
-        this.infoURL = "https://forensicswiki.xyz/wiki/index.php?title=Ssdeep";
+        this.infoURL = "https://forensics.wiki/ssdeep/";
         this.inputType = "string";
         this.outputType = "Number";
         this.args = [

src/core/operations/Diff.mjs

@@ -119,9 +119,9 @@ class Diff extends Operation {
 
         for (let i = 0; i < diff.length; i++) {
             if (diff[i].added) {
-                if (showAdded) output += "<span class='hl5'>" + Utils.escapeHtml(diff[i].value) + "</span>";
+                if (showAdded) output += "<ins>" + Utils.escapeHtml(diff[i].value) + "</ins>";
             } else if (diff[i].removed) {
-                if (showRemoved) output += "<span class='hl3'>" + Utils.escapeHtml(diff[i].value) + "</span>";
+                if (showRemoved) output += "<del>" + Utils.escapeHtml(diff[i].value) + "</del>";
             } else if (!showSubtraction) {
                 output += Utils.escapeHtml(diff[i].value);
             }

src/core/operations/ExtractFiles.mjs

@@ -39,7 +39,7 @@ class ExtractFiles extends Operation {
                 ${supportedExts.join("</li><li>")}
                 </li>
             </ul>Minimum File Size can be used to prune small false positives.`;
-        this.infoURL = "https://forensicswiki.xyz/wiki/index.php?title=File_Carving";
+        this.infoURL = "https://forensics.wiki/file_carving";
         this.inputType = "ArrayBuffer";
         this.outputType = "List<File>";
         this.presentType = "html";

src/core/operations/ExtractIPAddresses.mjs

@@ -66,7 +66,7 @@ class ExtractIPAddresses extends Operation {
     run(input, args) {
         const [includeIpv4, includeIpv6, removeLocal, displayTotal, sort, unique] = args,
             ipv4 = "(?:(?:\\d|[01]?\\d\\d|2[0-4]\\d|25[0-5])\\.){3}(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d|\\d)(?:\\/\\d{1,2})?",
-            ipv6 = "((?=.*::)(?!.*::.+::)(::)?([\\dA-F]{1,4}:(:|\\b)|){5}|([\\dA-F]{1,4}:){6})((([\\dA-F]{1,4}((?!\\3)::|:\\b|(?![\\dA-F])))|(?!\\2\\3)){2}|(((2[0-4]|1\\d|[1-9])?\\d|25[0-5])\\.?\\b){4})";
+            ipv6 = "((?=.*::)(?!.*::.+::)(::)?([\\dA-F]{1,4}:(:|\\b)|){5}|([\\dA-F]{1,4}:){6})(([\\dA-F]{1,4}((?!\\3)::|:\\b|(?![\\dA-F])))|(?!\\2\\3)){2}";
         let ips  = "";
 
         if (includeIpv4 && includeIpv6) {

src/core/operations/FernetDecrypt.mjs

@@ -0,0 +1,63 @@
+/**
+ * @author Karsten Silkenbäumer [github.com/kassi]
+ * @copyright Karsten Silkenbäumer 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation.mjs";
+import OperationError from "../errors/OperationError.mjs";
+import fernet from "fernet";
+
+/**
+ * FernetDecrypt operation
+ */
+class FernetDecrypt extends Operation {
+    /**
+     * FernetDecrypt constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Fernet Decrypt";
+        this.module = "Default";
+        this.description = "Fernet is a symmetric encryption method which makes sure that the message encrypted cannot be manipulated/read without the key. It uses URL safe encoding for the keys. Fernet uses 128-bit AES in CBC mode and PKCS7 padding, with HMAC using SHA256 for authentication. The IV is created from os.random().<br><br><b>Key:</b> The key must be 32 bytes (256 bits) encoded with Base64.";
+        this.infoURL = "https://asecuritysite.com/encryption/fer";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [
+            {
+                "name": "Key",
+                "type": "string",
+                "value": ""
+            },
+        ];
+        this.patterns = [
+            {
+                match: "^[A-Z\\d\\-_=]{20,}$",
+                flags: "i",
+                args: []
+            },
+        ];
+    }
+    /**
+     * @param {String} input
+     * @param {Object[]} args
+     * @returns {String}
+     */
+    run(input, args) {
+        const [secretInput] = args;
+        try {
+            const secret = new fernet.Secret(secretInput);
+            const token = new fernet.Token({
+                secret: secret,
+                token: input,
+                ttl: 0
+            });
+            return token.decode();
+        } catch (err) {
+            throw new OperationError(err);
+        }
+    }
+}
+
+export default FernetDecrypt;

src/core/operations/FernetEncrypt.mjs

@@ -0,0 +1,54 @@
+/**
+ * @author Karsten Silkenbäumer [github.com/kassi]
+ * @copyright Karsten Silkenbäumer 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation.mjs";
+import OperationError from "../errors/OperationError.mjs";
+import fernet from "fernet";
+
+/**
+ * FernetEncrypt operation
+ */
+class FernetEncrypt extends Operation {
+    /**
+     * FernetEncrypt constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Fernet Encrypt";
+        this.module = "Default";
+        this.description = "Fernet is a symmetric encryption method which makes sure that the message encrypted cannot be manipulated/read without the key. It uses URL safe encoding for the keys. Fernet uses 128-bit AES in CBC mode and PKCS7 padding, with HMAC using SHA256 for authentication. The IV is created from os.random().<br><br><b>Key:</b> The key must be 32 bytes (256 bits) encoded with Base64.";
+        this.infoURL = "https://asecuritysite.com/encryption/fer";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [
+            {
+                "name": "Key",
+                "type": "string",
+                "value": ""
+            },
+        ];
+    }
+    /**
+     * @param {String} input
+     * @param {Object[]} args
+     * @returns {String}
+     */
+    run(input, args) {
+        const [secretInput] = args;
+        try {
+            const secret = new fernet.Secret(secretInput);
+            const token = new fernet.Token({
+                secret: secret,
+            });
+            return token.encode(input);
+        } catch (err) {
+            throw new OperationError(err);
+        }
+    }
+}
+
+export default FernetEncrypt;

src/core/operations/FileTree.mjs

@@ -0,0 +1,93 @@
+/**
+ * @author sw5678
+ * @copyright Crown Copyright 2016
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation.mjs";
+import Utils from "../Utils.mjs";
+import {INPUT_DELIM_OPTIONS} from "../lib/Delim.mjs";
+
+/**
+ * Unique operation
+ */
+class FileTree extends Operation {
+
+    /**
+     * Unique constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "File Tree";
+        this.module = "Default";
+        this.description = "Creates file tree from list of file paths (similar to the tree command in Linux)";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [
+            {
+                name: "File Path Delimiter",
+                type: "binaryString",
+                value: "/"
+            },
+            {
+                name: "Delimiter",
+                type: "option",
+                value: INPUT_DELIM_OPTIONS
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+
+        // Set up arrow and pipe for nice output display
+        const ARROW = "|---";
+        const PIPE = "|   ";
+
+        // Get args from input
+        const fileDelim = args[0];
+        const entryDelim = Utils.charRep(args[1]);
+
+        // Store path to print
+        const completedList = [];
+        const printList = [];
+
+        // Loop through all entries
+        const filePaths = input.split(entryDelim).unique().sort();
+        for (let i = 0; i < filePaths.length; i++) {
+            // Split by file delimiter
+            let path = filePaths[i].split(fileDelim);
+
+            if (path[0] === "") {
+                path = path.slice(1, path.length);
+            }
+
+            for (let j = 0; j < path.length; j++) {
+                let printLine;
+                let key;
+                if (j === 0) {
+                    printLine = path[j];
+                    key = path[j];
+                } else {
+                    printLine = PIPE.repeat(j-1) + ARROW + path[j];
+                    key = path.slice(0, j+1).join("/");
+                }
+
+                // Check to see we have already added that path
+                if (!completedList.includes(key)) {
+                    completedList.push(key);
+                    printList.push(printLine);
+                }
+            }
+        }
+        return printList.join("\n");
+    }
+
+}
+
+export default FileTree;

src/core/operations/FromBase92.mjs

@@ -0,0 +1,55 @@
+/**
+ * @author sg5506844 [sg5506844@gmail.com]
+ * @copyright Crown Copyright 2021
+ * @license Apache-2.0
+ */
+
+import { base92Ord } from "../lib/Base92.mjs";
+import Operation from "../Operation.mjs";
+
+/**
+ * From Base92 operation
+ */
+class FromBase92 extends Operation {
+    /**
+     * FromBase92 constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "From Base92";
+        this.module = "Default";
+        this.description = "Base92 is a notation for encoding arbitrary byte data using a restricted set of symbols that can be conveniently used by humans and processed by computers.";
+        this.infoURL = "https://wikipedia.org/wiki/List_of_numeral_systems";
+        this.inputType = "string";
+        this.outputType = "byteArray";
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {byteArray}
+     */
+    run(input, args) {
+        const res = [];
+        let bitString = "";
+
+        for (let i = 0; i < input.length; i += 2) {
+            if (i + 1 !== input.length) {
+                const x = base92Ord(input[i]) * 91 + base92Ord(input[i + 1]);
+                bitString += x.toString(2).padStart(13, "0");
+            } else {
+                const x = base92Ord(input[i]);
+                bitString += x.toString(2).padStart(6, "0");
+            }
+            while (bitString.length >= 8) {
+                res.push(parseInt(bitString.slice(0, 8), 2));
+                bitString = bitString.slice(8);
+            }
+        }
+
+        return res;
+    }
+}
+
+export default FromBase92;

src/core/operations/LZNT1Decompress.mjs

@@ -0,0 +1,41 @@
+/**
+ * @author 0xThiebaut [thiebaut.dev]
+ * @copyright Crown Copyright 2023
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation.mjs";
+import {decompress} from "../lib/LZNT1.mjs";
+
+/**
+ * LZNT1 Decompress operation
+ */
+class LZNT1Decompress extends Operation {
+
+    /**
+     * LZNT1 Decompress constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "LZNT1 Decompress";
+        this.module = "Compression";
+        this.description = "Decompresses data using the LZNT1 algorithm.<br><br>Similar to the Windows API <code>RtlDecompressBuffer</code>.";
+        this.infoURL = "https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-xca/5655f4a3-6ba4-489b-959f-e1f407c52f15";
+        this.inputType = "byteArray";
+        this.outputType = "byteArray";
+        this.args = [];
+    }
+
+    /**
+     * @param {byteArray} input
+     * @param {Object[]} args
+     * @returns {byteArray}
+     */
+    run(input, args) {
+        return decompress(input);
+    }
+
+}
+
+export default LZNT1Decompress;

src/core/operations/MurmurHash3.mjs

@@ -0,0 +1,143 @@
+/**
+ * Based on murmurhash-js (https://github.com/garycourt/murmurhash-js)
+ * @author Gary Court
+ * @license MIT
+ *
+ * @author AliceGrey [alice@grey.systems]
+ * @copyright Crown Copyright 2024
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation.mjs";
+
+/**
+ * MurmurHash3 operation
+ */
+class MurmurHash3 extends Operation {
+
+    /**
+     * MurmurHash3 constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "MurmurHash3";
+        this.module = "Default";
+        this.description = "Generates a MurmurHash v3 for a string input and an optional seed input";
+        this.infoURL = "https://wikipedia.org/wiki/MurmurHash";
+        this.inputType = "string";
+        this.outputType = "number";
+        this.args = [
+            {
+                name: "Seed",
+                type: "number",
+                value: 0
+            },
+            {
+                name: "Convert to Signed",
+                type: "boolean",
+                value: false
+            }
+        ];
+    }
+
+ /**
+ * Calculates the MurmurHash3 hash of the input.
+ * Based on Gary Court's JS MurmurHash implementation
+ * @see http://github.com/garycourt/murmurhash-js
+ * @author AliceGrey [alice@grey.systems]
+ * @param {string} input ASCII only
+ * @param {number} seed Positive integer only
+ * @return {number} 32-bit positive integer hash
+ */
+    mmh3(input, seed) {
+        let h1b;
+        let k1;
+        const remainder = input.length & 3; // input.length % 4
+        const bytes = input.length - remainder;
+        let h1 = seed;
+        const c1 = 0xcc9e2d51;
+        const c2 = 0x1b873593;
+        let i = 0;
+
+        while (i < bytes) {
+            k1 =
+                ((input.charCodeAt(i) & 0xff)) |
+                ((input.charCodeAt(++i) & 0xff) << 8) |
+                ((input.charCodeAt(++i) & 0xff) << 16) |
+                ((input.charCodeAt(++i) & 0xff) << 24);
+            ++i;
+
+            k1 = ((((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16))) & 0xffffffff;
+            k1 = (k1 << 15) | (k1 >>> 17);
+            k1 = ((((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16))) & 0xffffffff;
+
+            h1 ^= k1;
+            h1 = (h1 << 13) | (h1 >>> 19);
+            h1b = ((((h1 & 0xffff) * 5) + ((((h1 >>> 16) * 5) & 0xffff) << 16))) & 0xffffffff;
+            h1 = (((h1b & 0xffff) + 0x6b64) + ((((h1b >>> 16) + 0xe654) & 0xffff) << 16));
+        }
+
+        k1 = 0;
+
+        if (remainder === 3) {
+            k1 ^= (input.charCodeAt(i + 2) & 0xff) << 16;
+        }
+
+        if (remainder === 3 || remainder === 2) {
+            k1 ^= (input.charCodeAt(i + 1) & 0xff) << 8;
+        }
+
+        if (remainder === 3 || remainder === 2 || remainder === 1) {
+            k1 ^= (input.charCodeAt(i) & 0xff);
+
+            k1 = (((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16)) & 0xffffffff;
+            k1 = (k1 << 15) | (k1 >>> 17);
+            k1 = (((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16)) & 0xffffffff;
+            h1 ^= k1;
+        }
+
+        h1 ^= input.length;
+
+        h1 ^= h1 >>> 16;
+        h1 = (((h1 & 0xffff) * 0x85ebca6b) + ((((h1 >>> 16) * 0x85ebca6b) & 0xffff) << 16)) & 0xffffffff;
+        h1 ^= h1 >>> 13;
+        h1 = ((((h1 & 0xffff) * 0xc2b2ae35) + ((((h1 >>> 16) * 0xc2b2ae35) & 0xffff) << 16))) & 0xffffffff;
+        h1 ^= h1 >>> 16;
+
+        return h1 >>> 0;
+    }
+
+    /**
+    * Converts an unsigned 32-bit integer to a signed 32-bit integer
+    * @author AliceGrey [alice@grey.systems]
+    * @param {value} 32-bit unsigned integer
+    * @return {number} 32-bit signed integer
+    */
+    unsignedToSigned(value) {
+        if (value & 0x80000000) {
+            return -0x100000000 + value;
+        } else {
+            return value;
+        }
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {number}
+     */
+    run(input, args) {
+        if (args && args.length >= 1) {
+            const seed = args[0];
+            const hash = this.mmh3(input, seed);
+            if (args.length > 1 && args[1]) {
+                return this.unsignedToSigned(hash);
+            }
+            return hash;
+        }
+        return this.mmh3(input);
+    }
+}
+
+export default MurmurHash3;

src/core/operations/ParseASN1HexString.mjs

@@ -20,7 +20,7 @@ class ParseASN1HexString extends Operation {
 
         this.name = "Parse ASN.1 hex string";
         this.module = "PublicKey";
-        this.description = "Abstract Syntax Notation One (ASN.1) is a standard and notation that describes rules and structures for representing, encoding, transmitting, and decoding data in telecommunications and computer networking.<br><br>This operation parses arbitrary ASN.1 data and presents the resulting tree.";
+        this.description = "Abstract Syntax Notation One (ASN.1) is a standard and notation that describes rules and structures for representing, encoding, transmitting, and decoding data in telecommunications and computer networking.<br><br>This operation parses arbitrary ASN.1 data (encoded as an hex string: use the 'To Hex' operation if necessary) and presents the resulting tree.";
         this.infoURL = "https://wikipedia.org/wiki/Abstract_Syntax_Notation_One";
         this.inputType = "string";
         this.outputType = "string";

src/core/operations/RAKE.mjs

@@ -0,0 +1,149 @@
+/**
+ * @author sw5678
+ * @copyright Crown Copyright 2024
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation.mjs";
+
+/**
+ * RAKE operation
+ */
+class RAKE extends Operation {
+
+    /**
+     * RAKE constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "RAKE";
+        this.module = "Default";
+        this.description = [
+            "Rapid Keyword Extraction (RAKE)",
+            "<br><br>",
+            "RAKE is a domain-independent keyword extraction algorithm in Natural Language Processing.",
+            "<br><br>",
+            "The list of stop words are from the NLTK python package",
+        ].join("\n");
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [
+            {
+                name: "Word Delimiter (Regex)",
+                type: "text",
+                value: "\\s"
+            },
+            {
+                name: "Sentence Delimiter (Regex)",
+                type: "text",
+                value: "\\.\\s|\\n"
+            },
+            {
+                name: "Stop Words",
+                type: "text",
+                value: "i,me,my,myself,we,our,ours,ourselves,you,you're,you've,you'll,you'd,your,yours,yourself,yourselves,he,him,his,himself,she,she's,her,hers,herself,it,it's,its,itsef,they,them,their,theirs,themselves,what,which,who,whom,this,that,that'll,these,those,am,is,are,was,were,be,been,being,have,has,had,having,do,does',did,doing,a,an,the,and,but,if,or,because,as,until,while,of,at,by,for,with,about,against,between,into,through,during,before,after,above,below,to,from,up,down,in,out,on,off,over,under,again,further,then,once,here,there,when,where,why,how,all,any,both,each,few,more,most,other,some,such,no,nor,not,only,own,same,so,than,too,very,s,t,can,will,just,don,don't,should,should've,now,d,ll,m,o,re,ve,y,ain,aren,aren't,couldn,couldn't,didn,didn't,doesn,doesn't,hadn,hadn't,hasn,hasn't,haven,haven't,isn,isn't,ma,mightn,mightn't,mustn,mustn't,needn,needn't,shan,shan't,shouldn,shouldn't,wasn,wasn't,weren,weren't,won,won't,wouldn,wouldn't"
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+
+        // Get delimiter regexs
+        const wordDelim =  new RegExp(args[0], "g");
+        const sentDelim = new RegExp(args[1], "g");
+
+        // Deduplicate the stop words and add the empty string
+        const stopWords = args[2].toLowerCase().replace(/ /g, "").split(",").unique();
+        stopWords.push("");
+
+        // Lower case input and remove start and ending whitespace
+        input = input.toLowerCase().trim();
+
+        // Get tokens, token count, and phrases
+        const tokens = [];
+        const wordFrequencies = [];
+        let phrases = [];
+
+        // Build up list of phrases and token counts
+        const sentences = input.split(sentDelim);
+        for (const sent of sentences) {
+
+            // Split sentence into words
+            const splitSent = sent.split(wordDelim);
+            let startIndex = 0;
+
+            for (let i = 0; i < splitSent.length; i++) {
+                const token = splitSent[i];
+                if (stopWords.includes(token)) {
+                    // If token is stop word then split to create phrase
+                    phrases.push(splitSent.slice(startIndex, i));
+                    startIndex = i + 1;
+                } else {
+                    // If token is not a stop word add to the count of the list of words
+                    if (tokens.includes(token)) {
+                        wordFrequencies[tokens.indexOf(token)]+=1;
+                    } else {
+                        tokens.push(token);
+                        wordFrequencies.push(1);
+                    }
+                }
+            }
+            phrases.push(splitSent.slice(startIndex));
+        }
+
+        // remove empty phrases
+        phrases = phrases.filter(subArray => subArray.length > 0);
+
+        // Remove duplicate phrases
+        const uniquePhrases = [...new Set(phrases.map(function (phrase) {
+            return phrase.join(" ");
+        }))];
+        phrases = uniquePhrases.map(function (phrase) {
+            return phrase.split(" ");
+        });
+
+        // Generate word_degree_matrix and populate
+        const wordDegreeMatrix = Array.from(Array(tokens.length), _ => Array(tokens.length).fill(0));
+        phrases.forEach(function (phrase) {
+            phrase.forEach(function (word1) {
+                phrase.forEach(function (word2) {
+                    wordDegreeMatrix[tokens.indexOf(word1)][tokens.indexOf(word2)]++;
+                });
+            });
+        });
+
+        // Calculate degree score for each token
+        const degreeScores = Array(tokens.length).fill(0);
+        for (let i=0; i<tokens.length; i++) {
+            let wordDegree = 0;
+            for (let j=0; j<wordDegreeMatrix.length; j++) {
+                wordDegree += wordDegreeMatrix[j][i];
+            }
+            degreeScores[i] = wordDegree / wordFrequencies[i];
+        }
+
+        // Calculate score for each phrase
+        const scores = phrases.map(function (phrase) {
+            let score = 0;
+            phrase.forEach(function (token) {
+                score += degreeScores[tokens.indexOf(token)];
+            });
+            return new Array(score, phrase.join(" "));
+        });
+        scores.sort((a, b) => b[0] - a[0]);
+        scores.unshift(new Array("Scores: ", "Keywords: "));
+
+        // Output works with the 'To Table' functionality already built into CC
+        return scores.map(function (score) {
+            return score.join(", ");
+        }).join("\n");
+    }
+}
+
+export default RAKE;

src/core/operations/RegularExpression.mjs

@@ -83,6 +83,10 @@ class RegularExpression extends Operation {
                         name: "Strings",
                         value: "[A-Za-z\\d/\\-:.,_$%\\x27\"()<>= !\\[\\]{}@]{4,}"
                     },
+                    {
+                        name: "UUID (any version)",
+                        value: "[0-9a-fA-F]{8}\\b-[0-9a-fA-F]{4}\\b-[0-9a-fA-F]{4}\\b-[0-9a-fA-F]{4}\\b-[0-9a-fA-F]{12}"
+                    },
                 ],
                 "target": 1
             },

src/core/operations/RisonDecode.mjs

@@ -0,0 +1,60 @@
+/**
+ * @author sg5506844 [sg5506844@gmail.com]
+ * @copyright Crown Copyright 2021
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation.mjs";
+import OperationError from "../errors/OperationError.mjs";
+import rison from "rison";
+
+/**
+ * Rison Decode operation
+ */
+class RisonDecode extends Operation {
+
+    /**
+     * RisonDecode constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Rison Decode";
+        this.module = "Default";
+        this.description = "Rison, a data serialization format optimized for compactness in URIs. Rison is a slight variation of JSON that looks vastly superior after URI encoding. Rison still expresses exactly the same set of data structures as JSON, so data can be translated back and forth without loss or guesswork.";
+        this.infoURL = "https://github.com/Nanonid/rison";
+        this.inputType = "string";
+        this.outputType = "Object";
+        this.args = [
+            {
+                name: "Decode Option",
+                type: "editableOption",
+                value: [
+                    { name: "Decode", value: "Decode", },
+                    { name: "Decode Object", value: "Decode Object", },
+                    { name: "Decode Array", value: "Decode Array", },
+                ]
+            },
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {Object}
+     */
+    run(input, args) {
+        const [decodeOption] = args;
+        switch (decodeOption) {
+            case "Decode":
+                return rison.decode(input);
+            case "Decode Object":
+                return rison.decode_object(input);
+            case "Decode Array":
+                return rison.decode_array(input);
+        }
+        throw new OperationError("Invalid Decode option");
+    }
+}
+
+export default RisonDecode;

src/core/operations/RisonEncode.mjs

@@ -0,0 +1,63 @@
+/**
+ * @author sg5506844 [sg5506844@gmail.com]
+ * @copyright Crown Copyright 2021
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation.mjs";
+import OperationError from "../errors/OperationError.mjs";
+import rison from "rison";
+
+/**
+ * Rison Encode operation
+ */
+class RisonEncode extends Operation {
+
+    /**
+     * RisonEncode constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Rison Encode";
+        this.module = "Default";
+        this.description = "Rison, a data serialization format optimized for compactness in URIs. Rison is a slight variation of JSON that looks vastly superior after URI encoding. Rison still expresses exactly the same set of data structures as JSON, so data can be translated back and forth without loss or guesswork.";
+        this.infoURL = "https://github.com/Nanonid/rison";
+        this.inputType = "Object";
+        this.outputType = "string";
+        this.args = [
+            {
+                name: "Encode Option",
+                type: "editableOption",
+                value: [
+                    { name: "Encode", value: "Encode", },
+                    { name: "Encode Object", value: "Encode Object", },
+                    { name: "Encode Array", value: "Encode Array", },
+                    { name: "Encode URI", value: "Encode URI", }
+                ]
+            },
+        ];
+    }
+
+    /**
+     * @param {Object} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        const [encodeOption] = args;
+        switch (encodeOption) {
+            case "Encode":
+                return rison.encode(input);
+            case "Encode Object":
+                return rison.encode_object(input);
+            case "Encode Array":
+                return rison.encode_array(input);
+            case "Encode URI":
+                return rison.encode_uri(input);
+        }
+        throw new OperationError("Invalid encode option");
+    }
+}
+
+export default RisonEncode;

src/core/operations/SSDEEP.mjs

@@ -21,7 +21,7 @@ class SSDEEP extends Operation {
         this.name = "SSDEEP";
         this.module = "Crypto";
         this.description = "SSDEEP is a program for computing context triggered piecewise hashes (CTPH). Also called fuzzy hashes, CTPH can match inputs that have homologies. Such inputs have sequences of identical bytes in the same order, although bytes in between these sequences may be different in both content and length.<br><br>SSDEEP hashes are now widely used for simple identification purposes (e.g. the 'Basic Properties' section in VirusTotal). Although 'better' fuzzy hashes are available, SSDEEP is still one of the primary choices because of its speed and being a de facto standard.<br><br>This operation is fundamentally the same as the CTPH operation, however their outputs differ in format.";
-        this.infoURL = "https://forensicswiki.xyz/wiki/index.php?title=Ssdeep";
+        this.infoURL = "https://forensics.wiki/ssdeep";
         this.inputType = "string";
         this.outputType = "string";
         this.args = [];

src/core/operations/ToBase92.mjs

@@ -0,0 +1,67 @@
+/**
+ * @author sg5506844 [sg5506844@gmail.com]
+ * @copyright Crown Copyright 2021
+ * @license Apache-2.0
+ */
+
+import { base92Chr } from "../lib/Base92.mjs";
+import Operation from "../Operation.mjs";
+
+/**
+ * To Base92 operation
+ */
+class ToBase92 extends Operation {
+    /**
+     * ToBase92 constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "To Base92";
+        this.module = "Default";
+        this.description = "Base92 is a notation for encoding arbitrary byte data using a restricted set of symbols that can be conveniently used by humans and processed by computers.";
+        this.infoURL = "https://wikipedia.org/wiki/List_of_numeral_systems";
+        this.inputType = "string";
+        this.outputType = "byteArray";
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {byteArray}
+     */
+    run(input, args) {
+        const res = [];
+        let bitString = "";
+
+        while (input.length > 0) {
+            while (bitString.length < 13 && input.length > 0) {
+                bitString += input[0].charCodeAt(0).toString(2).padStart(8, "0");
+                input = input.slice(1);
+            }
+            if (bitString.length < 13)
+                break;
+            const i = parseInt(bitString.slice(0, 13), 2);
+            res.push(base92Chr(Math.floor(i / 91)));
+            res.push(base92Chr(i % 91));
+            bitString = bitString.slice(13);
+        }
+
+        if (bitString.length > 0) {
+            if (bitString.length < 7) {
+                bitString = bitString.padEnd(6, "0");
+                res.push(base92Chr(parseInt(bitString, 2)));
+            } else {
+                bitString = bitString.padEnd(13, "0");
+                const i = parseInt(bitString.slice(0, 13), 2);
+                res.push(base92Chr(Math.floor(i / 91)));
+                res.push(base92Chr(i % 91));
+            }
+        }
+
+        return res;
+
+    }
+}
+
+export default ToBase92;

src/web/App.mjs

@@ -46,6 +46,8 @@ class App {
         this.appLoaded     = false;
         this.workerLoaded  = false;
         this.waitersLoaded = false;
+
+        this.snackbars     = [];
     }
 
 
@@ -500,22 +502,22 @@ class App {
         // Input Character Encoding
         // Must be set before the input is loaded
         if (this.uriParams.ienc) {
-            this.manager.input.chrEncChange(parseInt(this.uriParams.ienc, 10));
+            this.manager.input.chrEncChange(parseInt(this.uriParams.ienc, 10), true);
         }
 
         // Output Character Encoding
         if (this.uriParams.oenc) {
-            this.manager.output.chrEncChange(parseInt(this.uriParams.oenc, 10));
+            this.manager.output.chrEncChange(parseInt(this.uriParams.oenc, 10), true);
         }
 
         // Input EOL sequence
         if (this.uriParams.ieol) {
-            this.manager.input.eolChange(this.uriParams.ieol);
+            this.manager.input.eolChange(this.uriParams.ieol, true);
         }
 
         // Output EOL sequence
         if (this.uriParams.oeol) {
-            this.manager.output.eolChange(this.uriParams.oeol);
+            this.manager.output.eolChange(this.uriParams.oeol, true);
         }
 
         // Read in input data from URI params
@@ -708,14 +710,14 @@ class App {
         log.info("[" + time.toLocaleString() + "] " + str);
         if (silent) return;
 
-        this.currentSnackbar = $.snackbar({
+        this.snackbars.push($.snackbar({
             content: str,
             timeout: timeout,
             htmlAllowed: true,
             onClose: () => {
-                this.currentSnackbar.remove();
+                this.snackbars.shift().remove();
             }
-        });
+        }));
     }
 
 

src/web/HTMLOperation.mjs

@@ -157,9 +157,9 @@ function titleFromWikiLink(urlStr) {
         pageTitle = "";
 
     switch (urlObj.host) {
-        case "forensicswiki.xyz":
+        case "forensics.wiki":
             wikiName = "Forensics Wiki";
-            pageTitle = urlObj.query.substr(6).replace(/_/g, " "); // Chop off 'title='
+            pageTitle = Utils.toTitleCase(urlObj.path.replace(/\//g, "").replace(/_/g, " "));
             break;
         case "wikipedia.org":
             wikiName = "Wikipedia";

src/web/html/index.html

@@ -62,7 +62,8 @@
                 "Training branch predictor...",
                 "Timing cache hits...",
                 "Speculatively executing recipes...",
-                "Adding LLM hallucinations..."
+                "Adding LLM hallucinations...",
+                "Decompressing malware..."
             ];
 
             // Shuffle array using Durstenfeld algorithm

src/web/stylesheets/components/_operation.css

@@ -69,6 +69,10 @@ select.arg {
     min-width: 100px;
 }
 
+select.arg.form-control:not([size]):not([multiple]), select.custom-file-control:not([size]):not([multiple]) {
+    height: 100% !important;
+}
+
 textarea.arg {
     min-height: 74px;
     resize: vertical;
@@ -80,7 +84,7 @@ div.toggle-string {
 
 input.toggle-string {
     border-top-right-radius: 0 !important;
-    height: 42px !important;
+    height: 100%;
 }
 
 .operation [class^='bmd-label'],

src/web/stylesheets/index.css

@@ -36,4 +36,5 @@
 @import "./layout/_structure.css";
 
 /* Operations */
+@import "./operations/diff.css";
 @import "./operations/json.css";

src/web/stylesheets/operations/diff.css

@@ -0,0 +1,8 @@
+del {
+    background-color: var(--hl3);
+}
+
+ins {
+    text-decoration: underline; /* shouldn't be needed, but Chromium doesn't copy to clipboard without it */
+    background-color: var(--hl5);
+}

src/web/stylesheets/operations/json.css

@@ -44,7 +44,8 @@ ul.json-dict, ol.json-array {
     display: contents;
 }
 .json-summary {
-    display: contents;
+    display: inline;
+    list-style: none;
 }
 
 /* Display object and array brackets when closed */

src/web/utils/editorUtils.mjs

@@ -95,3 +95,42 @@ export function escapeControlChars(str, preserveWs=false, lineBreak="\n") {
         return n.outerHTML;
     });
 }
+
+/**
+ * Convert and EOL sequence to its name
+ */
+export const eolSeqToCode = {
+    "\u000a": "LF",
+    "\u000b": "VT",
+    "\u000c": "FF",
+    "\u000d": "CR",
+    "\u000d\u000a": "CRLF",
+    "\u0085": "NEL",
+    "\u2028": "LS",
+    "\u2029": "PS"
+};
+
+/**
+ * Convert an EOL name to its sequence
+ */
+export const eolCodeToSeq = {
+    "LF": "\u000a",
+    "VT": "\u000b",
+    "FF": "\u000c",
+    "CR": "\u000d",
+    "CRLF": "\u000d\u000a",
+    "NEL": "\u0085",
+    "LS": "\u2028",
+    "PS": "\u2029"
+};
+
+export const eolCodeToName = {
+    "LF": "Line Feed",
+    "VT": "Vertical Tab",
+    "FF": "Form Feed",
+    "CR": "Carriage Return",
+    "CRLF": "Carriage Return + Line Feed",
+    "NEL": "Next Line",
+    "LS": "Line Separator",
+    "PS": "Paragraph Separator"
+};

src/web/utils/statusBar.mjs

@@ -6,6 +6,7 @@
 
 import {showPanel} from "@codemirror/view";
 import {CHR_ENC_SIMPLE_LOOKUP, CHR_ENC_SIMPLE_REVERSE_LOOKUP} from "../../core/lib/ChrEnc.mjs";
+import { eolCodeToName, eolSeqToCode } from "./editorUtils.mjs";
 
 /**
  * A Status bar extension for CodeMirror
@@ -23,6 +24,8 @@ class StatusBarPanel {
         this.eolHandler = opts.eolHandler;
         this.chrEncHandler = opts.chrEncHandler;
         this.chrEncGetter = opts.chrEncGetter;
+        this.getEncodingState = opts.getEncodingState;
+        this.getEOLState = opts.getEOLState;
         this.htmlOutput = opts.htmlOutput;
 
         this.eolVal = null;
@@ -92,22 +95,12 @@ class StatusBarPanel {
         // preventDefault is required to stop the URL being modified and popState being triggered
         e.preventDefault();
 
-        const eolLookup = {
-            "LF": "\u000a",
-            "VT": "\u000b",
-            "FF": "\u000c",
-            "CR": "\u000d",
-            "CRLF": "\u000d\u000a",
-            "NEL": "\u0085",
-            "LS": "\u2028",
-            "PS": "\u2029"
-        };
-        const eolval = eolLookup[e.target.getAttribute("data-val")];
-
-        if (eolval === undefined) return;
+        const eolCode = e.target.getAttribute("data-val");
+        if (!eolCode) return;
 
         // Call relevant EOL change handler
-        this.eolHandler(eolval);
+        this.eolHandler(e.target.getAttribute("data-val"), true);
+
         hideElement(e.target.closest(".cm-status-bar-select-content"));
     }
 
@@ -124,7 +117,7 @@ class StatusBarPanel {
 
         if (isNaN(chrEncVal)) return;
 
-        this.chrEncHandler(chrEncVal);
+        this.chrEncHandler(chrEncVal, true);
         this.updateCharEnc(chrEncVal);
         hideElement(e.target.closest(".cm-status-bar-select-content"));
     }
@@ -221,25 +214,34 @@ class StatusBarPanel {
      * @param {EditorState} state
      */
     updateEOL(state) {
-        if (state.lineBreak === this.eolVal) return;
-
-        const eolLookup = {
-            "\u000a": ["LF", "Line Feed"],
-            "\u000b": ["VT", "Vertical Tab"],
-            "\u000c": ["FF", "Form Feed"],
-            "\u000d": ["CR", "Carriage Return"],
-            "\u000d\u000a": ["CRLF", "Carriage Return + Line Feed"],
-            "\u0085": ["NEL", "Next Line"],
-            "\u2028": ["LS", "Line Separator"],
-            "\u2029": ["PS", "Paragraph Separator"]
-        };
+        if (this.getEOLState() < 2 && state.lineBreak === this.eolVal) return;
 
         const val = this.dom.querySelector(".eol-value");
         const button = val.closest(".cm-status-bar-select-btn");
-        const eolName = eolLookup[state.lineBreak];
-        val.textContent = eolName[0];
-        button.setAttribute("title", `End of line sequence:<br>${eolName[1]}`);
-        button.setAttribute("data-original-title", `End of line sequence:<br>${eolName[1]}`);
+        let eolCode = eolSeqToCode[state.lineBreak];
+        let eolName = eolCodeToName[eolCode];
+
+        switch (this.getEOLState()) {
+            case 1: // Detected
+                val.classList.add("font-italic");
+                eolCode += " (detected)";
+                eolName += " (detected)";
+                // Pulse
+                val.classList.add("pulse");
+                setTimeout(() => {
+                    val.classList.remove("pulse");
+                }, 2000);
+                break;
+            case 0: // Unset
+            case 2: // Manually set
+            default:
+                val.classList.remove("font-italic");
+                break;
+        }
+
+        val.textContent = eolCode;
+        button.setAttribute("title", `End of line sequence:<br>${eolName}`);
+        button.setAttribute("data-original-title", `End of line sequence:<br>${eolName}`);
         this.eolVal = state.lineBreak;
     }
 
@@ -249,12 +251,30 @@ class StatusBarPanel {
      */
     updateCharEnc() {
         const chrEncVal = this.chrEncGetter();
-        if (chrEncVal === this.chrEncVal) return;
+        if (this.getEncodingState() < 2 && chrEncVal === this.chrEncVal) return;
 
-        const name = CHR_ENC_SIMPLE_REVERSE_LOOKUP[chrEncVal] ? CHR_ENC_SIMPLE_REVERSE_LOOKUP[chrEncVal] : "Raw Bytes";
+        let name = CHR_ENC_SIMPLE_REVERSE_LOOKUP[chrEncVal] ? CHR_ENC_SIMPLE_REVERSE_LOOKUP[chrEncVal] : "Raw Bytes";
 
         const val = this.dom.querySelector(".chr-enc-value");
         const button = val.closest(".cm-status-bar-select-btn");
+
+        switch (this.getEncodingState()) {
+            case 1: // Detected
+                val.classList.add("font-italic");
+                name += " (detected)";
+                // Pulse
+                val.classList.add("pulse");
+                setTimeout(() => {
+                    val.classList.remove("pulse");
+                }, 2000);
+                break;
+            case 0: // Unset
+            case 2: // Manually set
+            default:
+                val.classList.remove("font-italic");
+                break;
+        }
+
         val.textContent = name;
         button.setAttribute("title", `${this.label} character encoding:<br>${name}`);
         button.setAttribute("data-original-title", `${this.label} character encoding:<br>${name}`);
@@ -275,7 +295,6 @@ class StatusBarPanel {
             bakingTime.textContent = this.timing.duration(this.tabNumGetter());
 
             const info = this.timing.printStages(this.tabNumGetter()).replace(/\n/g, "<br>");
-            bakingTimeInfo.setAttribute("title", info);
             bakingTimeInfo.setAttribute("data-original-title", info);
         } else {
             bakingTimeInfo.style.display = "none";

src/web/waiters/ControlsWaiter.mjs

@@ -5,6 +5,7 @@
  */
 
 import Utils from "../../core/Utils.mjs";
+import { eolSeqToCode } from "../utils/editorUtils.mjs";
 
 
 /**
@@ -140,16 +141,16 @@ class ControlsWaiter {
 
         const inputChrEnc = this.manager.input.getChrEnc();
         const outputChrEnc = this.manager.output.getChrEnc();
-        const inputEOLSeq = this.manager.input.getEOLSeq();
-        const outputEOLSeq = this.manager.output.getEOLSeq();
+        const inputEOL = eolSeqToCode[this.manager.input.getEOLSeq()];
+        const outputEOL = eolSeqToCode[this.manager.output.getEOLSeq()];
 
         const params = [
             includeRecipe ? ["recipe", recipeStr] : undefined,
             includeInput && input.length ? ["input", Utils.escapeHtml(input)] : undefined,
             inputChrEnc !== 0 ? ["ienc", inputChrEnc] : undefined,
             outputChrEnc !== 0 ? ["oenc", outputChrEnc] : undefined,
-            inputEOLSeq !== "\n" ? ["ieol", inputEOLSeq] : undefined,
-            outputEOLSeq !== "\n" ? ["oeol", outputEOLSeq] : undefined
+            inputEOL !== "LF" ? ["ieol", inputEOL] : undefined,
+            outputEOL !== "LF" ? ["oeol", outputEOL] : undefined
         ];
 
         const hash = params

src/web/waiters/InputWaiter.mjs

@@ -42,7 +42,7 @@ import {
 
 import {statusBar} from "../utils/statusBar.mjs";
 import {fileDetailsPanel} from "../utils/fileDetails.mjs";
-import {renderSpecialChar} from "../utils/editorUtils.mjs";
+import {eolCodeToSeq, eolCodeToName, renderSpecialChar} from "../utils/editorUtils.mjs";
 
 
 /**
@@ -62,6 +62,8 @@ class InputWaiter {
 
         this.inputTextEl = document.getElementById("input-text");
         this.inputChrEnc = 0;
+        this.eolState = 0; // 0 = unset, 1 = detected, 2 = manual
+        this.encodingState = 0; // 0 = unset, 1 = detected, 2 = manual
         this.initEditor();
 
         this.inputWorker = null;
@@ -92,6 +94,7 @@ class InputWaiter {
             fileDetailsPanel: new Compartment
         };
 
+        const self = this;
         const initialState = EditorState.create({
             doc: null,
             extensions: [
@@ -114,7 +117,9 @@ class InputWaiter {
                     label: "Input",
                     eolHandler: this.eolChange.bind(this),
                     chrEncHandler: this.chrEncChange.bind(this),
-                    chrEncGetter: this.getChrEnc.bind(this)
+                    chrEncGetter: this.getChrEnc.bind(this),
+                    getEncodingState: this.getEncodingState.bind(this),
+                    getEOLState: this.getEOLState.bind(this)
                 }),
 
                 // Mutable state
@@ -141,10 +146,21 @@ class InputWaiter {
                     if (e.docChanged && !this.silentInputChange)
                         this.inputChange(e);
                     this.silentInputChange = false;
+                }),
+
+                // Event handlers
+                EditorView.domEventHandlers({
+                    paste(event, view) {
+                        setTimeout(() => {
+                            self.afterPaste(event);
+                        });
+                    }
                 })
             ]
         });
 
+
+        if (this.inputEditorView) this.inputEditorView.destroy();
         this.inputEditorView = new EditorView({
             state: initialState,
             parent: this.inputTextEl
@@ -154,12 +170,23 @@ class InputWaiter {
     /**
      * Handler for EOL change events
      * Sets the line separator
-     * @param {string} eolVal
+     * @param {string} eol
+     * @param {boolean} [manual=false]
      */
-    eolChange(eolVal) {
-        const oldInputVal = this.getInput();
+    eolChange(eol, manual=false) {
+        const eolVal = eolCodeToSeq[eol];
+        if (eolVal === undefined) return;
+
+        this.eolState = manual ? 2 : this.eolState;
+        if (this.eolState < 2 && eolVal === this.getEOLSeq()) return;
+
+        if (this.eolState === 1) {
+            // Alert
+            this.app.alert(`Input end of line separator has been detected and changed to ${eolCodeToName[eol]}`, 5000);
+        }
 
         // Update the EOL value
+        const oldInputVal = this.getInput();
         this.inputEditorView.dispatch({
             effects: this.inputEditorConf.eol.reconfigure(EditorState.lineSeparator.of(eolVal))
         });
@@ -176,14 +203,24 @@ class InputWaiter {
         return this.inputEditorView.state.lineBreak;
     }
 
+    /**
+     * Returns whether the input EOL sequence was set manually or has been detected automatically
+     * @returns {number} - 0 = unset, 1 = detected, 2 = manual
+     */
+    getEOLState() {
+        return this.eolState;
+    }
+
     /**
      * Handler for Chr Enc change events
      * Sets the input character encoding
      * @param {number} chrEncVal
+     * @param {boolean} [manual=false]
      */
-    chrEncChange(chrEncVal) {
+    chrEncChange(chrEncVal, manual=false) {
         if (typeof chrEncVal !== "number") return;
         this.inputChrEnc = chrEncVal;
+        this.encodingState = manual ? 2 : this.encodingState;
         this.inputChange();
     }
 
@@ -195,6 +232,14 @@ class InputWaiter {
         return this.inputChrEnc;
     }
 
+    /**
+     * Returns whether the input character encoding was set manually or has been detected automatically
+     * @returns {number} - 0 = unset, 1 = detected, 2 = manual
+     */
+    getEncodingState() {
+        return this.encodingState;
+    }
+
     /**
      * Sets word wrap on the input editor
      * @param {boolean} wrap
@@ -866,6 +911,55 @@ class InputWaiter {
         }, delay, "inputChange", this, [e])();
     }
 
+    /**
+     * Handler that fires just after input paste events.
+     * Checks whether the EOL separator or character encoding should be updated.
+     *
+     * @param {event} e
+     */
+    afterPaste(e) {
+        // If EOL has been fixed, skip this.
+        if (this.eolState > 1) return;
+
+        const inputText = this.getInput();
+
+        // Detect most likely EOL sequence
+        const eolCharCounts = {
+            "LF": inputText.count("\u000a"),
+            "VT": inputText.count("\u000b"),
+            "FF": inputText.count("\u000c"),
+            "CR": inputText.count("\u000d"),
+            "CRLF": inputText.count("\u000d\u000a"),
+            "NEL": inputText.count("\u0085"),
+            "LS": inputText.count("\u2028"),
+            "PS": inputText.count("\u2029")
+        };
+
+        // If all zero, leave alone
+        const total = Object.values(eolCharCounts).reduce((acc, curr) => {
+            return acc + curr;
+        }, 0);
+        if (total === 0) return;
+
+        // Find most prevalent line ending sequence
+        const highest = Object.entries(eolCharCounts).reduce((acc, curr) => {
+            return curr[1] > acc[1] ? curr : acc;
+        }, ["LF", 0]);
+        let choice = highest[0];
+
+        // If CRLF not zero and more than half the highest alternative, choose CRLF
+        if ((eolCharCounts.CRLF * 2) > highest[1]) {
+            choice = "CRLF";
+        }
+
+        const eolVal = eolCodeToSeq[choice];
+        if (eolVal === this.getEOLSeq()) return;
+
+        // Setting automatically
+        this.eolState = 1;
+        this.eolChange(choice);
+    }
+
     /**
      * Handler for input dragover events.
      * Gives the user a visual cue to show that items can be dropped here.
@@ -1199,6 +1293,14 @@ class InputWaiter {
         this.manager.output.removeAllOutputs();
         this.manager.output.terminateZipWorker();
 
+        this.eolState = 0;
+        this.encodingState = 0;
+        this.manager.output.eolState = 0;
+        this.manager.output.encodingState = 0;
+
+        this.initEditor();
+        this.manager.output.initEditor();
+
         const tabsList = document.getElementById("input-tabs");
         const tabsListChildren = tabsList.children;
 

src/web/waiters/OutputWaiter.mjs

@@ -7,6 +7,7 @@
 
 import Utils, {debounce} from "../../core/Utils.mjs";
 import Dish from "../../core/Dish.mjs";
+import {isUTF8, CHR_ENC_SIMPLE_REVERSE_LOOKUP} from "../../core/lib/ChrEnc.mjs";
 import {detectFileType} from "../../core/lib/FileType.mjs";
 import FileSaver from "file-saver";
 import ZipWorker from "worker-loader?inline=no-fallback!../workers/ZipWorker.mjs";
@@ -38,7 +39,7 @@ import {
 import {statusBar} from "../utils/statusBar.mjs";
 import {htmlPlugin} from "../utils/htmlWidget.mjs";
 import {copyOverride} from "../utils/copyOverride.mjs";
-import {renderSpecialChar} from "../utils/editorUtils.mjs";
+import {eolCodeToSeq, eolCodeToName, renderSpecialChar} from "../utils/editorUtils.mjs";
 
 
 /**
@@ -70,6 +71,8 @@ class OutputWaiter {
         this.zipWorker = null;
         this.maxTabs = this.manager.tabs.calcMaxTabs();
         this.tabTimeout = null;
+        this.eolState = 0; // 0 = unset, 1 = detected, 2 = manual
+        this.encodingState = 0; // 0 = unset, 1 = detected, 2 = manual
     }
 
     /**
@@ -109,6 +112,8 @@ class OutputWaiter {
                     eolHandler: this.eolChange.bind(this),
                     chrEncHandler: this.chrEncChange.bind(this),
                     chrEncGetter: this.getChrEnc.bind(this),
+                    getEncodingState: this.getEncodingState.bind(this),
+                    getEOLState: this.getEOLState.bind(this),
                     htmlOutput: this.htmlOutput
                 }),
                 htmlPlugin(this.htmlOutput),
@@ -137,6 +142,7 @@ class OutputWaiter {
             ]
         });
 
+        if (this.outputEditorView) this.outputEditorView.destroy();
         this.outputEditorView = new EditorView({
             state: initialState,
             parent: this.outputTextEl
@@ -146,9 +152,21 @@ class OutputWaiter {
     /**
      * Handler for EOL change events
      * Sets the line separator
-     * @param {string} eolVal
+     * @param {string} eol
+     * @param {boolean} [manual=false]
      */
-    async eolChange(eolVal) {
+    async eolChange(eol, manual=false) {
+        const eolVal = eolCodeToSeq[eol];
+        if (eolVal === undefined) return;
+
+        this.eolState = manual ? 2 : this.eolState;
+        if (this.eolState < 2 && eolVal === this.getEOLSeq()) return;
+
+        if (this.eolState === 1) {
+            // Alert
+            this.app.alert(`Output end of line separator has been detected and changed to ${eolCodeToName[eol]}`, 5000);
+        }
+
         const currentTabNum = this.manager.tabs.getActiveTab("output");
         if (currentTabNum >= 0) {
             this.outputs[currentTabNum].eolSequence = eolVal;
@@ -180,13 +198,23 @@ class OutputWaiter {
         return this.outputs[currentTabNum].eolSequence;
     }
 
+    /**
+     * Returns whether the output EOL sequence was set manually or has been detected automatically
+     * @returns {number} - 0 = unset, 1 = detected, 2 = manual
+     */
+    getEOLState() {
+        return this.eolState;
+    }
+
     /**
      * Handler for Chr Enc change events
      * Sets the output character encoding
      * @param {number} chrEncVal
+     * @param {boolean} [manual=false]
      */
-    async chrEncChange(chrEncVal) {
+    async chrEncChange(chrEncVal, manual=false) {
         if (typeof chrEncVal !== "number") return;
+        const currentEnc = this.getChrEnc();
 
         const currentTabNum = this.manager.tabs.getActiveTab("output");
         if (currentTabNum >= 0) {
@@ -195,10 +223,17 @@ class OutputWaiter {
             throw new Error(`Cannot change output ${currentTabNum} chrEnc to ${chrEncVal}`);
         }
 
-        // Reset the output, forcing it to re-decode the data with the new character encoding
-        await this.setOutput(this.currentOutputCache, true);
-        // Update the URL manually since we aren't firing a statechange event
-        this.app.updateURL(true);
+        this.encodingState = manual ? 2 : this.encodingState;
+
+        if (this.encodingState > 1) {
+            // Reset the output, forcing it to re-decode the data with the new character encoding
+            await this.setOutput(this.currentOutputCache, true);
+            // Update the URL manually since we aren't firing a statechange event
+            this.app.updateURL(true);
+        } else if (currentEnc !== chrEncVal) {
+            // Alert
+            this.app.alert(`Output character encoding has been detected and changed to ${CHR_ENC_SIMPLE_REVERSE_LOOKUP[chrEncVal] || "Raw Bytes"}`, 5000);
+        }
     }
 
     /**
@@ -213,6 +248,14 @@ class OutputWaiter {
         return this.outputs[currentTabNum].encoding;
     }
 
+    /**
+     * Returns whether the output character encoding was set manually or has been detected automatically
+     * @returns {number} - 0 = unset, 1 = detected, 2 = manual
+     */
+    getEncodingState() {
+        return this.encodingState;
+    }
+
     /**
      * Sets word wrap on the output editor
      * @param {boolean} wrap
@@ -248,6 +291,7 @@ class OutputWaiter {
         const tabNum = this.manager.tabs.getActiveTab("output");
         this.manager.timing.recordTime("outputDecodingStart", tabNum);
         if (data instanceof ArrayBuffer) {
+            await this.detectEncoding(data);
             data = await this.bufferToStr(data);
         }
         this.manager.timing.recordTime("outputDecodingEnd", tabNum);
@@ -276,6 +320,9 @@ class OutputWaiter {
         // If turning word wrap off, do it before we populate the editor for performance reasons
         if (!wrap) this.setWordWrap(wrap);
 
+        // Detect suitable EOL sequence
+        this.detectEOLSequence(data);
+
         // We use setTimeout here to delay the editor dispatch until the next event cycle,
         // ensuring all async actions have completed before attempting to set the contents
         // of the editor. This is mainly with the above call to setWordWrap() in mind.
@@ -345,6 +392,85 @@ class OutputWaiter {
         });
     }
 
+    /**
+     * Checks whether the EOL separator should be updated
+     *
+     * @param {string} data
+     */
+    detectEOLSequence(data) {
+        // If EOL has been fixed, skip this.
+        if (this.eolState > 1) return;
+        // If data is too long, skip this.
+        if (data.length > 1000000) return;
+
+        // Detect most likely EOL sequence
+        const eolCharCounts = {
+            "LF": data.count("\u000a"),
+            "VT": data.count("\u000b"),
+            "FF": data.count("\u000c"),
+            "CR": data.count("\u000d"),
+            "CRLF": data.count("\u000d\u000a"),
+            "NEL": data.count("\u0085"),
+            "LS": data.count("\u2028"),
+            "PS": data.count("\u2029")
+        };
+
+        // If all zero, leave alone
+        const total = Object.values(eolCharCounts).reduce((acc, curr) => {
+            return acc + curr;
+        }, 0);
+        if (total === 0) return;
+
+        // Find most prevalent line ending sequence
+        const highest = Object.entries(eolCharCounts).reduce((acc, curr) => {
+            return curr[1] > acc[1] ? curr : acc;
+        }, ["LF", 0]);
+        let choice = highest[0];
+
+        // If CRLF not zero and more than half the highest alternative, choose CRLF
+        if ((eolCharCounts.CRLF * 2) > highest[1]) {
+            choice = "CRLF";
+        }
+
+        const eolVal = eolCodeToSeq[choice];
+        if (eolVal === this.getEOLSeq()) return;
+
+        // Setting automatically
+        this.eolState = 1;
+        this.eolChange(choice);
+    }
+
+    /**
+     * Checks whether the character encoding should be updated.
+     *
+     * @param {ArrayBuffer} data
+     */
+    async detectEncoding(data) {
+        // If encoding has been fixed, skip this.
+        if (this.encodingState > 1) return;
+        // If data is too long, skip this.
+        if (data.byteLength > 1000000) return;
+
+        const enc = isUTF8(data); // 0 = not UTF8, 1 = ASCII, 2 = UTF8
+
+        switch (enc) {
+            case 0: // not UTF8
+                // Set to Raw Bytes
+                this.encodingState = 1;
+                await this.chrEncChange(0, false);
+                break;
+            case 2: // UTF8
+                // Set to UTF8
+                this.encodingState = 1;
+                await this.chrEncChange(65001, false);
+                break;
+            case 1: // ASCII
+            default:
+                // Ignore
+                break;
+        }
+    }
+
     /**
      * Calculates the maximum number of tabs to display
      */

tests/browser/00_nightwatch.js

@@ -194,6 +194,9 @@ module.exports = {
 
         // Open category
         browser
+            .useCss()
+            .waitForElementNotVisible("#snackbar-container", 10000)
+            .useXpath()
             .click(otherCat)
             .expect.element(genUUID).to.be.visible;
 
@@ -230,8 +233,8 @@ module.exports = {
 
         // Alert bar shows and contains correct content
         browser
+            .waitForElementNotVisible("#snackbar-container")
             .click("#copy-output")
-            .waitForElementVisible("#snackbar-container")
             .waitForElementVisible("#snackbar-container .snackbar-content")
             .expect.element("#snackbar-container .snackbar-content").text.to.equal("Copied raw output successfully.");
 

tests/browser/01_io.js

@@ -545,8 +545,8 @@ module.exports = {
         browser.expect.element("#output-text .cm-status-bar .stats-lines-value").text.to.equal("2");
 
         /* Line endings appear in the URL */
-        browser.assert.urlContains("ieol=%0D%0A");
-        browser.assert.urlContains("oeol=%0D");
+        browser.assert.urlContains("ieol=CRLF");
+        browser.assert.urlContains("oeol=CR");
 
         /* Preserved when changing tabs */
         browser
@@ -643,7 +643,7 @@ module.exports = {
     "Loading from URL": browser => {
         /* Complex deep link populates the input correctly (encoding, eol, input) */
         browser
-            .urlHash("recipe=To_Base64('A-Za-z0-9%2B/%3D')&input=VGhlIHNoaXBzIGh1bmcgaW4gdGhlIHNreSBpbiBtdWNoIHRoZSBzYW1lIHdheSB0aGF0IGJyaWNrcyBkb24ndC4M&ienc=21866&oenc=1201&ieol=%0C&oeol=%E2%80%A9")
+            .urlHash("recipe=To_Base64('A-Za-z0-9%2B/%3D')&input=VGhlIHNoaXBzIGh1bmcgaW4gdGhlIHNreSBpbiBtdWNoIHRoZSBzYW1lIHdheSB0aGF0IGJyaWNrcyBkb24ndC4M&ienc=21866&oenc=1201&ieol=FF&oeol=PS")
             .waitForElementVisible("#rec-list li.operation");
 
         browser.expect.element(`#input-text .cm-content`).to.have.property("textContent").match(/^.{65}$/);

tests/browser/02_ops.js

@@ -108,7 +108,7 @@ module.exports = {
         // testOp(browser, "Derive EVP key", "test input", "test_output");
         // testOp(browser, "Derive PBKDF2 key", "test input", "test_output");
         // testOp(browser, "Detect File Type", "test input", "test_output");
-        testOpHtml(browser, "Diff", "The cat sat on the mat\n\nThe mat cat on the sat", ".hl5:first-child", "mat", ["\\n\\n", "Word", true, true, false, false]);
+        testOpHtml(browser, "Diff", "The cat sat on the mat\n\nThe mat cat on the sat", "ins:first-child", "mat", ["\\n\\n", "Word", true, true, false, false]);
         // testOp(browser, "Disassemble x86", "test input", "test_output");
         testOpImage(browser, "Dither Image", "files/Hitchhikers_Guide.jpeg");
     // testOp(browser, "Divide", "test input", "test_output");

tests/browser/browserUtils.js

@@ -65,6 +65,7 @@ function setChrEnc(browser, io, enc) {
     io = `#${io}-text`;
     browser
         .useCss()
+        .waitForElementNotVisible("#snackbar-container", 6000)
         .click(io + " .chr-enc-value")
         .waitForElementVisible(io + " .chr-enc-select .cm-status-bar-select-scroll")
         .click("link text", enc)
@@ -83,6 +84,7 @@ function setEOLSeq(browser, io, eol) {
     io = `#${io}-text`;
     browser
         .useCss()
+        .waitForElementNotVisible("#snackbar-container", 6000)
         .click(io + " .eol-value")
         .waitForElementVisible(io + " .eol-select .cm-status-bar-select-content")
         .click(`${io} .cm-status-bar-select-content a[data-val=${eol}]`)

tests/node/tests/nodeApi.mjs

@@ -136,7 +136,7 @@ TestRegister.addApiTests([
 
     it("chef.help: returns multiple results", () => {
         const result = chef.help("base 64");
-        assert.strictEqual(result.length, 11);
+        assert.strictEqual(result.length, 13);
     }),
 
     it("chef.help: looks in description for matches too", () => {

tests/node/tests/operations.mjs

@@ -635,6 +635,10 @@ WWFkYSBZYWRh\r
         assert.strictEqual(chef.keccak("Flea Market").toString(), "c2a06880b19e453ee5440e8bd4c2024bedc15a6630096aa3f609acfd2b8f15f27cd293e1cc73933e81432269129ce954a6138889ce87831179d55dcff1cc7587");
     }),
 
+    it("LZNT1 Decompress", () => {
+        assert.strictEqual(chef.LZNT1Decompress("\x1a\xb0\x00compress\x00edtestda\x04ta\x07\x88alot").toString(), "compressedtestdatacompressedalot");
+    }),
+
     it("MD6", () => {
         assert.strictEqual(chef.MD6("Head Over Heels", {key: "arty"}).toString(), "d8f7fe4931fbaa37316f76283d5f615f50ddd54afdc794b61da522556aee99ad");
     }),

tests/operations/index.mjs

@@ -17,127 +17,132 @@ import {
 } from "../lib/utils.mjs";
 
 import TestRegister from "../lib/TestRegister.mjs";
-import "./tests/BCD.mjs";
-import "./tests/BSON.mjs";
+import "./tests/AESKeyWrap.mjs";
+import "./tests/AvroToJSON.mjs";
 import "./tests/BaconCipher.mjs";
 import "./tests/Base45.mjs";
 import "./tests/Base58.mjs";
-import "./tests/Base64.mjs";
 import "./tests/Base62.mjs";
+import "./tests/Base64.mjs";
 import "./tests/Base85.mjs";
+import "./tests/Base92.mjs";
+import "./tests/BCD.mjs";
 import "./tests/BitwiseOp.mjs";
+import "./tests/BLAKE2b.mjs";
+import "./tests/BLAKE2s.mjs";
+import "./tests/Bombe.mjs";
+import "./tests/BSON.mjs";
 import "./tests/ByteRepr.mjs";
+import "./tests/CaesarBoxCipher.mjs";
+import "./tests/CaretMdecode.mjs";
 import "./tests/CartesianProduct.mjs";
-import "./tests/CetaceanCipherEncode.mjs";
+import "./tests/CBORDecode.mjs";
+import "./tests/CBOREncode.mjs";
 import "./tests/CetaceanCipherDecode.mjs";
+import "./tests/CetaceanCipherEncode.mjs";
 import "./tests/ChaCha.mjs";
-import "./tests/CharEnc.mjs";
 import "./tests/ChangeIPFormat.mjs";
+import "./tests/CharEnc.mjs";
 import "./tests/Charts.mjs";
 import "./tests/Checksum.mjs";
 import "./tests/Ciphers.mjs";
+import "./tests/CipherSaber2.mjs";
+import "./tests/CMAC.mjs";
 import "./tests/Code.mjs";
+import "./tests/Colossus.mjs";
 import "./tests/Comment.mjs";
 import "./tests/Compress.mjs";
 import "./tests/ConditionalJump.mjs";
+import "./tests/ConvertCoordinateFormat.mjs";
+import "./tests/ConvertToNATOAlphabet.mjs";
 import "./tests/Crypt.mjs";
 import "./tests/CSV.mjs";
 import "./tests/DateTime.mjs";
+import "./tests/DefangIP.mjs";
+import "./tests/ELFInfo.mjs";
+import "./tests/Enigma.mjs";
 import "./tests/ExtractEmailAddresses.mjs";
+import "./tests/FileTree.mjs";
+import "./tests/FletcherChecksum.mjs";
 import "./tests/Fork.mjs";
 import "./tests/FromDecimal.mjs";
 import "./tests/GenerateAllHashes.mjs";
-import "./tests/Gzip.mjs";
+import "./tests/GenerateDeBruijnSequence.mjs";
+import "./tests/GetAllCasings.mjs";
+import "./tests/GOST.mjs";
 import "./tests/Gunzip.mjs";
+import "./tests/Gzip.mjs";
 import "./tests/Hash.mjs";
+import "./tests/HASSH.mjs";
 import "./tests/HaversineDistance.mjs";
 import "./tests/Hex.mjs";
 import "./tests/Hexdump.mjs";
+import "./tests/HKDF.mjs";
 import "./tests/Image.mjs";
 import "./tests/IndexOfCoincidence.mjs";
-import "./tests/Jump.mjs";
+import "./tests/JA3Fingerprint.mjs";
+import "./tests/JA3SFingerprint.mjs";
 import "./tests/JSONBeautify.mjs";
 import "./tests/JSONMinify.mjs";
 import "./tests/JSONtoCSV.mjs";
+import "./tests/Jump.mjs";
 import "./tests/JWTDecode.mjs";
 import "./tests/JWTSign.mjs";
 import "./tests/JWTVerify.mjs";
-import "./tests/MS.mjs";
+import "./tests/LevenshteinDistance.mjs";
+import "./tests/Lorenz.mjs";
+import "./tests/LS47.mjs";
+import "./tests/LuhnChecksum.mjs";
+import "./tests/LZNT1Decompress.mjs";
+import "./tests/LZString.mjs";
 import "./tests/Magic.mjs";
+import "./tests/Media.mjs";
 import "./tests/MorseCode.mjs";
+import "./tests/MS.mjs";
+import "./tests/MultipleBombe.mjs";
+import "./tests/MurmurHash3.mjs";
 import "./tests/NetBIOS.mjs";
 import "./tests/NormaliseUnicode.mjs";
+import "./tests/NTLM.mjs";
 import "./tests/OTP.mjs";
+import "./tests/ParseIPRange.mjs";
+import "./tests/ParseObjectIDTimestamp.mjs";
+import "./tests/ParseQRCode.mjs";
+import "./tests/ParseSSHHostKey.mjs";
+import "./tests/ParseTCP.mjs";
+import "./tests/ParseTLV.mjs";
+import "./tests/ParseUDP.mjs";
+import "./tests/PEMtoHex.mjs";
 import "./tests/PGP.mjs";
 import "./tests/PHP.mjs";
-import "./tests/ParseIPRange.mjs";
-import "./tests/ParseQRCode.mjs";
-import "./tests/PEMtoHex.mjs";
 import "./tests/PowerSet.mjs";
+import "./tests/Protobuf.mjs";
+import "./tests/Rabbit.mjs";
+import "./tests/RAKE.mjs";
 import "./tests/Regex.mjs";
 import "./tests/Register.mjs";
+import "./tests/RisonEncodeDecode.mjs";
 import "./tests/Rotate.mjs";
+import "./tests/RSA.mjs";
 import "./tests/SeqUtils.mjs";
 import "./tests/SetDifference.mjs";
 import "./tests/SetIntersection.mjs";
 import "./tests/SetUnion.mjs";
+import "./tests/Shuffle.mjs";
+import "./tests/SIGABA.mjs";
 import "./tests/SM4.mjs";
+// import "./tests/SplitColourChannels.mjs"; // Cannot test operations that use the File type yet
 import "./tests/StrUtils.mjs";
+import "./tests/Subsection.mjs";
+import "./tests/SwapCase.mjs";
 import "./tests/SymmetricDifference.mjs";
 import "./tests/TextEncodingBruteForce.mjs";
-import "./tests/TranslateDateTimeFormat.mjs";
-import "./tests/Magic.mjs";
-import "./tests/ParseTLV.mjs";
-import "./tests/Media.mjs";
 import "./tests/ToFromInsensitiveRegex.mjs";
-import "./tests/YARA.mjs";
-import "./tests/ConvertCoordinateFormat.mjs";
-import "./tests/Enigma.mjs";
-import "./tests/Bombe.mjs";
-import "./tests/MultipleBombe.mjs";
+import "./tests/TranslateDateTimeFormat.mjs";
 import "./tests/Typex.mjs";
-import "./tests/BLAKE2b.mjs";
-import "./tests/BLAKE2s.mjs";
-import "./tests/Protobuf.mjs";
-import "./tests/ParseSSHHostKey.mjs";
-import "./tests/DefangIP.mjs";
-import "./tests/ParseUDP.mjs";
-import "./tests/ParseTCP.mjs";
-import "./tests/AvroToJSON.mjs";
-import "./tests/Lorenz.mjs";
-import "./tests/LuhnChecksum.mjs";
-import "./tests/CipherSaber2.mjs";
-import "./tests/Colossus.mjs";
-import "./tests/ParseObjectIDTimestamp.mjs";
-import "./tests/Unicode.mjs";
-import "./tests/RSA.mjs";
-import "./tests/CBOREncode.mjs";
-import "./tests/CBORDecode.mjs";
-import "./tests/JA3Fingerprint.mjs";
-import "./tests/JA3SFingerprint.mjs";
-import "./tests/HASSH.mjs";
-import "./tests/GetAllCasings.mjs";
-import "./tests/SIGABA.mjs";
-import "./tests/ELFInfo.mjs";
-import "./tests/Subsection.mjs";
-import "./tests/CaesarBoxCipher.mjs";
 import "./tests/UnescapeString.mjs";
-import "./tests/LS47.mjs";
-import "./tests/LZString.mjs";
-import "./tests/NTLM.mjs";
-import "./tests/Shuffle.mjs";
-import "./tests/FletcherChecksum.mjs";
-import "./tests/CMAC.mjs";
-import "./tests/AESKeyWrap.mjs";
-import "./tests/Rabbit.mjs";
-import "./tests/LevenshteinDistance.mjs";
-import "./tests/SwapCase.mjs";
-import "./tests/HKDF.mjs";
-import "./tests/GenerateDeBruijnSequence.mjs";
-import "./tests/GOST.mjs";
-
-// Cannot test operations that use the File type yet
-// import "./tests/SplitColourChannels.mjs";
+import "./tests/Unicode.mjs";
+import "./tests/YARA.mjs";
 
 const testStatus = {
     allTestsPassing: true,

tests/operations/tests/Base92.mjs

@@ -0,0 +1,89 @@
+/**
+ * Base92 tests.
+ *
+ * @author sg5506844 [sg5506844@gmail.com]
+ * @copyright Crown Copyright 2021
+ * @license Apache-2.0
+ */
+
+import TestRegister from "../../lib/TestRegister.mjs";
+
+TestRegister.addTests([
+    {
+        name: "To Base92: nothing",
+        input: "",
+        expectedOutput: "",
+        recipeConfig: [
+            {
+                op: "To Base92",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "To Base92: Spec encoding example 1",
+        input: "AB",
+        expectedOutput: "8y2",
+        recipeConfig: [
+            {
+                op: "To Base92",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "To Base92: Spec encoding example 2",
+        input: "Hello!!",
+        expectedOutput: ";K_$aOTo&",
+        recipeConfig: [
+            {
+                op: "To Base92",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "To Base92: Spec encoding example 3",
+        input: "base-92",
+        expectedOutput: "DX2?V<Y(*",
+        recipeConfig: [
+            {
+                op: "To Base92",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "From Base92: nothing",
+        input: "",
+        expectedOutput: "",
+        recipeConfig: [
+            {
+                op: "From Base92",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "From Base92: Spec decoding example 1",
+        input: "G'_DW[B",
+        expectedOutput: "ietf!",
+        recipeConfig: [
+            {
+                op: "From Base92",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "From Base92: Invalid character",
+        input: "~",
+        expectedOutput: "~ is not a base92 character",
+        recipeConfig: [
+            {
+                op: "From Base92",
+                args: [],
+            },
+        ],
+    },
+]);

tests/operations/tests/CaretMdecode.mjs

@@ -0,0 +1,39 @@
+/**
+ * Caesar Box Cipher tests.
+ *
+ * @author tedk [tedk@ted.do]
+ *
+ * @copyright Crown Copyright 2020
+ * @license Apache-2.0
+ */
+import TestRegister from "../../lib/TestRegister.mjs";
+
+TestRegister.addTests([
+    {
+        name: "Caret/M-decode: nothing",
+        input: "",
+        expectedOutput: "",
+        recipeConfig: [
+            {
+                op: "Caret/M-decode",
+                args: [],
+            },
+        ],
+    },
+    {
+        /*
+         * Tests the full range.
+         * Everything except "^_" (\x5e\x5f) will decode correctly.
+         */
+        name: "Caret/M-decode: Full set",
+        input: "^@^A^B^C^D^E^F^G^H^I^J^K^L^M^N^O^P^Q^R^S^T^U^V^W^X^Y^Z^[^\\^]^^^_ !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~^?M-^@M-^AM-^BM-^CM-^DM-^EM-^FM-^GM-^HM-^IM-^JM-^KM-^LM-^MM-^NM-^OM-^PM-^QM-^RM-^SM-^TM-^UM-^VM-^WM-^XM-^YM-^ZM-^[M-^\\M-^]M-^^M-^_M- M-!M-\"M-#M-$M-%M-&M-'M-(M-)M-*M-+M-,M--M-.M-/M-0M-1M-2M-3M-4M-5M-6M-7M-8M-9M-:M-;M-<M-=M->M-?M-@M-AM-BM-CM-DM-EM-FM-GM-HM-IM-JM-KM-LM-MM-NM-OM-PM-QM-RM-SM-TM-UM-VM-WM-XM-YM-ZM-[M-\\M-]M-^M-_M-`M-aM-bM-cM-dM-eM-fM-gM-hM-iM-jM-kM-lM-mM-nM-oM-pM-qM-rM-sM-tM-uM-vM-wM-xM-yM-zM-{M-|M-}M-~M-^?",
+        expectedOutput: "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x1f\x60\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\x8d\x2d\x5f\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+        recipeConfig: [
+            {
+                op: "Caret/M-decode",
+                args: [],
+            },
+        ],
+    },
+]);
+

tests/operations/tests/Fernet.mjs

@@ -0,0 +1,80 @@
+/**
+ * Fernet tests.
+ *
+ * @author Karsten Silkenbäumer [github.com/kassi]
+ * @copyright Karsten Silkenbäumer 2019
+ * @license Apache-2.0
+ */
+import TestRegister from "../TestRegister";
+
+TestRegister.addTests([
+    {
+        name: "Fernet Decrypt: no input",
+        input: "",
+        expectedOutput: "Error: Invalid version",
+        recipeConfig: [
+            {
+                op: "Fernet Decrypt",
+                args: ["MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI="]
+            }
+        ],
+    },
+    {
+        name: "Fernet Decrypt: no secret",
+        input: "gAAAAABce-Tycae8klRxhDX2uenJ-uwV8-A1XZ2HRnfOXlNzkKKfRxviNLlgtemhT_fd1Fw5P_zFUAjd69zaJBQyWppAxVV00SExe77ql8c5n62HYJOnoIU=",
+        expectedOutput: "Error: Secret must be 32 url-safe base64-encoded bytes.",
+        recipeConfig: [
+            {
+                op: "Fernet Decrypt",
+                args: [""]
+            }
+        ],
+    },
+    {
+        name: "Fernet Decrypt: valid arguments",
+        input: "gAAAAABce-Tycae8klRxhDX2uenJ-uwV8-A1XZ2HRnfOXlNzkKKfRxviNLlgtemhT_fd1Fw5P_zFUAjd69zaJBQyWppAxVV00SExe77ql8c5n62HYJOnoIU=",
+        expectedOutput: "This is a secret message.\n",
+        recipeConfig: [
+            {
+                op: "Fernet Decrypt",
+                args: ["VGhpc0lzVGhpcnR5VHdvQ2hhcmFjdGVyc0xvbmdLZXk="]
+            }
+        ],
+    }
+]);
+
+TestRegister.addTests([
+    {
+        name: "Fernet Encrypt: no input",
+        input: "",
+        expectedMatch: /^gAAAAABce-[\w-]+={0,2}$/,
+        recipeConfig: [
+            {
+                op: "Fernet Encrypt",
+                args: ["MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI="]
+            }
+        ],
+    },
+    {
+        name: "Fernet Encrypt: no secret",
+        input: "This is a secret message.\n",
+        expectedOutput: "Error: Secret must be 32 url-safe base64-encoded bytes.",
+        recipeConfig: [
+            {
+                op: "Fernet Encrypt",
+                args: [""]
+            }
+        ],
+    },
+    {
+        name: "Fernet Encrypt: valid arguments",
+        input: "This is a secret message.\n",
+        expectedMatch: /^gAAAAABce-[\w-]+={0,2}$/,
+        recipeConfig: [
+            {
+                op: "Fernet Encrypt",
+                args: ["MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI="]
+            }
+        ],
+    }
+]);

tests/operations/tests/FileTree.mjs

@@ -0,0 +1,22 @@
+/**
+ * File tree tests.
+ *
+ * @author sw5678
+ * @copyright Crown Copyright 2023
+ * @license Apache-2.0
+ */
+import TestRegister from "../../lib/TestRegister.mjs";
+
+TestRegister.addTests([
+    {
+        "name": "File Tree: basic example",
+        "input": "/test_dir1/test_file1.txt\n/test_dir1/test_file2.txt\n/test_dir2/test_file1.txt",
+        "expectedOutput": "test_dir1\n|---test_file1.txt\n|---test_file2.txt\ntest_dir2\n|---test_file1.txt",
+        "recipeConfig": [
+            {
+                "op": "File Tree",
+                "args": ["/", "Line feed"],
+            },
+        ],
+    }
+]);

tests/operations/tests/FromGeohash.mjs

@@ -1,55 +0,0 @@
-/**
- * To Geohash tests
- *
- * @author gchq77703
- * @copyright Crown Copyright 2018
- * @license Apache-2.0
- */
-import TestRegister from "../../lib/TestRegister.mjs";
-
-TestRegister.addTests([
-    {
-        name: "From Geohash",
-        input: "ww8p1r4t8",
-        expectedOutput: "37.83238649368286,112.55838632583618",
-        recipeConfig: [
-            {
-                op: "From Geohash",
-                args: [],
-            },
-        ],
-    },
-    {
-        name: "From Geohash",
-        input: "ww8p1r",
-        expectedOutput: "37.83416748046875,112.5604248046875",
-        recipeConfig: [
-            {
-                op: "From Geohash",
-                args: [],
-            },
-        ],
-    },
-    {
-        name: "From Geohash",
-        input: "ww8",
-        expectedOutput: "37.265625,113.203125",
-        recipeConfig: [
-            {
-                op: "From Geohash",
-                args: [],
-            },
-        ],
-    },
-    {
-        name: "From Geohash",
-        input: "w",
-        expectedOutput: "22.5,112.5",
-        recipeConfig: [
-            {
-                op: "From Geohash",
-                args: [],
-            },
-        ],
-    },
-]);

tests/operations/tests/LZNT1Decompress.mjs

@@ -0,0 +1,22 @@
+/**
+ * LZNT1 Decompress tests.
+ *
+ * @author 0xThiebaut [thiebaut.dev]
+ * @copyright Crown Copyright 2023
+ * @license Apache-2.0
+ */
+import TestRegister from "../../lib/TestRegister.mjs";
+
+TestRegister.addTests([
+    {
+        name: "LZNT1 Decompress",
+        input: "\x1a\xb0\x00compress\x00edtestda\x04ta\x07\x88alot",
+        expectedOutput: "compressedtestdatacompressedalot",
+        recipeConfig: [
+            {
+                op: "LZNT1 Decompress",
+                args: []
+            }
+        ],
+    }
+]);

tests/operations/tests/MurmurHash3.mjs

@@ -0,0 +1,77 @@
+/**
+ * MurmurHash3 tests
+ * @author AliceGrey [alice@grey.systems]
+ * @copyright Crown Copyright 2024
+ * @license Apache-2.0
+ */
+
+import TestRegister from "../../lib/TestRegister.mjs";
+
+TestRegister.addTests([
+    {
+        name: "To MurmurHash3: nothing",
+        input: "",
+        expectedOutput: "0",
+        recipeConfig: [
+            {
+                op: "MurmurHash3",
+                args: [0],
+            },
+        ],
+    },
+    {
+        name: "To MurmurHash3: 1",
+        input: "1",
+        expectedOutput: "2484513939",
+        recipeConfig: [
+            {
+                op: "MurmurHash3",
+                args: [0],
+            },
+        ],
+    },
+    {
+        name: "To MurmurHash3: Hello World!",
+        input: "Hello World!",
+        expectedOutput: "3691591037",
+        recipeConfig: [
+            {
+                op: "MurmurHash3",
+                args: [0],
+            },
+        ],
+    },
+    {
+        name: "To MurmurHash3: Hello World! with seed",
+        input: "Hello World!",
+        expectedOutput: "1148600031",
+        recipeConfig: [
+            {
+                op: "MurmurHash3",
+                args: [1337],
+            },
+        ],
+    },
+    {
+        name: "To MurmurHash3: foo",
+        input: "foo",
+        expectedOutput: "4138058784",
+        recipeConfig: [
+            {
+                op: "MurmurHash3",
+                args: [0],
+            },
+        ],
+    },
+    {
+        name: "To MurmurHash3: foo signed",
+        input: "foo",
+        expectedOutput: "-156908512",
+        recipeConfig: [
+            {
+                op: "MurmurHash3",
+                args: [0, true],
+            },
+        ],
+    }
+]);

tests/operations/tests/RAKE.mjs

@@ -0,0 +1,22 @@
+/**
+ * RAKE, Rapid Automatic Keyword Extraction tests.
+ *
+ * @author sw5678
+ * @copyright Crown Copyright 2024
+ * @license Apache-2.0
+ */
+import TestRegister from "../../lib/TestRegister.mjs";
+
+TestRegister.addTests([
+    {
+        "name": "RAKE: Basic Example",
+        "input": "test1 test2. test2",
+        "expectedOutput": "Scores: , Keywords: \n3.5, test1 test2\n1.5, test2",
+        "recipeConfig": [
+            {
+                "op": "RAKE",
+                "args": ["\\s", "\\.\\s|\\n", "i,me,my,myself,we,our"]
+            },
+        ],
+    }
+]);

tests/operations/tests/RisonEncodeDecode.mjs

@@ -0,0 +1,66 @@
+/**
+ * @author sg5506844 [sg5506844@gmail.com]
+ * @copyright Crown Copyright 2021
+ * @license Apache-2.0
+ */
+
+import TestRegister from "../../lib/TestRegister.mjs";
+
+TestRegister.addTests([
+    {
+        name: "Rison Encode: Encoding example 1",
+        input: JSON.stringify({ any: "json", yes: true }),
+        expectedOutput: "(any:json,yes:!t)",
+        recipeConfig: [
+            {
+                op: "Rison Encode",
+                args: ["Encode"]
+            }
+        ]
+    },
+    {
+        name: "Rison Encode: Encoding example 2",
+        input: JSON.stringify({ supportsObjects: true, ints: 435 }),
+        expectedOutput: "ints:435,supportsObjects:!t",
+        recipeConfig: [
+            {
+                op: "Rison Encode",
+                args: ["Encode Object"]
+            }
+        ]
+    },
+    {
+        name: "Rison Encode: Encoding example 3",
+        input: JSON.stringify(["A", "B", { supportsObjects: true }]),
+        expectedOutput: "A,B,(supportsObjects:!t)",
+        recipeConfig: [
+            {
+                op: "Rison Encode",
+                args: ["Encode Array"]
+            }
+        ]
+    },
+    {
+        name: "Rison Encode: Object for an array",
+        input: JSON.stringify({ supportsObjects: true, ints: 435 }),
+        expectedOutput: "Rison Encode - rison.encode_array expects an array argument",
+        expectedError: "Rison Encode - rison.encode_array expects an array argument",
+        recipeConfig: [
+            {
+                op: "Rison Encode",
+                args: ["Encode Array"]
+            }
+        ]
+    },
+    {
+        name: "Rison Decode: Decoding example 1",
+        input: "(any:json,yes:!t)",
+        expectedOutput: JSON.stringify({ any: "json", yes: true }, null, 4),
+        recipeConfig: [
+            {
+                op: "Rison Decode",
+                args: ["Decode"]
+            }
+        ]
+    }
+]);

tests/operations/tests/StrUtils.mjs

@@ -11,7 +11,7 @@ TestRegister.addTests([
     {
         name: "Diff, basic usage",
         input: "testing23\n\ntesting123",
-        expectedOutput: "testing<span class='hl5'>1</span>23",
+        expectedOutput: "testing<ins>1</ins>23",
         recipeConfig: [
             {
                 "op": "Diff",
@@ -22,7 +22,7 @@ TestRegister.addTests([
     {
         name: "Diff added with subtraction, basic usage",
         input: "testing23\n\ntesting123",
-        expectedOutput: "<span class='hl5'>1</span>",
+        expectedOutput: "<ins>1</ins>",
         recipeConfig: [
             {
                 "op": "Diff",
@@ -33,7 +33,7 @@ TestRegister.addTests([
     {
         name: "Diff removed with subtraction, basic usage",
         input: "testing123\n\ntesting3",
-        expectedOutput: "<span class='hl3'>12</span>",
+        expectedOutput: "<del>12</del>",
         recipeConfig: [
             {
                 "op": "Diff",

tests/operations/tests/ToGeohash.mjs

@@ -1,55 +0,0 @@
-/**
- * To Geohash tests
- *
- * @author gchq77703
- * @copyright Crown Copyright 2018
- * @license Apache-2.0
- */
-import TestRegister from "../../lib/TestRegister.mjs";
-
-TestRegister.addTests([
-    {
-        name: "To Geohash",
-        input: "37.8324,112.5584",
-        expectedOutput: "ww8p1r4t8",
-        recipeConfig: [
-            {
-                op: "To Geohash",
-                args: [9],
-            },
-        ],
-    },
-    {
-        name: "To Geohash",
-        input: "37.9324,-112.2584",
-        expectedOutput: "9w8pv3ruj",
-        recipeConfig: [
-            {
-                op: "To Geohash",
-                args: [9],
-            },
-        ],
-    },
-    {
-        name: "To Geohash",
-        input: "37.8324,112.5584",
-        expectedOutput: "ww8",
-        recipeConfig: [
-            {
-                op: "To Geohash",
-                args: [3],
-            },
-        ],
-    },
-    {
-        name: "To Geohash",
-        input: "37.9324,-112.2584",
-        expectedOutput: "9w8pv3rujxy5b99",
-        recipeConfig: [
-            {
-                op: "To Geohash",
-                args: [15],
-            },
-        ],
-    },
-]);