8.26.3

Fix disassembler issue properly

.babelrc

@@ -1,19 +0,0 @@
-{
-    "presets": [
-        ["env", {
-            "targets": {
-                "chrome": 40,
-                "firefox": 35,
-                "edge": 14,
-                "node": "6.5"
-            },
-            "modules": false,
-            "useBuiltIns": true
-        }]
-    ],
-    "plugins": [
-        ["babel-plugin-transform-builtin-extend", {
-            "globals": ["Error"]
-        }]
-    ]
-}

.editorconfig

@@ -9,6 +9,6 @@ trim_trailing_whitespace = true
 indent_style = space
 indent_size = 4
 
-[{package.json,.travis.yml}]
+[{package.json,.travis.yml,nightwatch.json}]
 indent_style = space
 indent_size = 2

.eslintrc.json

@@ -1,10 +1,12 @@
 {
+    "parser": "babel-eslint",
     "parserOptions": {
         "ecmaVersion": 9,
         "ecmaFeatures": {
             "impliedStrict": true
         },
-        "sourceType": "module"
+        "sourceType": "module",
+        "allowImportExportEverywhere": true
     },
     "env": {
         "browser": true,
@@ -87,10 +89,20 @@
         "no-var": "error",
         "prefer-const": "error"
     },
+    "overrides": [
+        {
+            "files": "tests/**/*",
+            "rules": {
+                "no-unused-expressions": "off",
+                "no-console": "off"
+            }
+        }
+    ],
     "globals": {
         "$": false,
         "jQuery": false,
         "log": false,
+        "app": false,
 
         "COMPILE_TIME": false,
         "COMPILE_MSG": false,

.gitignore

@@ -9,4 +9,5 @@ docs/*
 src/core/config/modules/*
 src/core/config/OperationConfig.json
 src/core/operations/index.mjs
+tests/browser/output/*
 

.travis.yml

@@ -1,15 +1,19 @@
 language: node_js
 node_js:
   - node
+addons:
+  chrome: stable
 install: npm install
 before_script:
   - npm install -g grunt
+  - export NODE_OPTIONS=--max_old_space_size=2048
 script:
   - grunt lint
   - grunt test
   - grunt docs
   - grunt node
   - grunt prod --msg="$COMPILE_MSG"
+  - xvfb-run --server-args="-screen 0 1200x800x24" grunt testui
 before_deploy:
   - grunt exec:sitemap
   - grunt copy:ghPages
@@ -36,7 +40,7 @@ deploy:
     skip_cleanup: true
     email: "n1474335@gmail.com"
     api_key:
-      secure: "Z3FK6bm4RfQEIRXZ1lBNzQkVIoHpivThr9U+XBHmsBgIfdrK/XUnzs/slugo+NIz8nPiGmMx4gxyJonBCLHDGb1ysky2aEWTl26c0teaF4DeQEjWC1ZaGzv8MV1/GkUamnr1qouXjyUhyEAp33rd8ccN9Rq3QNYB/qLDcA9/FCme7JCW6sCd4zWO0LGEYMJEMc2FzAUkqhqsI05hegGhSDgKXRn5PmLARek4yHD+Hx7pstaTeQIy0WoGJjdzoB3iJIMmo/hWZGzZafktUOh223c5qzx4zMpDRNmMngBUw6R94nKd4KvplYRgB87Y3L/aiVU4CF+axwLmK8RPaC1wbJnlHf06zxHPdiFmsY/zKPpNel+nOnxzRrF5l2KMU4TU6gug3s9Jnzp9T5UMfhp0jW3YkxHGeuOPOeE1i0lTUWUGWrPHLQquAhLfkr2zxaU4ETk/y85hq9W4LAy0ENEDVXX2jP7FnI4Z1fdpmljpmVNJR+outPg6t+Coqgvil7v7XpMtDm8lKQanVYuxwmkb/ncOWFRWuM2j5zIEg3CHnFDcJ9bYrfKRg0b0tb/2BWD14pQnV76goVwzJQYVzdPc8TKIYJw2BZ1Nh9c0iruQVebe/6l1FX9fDCkz8VMmltni61/LxZrf8y0NT1YaU1raeNY2dH5UWvEa9p72FPMI6Eg="
+      secure: "UnDQL3Kh+GK2toL0TK3FObO0ujVssU3Eg4BBuYdjwLB81GhiGE5/DTh7THdZPOpbLo6wQeOwfZDuMeKC1OU+0Uf4NsdYFu1aq6xMO20qBQ4qUfgsyiK4Qgywj9gk0p1+OFZdGAZ/j1CNRAaF71XQIY6iV84c+SO4WoizXYrNT0Jh4sr2DA4/97G2xmJtPi0qOzYrJ09R56ZUozmqeik5G0pMRIuJRbpjS/7bZXV+N7WV0ombZc9RkUaetbabEVOLQ+Xx5YAIVq+VuEeMe9VBSnxY/FfCLmy1wJsjGzpLCyBI9nbrG4nw8Wgc2m8NfK9rcpIvBTGner9r2j60NVDkZ8kLZPrqXhq6AZMwa+oz6K5UQCqRo2RRQzSGwXxg67HY5Tcq+oNmjd+DqpPg4LZ3eGlluyP5XfG+hpSr9Ya4d8q8SrUWLxkoLHI6ZKMtoKFbTCSSQPiluW5hsZxjz3yDkkjsJw64M/EM8UyJrgaXqDklQu+7rBGKLfsK6os7RDiqjBWpQ7gwpo8HvY0O8yqEAabPz+QGkanpjcCOZCXFbSkzWxYy37RMAPu88iINVZVlZE4l+WJenCpZY95ueyy0mG9cyMSzVRPyX6A+/n4H6VMFPFjpGDLTD588ACEjY1lmHfS/eXwXJcgqPPD2gW0XdRdUheU/ssqlfCfGWQMTDXs="
     on:
       tags: true
       branch: master

CHANGELOG.md

@@ -1,40 +1,134 @@
 # Changelog
-All notable changes to this project will be documented in this file.
+All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master).
+
+
+### [8.24.0] - 2019-02-08
+- 'DNS over HTTPS' operation added [@h345983745] | [#489]
+
+### [8.23.1] - 2019-01-18
+- 'Convert co-ordinate format' operation added [@j433866] | [#476]
+
+### [8.23.0] - 2019-01-18
+- 'YARA Rules' operation added [@artemisbot] | [#468]
+
+### [8.22.0] - 2019-01-10
+- 'Subsection' operation added [@j433866] | [#467]
+
+### [8.21.0] - 2019-01-10
+- 'To Case Insensitive Regex' and 'From Case Insensitive Regex' operations added [@masq] | [#461]
+
+### [8.20.0] - 2019-01-09
+- 'Generate Lorem Ipsum' operation added [@klaxon1] | [#455]
+
+### [8.19.0] - 2018-12-30
+- UI test suite added to confirm that the app loads correctly in a reasonable time and that various operations from each module can be run [@n1474335] | [#458]
+
+### [8.18.0] - 2018-12-26
+- 'Split Colour Channels' operation added [@artemisbot] | [#449]
+
+### [8.17.0] - 2018-12-25
+- 'Generate QR Code' and 'Parse QR Code' operations added [@j433866] | [#448]
+
+### [8.16.0] - 2018-12-19
+- 'Play Media' operation added [@anthony-arnold] | [#446]
+
+### [8.15.0] - 2018-12-18
+- 'Text Encoding Brute Force' operation added [@Cynser] | [#439]
+
+### [8.14.0] - 2018-12-18
+- 'To Base62' and 'From Base62' operations added [@tcode2k16] | [#443]
+
+### [8.13.0] - 2018-12-15
+- 'A1Z26 Cipher Encode' and 'A1Z26 Cipher Decode' operations added [@jarmovanlenthe] | [#441]
+
+### [8.12.0] - 2018-11-21
+- 'Citrix CTX1 Encode' and 'Citrix CTX1 Decode' operations added [@bwhitn] | [#428]
+
+### [8.11.0] - 2018-11-13
+- 'CSV to JSON' and 'JSON to CSV' operations added [@n1474335] | [#277]
+
+### [8.10.0] - 2018-11-07
+- 'Remove Diacritics' operation added [@klaxon1] | [#387]
+
+### [8.9.0] - 2018-11-07
+- 'Defang URL' operation added [@arnydo] | [#394]
+
+### [8.8.0] - 2018-10-10
+- 'Parse TLV' operation added [@GCHQ77703] | [#351]
+
+### [8.7.0] - 2018-08-31
+- 'JWT Sign', 'JWT Verify' and 'JWT Decode' operations added [@GCHQ77703] | [#348]
+
+### [8.6.0] - 2018-08-29
+- 'To Geohash' and 'From Geohash' operations added [@GCHQ77703] | [#344]
+
+### [8.5.0] - 2018-08-23
+- 'To Braille' and 'From Braille' operations added [@n1474335] | [#255]
+
+### [8.4.0] - 2018-08-23
+- 'To Base85' and 'From Base85' operations added [@PenguinGeorge] | [#340]
+
+### [8.3.0] - 2018-08-21
+- 'To MessagePack' and 'From MessagePack' operations added [@artemisbot] | [#338]
 
 ### [8.2.0] - 2018-08-21
-- Added information links to most operations, accessible in the description popover @PenguinGeorge #298
+- Information links added to most operations, accessible in the description popover [@PenguinGeorge] | [#298]
 
 ### [8.1.0] - 2018-08-19
-- 'Dechunk HTTP response' operation added @sevzero #311
+- 'Dechunk HTTP response' operation added [@sevzero] | [#311]
 
 ## [8.0.0] - 2018-08-05
-- Codebase rewritten using [ES modules](https://hacks.mozilla.org/2018/03/es-modules-a-cartoon-deep-dive/) and [classes](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Classes) #284
-- Operation architecture restructured to make adding new operations a lot simpler #284
-- A script has been added to aid in the creation of new operations by running `npm run newop` @n1474335 #284
-- 'Magic' operation added - [automated detection of encoded data](https://github.com/gchq/CyberChef/wiki/Automatic-detection-of-encoded-data-using-CyberChef-Magic) @n1474335 #239
-- UI updated to use [Bootstrap Material Design](https://fezvrasta.github.io/bootstrap-material-design/) @n1474335 #248
-- `JSON`, `File` and `List<File>` Dish types added @n1474335 #284
-- `OperationError` type added for better handling of errors thrown by operations @d98762625 #296
-- A `present()` method has been added, allowing operations to pass machine-friendly data to subsequent operations whilst presenting human-friendly data to the user @n1474335 #284
-- Set operations added @d98762625 #281
-- 'To Table' operation added @JustAnotherMark #294
-- 'Haversine distance' operation added @Dachande663 #325
-- Started keeping a changelog @n1474335
+- Codebase rewritten using [ES modules](https://hacks.mozilla.org/2018/03/es-modules-a-cartoon-deep-dive/) and [classes](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Classes) [@n1474335] [@d98762625] [@artemisbot] [@picapi] | [#284]
+- Operation architecture restructured to make adding new operations a lot simpler [@n1474335] | [#284]
+- A script has been added to aid in the creation of new operations by running `npm run newop` [@n1474335] | [#284]
+- 'Magic' operation added - [automated detection of encoded data](https://github.com/gchq/CyberChef/wiki/Automatic-detection-of-encoded-data-using-CyberChef-Magic) [@n1474335] | [#239]
+- UI updated to use [Bootstrap Material Design](https://fezvrasta.github.io/bootstrap-material-design/) [@n1474335] | [#248]
+- `JSON`, `File` and `List<File>` Dish types added [@n1474335] | [#284]
+- `OperationError` type added for better handling of errors thrown by operations [@d98762625] | [#296]
+- A `present()` method has been added, allowing operations to pass machine-friendly data to subsequent operations whilst presenting human-friendly data to the user [@n1474335] | [#284]
+- Set operations added [@d98762625] | [#281]
+- 'To Table' operation added [@JustAnotherMark] | [#294]
+- 'Haversine distance' operation added [@Dachande663] | [#325]
+- Started keeping a changelog [@n1474335]
 
 ## [7.0.0] - 2017-12-28
-- Added support for loading, processing and downloading files up to 500MB @n1474335 #224
+- Added support for loading, processing and downloading files up to 500MB [@n1474335] | [#224]
 
 ## [6.0.0] - 2017-09-19
-- Added threading support, moving all recipe processing into a [Web Worker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers) to increase performance and allow long-running operations to be cancelled @n1474335 #173
-- Created modules so that operations relying on large libraries can be downloaded separately as required, reducing the initial loading time for the app @n1474335 #173
+- Threading support added. All recipe processing moved into a [Web Worker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers) to increase performance and to allow long-running operations to be cancelled [@n1474335] | [#173]
+- Module system created so that operations relying on large libraries can be downloaded separately as required, reducing the initial loading time for the app [@n1474335] | [#173]
 
 ## [5.0.0] - 2017-03-30
--  Configured Webpack build process, Babel transpilation and ES6 imports and exports @n1474335 #95
+-  Webpack build process configured with Babel transpilation and ES6 imports and exports [@n1474335] | [#95]
 
 ## [4.0.0] - 2016-11-28
--  Initial open source commit @n1474335
+-  Initial open source commit [@n1474335] | [b1d73a72](https://github.com/gchq/CyberChef/commit/b1d73a725dc7ab9fb7eb789296efd2b7e4b08306)
 
 
+
+[8.24.0]: https://github.com/gchq/CyberChef/releases/tag/v8.24.0
+[8.23.1]: https://github.com/gchq/CyberChef/releases/tag/v8.23.1
+[8.23.0]: https://github.com/gchq/CyberChef/releases/tag/v8.23.0
+[8.22.0]: https://github.com/gchq/CyberChef/releases/tag/v8.22.0
+[8.21.0]: https://github.com/gchq/CyberChef/releases/tag/v8.21.0
+[8.20.0]: https://github.com/gchq/CyberChef/releases/tag/v8.20.0
+[8.19.0]: https://github.com/gchq/CyberChef/releases/tag/v8.19.0
+[8.18.0]: https://github.com/gchq/CyberChef/releases/tag/v8.18.0
+[8.17.0]: https://github.com/gchq/CyberChef/releases/tag/v8.17.0
+[8.16.0]: https://github.com/gchq/CyberChef/releases/tag/v8.16.0
+[8.15.0]: https://github.com/gchq/CyberChef/releases/tag/v8.15.0
+[8.14.0]: https://github.com/gchq/CyberChef/releases/tag/v8.14.0
+[8.13.0]: https://github.com/gchq/CyberChef/releases/tag/v8.13.0
+[8.12.0]: https://github.com/gchq/CyberChef/releases/tag/v8.12.0
+[8.11.0]: https://github.com/gchq/CyberChef/releases/tag/v8.11.0
+[8.10.0]: https://github.com/gchq/CyberChef/releases/tag/v8.10.0
+[8.9.0]: https://github.com/gchq/CyberChef/releases/tag/v8.9.0
+[8.8.0]: https://github.com/gchq/CyberChef/releases/tag/v8.8.0
+[8.7.0]: https://github.com/gchq/CyberChef/releases/tag/v8.7.0
+[8.6.0]: https://github.com/gchq/CyberChef/releases/tag/v8.6.0
+[8.5.0]: https://github.com/gchq/CyberChef/releases/tag/v8.5.0
+[8.4.0]: https://github.com/gchq/CyberChef/releases/tag/v8.4.0
+[8.3.0]: https://github.com/gchq/CyberChef/releases/tag/v8.3.0
 [8.2.0]: https://github.com/gchq/CyberChef/releases/tag/v8.2.0
 [8.1.0]: https://github.com/gchq/CyberChef/releases/tag/v8.1.0
 [8.0.0]: https://github.com/gchq/CyberChef/releases/tag/v8.0.0
@@ -42,3 +136,59 @@ All notable changes to this project will be documented in this file.
 [6.0.0]: https://github.com/gchq/CyberChef/releases/tag/v6.0.0
 [5.0.0]: https://github.com/gchq/CyberChef/releases/tag/v5.0.0
 [4.0.0]: https://github.com/gchq/CyberChef/commit/b1d73a725dc7ab9fb7eb789296efd2b7e4b08306
+
+[@n1474335]: https://github.com/n1474335
+[@d98762625]: https://github.com/d98762625
+[@j433866]: https://github.com/j433866
+[@GCHQ77703]: https://github.com/GCHQ77703
+[@h345983745]: https://github.com/h345983745
+[@artemisbot]: https://github.com/artemisbot
+[@picapi]: https://github.com/picapi
+[@Dachande663]: https://github.com/Dachande663
+[@JustAnotherMark]: https://github.com/JustAnotherMark
+[@sevzero]: https://github.com/sevzero
+[@PenguinGeorge]: https://github.com/PenguinGeorge
+[@arnydo]: https://github.com/arnydo
+[@klaxon1]: https://github.com/klaxon1
+[@bwhitn]: https://github.com/bwhitn
+[@jarmovanlenthe]: https://github.com/jarmovanlenthe
+[@tcode2k16]: https://github.com/tcode2k16
+[@Cynser]: https://github.com/Cynser
+[@anthony-arnold]: https://github.com/anthony-arnold
+[@masq]: https://github.com/masq
+
+[#95]: https://github.com/gchq/CyberChef/pull/299
+[#173]: https://github.com/gchq/CyberChef/pull/173
+[#224]: https://github.com/gchq/CyberChef/pull/224
+[#239]: https://github.com/gchq/CyberChef/pull/239
+[#248]: https://github.com/gchq/CyberChef/pull/248
+[#255]: https://github.com/gchq/CyberChef/issues/255
+[#277]: https://github.com/gchq/CyberChef/issues/277
+[#281]: https://github.com/gchq/CyberChef/pull/281
+[#284]: https://github.com/gchq/CyberChef/pull/284
+[#294]: https://github.com/gchq/CyberChef/pull/294
+[#296]: https://github.com/gchq/CyberChef/pull/296
+[#298]: https://github.com/gchq/CyberChef/pull/298
+[#311]: https://github.com/gchq/CyberChef/pull/311
+[#325]: https://github.com/gchq/CyberChef/pull/325
+[#338]: https://github.com/gchq/CyberChef/pull/338
+[#340]: https://github.com/gchq/CyberChef/pull/340
+[#344]: https://github.com/gchq/CyberChef/pull/344
+[#348]: https://github.com/gchq/CyberChef/pull/348
+[#351]: https://github.com/gchq/CyberChef/pull/351
+[#387]: https://github.com/gchq/CyberChef/pull/387
+[#394]: https://github.com/gchq/CyberChef/pull/394
+[#428]: https://github.com/gchq/CyberChef/pull/428
+[#439]: https://github.com/gchq/CyberChef/pull/439
+[#441]: https://github.com/gchq/CyberChef/pull/441
+[#443]: https://github.com/gchq/CyberChef/pull/443
+[#446]: https://github.com/gchq/CyberChef/pull/446
+[#448]: https://github.com/gchq/CyberChef/pull/448
+[#449]: https://github.com/gchq/CyberChef/pull/449
+[#455]: https://github.com/gchq/CyberChef/pull/455
+[#458]: https://github.com/gchq/CyberChef/pull/458
+[#461]: https://github.com/gchq/CyberChef/pull/461
+[#467]: https://github.com/gchq/CyberChef/pull/467
+[#468]: https://github.com/gchq/CyberChef/pull/468
+[#476]: https://github.com/gchq/CyberChef/pull/476
+[#489]: https://github.com/gchq/CyberChef/pull/489

Gruntfile.js

@@ -2,6 +2,7 @@
 
 const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
+const BundleAnalyzerPlugin = require("webpack-bundle-analyzer").BundleAnalyzerPlugin;
 const NodeExternals = require("webpack-node-externals");
 const Inliner = require("web-resource-inliner");
 const glob = require("glob");
@@ -22,15 +23,19 @@ module.exports = function (grunt) {
     // Tasks
     grunt.registerTask("dev",
         "A persistent task which creates a development build whenever source files are modified.",
-        ["clean:dev", "exec:generateConfig", "concurrent:dev"]);
+        ["clean:dev", "clean:config", "exec:generateConfig", "concurrent:dev"]);
 
     grunt.registerTask("node",
         "Compiles CyberChef into a single NodeJS module.",
         ["clean:node", "clean:config", "exec:generateConfig", "webpack:node", "chmod:build"]);
 
     grunt.registerTask("test",
-        "A task which runs all the tests in test/tests.",
-        ["exec:generateConfig", "exec:tests"]);
+        "A task which runs all the operation tests in the tests directory.",
+        ["exec:generateConfig", "exec:opTests"]);
+
+    grunt.registerTask("testui",
+        "A task which runs all the UI tests in the tests directory. The prod task must already have been run.",
+        ["connect:prod", "exec:browserTests"]);
 
     grunt.registerTask("docs",
         "Compiles documentation in the /docs directory.",
@@ -38,7 +43,7 @@ module.exports = function (grunt) {
 
     grunt.registerTask("prod",
         "Creates a production-ready build. Use the --msg flag to add a compile message.",
-        ["eslint", "clean:prod", "exec:generateConfig", "webpack:web", "inline", "chmod"]);
+        ["eslint", "clean:prod", "clean:config", "exec:generateConfig", "webpack:web", "inline", "chmod"]);
 
     grunt.registerTask("default",
         "Lints the code base",
@@ -66,6 +71,7 @@ module.exports = function (grunt) {
     grunt.loadNpmTasks("grunt-exec");
     grunt.loadNpmTasks("grunt-accessibility");
     grunt.loadNpmTasks("grunt-concurrent");
+    grunt.loadNpmTasks("grunt-contrib-connect");
 
 
     // Project configuration
@@ -143,11 +149,11 @@ module.exports = function (grunt) {
             options: {
                 configFile: "./.eslintrc.json"
             },
-            configs: ["Gruntfile.js"],
+            configs: ["*.{js,mjs}"],
             core: ["src/core/**/*.{js,mjs}", "!src/core/vendor/**/*", "!src/core/operations/legacy/**/*"],
             web: ["src/web/**/*.{js,mjs}"],
             node: ["src/node/**/*.{js,mjs}"],
-            tests: ["test/**/*.{js,mjs}"],
+            tests: ["tests/**/*.{js,mjs}"],
         },
         jsdoc: {
             options: {
@@ -179,37 +185,45 @@ module.exports = function (grunt) {
         },
         webpack: {
             options: webpackConfig,
-            web: {
-                mode: "production",
-                target: "web",
-                entry: Object.assign({
-                    main: "./src/web/index.js",
-                    sitemap: "./src/web/static/sitemap.js"
-                }, moduleEntryPoints),
-                output: {
-                    path: __dirname + "/build/prod"
-                },
-                resolve: {
-                    alias: {
-                        "./config/modules/OpModules": "./config/modules/Default"
-                    }
-                },
-                plugins: [
-                    new webpack.DefinePlugin(BUILD_CONSTANTS),
-                    new HtmlWebpackPlugin({
-                        filename: "index.html",
-                        template: "./src/web/html/index.html",
-                        chunks: ["main"],
-                        compileTime: compileTime,
-                        version: pkg.version,
-                        minify: {
-                            removeComments: true,
-                            collapseWhitespace: true,
-                            minifyJS: true,
-                            minifyCSS: true
+            web: () => {
+                return {
+                    mode: "production",
+                    target: "web",
+                    entry: Object.assign({
+                        main: "./src/web/index.js",
+                        sitemap: "./src/web/static/sitemap.js"
+                    }, moduleEntryPoints),
+                    output: {
+                        path: __dirname + "/build/prod",
+                        globalObject: "this"
+                    },
+                    resolve: {
+                        alias: {
+                            "./config/modules/OpModules": "./config/modules/Default"
                         }
-                    }),
-                ]
+                    },
+                    plugins: [
+                        new webpack.DefinePlugin(BUILD_CONSTANTS),
+                        new HtmlWebpackPlugin({
+                            filename: "index.html",
+                            template: "./src/web/html/index.html",
+                            chunks: ["main"],
+                            compileTime: compileTime,
+                            version: pkg.version,
+                            minify: {
+                                removeComments: true,
+                                collapseWhitespace: true,
+                                minifyJS: true,
+                                minifyCSS: true
+                            }
+                        }),
+                        new BundleAnalyzerPlugin({
+                            analyzerMode: "static",
+                            reportFilename: "BundleAnalyzerReport.html",
+                            openAnalyzer: false
+                        }),
+                    ]
+                };
             },
             webInline: {
                 mode: "production",
@@ -238,19 +252,6 @@ module.exports = function (grunt) {
                     }),
                 ]
             },
-            tests: {
-                mode: "development",
-                target: "node",
-                entry: "./test/index.mjs",
-                externals: [NodeExternals()],
-                output: {
-                    filename: "index.js",
-                    path: __dirname + "/build/test"
-                },
-                plugins: [
-                    new webpack.DefinePlugin(BUILD_CONSTANTS)
-                ]
-            },
             node: {
                 mode: "production",
                 target: "node",
@@ -280,7 +281,11 @@ module.exports = function (grunt) {
                     chunks: false,
                     modules: false,
                     entrypoints: false,
-                    warningsFilter: [/source-map/, /dependency is an expression/],
+                    warningsFilter: [
+                        /source-map/,
+                        /dependency is an expression/,
+                        /export 'default'/
+                    ],
                 }
             },
             start: {
@@ -295,6 +300,9 @@ module.exports = function (grunt) {
                             "./config/modules/OpModules": "./config/modules/Default"
                         }
                     },
+                    output: {
+                        globalObject: "this",
+                    },
                     plugins: [
                         new webpack.DefinePlugin(BUILD_CONSTANTS),
                         new HtmlWebpackPlugin({
@@ -308,6 +316,14 @@ module.exports = function (grunt) {
                 }
             }
         },
+        connect: {
+            prod: {
+                options: {
+                    port: 8000,
+                    base: "build/prod/"
+                }
+            }
+        },
         copy: {
             ghPages: {
                 options: {
@@ -379,16 +395,17 @@ module.exports = function (grunt) {
             generateConfig: {
                 command: [
                     "echo '\n--- Regenerating config files. ---'",
-                    "mkdir -p src/core/config/modules",
-                    "echo 'export default {};\n' > src/core/config/modules/OpModules.mjs",
-                    "echo '[]\n' > src/core/config/OperationConfig.json",
-                    "node --experimental-modules src/core/config/scripts/generateOpsIndex.mjs",
-                    "node --experimental-modules src/core/config/scripts/generateConfig.mjs",
+                    "echo [] > src/core/config/OperationConfig.json",
+                    "node --experimental-modules --no-warnings --no-deprecation src/core/config/scripts/generateOpsIndex.mjs",
+                    "node --experimental-modules --no-warnings --no-deprecation src/core/config/scripts/generateConfig.mjs",
                     "echo '--- Config scripts finished. ---\n'"
                 ].join(";")
             },
-            tests: {
-                command: "node --experimental-modules test/index.mjs"
+            opTests: {
+                command: "node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs"
+            },
+            browserTests: {
+                command: "./node_modules/.bin/nightwatch --env prod,inline"
             }
         },
     });

babel.config.js

@@ -0,0 +1,24 @@
+module.exports = function(api) {
+    api.cache.forever();
+
+    return {
+        "presets": [
+            ["@babel/preset-env", {
+                "targets": {
+                    "chrome": 40,
+                    "firefox": 35,
+                    "edge": 14,
+                    "node": "6.5"
+                },
+                "modules": false,
+                "useBuiltIns": "entry"
+            }]
+        ],
+        "plugins": [
+            "babel-plugin-syntax-dynamic-import",
+            ["babel-plugin-transform-builtin-extend", {
+                "globals": ["Error"]
+            }]
+        ]
+    };
+};

nightwatch.json

@@ -0,0 +1,34 @@
+{
+  "src_folders": ["tests/browser"],
+  "output_folder": "tests/browser/output",
+
+  "test_settings": {
+
+    "default": {
+      "launch_url": "http://localhost:8080",
+      "webdriver": {
+        "start_process": true,
+        "server_path": "./node_modules/.bin/chromedriver",
+        "port": 9515,
+        "log_path": false
+      },
+      "desiredCapabilities": {
+        "browserName": "chrome"
+      }
+    },
+
+    "dev": {
+      "launch_url": "http://localhost:8080"
+    },
+
+    "prod": {
+      "launch_url": "http://localhost:8000/index.html"
+    },
+
+    "inline": {
+      "launch_url": "http://localhost:8000/cyberchef.htm"
+    }
+
+  }
+}
+

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.2.0",
+  "version": "8.26.3",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",
@@ -30,45 +30,50 @@
   "main": "build/node/CyberChef.js",
   "bugs": "https://github.com/gchq/CyberChef/issues",
   "devDependencies": {
-    "autoprefixer": "^9.1.0",
-    "babel-core": "^6.26.3",
-    "babel-loader": "^7.1.5",
-    "babel-preset-env": "^1.7.0",
-    "bootstrap": "^4.1.3",
-    "colors": "^1.3.1",
-    "css-loader": "^1.0.0",
-    "eslint": "^5.3.0",
+    "@babel/core": "^7.2.2",
+    "@babel/preset-env": "^7.2.3",
+    "autoprefixer": "^9.4.3",
+    "babel-eslint": "^10.0.1",
+    "babel-loader": "^8.0.4",
+    "babel-plugin-syntax-dynamic-import": "^6.18.0",
+    "bootstrap": "^4.2.1",
+    "chromedriver": "^2.45.0",
+    "colors": "^1.3.3",
+    "css-loader": "^2.1.0",
+    "eslint": "^5.12.1",
     "exports-loader": "^0.7.0",
-    "extract-text-webpack-plugin": "^4.0.0-alpha0",
-    "file-loader": "^1.1.11",
+    "file-loader": "^3.0.1",
     "grunt": "^1.0.3",
     "grunt-accessibility": "~6.0.0",
     "grunt-chmod": "~1.1.1",
     "grunt-concurrent": "^2.3.1",
-    "grunt-contrib-clean": "~1.1.0",
+    "grunt-contrib-clean": "~2.0.0",
+    "grunt-contrib-connect": "^2.0.0",
     "grunt-contrib-copy": "~1.0.0",
     "grunt-contrib-watch": "^1.1.0",
     "grunt-eslint": "^21.0.0",
     "grunt-exec": "~3.0.0",
-    "grunt-jsdoc": "^2.2.1",
-    "grunt-webpack": "^3.1.2",
+    "grunt-jsdoc": "^2.3.0",
+    "grunt-webpack": "^3.1.3",
     "html-webpack-plugin": "^3.2.0",
     "imports-loader": "^0.8.0",
     "ink-docstrap": "^1.3.2",
-    "js-to-mjs": "^0.2.0",
-    "jsdoc-babel": "^0.4.0",
-    "node-sass": "^4.9.2",
-    "postcss-css-variables": "^0.9.0",
-    "postcss-import": "^12.0.0",
-    "postcss-loader": "^2.1.6",
+    "jsdoc-babel": "^0.5.0",
+    "mini-css-extract-plugin": "^0.5.0",
+    "nightwatch": "^1.0.18",
+    "node-sass": "^4.11.0",
+    "postcss-css-variables": "^0.11.0",
+    "postcss-import": "^12.0.1",
+    "postcss-loader": "^3.0.0",
     "prompt": "^1.0.0",
     "sass-loader": "^7.1.0",
-    "sitemap": "^1.13.0",
-    "style-loader": "^0.21.0",
-    "url-loader": "^1.0.1",
+    "sitemap": "^2.1.0",
+    "style-loader": "^0.23.1",
+    "url-loader": "^1.1.2",
     "web-resource-inliner": "^4.2.1",
-    "webpack": "^4.16.4",
-    "webpack-dev-server": "^3.1.5",
+    "webpack": "^4.28.3",
+    "webpack-bundle-analyzer": "^3.0.3",
+    "webpack-dev-server": "^3.1.14",
     "webpack-node-externals": "^1.7.2",
     "worker-loader": "^2.0.0"
   },
@@ -77,60 +82,67 @@
     "babel-plugin-transform-builtin-extend": "1.1.2",
     "babel-polyfill": "^6.26.0",
     "bcryptjs": "^2.4.3",
-    "bignumber.js": "^7.2.1",
+    "bignumber.js": "^8.0.2",
     "bootstrap-colorpicker": "^2.5.3",
     "bootstrap-material-design": "^4.1.1",
-    "bson": "^3.0.2",
+    "bson": "^4.0.1",
     "chi-squared": "^1.1.0",
-    "crypto-api": "^0.8.0",
+    "crypto-api": "^0.8.3",
     "crypto-js": "^3.1.9-1",
     "ctph.js": "0.0.5",
     "diff": "^3.5.0",
-    "es6-promisify": "^6.0.0",
+    "es6-promisify": "^6.0.1",
     "escodegen": "^1.11.0",
     "esmangle": "^1.0.1",
     "esprima": "^4.0.1",
     "exif-parser": "^0.1.12",
-    "file-saver": "^1.3.8",
-    "highlight.js": "^9.12.0",
+    "file-saver": "^2.0.0",
+    "geodesy": "^1.1.3",
+    "highlight.js": "^9.13.1",
+    "jimp": "^0.6.0",
     "jquery": "^3.3.1",
     "js-crc": "^0.2.0",
-    "js-sha3": "^0.7.0",
-    "jsbn": "^1.1.0",
-    "jsesc": "^2.5.1",
+    "js-sha3": "^0.8.0",
+    "jsesc": "^2.5.2",
     "jsonpath": "^1.0.0",
+    "jsonwebtoken": "^8.4.0",
+    "jsqr": "^1.1.1",
     "jsrsasign": "8.0.12",
-    "kbpgp": "^2.0.77",
-    "lodash": "^4.17.10",
+    "kbpgp": "^2.0.82",
+    "libyara-wasm": "0.0.12",
+    "lodash": "^4.17.11",
     "loglevel": "^1.6.1",
     "loglevel-message-prefix": "^3.0.0",
-    "moment": "^2.22.2",
-    "moment-timezone": "^0.5.21",
-    "node-forge": "^0.7.5",
+    "moment": "^2.23.0",
+    "moment-timezone": "^0.5.23",
+    "ngeohash": "^0.6.3",
+    "node-forge": "^0.7.6",
     "node-md6": "^0.1.0",
+    "notepack.io": "^2.2.0",
     "nwmatcher": "^1.4.4",
     "otp": "^0.1.3",
-    "popper.js": "^1.14.4",
+    "popper.js": "^1.14.6",
+    "qr-image": "^3.2.0",
     "scryptsy": "^2.0.0",
     "snackbarjs": "^1.1.0",
-    "sortablejs": "^1.7.0",
-    "split.js": "^1.3.5",
+    "sortablejs": "^1.8.0-rc1",
+    "split.js": "^1.5.10",
     "ssdeep.js": "0.0.2",
-    "ua-parser-js": "^0.7.18",
+    "ua-parser-js": "^0.7.19",
     "utf8": "^3.0.0",
     "vkbeautify": "^0.99.3",
     "xmldom": "^0.1.27",
     "xpath": "0.0.27",
-    "xregexp": "^4.2.0",
+    "xregexp": "^4.2.4",
     "zlibjs": "^0.3.1"
   },
   "scripts": {
     "start": "grunt dev",
     "build": "grunt prod",
     "test": "grunt test",
+    "testui": "grunt testui",
     "docs": "grunt docs",
     "lint": "grunt lint",
-    "newop": "node --experimental-modules src/core/config/scripts/newOperation.mjs",
-    "postinstall": "[ -f node_modules/crypto-api/src/crypto-api.mjs ] || npx j2m node_modules/crypto-api/src/crypto-api.js"
+    "newop": "node --experimental-modules src/core/config/scripts/newOperation.mjs"
   }
 }

src/core/Chef.mjs

@@ -157,9 +157,9 @@ class Chef {
      * @param {number} pos.end - The end offset.
      * @returns {Object}
      */
-    calculateHighlights(recipeConfig, direction, pos) {
+    async calculateHighlights(recipeConfig, direction, pos) {
         const recipe = new Recipe(recipeConfig);
-        const highlights = recipe.generateHighlightList();
+        const highlights = await recipe.generateHighlightList();
 
         if (!highlights) return false;
 

src/core/ChefWorker.js

@@ -157,8 +157,8 @@ async function getDishAs(data) {
  * @param {number} pos.start - The start offset.
  * @param {number} pos.end - The end offset.
  */
-function calculateHighlights(recipeConfig, direction, pos) {
-    pos = self.chef.calculateHighlights(recipeConfig, direction, pos);
+async function calculateHighlights(recipeConfig, direction, pos) {
+    pos = await self.chef.calculateHighlights(recipeConfig, direction, pos);
 
     self.postMessage({
         action: "highlightsCalculated",

src/core/Dish.mjs

@@ -6,6 +6,7 @@
  */
 
 import Utils from "./Utils";
+import DishError from "./errors/DishError";
 import BigNumber from "bignumber.js";
 import log from "loglevel";
 
@@ -61,7 +62,7 @@ class Dish {
             case "list<file>":
                 return Dish.LIST_FILE;
             default:
-                throw "Invalid data type string. No matching enum.";
+                throw new DishError("Invalid data type string. No matching enum.");
         }
     }
 
@@ -93,7 +94,7 @@ class Dish {
             case Dish.LIST_FILE:
                 return "List<File>";
             default:
-                throw "Invalid data type enum. No matching type.";
+                throw new DishError("Invalid data type enum. No matching type.");
         }
     }
 
@@ -117,7 +118,7 @@ class Dish {
 
         if (!this.valid()) {
             const sample = Utils.truncate(JSON.stringify(this.value), 13);
-            throw "Data is not a valid " + Dish.enumLookup(type) + ": " + sample;
+            throw new DishError(`Data is not a valid ${Dish.enumLookup(type)}: ${sample}`);
         }
     }
 
@@ -151,77 +152,85 @@ class Dish {
         const byteArrayToStr = notUTF8 ? Utils.byteArrayToChars : Utils.byteArrayToUtf8;
 
         // Convert data to intermediate byteArray type
-        switch (this.type) {
-            case Dish.STRING:
-                this.value = this.value ? Utils.strToByteArray(this.value) : [];
-                break;
-            case Dish.NUMBER:
-                this.value = typeof this.value === "number" ? Utils.strToByteArray(this.value.toString()) : [];
-                break;
-            case Dish.HTML:
-                this.value = this.value ? Utils.strToByteArray(Utils.unescapeHtml(Utils.stripHtmlTags(this.value, true))) : [];
-                break;
-            case Dish.ARRAY_BUFFER:
-                // Array.from() would be nicer here, but it's slightly slower
-                this.value = Array.prototype.slice.call(new Uint8Array(this.value));
-                break;
-            case Dish.BIG_NUMBER:
-                this.value = this.value instanceof BigNumber ? Utils.strToByteArray(this.value.toFixed()) : [];
-                break;
-            case Dish.JSON:
-                this.value = this.value ? Utils.strToByteArray(JSON.stringify(this.value)) : [];
-                break;
-            case Dish.FILE:
-                this.value = await Utils.readFile(this.value);
-                this.value = Array.prototype.slice.call(this.value);
-                break;
-            case Dish.LIST_FILE:
-                this.value = await Promise.all(this.value.map(async f => Utils.readFile(f)));
-                this.value = this.value.map(b => Array.prototype.slice.call(b));
-                this.value = [].concat.apply([], this.value);
-                break;
-            default:
-                break;
+        try {
+            switch (this.type) {
+                case Dish.STRING:
+                    this.value = this.value ? Utils.strToByteArray(this.value) : [];
+                    break;
+                case Dish.NUMBER:
+                    this.value = typeof this.value === "number" ? Utils.strToByteArray(this.value.toString()) : [];
+                    break;
+                case Dish.HTML:
+                    this.value = this.value ? Utils.strToByteArray(Utils.unescapeHtml(Utils.stripHtmlTags(this.value, true))) : [];
+                    break;
+                case Dish.ARRAY_BUFFER:
+                    // Array.from() would be nicer here, but it's slightly slower
+                    this.value = Array.prototype.slice.call(new Uint8Array(this.value));
+                    break;
+                case Dish.BIG_NUMBER:
+                    this.value = BigNumber.isBigNumber(this.value) ? Utils.strToByteArray(this.value.toFixed()) : [];
+                    break;
+                case Dish.JSON:
+                    this.value = this.value ? Utils.strToByteArray(JSON.stringify(this.value, null, 4)) : [];
+                    break;
+                case Dish.FILE:
+                    this.value = await Utils.readFile(this.value);
+                    this.value = Array.prototype.slice.call(this.value);
+                    break;
+                case Dish.LIST_FILE:
+                    this.value = await Promise.all(this.value.map(async f => Utils.readFile(f)));
+                    this.value = this.value.map(b => Array.prototype.slice.call(b));
+                    this.value = [].concat.apply([], this.value);
+                    break;
+                default:
+                    break;
+            }
+        } catch (err) {
+            throw new DishError(`Error translating from ${Dish.enumLookup(this.type)} to byteArray: ${err}`);
         }
 
         this.type = Dish.BYTE_ARRAY;
 
         // Convert from byteArray to toType
-        switch (toType) {
-            case Dish.STRING:
-            case Dish.HTML:
-                this.value = this.value ? byteArrayToStr(this.value) : "";
-                this.type = Dish.STRING;
-                break;
-            case Dish.NUMBER:
-                this.value = this.value ? parseFloat(byteArrayToStr(this.value)) : 0;
-                this.type = Dish.NUMBER;
-                break;
-            case Dish.ARRAY_BUFFER:
-                this.value = new Uint8Array(this.value).buffer;
-                this.type = Dish.ARRAY_BUFFER;
-                break;
-            case Dish.BIG_NUMBER:
-                try {
-                    this.value = new BigNumber(byteArrayToStr(this.value));
-                } catch (err) {
-                    this.value = new BigNumber(NaN);
-                }
-                this.type = Dish.BIG_NUMBER;
-                break;
-            case Dish.JSON:
-                this.value = JSON.parse(byteArrayToStr(this.value));
-                this.type = Dish.JSON;
-                break;
-            case Dish.FILE:
-                this.value = new File(this.value, "unknown");
-                break;
-            case Dish.LIST_FILE:
-                this.value = [new File(this.value, "unknown")];
-                this.type = Dish.LIST_FILE;
-                break;
-            default:
-                break;
+        try {
+            switch (toType) {
+                case Dish.STRING:
+                case Dish.HTML:
+                    this.value = this.value ? byteArrayToStr(this.value) : "";
+                    this.type = Dish.STRING;
+                    break;
+                case Dish.NUMBER:
+                    this.value = this.value ? parseFloat(byteArrayToStr(this.value)) : 0;
+                    this.type = Dish.NUMBER;
+                    break;
+                case Dish.ARRAY_BUFFER:
+                    this.value = new Uint8Array(this.value).buffer;
+                    this.type = Dish.ARRAY_BUFFER;
+                    break;
+                case Dish.BIG_NUMBER:
+                    try {
+                        this.value = new BigNumber(byteArrayToStr(this.value));
+                    } catch (err) {
+                        this.value = new BigNumber(NaN);
+                    }
+                    this.type = Dish.BIG_NUMBER;
+                    break;
+                case Dish.JSON:
+                    this.value = JSON.parse(byteArrayToStr(this.value));
+                    this.type = Dish.JSON;
+                    break;
+                case Dish.FILE:
+                    this.value = new File(this.value, "unknown");
+                    break;
+                case Dish.LIST_FILE:
+                    this.value = [new File(this.value, "unknown")];
+                    this.type = Dish.LIST_FILE;
+                    break;
+                default:
+                    break;
+            }
+        } catch (err) {
+            throw new DishError(`Error translating from byteArray to ${Dish.enumLookup(toType)}: ${err}`);
         }
     }
 
@@ -256,7 +265,7 @@ class Dish {
             case Dish.ARRAY_BUFFER:
                 return this.value instanceof ArrayBuffer;
             case Dish.BIG_NUMBER:
-                return this.value instanceof BigNumber;
+                return BigNumber.isBigNumber(this.value);
             case Dish.JSON:
                 // All values can be serialised in some manner, so we return true in all cases
                 return true;
@@ -357,7 +366,7 @@ class Dish {
                 );
                 break;
             default:
-                throw new Error("Cannot clone Dish, unknown type");
+                throw new DishError("Cannot clone Dish, unknown type");
         }
 
         return newDish;

src/core/Ingredient.mjs

@@ -23,8 +23,13 @@ class Ingredient {
         this._value = null;
         this.disabled = false;
         this.hint = "";
+        this.rows = 0;
         this.toggleValues = [];
         this.target = null;
+        this.defaultIndex = 0;
+        this.min = null;
+        this.max = null;
+        this.step = 1;
 
         if (ingredientConfig) {
             this._parseConfig(ingredientConfig);
@@ -44,8 +49,13 @@ class Ingredient {
         this.defaultValue = ingredientConfig.value;
         this.disabled = !!ingredientConfig.disabled;
         this.hint = ingredientConfig.hint || false;
+        this.rows = ingredientConfig.rows || false;
         this.toggleValues = ingredientConfig.toggleValues;
         this.target = typeof ingredientConfig.target !== "undefined" ? ingredientConfig.target : null;
+        this.defaultIndex = typeof ingredientConfig.defaultIndex !== "undefined" ? ingredientConfig.defaultIndex : 0;
+        this.min = ingredientConfig.min;
+        this.max = ingredientConfig.max;
+        this.step = ingredientConfig.step;
     }
 
 
@@ -93,6 +103,7 @@ class Ingredient {
             case "binaryString":
             case "binaryShortString":
             case "editableOption":
+            case "editableOptionShort":
                 return Utils.parseEscapedChars(data);
             case "byteArray":
                 if (typeof data == "string") {

src/core/Operation.mjs

@@ -23,6 +23,7 @@ class Operation {
         this._breakpoint      = false;
         this._disabled        = false;
         this._flowControl     = false;
+        this._manualBake      = false;
         this._ingList         = [];
 
         // Public fields
@@ -179,8 +180,13 @@ class Operation {
 
             if (ing.toggleValues) conf.toggleValues = ing.toggleValues;
             if (ing.hint) conf.hint = ing.hint;
+            if (ing.rows) conf.rows = ing.rows;
             if (ing.disabled) conf.disabled = ing.disabled;
             if (ing.target) conf.target = ing.target;
+            if (ing.defaultIndex) conf.defaultIndex = ing.defaultIndex;
+            if (typeof ing.min === "number") conf.min = ing.min;
+            if (typeof ing.max === "number") conf.max = ing.max;
+            if (ing.step) conf.step = ing.step;
             return conf;
         });
     }
@@ -280,6 +286,7 @@ class Operation {
         return this._flowControl;
     }
 
+
     /**
      * Set whether this Operation is a flowcontrol op.
      *
@@ -289,6 +296,26 @@ class Operation {
         this._flowControl = !!value;
     }
 
+
+    /**
+     * Returns true if this Operation should not trigger AutoBake.
+     *
+     * @returns {boolean}
+     */
+    get manualBake() {
+        return this._manualBake;
+    }
+
+
+    /**
+     * Set whether this Operation should trigger AutoBake.
+     *
+     * @param {boolean} value
+     */
+    set manualBake(value) {
+        this._manualBake = !!value;
+    }
+
 }
 
 export default Operation;

src/core/Recipe.mjs

@@ -4,12 +4,15 @@
  * @license Apache-2.0
  */
 
-// import Operation from "./Operation.js";
-import OpModules from "./config/modules/OpModules";
 import OperationConfig from "./config/OperationConfig.json";
 import OperationError from "./errors/OperationError";
+import Operation from "./Operation";
+import DishError from "./errors/DishError";
 import log from "loglevel";
 
+// Cache container for modules
+let modules = null;
+
 /**
  * The Recipe controls a list of Operations and the Dish they operate on.
  */
@@ -36,16 +39,43 @@ class Recipe  {
      * @param {Object} recipeConfig
      */
     _parseConfig(recipeConfig) {
-        for (let c = 0; c < recipeConfig.length; c++) {
-            const operationName = recipeConfig[c].op;
-            const opConf = OperationConfig[operationName];
-            const opObj = OpModules[opConf.module][operationName];
-            const operation = new opObj();
-            operation.ingValues = recipeConfig[c].args;
-            operation.breakpoint = recipeConfig[c].breakpoint;
-            operation.disabled = recipeConfig[c].disabled;
-            this.addOperation(operation);
+        recipeConfig.forEach(c => {
+            this.opList.push({
+                name: c.op,
+                module: OperationConfig[c.op].module,
+                ingValues: c.args,
+                breakpoint: c.breakpoint,
+                disabled: c.disabled,
+            });
+        });
+    }
+
+
+    /**
+     * Populate elements of opList with operation instances.
+     * Dynamic import here removes top-level cyclic dependency issue.
+     *
+     * @private
+     */
+    async _hydrateOpList() {
+        if (!modules) {
+            // Using Webpack Magic Comments to force the dynamic import to be included in the main chunk
+            // https://webpack.js.org/api/module-methods/
+            modules = await import(/* webpackMode: "eager" */ "./config/modules/OpModules");
+            modules = modules.default;
         }
+
+        this.opList = this.opList.map(o => {
+            if (o instanceof Operation) {
+                return o;
+            } else {
+                const op = new modules[o.module][o.name]();
+                op.ingValues = o.ingValues;
+                op.breakpoint = o.breakpoint;
+                op.disabled = o.disabled;
+                return op;
+            }
+        });
     }
 
 
@@ -55,7 +85,10 @@ class Recipe  {
      * @returns {Object[]}
      */
     get config() {
-        return this.opList.map(op => op.config);
+        return this.opList.map(op => ({
+            op: op.name,
+            args: op.ingValues,
+        }));
     }
 
 
@@ -75,7 +108,19 @@ class Recipe  {
      * @param {Operation[]} operations
      */
     addOperations(operations) {
-        this.opList = this.opList.concat(operations);
+        operations.forEach(o => {
+            if (o instanceof Operation) {
+                this.opList.push(o);
+            } else {
+                this.opList.push({
+                    name: o.name,
+                    module: o.module,
+                    ingValues: o.args,
+                    breakpoint: o.breakpoint,
+                    disabled: o.disabled,
+                });
+            }
+        });
     }
 
 
@@ -108,7 +153,7 @@ class Recipe  {
 
 
     /**
-     * Returns true if there is an Flow Control Operation in this Recipe.
+     * Returns true if there is a Flow Control Operation in this Recipe.
      *
      * @returns {boolean}
      */
@@ -137,6 +182,8 @@ class Recipe  {
 
         if (startFrom === 0) this.lastRunOp = null;
 
+        await this._hydrateOpList();
+
         log.debug(`[*] Executing recipe of ${this.opList.length} operations, starting at ${startFrom}`);
 
         for (let i = startFrom; i < this.opList.length; i++) {
@@ -183,6 +230,10 @@ class Recipe  {
                     // native types is not fully supported yet.
                     dish.set(err.message, "string");
                     return i;
+                } else if (err instanceof DishError ||
+                    (err.type && err.type === "DishError")) {
+                    dish.set(err.message, "string");
+                    return i;
                 } else {
                     const e = typeof err == "string" ? { message: err } : err;
 
@@ -250,7 +301,8 @@ class Recipe  {
      * @returns {function} highlights[].b
      * @returns {Object[]} highlights[].args
      */
-    generateHighlightList() {
+    async generateHighlightList() {
+        await this._hydrateOpList();
         const highlights = [];
 
         for (let i = 0; i < this.opList.length; i++) {

src/core/Utils.mjs

@@ -5,10 +5,10 @@
  */
 
 import utf8 from "utf8";
-import moment from "moment-timezone";
-import {fromBase64} from "./lib/Base64";
+import {fromBase64, toBase64} from "./lib/Base64";
 import {fromHex} from "./lib/Hex";
 import {fromDecimal} from "./lib/Decimal";
+import {fromBinary} from "./lib/Binary";
 
 
 /**
@@ -298,7 +298,7 @@ class Utils {
      * Accepts hex, Base64, UTF8 and Latin1 strings.
      *
      * @param {string} str
-     * @param {string} type - One of "Hex", "Decimal", "Base64", "UTF8" or "Latin1"
+     * @param {string} type - One of "Binary", "Hex", "Decimal", "Base64", "UTF8" or "Latin1"
      * @returns {byteArray}
      *
      * @example
@@ -313,6 +313,8 @@ class Utils {
      */
     static convertToByteArray(str, type) {
         switch (type.toLowerCase()) {
+            case "binary":
+                return fromBinary(str);
             case "hex":
                 return fromHex(str);
             case "decimal":
@@ -333,7 +335,7 @@ class Utils {
      * Accepts hex, Base64, UTF8 and Latin1 strings.
      *
      * @param {string} str
-     * @param {string} type - One of "Hex", "Decimal", "Base64", "UTF8" or "Latin1"
+     * @param {string} type - One of "Binary", "Hex", "Decimal", "Base64", "UTF8" or "Latin1"
      * @returns {string}
      *
      * @example
@@ -348,6 +350,8 @@ class Utils {
      */
     static convertToByteString(str, type) {
         switch (type.toLowerCase()) {
+            case "binary":
+                return Utils.byteArrayToChars(fromBinary(str));
             case "hex":
                 return Utils.byteArrayToChars(fromHex(str));
             case "decimal":
@@ -555,8 +559,6 @@ class Utils {
             if (renderNext) {
                 cell += b;
                 renderNext = false;
-            } else if (b === "\\") {
-                renderNext = true;
             } else if (b === "\"" && !inString) {
                 inString = true;
             } else if (b === "\"" && inString) {
@@ -570,6 +572,10 @@ class Utils {
                 cell = "";
                 lines.push(line);
                 line = [];
+                // Skip next byte if it is also a line delim (e.g. \r\n)
+                if (lineDelims.indexOf(next) >= 0 && next !== b) {
+                    i++;
+                }
             } else {
                 cell += b;
             }
@@ -790,38 +796,6 @@ class Utils {
     }
 
 
-    /**
-     * Expresses a number of milliseconds in a human readable format.
-     *
-     * Range                        | Sample Output
-     * -----------------------------|-------------------------------
-     * 0 to 45 seconds              | a few seconds ago
-     * 45 to 90 seconds             | a minute ago
-     * 90 seconds to 45 minutes     | 2 minutes ago ... 45 minutes ago
-     * 45 to 90 minutes             | an hour ago
-     * 90 minutes to 22 hours       | 2 hours ago ... 22 hours ago
-     * 22 to 36 hours               | a day ago
-     * 36 hours to 25 days          | 2 days ago ... 25 days ago
-     * 25 to 45 days                | a month ago
-     * 45 to 345 days               | 2 months ago ... 11 months ago
-     * 345 to 545 days (1.5 years)  | a year ago
-     * 546 days+                    | 2 years ago ... 20 years ago
-     *
-     * @param {number} ms
-     * @returns {string}
-     *
-     * @example
-     * // returns "3 minutes"
-     * Utils.fuzzyTime(152435);
-     *
-     * // returns "5 days"
-     * Utils.fuzzyTime(456851321);
-     */
-    static fuzzyTime(ms) {
-        return moment.duration(ms, "milliseconds").humanize();
-    }
-
-
     /**
      * Formats a list of files or directories.
      *
@@ -843,12 +817,24 @@ class Utils {
             return html;
         };
 
+        const formatContent = function (buff, type) {
+            if (type.startsWith("image")) {
+                let dataURI = "data:";
+                dataURI += type + ";";
+                dataURI += "base64," + toBase64(buff);
+                return "<img style='max-width: 100%;' src='" + dataURI + "'>";
+            } else {
+                return `<pre>${Utils.escapeHtml(Utils.arrayBufferToStr(buff.buffer))}</pre>`;
+            }
+        };
+
         const formatFile = async function(file, i) {
             const buff = await Utils.readFile(file);
             const blob = new Blob(
                 [buff],
-                {type: "octet/stream"}
+                {type: file.type || "octet/stream"}
             );
+            const blobURL = URL.createObjectURL(blob);
 
             const html = `<div class='card' style='white-space: normal;'>
                     <div class='card-header' id='heading${i}'>
@@ -863,16 +849,25 @@ class Utils {
                             <span class='float-right' style="margin-top: -3px">
                                 ${file.size.toLocaleString()} bytes
                                 <a title="Download ${Utils.escapeHtml(file.name)}"
-                                    href='${URL.createObjectURL(blob)}'
-                                    download='${Utils.escapeHtml(file.name)}'>
+                                    href="${blobURL}"
+                                    download="${Utils.escapeHtml(file.name)}"
+                                    data-toggle="tooltip">
                                     <i class="material-icons" style="vertical-align: bottom">save</i>
                                 </a>
+                                <a title="Move to input"
+                                    href="#"
+                                    blob-url="${blobURL}"
+                                    file-name="${Utils.escapeHtml(file.name)}"
+                                    class="extract-file"
+                                    data-toggle="tooltip">
+                                    <i class="material-icons" style="vertical-align: bottom">open_in_browser</i>
+                                </a>
                             </span>
                         </h6>
                     </div>
                     <div id='collapse${i}' class='collapse' aria-labelledby='heading${i}' data-parent="#files">
                         <div class='card-body'>
-                            <pre>${Utils.escapeHtml(Utils.arrayBufferToStr(buff.buffer))}</pre>
+                            ${formatContent(buff, file.type)}
                         </div>
                     </div>
                 </div>`;
@@ -1178,6 +1173,21 @@ String.prototype.count = function(chr) {
 };
 
 
+/**
+ * Wrapper for self.sendStatusMessage to handle different environments.
+ *
+ * @param {string} msg
+ */
+export function sendStatusMessage(msg) {
+    if (ENVIRONMENT_IS_WORKER())
+        self.sendStatusMessage(msg);
+    else if (ENVIRONMENT_IS_WEB())
+        app.alert(msg, 10000);
+    else if (ENVIRONMENT_IS_NODE())
+        log.debug(msg);
+}
+
+
 /*
  * Polyfills
  */

src/core/config/Categories.json

@@ -25,6 +25,10 @@
             "From Base32",
             "To Base58",
             "From Base58",
+            "To Base62",
+            "From Base62",
+            "To Base85",
+            "From Base85",
             "To Base",
             "From Base",
             "To BCD",
@@ -47,7 +51,15 @@
             "Change IP format",
             "Encode text",
             "Decode text",
-            "Swap endianness"
+            "Text Encoding Brute Force",
+            "Swap endianness",
+            "To MessagePack",
+            "From MessagePack",
+            "To Braille",
+            "From Braille",
+            "Parse TLV",
+            "CSV to JSON",
+            "JSON to CSV"
         ]
     },
     {
@@ -77,12 +89,19 @@
             "Bifid Cipher Decode",
             "Affine Cipher Encode",
             "Affine Cipher Decode",
+            "A1Z26 Cipher Encode",
+            "A1Z26 Cipher Decode",
             "Atbash Cipher",
             "Substitute",
             "Derive PBKDF2 key",
             "Derive EVP key",
             "Bcrypt",
             "Scrypt",
+            "JWT Sign",
+            "JWT Verify",
+            "JWT Decode",
+            "Citrix CTX1 Encode",
+            "Citrix CTX1 Decode",
             "Pseudo-Random Number Generator"
         ]
     },
@@ -136,6 +155,7 @@
         "name": "Networking",
         "ops": [
             "HTTP request",
+            "DNS over HTTPS",
             "Strip HTTP headers",
             "Dechunk HTTP response",
             "Parse User Agent",
@@ -149,7 +169,8 @@
             "Change IP format",
             "Group IP addresses",
             "Encode NetBIOS Name",
-            "Decode NetBIOS Name"
+            "Decode NetBIOS Name",
+            "Defang URL"
         ]
     },
     {
@@ -157,6 +178,7 @@
         "ops": [
             "Encode text",
             "Decode text",
+            "Remove Diacritics",
             "Unescape Unicode Characters"
         ]
     },
@@ -168,6 +190,8 @@
             "Remove null bytes",
             "To Upper case",
             "To Lower case",
+			"To Case Insensitive Regex",
+			"From Case Insensitive Regex",
             "Add line numbers",
             "Remove line numbers",
             "To Table",
@@ -192,6 +216,7 @@
             "Convert mass",
             "Convert speed",
             "Convert data units",
+            "Convert co-ordinate format",
             "Parse UNIX file permissions",
             "Swap endianness",
             "Parse colour code",
@@ -229,7 +254,8 @@
             "XPath expression",
             "JPath expression",
             "CSS selector",
-            "Extract EXIF"
+            "Extract EXIF",
+            "Extract Files"
         ]
     },
     {
@@ -313,7 +339,42 @@
             "To Camel case",
             "To Kebab case",
             "BSON serialise",
-            "BSON deserialise"
+            "BSON deserialise",
+            "To MessagePack",
+            "From MessagePack"
+        ]
+    },
+    {
+        "name": "Forensics",
+        "ops": [
+            "Detect File Type",
+            "Scan for Embedded Files",
+            "Extract Files",
+            "Remove EXIF",
+            "Extract EXIF"
+        ]
+    },
+    {
+        "name": "Multimedia",
+        "ops": [
+            "Render Image",
+            "Play Media",
+            "Remove EXIF",
+            "Extract EXIF",
+            "Split Colour Channels",
+            "Rotate Image",
+            "Resize Image",
+            "Blur Image",
+            "Dither Image",
+            "Invert Image",
+            "Flip Image",
+            "Crop Image",
+            "Image Brightness / Contrast",
+            "Image Opacity",
+            "Image Filter",
+            "Contain Image",
+            "Cover Image",
+            "Image Hue/Saturation/Lightness"
         ]
     },
     {
@@ -322,17 +383,15 @@
             "Entropy",
             "Frequency distribution",
             "Chi Square",
-            "Detect File Type",
-            "Scan for Embedded Files",
             "Disassemble x86",
             "Pseudo-Random Number Generator",
             "Generate UUID",
             "Generate TOTP",
             "Generate HOTP",
+            "Generate QR Code",
+            "Parse QR Code",
             "Haversine distance",
-            "Render Image",
-            "Remove EXIF",
-            "Extract EXIF",
+            "Generate Lorem Ipsum",
             "Numberwang",
             "XKCD Random Number"
         ]
@@ -342,6 +401,7 @@
         "ops": [
             "Magic",
             "Fork",
+            "Subsection",
             "Merge",
             "Register",
             "Label",

src/core/config/scripts/generateConfig.mjs

@@ -41,6 +41,7 @@ for (const opObj in Ops) {
         inputType:   op.inputType,
         outputType:  op.presentType,
         flowControl: op.flowControl,
+        manualBake:  op.manualBake,
         args:        op.args
     };
 

src/core/config/scripts/newOperation.mjs

@@ -222,7 +222,7 @@ export default ${moduleName};
     console.log(`\nNext steps:
 1. Add your operation to ${colors.green("src/core/config/Categories.json")}
 2. Write your operation code.
-3. Write tests in ${colors.green("test/tests/operations/")}
+3. Write tests in ${colors.green("tests/operations/tests/")}
 4. Run ${colors.cyan("npm run lint")} and ${colors.cyan("npm run test")}
 5. Submit a Pull Request to get your operation added to the official CyberChef repository.`);
 

src/core/errors/DishError.mjs

@@ -0,0 +1,26 @@
+/**
+ * Custom error type for handling Dish type errors.
+ * i.e. where the Dish cannot be successfully translated between types
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+class DishError extends Error {
+    /**
+     * Standard error constructor. Adds no new behaviour.
+     *
+     * @param args - Standard error args
+     */
+    constructor(...args) {
+        super(...args);
+
+        this.type = "DishError";
+
+        if (Error.captureStackTrace) {
+            Error.captureStackTrace(this, DishError);
+        }
+    }
+}
+
+export default DishError;

src/core/lib/Base64.mjs

@@ -81,6 +81,7 @@ export function fromBase64(data, alphabet="A-Za-z0-9+/=", returnType="string", r
         return returnType === "string" ? "" : [];
     }
 
+    alphabet = alphabet || "A-Za-z0-9+/=";
     alphabet = Utils.expandAlphRange(alphabet).join("");
 
     const output = [];
@@ -125,14 +126,14 @@ export function fromBase64(data, alphabet="A-Za-z0-9+/=", returnType="string", r
  * Base64 alphabets.
  */
 export const ALPHABET_OPTIONS = [
-    {name: "Standard: A-Za-z0-9+/=", value: "A-Za-z0-9+/="},
-    {name: "URL safe: A-Za-z0-9-_", value: "A-Za-z0-9-_"},
+    {name: "Standard (RFC 4648): A-Za-z0-9+/=", value: "A-Za-z0-9+/="},
+    {name: "URL safe (RFC 4648 \u00A75): A-Za-z0-9-_", value: "A-Za-z0-9-_"},
     {name: "Filename safe: A-Za-z0-9+-=", value: "A-Za-z0-9+\\-="},
     {name: "itoa64: ./0-9A-Za-z=", value: "./0-9A-Za-z="},
     {name: "XML: A-Za-z0-9_.", value: "A-Za-z0-9_."},
     {name: "y64: A-Za-z0-9._-", value: "A-Za-z0-9._-"},
     {name: "z64: 0-9a-zA-Z+/=", value: "0-9a-zA-Z+/="},
-    {name: "Radix-64: 0-9A-Za-z+/=", value: "0-9A-Za-z+/="},
+    {name: "Radix-64 (RFC 4880): 0-9A-Za-z+/=", value: "0-9A-Za-z+/="},
     {name: "Uuencoding: [space]-_", value: " -_"},
     {name: "Xxencoding: +-0-9A-Za-z", value: "+\\-0-9A-Za-z"},
     {name: "BinHex: !-,-0-689@A-NP-VX-Z[`a-fh-mp-r", value: "!-,-0-689@A-NP-VX-Z[`a-fh-mp-r"},

src/core/lib/Base85.mjs

@@ -0,0 +1,45 @@
+/**
+ * Base85 resources.
+ *
+ * @author PenguinGeorge [george@penguingeorge.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+/**
+ * Base85 alphabet options.
+ */
+export const ALPHABET_OPTIONS = [
+    {
+        name: "Standard",
+        value: "!-u",
+    },
+    {
+        name: "Z85 (ZeroMQ)",
+        value: "0-9a-zA-Z.\\-:+=^!/*?&<>()[]{}@%$#",
+    },
+    {
+        name: "IPv6",
+        value: "0-9A-Za-z!#$%&()*+\\-;<=>?@^_`{|~}",
+    }
+];
+
+
+/**
+ * Returns the name of the alphabet, when given the alphabet.
+ *
+ * @param {string} alphabet
+ * @returns {string}
+ */
+export function alphabetName(alphabet) {
+    alphabet = alphabet.replace("'", "&apos;");
+    alphabet = alphabet.replace("\"", "&quot;");
+    alphabet = alphabet.replace("\\", "&bsol;");
+    let name;
+
+    ALPHABET_OPTIONS.forEach(function(a) {
+        if (escape(alphabet) === escape(a.value)) name = a.name;
+    });
+
+    return name;
+}

src/core/lib/Binary.mjs

@@ -0,0 +1,70 @@
+/**
+ * Binary functions.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Utils from "../Utils";
+
+
+/**
+ * Convert a byte array into a binary string.
+ *
+ * @param {Uint8Array|byteArray} data
+ * @param {string} [delim="Space"]
+ * @param {number} [padding=8]
+ * @returns {string}
+ *
+ * @example
+ * // returns "00010000 00100000 00110000"
+ * toBinary([10,20,30]);
+ *
+ * // returns "00010000 00100000 00110000"
+ * toBinary([10,20,30], ":");
+ */
+export function toBinary(data, delim="Space", padding=8) {
+    if (!data) return "";
+
+    delim = Utils.charRep(delim);
+    let output = "";
+
+    for (let i = 0; i < data.length; i++) {
+        output += data[i].toString(2).padStart(padding, "0") + delim;
+    }
+
+    if (delim.length) {
+        return output.slice(0, -delim.length);
+    } else {
+        return output;
+    }
+}
+
+
+/**
+ * Convert a binary string into a byte array.
+ *
+ * @param {string} data
+ * @param {string} [delim]
+ * @param {number} [byteLen=8]
+ * @returns {byteArray}
+ *
+ * @example
+ * // returns [10,20,30]
+ * fromBinary("00010000 00100000 00110000");
+ *
+ * // returns [10,20,30]
+ * fromBinary("00010000:00100000:00110000", "Colon");
+ */
+export function fromBinary(data, delim="Space", byteLen=8) {
+    const delimRegex = Utils.regexRep(delim);
+    data = data.replace(delimRegex, "");
+
+    const output = [];
+    for (let i = 0; i < data.length; i += byteLen) {
+        output.push(parseInt(data.substr(i, byteLen), 2));
+    }
+    return output;
+}
+

src/core/lib/BitwiseOp.mjs

@@ -25,6 +25,7 @@ export function bitOp (input, key, func, nullPreserving, scheme) {
 
     for (let i = 0; i < input.length; i++) {
         k = key[i % key.length];
+        if (scheme === "Cascade") k = input[i + 1] || 0;
         o = input[i];
         x = nullPreserving && (o === 0 || o === k) ? o : func(o, k);
         result.push(x);
@@ -115,3 +116,9 @@ export function sub(operand, key) {
     const result = operand - key;
     return (result < 0) ? 256 + result : result;
 }
+
+
+/**
+ * Delimiter options for bitwise operations
+ */
+export const BITWISE_OP_DELIMS = ["Hex", "Decimal", "Binary", "Base64", "UTF8", "Latin1"];

src/core/lib/Braille.mjs

@@ -0,0 +1,15 @@
+/**
+ * Braille resources.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+/**
+ * Braille lookup table.
+ */
+export const BRAILLE_LOOKUP = {
+    ascii: " A1B'K2L@CIF/MSP\"E3H9O6R^DJG>NTQ,*5<-U8V.%[$+X!&;:4\\0Z7(_?W]#Y)=",
+    dot6:  "⠀⠁⠂⠃⠄⠅⠆⠇⠈⠉⠊⠋⠌⠍⠎⠏⠐⠑⠒⠓⠔⠕⠖⠗⠘⠙⠚⠛⠜⠝⠞⠟⠠⠡⠢⠣⠤⠥⠦⠧⠨⠩⠪⠫⠬⠭⠮⠯⠰⠱⠲⠳⠴⠵⠶⠷⠸⠹⠺⠻⠼⠽⠾⠿"
+};

src/core/lib/ConvertCoordinates.mjs

@@ -0,0 +1,655 @@
+/**
+ * Co-ordinate conversion resources.
+ *
+ * @author j433866 [j433866@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import geohash from "ngeohash";
+import geodesy from "geodesy";
+import OperationError from "../errors/OperationError";
+
+/**
+ * Co-ordinate formats
+ */
+export const FORMATS = [
+    "Degrees Minutes Seconds",
+    "Degrees Decimal Minutes",
+    "Decimal Degrees",
+    "Geohash",
+    "Military Grid Reference System",
+    "Ordnance Survey National Grid",
+    "Universal Transverse Mercator"
+];
+
+/**
+ * Formats that should be passed to the conversion module as-is
+ */
+const NO_CHANGE = [
+    "Geohash",
+    "Military Grid Reference System",
+    "Ordnance Survey National Grid",
+    "Universal Transverse Mercator",
+];
+
+/**
+ * Convert a given latitude and longitude into a different format.
+ *
+ * @param {string} input - Input string to be converted
+ * @param {string} inFormat - Format of the input coordinates
+ * @param {string} inDelim - The delimiter splitting the lat/long of the input
+ * @param {string} outFormat - Format to convert to
+ * @param {string} outDelim - The delimiter to separate the output with
+ * @param {string} includeDir - Whether or not to include the compass direction in the output
+ * @param {number} precision - Precision of the result
+ * @returns {string} A formatted string of the converted co-ordinates
+ */
+export function convertCoordinates (input, inFormat, inDelim, outFormat, outDelim, includeDir, precision) {
+    let isPair = false,
+        split,
+        latlon,
+        convLat,
+        convLon,
+        conv,
+        hash,
+        utm,
+        mgrs,
+        osng,
+        splitLat,
+        splitLong,
+        lat,
+        lon;
+
+    // Can't have a precision less than 0!
+    if (precision < 0) {
+        precision = 0;
+    }
+
+    if (inDelim === "Auto") {
+        // Try to detect a delimiter in the input.
+        inDelim = findDelim(input);
+        if (inDelim === null) {
+            throw new OperationError("Unable to detect the input delimiter automatically.");
+        }
+    } else if (!inDelim.includes("Direction")) {
+        // Convert the delimiter argument value to the actual character
+        inDelim = realDelim(inDelim);
+    }
+
+    if (inFormat === "Auto") {
+        // Try to detect the format of the input data
+        inFormat = findFormat(input, inDelim);
+        if (inFormat === null) {
+            throw new OperationError("Unable to detect the input format automatically.");
+        }
+    }
+
+    // Convert the output delimiter argument to the real character
+    outDelim = realDelim(outDelim);
+
+    if (!NO_CHANGE.includes(inFormat)) {
+        if (inDelim.includes("Direction")) {
+            // Split on directions
+            split = input.split(/[NnEeSsWw]/g);
+            if (split[0] === "") {
+                // Remove first element if direction preceding
+                split = split.slice(1);
+            }
+        } else {
+            split = input.split(inDelim);
+        }
+        // Replace any co-ordinate symbols with spaces so we can split on them later
+        for (let i = 0; i < split.length; i++) {
+            split[i] = split[i].replace(/[°˝´'"]/g, " ");
+        }
+        if (split.length > 1) {
+            isPair = true;
+        }
+    } else {
+        // Remove any delimiters from the input
+        input = input.replace(inDelim, "");
+        isPair = true;
+    }
+
+    // Conversions from the input format into a geodesy latlon object
+    switch (inFormat) {
+        case "Geohash":
+            hash = geohash.decode(input.replace(/[^A-Za-z0-9]/g, ""));
+            latlon = new geodesy.LatLonEllipsoidal(hash.latitude, hash.longitude);
+            break;
+        case "Military Grid Reference System":
+            utm = geodesy.Mgrs.parse(input.replace(/[^A-Za-z0-9]/g, "")).toUtm();
+            latlon = utm.toLatLonE();
+            break;
+        case "Ordnance Survey National Grid":
+            osng = geodesy.OsGridRef.parse(input.replace(/[^A-Za-z0-9]/g, ""));
+            latlon = geodesy.OsGridRef.osGridToLatLon(osng);
+            break;
+        case "Universal Transverse Mercator":
+            // Geodesy needs a space between the first 2 digits and the next letter
+            if (/^[\d]{2}[A-Za-z]/.test(input)) {
+                input = input.slice(0, 2) + " " + input.slice(2);
+            }
+            utm = geodesy.Utm.parse(input);
+            latlon = utm.toLatLonE();
+            break;
+        case "Degrees Minutes Seconds":
+            if (isPair) {
+                // Split up the lat/long into degrees / minutes / seconds values
+                splitLat = splitInput(split[0]);
+                splitLong = splitInput(split[1]);
+
+                if (splitLat.length >= 3 && splitLong.length >= 3) {
+                    lat = convDMSToDD(splitLat[0], splitLat[1], splitLat[2], 10);
+                    lon = convDMSToDD(splitLong[0], splitLong[1], splitLong[2], 10);
+                    latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lon.degrees);
+                } else {
+                    throw new OperationError("Invalid co-ordinate format for Degrees Minutes Seconds");
+                }
+            } else {
+                // Not a pair, so only try to convert one set of co-ordinates
+                splitLat = splitInput(split[0]);
+                if (splitLat.length >= 3) {
+                    lat = convDMSToDD(splitLat[0], splitLat[1], splitLat[2]);
+                    latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lat.degrees);
+                } else {
+                    throw new OperationError("Invalid co-ordinate format for Degrees Minutes Seconds");
+                }
+            }
+            break;
+        case "Degrees Decimal Minutes":
+            if (isPair) {
+                splitLat = splitInput(split[0]);
+                splitLong = splitInput(split[1]);
+                if (splitLat.length !== 2 || splitLong.length !== 2) {
+                    throw new OperationError("Invalid co-ordinate format for Degrees Decimal Minutes.");
+                }
+                // Convert to decimal degrees, and then convert to a geodesy object
+                lat = convDDMToDD(splitLat[0], splitLat[1], 10);
+                lon = convDDMToDD(splitLong[0], splitLong[1], 10);
+                latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lon.degrees);
+            } else {
+                // Not a pair, so only try to convert one set of co-ordinates
+                splitLat = splitInput(input);
+                if (splitLat.length !== 2) {
+                    throw new OperationError("Invalid co-ordinate format for Degrees Decimal Minutes.");
+                }
+                lat = convDDMToDD(splitLat[0], splitLat[1], 10);
+                latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lat.degrees);
+            }
+            break;
+        case "Decimal Degrees":
+            if (isPair) {
+                splitLat =  splitInput(split[0]);
+                splitLong = splitInput(split[1]);
+                if (splitLat.length !== 1 || splitLong.length !== 1) {
+                    throw new OperationError("Invalid co-ordinate format for Decimal Degrees.");
+                }
+                latlon = new geodesy.LatLonEllipsoidal(splitLat[0], splitLong[0]);
+            } else {
+                // Not a pair, so only try to convert one set of co-ordinates
+                splitLat = splitInput(split[0]);
+                if (splitLat.length !== 1) {
+                    throw new OperationError("Invalid co-ordinate format for Decimal Degrees.");
+                }
+                latlon = new geodesy.LatLonEllipsoidal(splitLat[0], splitLat[0]);
+            }
+            break;
+        default:
+            throw new OperationError(`Unknown input format '${inFormat}'`);
+    }
+
+    // Everything is now a geodesy latlon object
+    // These store the latitude and longitude as decimal
+    if (inFormat.includes("Degrees")) {
+        // If the input string contains directions, we need to check if they're S or W.
+        // If either of the directions are, we should make the decimal value negative
+        const dirs = input.toUpperCase().match(/[NESW]/g);
+        if (dirs && dirs.length >= 1) {
+            // Make positive lat/lon values with S/W directions into negative values
+            if (dirs[0] === "S" || dirs[0] === "W" && latlon.lat > 0) {
+                latlon.lat = -latlon.lat;
+            }
+            if (dirs.length >= 2) {
+                if (dirs[1] === "S" || dirs[1] === "W" && latlon.lon > 0) {
+                    latlon.lon = -latlon.lon;
+                }
+            }
+        }
+    }
+
+    // Try to find the compass directions of the lat and long
+    const [latDir, longDir] = findDirs(latlon.lat + "," + latlon.lon, ",");
+
+    // Output conversions for each output format
+    switch (outFormat) {
+        case "Decimal Degrees":
+            // We could use the built in latlon.toString(),
+            // but this makes adjusting the output harder
+            lat = convDDToDD(latlon.lat, precision);
+            lon = convDDToDD(latlon.lon, precision);
+            convLat = lat.string;
+            convLon = lon.string;
+            break;
+        case "Degrees Decimal Minutes":
+            lat = convDDToDDM(latlon.lat, precision);
+            lon = convDDToDDM(latlon.lon, precision);
+            convLat = lat.string;
+            convLon = lon.string;
+            break;
+        case "Degrees Minutes Seconds":
+            lat = convDDToDMS(latlon.lat, precision);
+            lon = convDDToDMS(latlon.lon, precision);
+            convLat = lat.string;
+            convLon = lon.string;
+            break;
+        case "Geohash":
+            convLat = geohash.encode(latlon.lat, latlon.lon, precision);
+            break;
+        case "Military Grid Reference System":
+            utm = latlon.toUtm();
+            mgrs = utm.toMgrs();
+            // MGRS wants a precision that's an even number between 2 and 10
+            if (precision % 2 !== 0) {
+                precision = precision + 1;
+            }
+            if (precision > 10) {
+                precision = 10;
+            }
+            convLat = mgrs.toString(precision);
+            break;
+        case "Ordnance Survey National Grid":
+            osng = geodesy.OsGridRef.latLonToOsGrid(latlon);
+            if (osng.toString() === "") {
+                throw new OperationError("Could not convert co-ordinates to OS National Grid. Are the co-ordinates in range?");
+            }
+            // OSNG wants a precision that's an even number between 2 and 10
+            if (precision % 2 !== 0) {
+                precision = precision + 1;
+            }
+            if (precision > 10) {
+                precision = 10;
+            }
+            convLat = osng.toString(precision);
+            break;
+        case "Universal Transverse Mercator":
+            utm = latlon.toUtm();
+            convLat = utm.toString(precision);
+            break;
+    }
+
+    if (convLat === undefined) {
+        throw new OperationError("Error converting co-ordinates.");
+    }
+
+    if (outFormat.includes("Degrees")) {
+        // Format DD/DDM/DMS for output
+        // If we're outputting a compass direction, remove the negative sign
+        if (latDir === "S" && includeDir !== "None") {
+            convLat = convLat.replace("-", "");
+        }
+        if (longDir === "W" && includeDir !== "None") {
+            convLon = convLon.replace("-", "");
+        }
+
+        let outConv = "";
+        if (includeDir === "Before") {
+            outConv += latDir + " ";
+        }
+
+        outConv += convLat;
+        if (includeDir === "After") {
+            outConv += " " + latDir;
+        }
+        outConv += outDelim;
+        if (isPair) {
+            if (includeDir === "Before") {
+                outConv += longDir + " ";
+            }
+            outConv += convLon;
+            if (includeDir === "After") {
+                outConv += " " + longDir;
+            }
+            outConv += outDelim;
+        }
+        conv = outConv;
+    } else {
+        conv = convLat + outDelim;
+    }
+
+    return conv;
+}
+
+/**
+ * Split up the input using a space or degrees signs, and sanitise the result
+ *
+ * @param {string} input - The input data to be split
+ * @returns {number[]} An array of the different items in the string, stored as floats
+ */
+function splitInput (input){
+    const split = [];
+
+    input.split(/\s+/).forEach(item => {
+        // Remove any character that isn't a digit, decimal point or negative sign
+        item = item.replace(/[^0-9.-]/g, "");
+        if (item.length > 0){
+            // Turn the item into a float
+            split.push(parseFloat(item));
+        }
+    });
+    return split;
+}
+
+/**
+ * Convert Degrees Minutes Seconds to Decimal Degrees
+ *
+ * @param {number} degrees - The degrees of the input co-ordinates
+ * @param {number} minutes - The minutes of the input co-ordinates
+ * @param {number} seconds - The seconds of the input co-ordinates
+ * @param {number} precision - The precision the result should be rounded to
+ * @returns {{string: string, degrees: number}} An object containing the raw converted value (obj.degrees), and a formatted string version (obj.string)
+ */
+function convDMSToDD (degrees, minutes, seconds, precision){
+    const absDegrees = Math.abs(degrees);
+    let conv = absDegrees + (minutes / 60) + (seconds / 3600);
+    let outString = round(conv, precision) + "°";
+    if (isNegativeZero(degrees) || degrees < 0) {
+        conv = -conv;
+        outString = "-" + outString;
+    }
+    return {
+        "degrees": conv,
+        "string": outString
+    };
+}
+
+/**
+ * Convert Decimal Degrees Minutes to Decimal Degrees
+ *
+ * @param {number} degrees - The input degrees to be converted
+ * @param {number} minutes - The input minutes to be converted
+ * @param {number} precision - The precision which the result should be rounded to
+ * @returns {{string: string, degrees: number}} An object containing the raw converted value (obj.degrees), and a formatted string version (obj.string)
+ */
+function convDDMToDD (degrees, minutes, precision) {
+    const absDegrees = Math.abs(degrees);
+    let conv = absDegrees + minutes / 60;
+    let outString = round(conv, precision) + "°";
+    if (isNegativeZero(degrees) || degrees < 0) {
+        conv = -conv;
+        outString = "-" + outString;
+    }
+    return {
+        "degrees": conv,
+        "string": outString
+    };
+}
+
+/**
+ * Convert Decimal Degrees to Decimal Degrees
+ *
+ * Doesn't affect the input, just puts it into an object
+ * @param {number} degrees - The input degrees to be converted
+ * @param {number} precision - The precision which the result should be rounded to
+ * @returns {{string: string, degrees: number}} An object containing the raw converted value (obj.degrees), and a formatted string version (obj.string)
+ */
+function convDDToDD (degrees, precision) {
+    return {
+        "degrees": degrees,
+        "string": round(degrees, precision) + "°"
+    };
+}
+
+/**
+ * Convert Decimal Degrees to Degrees Minutes Seconds
+ *
+ * @param {number} decDegrees - The input data to be converted
+ * @param {number} precision - The precision which the result should be rounded to
+ * @returns {{string: string, degrees: number, minutes: number, seconds: number}} An object containing the raw converted value as separate numbers (.degrees, .minutes, .seconds), and a formatted string version (obj.string)
+ */
+function convDDToDMS (decDegrees, precision) {
+    const absDegrees = Math.abs(decDegrees);
+    let degrees = Math.floor(absDegrees);
+    const minutes = Math.floor(60 * (absDegrees - degrees)),
+        seconds = round(3600 * (absDegrees - degrees) - 60 * minutes, precision);
+    let outString = degrees + "° " + minutes + "' " + seconds + "\"";
+    if (isNegativeZero(decDegrees) || decDegrees < 0) {
+        degrees = -degrees;
+        outString = "-" + outString;
+    }
+    return {
+        "degrees": degrees,
+        "minutes": minutes,
+        "seconds": seconds,
+        "string": outString
+    };
+}
+
+/**
+ * Convert Decimal Degrees to Degrees Decimal Minutes
+ *
+ * @param {number} decDegrees - The input degrees to be converted
+ * @param {number} precision - The precision the input data should be rounded to
+ * @returns {{string: string, degrees: number, minutes: number}} An object containing the raw converted value as separate numbers (.degrees, .minutes), and a formatted string version (obj.string)
+ */
+function convDDToDDM (decDegrees, precision) {
+    const absDegrees = Math.abs(decDegrees);
+    let degrees = Math.floor(absDegrees);
+    const minutes = absDegrees - degrees,
+        decMinutes = round(minutes * 60, precision);
+    let outString = degrees + "° " + decMinutes + "'";
+    if (decDegrees < 0 || isNegativeZero(decDegrees)) {
+        degrees = -degrees;
+        outString = "-" + outString;
+    }
+
+    return {
+        "degrees": degrees,
+        "minutes": decMinutes,
+        "string": outString,
+    };
+}
+
+/**
+ * Finds and returns the compass directions in an input string
+ *
+ * @param {string} input - The input co-ordinates containing the direction
+ * @param {string} delim - The delimiter separating latitide and longitude
+ * @returns {string[]} String array containing the latitude and longitude directions
+ */
+export function findDirs(input, delim) {
+    const upperInput = input.toUpperCase();
+    const dirExp = new RegExp(/[NESW]/g);
+
+    const dirs = upperInput.match(dirExp);
+
+    if (dirs) {
+        // If there's actually compass directions
+        // in the input, use these to work out the direction
+        if (dirs.length <= 2 && dirs.length >= 1) {
+            return dirs.length === 2 ? [dirs[0], dirs[1]] : [dirs[0], ""];
+        }
+    }
+
+    // Nothing was returned, so guess the directions
+    let lat = upperInput,
+        long,
+        latDir = "",
+        longDir = "";
+    if (!delim.includes("Direction")) {
+        if (upperInput.includes(delim)) {
+            const split = upperInput.split(delim);
+            if (split.length >= 1) {
+                if (split[0] !== "") {
+                    lat = split[0];
+                }
+                if (split.length >= 2 && split[1] !== "") {
+                    long = split[1];
+                }
+            }
+        }
+    } else {
+        const split = upperInput.split(dirExp);
+        if (split.length > 1) {
+            lat = split[0] === "" ? split[1] : split[0];
+            if (split.length > 2 && split[2] !== "") {
+                long = split[2];
+            }
+        }
+    }
+
+    if (lat) {
+        lat = parseFloat(lat);
+        latDir = lat < 0 ? "S" : "N";
+    }
+
+    if (long) {
+        long = parseFloat(long);
+        longDir = long < 0 ? "W" : "E";
+    }
+
+    return [latDir, longDir];
+}
+
+/**
+ * Detects the co-ordinate format of the input data
+ *
+ * @param {string} input - The input data whose format we need to detect
+ * @param {string} delim - The delimiter separating the data in input
+ * @returns {string} The input format
+ */
+export function findFormat (input, delim) {
+    let testData;
+    const mgrsPattern = new RegExp(/^[0-9]{2}\s?[C-HJ-NP-X]{1}\s?[A-HJ-NP-Z][A-HJ-NP-V]\s?[0-9\s]+/),
+        osngPattern = new RegExp(/^[A-HJ-Z]{2}\s+[0-9\s]+$/),
+        geohashPattern = new RegExp(/^[0123456789BCDEFGHJKMNPQRSTUVWXYZ]+$/),
+        utmPattern = new RegExp(/^[0-9]{2}\s?[C-HJ-NP-X]\s[0-9.]+\s?[0-9.]+$/),
+        degPattern = new RegExp(/[°'"]/g);
+
+    input = input.trim();
+
+    if (delim !== null && delim.includes("Direction")) {
+        const split = input.split(/[NnEeSsWw]/);
+        if (split.length > 1) {
+            testData = split[0] === "" ? split[1] : split[0];
+        }
+    } else if (delim !== null && delim !== "") {
+        if (input.includes(delim)) {
+            const split = input.split(delim);
+            if (split.length > 1) {
+                testData = split[0] === "" ? split[1] : split[0];
+            }
+        } else {
+            testData = input;
+        }
+    }
+
+    // Test non-degrees formats
+    if (!degPattern.test(input)) {
+        const filteredInput = input.toUpperCase().replace(delim, "");
+
+        if (utmPattern.test(filteredInput)) {
+            return "Universal Transverse Mercator";
+        }
+        if (mgrsPattern.test(filteredInput)) {
+            return "Military Grid Reference System";
+        }
+        if (osngPattern.test(filteredInput)) {
+            return "Ordnance Survey National Grid";
+        }
+        if (geohashPattern.test(filteredInput)) {
+            return "Geohash";
+        }
+    }
+
+    // Test DMS/DDM/DD formats
+    if (testData !== undefined) {
+        const split = splitInput(testData);
+        switch (split.length){
+            case 3:
+                return "Degrees Minutes Seconds";
+            case 2:
+                return "Degrees Decimal Minutes";
+            case 1:
+                return "Decimal Degrees";
+        }
+    }
+    return null;
+}
+
+/**
+ * Automatically find the delimeter type from the given input
+ *
+ * @param {string} input
+ * @returns {string} Delimiter type
+ */
+export function findDelim (input) {
+    input = input.trim();
+    const delims = [",", ";", ":"];
+    const testDir = input.match(/[NnEeSsWw]/g);
+    if (testDir !== null && testDir.length > 0 && testDir.length < 3) {
+        // Possibly contains a direction
+        const splitInput = input.split(/[NnEeSsWw]/);
+        if (splitInput.length <= 3 && splitInput.length > 0) {
+            // If there's 3 splits (one should be empty), then assume we have directions
+            if (splitInput[0] === "") {
+                return "Direction Preceding";
+            } else if (splitInput[splitInput.length - 1] === "") {
+                return "Direction Following";
+            }
+        }
+    }
+
+    // Loop through the standard delimiters, and try to find them in the input
+    for (let i = 0; i < delims.length; i++) {
+        const delim = delims[i];
+        if (input.includes(delim)) {
+            const splitInput = input.split(delim);
+            if (splitInput.length <= 3 && splitInput.length > 0) {
+                // Don't want to try and convert more than 2 co-ordinates
+                return delim;
+            }
+        }
+    }
+    return null;
+}
+
+/**
+ * Gets the real string for a delimiter name.
+ *
+ * @param {string} delim The delimiter to be matched
+ * @returns {string}
+ */
+export function realDelim (delim) {
+    return {
+        "Auto":         "Auto",
+        "Space":        " ",
+        "\\n":          "\n",
+        "Comma":        ",",
+        "Semi-colon":   ";",
+        "Colon":        ":"
+    }[delim];
+}
+
+/**
+ * Returns true if a zero is negative
+ *
+ * @param {number} zero
+ * @returns {boolean}
+ */
+function isNegativeZero(zero) {
+    return zero === 0 && (1/zero < 0);
+}
+
+/**
+ * Rounds a number to a specified number of decimal places
+ *
+ * @param {number} input - The number to be rounded
+ * @param {precision} precision - The number of decimal places the number should be rounded to
+ * @returns {number}
+ */
+function round(input, precision) {
+    precision = Math.pow(10, precision);
+    return Math.round(input * precision) / precision;
+}

src/core/lib/Extract.mjs

@@ -39,3 +39,21 @@ export function search (input, searchRegex, removeRegex, includeTotal) {
 
     return output;
 }
+
+
+/**
+ * URL regular expression
+ */
+const protocol = "[A-Z]+://",
+    hostname = "[-\\w]+(?:\\.\\w[-\\w]*)+",
+    port = ":\\d+",
+    path = "/[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]*" +
+        "(?:[.!,?]+[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]+)*";
+
+export const URL_REGEX = new RegExp(protocol + hostname + "(?:" + port + ")?(?:" + path + ")?", "ig");
+
+
+/**
+ * Domain name regular expression
+ */
+export const DOMAIN_REGEX = /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/ig;

src/core/lib/FileType.mjs

@@ -0,0 +1,263 @@
+/**
+ * File type functions
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ *
+ */
+import {FILE_SIGNATURES} from "./FileSignatures";
+import {sendStatusMessage} from "../Utils";
+
+
+/**
+ * Checks whether a signature matches a buffer.
+ *
+ * @param {Object|Object[]} sig - A dictionary of offsets with values assigned to them.
+ *   These values can be numbers for static checks, arrays of potential valid matches,
+ *   or bespoke functions to check the validity of the buffer value at that offset.
+ * @param {Uint8Array} buf
+ * @param {number} [offset=0] Where in the buffer to start searching from
+ * @returns {boolean}
+ */
+function signatureMatches(sig, buf, offset=0) {
+    // Using a length check seems to be more performant than `sig instanceof Array`
+    if (sig.length) {
+        // sig is an Array - return true if any of them match
+        // The following `reduce` method is nice, but performance matters here, so we
+        // opt for a faster, if less elegant, for loop.
+        // return sig.reduce((acc, s) => acc || bytesMatch(s, buf, offset), false);
+        for (let i = 0; i < sig.length; i++) {
+            if (bytesMatch(sig[i], buf, offset)) return true;
+        }
+        return false;
+    } else {
+        return bytesMatch(sig, buf, offset);
+    }
+}
+
+
+/**
+ * Checks whether a set of bytes match the given buffer.
+ *
+ * @param {Object} sig - A dictionary of offsets with values assigned to them.
+ *   These values can be numbers for static checks, arrays of potential valid matches,
+ *   or bespoke functions to check the validity of the buffer value at that offset.
+ * @param {Uint8Array} buf
+ * @param {number} [offset=0] Where in the buffer to start searching from
+ * @returns {boolean}
+ */
+function bytesMatch(sig, buf, offset=0) {
+    for (const sigoffset in sig) {
+        const pos = parseInt(sigoffset, 10) + offset;
+        switch (typeof sig[sigoffset]) {
+            case "number": // Static check
+                if (buf[pos] !== sig[sigoffset])
+                    return false;
+                break;
+            case "object": // Array of options
+                if (sig[sigoffset].indexOf(buf[pos]) < 0)
+                    return false;
+                break;
+            case "function": // More complex calculation
+                if (!sig[sigoffset](buf[pos]))
+                    return false;
+                break;
+            default:
+                throw new Error(`Unrecognised signature type at offset ${sigoffset}`);
+        }
+    }
+    return true;
+}
+
+
+/**
+ * Given a buffer, detects magic byte sequences at specific positions and returns the
+ * extension and mime type.
+ *
+ * @param {Uint8Array} buf
+ * @param {string[]} [categories=All] - Which categories of file to look for
+ * @returns {Object[]} types
+ * @returns {string} type.name - Name of file type
+ * @returns {string} type.ext - File extension
+ * @returns {string} type.mime - Mime type
+ * @returns {string} [type.desc] - Description
+ */
+export function detectFileType(buf, categories=Object.keys(FILE_SIGNATURES)) {
+    if (!(buf && buf.length > 1)) {
+        return [];
+    }
+
+    const matchingFiles = [];
+    const signatures = {};
+
+    for (const cat in FILE_SIGNATURES) {
+        if (categories.includes(cat)) {
+            signatures[cat] = FILE_SIGNATURES[cat];
+        }
+    }
+
+    for (const cat in signatures) {
+        const category = signatures[cat];
+
+        category.forEach(filetype => {
+            if (signatureMatches(filetype.signature, buf)) {
+                matchingFiles.push(filetype);
+            }
+        });
+    }
+    return matchingFiles;
+}
+
+
+/**
+ * Given a buffer, searches for magic byte sequences at all possible positions and returns
+ * the extensions and mime types.
+ *
+ * @param {Uint8Array} buf
+ * @param {string[]} [categories=All] - Which categories of file to look for
+ * @returns {Object[]} foundFiles
+ * @returns {number} foundFiles.offset - The position in the buffer at which this file was found
+ * @returns {Object} foundFiles.fileDetails
+ * @returns {string} foundFiles.fileDetails.name - Name of file type
+ * @returns {string} foundFiles.fileDetails.ext - File extension
+ * @returns {string} foundFiles.fileDetails.mime - Mime type
+ * @returns {string} [foundFiles.fileDetails.desc] - Description
+ */
+export function scanForFileTypes(buf, categories=Object.keys(FILE_SIGNATURES)) {
+    if (!(buf && buf.length > 1)) {
+        return [];
+    }
+
+    const foundFiles = [];
+    const signatures = {};
+
+    for (const cat in FILE_SIGNATURES) {
+        if (categories.includes(cat)) {
+            signatures[cat] = FILE_SIGNATURES[cat];
+        }
+    }
+
+    for (const cat in signatures) {
+        const category = signatures[cat];
+
+        for (let i = 0; i < category.length; i++) {
+            const filetype = category[i];
+            const sigs = filetype.signature.length ? filetype.signature : [filetype.signature];
+
+            sigs.forEach(sig => {
+                let pos = 0;
+                while ((pos = locatePotentialSig(buf, sig, pos)) >= 0) {
+                    if (bytesMatch(sig, buf, pos)) {
+                        sendStatusMessage(`Found potential signature for ${filetype.name} at pos ${pos}`);
+                        foundFiles.push({
+                            offset: pos,
+                            fileDetails: filetype
+                        });
+                    }
+                    pos++;
+                }
+            });
+        }
+    }
+
+    // Return found files in order of increasing offset
+    return foundFiles.sort((a, b) => {
+        return a.offset - b.offset;
+    });
+}
+
+
+/**
+ * Fastcheck function to quickly scan the buffer for the first byte in a signature.
+ *
+ * @param {Uint8Array} buf - The buffer to search
+ * @param {Object} sig - A single signature object (Not an array of signatures)
+ * @param {number} offset - Where to start search from
+ * @returs {number} The position of the match or -1 if one cannot be found.
+ */
+function locatePotentialSig(buf, sig, offset) {
+    // Find values for first key and value in sig
+    const k = parseInt(Object.keys(sig)[0], 10);
+    const v = Object.values(sig)[0];
+    switch (typeof v) {
+        case "number":
+            return buf.indexOf(v, offset + k) - k;
+        case "object":
+            for (let i = offset + k; i < buf.length; i++) {
+                if (v.indexOf(buf[i]) >= 0) return i - k;
+            }
+            return -1;
+        case "function":
+            for (let i = offset + k; i < buf.length; i++) {
+                if (v(buf[i])) return i - k;
+            }
+            return -1;
+        default:
+            throw new Error("Unrecognised signature type");
+    }
+}
+
+
+/**
+ * Detects whether the given buffer is a file of the type specified.
+ *
+ * @param {string|RegExp} type
+ * @param {Uint8Array} buf
+ * @returns {string|false} The mime type or false if the type does not match
+ */
+export function isType(type, buf) {
+    const types = detectFileType(buf);
+
+    if (!(types && types.length)) return false;
+
+    if (typeof type === "string") {
+        return types.reduce((acc, t) => {
+            const mime = t.mime.startsWith(type) ? t.mime : false;
+            return acc || mime;
+        }, false);
+    } else if (type instanceof RegExp) {
+        return types.reduce((acc, t) => {
+            const mime = type.test(t.mime) ? t.mime : false;
+            return acc || mime;
+        }, false);
+    } else {
+        throw new Error("Invalid type input.");
+    }
+}
+
+
+/**
+ * Detects whether the given buffer contains an image file.
+ *
+ * @param {Uint8Array} buf
+ * @returns {string|false} The mime type or false if the type does not match
+ */
+export function isImage(buf) {
+    return isType("image", buf);
+}
+
+
+/**
+ * Attempts to extract a file from a data stream given its offset and extractor function.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Object} fileDetail
+ * @param {string} fileDetail.mime
+ * @param {string} fileDetail.extension
+ * @param {Function} fileDetail.extractor
+ * @param {number} offset
+ * @returns {File}
+ */
+export function extractFile(bytes, fileDetail, offset) {
+    if (fileDetail.extractor) {
+        sendStatusMessage(`Attempting to extract ${fileDetail.name} at pos ${offset}...`);
+        const fileData = fileDetail.extractor(bytes, offset);
+        const ext = fileDetail.extension.split(",")[0];
+        return new File([fileData], `extracted_at_0x${offset.toString(16)}.${ext}`, {
+            type: fileDetail.mime
+        });
+    }
+
+    throw new Error(`No extraction algorithm available for "${fileDetail.mime}" files`);
+}

src/core/lib/IP.mjs

@@ -3,6 +3,7 @@
  *
  * @author picapi
  * @author n1474335 [n1474335@gmail.com]
+ * @author Klaxon [klaxon@veyr.com]
  * @copyright Crown Copyright 2016
  * @license Apache-2.0
  */
@@ -109,8 +110,8 @@ export function ipv6CidrRange(cidr, includeNetworkInfo) {
  * @returns {string}
  */
 export function ipv4HyphenatedRange(range, includeNetworkInfo, enumerateAddresses, allowLargeList) {
-    const ip1 = strToIpv4(range[1]),
-        ip2 = strToIpv4(range[2]);
+    const ip1 = strToIpv4(range[0].split("-")[0].trim()),
+        ip2 = strToIpv4(range[0].split("-")[1].trim());
 
     let output = "";
 
@@ -162,8 +163,8 @@ Total addresses in range: ${(((ip2 - ip1) >>> 0) + 1)}
  * @returns {string}
  */
 export function ipv6HyphenatedRange(range, includeNetworkInfo) {
-    const ip1 = strToIpv6(range[1]),
-        ip2 = strToIpv6(range[14]),
+    const ip1 = strToIpv6(range[0].split("-")[0].trim()),
+        ip2 = strToIpv6(range[0].split("-")[1].trim()),
         total = new Array(128).fill();
 
     let output = "",
@@ -188,6 +189,93 @@ export function ipv6HyphenatedRange(range, includeNetworkInfo) {
     return output;
 }
 
+/**
+ * Parses a list of IPv4 addresses separated by a new line (\n) and displays information
+ * about it.
+ *
+ * @param {RegExp} list
+ * @param {boolean} includeNetworkInfo
+ * @param {boolean} enumerateAddresses
+ * @param {boolean} allowLargeList
+ * @returns {string}
+ */
+export function ipv4ListedRange(match, includeNetworkInfo, enumerateAddresses, allowLargeList) {
+
+    let ipv4List = match[0].split("\n");
+    ipv4List = ipv4List.filter(Boolean);
+
+    const ipv4CidrList = ipv4List.filter(function(a) {
+        return a.includes("/");
+    });
+    for (let i = 0; i < ipv4CidrList.length; i++) {
+        const network = strToIpv4(ipv4CidrList[i].split("/")[0]);
+        const cidrRange = parseInt(ipv4CidrList[i].split("/")[1], 10);
+        if (cidrRange < 0 || cidrRange > 31) {
+            return "IPv4 CIDR must be less than 32";
+        }
+        const mask = ~(0xFFFFFFFF >>> cidrRange),
+            cidrIp1 = network & mask,
+            cidrIp2 = cidrIp1 | ~mask;
+        ipv4List.splice(ipv4List.indexOf(ipv4CidrList[i]), 1);
+        ipv4List.push(ipv4ToStr(cidrIp1), ipv4ToStr(cidrIp2));
+    }
+
+    ipv4List = ipv4List.sort(ipv4Compare);
+    const ip1 = ipv4List[0];
+    const ip2 = ipv4List[ipv4List.length - 1];
+    const range = [ip1 + " - " + ip2];
+    return ipv4HyphenatedRange(range, includeNetworkInfo, enumerateAddresses, allowLargeList);
+}
+
+/**
+ * Parses a list of IPv6 addresses separated by a new line (\n) and displays information
+ * about it.
+ *
+ * @param {RegExp} list
+ * @param {boolean} includeNetworkInfo
+ * @returns {string}
+ */
+export function ipv6ListedRange(match, includeNetworkInfo) {
+
+    let ipv6List = match[0].split("\n");
+    ipv6List = ipv6List.filter(function(str) {
+        return str.trim();
+    });
+    for (let i =0; i < ipv6List.length; i++){
+        ipv6List[i] = ipv6List[i].trim();
+    }
+    const ipv6CidrList = ipv6List.filter(function(a) {
+        return a.includes("/");
+    });
+
+    for (let i = 0; i < ipv6CidrList.length; i++) {
+
+        const network = strToIpv6(ipv6CidrList[i].split("/")[0]);
+        const cidrRange = parseInt(ipv6CidrList[i].split("/")[1], 10);
+
+        if (cidrRange < 0 || cidrRange > 127) {
+            return "IPv6 CIDR must be less than 128";
+        }
+
+        const cidrIp1 = new Array(8),
+            cidrIp2 = new Array(8);
+
+        const mask = genIpv6Mask(cidrRange);
+
+        for (let j = 0; j < 8; j++) {
+            cidrIp1[j] = network[j] & mask[j];
+            cidrIp2[j] = cidrIp1[j] | (~mask[j] & 0x0000FFFF);
+        }
+        ipv6List.splice(ipv6List.indexOf(ipv6CidrList[i]), 1);
+        ipv6List.push(ipv6ToStr(cidrIp1), ipv6ToStr(cidrIp2));
+    }
+    ipv6List = ipv6List.sort(ipv6Compare);
+    const ip1 = ipv6List[0];
+    const ip2 = ipv6List[ipv6List.length - 1];
+    const range = [ip1 + " - " + ip2];
+    return ipv6HyphenatedRange(range, includeNetworkInfo);
+}
+
 /**
  * Converts an IPv4 address from string format to numerical format.
  *
@@ -391,6 +479,37 @@ export function genIpv6Mask(cidr) {
     return mask;
 }
 
+/**
+ * Comparison operation for sorting of IPv4 addresses.
+ *
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function ipv4Compare(a, b) {
+    return strToIpv4(a) - strToIpv4(b);
+}
+
+/**
+ * Comparison operation for sorting of IPv6 addresses.
+ *
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function ipv6Compare(a, b) {
+
+    const a_ = strToIpv6(a),
+        b_ = strToIpv6(b);
+
+    for (let i = 0; i < a_.length; i++){
+        if (a_[i] !== b_[i]){
+            return a_[i] - b_[i];
+        }
+    }
+    return 0;
+}
+
 const _LARGE_RANGE_ERROR = "The specified range contains more than 65,536 addresses. Running this query could crash your browser. If you want to run it, select the \"Allow large queries\" option. You are advised to turn off \"Auto Bake\" whilst editing large ranges.";
 
 /**

src/core/lib/LoremIpsum.mjs

@@ -0,0 +1,230 @@
+/**
+ * Lorem Ipsum generator.
+ *
+ * @author Klaxon [klaxon@veyr.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+/**
+ * Generate lorem ipsum paragraphs.
+ *
+ * @param {number} length
+ * @returns {string}
+ */
+export function GenerateParagraphs(length=3) {
+    const paragraphs = [];
+    while (paragraphs.length < length) {
+        const paragraphLength = getRandomLength(PARAGRAPH_LENGTH_MEAN, PARAGRAPH_LENGTH_STD_DEV);
+        const sentences = [];
+        while (sentences.length < paragraphLength) {
+            const sentenceLength = getRandomLength(SENTENCE_LENGTH_MEAN, SENTENCE_LENGTH_STD_DEV);
+            const sentence = getWords(sentenceLength);
+            sentences.push(formatSentence(sentence));
+        }
+        paragraphs.push(formatParagraph(sentences));
+    }
+    paragraphs[paragraphs.length-1] = paragraphs[paragraphs.length-1].slice(0, -2);
+    paragraphs[0] = replaceStart(paragraphs[0]);
+    return paragraphs.join("");
+}
+
+
+/**
+ * Generate lorem ipsum sentences.
+ *
+ * @param {number} length
+ * @returns {string}
+ */
+export function GenerateSentences(length=3) {
+    const sentences = [];
+    while (sentences.length < length) {
+        const sentenceLength = getRandomLength(SENTENCE_LENGTH_MEAN, SENTENCE_LENGTH_STD_DEV);
+        const sentence = getWords(sentenceLength);
+        sentences.push(formatSentence(sentence));
+    }
+    const paragraphs = sentencesToParagraphs(sentences);
+    return paragraphs.join("");
+}
+
+
+/**
+ * Generate lorem ipsum words.
+ *
+ * @param {number} length
+ * @returns {string}
+ */
+export function GenerateWords(length=3) {
+    const words = getWords(length);
+    const sentences = wordsToSentences(words);
+    const paragraphs = sentencesToParagraphs(sentences);
+    return paragraphs.join("");
+}
+
+
+/**
+ * Generate lorem ipsum bytes.
+ *
+ * @param {number} length
+ * @returns {string}
+ */
+export function GenerateBytes(length=3) {
+    const str = GenerateWords(length/3);
+    return str.slice(0, length);
+}
+
+
+/**
+ * Get array of randomly selected words from the lorem ipsum wordList.
+ *
+ * @param {number} length
+ * @returns {string[]}
+ * @private
+ */
+function getWords(length=3) {
+    const words = [];
+    let word;
+    let previousWord;
+    while (words.length < length){
+        do {
+            word = wordList[Math.floor(Math.random() * wordList.length)];
+        } while (previousWord === word);
+        words.push(word);
+        previousWord = word;
+    }
+    return words;
+}
+
+
+/**
+ * Convert an array of words into an array of sentences
+ *
+ * @param {string[]} words
+ * @returns {string[]}
+ * @private
+ */
+function wordsToSentences(words) {
+    const sentences = [];
+    while (words.length > 0) {
+        const sentenceLength = getRandomLength(SENTENCE_LENGTH_MEAN, SENTENCE_LENGTH_STD_DEV);
+        if (sentenceLength <= words.length) {
+            sentences.push(formatSentence(words.splice(0, sentenceLength)));
+        } else {
+            sentences.push(formatSentence(words.splice(0, words.length)));
+        }
+    }
+    return sentences;
+}
+
+
+/**
+ * Convert an array of sentences into an array of paragraphs
+ *
+ * @param {string[]} sentences
+ * @returns {string[]}
+ * @private
+ */
+function sentencesToParagraphs(sentences) {
+    const paragraphs = [];
+    while (sentences.length > 0) {
+        const paragraphLength = getRandomLength(PARAGRAPH_LENGTH_MEAN, PARAGRAPH_LENGTH_STD_DEV);
+        paragraphs.push(formatParagraph(sentences.splice(0, paragraphLength)));
+    }
+    paragraphs[paragraphs.length-1] = paragraphs[paragraphs.length-1].slice(0, -1);
+    paragraphs[0] = replaceStart(paragraphs[0]);
+    return paragraphs;
+}
+
+
+/**
+ * Format an array of words into a sentence.
+ *
+ * @param {string[]} words
+ * @returns {string}
+ * @private
+ */
+function formatSentence(words) {
+    // 0.35 chance of a  comma being added randomly to the sentence.
+    if (Math.random() < PROBABILITY_OF_A_COMMA) {
+        const pos = Math.round(Math.random()*(words.length-1));
+        words[pos] +=",";
+    }
+    let sentence = words.join(" ");
+    sentence = sentence.charAt(0).toUpperCase() + sentence.slice(1);
+    sentence += ".";
+    return sentence;
+}
+
+
+/**
+ * Format an array of sentences into a paragraph.
+ *
+ * @param {string[]} sentences
+ * @returns {string}
+ * @private
+ */
+function formatParagraph(sentences) {
+    let paragraph = sentences.join(" ");
+    paragraph += "\n\n";
+    return paragraph;
+}
+
+
+/**
+ * Get a random number based on a mean and standard deviation.
+ *
+ * @param {number} mean
+ * @param {number} stdDev
+ * @returns {number}
+ * @private
+ */
+function getRandomLength(mean, stdDev) {
+    let length;
+    do {
+        length =  Math.round((Math.random()*2-1)+(Math.random()*2-1)+(Math.random()*2-1)*stdDev+mean);
+    } while (length <= 0);
+    return length;
+}
+
+
+/**
+ * Replace first 5 words with "Lorem ipsum dolor sit amet"
+ *
+ * @param {string[]} str
+ * @returns {string[]}
+ * @private
+ */
+function replaceStart(str) {
+    let words = str.split(" ");
+    if (words.length > 5) {
+        words.splice(0, 5, "Lorem", "ipsum", "dolor", "sit", "amet");
+        return words.join(" ");
+    } else {
+        const lorem = ["Lorem", "ipsum", "dolor", "sit", "amet"];
+        words = lorem.slice(0, words.length);
+        str = words.join(" ");
+        str += ".";
+        return str;
+    }
+}
+
+
+const SENTENCE_LENGTH_MEAN = 15;
+const SENTENCE_LENGTH_STD_DEV = 9;
+const PARAGRAPH_LENGTH_MEAN = 5;
+const PARAGRAPH_LENGTH_STD_DEV = 2;
+const PROBABILITY_OF_A_COMMA = 0.35;
+
+const wordList = [
+    "ad", "adipisicing", "aliqua", "aliquip", "amet", "anim",
+    "aute", "cillum", "commodo", "consectetur", "consequat", "culpa",
+    "cupidatat", "deserunt", "do", "dolor", "dolore", "duis",
+    "ea", "eiusmod", "elit", "enim", "esse", "est",
+    "et", "eu", "ex", "excepteur", "exercitation", "fugiat",
+    "id", "in", "incididunt", "ipsum", "irure", "labore",
+    "laboris", "laborum", "Lorem", "magna", "minim", "mollit",
+    "nisi", "non", "nostrud", "nulla", "occaecat", "officia",
+    "pariatur", "proident", "qui", "quis", "reprehenderit", "sint",
+    "sit", "sunt", "tempor", "ullamco", "ut", "velit",
+    "veniam", "voluptate",
+];

src/core/lib/Magic.mjs

@@ -2,6 +2,7 @@ import OperationConfig from "../config/OperationConfig.json";
 import Utils from "../Utils";
 import Recipe from "../Recipe";
 import Dish from "../Dish";
+import {detectFileType} from "./FileType";
 import chiSquared from "chi-squared";
 
 /**
@@ -92,7 +93,14 @@ class Magic {
      * @returns {string} [type.desc] - Description
      */
     detectFileType() {
-        return Magic.magicFileType(this.inputBuffer);
+        const fileType = detectFileType(this.inputBuffer);
+
+        if (!fileType.length) return null;
+        return {
+            ext: fileType[0].extension,
+            mime: fileType[0].mime,
+            desc: fileType[0].description
+        };
     }
 
     /**
@@ -265,9 +273,10 @@ class Magic {
      *                                      performance)
      * @param {Object[]} [recipeConfig=[]] - The recipe configuration up to this point
      * @param {boolean} [useful=false] - Whether the current recipe should be scored highly
+     * @param {string} [crib=null] - The regex crib provided by the user, for filtering the operation output
      * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding
      */
-    async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false) {
+    async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false, crib=null) {
         if (depth < 0) return [];
 
         // Find any operations that can be run on this data
@@ -284,9 +293,9 @@ class Magic {
             isUTF8: this.isUTF8(),
             entropy: this.calcEntropy(),
             matchingOps: matchingOps,
-            useful: useful
+            useful: useful,
+            matchesCrib: crib && crib.test(this.inputStr)
         });
-
         const prevOp = recipeConfig[recipeConfig.length - 1];
 
         // Execute each of the matching operations, then recursively call the speculativeExecution()
@@ -305,7 +314,7 @@ class Magic {
 
             const magic = new Magic(output, this.opPatterns),
                 speculativeResults = await magic.speculativeExecution(
-                    depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful);
+                    depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, crib);
 
             results = results.concat(speculativeResults);
         }));
@@ -317,7 +326,7 @@ class Magic {
             await Promise.all(bfEncodings.map(async enc => {
                 const magic = new Magic(enc.data, this.opPatterns),
                     bfResults = await magic.speculativeExecution(
-                        depth-1, extLang, false, [...recipeConfig, enc.conf]);
+                        depth-1, extLang, false, [...recipeConfig, enc.conf], false, crib);
 
                 results = results.concat(bfResults);
             }));
@@ -784,452 +793,9 @@ class Magic {
         }[code];
     }
 
-
-    /**
-     * Given a buffer, detects magic byte sequences at specific positions and returns the
-     * extension and mime type.
-     *
-     * @param {Uint8Array} buf
-     * @returns {Object} type
-     * @returns {string} type.ext - File extension
-     * @returns {string} type.mime - Mime type
-     * @returns {string} [type.desc] - Description
-     */
-    static magicFileType(buf) {
-        if (!(buf && buf.length > 1)) {
-            return null;
-        }
-
-        if (buf[0] === 0xFF && buf[1] === 0xD8 && buf[2] === 0xFF) {
-            return {
-                ext: "jpg",
-                mime: "image/jpeg"
-            };
-        }
-
-        if (buf[0] === 0x89 && buf[1] === 0x50 && buf[2] === 0x4E && buf[3] === 0x47) {
-            return {
-                ext: "png",
-                mime: "image/png"
-            };
-        }
-
-        if (buf[0] === 0x47 && buf[1] === 0x49 && buf[2] === 0x46) {
-            return {
-                ext: "gif",
-                mime: "image/gif"
-            };
-        }
-
-        if (buf[8] === 0x57 && buf[9] === 0x45 && buf[10] === 0x42 && buf[11] === 0x50) {
-            return {
-                ext: "webp",
-                mime: "image/webp"
-            };
-        }
-
-        // needs to be before `tif` check
-        if (((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) && buf[8] === 0x43 && buf[9] === 0x52) {
-            return {
-                ext: "cr2",
-                mime: "image/x-canon-cr2"
-            };
-        }
-
-        if ((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) {
-            return {
-                ext: "tif",
-                mime: "image/tiff"
-            };
-        }
-
-        if (buf[0] === 0x42 && buf[1] === 0x4D) {
-            return {
-                ext: "bmp",
-                mime: "image/bmp"
-            };
-        }
-
-        if (buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0xBC) {
-            return {
-                ext: "jxr",
-                mime: "image/vnd.ms-photo"
-            };
-        }
-
-        if (buf[0] === 0x38 && buf[1] === 0x42 && buf[2] === 0x50 && buf[3] === 0x53) {
-            return {
-                ext: "psd",
-                mime: "image/vnd.adobe.photoshop"
-            };
-        }
-
-        // needs to be before `zip` check
-        if (buf[0] === 0x50 && buf[1] === 0x4B && buf[2] === 0x3 && buf[3] === 0x4 && buf[30] === 0x6D && buf[31] === 0x69 && buf[32] === 0x6D && buf[33] === 0x65 && buf[34] === 0x74 && buf[35] === 0x79 && buf[36] === 0x70 && buf[37] === 0x65 && buf[38] === 0x61 && buf[39] === 0x70 && buf[40] === 0x70 && buf[41] === 0x6C && buf[42] === 0x69 && buf[43] === 0x63 && buf[44] === 0x61 && buf[45] === 0x74 && buf[46] === 0x69 && buf[47] === 0x6F && buf[48] === 0x6E && buf[49] === 0x2F && buf[50] === 0x65 && buf[51] === 0x70 && buf[52] === 0x75 && buf[53] === 0x62 && buf[54] === 0x2B && buf[55] === 0x7A && buf[56] === 0x69 && buf[57] === 0x70) {
-            return {
-                ext: "epub",
-                mime: "application/epub+zip"
-            };
-        }
-
-        if (buf[0] === 0x50 && buf[1] === 0x4B && (buf[2] === 0x3 || buf[2] === 0x5 || buf[2] === 0x7) && (buf[3] === 0x4 || buf[3] === 0x6 || buf[3] === 0x8)) {
-            return {
-                ext: "zip",
-                mime: "application/zip"
-            };
-        }
-
-        if (buf[257] === 0x75 && buf[258] === 0x73 && buf[259] === 0x74 && buf[260] === 0x61 && buf[261] === 0x72) {
-            return {
-                ext: "tar",
-                mime: "application/x-tar"
-            };
-        }
-
-        if (buf[0] === 0x52 && buf[1] === 0x61 && buf[2] === 0x72 && buf[3] === 0x21 && buf[4] === 0x1A && buf[5] === 0x7 && (buf[6] === 0x0 || buf[6] === 0x1)) {
-            return {
-                ext: "rar",
-                mime: "application/x-rar-compressed"
-            };
-        }
-
-        if (buf[0] === 0x1F && buf[1] === 0x8B && buf[2] === 0x8) {
-            return {
-                ext: "gz",
-                mime: "application/gzip"
-            };
-        }
-
-        if (buf[0] === 0x42 && buf[1] === 0x5A && buf[2] === 0x68) {
-            return {
-                ext: "bz2",
-                mime: "application/x-bzip2"
-            };
-        }
-
-        if (buf[0] === 0x37 && buf[1] === 0x7A && buf[2] === 0xBC && buf[3] === 0xAF && buf[4] === 0x27 && buf[5] === 0x1C) {
-            return {
-                ext: "7z",
-                mime: "application/x-7z-compressed"
-            };
-        }
-
-        if (buf[0] === 0x78 && buf[1] === 0x01) {
-            return {
-                ext: "dmg, zlib",
-                mime: "application/x-apple-diskimage, application/x-deflate"
-            };
-        }
-
-        if ((buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && (buf[3] === 0x18 || buf[3] === 0x20) && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) || (buf[0] === 0x33 && buf[1] === 0x67 && buf[2] === 0x70 && buf[3] === 0x35) || (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x1C && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x6D && buf[9] === 0x70 && buf[10] === 0x34 && buf[11] === 0x32 && buf[16] === 0x6D && buf[17] === 0x70 && buf[18] === 0x34 && buf[19] === 0x31 && buf[20] === 0x6D && buf[21] === 0x70 && buf[22] === 0x34 && buf[23] === 0x32 && buf[24] === 0x69 && buf[25] === 0x73 && buf[26] === 0x6F && buf[27] === 0x6D)) {
-            return {
-                ext: "mp4",
-                mime: "video/mp4"
-            };
-        }
-
-        if ((buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x1C && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x4D && buf[9] === 0x34 && buf[10] === 0x56)) {
-            return {
-                ext: "m4v",
-                mime: "video/x-m4v"
-            };
-        }
-
-        if (buf[0] === 0x4D && buf[1] === 0x54 && buf[2] === 0x68 && buf[3] === 0x64) {
-            return {
-                ext: "mid",
-                mime: "audio/midi"
-            };
-        }
-
-        // needs to be before the `webm` check
-        if (buf[31] === 0x6D && buf[32] === 0x61 && buf[33] === 0x74 && buf[34] === 0x72 && buf[35] === 0x6f && buf[36] === 0x73 && buf[37] === 0x6B && buf[38] === 0x61) {
-            return {
-                ext: "mkv",
-                mime: "video/x-matroska"
-            };
-        }
-
-        if (buf[0] === 0x1A && buf[1] === 0x45 && buf[2] === 0xDF && buf[3] === 0xA3) {
-            return {
-                ext: "webm",
-                mime: "video/webm"
-            };
-        }
-
-        if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x14 && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) {
-            return {
-                ext: "mov",
-                mime: "video/quicktime"
-            };
-        }
-
-        if (buf[0] === 0x52 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x46 && buf[8] === 0x41 && buf[9] === 0x56 && buf[10] === 0x49) {
-            return {
-                ext: "avi",
-                mime: "video/x-msvideo"
-            };
-        }
-
-        if (buf[0] === 0x30 && buf[1] === 0x26 && buf[2] === 0xB2 && buf[3] === 0x75 && buf[4] === 0x8E && buf[5] === 0x66 && buf[6] === 0xCF && buf[7] === 0x11 && buf[8] === 0xA6 && buf[9] === 0xD9) {
-            return {
-                ext: "wmv",
-                mime: "video/x-ms-wmv"
-            };
-        }
-
-        if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x1 && buf[3].toString(16)[0] === "b") {
-            return {
-                ext: "mpg",
-                mime: "video/mpeg"
-            };
-        }
-
-        if ((buf[0] === 0x49 && buf[1] === 0x44 && buf[2] === 0x33) || (buf[0] === 0xFF && buf[1] === 0xfb)) {
-            return {
-                ext: "mp3",
-                mime: "audio/mpeg"
-            };
-        }
-
-        if ((buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x4D && buf[9] === 0x34 && buf[10] === 0x41) || (buf[0] === 0x4D && buf[1] === 0x34 && buf[2] === 0x41 && buf[3] === 0x20)) {
-            return {
-                ext: "m4a",
-                mime: "audio/m4a"
-            };
-        }
-
-        if (buf[0] === 0x4F && buf[1] === 0x67 && buf[2] === 0x67 && buf[3] === 0x53) {
-            return {
-                ext: "ogg",
-                mime: "audio/ogg"
-            };
-        }
-
-        if (buf[0] === 0x66 && buf[1] === 0x4C && buf[2] === 0x61 && buf[3] === 0x43) {
-            return {
-                ext: "flac",
-                mime: "audio/x-flac"
-            };
-        }
-
-        if (buf[0] === 0x52 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x46 && buf[8] === 0x57 && buf[9] === 0x41 && buf[10] === 0x56 && buf[11] === 0x45) {
-            return {
-                ext: "wav",
-                mime: "audio/x-wav"
-            };
-        }
-
-        if (buf[0] === 0x23 && buf[1] === 0x21 && buf[2] === 0x41 && buf[3] === 0x4D && buf[4] === 0x52 && buf[5] === 0x0A) {
-            return {
-                ext: "amr",
-                mime: "audio/amr"
-            };
-        }
-
-        if (buf[0] === 0x25 && buf[1] === 0x50 && buf[2] === 0x44 && buf[3] === 0x46) {
-            return {
-                ext: "pdf",
-                mime: "application/pdf"
-            };
-        }
-
-        if (buf[0] === 0x4D && buf[1] === 0x5A) {
-            return {
-                ext: "exe",
-                mime: "application/x-msdownload"
-            };
-        }
-
-        if ((buf[0] === 0x43 || buf[0] === 0x46) && buf[1] === 0x57 && buf[2] === 0x53) {
-            return {
-                ext: "swf",
-                mime: "application/x-shockwave-flash"
-            };
-        }
-
-        if (buf[0] === 0x7B && buf[1] === 0x5C && buf[2] === 0x72 && buf[3] === 0x74 && buf[4] === 0x66) {
-            return {
-                ext: "rtf",
-                mime: "application/rtf"
-            };
-        }
-
-        if (buf[0] === 0x77 && buf[1] === 0x4F && buf[2] === 0x46 && buf[3] === 0x46 && buf[4] === 0x00 && buf[5] === 0x01 && buf[6] === 0x00 && buf[7] === 0x00) {
-            return {
-                ext: "woff",
-                mime: "application/font-woff"
-            };
-        }
-
-        if (buf[0] === 0x77 && buf[1] === 0x4F && buf[2] === 0x46 && buf[3] === 0x32 && buf[4] === 0x00 && buf[5] === 0x01 && buf[6] === 0x00 && buf[7] === 0x00) {
-            return {
-                ext: "woff2",
-                mime: "application/font-woff"
-            };
-        }
-
-        if (buf[34] === 0x4C && buf[35] === 0x50 && ((buf[8] === 0x02 && buf[9] === 0x00 && buf[10] === 0x01) || (buf[8] === 0x01 && buf[9] === 0x00 && buf[10] === 0x00) || (buf[8] === 0x02 && buf[9] === 0x00 && buf[10] === 0x02))) {
-            return {
-                ext: "eot",
-                mime: "application/octet-stream"
-            };
-        }
-
-        if (buf[0] === 0x00 && buf[1] === 0x01 && buf[2] === 0x00 && buf[3] === 0x00 && buf[4] === 0x00) {
-            return {
-                ext: "ttf",
-                mime: "application/font-sfnt"
-            };
-        }
-
-        if (buf[0] === 0x4F && buf[1] === 0x54 && buf[2] === 0x54 && buf[3] === 0x4F && buf[4] === 0x00) {
-            return {
-                ext: "otf",
-                mime: "application/font-sfnt"
-            };
-        }
-
-        if (buf[0] === 0x00 && buf[1] === 0x00 && buf[2] === 0x01 && buf[3] === 0x00) {
-            return {
-                ext: "ico",
-                mime: "image/x-icon"
-            };
-        }
-
-        if (buf[0] === 0x46 && buf[1] === 0x4C && buf[2] === 0x56 && buf[3] === 0x01) {
-            return {
-                ext: "flv",
-                mime: "video/x-flv"
-            };
-        }
-
-        if (buf[0] === 0x25 && buf[1] === 0x21) {
-            return {
-                ext: "ps",
-                mime: "application/postscript"
-            };
-        }
-
-        if (buf[0] === 0xFD && buf[1] === 0x37 && buf[2] === 0x7A && buf[3] === 0x58 && buf[4] === 0x5A && buf[5] === 0x00) {
-            return {
-                ext: "xz",
-                mime: "application/x-xz"
-            };
-        }
-
-        if (buf[0] === 0x53 && buf[1] === 0x51 && buf[2] === 0x4C && buf[3] === 0x69) {
-            return {
-                ext: "sqlite",
-                mime: "application/x-sqlite3"
-            };
-        }
-
-        /**
-         *
-         * Added by n1474335 [n1474335@gmail.com] from here on
-         *
-         */
-        if ((buf[0] === 0x1F && buf[1] === 0x9D) || (buf[0] === 0x1F && buf[1] === 0xA0)) {
-            return {
-                ext: "z, tar.z",
-                mime: "application/x-gtar"
-            };
-        }
-
-        if (buf[0] === 0x7F && buf[1] === 0x45 && buf[2] === 0x4C && buf[3] === 0x46) {
-            return {
-                ext: "none, axf, bin, elf, o, prx, puff, so",
-                mime: "application/x-executable",
-                desc: "Executable and Linkable Format file. No standard file extension."
-            };
-        }
-
-        if (buf[0] === 0xCA && buf[1] === 0xFE && buf[2] === 0xBA && buf[3] === 0xBE) {
-            return {
-                ext: "class",
-                mime: "application/java-vm"
-            };
-        }
-
-        if (buf[0] === 0xEF && buf[1] === 0xBB && buf[2] === 0xBF) {
-            return {
-                ext: "txt",
-                mime: "text/plain",
-                desc: "UTF-8 encoded Unicode byte order mark detected, commonly but not exclusively seen in text files."
-            };
-        }
-
-        // Must be before Little-endian UTF-16 BOM
-        if (buf[0] === 0xFF && buf[1] === 0xFE && buf[2] === 0x00 && buf[3] === 0x00) {
-            return {
-                ext: "UTF32LE",
-                mime: "charset/utf32le",
-                desc: "Little-endian UTF-32 encoded Unicode byte order mark detected."
-            };
-        }
-
-        if (buf[0] === 0xFF && buf[1] === 0xFE) {
-            return {
-                ext: "UTF16LE",
-                mime: "charset/utf16le",
-                desc: "Little-endian UTF-16 encoded Unicode byte order mark detected."
-            };
-        }
-
-        if ((buf[0x8001] === 0x43 && buf[0x8002] === 0x44 && buf[0x8003] === 0x30 && buf[0x8004] === 0x30 && buf[0x8005] === 0x31) ||
-            (buf[0x8801] === 0x43 && buf[0x8802] === 0x44 && buf[0x8803] === 0x30 && buf[0x8804] === 0x30 && buf[0x8805] === 0x31) ||
-            (buf[0x9001] === 0x43 && buf[0x9002] === 0x44 && buf[0x9003] === 0x30 && buf[0x9004] === 0x30 && buf[0x9005] === 0x31)) {
-            return {
-                ext: "iso",
-                mime: "application/octet-stream",
-                desc: "ISO 9660 CD/DVD image file"
-            };
-        }
-
-        if (buf[0] === 0xD0 && buf[1] === 0xCF && buf[2] === 0x11 && buf[3] === 0xE0 && buf[4] === 0xA1 && buf[5] === 0xB1 && buf[6] === 0x1A && buf[7] === 0xE1) {
-            return {
-                ext: "doc, xls, ppt",
-                mime: "application/msword, application/vnd.ms-excel, application/vnd.ms-powerpoint",
-                desc: "Microsoft Office documents"
-            };
-        }
-
-        if (buf[0] === 0x64 && buf[1] === 0x65 && buf[2] === 0x78 && buf[3] === 0x0A && buf[4] === 0x30 && buf[5] === 0x33 && buf[6] === 0x35 && buf[7] === 0x00) {
-            return {
-                ext: "dex",
-                mime: "application/octet-stream",
-                desc: "Dalvik Executable (Android)"
-            };
-        }
-
-        if (buf[0] === 0x4B && buf[1] === 0x44 && buf[2] === 0x4D) {
-            return {
-                ext: "vmdk",
-                mime: "application/vmdk, application/x-virtualbox-vmdk"
-            };
-        }
-
-        if (buf[0] === 0x43 && buf[1] === 0x72 && buf[2] === 0x32 && buf[3] === 0x34) {
-            return {
-                ext: "crx",
-                mime: "application/crx",
-                desc: "Google Chrome extension or packaged app"
-            };
-        }
-
-        if (buf[0] === 0x78 && (buf[1] === 0x01 || buf[1] === 0x9C || buf[1] === 0xDA || buf[1] === 0x5e)) {
-            return {
-                ext: "zlib",
-                mime: "application/x-deflate"
-            };
-        }
-
-        return null;
-    }
-
 }
 
+
 /**
  * Byte frequencies of various languages generated from Wikipedia dumps taken in late 2017 and early 2018.
  * The Chi-Squared test cannot accept expected values of 0, so 0.0001 has been used to account for bytes

src/core/lib/Stream.mjs

@@ -0,0 +1,263 @@
+/**
+ * Stream class for parsing binary protocols.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @author tlwr [toby@toby.codes]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ *
+ */
+
+/**
+ * A Stream can be used to traverse a binary blob, interpreting sections of it
+ * as various data types.
+ */
+export default class Stream {
+
+    /**
+     * Stream constructor.
+     *
+     * @param {Uint8Array} input
+     */
+    constructor(input) {
+        this.bytes = input;
+        this.length = this.bytes.length;
+        this.position = 0;
+        this.bitPos = 0;
+    }
+
+    /**
+     * Get a number of bytes from the current position.
+     *
+     * @param {number} numBytes
+     * @returns {Uint8Array}
+     */
+    getBytes(numBytes) {
+        if (this.position > this.length) return undefined;
+
+        const newPosition = this.position + numBytes;
+        const bytes = this.bytes.slice(this.position, newPosition);
+        this.position = newPosition;
+        this.bitPos = 0;
+        return bytes;
+    }
+
+    /**
+     * Interpret the following bytes as a string, stopping at the next null byte or
+     * the supplied limit.
+     *
+     * @param {number} numBytes
+     * @returns {string}
+     */
+    readString(numBytes) {
+        if (this.position > this.length) return undefined;
+
+        let result = "";
+        for (let i = this.position; i < this.position + numBytes; i++) {
+            const currentByte = this.bytes[i];
+            if (currentByte === 0) break;
+            result += String.fromCharCode(currentByte);
+        }
+        this.position += numBytes;
+        this.bitPos = 0;
+        return result;
+    }
+
+    /**
+     * Interpret the following bytes as an integer in big or little endian.
+     *
+     * @param {number} numBytes
+     * @param {string} [endianness="be"]
+     * @returns {number}
+     */
+    readInt(numBytes, endianness="be") {
+        if (this.position > this.length) return undefined;
+
+        let val = 0;
+        if (endianness === "be") {
+            for (let i = this.position; i < this.position + numBytes; i++) {
+                val = val << 8;
+                val |= this.bytes[i];
+            }
+        } else {
+            for (let i = this.position + numBytes - 1; i >= this.position; i--) {
+                val = val << 8;
+                val |= this.bytes[i];
+            }
+        }
+        this.position += numBytes;
+        this.bitPos = 0;
+        return val;
+    }
+
+    /**
+     * Reads a number of bits from the buffer.
+     *
+     * @TODO Add endianness
+     *
+     * @param {number} numBits
+     * @returns {number}
+     */
+    readBits(numBits) {
+        if (this.position > this.length) return undefined;
+
+        let bitBuf = 0,
+            bitBufLen = 0;
+
+        // Add remaining bits from current byte
+        bitBuf = (this.bytes[this.position++] & bitMask(this.bitPos)) >>> this.bitPos;
+        bitBufLen = 8 - this.bitPos;
+        this.bitPos = 0;
+
+        // Not enough bits yet
+        while (bitBufLen < numBits) {
+            bitBuf |= this.bytes[this.position++] << bitBufLen;
+            bitBufLen += 8;
+        }
+
+        // Reverse back to numBits
+        if (bitBufLen > numBits) {
+            const excess = bitBufLen - numBits;
+            bitBuf &= (1 << numBits) - 1;
+            bitBufLen -= excess;
+            this.position--;
+            this.bitPos = 8 - excess;
+        }
+
+        return bitBuf;
+
+        /**
+         * Calculates the bit mask based on the current bit position.
+         *
+         * @param {number} bitPos
+         * @returns {number} The bit mask
+         */
+        function bitMask(bitPos) {
+            return 256 - (1 << bitPos);
+        }
+    }
+
+    /**
+     * Consume the stream until we reach the specified byte or sequence of bytes.
+     *
+     * @param {number|List<number>} val
+     */
+    continueUntil(val) {
+        if (this.position > this.length) return;
+
+        this.bitPos = 0;
+
+        if (typeof val === "number") {
+            while (++this.position < this.length && this.bytes[this.position] !== val) {
+                continue;
+            }
+            return;
+        }
+
+        // val is an array
+        let found = false;
+        while (!found && this.position < this.length) {
+            while (++this.position < this.length && this.bytes[this.position] !== val[0]) {
+                continue;
+            }
+            found = true;
+            for (let i = 1; i < val.length; i++) {
+                if (this.position + i > this.length || this.bytes[this.position + i] !== val[i])
+                    found = false;
+            }
+        }
+    }
+
+    /**
+     * Consume the next byte if it matches the supplied value.
+     *
+     * @param {number} val
+     */
+    consumeIf(val) {
+        if (this.bytes[this.position] === val) {
+            this.position++;
+            this.bitPos = 0;
+        }
+    }
+
+    /**
+     * Move forwards through the stream by the specified number of bytes.
+     *
+     * @param {number} numBytes
+     */
+    moveForwardsBy(numBytes) {
+        const pos = this.position + numBytes;
+        if (pos < 0 || pos > this.length)
+            throw new Error("Cannot move to position " + pos + " in stream. Out of bounds.");
+        this.position = pos;
+        this.bitPos = 0;
+    }
+
+    /**
+     * Move backwards through the stream by the specified number of bytes.
+     *
+     * @param {number} numBytes
+     */
+    moveBackwardsBy(numBytes) {
+        const pos = this.position - numBytes;
+        if (pos < 0 || pos > this.length)
+            throw new Error("Cannot move to position " + pos + " in stream. Out of bounds.");
+        this.position = pos;
+        this.bitPos = 0;
+    }
+
+    /**
+     * Move backwards through the strem by the specified number of bits.
+     *
+     * @param {number} numBits
+     */
+    moveBackwardsByBits(numBits) {
+        if (numBits <= this.bitPos) {
+            this.bitPos -= numBits;
+        } else {
+            if (this.bitPos > 0) {
+                numBits -= this.bitPos;
+                this.bitPos = 0;
+            }
+
+            while (numBits > 0) {
+                this.moveBackwardsBy(1);
+                this.bitPos = 8;
+                this.moveBackwardsByBits(numBits);
+                numBits -= 8;
+            }
+        }
+    }
+
+    /**
+     * Move to a specified position in the stream.
+     *
+     * @param {number} pos
+     */
+    moveTo(pos) {
+        if (pos < 0 || pos > this.length)
+            throw new Error("Cannot move to position " + pos + " in stream. Out of bounds.");
+        this.position = pos;
+        this.bitPos = 0;
+    }
+
+    /**
+     * Returns true if there are more bytes left in the stream.
+     *
+     * @returns {boolean}
+     */
+    hasMore() {
+        return this.position < this.length;
+    }
+
+    /**
+     * Returns a slice of the stream up to the current position.
+     *
+     * @returns {Uint8Array}
+     */
+    carve() {
+        if (this.bitPos > 0) this.position++;
+        return this.bytes.slice(0, this.position);
+    }
+
+}

src/core/lib/TLVParser.mjs

@@ -0,0 +1,78 @@
+/**
+ * Parser for Type-length-value data.
+ *
+ * @author gchq77703 []
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+const defaults = {
+    location: 0,
+    bytesInLength: 1,
+    basicEncodingRules: false
+};
+
+/**
+ * TLVParser library
+ */
+export default class TLVParser {
+
+    /**
+     * TLVParser constructor
+     *
+     * @param {byteArray} input
+     * @param {Object} options
+     */
+    constructor(input, options) {
+        this.input = input;
+        Object.assign(this, defaults, options);
+    }
+
+    /**
+     * @returns {number}
+     */
+    getLength() {
+        if (this.basicEncodingRules) {
+            const bit = this.input[this.location];
+            if (bit & 0x80) {
+                this.bytesInLength = bit & ~0x80;
+            } else {
+                this.location++;
+                return bit & ~0x80;
+            }
+        }
+
+        let length = 0;
+
+        for (let i = 0; i < this.bytesInLength; i++) {
+            length += this.input[this.location] * Math.pow(Math.pow(2, 8), i);
+            this.location++;
+        }
+
+        return length;
+    }
+
+    /**
+     * @param {number} length
+     * @returns {number[]}
+     */
+    getValue(length) {
+        const value = [];
+
+        for (let i = 0; i < length; i++) {
+            if (this.location > this.input.length) return value;
+            value.push(this.input[this.location]);
+            this.location++;
+        }
+
+        return value;
+    }
+
+    /**
+     * @returns {boolean}
+     */
+    atEnd() {
+        return this.input.length <= this.location;
+    }
+}

src/core/operations/A1Z26CipherDecode.mjs

@@ -0,0 +1,63 @@
+/**
+ * @author Jarmo van Lenthe [github.com/jarmovanlenthe]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import Utils from "../Utils";
+import {DELIM_OPTIONS} from "../lib/Delim";
+import OperationError from "../errors/OperationError";
+
+/**
+ * A1Z26 Cipher Decode operation
+ */
+class A1Z26CipherDecode extends Operation {
+
+    /**
+     * A1Z26CipherDecode constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "A1Z26 Cipher Decode";
+        this.module = "Ciphers";
+        this.description = "Converts alphabet order numbers into their corresponding  alphabet character.<br><br>e.g. <code>1</code> becomes <code>a</code> and <code>2</code> becomes <code>b</code>.";
+        this.infoURL = "";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [
+            {
+                name: "Delimiter",
+                type: "option",
+                value: DELIM_OPTIONS
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        const delim = Utils.charRep(args[0] || "Space");
+
+        if (input.length === 0) {
+            return [];
+        }
+
+        const bites = input.split(delim);
+        let latin1 = "";
+        for (let i = 0; i < bites.length; i++) {
+            if (bites[i] < 1 || bites[i] > 26) {
+                throw new OperationError("Error: all numbers must be between 1 and 26.");
+            }
+            latin1 += Utils.chr(parseInt(bites[i], 10) + 96);
+        }
+        return latin1;
+    }
+
+}
+
+export default A1Z26CipherDecode;

src/core/operations/A1Z26CipherEncode.mjs

@@ -0,0 +1,61 @@
+/**
+ * @author Jarmo van Lenthe [github.com/jarmovanlenthe]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import Utils from "../Utils";
+import {DELIM_OPTIONS} from "../lib/Delim";
+
+/**
+ * A1Z26 Cipher Encode operation
+ */
+class A1Z26CipherEncode extends Operation {
+
+    /**
+     * A1Z26CipherEncode constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "A1Z26 Cipher Encode";
+        this.module = "Ciphers";
+        this.description = "Converts alphabet characters into their corresponding alphabet order number.<br><br>e.g. <code>a</code> becomes <code>1</code> and <code>b</code> becomes <code>2</code>.<br><br>Non-alphabet characters are dropped.";
+        this.infoURL = "";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [
+            {
+                name: "Delimiter",
+                type: "option",
+                value: DELIM_OPTIONS
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        const delim = Utils.charRep(args[0] || "Space");
+        let output = "";
+
+        const sanitizedinput = input.toLowerCase(),
+            charcode = Utils.strToCharcode(sanitizedinput);
+
+        for (let i = 0; i < charcode.length; i++) {
+            const ordinal = charcode[i] - 96;
+
+            if (ordinal > 0 && ordinal <= 26) {
+                output += ordinal.toString(10) + delim;
+            }
+        }
+        return output.slice(0, -delim.length);
+    }
+
+}
+
+export default A1Z26CipherEncode;

src/core/operations/ADD.mjs

@@ -6,7 +6,7 @@
 
 import Operation from "../Operation";
 import Utils from "../Utils";
-import { bitOp, add } from "../lib/BitwiseOp";
+import { bitOp, add, BITWISE_OP_DELIMS } from "../lib/BitwiseOp";
 
 /**
  * ADD operation
@@ -30,7 +30,7 @@ class ADD extends Operation {
                 "name": "Key",
                 "type": "toggleString",
                 "value": "",
-                "toggleValues": ["Hex", "Decimal", "Base64", "UTF8", "Latin1"]
+                "toggleValues": BITWISE_OP_DELIMS
             }
         ];
     }

src/core/operations/AND.mjs

@@ -6,7 +6,7 @@
 
 import Operation from "../Operation";
 import Utils from "../Utils";
-import { bitOp, and } from "../lib/BitwiseOp";
+import { bitOp, and, BITWISE_OP_DELIMS } from "../lib/BitwiseOp";
 
 /**
  * AND operation
@@ -30,7 +30,7 @@ class AND extends Operation {
                 "name": "Key",
                 "type": "toggleString",
                 "value": "",
-                "toggleValues": ["Hex", "Decimal", "Base64", "UTF8", "Latin1"]
+                "toggleValues": BITWISE_OP_DELIMS
             }
         ];
     }

src/core/operations/Adler32Checksum.mjs

@@ -19,7 +19,7 @@ class Adler32Checksum extends Operation {
         super();
 
         this.name = "Adler-32 Checksum";
-        this.module = "Hashing";
+        this.module = "Crypto";
         this.description = "Adler-32 is a checksum algorithm which was invented by Mark Adler in 1995, and is a modification of the Fletcher checksum. Compared to a cyclic redundancy check of the same length, it trades reliability for speed (preferring the latter).<br><br>Adler-32 is more reliable than Fletcher-16, and slightly less reliable than Fletcher-32.";
         this.infoURL = "https://wikipedia.org/wiki/Adler-32";
         this.inputType = "byteArray";

src/core/operations/AnalyseHash.mjs

@@ -19,7 +19,7 @@ class AnalyseHash extends Operation {
         super();
 
         this.name = "Analyse hash";
-        this.module = "Hashing";
+        this.module = "Crypto";
         this.description = "Tries to determine information about a given hash and suggests which algorithm may have been used to generate it based on its length.";
         this.infoURL = "https://wikipedia.org/wiki/Comparison_of_cryptographic_hash_functions";
         this.inputType = "string";

src/core/operations/BSONDeserialise.mjs

@@ -5,7 +5,7 @@
  */
 
 import Operation from "../Operation";
-import bsonjs from "bson";
+import bson from "bson";
 import OperationError from "../errors/OperationError";
 
 /**
@@ -36,8 +36,6 @@ class BSONDeserialise extends Operation {
     run(input, args) {
         if (!input.byteLength) return "";
 
-        const bson = new bsonjs();
-
         try {
             const data = bson.deserialize(new Buffer(input));
             return JSON.stringify(data, null, 2);

src/core/operations/BSONSerialise.mjs

@@ -5,7 +5,7 @@
  */
 
 import Operation from "../Operation";
-import bsonjs from "bson";
+import bson from "bson";
 import OperationError from "../errors/OperationError";
 
 /**
@@ -36,8 +36,6 @@ class BSONSerialise extends Operation {
     run(input, args) {
         if (!input) return new ArrayBuffer();
 
-        const bson = new bsonjs();
-
         try {
             const data = JSON.parse(input);
             return bson.serialize(data).buffer;

src/core/operations/Bcrypt.mjs

@@ -19,7 +19,7 @@ class Bcrypt extends Operation {
         super();
 
         this.name = "Bcrypt";
-        this.module = "Hashing";
+        this.module = "Crypto";
         this.description = "bcrypt is a password hashing function designed by Niels Provos and David Mazi\xe8res, based on the Blowfish cipher, and presented at USENIX in 1999. Besides incorporating a salt to protect against rainbow table attacks, bcrypt is an adaptive function: over time, the iteration count (rounds) can be increased to make it slower, so it remains resistant to brute-force search attacks even with increasing computation power.<br><br>Enter the password in the input to generate its hash.";
         this.infoURL = "https://wikipedia.org/wiki/Bcrypt";
         this.inputType = "string";

src/core/operations/BcryptCompare.mjs

@@ -19,7 +19,7 @@ class BcryptCompare extends Operation {
         super();
 
         this.name = "Bcrypt compare";
-        this.module = "Hashing";
+        this.module = "Crypto";
         this.description = "Tests whether the input matches the given bcrypt hash. To test multiple possible passwords, use the 'Fork' operation.";
         this.infoURL = "https://wikipedia.org/wiki/Bcrypt";
         this.inputType = "string";

src/core/operations/BcryptParse.mjs

@@ -20,7 +20,7 @@ class BcryptParse extends Operation {
         super();
 
         this.name = "Bcrypt parse";
-        this.module = "Hashing";
+        this.module = "Crypto";
         this.description = "Parses a bcrypt hash to determine the number of rounds used, the salt, and the password hash.";
         this.infoURL = "https://wikipedia.org/wiki/Bcrypt";
         this.inputType = "string";

src/core/operations/BlurImage.mjs

@@ -0,0 +1,102 @@
+/**
+ * @author j433866 [j433866@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import { isImage } from "../lib/FileType";
+import { toBase64 } from "../lib/Base64";
+import jimp from "jimp";
+
+/**
+ * Blur Image operation
+ */
+class BlurImage extends Operation {
+
+    /**
+     * BlurImage constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Blur Image";
+        this.module = "Image";
+        this.description = "Applies a blur effect to the image.<br><br>Gaussian blur is much slower than fast blur, but produces better results.";
+        this.infoURL = "https://wikipedia.org/wiki/Gaussian_blur";
+        this.inputType = "byteArray";
+        this.outputType = "byteArray";
+        this.presentType = "html";
+        this.args = [
+            {
+                name: "Amount",
+                type: "number",
+                value: 5,
+                min: 1
+            },
+            {
+                name: "Type",
+                type: "option",
+                value: ["Fast", "Gaussian"]
+            }
+        ];
+    }
+
+    /**
+     * @param {byteArray} input
+     * @param {Object[]} args
+     * @returns {byteArray}
+     */
+    async run(input, args) {
+        const [blurAmount, blurType] = args;
+
+        if (!isImage(input)) {
+            throw new OperationError("Invalid file type.");
+        }
+
+        let image;
+        try {
+            image = await jimp.read(Buffer.from(input));
+        } catch (err) {
+            throw new OperationError(`Error loading image. (${err})`);
+        }
+        try {
+            switch (blurType){
+                case "Fast":
+                    image.blur(blurAmount);
+                    break;
+                case "Gaussian":
+                    if (ENVIRONMENT_IS_WORKER())
+                        self.sendStatusMessage("Gaussian blurring image. This may take a while...");
+                    image.gaussian(blurAmount);
+                    break;
+            }
+
+            const imageBuffer = await image.getBufferAsync(jimp.AUTO);
+            return [...imageBuffer];
+        } catch (err) {
+            throw new OperationError(`Error blurring image. (${err})`);
+        }
+    }
+
+    /**
+     * Displays the blurred image using HTML for web apps
+     *
+     * @param {byteArray} data
+     * @returns {html}
+     */
+    present(data) {
+        if (!data.length) return "";
+
+        const type = isImage(data);
+        if (!type) {
+            throw new OperationError("Invalid file type.");
+        }
+
+        return `<img src="data:${type};base64,${toBase64(data)}">`;
+    }
+
+}
+
+export default BlurImage;

src/core/operations/Bzip2Decompress.mjs

@@ -5,7 +5,7 @@
  */
 
 import Operation from "../Operation";
-import bzip2 from "../vendor/bzip2.js";
+import bzip2 from "../vendor/bzip2";
 import OperationError from "../errors/OperationError";
 
 /**

src/core/operations/CRC16Checksum.mjs

@@ -19,7 +19,7 @@ class CRC16Checksum extends Operation {
         super();
 
         this.name = "CRC-16 Checksum";
-        this.module = "Hashing";
+        this.module = "Crypto";
         this.description = "A cyclic redundancy check (CRC) is an error-detecting code commonly used in digital networks and storage devices to detect accidental changes to raw data.<br><br>The CRC was invented by W. Wesley Peterson in 1961.";
         this.infoURL = "https://wikipedia.org/wiki/Cyclic_redundancy_check";
         this.inputType = "ArrayBuffer";

src/core/operations/CRC32Checksum.mjs

@@ -19,7 +19,7 @@ class CRC32Checksum extends Operation {
         super();
 
         this.name = "CRC-32 Checksum";
-        this.module = "Hashing";
+        this.module = "Crypto";
         this.description = "A cyclic redundancy check (CRC) is an error-detecting code commonly used in digital networks and storage devices to detect accidental changes to raw data.<br><br>The CRC was invented by W. Wesley Peterson in 1961; the 32-bit CRC function of Ethernet and many other standards is the work of several researchers and was published in 1975.";
         this.infoURL = "https://wikipedia.org/wiki/Cyclic_redundancy_check";
         this.inputType = "ArrayBuffer";

src/core/operations/CSVToJSON.mjs

@@ -0,0 +1,80 @@
+/**
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import Utils from "../Utils";
+
+/**
+ * CSV to JSON operation
+ */
+class CSVToJSON extends Operation {
+
+    /**
+     * CSVToJSON constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "CSV to JSON";
+        this.module = "Default";
+        this.description = "Converts a CSV file to JSON format.";
+        this.infoURL = "https://wikipedia.org/wiki/Comma-separated_values";
+        this.inputType = "string";
+        this.outputType = "JSON";
+        this.args = [
+            {
+                name: "Cell delimiters",
+                type: "binaryShortString",
+                value: ","
+            },
+            {
+                name: "Row delimiters",
+                type: "binaryShortString",
+                value: "\\r\\n"
+            },
+            {
+                name: "Format",
+                type: "option",
+                value: ["Array of dictionaries", "Array of arrays"]
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {JSON}
+     */
+    run(input, args) {
+        const [cellDelims, rowDelims, format] = args;
+        let json, header;
+
+        try {
+            json = Utils.parseCSV(input, cellDelims.split(""), rowDelims.split(""));
+        } catch (err) {
+            throw new OperationError("Unable to parse CSV: " + err);
+        }
+
+        switch (format) {
+            case "Array of dictionaries":
+                header = json[0];
+                return json.slice(1).map(row => {
+                    const obj = {};
+                    header.forEach((h, i) => {
+                        obj[h] = row[i];
+                    });
+                    return obj;
+                });
+            case "Array of arrays":
+            default:
+                return json;
+        }
+    }
+
+}
+
+export default CSVToJSON;

src/core/operations/CTPH.mjs

@@ -19,7 +19,7 @@ class CTPH extends Operation {
         super();
 
         this.name = "CTPH";
-        this.module = "Hashing";
+        this.module = "Crypto";
         this.description = "Context Triggered Piecewise Hashing, also called Fuzzy Hashing, can match inputs that have homologies. Such inputs have sequences of identical bytes in the same order, although bytes in between these sequences may be different in both content and length.<br><br>CTPH was originally based on the work of Dr. Andrew Tridgell and a spam email detector called SpamSum. This method was adapted by Jesse Kornblum and published at the DFRWS conference in 2006 in a paper 'Identifying Almost Identical Files Using Context Triggered Piecewise Hashing'.";
         this.infoURL = "https://forensicswiki.org/wiki/Context_Triggered_Piecewise_Hashing";
         this.inputType = "string";

src/core/operations/CitrixCTX1Decode.mjs

@@ -0,0 +1,58 @@
+/**
+ * @author bwhitn [brian.m.whitney@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import cptable from "../vendor/js-codepage/cptable.js";
+
+/**
+ * Citrix CTX1 Decode operation
+ */
+class CitrixCTX1Decode extends Operation {
+
+    /**
+     * CitrixCTX1Decode constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Citrix CTX1 Decode";
+        this.module = "Encodings";
+        this.description = "Decodes strings in a Citrix CTX1 password format to plaintext.";
+        this.infoURL = "https://www.reddit.com/r/AskNetsec/comments/1s3r6y/citrix_ctx1_hash_decoding/";
+        this.inputType = "byteArray";
+        this.outputType = "string";
+        this.args = [];
+    }
+
+    /**
+     * @param {byteArray} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        if (input.length % 4 !== 0) {
+            throw new OperationError("Incorrect hash length");
+        }
+        const revinput = input.reverse();
+        const result = [];
+        let temp = 0;
+        for (let i = 0; i < revinput.length; i += 2) {
+            if (i + 2 >= revinput.length) {
+                temp = 0;
+            } else {
+                temp = ((revinput[i + 2] - 0x41) & 0xf) ^ (((revinput[i + 3]- 0x41) << 4) & 0xf0);
+            }
+            temp = (((revinput[i] - 0x41) & 0xf) ^ (((revinput[i + 1] - 0x41) << 4) & 0xf0)) ^ 0xa5 ^ temp;
+            result.push(temp);
+        }
+        // Decodes a utf-16le string
+        return cptable.utils.decode(1200, result.reverse());
+    }
+
+}
+
+export default CitrixCTX1Decode;

src/core/operations/CitrixCTX1Encode.mjs

@@ -0,0 +1,50 @@
+/**
+ * @author bwhitn [brian.m.whitney@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import cptable from "../vendor/js-codepage/cptable.js";
+
+/**
+ * Citrix CTX1 Encode operation
+ */
+class CitrixCTX1Encode extends Operation {
+
+    /**
+     * CitrixCTX1Encode constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Citrix CTX1 Encode";
+        this.module = "Encodings";
+        this.description = "Encodes strings to Citrix CTX1 password format.";
+        this.infoURL = "https://www.reddit.com/r/AskNetsec/comments/1s3r6y/citrix_ctx1_hash_decoding/";
+        this.inputType = "string";
+        this.outputType = "byteArray";
+        this.args = [];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {byteArray}
+     */
+    run(input, args) {
+        const utf16pass = Array.from(cptable.utils.encode(1200, input));
+        const result = [];
+        let temp = 0;
+        for (let i = 0; i < utf16pass.length; i++) {
+            temp = utf16pass[i] ^ 0xa5 ^ temp;
+            result.push(((temp >>> 4) & 0xf) + 0x41);
+            result.push((temp & 0xf) + 0x41);
+        }
+
+        return result;
+    }
+
+}
+
+export default CitrixCTX1Encode;

src/core/operations/CompareCTPHHashes.mjs

@@ -22,7 +22,7 @@ class CompareCTPHHashes extends Operation {
         super();
 
         this.name = "Compare CTPH hashes";
-        this.module = "Hashing";
+        this.module = "Crypto";
         this.description = "Compares two Context Triggered Piecewise Hashing (CTPH) fuzzy hashes to determine the similarity between them on a scale of 0 to 100.";
         this.infoURL = "https://forensicswiki.org/wiki/Context_Triggered_Piecewise_Hashing";
         this.inputType = "string";

src/core/operations/CompareSSDEEPHashes.mjs

@@ -22,7 +22,7 @@ class CompareSSDEEPHashes extends Operation {
         super();
 
         this.name = "Compare SSDEEP hashes";
-        this.module = "Hashing";
+        this.module = "Crypto";
         this.description = "Compares two SSDEEP fuzzy hashes to determine the similarity between them on a scale of 0 to 100.";
         this.infoURL = "https://forensicswiki.org/wiki/Ssdeep";
         this.inputType = "string";

src/core/operations/ContainImage.mjs

@@ -0,0 +1,143 @@
+/**
+ * @author j433866 [j433866@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import { isImage } from "../lib/FileType";
+import { toBase64 } from "../lib/Base64.mjs";
+import jimp from "jimp";
+
+/**
+ * Contain Image operation
+ */
+class ContainImage extends Operation {
+
+    /**
+     * ContainImage constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Contain Image";
+        this.module = "Image";
+        this.description = "Scales an image to the specified width and height, maintaining the aspect ratio. The image may be letterboxed.";
+        this.infoURL = "";
+        this.inputType = "byteArray";
+        this.outputType = "byteArray";
+        this.presentType = "html";
+        this.args = [
+            {
+                name: "Width",
+                type: "number",
+                value: 100,
+                min: 1
+            },
+            {
+                name: "Height",
+                type: "number",
+                value: 100,
+                min: 1
+            },
+            {
+                name: "Horizontal align",
+                type: "option",
+                value: [
+                    "Left",
+                    "Center",
+                    "Right"
+                ],
+                defaultIndex: 1
+            },
+            {
+                name: "Vertical align",
+                type: "option",
+                value: [
+                    "Top",
+                    "Middle",
+                    "Bottom"
+                ],
+                defaultIndex: 1
+            },
+            {
+                name: "Resizing algorithm",
+                type: "option",
+                value: [
+                    "Nearest Neighbour",
+                    "Bilinear",
+                    "Bicubic",
+                    "Hermite",
+                    "Bezier"
+                ],
+                defaultIndex: 1
+            }
+        ];
+    }
+
+    /**
+     * @param {byteArray} input
+     * @param {Object[]} args
+     * @returns {byteArray}
+     */
+    async run(input, args) {
+        const [width, height, hAlign, vAlign, alg] = args;
+
+        const resizeMap = {
+            "Nearest Neighbour": jimp.RESIZE_NEAREST_NEIGHBOR,
+            "Bilinear": jimp.RESIZE_BILINEAR,
+            "Bicubic": jimp.RESIZE_BICUBIC,
+            "Hermite": jimp.RESIZE_HERMITE,
+            "Bezier": jimp.RESIZE_BEZIER
+        };
+
+        const alignMap = {
+            "Left": jimp.HORIZONTAL_ALIGN_LEFT,
+            "Center": jimp.HORIZONTAL_ALIGN_CENTER,
+            "Right": jimp.HORIZONTAL_ALIGN_RIGHT,
+            "Top": jimp.VERTICAL_ALIGN_TOP,
+            "Middle": jimp.VERTICAL_ALIGN_MIDDLE,
+            "Bottom": jimp.VERTICAL_ALIGN_BOTTOM
+        };
+
+        if (!isImage(input)) {
+            throw new OperationError("Invalid file type.");
+        }
+
+        let image;
+        try {
+            image = await jimp.read(Buffer.from(input));
+        } catch (err) {
+            throw new OperationError(`Error loading image. (${err})`);
+        }
+        try {
+            if (ENVIRONMENT_IS_WORKER())
+                self.sendStatusMessage("Containing image...");
+            image.contain(width, height, alignMap[hAlign] | alignMap[vAlign], resizeMap[alg]);
+            const imageBuffer = await image.getBufferAsync(jimp.AUTO);
+            return [...imageBuffer];
+        } catch (err) {
+            throw new OperationError(`Error containing image. (${err})`);
+        }
+    }
+
+    /**
+     * Displays the contained image using HTML for web apps
+     * @param {byteArray} data
+     * @returns {html}
+     */
+    present(data) {
+        if (!data.length) return "";
+
+        const type = isImage(data);
+        if (!type) {
+            throw new OperationError("Invalid file type.");
+        }
+
+        return `<img src="data:${type};base64,${toBase64(data)}">`;
+    }
+
+}
+
+export default ContainImage;

src/core/operations/ConvertCoordinateFormat.mjs

@@ -0,0 +1,95 @@
+/**
+ * @author j433866 [j433866@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import {FORMATS, convertCoordinates} from "../lib/ConvertCoordinates";
+
+/**
+ * Convert co-ordinate format operation
+ */
+class ConvertCoordinateFormat extends Operation {
+
+    /**
+     * ConvertCoordinateFormat constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Convert co-ordinate format";
+        this.module = "Hashing";
+        this.description = "Converts geographical coordinates between different formats.<br><br>Supported formats:<ul><li>Degrees Minutes Seconds (DMS)</li><li>Degrees Decimal Minutes (DDM)</li><li>Decimal Degrees (DD)</li><li>Geohash</li><li>Military Grid Reference System (MGRS)</li><li>Ordnance Survey National Grid (OSNG)</li><li>Universal Transverse Mercator (UTM)</li></ul><br>The operation can try to detect the input co-ordinate format and delimiter automatically, but this may not always work correctly.";
+        this.infoURL = "https://wikipedia.org/wiki/Geographic_coordinate_conversion";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [
+            {
+                "name": "Input Format",
+                "type": "option",
+                "value": ["Auto"].concat(FORMATS)
+            },
+            {
+                "name": "Input Delimiter",
+                "type": "option",
+                "value": [
+                    "Auto",
+                    "Direction Preceding",
+                    "Direction Following",
+                    "\\n",
+                    "Comma",
+                    "Semi-colon",
+                    "Colon"
+                ]
+            },
+            {
+                "name": "Output Format",
+                "type": "option",
+                "value": FORMATS
+            },
+            {
+                "name": "Output Delimiter",
+                "type": "option",
+                "value": [
+                    "Space",
+                    "\\n",
+                    "Comma",
+                    "Semi-colon",
+                    "Colon"
+                ]
+            },
+            {
+                "name": "Include Compass Directions",
+                "type": "option",
+                "value": [
+                    "None",
+                    "Before",
+                    "After"
+                ]
+            },
+            {
+                "name": "Precision",
+                "type": "number",
+                "value": 3
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        if (input.replace(/[\s+]/g, "") !== "") {
+            const [inFormat, inDelim, outFormat, outDelim, incDirection, precision] = args;
+            const result = convertCoordinates(input, inFormat, inDelim, outFormat, outDelim, incDirection, precision);
+            return result;
+        } else {
+            return input;
+        }
+    }
+}
+
+export default ConvertCoordinateFormat;

src/core/operations/CoverImage.mjs

@@ -0,0 +1,143 @@
+/**
+ * @author j433866 [j433866@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import { isImage } from "../lib/FileType";
+import { toBase64 } from "../lib/Base64.mjs";
+import jimp from "jimp";
+
+/**
+ * Cover Image operation
+ */
+class CoverImage extends Operation {
+
+    /**
+     * CoverImage constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Cover Image";
+        this.module = "Image";
+        this.description = "Scales the image to the given width and height, keeping the aspect ratio. The image may be clipped.";
+        this.infoURL = "";
+        this.inputType = "byteArray";
+        this.outputType = "byteArray";
+        this.presentType = "html";
+        this.args = [
+            {
+                name: "Width",
+                type: "number",
+                value: 100,
+                min: 1
+            },
+            {
+                name: "Height",
+                type: "number",
+                value: 100,
+                min: 1
+            },
+            {
+                name: "Horizontal align",
+                type: "option",
+                value: [
+                    "Left",
+                    "Center",
+                    "Right"
+                ],
+                defaultIndex: 1
+            },
+            {
+                name: "Vertical align",
+                type: "option",
+                value: [
+                    "Top",
+                    "Middle",
+                    "Bottom"
+                ],
+                defaultIndex: 1
+            },
+            {
+                name: "Resizing algorithm",
+                type: "option",
+                value: [
+                    "Nearest Neighbour",
+                    "Bilinear",
+                    "Bicubic",
+                    "Hermite",
+                    "Bezier"
+                ],
+                defaultIndex: 1
+            }
+        ];
+    }
+
+    /**
+     * @param {byteArray} input
+     * @param {Object[]} args
+     * @returns {byteArray}
+     */
+    async run(input, args) {
+        const [width, height, hAlign, vAlign, alg] = args;
+
+        const resizeMap = {
+            "Nearest Neighbour": jimp.RESIZE_NEAREST_NEIGHBOR,
+            "Bilinear": jimp.RESIZE_BILINEAR,
+            "Bicubic": jimp.RESIZE_BICUBIC,
+            "Hermite": jimp.RESIZE_HERMITE,
+            "Bezier": jimp.RESIZE_BEZIER
+        };
+
+        const alignMap = {
+            "Left": jimp.HORIZONTAL_ALIGN_LEFT,
+            "Center": jimp.HORIZONTAL_ALIGN_CENTER,
+            "Right": jimp.HORIZONTAL_ALIGN_RIGHT,
+            "Top": jimp.VERTICAL_ALIGN_TOP,
+            "Middle": jimp.VERTICAL_ALIGN_MIDDLE,
+            "Bottom": jimp.VERTICAL_ALIGN_BOTTOM
+        };
+
+        if (!isImage(input)) {
+            throw new OperationError("Invalid file type.");
+        }
+
+        let image;
+        try {
+            image = await jimp.read(Buffer.from(input));
+        } catch (err) {
+            throw new OperationError(`Error loading image. (${err})`);
+        }
+        try {
+            if (ENVIRONMENT_IS_WORKER())
+                self.sendStatusMessage("Covering image...");
+            image.cover(width, height, alignMap[hAlign] | alignMap[vAlign], resizeMap[alg]);
+            const imageBuffer = await image.getBufferAsync(jimp.AUTO);
+            return [...imageBuffer];
+        } catch (err) {
+            throw new OperationError(`Error covering image. (${err})`);
+        }
+    }
+
+    /**
+     * Displays the covered image using HTML for web apps
+     * @param {byteArray} data
+     * @returns {html}
+     */
+    present(data) {
+        if (!data.length) return "";
+
+        const type = isImage(data);
+        if (!type) {
+            throw new OperationError("Invalid file type.");
+        }
+
+        return `<img src="data:${type};base64,${toBase64(data)}">`;
+    }
+
+}
+
+export default CoverImage;

src/core/operations/CropImage.mjs

@@ -0,0 +1,144 @@
+/**
+ * @author j433866 [j433866@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import { isImage } from "../lib/FileType";
+import { toBase64 } from "../lib/Base64.mjs";
+import jimp from "jimp";
+
+/**
+ * Crop Image operation
+ */
+class CropImage extends Operation {
+
+    /**
+     * CropImage constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Crop Image";
+        this.module = "Image";
+        this.description = "Crops an image to the specified region, or automatically crops edges.<br><br><b><u>Autocrop</u></b><br>Automatically crops same-colour borders from the image.<br><br><u>Autocrop tolerance</u><br>A percentage value for the tolerance of colour difference between pixels.<br><br><u>Only autocrop frames</u><br>Only crop real frames (all sides must have the same border)<br><br><u>Symmetric autocrop</u><br>Force autocrop to be symmetric (top/bottom and left/right are cropped by the same amount)<br><br><u>Autocrop keep border</u><br>The number of pixels of border to leave around the image.";
+        this.infoURL = "https://wikipedia.org/wiki/Cropping_(image)";
+        this.inputType = "byteArray";
+        this.outputType = "byteArray";
+        this.presentType = "html";
+        this.args = [
+            {
+                name: "X Position",
+                type: "number",
+                value: 0,
+                min: 0
+            },
+            {
+                name: "Y Position",
+                type: "number",
+                value: 0,
+                min: 0
+            },
+            {
+                name: "Width",
+                type: "number",
+                value: 10,
+                min: 1
+            },
+            {
+                name: "Height",
+                type: "number",
+                value: 10,
+                min: 1
+            },
+            {
+                name: "Autocrop",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Autocrop tolerance (%)",
+                type: "number",
+                value: 0.02,
+                min: 0,
+                max: 100,
+                step: 0.01
+            },
+            {
+                name: "Only autocrop frames",
+                type: "boolean",
+                value: true
+            },
+            {
+                name: "Symmetric autocrop",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Autocrop keep border (px)",
+                type: "number",
+                value: 0,
+                min: 0
+            }
+        ];
+    }
+
+    /**
+     * @param {byteArray} input
+     * @param {Object[]} args
+     * @returns {byteArray}
+     */
+    async run(input, args) {
+        const [xPos, yPos, width, height, autocrop, autoTolerance, autoFrames, autoSymmetric, autoBorder] = args;
+        if (!isImage(input)) {
+            throw new OperationError("Invalid file type.");
+        }
+
+        let image;
+        try {
+            image = await jimp.read(Buffer.from(input));
+        } catch (err) {
+            throw new OperationError(`Error loading image. (${err})`);
+        }
+        try {
+            if (ENVIRONMENT_IS_WORKER())
+                self.sendStatusMessage("Cropping image...");
+            if (autocrop) {
+                image.autocrop({
+                    tolerance: (autoTolerance / 100),
+                    cropOnlyFrames: autoFrames,
+                    cropSymmetric: autoSymmetric,
+                    leaveBorder: autoBorder
+                });
+            } else {
+                image.crop(xPos, yPos, width, height);
+            }
+
+            const imageBuffer = await image.getBufferAsync(jimp.AUTO);
+            return [...imageBuffer];
+        } catch (err) {
+            throw new OperationError(`Error cropping image. (${err})`);
+        }
+    }
+
+    /**
+     * Displays the cropped image using HTML for web apps
+     * @param {byteArray} data
+     * @returns {html}
+     */
+    present(data) {
+        if (!data.length) return "";
+
+        const type = isImage(data);
+        if (!type) {
+            throw new OperationError("Invalid file type.");
+        }
+
+        return `<img src="data:${type};base64,${toBase64(data)}">`;
+    }
+
+}
+
+export default CropImage;

src/core/operations/DNSOverHTTPS.mjs

@@ -0,0 +1,125 @@
+/**
+ * @author h345983745
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+
+/**
+ * DNS over HTTPS operation
+ */
+class DNSOverHTTPS extends Operation {
+
+    /**
+     * DNSOverHTTPS constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "DNS over HTTPS";
+        this.module = "Default";
+        this.description = [
+            "Takes a single domain name and performs a DNS lookup using DNS over HTTPS.",
+            "<br><br>",
+            "By default, <a href='https://developers.cloudflare.com/1.1.1.1/dns-over-https/'>Cloudflare</a> and <a href='https://developers.google.com/speed/public-dns/docs/dns-over-https'>Google</a> DNS over HTTPS services are supported.",
+            "<br><br>",
+            "Can be used with any service that supports the GET parameters <code>name</code> and <code>type</code>."
+        ].join("\n");
+        this.infoURL = "https://wikipedia.org/wiki/DNS_over_HTTPS";
+        this.inputType = "string";
+        this.outputType = "JSON";
+        this.manualBake = true;
+        this.args = [
+            {
+                name: "Resolver",
+                type: "editableOption",
+                value: [
+                    {
+                        name: "Google",
+                        value: "https://dns.google.com/resolve"
+                    },
+                    {
+                        name: "Cloudflare",
+                        value: "https://cloudflare-dns.com/dns-query"
+                    }
+                ]
+            },
+            {
+                name: "Request Type",
+                type: "option",
+                value: [
+                    "A",
+                    "AAAA",
+                    "TXT",
+                    "MX",
+                    "DNSKEY",
+                    "NS"
+                ]
+            },
+            {
+                name: "Answer Data Only",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Validate DNSSEC",
+                type: "boolean",
+                value: true
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {JSON}
+     */
+    run(input, args) {
+        const [resolver, requestType, justAnswer, DNSSEC] = args;
+        let url = URL;
+        try {
+            url = new URL(resolver);
+        } catch (error) {
+            throw new OperationError(error.toString() +
+            "\n\nThis error could be caused by one of the following:\n" +
+            " - An invalid Resolver URL\n");
+        }
+        const params = {name: input, type: requestType, cd: DNSSEC};
+
+        url.search = new URLSearchParams(params);
+
+        return fetch(url, {headers: {"accept": "application/dns-json"}}).then(response => {
+            return response.json();
+        }).then(data => {
+            if (justAnswer) {
+                return extractData(data.Answer);
+            }
+            return data;
+        }).catch(e => {
+            throw new OperationError(`Error making request to ${url}\n${e.toString()}`);
+        });
+
+    }
+}
+
+/**
+ * Construct an array of just data from a DNS Answer section
+ *
+ * @private
+ * @param {JSON} data
+ * @returns {JSON}
+ */
+function extractData(data) {
+    if (typeof(data) == "undefined"){
+        return [];
+    } else {
+        const dataValues = [];
+        data.forEach(element => {
+            dataValues.push(element.data);
+        });
+        return dataValues;
+    }
+}
+
+export default DNSOverHTTPS;

src/core/operations/DecodeText.mjs

@@ -20,7 +20,7 @@ class DecodeText extends Operation {
         super();
 
         this.name = "Decode text";
-        this.module = "CharEnc";
+        this.module = "Encodings";
         this.description = [
             "Decodes text from the chosen character encoding.",
             "<br><br>",

src/core/operations/DefangURL.mjs

@@ -0,0 +1,102 @@
+/**
+ * @author arnydo [arnydo@protonmail.com]
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import {URL_REGEX, DOMAIN_REGEX} from "../lib/Extract";
+
+/**
+ * DefangURL operation
+ */
+class DefangURL extends Operation {
+
+    /**
+     * DefangURL constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Defang URL";
+        this.module = "Default";
+        this.description = "Takes a Universal Resource Locator (URL) and 'Defangs' it; meaning the URL becomes invalid, neutralising the risk of accidentally clicking on a malicious link.<br><br>This is often used when dealing with malicious links or IOCs.<br><br>Works well when combined with the 'Extract URLs' operation.";
+        this.infoURL = "https://isc.sans.edu/forums/diary/Defang+all+the+things/22744/";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [
+            {
+                name: "Escape dots",
+                type: "boolean",
+                value: true
+            },
+            {
+                name: "Escape http",
+                type: "boolean",
+                value: true
+            },
+            {
+                name: "Escape ://",
+                type: "boolean",
+                value: true
+            },
+            {
+                name: "Process",
+                type: "option",
+                value: ["Valid domains and full URLs", "Only full URLs", "Everything"]
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        const [dots, http, slashes, process] = args;
+
+        switch (process) {
+            case "Valid domains and full URLs":
+                input = input.replace(URL_REGEX, x => {
+                    return defangURL(x, dots, http, slashes);
+                });
+                input = input.replace(DOMAIN_REGEX, x => {
+                    return defangURL(x, dots, http, slashes);
+                });
+                break;
+            case "Only full URLs":
+                input = input.replace(URL_REGEX, x => {
+                    return defangURL(x, dots, http, slashes);
+                });
+                break;
+            case "Everything":
+                input = defangURL(input, dots, http, slashes);
+                break;
+        }
+
+        return input;
+    }
+
+}
+
+
+/**
+ * Defangs a given URL
+ *
+ * @param {string} url
+ * @param {boolean} dots
+ * @param {boolean} http
+ * @param {boolean} slashes
+ * @returns {string}
+ */
+function defangURL(url, dots, http, slashes) {
+    if (dots) url = url.replace(/\./g, "[.]");
+    if (http) url = url.replace(/http/gi, "hxxp");
+    if (slashes) url = url.replace(/:\/\//g, "[://]");
+
+    return url;
+}
+
+export default DefangURL;

src/core/operations/DetectFileType.mjs

@@ -5,7 +5,8 @@
  */
 
 import Operation from "../Operation";
-import Magic from "../lib/Magic";
+import {detectFileType} from "../lib/FileType";
+import {FILE_SIGNATURES} from "../lib/FileSignatures";
 
 /**
  * Detect File Type operation
@@ -24,7 +25,13 @@ class DetectFileType extends Operation {
         this.infoURL = "https://wikipedia.org/wiki/List_of_file_signatures";
         this.inputType = "ArrayBuffer";
         this.outputType = "string";
-        this.args = [];
+        this.args = Object.keys(FILE_SIGNATURES).map(cat => {
+            return {
+                name: cat,
+                type: "boolean",
+                value: true
+            };
+        });
     }
 
     /**
@@ -34,17 +41,27 @@ class DetectFileType extends Operation {
      */
     run(input, args) {
         const data = new Uint8Array(input),
-            type = Magic.magicFileType(data);
+            categories = [];
 
-        if (!type) {
+        args.forEach((cat, i) => {
+            if (cat) categories.push(Object.keys(FILE_SIGNATURES)[i]);
+        });
+
+        const types = detectFileType(data, categories);
+
+        if (!types.length) {
             return "Unknown file type. Have you tried checking the entropy of this data to determine whether it might be encrypted or compressed?";
         } else {
-            let output = "File extension: " + type.ext + "\n" +
-                "MIME type:      " + type.mime;
+            let output = "";
 
-            if (type.desc && type.desc.length) {
-                output += "\nDescription:    " + type.desc;
-            }
+            types.forEach(type => {
+                output += "File extension: " + type.extension + "\n" +
+                    "MIME type:      " + type.mime + "\n";
+
+                if (type.description && type.description.length) {
+                    output += "\nDescription:    " + type.description + "\n";
+                }
+            });
 
             return output;
         }

src/core/operations/DitherImage.mjs

@@ -0,0 +1,79 @@
+/**
+ * @author j433866 [j433866@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import { isImage } from "../lib/FileType";
+import { toBase64 } from "../lib/Base64";
+import jimp from "jimp";
+
+/**
+ * Image Dither operation
+ */
+class DitherImage extends Operation {
+
+    /**
+     * DitherImage constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Dither Image";
+        this.module = "Image";
+        this.description = "Apply a dither effect to an image.";
+        this.infoURL = "https://wikipedia.org/wiki/Dither";
+        this.inputType = "byteArray";
+        this.outputType = "byteArray";
+        this.presentType = "html";
+        this.args = [];
+    }
+
+    /**
+     * @param {byteArray} input
+     * @param {Object[]} args
+     * @returns {byteArray}
+     */
+    async run(input, args) {
+        if (!isImage(input)) {
+            throw new OperationError("Invalid file type.");
+        }
+
+        let image;
+        try {
+            image = await jimp.read(Buffer.from(input));
+        } catch (err) {
+            throw new OperationError(`Error loading image. (${err})`);
+        }
+        try {
+            if (ENVIRONMENT_IS_WORKER())
+                self.sendStatusMessage("Applying dither to image...");
+            image.dither565();
+            const imageBuffer = await image.getBufferAsync(jimp.AUTO);
+            return [...imageBuffer];
+        } catch (err) {
+            throw new OperationError(`Error applying dither to image. (${err})`);
+        }
+    }
+
+    /**
+     * Displays the dithered image using HTML for web apps
+     * @param {byteArray} data
+     * @returns {html}
+     */
+    present(data) {
+        if (!data.length) return "";
+
+        const type = isImage(data);
+        if (!type) {
+            throw new OperationError("Invalid file type.");
+        }
+
+        return `<img src="data:${type};base64,${toBase64(data)}">`;
+    }
+
+}
+
+export default DitherImage;

src/core/operations/Divide.mjs

@@ -43,7 +43,7 @@ class Divide extends Operation {
      */
     run(input, args) {
         const val = div(createNumArray(input, args[0]));
-        return val instanceof BigNumber ? val : new BigNumber(NaN);
+        return BigNumber.isBigNumber(val) ? val : new BigNumber(NaN);
     }
 
 }

src/core/operations/DropBytes.mjs

@@ -5,7 +5,6 @@
  */
 
 import Operation from "../Operation";
-import OperationError from "../errors/OperationError";
 
 /**
  * Drop bytes operation
@@ -20,7 +19,7 @@ class DropBytes extends Operation {
 
         this.name = "Drop bytes";
         this.module = "Default";
-        this.description = "Cuts a slice of the specified number of bytes out of the data.";
+        this.description = "Cuts a slice of the specified number of bytes out of the data. Negative values are allowed.";
         this.inputType = "ArrayBuffer";
         this.outputType = "ArrayBuffer";
         this.args = [
@@ -50,14 +49,25 @@ class DropBytes extends Operation {
      * @throws {OperationError} if invalid input
      */
     run(input, args) {
-        const start = args[0],
-            length = args[1],
-            applyToEachLine = args[2];
-
-        if (start < 0 || length < 0)
-            throw new OperationError("Error: Invalid value");
+        let start = args[0],
+            length = args[1];
+        const applyToEachLine = args[2];
 
         if (!applyToEachLine) {
+            if (start < 0) { // Take from the end
+                start = input.byteLength + start;
+            }
+
+            if (length < 0) { // Flip start point
+                start = start + length;
+                if (start < 0) {
+                    start = input.byteLength + start;
+                    length = start - length;
+                } else {
+                    length = -length;
+                }
+            }
+
             const left = input.slice(0, start),
                 right = input.slice(start + length, input.byteLength);
             const result = new Uint8Array(left.byteLength + right.byteLength);
@@ -82,10 +92,28 @@ class DropBytes extends Operation {
         }
         lines.push(line);
 
-        let output = [];
+        let output = [],
+            s = start,
+            l = length;
         for (i = 0; i < lines.length; i++) {
-            output = output.concat(lines[i].slice(0, start).concat(lines[i].slice(start+length, lines[i].length)));
+            if (s < 0) { // Take from the end
+                s = lines[i].length + s;
+            }
+
+            if (l < 0) { // Flip start point
+                s = s + l;
+                if (s < 0) {
+                    s = lines[i].length + s;
+                    l = s - l;
+                } else {
+                    l = -l;
+                }
+            }
+
+            output = output.concat(lines[i].slice(0, s).concat(lines[i].slice(s+l, lines[i].length)));
             output.push(0x0a);
+            s = start;
+            l = length;
         }
         return new Uint8Array(output.slice(0, output.length-1)).buffer;
     }

src/core/operations/EncodeText.mjs

@@ -20,7 +20,7 @@ class EncodeText extends Operation {
         super();
 
         this.name = "Encode text";
-        this.module = "CharEnc";
+        this.module = "Encodings";
         this.description = [
             "Encodes text into the chosen character encoding.",
             "<br><br>",

src/core/operations/ExtractDomains.mjs

@@ -5,7 +5,7 @@
  */
 
 import Operation from "../Operation";
-import { search } from "../lib/Extract";
+import { search, DOMAIN_REGEX } from "../lib/Extract";
 
 /**
  * Extract domains operation
@@ -38,10 +38,8 @@ class ExtractDomains extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const displayTotal = args[0],
-            regex = /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/ig;
-
-        return search(input, regex, null, displayTotal);
+        const displayTotal = args[0];
+        return search(input, DOMAIN_REGEX, null, displayTotal);
     }
 
 }

src/core/operations/ExtractEmailAddresses.mjs

@@ -39,8 +39,8 @@ class ExtractEmailAddresses extends Operation {
      */
     run(input, args) {
         const displayTotal = args[0],
-            regex = /\b\w[-.\w]*@[-\w]+(?:\.[-\w]+)*\.[A-Z]{2,4}\b/ig;
-
+        // email regex from: https://www.regextester.com/98066
+            regex = /(?:[\u00A0-\uD7FF\uE000-\uFFFF-a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[\u00A0-\uD7FF\uE000-\uFFFF-a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[\u00A0-\uD7FF\uE000-\uFFFF-a-z0-9](?:[\u00A0-\uD7FF\uE000-\uFFFF-a-z0-9-]*[\u00A0-\uD7FF\uE000-\uFFFF-a-z0-9])?\.)+[\u00A0-\uD7FF\uE000-\uFFFF-a-z0-9](?:[\u00A0-\uD7FF\uE000-\uFFFF-a-z0-9-]*[\u00A0-\uD7FF\uE000-\uFFFF-a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])/ig;
         return search(input, regex, null, displayTotal);
     }
 

src/core/operations/ExtractFiles.mjs

@@ -0,0 +1,100 @@
+/**
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import Utils from "../Utils";
+import {scanForFileTypes, extractFile} from "../lib/FileType";
+import {FILE_SIGNATURES} from "../lib/FileSignatures";
+
+/**
+ * Extract Files operation
+ */
+class ExtractFiles extends Operation {
+
+    /**
+     * ExtractFiles constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Extract Files";
+        this.module = "Default";
+        this.description = "TODO";
+        this.infoURL = "https://forensicswiki.org/wiki/File_Carving";
+        this.inputType = "ArrayBuffer";
+        this.outputType = "List<File>";
+        this.presentType = "html";
+        this.args = Object.keys(FILE_SIGNATURES).map(cat => {
+            return {
+                name: cat,
+                type: "boolean",
+                value: cat === "Miscellaneous" ? false : true
+            };
+        }).concat([
+            {
+                name: "Ignore failed extractions",
+                type: "boolean",
+                value: "true"
+            }
+        ]);
+    }
+
+    /**
+     * @param {ArrayBuffer} input
+     * @param {Object[]} args
+     * @returns {List<File>}
+     */
+    run(input, args) {
+        const bytes = new Uint8Array(input),
+            categories = [],
+            ignoreFailedExtractions = args.pop(1);
+
+        args.forEach((cat, i) => {
+            if (cat) categories.push(Object.keys(FILE_SIGNATURES)[i]);
+        });
+
+        // Scan for embedded files
+        const detectedFiles = scanForFileTypes(bytes, categories);
+
+        // Extract each file that we support
+        const files = [];
+        const errors = [];
+        detectedFiles.forEach(detectedFile => {
+            try {
+                files.push(extractFile(bytes, detectedFile.fileDetails, detectedFile.offset));
+            } catch (err) {
+                if (!ignoreFailedExtractions && err.message.indexOf("No extraction algorithm available") < 0) {
+                    errors.push(
+                        `Error while attempting to extract ${detectedFile.fileDetails.name} ` +
+                        `at offset ${detectedFile.offset}:\n` +
+                        `${err.message}`
+                    );
+                }
+            }
+        });
+
+        if (errors.length) {
+            throw new OperationError(errors.join("\n\n"));
+        }
+
+        return files;
+    }
+
+
+    /**
+     * Displays the files in HTML for web apps.
+     *
+     * @param {File[]} files
+     * @returns {html}
+     */
+    async present(files) {
+        return await Utils.displayFilesAsHTML(files);
+    }
+
+}
+
+export default ExtractFiles;

src/core/operations/ExtractURLs.mjs

@@ -5,7 +5,7 @@
  */
 
 import Operation from "../Operation";
-import { search } from "../lib/Extract";
+import { search, URL_REGEX } from "../lib/Extract";
 
 /**
  * Extract URLs operation
@@ -38,16 +38,8 @@ class ExtractURLs extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const displayTotal = args[0],
-            protocol = "[A-Z]+://",
-            hostname = "[-\\w]+(?:\\.\\w[-\\w]*)+",
-            port = ":\\d+";
-        let path = "/[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]*";
-
-        path += "(?:[.!,?]+[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]+)*";
-        const regex = new RegExp(protocol + hostname + "(?:" + port +
-            ")?(?:" + path + ")?", "ig");
-        return search(input, regex, null, displayTotal);
+        const displayTotal = args[0];
+        return search(input, URL_REGEX, null, displayTotal);
     }
 
 }

src/core/operations/Filter.mjs

@@ -8,6 +8,7 @@ import Operation from "../Operation";
 import Utils from "../Utils";
 import {INPUT_DELIM_OPTIONS} from "../lib/Delim";
 import OperationError from "../errors/OperationError";
+import XRegExp from "xregexp";
 
 /**
  * Filter operation
@@ -21,7 +22,7 @@ class Filter extends Operation {
         super();
 
         this.name = "Filter";
-        this.module = "Default";
+        this.module = "Regex";
         this.description = "Splits up the input using the specified delimiter and then filters each branch based on a regular expression.";
         this.inputType = "string";
         this.outputType = "string";
@@ -55,7 +56,7 @@ class Filter extends Operation {
         let regex;
 
         try {
-            regex = new RegExp(args[1]);
+            regex = new XRegExp(args[1]);
         } catch (err) {
             throw new OperationError(`Invalid regex. Details: ${err.message}`);
         }

src/core/operations/FindReplace.mjs

@@ -6,6 +6,7 @@
 
 import Operation from "../Operation";
 import Utils from "../Utils";
+import XRegExp from "xregexp";
 
 /**
  * Find / Replace operation
@@ -50,6 +51,11 @@ class FindReplace extends Operation {
                 "name": "Multiline matching",
                 "type": "boolean",
                 "value": true
+            },
+            {
+                "name": "Dot matches all",
+                "type": "boolean",
+                "value": false
             }
         ];
     }
@@ -60,16 +66,17 @@ class FindReplace extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const [{option: type}, replace, g, i, m] = args;
+        const [{option: type}, replace, g, i, m, s] = args;
         let find = args[0].string,
             modifiers = "";
 
         if (g) modifiers += "g";
         if (i) modifiers += "i";
         if (m) modifiers += "m";
+        if (s) modifiers += "s";
 
         if (type === "Regex") {
-            find = new RegExp(find, modifiers);
+            find = new XRegExp(find, modifiers);
             return input.replace(find, replace);
         }
 
@@ -77,7 +84,7 @@ class FindReplace extends Operation {
             find = Utils.parseEscapedChars(find);
         }
 
-        find = new RegExp(Utils.escapeRegex(find), modifiers);
+        find = new XRegExp(Utils.escapeRegex(find), modifiers);
 
         return input.replace(find, replace);
     }

src/core/operations/Fletcher16Checksum.mjs

@@ -19,7 +19,7 @@ class Fletcher16Checksum extends Operation {
         super();
 
         this.name = "Fletcher-16 Checksum";
-        this.module = "Hashing";
+        this.module = "Crypto";
         this.description = "The Fletcher checksum is an algorithm for computing a position-dependent checksum devised by John Gould Fletcher at Lawrence Livermore Labs in the late 1970s.<br><br>The objective of the Fletcher checksum was to provide error-detection properties approaching those of a cyclic redundancy check but with the lower computational effort associated with summation techniques.";
         this.infoURL = "https://wikipedia.org/wiki/Fletcher%27s_checksum#Fletcher-16";
         this.inputType = "byteArray";

src/core/operations/Fletcher32Checksum.mjs

@@ -19,7 +19,7 @@ class Fletcher32Checksum extends Operation {
         super();
 
         this.name = "Fletcher-32 Checksum";
-        this.module = "Hashing";
+        this.module = "Crypto";
         this.description = "The Fletcher checksum is an algorithm for computing a position-dependent checksum devised by John Gould Fletcher at Lawrence Livermore Labs in the late 1970s.<br><br>The objective of the Fletcher checksum was to provide error-detection properties approaching those of a cyclic redundancy check but with the lower computational effort associated with summation techniques.";
         this.infoURL = "https://wikipedia.org/wiki/Fletcher%27s_checksum#Fletcher-32";
         this.inputType = "byteArray";

src/core/operations/Fletcher64Checksum.mjs

@@ -19,7 +19,7 @@ class Fletcher64Checksum extends Operation {
         super();
 
         this.name = "Fletcher-64 Checksum";
-        this.module = "Hashing";
+        this.module = "Crypto";
         this.description = "The Fletcher checksum is an algorithm for computing a position-dependent checksum devised by John Gould Fletcher at Lawrence Livermore Labs in the late 1970s.<br><br>The objective of the Fletcher checksum was to provide error-detection properties approaching those of a cyclic redundancy check but with the lower computational effort associated with summation techniques.";
         this.infoURL = "https://wikipedia.org/wiki/Fletcher%27s_checksum#Fletcher-64";
         this.inputType = "byteArray";

src/core/operations/Fletcher8Checksum.mjs

@@ -19,7 +19,7 @@ class Fletcher8Checksum extends Operation {
         super();
 
         this.name = "Fletcher-8 Checksum";
-        this.module = "Hashing";
+        this.module = "Crypto";
         this.description = "The Fletcher checksum is an algorithm for computing a position-dependent checksum devised by John Gould Fletcher at Lawrence Livermore Labs in the late 1970s.<br><br>The objective of the Fletcher checksum was to provide error-detection properties approaching those of a cyclic redundancy check but with the lower computational effort associated with summation techniques.";
         this.infoURL = "https://wikipedia.org/wiki/Fletcher%27s_checksum";
         this.inputType = "byteArray";

src/core/operations/FlipImage.mjs

@@ -0,0 +1,94 @@
+/**
+ * @author j433866 [j433866@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import { isImage } from "../lib/FileType";
+import { toBase64 } from "../lib/Base64";
+import jimp from "jimp";
+
+/**
+ * Flip Image operation
+ */
+class FlipImage extends Operation {
+
+    /**
+     * FlipImage constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Flip Image";
+        this.module = "Image";
+        this.description = "Flips an image along its X or Y axis.";
+        this.infoURL = "";
+        this.inputType = "byteArray";
+        this.outputType = "byteArray";
+        this.presentType = "html";
+        this.args = [
+            {
+                name: "Axis",
+                type: "option",
+                value: ["Horizontal", "Vertical"]
+            }
+        ];
+    }
+
+    /**
+     * @param {byteArray} input
+     * @param {Object[]} args
+     * @returns {byteArray}
+     */
+    async run(input, args) {
+        const [flipAxis] = args;
+        if (!isImage(input)) {
+            throw new OperationError("Invalid input file type.");
+        }
+
+        let image;
+        try {
+            image = await jimp.read(Buffer.from(input));
+        } catch (err) {
+            throw new OperationError(`Error loading image. (${err})`);
+        }
+        try {
+            if (ENVIRONMENT_IS_WORKER())
+                self.sendStatusMessage("Flipping image...");
+            switch (flipAxis){
+                case "Horizontal":
+                    image.flip(true, false);
+                    break;
+                case "Vertical":
+                    image.flip(false, true);
+                    break;
+            }
+
+            const imageBuffer = await image.getBufferAsync(jimp.AUTO);
+            return [...imageBuffer];
+        } catch (err) {
+            throw new OperationError(`Error flipping image. (${err})`);
+        }
+    }
+
+    /**
+     * Displays the flipped image using HTML for web apps
+     * @param {byteArray} data
+     * @returns {html}
+     */
+    present(data) {
+        if (!data.length) return "";
+
+        const type = isImage(data);
+        if (!type) {
+            throw new OperationError("Invalid file type.");
+        }
+
+        return `<img src="data:${type};base64,${toBase64(data)}">`;
+    }
+
+}
+
+export default FlipImage;

src/core/operations/Fork.mjs

@@ -89,7 +89,7 @@ class Fork extends Operation {
         // Run recipe over each tranche
         for (i = 0; i < inputs.length; i++) {
             // Baseline ing values for each tranche so that registers are reset
-            subOpList.forEach((op, i) => {
+            recipe.opList.forEach((op, i) => {
                 op.ingValues = JSON.parse(JSON.stringify(ingValues[i]));
             });
 

src/core/operations/FromBase58.mjs

@@ -71,6 +71,11 @@ class FromBase58 extends Operation {
 
         if (input.length === 0) return [];
 
+        let zeroPrefix = 0;
+        for (let i = 0; i < input.length && input[i] === alphabet[0]; i++) {
+            zeroPrefix++;
+        }
+
         [].forEach.call(input, function(c, charIndex) {
             const index = alphabet.indexOf(c);
 
@@ -98,6 +103,10 @@ class FromBase58 extends Operation {
             }
         });
 
+        while (zeroPrefix--) {
+            result.push(0);
+        }
+
         return result.reverse();
     }
 

src/core/operations/FromBase62.mjs

@@ -0,0 +1,58 @@
+/**
+ * @author tcode2k16 [tcode2k16@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import BigNumber from "bignumber.js";
+import Utils from "../Utils";
+
+
+/**
+ * From Base62 operation
+ */
+class FromBase62 extends Operation {
+
+    /**
+     * FromBase62 constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "From Base62";
+        this.module = "Default";
+        this.description = "Base62 is a notation for encoding arbitrary byte data using a restricted set of symbols that can be conveniently used by humans and processed by computers. The high number base results in shorter strings than with the decimal or hexadecimal system.";
+        this.infoURL = "https://wikipedia.org/wiki/List_of_numeral_systems";
+        this.inputType = "string";
+        this.outputType = "byteArray";
+        this.args = [
+            {
+                name: "Alphabet",
+                type: "string",
+                value: "0-9A-Za-z"
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {byteArray}
+     */
+    run(input, args) {
+        if (input.length < 1) return [];
+        const ALPHABET = Utils.expandAlphRange(args[0]).join("");
+        const BN = BigNumber.clone({ ALPHABET });
+
+        const re = new RegExp("[^" + ALPHABET.replace(/[[\]\\\-^$]/g, "\\$&") + "]", "g");
+        input = input.replace(re, "");
+
+        const number = new BN(input, 62);
+
+        return Utils.convertToByteArray(number.toString(16), "Hex");
+    }
+
+}
+
+export default FromBase62;

src/core/operations/FromBase64.mjs

@@ -38,44 +38,44 @@ class FromBase64 extends Operation {
         ];
         this.patterns = [
             {
-                match: "^(?:[A-Z\\d+/]{4})+(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
+                match: "^\\s*(?:[A-Z\\d+/]{4})+(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
                 flags: "i",
-                args: ["A-Za-z0-9+/=", false]
+                args: ["A-Za-z0-9+/=", true]
             },
             {
-                match: "^[A-Z\\d\\-_]{20,}$",
+                match: "^\\s*[A-Z\\d\\-_]{20,}\\s*$",
                 flags: "i",
-                args: ["A-Za-z0-9-_", false]
+                args: ["A-Za-z0-9-_", true]
             },
             {
-                match: "^(?:[A-Z\\d+\\-]{4}){5,}(?:[A-Z\\d+\\-]{2}==|[A-Z\\d+\\-]{3}=)?$",
+                match: "^\\s*(?:[A-Z\\d+\\-]{4}){5,}(?:[A-Z\\d+\\-]{2}==|[A-Z\\d+\\-]{3}=)?\\s*$",
                 flags: "i",
-                args: ["A-Za-z0-9+\\-=", false]
+                args: ["A-Za-z0-9+\\-=", true]
             },
             {
-                match: "^(?:[A-Z\\d./]{4}){5,}(?:[A-Z\\d./]{2}==|[A-Z\\d./]{3}=)?$",
+                match: "^\\s*(?:[A-Z\\d./]{4}){5,}(?:[A-Z\\d./]{2}==|[A-Z\\d./]{3}=)?\\s*$",
                 flags: "i",
-                args: ["./0-9A-Za-z=", false]
+                args: ["./0-9A-Za-z=", true]
             },
             {
-                match: "^[A-Z\\d_.]{20,}$",
+                match: "^\\s*[A-Z\\d_.]{20,}\\s*$",
                 flags: "i",
-                args: ["A-Za-z0-9_.", false]
+                args: ["A-Za-z0-9_.", true]
             },
             {
-                match: "^(?:[A-Z\\d._]{4}){5,}(?:[A-Z\\d._]{2}--|[A-Z\\d._]{3}-)?$",
+                match: "^\\s*(?:[A-Z\\d._]{4}){5,}(?:[A-Z\\d._]{2}--|[A-Z\\d._]{3}-)?\\s*$",
                 flags: "i",
-                args: ["A-Za-z0-9._-", false]
+                args: ["A-Za-z0-9._-", true]
             },
             {
-                match: "^(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
+                match: "^\\s*(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
                 flags: "i",
-                args: ["0-9a-zA-Z+/=", false]
+                args: ["0-9a-zA-Z+/=", true]
             },
             {
-                match: "^(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
+                match: "^\\s*(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
                 flags: "i",
-                args: ["0-9A-Za-z+/=", false]
+                args: ["0-9A-Za-z+/=", true]
             },
             {
                 match: "^[ !\"#$%&'()*+,\\-./\\d:;<=>?@A-Z[\\\\\\]^_]{20,}$",
@@ -83,24 +83,24 @@ class FromBase64 extends Operation {
                 args: [" -_", false]
             },
             {
-                match: "^[A-Z\\d+\\-]{20,}$",
+                match: "^\\s*[A-Z\\d+\\-]{20,}\\s*$",
                 flags: "i",
-                args: ["+\\-0-9A-Za-z", false]
+                args: ["+\\-0-9A-Za-z", true]
             },
             {
-                match: "^[!\"#$%&'()*+,\\-0-689@A-NP-VX-Z[`a-fh-mp-r]{20,}$",
+                match: "^\\s*[!\"#$%&'()*+,\\-0-689@A-NP-VX-Z[`a-fh-mp-r]{20,}\\s*$",
                 flags: "",
-                args: ["!-,-0-689@A-NP-VX-Z[`a-fh-mp-r", false]
+                args: ["!-,-0-689@A-NP-VX-Z[`a-fh-mp-r", true]
             },
             {
-                match: "^(?:[N-ZA-M\\d+/]{4}){5,}(?:[N-ZA-M\\d+/]{2}==|[N-ZA-M\\d+/]{3}=)?$",
+                match: "^\\s*(?:[N-ZA-M\\d+/]{4}){5,}(?:[N-ZA-M\\d+/]{2}==|[N-ZA-M\\d+/]{3}=)?\\s*$",
                 flags: "i",
-                args: ["N-ZA-Mn-za-m0-9+/=", false]
+                args: ["N-ZA-Mn-za-m0-9+/=", true]
             },
             {
-                match: "^[A-Z\\d./]{20,}$",
+                match: "^\\s*[A-Z\\d./]{20,}\\s*$",
                 flags: "i",
-                args: ["./0-9A-Za-z", false]
+                args: ["./0-9A-Za-z", true]
             },
         ];
     }

src/core/operations/FromBase85.mjs

@@ -0,0 +1,105 @@
+/**
+ * @author PenguinGeorge [george@penguingeorge.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import Utils from "../Utils";
+import {alphabetName, ALPHABET_OPTIONS} from "../lib/Base85";
+
+/**
+ * From Base85 operation
+ */
+class FromBase85 extends Operation {
+
+    /**
+     * From Base85 constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "From Base85";
+        this.module = "Default";
+        this.description = "Base85 (also called Ascii85) is a notation for encoding arbitrary byte data. It is usually more efficient that Base64.<br><br>This operation decodes data from an ASCII string (with an alphabet of your choosing, presets included).<br><br>e.g. <code>BOu!rD]j7BEbo7</code> becomes <code>hello world</code><br><br>Base85 is commonly used in Adobe's PostScript and PDF file formats.";
+        this.infoURL = "https://wikipedia.org/wiki/Ascii85";
+        this.inputType = "string";
+        this.outputType = "byteArray";
+        this.args = [
+            {
+                name: "Alphabet",
+                type: "editableOption",
+                value: ALPHABET_OPTIONS
+            },
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {byteArray}
+     */
+    run(input, args) {
+        const alphabet = Utils.expandAlphRange(args[0]).join(""),
+            encoding = alphabetName(alphabet),
+            result = [];
+
+        if (alphabet.length !== 85 ||
+            [].unique.call(alphabet).length !== 85) {
+            throw new OperationError("Alphabet must be of length 85");
+        }
+
+        if (input.length === 0) return [];
+
+        const matches = input.match(/<~(.+?)~>/);
+        if (matches !== null) input = matches[1];
+
+        let i = 0;
+        let block, blockBytes;
+        while (i < input.length) {
+            if (encoding === "Standard" && input[i] === "z") {
+                result.push(0, 0, 0, 0);
+                i++;
+            } else {
+                let digits = [];
+                digits = input
+                    .substr(i, 5)
+                    .split("")
+                    .map((chr, idx) => {
+                        const digit = alphabet.indexOf(chr);
+                        if (digit < 0 || digit > 84) {
+                            throw `Invalid character '${chr}' at index ${idx}`;
+                        }
+                        return digit;
+                    });
+
+                block =
+                    digits[0] * 52200625 +
+                    digits[1] * 614125 +
+                    (i + 2 < input.length ? digits[2] : 84) * 7225 +
+                    (i + 3 < input.length ? digits[3] : 84) * 85 +
+                    (i + 4 < input.length ? digits[4] : 84);
+
+                blockBytes = [
+                    (block >> 24) & 0xff,
+                    (block >> 16) & 0xff,
+                    (block >> 8) & 0xff,
+                    block & 0xff
+                ];
+
+                if (input.length < i + 5) {
+                    blockBytes.splice(input.length - (i + 5), 5);
+                }
+
+                result.push.apply(result, blockBytes);
+                i += 5;
+            }
+        }
+
+        return result;
+    }
+
+}
+
+export default FromBase85;

src/core/operations/FromBinary.mjs

@@ -7,6 +7,7 @@
 import Operation from "../Operation";
 import Utils from "../Utils";
 import {BIN_DELIM_OPTIONS} from "../lib/Delim";
+import {fromBinary} from "../lib/Binary";
 
 /**
  * From Binary operation
@@ -77,15 +78,7 @@ class FromBinary extends Operation {
      * @returns {byteArray}
      */
     run(input, args) {
-        const delimRegex = Utils.regexRep(args[0] || "Space");
-        input = input.replace(delimRegex, "");
-
-        const output = [];
-        const byteLen = 8;
-        for (let i = 0; i < input.length; i += byteLen) {
-            output.push(parseInt(input.substr(i, byteLen), 2));
-        }
-        return output;
+        return fromBinary(input, args[0]);
     }
 
     /**

src/core/operations/FromBraille.mjs

@@ -0,0 +1,70 @@
+/**
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import {BRAILLE_LOOKUP} from "../lib/Braille";
+
+/**
+ * From Braille operation
+ */
+class FromBraille extends Operation {
+
+    /**
+     * FromBraille constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "From Braille";
+        this.module = "Default";
+        this.description = "Converts six-dot braille symbols to text.";
+        this.infoURL = "https://wikipedia.org/wiki/Braille";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        return input.split("").map(b => {
+            const idx = BRAILLE_LOOKUP.dot6.indexOf(b);
+            return idx < 0 ? b : BRAILLE_LOOKUP.ascii[idx];
+        }).join("");
+    }
+
+    /**
+     * Highlight From Braille
+     *
+     * @param {Object[]} pos
+     * @param {number} pos[].start
+     * @param {number} pos[].end
+     * @param {Object[]} args
+     * @returns {Object[]} pos
+     */
+    highlight(pos, args) {
+        return pos;
+    }
+
+    /**
+     * Highlight From Braille in reverse
+     *
+     * @param {Object[]} pos
+     * @param {number} pos[].start
+     * @param {number} pos[].end
+     * @param {Object[]} args
+     * @returns {Object[]} pos
+     */
+    highlightReverse(pos, args) {
+        return pos;
+    }
+
+}
+
+export default FromBraille;

src/core/operations/FromCaseInsensitiveRegex.mjs

@@ -0,0 +1,39 @@
+/**
+ * @author masq [github.cyberchef@masq.cc]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+
+/**
+ * From Case Insensitive Regex operation
+ */
+class FromCaseInsensitiveRegex extends Operation {
+
+    /**
+     * FromCaseInsensitiveRegex constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "From Case Insensitive Regex";
+        this.module = "Default";
+        this.description = "Converts a case-insensitive regex string to a case sensitive regex string (no guarantee on it being the proper original casing) in case the i flag wasn't available at the time but now is, or you need it to be case-sensitive again.<br><br>e.g. <code>[mM][oO][zZ][iI][lL][lL][aA]/[0-9].[0-9] .*</code> becomes <code>Mozilla/[0-9].[0-9] .*</code>";
+        this.infoURL = "https://wikipedia.org/wiki/Regular_expression";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        return input.replace(/\[[a-z]{2}\]/ig, m => m[1].toUpperCase() === m[2].toUpperCase() ? m[1] : m);
+    }
+}
+
+export default FromCaseInsensitiveRegex;

src/core/operations/FromDecimal.mjs

@@ -29,38 +29,43 @@ class FromDecimal extends Operation {
                 "name": "Delimiter",
                 "type": "option",
                 "value": DELIM_OPTIONS
+            },
+            {
+                "name": "Support signed values",
+                "type": "boolean",
+                "value": false
             }
         ];
         this.patterns = [
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?: (?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["Space"]
+                args: ["Space", false]
             },
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:,(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["Comma"]
+                args: ["Comma", false]
             },
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:;(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["Semi-colon"]
+                args: ["Semi-colon", false]
             },
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?::(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["Colon"]
+                args: ["Colon", false]
             },
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["Line feed"]
+                args: ["Line feed", false]
             },
             {
                 match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\r\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
-                args: ["CRLF"]
+                args: ["CRLF", false]
             },
         ];
     }
@@ -71,7 +76,11 @@ class FromDecimal extends Operation {
      * @returns {byteArray}
      */
     run(input, args) {
-        return fromDecimal(input, args[0]);
+        let data = fromDecimal(input, args[0]);
+        if (args[1]) { // Convert negatives
+            data = data.map(v => v < 0 ? 0xFF + v + 1 : v);
+        }
+        return data;
     }
 
 }

src/core/operations/FromMessagePack.mjs

@@ -0,0 +1,47 @@
+/**
+ * @author Matt C [matt@artemisbot.uk]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import notepack from "notepack.io";
+
+/**
+ * From MessagePack operation
+ */
+class FromMessagePack extends Operation {
+
+    /**
+     * FromMessagePack constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "From MessagePack";
+        this.module = "Code";
+        this.description = "Converts MessagePack encoded data to JSON. MessagePack is a computer data interchange format. It is a binary form for representing simple data structures like arrays and associative arrays.";
+        this.infoURL = "https://wikipedia.org/wiki/MessagePack";
+        this.inputType = "ArrayBuffer";
+        this.outputType = "JSON";
+        this.args = [];
+    }
+
+    /**
+     * @param {ArrayBuffer} input
+     * @param {Object[]} args
+     * @returns {JSON}
+     */
+    run(input, args) {
+        try {
+            const buf = Buffer.from(new Uint8Array(input));
+            return notepack.decode(buf);
+        } catch (err) {
+            throw new OperationError(`Could not decode MessagePack to JSON: ${err}`);
+        }
+    }
+
+}
+
+export default FromMessagePack;

src/core/operations/FromQuotedPrintable.mjs

@@ -30,7 +30,7 @@ class FromQuotedPrintable extends Operation {
         this.args = [];
         this.patterns = [
             {
-                match: "^[\\x21-\\x3d\\x3f-\\x7e \\t]*(?:=[\\da-f]{2}|=\\r?\\n)(?:[\\x21-\\x3d\\x3f-\\x7e \\t]|=[\\da-f]{2}|=\\r?\\n)*$",
+                match: "^[\\x21-\\x3d\\x3f-\\x7e \\t]{0,76}(?:=[\\da-f]{2}|=\\r?\\n)(?:[\\x21-\\x3d\\x3f-\\x7e \\t]|=[\\da-f]{2}|=\\r?\\n)*$",
                 flags: "i",
                 args: []
             },

src/core/operations/GenerateAllHashes.mjs

@@ -41,7 +41,7 @@ class GenerateAllHashes extends Operation {
         super();
 
         this.name = "Generate all hashes";
-        this.module = "Hashing";
+        this.module = "Crypto";
         this.description = "Generates all available hashes and checksums for the input.";
         this.infoURL = "https://wikipedia.org/wiki/Comparison_of_cryptographic_hash_functions";
         this.inputType = "ArrayBuffer";

src/core/operations/GenerateLoremIpsum.mjs

@@ -0,0 +1,70 @@
+/**
+ * @author klaxon [klaxon@veyr.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import { GenerateParagraphs, GenerateSentences, GenerateWords, GenerateBytes } from "../lib/LoremIpsum";
+
+/**
+ * Generate Lorem Ipsum operation
+ */
+class GenerateLoremIpsum extends Operation {
+
+    /**
+     * GenerateLoremIpsum constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Generate Lorem Ipsum";
+        this.module = "Default";
+        this.description = "Generate varying length lorem ipsum placeholder text.";
+        this.infoURL = "https://wikipedia.org/wiki/Lorem_ipsum";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [
+            {
+                "name": "Length",
+                "type": "number",
+                "value": "3"
+            },
+            {
+                "name": "Length in",
+                "type": "option",
+                "value": ["Paragraphs", "Sentences", "Words", "Bytes"]
+            }
+
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        const [length, lengthType] = args;
+        if (length < 1){
+            throw new OperationError("Length must be greater than 0");
+        }
+        switch (lengthType) {
+            case "Paragraphs":
+                return GenerateParagraphs(length);
+            case "Sentences":
+                return GenerateSentences(length);
+            case "Words":
+                return GenerateWords(length);
+            case "Bytes":
+                return GenerateBytes(length);
+            default:
+                throw new OperationError("Invalid length type");
+
+        }
+    }
+
+}
+
+export default GenerateLoremIpsum;

src/core/operations/GenerateQRCode.mjs

@@ -0,0 +1,119 @@
+/**
+ * @author j433866 [j433866@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import qr from "qr-image";
+import { toBase64 } from "../lib/Base64";
+import { isImage } from "../lib/FileType";
+import Utils from "../Utils";
+
+/**
+ * Generate QR Code operation
+ */
+class GenerateQRCode extends Operation {
+
+    /**
+     * GenerateQRCode constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Generate QR Code";
+        this.module = "Image";
+        this.description = "Generates a Quick Response (QR) code from the input text.<br><br>A QR code is a type of matrix barcode (or two-dimensional barcode) first designed in 1994 for the automotive industry in Japan. A barcode is a machine-readable optical label that contains information about the item to which it is attached.";
+        this.infoURL = "https://wikipedia.org/wiki/QR_code";
+        this.inputType = "string";
+        this.outputType = "byteArray";
+        this.presentType = "html";
+        this.args = [
+            {
+                "name": "Image Format",
+                "type": "option",
+                "value": ["PNG", "SVG", "EPS", "PDF"]
+            },
+            {
+                "name": "Module size (px)",
+                "type": "number",
+                "value": 5
+            },
+            {
+                "name": "Margin (num modules)",
+                "type": "number",
+                "value": 2
+            },
+            {
+                "name": "Error correction",
+                "type": "option",
+                "value": ["Low", "Medium", "Quartile", "High"],
+                "defaultIndex": 1
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {byteArray}
+     */
+    run(input, args) {
+        const [format, size, margin, errorCorrection] = args;
+
+        // Create new QR image from the input data, and convert it to a buffer
+        const qrImage = qr.imageSync(input, {
+            type: format,
+            size: size,
+            margin: margin,
+            "ec_level": errorCorrection.charAt(0).toUpperCase()
+        });
+
+        if (qrImage == null) {
+            throw new OperationError("Error generating QR code.");
+        }
+
+        switch (format) {
+            case "SVG":
+            case "EPS":
+            case "PDF":
+                return [...Buffer.from(qrImage)];
+            case "PNG":
+                // Return the QR image buffer as a byte array
+                return [...qrImage];
+            default:
+                throw new OperationError("Unsupported QR code format.");
+        }
+    }
+
+    /**
+     * Displays the QR image using HTML for web apps
+     *
+     * @param {byteArray} data
+     * @returns {html}
+     */
+    present(data, args) {
+        if (!data.length) return "";
+
+        const [format] = args;
+
+        if (format === "PNG") {
+            let dataURI = "data:";
+            const mime = isImage(data);
+            if (mime){
+                dataURI += mime + ";";
+            } else {
+                throw new OperationError("Invalid PNG file generated by QR image");
+            }
+            dataURI += "base64," + toBase64(data);
+
+            return `<img src="${dataURI}">`;
+        }
+
+        return Utils.byteArrayToChars(data);
+    }
+
+}
+
+export default GenerateQRCode;

src/core/operations/GroupIPAddresses.mjs

@@ -22,7 +22,7 @@ class GroupIPAddresses extends Operation {
         super();
 
         this.name = "Group IP addresses";
-        this.module = "JSBN";
+        this.module = "Default";
         this.description = "Groups a list of IP addresses into subnets. Supports both IPv4 and IPv6 addresses.";
         this.infoURL = "https://wikipedia.org/wiki/Subnetwork";
         this.inputType = "string";

src/core/operations/HAS160.mjs

@@ -19,7 +19,7 @@ class HAS160 extends Operation {
         super();
 
         this.name = "HAS-160";
-        this.module = "Hashing";
+        this.module = "Crypto";
         this.description = "HAS-160 is a cryptographic hash function designed for use with the Korean KCDSA digital signature algorithm. It is derived from SHA-1, with assorted changes intended to increase its security. It produces a 160-bit output.<br><br>HAS-160 is used in the same way as SHA-1. First it divides input in blocks of 512 bits each and pads the final block. A digest function updates the intermediate hash value by processing the input blocks in turn.<br><br>The message digest algorithm consists of 80 rounds.";
         this.infoURL = "https://wikipedia.org/wiki/HAS-160";
         this.inputType = "ArrayBuffer";

src/core/operations/HMAC.mjs

@@ -20,7 +20,7 @@ class HMAC extends Operation {
         super();
 
         this.name = "HMAC";
-        this.module = "Hashing";
+        this.module = "Crypto";
         this.description = "Keyed-Hash Message Authentication Codes (HMAC) are a mechanism for message authentication using cryptographic hash functions.";
         this.infoURL = "https://wikipedia.org/wiki/HMAC";
         this.inputType = "ArrayBuffer";
@@ -28,8 +28,9 @@ class HMAC extends Operation {
         this.args = [
             {
                 "name": "Key",
-                "type": "binaryString",
-                "value": ""
+                "type": "toggleString",
+                "value": "",
+                "toggleValues": ["Hex", "Decimal", "Base64", "UTF8", "Latin1"]
             },
             {
                 "name": "Hashing function",
@@ -66,19 +67,12 @@ class HMAC extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const key = args[0],
+        const key = Utils.convertToByteString(args[0].string || "", args[0].option),
             hashFunc = args[1].toLowerCase(),
             msg = Utils.arrayBufferToStr(input, false),
             hasher = CryptoApi.getHasher(hashFunc);
 
-        // Horrible shim to fix constructor bug. Reported in nf404/crypto-api#8
-        hasher.reset = () => {
-            hasher.state = {};
-            const tmp = new hasher.constructor();
-            hasher.state = tmp.state;
-        };
-
-        const mac = CryptoApi.getHmac(CryptoApi.encoder.fromUtf(key), hasher);
+        const mac = CryptoApi.getHmac(key, hasher);
         mac.update(msg);
         return CryptoApi.encoder.toHex(mac.finalize());
     }