From 23763363bab5c24338c89f7131ebed7921dc5951 Mon Sep 17 00:00:00 2001 From: Philipp Arnold Date: Fri, 10 Dec 2021 23:22:57 +0100 Subject: [PATCH 001/101] Added a JSON to YAML and a YALM to JSON operation --- src/core/config/Categories.json | 4 ++- src/core/operations/JSONToYAML.mjs | 46 ++++++++++++++++++++++++++++++ src/core/operations/YAMLToJSON.mjs | 45 +++++++++++++++++++++++++++++ tests/operations/index.mjs | 1 + tests/operations/tests/YAML.mjs | 42 +++++++++++++++++++++++++++ 5 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 src/core/operations/JSONToYAML.mjs create mode 100644 src/core/operations/YAMLToJSON.mjs create mode 100644 tests/operations/tests/YAML.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 09ee8d15b..e35bc8519 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -63,7 +63,9 @@ "JSON to CSV", "Avro to JSON", "CBOR Encode", - "CBOR Decode" + "CBOR Decode", + "YAML to JSON", + "JSON to YAML" ] }, { diff --git a/src/core/operations/JSONToYAML.mjs b/src/core/operations/JSONToYAML.mjs new file mode 100644 index 000000000..2c8102e06 --- /dev/null +++ b/src/core/operations/JSONToYAML.mjs @@ -0,0 +1,46 @@ +/** + * @author ccarpo [ccarpo@gmx.net] + * @copyright Crown Copyright 2021 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; +import YAML from "yaml"; + +/** + * JSON to YAML operation + */ +class JSONToYAML extends Operation { + + /** + * JSONToYAML constructor + */ + constructor() { + super(); + + this.name = "JSON to YAML"; + this.module = "Default"; + this.description = "Converts a JSON into a YAML"; + this.infoURL = "https://en.wikipedia.org/wiki/YAML"; + this.inputType = "JSON"; + this.outputType = "string"; + this.args = []; + } + + /** + * @param {JSON} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + try { + return YAML.stringify(input); + } catch (err) { + throw new OperationError("Test"); + } + } + +} + +export default JSONToYAML; diff --git a/src/core/operations/YAMLToJSON.mjs b/src/core/operations/YAMLToJSON.mjs new file mode 100644 index 000000000..4f77f4fde --- /dev/null +++ b/src/core/operations/YAMLToJSON.mjs @@ -0,0 +1,45 @@ +/** + * @author ccarpo [ccarpo@gmx.net] + * @copyright Crown Copyright 2021 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; +import jsYaml from "js-yaml"; +/** + * YAML to JSON operation + */ +class YAMLToJSON extends Operation { + + /** + * YAMLToJSON constructor + */ + constructor() { + super(); + + this.name = "YAML to JSON"; + this.module = "Default"; + this.description = "Converts a YAML to JSON"; + this.infoURL = "https://en.wikipedia.org/wiki/YAML"; + this.inputType = "string"; + this.outputType = "JSON"; + this.args = []; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {JSON} + */ + run(input, args) { + try { + return jsYaml.load(input); + } catch (err) { + throw new OperationError("Unable to parse YAML: " + err); + } + } + +} + +export default YAMLToJSON; diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index 9add20b9b..3f7a7457f 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -107,6 +107,7 @@ import "./tests/CBORDecode.mjs"; import "./tests/JA3Fingerprint.mjs"; import "./tests/JA3SFingerprint.mjs"; import "./tests/HASSH.mjs"; +import "./tests/YAML.mjs"; // Cannot test operations that use the File type yet diff --git a/tests/operations/tests/YAML.mjs b/tests/operations/tests/YAML.mjs new file mode 100644 index 000000000..ba72ed436 --- /dev/null +++ b/tests/operations/tests/YAML.mjs @@ -0,0 +1,42 @@ + +/** + * YAML tests. + * + * @author ccarpo [ccarpo@gmx.net] + * + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ +import TestRegister from "../../lib/TestRegister.mjs"; + +const EXAMPLE_YAML = `number: 3\nplain: string\nblock: |\n two\n lines`; +const EXAMPLE_JSON = `{ "number": 3, "plain": "string" }`; + +TestRegister.addTests([ + { + name: "YAML to JSON", + input: EXAMPLE_YAML, + expectedOutput: JSON.stringify({ + "number": 3, + "plain": "string", + "block": "two\nlines\n" + }, null, 4), + recipeConfig: [ + { + op: "YAML to JSON", + args: [], + } + ], + }, + { + name: "JSON to YAML", + input: EXAMPLE_JSON, + expectedOutput: `number: 3\nplain: string\n`, + recipeConfig: [ + { + op: "JSON to YAML", + args: [], + } + ], + }, +]); From 3e8c5d945cd65e920930ddd6dfb012123cd6a0a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Mon, 27 Feb 2023 06:09:30 +0000 Subject: [PATCH 002/101] Add "XOR Checksum" operation --- src/core/config/Categories.json | 3 +- src/core/operations/XORChecksum.mjs | 59 +++++++++++++++++++ tests/operations/tests/Checksum.mjs | 90 ++++++++++++++++++++++++++++- 3 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 src/core/operations/XORChecksum.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 075e8d666..97b20bd76 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -390,7 +390,8 @@ "CRC-8 Checksum", "CRC-16 Checksum", "CRC-32 Checksum", - "TCP/IP Checksum" + "TCP/IP Checksum", + "XOR Checksum" ] }, { diff --git a/src/core/operations/XORChecksum.mjs b/src/core/operations/XORChecksum.mjs new file mode 100644 index 000000000..1603a2651 --- /dev/null +++ b/src/core/operations/XORChecksum.mjs @@ -0,0 +1,59 @@ +/** + * @author Thomas Weißschuh [thomas@t-8ch.de] + * @copyright Crown Copyright 2023 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import Utils from "../Utils.mjs"; +import { toHex } from "../lib/Hex.mjs"; + +/** + * XOR Checksum operation + */ +class XORChecksum extends Operation { + + /** + * XORChecksum constructor + */ + constructor() { + super(); + + this.name = "XOR Checksum"; + this.module = "Crypto"; + this.description = "XOR Checksum splits the input into blocks of a configurable size and performs the XOR operation on these blocks."; + this.infoURL = "https://wikipedia.org/wiki/XOR"; + this.inputType = "ArrayBuffer"; + this.outputType = "string"; + this.args = [ + { + name: "Blocksize", + type: "number", + value: 4 + }, + ]; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + const blocksize = args[0]; + input = new Uint8Array(input); + + const res = Array(blocksize); + res.fill(0); + + for (const chunk of Utils.chunked(input, blocksize)) { + for (let i = 0; i < blocksize; i++) { + res[i] ^= chunk[i]; + } + } + + return toHex(res, ""); + } +} + +export default XORChecksum; diff --git a/tests/operations/tests/Checksum.mjs b/tests/operations/tests/Checksum.mjs index 142ee2678..5266ab992 100644 --- a/tests/operations/tests/Checksum.mjs +++ b/tests/operations/tests/Checksum.mjs @@ -237,5 +237,93 @@ TestRegister.addTests([ "args": [] } ] - } + }, + { + name: "XOR Checksum (1): nothing", + input: "", + expectedOutput: "00", + recipeConfig: [ + { + "op": "XOR Checksum", + "args": [1] + } + ] + }, + { + name: "XOR Checksum (1): basic string", + input: BASIC_STRING, + expectedOutput: "08", + recipeConfig: [ + { + "op": "XOR Checksum", + "args": [1] + } + ] + }, + { + name: "XOR Checksum (1): UTF-8", + input: UTF8_STR, + expectedOutput: "df", + recipeConfig: [ + { + "op": "XOR Checksum", + "args": [1] + } + ] + }, + { + name: "XOR Checksum (1): all bytes", + input: ALL_BYTES, + expectedOutput: "00", + recipeConfig: [ + { + "op": "XOR Checksum", + "args": [1] + } + ] + }, + { + name: "XOR Checksum (4): nothing", + input: "", + expectedOutput: "00000000", + recipeConfig: [ + { + "op": "XOR Checksum", + "args": [4] + } + ] + }, + { + name: "XOR Checksum (4): basic string", + input: BASIC_STRING, + expectedOutput: "4918421b", + recipeConfig: [ + { + "op": "XOR Checksum", + "args": [4] + } + ] + }, + { + name: "XOR Checksum (4): UTF-8", + input: UTF8_STR, + expectedOutput: "83a424dc", + recipeConfig: [ + { + "op": "XOR Checksum", + "args": [4] + } + ] + }, + { + name: "XOR Checksum (4): all bytes", + input: ALL_BYTES, + expectedOutput: "00000000", + recipeConfig: [ + { + "op": "XOR Checksum", + "args": [4] + } + ] + }, ]); From 73a39fa3ceba9f04bda0d7c96d6993732700add3 Mon Sep 17 00:00:00 2001 From: starplanet Date: Wed, 5 Apr 2023 18:25:37 +0800 Subject: [PATCH 003/101] fix: fix ToDecimal signed logic --- src/core/operations/ToDecimal.mjs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/operations/ToDecimal.mjs b/src/core/operations/ToDecimal.mjs index 65798a7cb..50db41f68 100644 --- a/src/core/operations/ToDecimal.mjs +++ b/src/core/operations/ToDecimal.mjs @@ -45,11 +45,12 @@ class ToDecimal extends Operation { * @returns {string} */ run(input, args) { - input = new Uint8Array(input); const delim = Utils.charRep(args[0]), signed = args[1]; if (signed) { - input = input.map(v => v > 0x7F ? v - 0xFF - 1 : v); + input = new Int8Array(input); + } else { + input = new Uint8Array(input); } return input.join(delim); } From bf1e708a4cd465b35df6bede05a75f4b81fe1394 Mon Sep 17 00:00:00 2001 From: Brunon Blok <43315279+brun0ne@users.noreply.github.com> Date: Thu, 6 Apr 2023 20:32:50 +0000 Subject: [PATCH 004/101] added PHP Serialize operation --- src/core/operations/PHPSerialize.mjs | 144 ++++++++++++++++++++++++ tests/operations/tests/PHPSerialize.mjs | 112 ++++++++++++++++++ 2 files changed, 256 insertions(+) create mode 100644 src/core/operations/PHPSerialize.mjs create mode 100644 tests/operations/tests/PHPSerialize.mjs diff --git a/src/core/operations/PHPSerialize.mjs b/src/core/operations/PHPSerialize.mjs new file mode 100644 index 000000000..94f42cbb4 --- /dev/null +++ b/src/core/operations/PHPSerialize.mjs @@ -0,0 +1,144 @@ +/** + * @author brun0ne [brunonblok@gmail.com] + * @copyright Crown Copyright 2023 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; + +/** + * PHP Serialize operation + */ +class PHPSerialize extends Operation { + + /** + * PHPSerialize constructor + */ + constructor() { + super(); + + this.name = "PHP Serialize"; + this.module = "Default"; + this.description = "Performs PHP serialization on JSON data.

This function does not support object tags.

Since PHP doesn't distinguish dicts and arrays, this operation is not always symmetric to PHP Deserialize.

Example:
[5,"abc",true]
becomes
a:3:{i:0;i:5;i:1;s:3:"abc";i:2;b:1;}"; + this.infoURL = "https://www.phpinternalsbook.com/php5/classes_objects/serialization.html"; + this.inputType = "JSON"; + this.outputType = "string"; + this.args = []; + } + + /** + * @param {JSON} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + /** + * Determines if a number is an integer + * @param {number} value + * @returns {boolean} + */ + function isInteger(value) { + return typeof value === "number" && parseInt(value.toString(), 10) === value; + } + + /** + * Serialize basic types + * @param {string | number | boolean} content + * @returns {string} + */ + function serializeBasicTypes(content) { + const basicTypes = { + "string": "s", + "integer": "i", + "float": "d", + "boolean": "b" + }; + + /** + * Booleans + * cast to 0 or 1 + */ + if (typeof content === "boolean"){ + return `${basicTypes["boolean"]}:${content ? 1 : 0}`; + } + + /** + * Numbers + */ + if (typeof content === "number"){ + if (isInteger(content)){ + return `${basicTypes["integer"]}:${content.toString()}` + } + else { + return `${basicTypes["float"]}:${content.toString()}` + } + } + + /** + * Strings + */ + if (typeof content === "string") + return `${basicTypes["string"]}:${content.length}:"${content}"`; + + /** This should be unreachable */ + throw new OperationError(`Encountered a non-implemented type: ${typeof content}`); + } + + /** + * Recursively serialize + * @param {*} object + * @returns {string} + */ + function serialize(object) { + /** + * Null + */ + if (object == null) { + return `N;` + } + + /** + * Basic types + */ + if (typeof object !== "object"){ + return `${serializeBasicTypes(object)};`; + } + + /** + * Arrays + */ + else if (object instanceof Array) { + const serializedElements = []; + + for (let i = 0; i < object.length; i++) { + serializedElements.push(`${serialize(i)}${serialize(object[i])}`); + } + + return `a:${object.length}:{${serializedElements.join("")}}` + } + + /** + * Objects + * Note: the output cannot be guaranteed to be in the same order as the input + */ + else if (object instanceof Object) { + const serializedElements = []; + const keys = Object.keys(object); + + for (const key of keys) { + serializedElements.push(`${serialize(key)}${serialize(object[key])}`); + } + + return `a:${keys.length}:{${serializedElements.join("")}}` + } + + /** This should be unreachable */ + throw new OperationError(`Encountered a non-implemented type: ${typeof object}`); + } + + return serialize(input); + } +} + +export default PHPSerialize; diff --git a/tests/operations/tests/PHPSerialize.mjs b/tests/operations/tests/PHPSerialize.mjs new file mode 100644 index 000000000..fa6e87c53 --- /dev/null +++ b/tests/operations/tests/PHPSerialize.mjs @@ -0,0 +1,112 @@ +/** + * PHP Serialization tests. + * + * @author brun0ne [brunonblok@gmail.com] + * + * @copyright Crown Copyright 2023 + * @license Apache-2.0 + */ + +import TestRegister from "../../lib/TestRegister.mjs"; + +TestRegister.addTests([ + { + name: "PHP Serialize empty array", + input: "[]", + expectedOutput: "a:0:{}", + recipeConfig: [ + { + op: "PHP Serialize", + args: [] + } + ] + }, + { + name: "PHP Serialize empty object", + input: "{}", + expectedOutput: "a:0:{}", + recipeConfig: [ + { + op: "PHP Serialize", + args: [] + } + ] + }, + { + name: "PHP Serialize null", + input: "null", + expectedOutput: "N;", + recipeConfig: [ + { + op: "PHP Serialize", + args: [] + } + ] + }, + { + name: "PHP Serialize integer", + input: "10", + expectedOutput: "i:10;", + recipeConfig: [ + { + op: "PHP Serialize", + args: [] + } + ] + }, + { + name: "PHP Serialize float", + input: "14.523", + expectedOutput: "d:14.523;", + recipeConfig: [ + { + op: "PHP Serialize", + args: [] + } + ] + }, + { + name: "PHP Serialize boolean", + input: "[true, false]", + expectedOutput: "a:2:{i:0;b:1;i:1;b:0;}", + recipeConfig: [ + { + op: "PHP Serialize", + args: [] + } + ] + }, + { + name: "PHP Serialize string", + input: "\"Test string to serialize\"", + expectedOutput: "s:24:\"Test string to serialize\";", + recipeConfig: [ + { + op: "PHP Serialize", + args: [] + } + ] + }, + { + name: "PHP Serialize object", + input: "{\"a\": 10,\"0\": {\"ab\": true}}", + expectedOutput: "a:2:{s:1:\"0\";a:1:{s:2:\"ab\";b:1;}s:1:\"a\";i:10;}", + recipeConfig: [ + { + op: "PHP Serialize", + args: [] + } + ] + }, + { + name: "PHP Serialize array", + input: "[1,\"abc\",true,{\"x\":1,\"y\":2}]", + expectedOutput: "a:4:{i:0;i:1;i:1;s:3:\"abc\";i:2;b:1;i:3;a:2:{s:1:\"x\";i:1;s:1:\"y\";i:2;}}", + recipeConfig: [ + { + op: "PHP Serialize", + args: [] + } + ] + } +]); From 15b426ebb6e18414259f000640f5287c431b5a25 Mon Sep 17 00:00:00 2001 From: Brunon Blok <43315279+brun0ne@users.noreply.github.com> Date: Thu, 6 Apr 2023 21:03:46 +0000 Subject: [PATCH 005/101] clean up code formatting and fix missing entries --- src/core/config/Categories.json | 1 + src/core/operations/PHPSerialize.mjs | 74 +++++++++++++--------------- tests/operations/index.mjs | 1 + 3 files changed, 35 insertions(+), 41 deletions(-) diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index ce2f01f5e..c95637eff 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -418,6 +418,7 @@ "JPath expression", "CSS selector", "PHP Deserialize", + "PHP Serialize", "Microsoft Script Decoder", "Strip HTML tags", "Diff", diff --git a/src/core/operations/PHPSerialize.mjs b/src/core/operations/PHPSerialize.mjs index 94f42cbb4..221af3274 100644 --- a/src/core/operations/PHPSerialize.mjs +++ b/src/core/operations/PHPSerialize.mjs @@ -35,7 +35,7 @@ class PHPSerialize extends Operation { run(input, args) { /** * Determines if a number is an integer - * @param {number} value + * @param {number} value * @returns {boolean} */ function isInteger(value) { @@ -44,7 +44,7 @@ class PHPSerialize extends Operation { /** * Serialize basic types - * @param {string | number | boolean} content + * @param {string | number | boolean} content * @returns {string} */ function serializeBasicTypes(content) { @@ -54,40 +54,36 @@ class PHPSerialize extends Operation { "float": "d", "boolean": "b" }; - /** * Booleans * cast to 0 or 1 */ - if (typeof content === "boolean"){ - return `${basicTypes["boolean"]}:${content ? 1 : 0}`; + if (typeof content === "boolean") { + return `${basicTypes.boolean}:${content ? 1 : 0}`; } - /** * Numbers */ - if (typeof content === "number"){ - if (isInteger(content)){ - return `${basicTypes["integer"]}:${content.toString()}` - } - else { - return `${basicTypes["float"]}:${content.toString()}` + if (typeof content === "number") { + if (isInteger(content)) { + return `${basicTypes.integer}:${content.toString()}`; + } else { + return `${basicTypes.float}:${content.toString()}`; } } - /** * Strings */ if (typeof content === "string") - return `${basicTypes["string"]}:${content.length}:"${content}"`; - + return `${basicTypes.string}:${content.length}:"${content}"`; + /** This should be unreachable */ throw new OperationError(`Encountered a non-implemented type: ${typeof content}`); } /** * Recursively serialize - * @param {*} object + * @param {*} object * @returns {string} */ function serialize(object) { @@ -95,42 +91,38 @@ class PHPSerialize extends Operation { * Null */ if (object == null) { - return `N;` + return `N;`; } - - /** - * Basic types - */ - if (typeof object !== "object"){ + + if (typeof object !== "object") { + /** + * Basic types + */ return `${serializeBasicTypes(object)};`; - } - - /** - * Arrays - */ - else if (object instanceof Array) { + } else if (object instanceof Array) { + /** + * Arrays + */ const serializedElements = []; - + for (let i = 0; i < object.length; i++) { serializedElements.push(`${serialize(i)}${serialize(object[i])}`); } - - return `a:${object.length}:{${serializedElements.join("")}}` - } - - /** - * Objects - * Note: the output cannot be guaranteed to be in the same order as the input - */ - else if (object instanceof Object) { + + return `a:${object.length}:{${serializedElements.join("")}}`; + } else if (object instanceof Object) { + /** + * Objects + * Note: the output cannot be guaranteed to be in the same order as the input + */ const serializedElements = []; const keys = Object.keys(object); - + for (const key of keys) { serializedElements.push(`${serialize(key)}${serialize(object[key])}`); } - - return `a:${keys.length}:{${serializedElements.join("")}}` + + return `a:${keys.length}:{${serializedElements.join("")}}`; } /** This should be unreachable */ diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index 56f432e09..0bf8fd3bf 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -70,6 +70,7 @@ import "./tests/NormaliseUnicode.mjs"; import "./tests/OTP.mjs"; import "./tests/PGP.mjs"; import "./tests/PHP.mjs"; +import "./tests/PHPSerialize.mjs"; import "./tests/ParseIPRange.mjs"; import "./tests/ParseQRCode.mjs"; import "./tests/PEMtoHex.mjs"; From c0e84dcd501d12d0b322b2595f2c2424d6489b0e Mon Sep 17 00:00:00 2001 From: Brunon Blok <43315279+brun0ne@users.noreply.github.com> Date: Sun, 9 Apr 2023 19:06:59 +0000 Subject: [PATCH 006/101] change comments --- src/core/operations/PHPSerialize.mjs | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/src/core/operations/PHPSerialize.mjs b/src/core/operations/PHPSerialize.mjs index 221af3274..00fb1380b 100644 --- a/src/core/operations/PHPSerialize.mjs +++ b/src/core/operations/PHPSerialize.mjs @@ -61,9 +61,7 @@ class PHPSerialize extends Operation { if (typeof content === "boolean") { return `${basicTypes.boolean}:${content ? 1 : 0}`; } - /** - * Numbers - */ + /* Numbers */ if (typeof content === "number") { if (isInteger(content)) { return `${basicTypes.integer}:${content.toString()}`; @@ -71,9 +69,7 @@ class PHPSerialize extends Operation { return `${basicTypes.float}:${content.toString()}`; } } - /** - * Strings - */ + /* Strings */ if (typeof content === "string") return `${basicTypes.string}:${content.length}:"${content}"`; @@ -87,22 +83,16 @@ class PHPSerialize extends Operation { * @returns {string} */ function serialize(object) { - /** - * Null - */ + /* Null */ if (object == null) { return `N;`; } if (typeof object !== "object") { - /** - * Basic types - */ + /* Basic types */ return `${serializeBasicTypes(object)};`; } else if (object instanceof Array) { - /** - * Arrays - */ + /* Arrays */ const serializedElements = []; for (let i = 0; i < object.length; i++) { From d2da11c79a59412b3ae480a5c2e05ff4421bd113 Mon Sep 17 00:00:00 2001 From: zhzy0077 Date: Tue, 25 Jul 2023 14:42:43 +0800 Subject: [PATCH 007/101] Support jq. --- package-lock.json | 11 +++++++ package.json | 1 + src/core/config/Categories.json | 1 + src/core/operations/Jq.mjs | 57 +++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+) create mode 100644 src/core/operations/Jq.mjs diff --git a/package-lock.json b/package-lock.json index 3cccfa6ca..a53ba37ed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -45,6 +45,7 @@ "geodesy": "1.1.3", "highlight.js": "^11.7.0", "jimp": "^0.16.13", + "jq-web": "^0.5.1", "jquery": "3.6.4", "js-crc": "^0.2.0", "js-sha3": "^0.8.0", @@ -8662,6 +8663,11 @@ "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==" }, + "node_modules/jq-web": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/jq-web/-/jq-web-0.5.1.tgz", + "integrity": "sha512-3Fa3E6g3U1O1j46ljy0EM10yRr4txzILga8J7bqOG8F89gZ6Lilz82WG9z6TItWpYEO0YGa4W8yFGj+NMM1xqQ==" + }, "node_modules/jquery": { "version": "3.6.4", "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.6.4.tgz", @@ -19681,6 +19687,11 @@ "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==" }, + "jq-web": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/jq-web/-/jq-web-0.5.1.tgz", + "integrity": "sha512-3Fa3E6g3U1O1j46ljy0EM10yRr4txzILga8J7bqOG8F89gZ6Lilz82WG9z6TItWpYEO0YGa4W8yFGj+NMM1xqQ==" + }, "jquery": { "version": "3.6.4", "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.6.4.tgz", diff --git a/package.json b/package.json index 45328dd19..7bb14e9b4 100644 --- a/package.json +++ b/package.json @@ -127,6 +127,7 @@ "geodesy": "1.1.3", "highlight.js": "^11.7.0", "jimp": "^0.16.13", + "jq-web": "^0.5.1", "jquery": "3.6.4", "js-crc": "^0.2.0", "js-sha3": "^0.8.0", diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index cf4d91be0..3cf600fdf 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -422,6 +422,7 @@ "CSS Minify", "XPath expression", "JPath expression", + "Jq", "CSS selector", "PHP Deserialize", "Microsoft Script Decoder", diff --git a/src/core/operations/Jq.mjs b/src/core/operations/Jq.mjs new file mode 100644 index 000000000..38aa5abb0 --- /dev/null +++ b/src/core/operations/Jq.mjs @@ -0,0 +1,57 @@ +/** + * @author zhzy0077 [zhzy0077@hotmail.com] + * @copyright Crown Copyright 2023 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; +import jq from "jq-web"; + +/** + * jq operation + */ +class Jq extends Operation { + + /** + * Jq constructor + */ + constructor() { + super(); + + this.name = "jq"; + this.module = "Code"; + this.description = "jq is a lightweight and flexible command-line JSON processor."; + this.infoURL = "https://github.com/jqlang/jq"; + this.inputType = "JSON"; + this.outputType = "string"; + this.args = [ + { + name: "Query", + type: "string", + value: "" + } + ]; + } + + /** + * @param {JSON} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + const [query] = args; + let result; + + try { + result = jq.json(input, query); + } catch (err) { + throw new OperationError(`Invalid jq expression: ${err.message}`); + } + + return JSON.stringify(result); + } + +} + +export default Jq; From 721061d0548a4b9cc9e1d48aa9043ce5c38e8ef2 Mon Sep 17 00:00:00 2001 From: zhzy0077 Date: Tue, 25 Jul 2023 14:48:03 +0800 Subject: [PATCH 008/101] fix typo. --- src/core/config/Categories.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 3cf600fdf..9283fe65d 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -422,7 +422,7 @@ "CSS Minify", "XPath expression", "JPath expression", - "Jq", + "jq", "CSS selector", "PHP Deserialize", "Microsoft Script Decoder", From 7353315baaedb2113cda178d03a88cafc7f02a4e Mon Sep 17 00:00:00 2001 From: ccarpo Date: Tue, 21 Nov 2023 08:56:20 +0100 Subject: [PATCH 009/101] Renamed JSON to Yaml to Beautify JSON --- src/core/config/Categories.json | 2 +- .../operations/{JSONToYAML.mjs => BeautifyYAML.mjs} | 12 ++++++------ src/core/operations/YAMLToJSON.mjs | 2 +- tests/operations/index.mjs | 2 +- .../operations/tests/{YAML.mjs => BeautifyYAML.mjs} | 6 +++--- 5 files changed, 12 insertions(+), 12 deletions(-) rename src/core/operations/{JSONToYAML.mjs => BeautifyYAML.mjs} (76%) rename tests/operations/tests/{YAML.mjs => BeautifyYAML.mjs} (89%) diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index e35bc8519..ce1a639be 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -65,7 +65,7 @@ "CBOR Encode", "CBOR Decode", "YAML to JSON", - "JSON to YAML" + "Beatufiy YAML" ] }, { diff --git a/src/core/operations/JSONToYAML.mjs b/src/core/operations/BeautifyYAML.mjs similarity index 76% rename from src/core/operations/JSONToYAML.mjs rename to src/core/operations/BeautifyYAML.mjs index 2c8102e06..b87573d8f 100644 --- a/src/core/operations/JSONToYAML.mjs +++ b/src/core/operations/BeautifyYAML.mjs @@ -9,19 +9,19 @@ import OperationError from "../errors/OperationError.mjs"; import YAML from "yaml"; /** - * JSON to YAML operation + * Beautify YAML operation */ -class JSONToYAML extends Operation { +class BeautifyYAML extends Operation { /** - * JSONToYAML constructor + * BeautifyYAML constructor */ constructor() { super(); - this.name = "JSON to YAML"; + this.name = "Beautify YAML"; this.module = "Default"; - this.description = "Converts a JSON into a YAML"; + this.description = "Format a JSON object into YAML"; this.infoURL = "https://en.wikipedia.org/wiki/YAML"; this.inputType = "JSON"; this.outputType = "string"; @@ -43,4 +43,4 @@ class JSONToYAML extends Operation { } -export default JSONToYAML; +export default BeautifyYAML; diff --git a/src/core/operations/YAMLToJSON.mjs b/src/core/operations/YAMLToJSON.mjs index 4f77f4fde..5b9865752 100644 --- a/src/core/operations/YAMLToJSON.mjs +++ b/src/core/operations/YAMLToJSON.mjs @@ -20,7 +20,7 @@ class YAMLToJSON extends Operation { this.name = "YAML to JSON"; this.module = "Default"; - this.description = "Converts a YAML to JSON"; + this.description = "Convert YAML to JSON"; this.infoURL = "https://en.wikipedia.org/wiki/YAML"; this.inputType = "string"; this.outputType = "JSON"; diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index 3f7a7457f..475d6d310 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -107,7 +107,7 @@ import "./tests/CBORDecode.mjs"; import "./tests/JA3Fingerprint.mjs"; import "./tests/JA3SFingerprint.mjs"; import "./tests/HASSH.mjs"; -import "./tests/YAML.mjs"; +import "./tests/BeautifyYAML.mjs"; // Cannot test operations that use the File type yet diff --git a/tests/operations/tests/YAML.mjs b/tests/operations/tests/BeautifyYAML.mjs similarity index 89% rename from tests/operations/tests/YAML.mjs rename to tests/operations/tests/BeautifyYAML.mjs index ba72ed436..f2cc58fbe 100644 --- a/tests/operations/tests/YAML.mjs +++ b/tests/operations/tests/BeautifyYAML.mjs @@ -4,7 +4,7 @@ * * @author ccarpo [ccarpo@gmx.net] * - * @copyright Crown Copyright 2018 + * @copyright Crown Copyright 2021 * @license Apache-2.0 */ import TestRegister from "../../lib/TestRegister.mjs"; @@ -29,12 +29,12 @@ TestRegister.addTests([ ], }, { - name: "JSON to YAML", + name: "Beautify YAML", input: EXAMPLE_JSON, expectedOutput: `number: 3\nplain: string\n`, recipeConfig: [ { - op: "JSON to YAML", + op: "Beautify YAML", args: [], } ], From 4255d8d543226969381adc1c6a06f421e76ac6d8 Mon Sep 17 00:00:00 2001 From: ccarpo Date: Tue, 21 Nov 2023 09:19:58 +0100 Subject: [PATCH 010/101] Renamed JSON to Yaml to Beautify JSON --- src/core/config/Categories.json | 2 +- .../operations/{JSONToYAML.mjs => BeautifyYAML.mjs} | 12 ++++++------ src/core/operations/YAMLToJSON.mjs | 2 +- tests/operations/index.mjs | 2 +- .../operations/tests/{YAML.mjs => BeautifyYAML.mjs} | 6 +++--- 5 files changed, 12 insertions(+), 12 deletions(-) rename src/core/operations/{JSONToYAML.mjs => BeautifyYAML.mjs} (76%) rename tests/operations/tests/{YAML.mjs => BeautifyYAML.mjs} (89%) diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index e35bc8519..6e5e7f040 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -65,7 +65,7 @@ "CBOR Encode", "CBOR Decode", "YAML to JSON", - "JSON to YAML" + "Beautify YAML" ] }, { diff --git a/src/core/operations/JSONToYAML.mjs b/src/core/operations/BeautifyYAML.mjs similarity index 76% rename from src/core/operations/JSONToYAML.mjs rename to src/core/operations/BeautifyYAML.mjs index 2c8102e06..b87573d8f 100644 --- a/src/core/operations/JSONToYAML.mjs +++ b/src/core/operations/BeautifyYAML.mjs @@ -9,19 +9,19 @@ import OperationError from "../errors/OperationError.mjs"; import YAML from "yaml"; /** - * JSON to YAML operation + * Beautify YAML operation */ -class JSONToYAML extends Operation { +class BeautifyYAML extends Operation { /** - * JSONToYAML constructor + * BeautifyYAML constructor */ constructor() { super(); - this.name = "JSON to YAML"; + this.name = "Beautify YAML"; this.module = "Default"; - this.description = "Converts a JSON into a YAML"; + this.description = "Format a JSON object into YAML"; this.infoURL = "https://en.wikipedia.org/wiki/YAML"; this.inputType = "JSON"; this.outputType = "string"; @@ -43,4 +43,4 @@ class JSONToYAML extends Operation { } -export default JSONToYAML; +export default BeautifyYAML; diff --git a/src/core/operations/YAMLToJSON.mjs b/src/core/operations/YAMLToJSON.mjs index 4f77f4fde..5b9865752 100644 --- a/src/core/operations/YAMLToJSON.mjs +++ b/src/core/operations/YAMLToJSON.mjs @@ -20,7 +20,7 @@ class YAMLToJSON extends Operation { this.name = "YAML to JSON"; this.module = "Default"; - this.description = "Converts a YAML to JSON"; + this.description = "Convert YAML to JSON"; this.infoURL = "https://en.wikipedia.org/wiki/YAML"; this.inputType = "string"; this.outputType = "JSON"; diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index 3f7a7457f..475d6d310 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -107,7 +107,7 @@ import "./tests/CBORDecode.mjs"; import "./tests/JA3Fingerprint.mjs"; import "./tests/JA3SFingerprint.mjs"; import "./tests/HASSH.mjs"; -import "./tests/YAML.mjs"; +import "./tests/BeautifyYAML.mjs"; // Cannot test operations that use the File type yet diff --git a/tests/operations/tests/YAML.mjs b/tests/operations/tests/BeautifyYAML.mjs similarity index 89% rename from tests/operations/tests/YAML.mjs rename to tests/operations/tests/BeautifyYAML.mjs index ba72ed436..f2cc58fbe 100644 --- a/tests/operations/tests/YAML.mjs +++ b/tests/operations/tests/BeautifyYAML.mjs @@ -4,7 +4,7 @@ * * @author ccarpo [ccarpo@gmx.net] * - * @copyright Crown Copyright 2018 + * @copyright Crown Copyright 2021 * @license Apache-2.0 */ import TestRegister from "../../lib/TestRegister.mjs"; @@ -29,12 +29,12 @@ TestRegister.addTests([ ], }, { - name: "JSON to YAML", + name: "Beautify YAML", input: EXAMPLE_JSON, expectedOutput: `number: 3\nplain: string\n`, recipeConfig: [ { - op: "JSON to YAML", + op: "Beautify YAML", args: [], } ], From cc21fe18ca450e05a8e71c1d1c0b58c460127d94 Mon Sep 17 00:00:00 2001 From: ccarpo Date: Tue, 21 Nov 2023 09:24:54 +0100 Subject: [PATCH 011/101] fixed typo. --- src/core/config/Categories.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index ce1a639be..6e5e7f040 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -65,7 +65,7 @@ "CBOR Encode", "CBOR Decode", "YAML to JSON", - "Beatufiy YAML" + "Beautify YAML" ] }, { From fb3c36af85a528ee2728a051b931c3bcba6d1ecb Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Wed, 24 Jul 2024 14:23:40 +0200 Subject: [PATCH 012/101] Add fingerprints to 'Parse X.509 certificate' operation --- src/core/operations/ParseX509Certificate.mjs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/core/operations/ParseX509Certificate.mjs b/src/core/operations/ParseX509Certificate.mjs index 11e63424d..cdd1e9c7c 100644 --- a/src/core/operations/ParseX509Certificate.mjs +++ b/src/core/operations/ParseX509Certificate.mjs @@ -6,7 +6,8 @@ import r from "jsrsasign"; import { fromBase64 } from "../lib/Base64.mjs"; -import { toHex } from "../lib/Hex.mjs"; +import { runHash } from "../lib/Hash.mjs"; +import { fromHex, toHex } from "../lib/Hex.mjs"; import { formatByteStr, formatDnObj } from "../lib/PublicKey.mjs"; import Operation from "../Operation.mjs"; import Utils from "../Utils.mjs"; @@ -81,7 +82,8 @@ class ParseX509Certificate extends Operation { } if (undefinedInputFormat) throw "Undefined input format"; - const sn = cert.getSerialNumberHex(), + const hex = Utils.strToArrayBuffer(Utils.byteArrayToChars(fromHex(cert.hex))), + sn = cert.getSerialNumberHex(), issuer = cert.getIssuer(), subject = cert.getSubject(), pk = cert.getPublicKey(), @@ -191,6 +193,10 @@ Issuer ${issuerStr} Subject ${subjectStr} +Fingerprints + MD5: ${runHash("md5", hex)} + SHA1: ${runHash("sha1", hex)} + SHA256: ${runHash("sha256", hex)} Public Key ${pkStr.slice(0, -1)} Certificate Signature From 77c2b7923dfa7d4acd2323ea752f4686a5f6d69c Mon Sep 17 00:00:00 2001 From: remingtr Date: Fri, 9 Aug 2024 16:20:38 -0700 Subject: [PATCH 013/101] Add more clear build instructions Added more clear build instructions to the README. --- README.md | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5549bda2a..7add97261 100755 --- a/README.md +++ b/README.md @@ -20,21 +20,36 @@ Cryptographic operations in CyberChef should not be relied upon to provide secur [A live demo can be found here][1] - have fun! -## Containers +## Running Locally with Docker -If you would like to try out CyberChef locally you can either build it yourself: +**Prerequisites:** +- [Docker](hhttps://www.docker.com/products/docker-desktop/) + - Docker Desktop must be open and running on your machine + + +#### Option 1: Build the Docker Image Yourself + +1. Build the docker image ```bash docker build --tag cyberchef --ulimit nofile=10000 . +``` +2. Run the docker container +```bash docker run -it -p 8080:80 cyberchef ``` +3. Navigate to `http://localhost:8080` in your browser -Or you can use our image directly: +#### Option 2: Use the pre-built Docker Image + +If you prefer to skip the build process, you can use the pre-built image ```bash docker run -it -p 8080:80 ghcr.io/gchq/cyberchef:latest ``` +Just like before, navigate to `http://localhost:8080` in your browser. + This image is built and published through our [GitHub Workflows](.github/workflows/releases.yml) ## How it works From d58e162bfafc6cb5b9dc1bbfb57ab29dd6162e4e Mon Sep 17 00:00:00 2001 From: Niall Date: Fri, 23 Aug 2024 19:37:00 +0000 Subject: [PATCH 014/101] Fixes an issue with ShowOnMap where previous tileUrl would result in 403 errors when pulling images, also bumped to leaflet 1.9.4 as the latest stable version Changed tileUrl provider to openstreetmap directly and updated tileAttribution, as well as infoURL --- src/core/operations/ShowOnMap.mjs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/core/operations/ShowOnMap.mjs b/src/core/operations/ShowOnMap.mjs index c2ac1c6e3..d75c2aa6a 100644 --- a/src/core/operations/ShowOnMap.mjs +++ b/src/core/operations/ShowOnMap.mjs @@ -1,6 +1,7 @@ /** * @author j433866 [j433866@gmail.com] - * @copyright Crown Copyright 2019 + * @author 0xff1ce [github.com/0xff1ce] + * @copyright Crown Copyright 2024 * @license Apache-2.0 */ @@ -22,7 +23,7 @@ class ShowOnMap extends Operation { this.name = "Show on map"; this.module = "Hashing"; this.description = "Displays co-ordinates on a slippy map.

Co-ordinates will be converted to decimal degrees before being shown on the map.

Supported formats:
  • Degrees Minutes Seconds (DMS)
  • Degrees Decimal Minutes (DDM)
  • Decimal Degrees (DD)
  • Geohash
  • Military Grid Reference System (MGRS)
  • Ordnance Survey National Grid (OSNG)
  • Universal Transverse Mercator (UTM)

This operation will not work offline."; - this.infoURL = "https://foundation.wikimedia.org/wiki/Maps_Terms_of_Use"; + this.infoURL = "https://osmfoundation.org/wiki/Terms_of_Use"; this.inputType = "string"; this.outputType = "string"; this.presentType = "html"; @@ -85,10 +86,10 @@ class ShowOnMap extends Operation { data = "0, 0"; } const zoomLevel = args[0]; - const tileUrl = "https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png", - tileAttribution = "Wikimedia maps | © OpenStreetMap contributors", - leafletUrl = "https://unpkg.com/leaflet@1.5.0/dist/leaflet.js", - leafletCssUrl = "https://unpkg.com/leaflet@1.5.0/dist/leaflet.css"; + const tileUrl = "https://tile.openstreetmap.org/{z}/{x}/{y}.png", + tileAttribution = "© OpenStreetMap contributors", + leafletUrl = "https://unpkg.com/leaflet@1.9.4/dist/leaflet.js", + leafletCssUrl = "https://unpkg.com/leaflet@1.9.4/dist/leaflet.css"; return `