mirror of
https://github.com/gchq/CyberChef
synced 2025-12-27 13:43:30 +00:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
05bfa9158d | ||
|
|
649016bc85 | ||
|
|
7492b874cf | ||
|
|
9ea21af61f | ||
|
|
dd18e52993 |
@@ -13,6 +13,9 @@ All major and minor version changes will be documented in this file. Details of
|
||||
|
||||
## Details
|
||||
|
||||
### [9.32.0] - 2021-08-18
|
||||
- 'Protobuf Encode' operation added and decode operation modified to allow decoding with full and partial schemas [@n1474335] | [dd18e52]
|
||||
|
||||
### [9.31.0] - 2021-08-10
|
||||
- 'HASSH Client Fingerprint' and 'HASSH Server Fingerprint' operations added [@n1474335] | [e9ca4dc]
|
||||
|
||||
@@ -268,6 +271,7 @@ All major and minor version changes will be documented in this file. Details of
|
||||
|
||||
|
||||
|
||||
[9.32.0]: https://github.com/gchq/CyberChef/releases/tag/v9.32.0
|
||||
[9.31.0]: https://github.com/gchq/CyberChef/releases/tag/v9.31.0
|
||||
[9.30.0]: https://github.com/gchq/CyberChef/releases/tag/v9.30.0
|
||||
[9.29.0]: https://github.com/gchq/CyberChef/releases/tag/v9.29.0
|
||||
@@ -385,6 +389,7 @@ All major and minor version changes will be documented in this file. Details of
|
||||
[9a33498]: https://github.com/gchq/CyberChef/commit/9a33498fed26a8df9c9f35f39a78a174bf50a513
|
||||
[289a417]: https://github.com/gchq/CyberChef/commit/289a417dfb5923de5e1694354ec42a08d9395bfe
|
||||
[e9ca4dc]: https://github.com/gchq/CyberChef/commit/e9ca4dc9caf98f33fd986431cd400c88082a42b8
|
||||
[dd18e52]: https://github.com/gchq/CyberChef/commit/dd18e529939078b89867297b181a584e8b2cc7da
|
||||
|
||||
[#95]: https://github.com/gchq/CyberChef/pull/299
|
||||
[#173]: https://github.com/gchq/CyberChef/pull/173
|
||||
|
||||
5340
package-lock.json
generated
5340
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
99
package.json
99
package.json
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "cyberchef",
|
||||
"version": "9.31.0",
|
||||
"version": "9.32.1",
|
||||
"description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
|
||||
"author": "n1474335 <n1474335@gmail.com>",
|
||||
"homepage": "https://gchq.github.io/CyberChef",
|
||||
@@ -36,22 +36,22 @@
|
||||
"node >= 10"
|
||||
],
|
||||
"devDependencies": {
|
||||
"@babel/core": "^7.12.16",
|
||||
"@babel/plugin-transform-runtime": "^7.12.15",
|
||||
"@babel/preset-env": "^7.12.16",
|
||||
"autoprefixer": "^10.2.4",
|
||||
"@babel/core": "^7.15.0",
|
||||
"@babel/plugin-transform-runtime": "^7.15.0",
|
||||
"@babel/preset-env": "^7.15.0",
|
||||
"autoprefixer": "^10.3.1",
|
||||
"babel-eslint": "^10.1.0",
|
||||
"babel-loader": "^8.2.2",
|
||||
"babel-plugin-dynamic-import-node": "^2.3.3",
|
||||
"chromedriver": "^92.0.0",
|
||||
"chromedriver": "^92.0.1",
|
||||
"cli-progress": "^3.9.0",
|
||||
"colors": "^1.4.0",
|
||||
"copy-webpack-plugin": "^7.0.0",
|
||||
"css-loader": "^5.0.2",
|
||||
"eslint": "^7.20.0",
|
||||
"exports-loader": "^2.0.0",
|
||||
"copy-webpack-plugin": "^9.0.1",
|
||||
"css-loader": "5.2.7",
|
||||
"eslint": "^7.32.0",
|
||||
"exports-loader": "^3.0.0",
|
||||
"file-loader": "^6.2.0",
|
||||
"grunt": "^1.3.0",
|
||||
"grunt": "^1.4.1",
|
||||
"grunt-chmod": "~1.1.1",
|
||||
"grunt-concurrent": "^3.0.0",
|
||||
"grunt-contrib-clean": "~2.0.0",
|
||||
@@ -60,56 +60,56 @@
|
||||
"grunt-contrib-watch": "^1.1.0",
|
||||
"grunt-eslint": "^23.0.0",
|
||||
"grunt-exec": "~3.0.0",
|
||||
"grunt-webpack": "^4.0.2",
|
||||
"grunt-webpack": "^4.0.3",
|
||||
"grunt-zip": "^0.18.2",
|
||||
"html-webpack-plugin": "^5.1.0",
|
||||
"imports-loader": "^2.0.0",
|
||||
"mini-css-extract-plugin": "^1.3.7",
|
||||
"nightwatch": "^1.7.7",
|
||||
"html-webpack-plugin": "^5.3.2",
|
||||
"imports-loader": "^3.0.0",
|
||||
"mini-css-extract-plugin": "1.3.7",
|
||||
"nightwatch": "^1.7.8",
|
||||
"node-sass": "^5.0.0",
|
||||
"postcss": "^8.2.6",
|
||||
"postcss-css-variables": "^0.17.0",
|
||||
"postcss-import": "^14.0.0",
|
||||
"postcss-loader": "^5.0.0",
|
||||
"postcss": "^8.3.6",
|
||||
"postcss-css-variables": "^0.18.0",
|
||||
"postcss-import": "^14.0.2",
|
||||
"postcss-loader": "^6.1.1",
|
||||
"prompt": "^1.1.0",
|
||||
"sass-loader": "^11.0.1",
|
||||
"sitemap": "^6.3.6",
|
||||
"style-loader": "^2.0.0",
|
||||
"sass-loader": "^12.1.0",
|
||||
"sitemap": "^7.0.0",
|
||||
"style-loader": "^3.2.1",
|
||||
"svg-url-loader": "^7.1.1",
|
||||
"url-loader": "^4.1.1",
|
||||
"webpack": "^5.22.0",
|
||||
"webpack-bundle-analyzer": "^4.4.0",
|
||||
"webpack-dev-server": "^3.11.2",
|
||||
"webpack": "^5.51.0",
|
||||
"webpack-bundle-analyzer": "^4.4.2",
|
||||
"webpack-dev-server": "3.11.2",
|
||||
"webpack-node-externals": "^3.0.0",
|
||||
"worker-loader": "^3.0.8"
|
||||
},
|
||||
"dependencies": {
|
||||
"@babel/polyfill": "^7.12.1",
|
||||
"@babel/runtime": "^7.12.13",
|
||||
"@babel/runtime": "^7.15.3",
|
||||
"arrive": "^2.4.1",
|
||||
"avsc": "^5.5.3",
|
||||
"avsc": "^5.7.3",
|
||||
"babel-plugin-transform-builtin-extend": "1.1.2",
|
||||
"bcryptjs": "^2.4.3",
|
||||
"bignumber.js": "^9.0.1",
|
||||
"blakejs": "^1.1.0",
|
||||
"blakejs": "^1.1.1",
|
||||
"bootstrap": "4.6.0",
|
||||
"bootstrap-colorpicker": "^3.2.0",
|
||||
"bootstrap-colorpicker": "^3.4.0",
|
||||
"bootstrap-material-design": "^4.1.3",
|
||||
"browserify-zlib": "^0.2.0",
|
||||
"bson": "^4.2.2",
|
||||
"bson": "^4.4.1",
|
||||
"buffer": "^6.0.3",
|
||||
"cbor": "^5.0.1",
|
||||
"cbor": "5.0.1",
|
||||
"chi-squared": "^1.1.0",
|
||||
"codepage": "^1.14.0",
|
||||
"core-js": "^3.8.3",
|
||||
"codepage": "^1.15.0",
|
||||
"core-js": "^3.16.2",
|
||||
"crypto-api": "^0.8.5",
|
||||
"crypto-browserify": "^3.12.0",
|
||||
"crypto-js": "^4.0.0",
|
||||
"crypto-js": "^4.1.1",
|
||||
"ctph.js": "0.0.5",
|
||||
"d3": "^6.5.0",
|
||||
"d3": "6.5.0",
|
||||
"d3-hexbin": "^0.2.2",
|
||||
"diff": "^5.0.0",
|
||||
"es6-promisify": "^6.1.1",
|
||||
"es6-promisify": "^7.0.0",
|
||||
"escodegen": "^2.0.0",
|
||||
"esm": "^3.2.25",
|
||||
"esprima": "^4.0.1",
|
||||
@@ -117,23 +117,23 @@
|
||||
"file-saver": "^2.0.5",
|
||||
"flat": "^5.0.2",
|
||||
"geodesy": "1.1.3",
|
||||
"highlight.js": "^10.6.0",
|
||||
"highlight.js": "^11.2.0",
|
||||
"jimp": "^0.16.1",
|
||||
"jquery": "3.5.1",
|
||||
"jquery": "3.6.0",
|
||||
"js-crc": "^0.2.0",
|
||||
"js-sha3": "^0.8.0",
|
||||
"jsesc": "^3.0.2",
|
||||
"jsonpath": "^1.1.1",
|
||||
"jsonwebtoken": "^8.5.1",
|
||||
"jsqr": "^1.3.1",
|
||||
"jsrsasign": "^10.3.0",
|
||||
"jsqr": "^1.4.0",
|
||||
"jsrsasign": "^10.4.0",
|
||||
"kbpgp": "2.1.15",
|
||||
"libbzip2-wasm": "0.0.4",
|
||||
"libyara-wasm": "^1.1.0",
|
||||
"lodash": "^4.17.21",
|
||||
"loglevel": "^1.7.1",
|
||||
"loglevel-message-prefix": "^3.0.0",
|
||||
"markdown-it": "^12.0.4",
|
||||
"markdown-it": "^12.2.0",
|
||||
"moment": "^2.29.1",
|
||||
"moment-timezone": "^0.5.33",
|
||||
"ngeohash": "^0.6.3",
|
||||
@@ -146,22 +146,23 @@
|
||||
"path": "^0.12.7",
|
||||
"popper.js": "^1.16.1",
|
||||
"process": "^0.11.10",
|
||||
"protobufjs": "^6.11.2",
|
||||
"qr-image": "^3.2.0",
|
||||
"scryptsy": "^2.1.0",
|
||||
"snackbarjs": "^1.1.0",
|
||||
"sortablejs": "^1.13.0",
|
||||
"split.js": "^1.6.2",
|
||||
"ssdeep.js": "0.0.2",
|
||||
"sortablejs": "^1.14.0",
|
||||
"split.js": "^1.6.4",
|
||||
"ssdeep.js": "0.0.3",
|
||||
"stream-browserify": "^3.0.0",
|
||||
"terser": "^5.6.0",
|
||||
"tesseract.js": "2.1.1",
|
||||
"ua-parser-js": "^0.7.24",
|
||||
"terser": "^5.7.1",
|
||||
"tesseract.js": "2.1.5",
|
||||
"ua-parser-js": "^0.7.28",
|
||||
"unorm": "^1.6.0",
|
||||
"utf8": "^3.0.0",
|
||||
"vkbeautify": "^0.99.3",
|
||||
"xmldom": "^0.6.0",
|
||||
"xpath": "0.0.32",
|
||||
"xregexp": "^5.0.1",
|
||||
"xregexp": "^5.1.0",
|
||||
"zlibjs": "^0.3.1"
|
||||
},
|
||||
"scripts": {
|
||||
|
||||
1
src/core/config/Categories.json
Executable file → Normal file
1
src/core/config/Categories.json
Executable file → Normal file
@@ -191,6 +191,7 @@
|
||||
"URL Encode",
|
||||
"URL Decode",
|
||||
"Protobuf Decode",
|
||||
"Protobuf Encode",
|
||||
"VarInt Encode",
|
||||
"VarInt Decode",
|
||||
"JA3 Fingerprint",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import Utils from "../Utils.mjs";
|
||||
import protobuf from "protobufjs";
|
||||
|
||||
/**
|
||||
* Protobuf lib. Contains functions to decode protobuf serialised
|
||||
@@ -32,9 +33,10 @@ class Protobuf {
|
||||
this.MSB = 0x80;
|
||||
this.VALUE = 0x7f;
|
||||
|
||||
// Declare offset and length
|
||||
// Declare offset, length, and field type object
|
||||
this.offset = 0;
|
||||
this.LENGTH = data.length;
|
||||
this.fieldTypes = {};
|
||||
}
|
||||
|
||||
// Public Functions
|
||||
@@ -76,15 +78,281 @@ class Protobuf {
|
||||
return pb._varInt();
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode input JSON according to the given schema
|
||||
*
|
||||
* @param {Object} input
|
||||
* @param {Object []} args
|
||||
* @returns {Object}
|
||||
*/
|
||||
static encode(input, args) {
|
||||
this.updateProtoRoot(args[0]);
|
||||
if (!this.mainMessageName) {
|
||||
throw new Error("Schema Error: Schema not defined");
|
||||
}
|
||||
const message = this.parsedProto.root.nested[this.mainMessageName];
|
||||
|
||||
// Convert input into instance of message, and verify instance
|
||||
input = message.fromObject(input);
|
||||
const error = message.verify(input);
|
||||
if (error) {
|
||||
throw new Error("Input Error: " + error);
|
||||
}
|
||||
// Encode input
|
||||
const output = message.encode(input).finish();
|
||||
return new Uint8Array(output).buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse Protobuf data
|
||||
*
|
||||
* @param {byteArray} input
|
||||
* @returns {Object}
|
||||
*/
|
||||
static decode(input) {
|
||||
static decode(input, args) {
|
||||
this.updateProtoRoot(args[0]);
|
||||
this.showUnknownFields = args[1];
|
||||
this.showTypes = args[2];
|
||||
return this.mergeDecodes(input);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the parsedProto, throw parsing errors
|
||||
*
|
||||
* @param {string} protoText
|
||||
*/
|
||||
static updateProtoRoot(protoText) {
|
||||
try {
|
||||
this.parsedProto = protobuf.parse(protoText);
|
||||
if (this.parsedProto.package) {
|
||||
this.parsedProto.root = this.parsedProto.root.nested[this.parsedProto.package];
|
||||
}
|
||||
this.updateMainMessageName();
|
||||
} catch (error) {
|
||||
throw new Error("Schema " + error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set mainMessageName to the first instance of a message defined in the schema that is not a submessage
|
||||
*
|
||||
*/
|
||||
static updateMainMessageName() {
|
||||
const messageNames = [];
|
||||
const fieldTypes = [];
|
||||
this.parsedProto.root.nestedArray.forEach(block => {
|
||||
if (block.constructor.name === "Type") {
|
||||
messageNames.push(block.name);
|
||||
this.parsedProto.root.nested[block.name].fieldsArray.forEach(field => {
|
||||
fieldTypes.push(field.type);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
if (messageNames.length === 0) {
|
||||
this.mainMessageName = null;
|
||||
} else {
|
||||
for (const name of messageNames) {
|
||||
if (!fieldTypes.includes(name)) {
|
||||
this.mainMessageName = name;
|
||||
break;
|
||||
}
|
||||
}
|
||||
this.mainMessageName = messageNames[0];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode input using Protobufjs package and raw methods, compare, and merge results
|
||||
*
|
||||
* @param {byteArray} input
|
||||
* @returns {Object}
|
||||
*/
|
||||
static mergeDecodes(input) {
|
||||
const pb = new Protobuf(input);
|
||||
return pb._parse();
|
||||
let rawDecode = pb._parse();
|
||||
let message;
|
||||
|
||||
if (this.showTypes) {
|
||||
rawDecode = this.showRawTypes(rawDecode, pb.fieldTypes);
|
||||
this.parsedProto.root = this.appendTypesToFieldNames(this.parsedProto.root);
|
||||
}
|
||||
|
||||
try {
|
||||
message = this.parsedProto.root.nested[this.mainMessageName];
|
||||
const packageDecode = message.toObject(message.decode(input), {
|
||||
bytes: String,
|
||||
longs: Number,
|
||||
enums: String,
|
||||
defualts: true
|
||||
});
|
||||
const output = {};
|
||||
|
||||
if (this.showUnknownFields) {
|
||||
output[message.name] = packageDecode;
|
||||
output["Unknown Fields"] = this.compareFields(rawDecode, message);
|
||||
return output;
|
||||
} else {
|
||||
return packageDecode;
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
if (message) {
|
||||
throw new Error("Input " + error);
|
||||
} else {
|
||||
return rawDecode;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace fieldnames with fieldname and type
|
||||
*
|
||||
* @param {Object} schemaRoot
|
||||
* @returns {Object}
|
||||
*/
|
||||
static appendTypesToFieldNames(schemaRoot) {
|
||||
for (const block of schemaRoot.nestedArray) {
|
||||
if (block.constructor.name === "Type") {
|
||||
for (const [fieldName, fieldData] of Object.entries(block.fields)) {
|
||||
schemaRoot.nested[block.name].remove(block.fields[fieldName]);
|
||||
schemaRoot.nested[block.name].add(new protobuf.Field(`${fieldName} (${fieldData.type})`, fieldData.id, fieldData.type, fieldData.rule));
|
||||
}
|
||||
}
|
||||
}
|
||||
return schemaRoot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add field type to field name for fields in the raw decoded output
|
||||
*
|
||||
* @param {Object} rawDecode
|
||||
* @param {Object} fieldTypes
|
||||
* @returns {Object}
|
||||
*/
|
||||
static showRawTypes(rawDecode, fieldTypes) {
|
||||
for (const [fieldNum, value] of Object.entries(rawDecode)) {
|
||||
const fieldType = fieldTypes[fieldNum];
|
||||
let outputFieldValue;
|
||||
let outputFieldType;
|
||||
|
||||
// Submessages
|
||||
if (isNaN(fieldType)) {
|
||||
outputFieldType = 2;
|
||||
|
||||
// Repeated submessages
|
||||
if (Array.isArray(value)) {
|
||||
const fieldInstances = [];
|
||||
for (const instance of Object.keys(value)) {
|
||||
if (typeof(value[instance]) !== "string") {
|
||||
fieldInstances.push(this.showRawTypes(value[instance], fieldType));
|
||||
} else {
|
||||
fieldInstances.push(value[instance]);
|
||||
}
|
||||
}
|
||||
outputFieldValue = fieldInstances;
|
||||
|
||||
// Single submessage
|
||||
} else {
|
||||
outputFieldValue = this.showRawTypes(value, fieldType);
|
||||
}
|
||||
|
||||
// Non-submessage field
|
||||
} else {
|
||||
outputFieldType = fieldType;
|
||||
outputFieldValue = value;
|
||||
}
|
||||
|
||||
// Substitute fieldNum with field number and type
|
||||
rawDecode[`field #${fieldNum}: ${this.getTypeInfo(outputFieldType)}`] = outputFieldValue;
|
||||
delete rawDecode[fieldNum];
|
||||
}
|
||||
return rawDecode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare raw decode to package decode and return discrepancies
|
||||
*
|
||||
* @param rawDecodedMessage
|
||||
* @param schemaMessage
|
||||
* @returns {Object}
|
||||
*/
|
||||
static compareFields(rawDecodedMessage, schemaMessage) {
|
||||
// Define message data using raw decode output and schema
|
||||
const schemaFieldProperties = {};
|
||||
const schemaFieldNames = Object.keys(schemaMessage.fields);
|
||||
schemaFieldNames.forEach(field => schemaFieldProperties[schemaMessage.fields[field].id] = field);
|
||||
|
||||
// Loop over each field present in the raw decode output
|
||||
for (const fieldName in rawDecodedMessage) {
|
||||
let fieldId;
|
||||
if (isNaN(fieldName)) {
|
||||
fieldId = fieldName.match(/^field #(\d+)/)[1];
|
||||
} else {
|
||||
fieldId = fieldName;
|
||||
}
|
||||
|
||||
// Check if this field is defined in the schema
|
||||
if (fieldId in schemaFieldProperties) {
|
||||
const schemaFieldName = schemaFieldProperties[fieldId];
|
||||
|
||||
// Extract the current field data from the raw decode and schema
|
||||
const rawFieldData = rawDecodedMessage[fieldName];
|
||||
const schemaField = schemaMessage.fields[schemaFieldName];
|
||||
|
||||
// Check for repeated fields
|
||||
if (Array.isArray(rawFieldData) && !schemaField.repeated) {
|
||||
rawDecodedMessage[`(${schemaMessage.name}) ${schemaFieldName} is a repeated field`] = rawFieldData;
|
||||
}
|
||||
|
||||
// Check for submessage fields
|
||||
if (schemaField.resolvedType !== null && schemaField.resolvedType.constructor.name === "Type") {
|
||||
const subMessageType = schemaMessage.fields[schemaFieldName].type;
|
||||
const schemaSubMessage = this.parsedProto.root.nested[subMessageType];
|
||||
const rawSubMessages = rawDecodedMessage[fieldName];
|
||||
let rawDecodedSubMessage = {};
|
||||
|
||||
// Squash multiple submessage instances into one submessage
|
||||
if (Array.isArray(rawSubMessages)) {
|
||||
rawSubMessages.forEach(subMessageInstance => {
|
||||
const instanceFields = Object.entries(subMessageInstance);
|
||||
instanceFields.forEach(subField => {
|
||||
rawDecodedSubMessage[subField[0]] = subField[1];
|
||||
});
|
||||
});
|
||||
} else {
|
||||
rawDecodedSubMessage = rawSubMessages;
|
||||
}
|
||||
|
||||
// Treat submessage as own message and compare its fields
|
||||
rawDecodedSubMessage = Protobuf.compareFields(rawDecodedSubMessage, schemaSubMessage);
|
||||
if (Object.entries(rawDecodedSubMessage).length !== 0) {
|
||||
rawDecodedMessage[`${schemaFieldName} (${subMessageType}) has missing fields`] = rawDecodedSubMessage;
|
||||
}
|
||||
}
|
||||
delete rawDecodedMessage[fieldName];
|
||||
}
|
||||
}
|
||||
return rawDecodedMessage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns wiretype information for input wiretype number
|
||||
*
|
||||
* @param {number} wireType
|
||||
* @returns {string}
|
||||
*/
|
||||
static getTypeInfo(wireType) {
|
||||
switch (wireType) {
|
||||
case 0:
|
||||
return "VarInt (e.g. int32, bool)";
|
||||
case 1:
|
||||
return "64-Bit (e.g. fixed64, double)";
|
||||
case 2:
|
||||
return "L-delim (e.g. string, message)";
|
||||
case 5:
|
||||
return "32-Bit (e.g. fixed32, float)";
|
||||
}
|
||||
}
|
||||
|
||||
// Private Class Functions
|
||||
@@ -143,6 +411,11 @@ class Protobuf {
|
||||
const header = this._fieldHeader();
|
||||
const type = header.type;
|
||||
const key = header.key;
|
||||
|
||||
if (typeof(this.fieldTypes[key]) !== "object") {
|
||||
this.fieldTypes[key] = type;
|
||||
}
|
||||
|
||||
switch (type) {
|
||||
// varint
|
||||
case 0:
|
||||
@@ -152,7 +425,7 @@ class Protobuf {
|
||||
return { "key": key, "value": this._uint64() };
|
||||
// length delimited
|
||||
case 2:
|
||||
return { "key": key, "value": this._lenDelim() };
|
||||
return { "key": key, "value": this._lenDelim(key) };
|
||||
// fixed 32
|
||||
case 5:
|
||||
return { "key": key, "value": this._uint32() };
|
||||
@@ -237,10 +510,10 @@ class Protobuf {
|
||||
* @returns {number}
|
||||
*/
|
||||
_uint64() {
|
||||
// Read off a Uint64
|
||||
let num = this.data[this.offset++] * 0x1000000 + (this.data[this.offset++] << 16) + (this.data[this.offset++] << 8) + this.data[this.offset++];
|
||||
num = num * 0x100000000 + this.data[this.offset++] * 0x1000000 + (this.data[this.offset++] << 16) + (this.data[this.offset++] << 8) + this.data[this.offset++];
|
||||
return num;
|
||||
// Read off a Uint64 with little-endian
|
||||
const lowerHalf = this.data[this.offset++] + (this.data[this.offset++] * 0x100) + (this.data[this.offset++] * 0x10000) + this.data[this.offset++] * 0x1000000;
|
||||
const upperHalf = this.data[this.offset++] + (this.data[this.offset++] * 0x100) + (this.data[this.offset++] * 0x10000) + this.data[this.offset++] * 0x1000000;
|
||||
return upperHalf * 0x100000000 + lowerHalf;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -249,7 +522,7 @@ class Protobuf {
|
||||
* @private
|
||||
* @returns {Object|string}
|
||||
*/
|
||||
_lenDelim() {
|
||||
_lenDelim(fieldNum) {
|
||||
// Read off the field length
|
||||
const length = this._varInt();
|
||||
const fieldBytes = this.data.slice(this.offset, this.offset + length);
|
||||
@@ -258,6 +531,10 @@ class Protobuf {
|
||||
// Attempt to parse as a new Protobuf Object
|
||||
const pbObject = new Protobuf(fieldBytes);
|
||||
field = pbObject._parse();
|
||||
|
||||
// Set field types object
|
||||
this.fieldTypes[fieldNum] = {...this.fieldTypes[fieldNum], ...pbObject.fieldTypes};
|
||||
|
||||
} catch (err) {
|
||||
// Otherwise treat as bytes
|
||||
field = Utils.byteArrayToChars(fieldBytes);
|
||||
@@ -276,7 +553,7 @@ class Protobuf {
|
||||
_uint32() {
|
||||
// Use a dataview to read off the integer
|
||||
const dataview = new DataView(new Uint8Array(this.data.slice(this.offset, this.offset + 4)).buffer);
|
||||
const value = dataview.getUint32(0);
|
||||
const value = dataview.getUint32(0, true);
|
||||
this.offset += 4;
|
||||
return value;
|
||||
}
|
||||
|
||||
@@ -20,12 +20,30 @@ class ProtobufDecode extends Operation {
|
||||
super();
|
||||
|
||||
this.name = "Protobuf Decode";
|
||||
this.module = "Default";
|
||||
this.description = "Decodes any Protobuf encoded data to a JSON representation of the data using the field number as the field key.";
|
||||
this.module = "Protobuf";
|
||||
this.description = "Decodes any Protobuf encoded data to a JSON representation of the data using the field number as the field key.<br><br>If a .proto schema is defined, the encoded data will be decoded with reference to the schema. Only one message instance will be decoded. <br><br><u>Show Unknown Fields</u><br>When a schema is used, this option shows fields that are present in the input data but not defined in the schema.<br><br><u>Show Types</u><br>Show the type of a field next to its name. For undefined fields, the wiretype and example types are shown instead.";
|
||||
this.infoURL = "https://wikipedia.org/wiki/Protocol_Buffers";
|
||||
this.inputType = "ArrayBuffer";
|
||||
this.outputType = "JSON";
|
||||
this.args = [];
|
||||
this.args = [
|
||||
{
|
||||
name: "Schema (.proto text)",
|
||||
type: "text",
|
||||
value: "",
|
||||
rows: 8,
|
||||
hint: "Drag and drop is enabled on this ingredient"
|
||||
},
|
||||
{
|
||||
name: "Show Unknown Fields",
|
||||
type: "boolean",
|
||||
value: false
|
||||
},
|
||||
{
|
||||
name: "Show Types",
|
||||
type: "boolean",
|
||||
value: false
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -36,7 +54,7 @@ class ProtobufDecode extends Operation {
|
||||
run(input, args) {
|
||||
input = new Uint8Array(input);
|
||||
try {
|
||||
return Protobuf.decode(input);
|
||||
return Protobuf.decode(input, args);
|
||||
} catch (err) {
|
||||
throw new OperationError(err);
|
||||
}
|
||||
|
||||
54
src/core/operations/ProtobufEncode.mjs
Normal file
54
src/core/operations/ProtobufEncode.mjs
Normal file
@@ -0,0 +1,54 @@
|
||||
/**
|
||||
* @author GCHQ Contributor [3]
|
||||
* @copyright Crown Copyright 2021
|
||||
* @license Apache-2.0
|
||||
*/
|
||||
|
||||
import Operation from "../Operation.mjs";
|
||||
import OperationError from "../errors/OperationError.mjs";
|
||||
import Protobuf from "../lib/Protobuf.mjs";
|
||||
|
||||
/**
|
||||
* Protobuf Encode operation
|
||||
*/
|
||||
class ProtobufEncode extends Operation {
|
||||
|
||||
/**
|
||||
* ProtobufEncode constructor
|
||||
*/
|
||||
constructor() {
|
||||
super();
|
||||
|
||||
this.name = "Protobuf Encode";
|
||||
this.module = "Protobuf";
|
||||
this.description = "Encodes a valid JSON object into a protobuf byte array using the input .proto schema.";
|
||||
this.infoURL = "https://developers.google.com/protocol-buffers/docs/encoding";
|
||||
this.inputType = "JSON";
|
||||
this.outputType = "ArrayBuffer";
|
||||
this.args = [
|
||||
{
|
||||
name: "Schema (.proto text)",
|
||||
type: "text",
|
||||
value: "",
|
||||
rows: 8,
|
||||
hint: "Drag and drop is enabled on this ingredient"
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Object} input
|
||||
* @param {Object[]} args
|
||||
* @returns {ArrayBuffer}
|
||||
*/
|
||||
run(input, args) {
|
||||
try {
|
||||
return Protobuf.encode(input, args);
|
||||
} catch (error) {
|
||||
throw new OperationError(error);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export default ProtobufEncode;
|
||||
@@ -10,10 +10,10 @@ import TestRegister from "../../lib/TestRegister.mjs";
|
||||
|
||||
TestRegister.addTests([
|
||||
{
|
||||
name: "Protobuf Decode",
|
||||
name: "Protobuf Decode: no schema",
|
||||
input: "0d1c0000001203596f751a024d65202b2a0a0a066162633132331200",
|
||||
expectedOutput: JSON.stringify({
|
||||
"1": 469762048,
|
||||
"1": 28,
|
||||
"2": "You",
|
||||
"3": "Me",
|
||||
"4": 43,
|
||||
@@ -29,7 +29,277 @@ TestRegister.addTests([
|
||||
},
|
||||
{
|
||||
"op": "Protobuf Decode",
|
||||
"args": []
|
||||
"args": ["", false, false]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
name: "Protobuf Decode: partial schema, no unknown fields",
|
||||
input: "0d1c0000001203596f751a024d65202b2a0a0a066162633132331200",
|
||||
expectedOutput: JSON.stringify({
|
||||
"Apple": [
|
||||
28
|
||||
],
|
||||
"Banana": "You",
|
||||
"Carrot": [
|
||||
"Me"
|
||||
]
|
||||
}, null, 4),
|
||||
recipeConfig: [
|
||||
{
|
||||
"op": "From Hex",
|
||||
"args": ["Auto"]
|
||||
},
|
||||
{
|
||||
"op": "Protobuf Decode",
|
||||
"args": [
|
||||
`message Test {
|
||||
repeated fixed32 Apple = 1;
|
||||
optional string Banana = 2;
|
||||
repeated string Carrot = 3;
|
||||
}`,
|
||||
false,
|
||||
false
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
name: "Protobuf Decode: partial schema, show unknown fields",
|
||||
input: "0d1c0000001203596f751a024d65202b2a0a0a066162633132331200",
|
||||
expectedOutput: JSON.stringify({
|
||||
"Test": {
|
||||
"Apple": [
|
||||
28
|
||||
],
|
||||
"Banana": "You",
|
||||
"Carrot": [
|
||||
"Me"
|
||||
]
|
||||
},
|
||||
"Unknown Fields": {
|
||||
"4": 43,
|
||||
"5": {
|
||||
"1": "abc123",
|
||||
"2": {}
|
||||
}
|
||||
}
|
||||
}, null, 4),
|
||||
recipeConfig: [
|
||||
{
|
||||
"op": "From Hex",
|
||||
"args": ["Auto"]
|
||||
},
|
||||
{
|
||||
"op": "Protobuf Decode",
|
||||
"args": [
|
||||
`message Test {
|
||||
repeated fixed32 Apple = 1;
|
||||
optional string Banana = 2;
|
||||
repeated string Carrot = 3;
|
||||
}`,
|
||||
true,
|
||||
false
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
name: "Protobuf Decode: full schema, no unknown fields",
|
||||
input: "0d1c0000001203596f751a024d65202b2a0a0a06616263313233120031ff00000000000000",
|
||||
expectedOutput: JSON.stringify({
|
||||
"Apple": [
|
||||
28
|
||||
],
|
||||
"Banana": "You",
|
||||
"Carrot": [
|
||||
"Me"
|
||||
],
|
||||
"Date": 43,
|
||||
"Elderberry": {
|
||||
"Fig": "abc123",
|
||||
"Grape": {}
|
||||
},
|
||||
"Huckleberry": 255
|
||||
}, null, 4),
|
||||
recipeConfig: [
|
||||
{
|
||||
"op": "From Hex",
|
||||
"args": ["Auto"]
|
||||
},
|
||||
{
|
||||
"op": "Protobuf Decode",
|
||||
"args": [
|
||||
`message Test {
|
||||
repeated fixed32 Apple = 1;
|
||||
optional string Banana = 2;
|
||||
repeated string Carrot = 3;
|
||||
optional int32 Date = 4;
|
||||
optional subTest Elderberry = 5;
|
||||
optional fixed64 Huckleberry = 6;
|
||||
}
|
||||
message subTest {
|
||||
optional string Fig = 1;
|
||||
optional subSubTest Grape = 2;
|
||||
}
|
||||
message subSubTest {}`,
|
||||
false,
|
||||
false
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
name: "Protobuf Decode: partial schema, show unknown fields, show types",
|
||||
input: "0d1c0000001203596f751a024d65202b2a0a0a06616263313233120031ba32a96cc10200003801",
|
||||
expectedOutput: JSON.stringify({
|
||||
"Test": {
|
||||
"Banana (string)": "You",
|
||||
"Carrot (string)": [
|
||||
"Me"
|
||||
],
|
||||
"Date (int32)": 43,
|
||||
"Imbe (Options)": "Option1"
|
||||
},
|
||||
"Unknown Fields": {
|
||||
"field #1: 32-Bit (e.g. fixed32, float)": 28,
|
||||
"field #5: L-delim (e.g. string, message)": {
|
||||
"field #1: L-delim (e.g. string, message)": "abc123",
|
||||
"field #2: L-delim (e.g. string, message)": {}
|
||||
},
|
||||
"field #6: 64-Bit (e.g. fixed64, double)": 3029774971578
|
||||
}
|
||||
}, null, 4),
|
||||
recipeConfig: [
|
||||
{
|
||||
"op": "From Hex",
|
||||
"args": ["Auto"]
|
||||
},
|
||||
{
|
||||
"op": "Protobuf Decode",
|
||||
"args": [
|
||||
`message Test {
|
||||
optional string Banana = 2;
|
||||
repeated string Carrot = 3;
|
||||
optional int32 Date = 4;
|
||||
optional Options Imbe = 7;
|
||||
}
|
||||
message subTest {
|
||||
optional string Fig = 1;
|
||||
optional subSubTest Grape = 2;
|
||||
}
|
||||
message subSubTest {}
|
||||
enum Options {
|
||||
Option0 = 0;
|
||||
Option1 = 1;
|
||||
Option2 = 2;
|
||||
}`,
|
||||
true,
|
||||
true
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
name: "Protobuf Encode",
|
||||
input: JSON.stringify({
|
||||
"Apple": [
|
||||
28
|
||||
],
|
||||
"Banana": "You",
|
||||
"Carrot": [
|
||||
"Me"
|
||||
],
|
||||
"Date": 43,
|
||||
"Elderberry": {
|
||||
"Fig": "abc123",
|
||||
"Grape": {}
|
||||
},
|
||||
"Huckleberry": [3029774971578],
|
||||
"Imbe": 1
|
||||
}, null, 4),
|
||||
expectedOutput: "0d1c0000001203596f751a024d65202b2a0a0a06616263313233120031ba32a96cc10200003801",
|
||||
recipeConfig: [
|
||||
{
|
||||
"op": "Protobuf Encode",
|
||||
"args": [
|
||||
`message Test {
|
||||
repeated fixed32 Apple = 1;
|
||||
optional string Banana = 2;
|
||||
repeated string Carrot = 3;
|
||||
optional int32 Date = 4;
|
||||
optional subTest Elderberry = 5;
|
||||
repeated fixed64 Huckleberry = 6;
|
||||
optional Options Imbe = 7;
|
||||
}
|
||||
message subTest {
|
||||
optional string Fig = 1;
|
||||
optional subSubTest Grape = 2;
|
||||
}
|
||||
message subSubTest {}
|
||||
enum Options {
|
||||
Option0 = 0;
|
||||
Option1 = 1;
|
||||
Option2 = 2;
|
||||
}`
|
||||
]
|
||||
},
|
||||
{
|
||||
"op": "To Hex",
|
||||
"args": [
|
||||
"None",
|
||||
0
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
name: "Protobuf Encode: incomplete schema",
|
||||
input: JSON.stringify({
|
||||
"Apple": [
|
||||
28
|
||||
],
|
||||
"Banana": "You",
|
||||
"Carrot": [
|
||||
"Me"
|
||||
],
|
||||
"Date": 43,
|
||||
"Elderberry": {
|
||||
"Fig": "abc123",
|
||||
"Grape": {}
|
||||
},
|
||||
"Huckleberry": [3029774971578],
|
||||
"Imbe": 1
|
||||
}, null, 4),
|
||||
expectedOutput: "1203596f75202b2a0a0a06616263313233120031ba32a96cc1020000",
|
||||
recipeConfig: [
|
||||
{
|
||||
"op": "Protobuf Encode",
|
||||
"args": [
|
||||
`message Test {
|
||||
optional string Banana = 2;
|
||||
optional int32 Date = 4;
|
||||
optional subTest Elderberry = 5;
|
||||
repeated fixed64 Huckleberry = 6;
|
||||
}
|
||||
message subTest {
|
||||
optional string Fig = 1;
|
||||
optional subSubTest Grape = 2;
|
||||
}
|
||||
message subSubTest {}
|
||||
enum Options {
|
||||
Option0 = 0;
|
||||
Option1 = 1;
|
||||
Option2 = 2;
|
||||
}`
|
||||
]
|
||||
},
|
||||
{
|
||||
"op": "To Hex",
|
||||
"args": [
|
||||
"None",
|
||||
0
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user