8.11.1

CHANGELOG.md

@@ -1,6 +1,9 @@
 # Changelog
 All notable changes to CyberChef will be documented in this file.
 
+### [8.11.0] - 2018-11-13
+- 'CSV to JSON' and 'JSON to CSV' operations added [@n1474335] | [#277]
+
 ### [8.10.0] - 2018-11-07
 - 'Remove Diacritics' operation added [@klaxon1] | [#387]
 
@@ -91,6 +94,7 @@ All notable changes to CyberChef will be documented in this file.
 [#239]: https://github.com/gchq/CyberChef/pull/239
 [#248]: https://github.com/gchq/CyberChef/pull/248
 [#255]: https://github.com/gchq/CyberChef/issues/255
+[#277]: https://github.com/gchq/CyberChef/issues/277
 [#281]: https://github.com/gchq/CyberChef/pull/281
 [#284]: https://github.com/gchq/CyberChef/pull/284
 [#294]: https://github.com/gchq/CyberChef/pull/294

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.10.2",
+  "version": "8.11.1",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",

src/core/Utils.mjs

@@ -573,6 +573,10 @@ class Utils {
                 cell = "";
                 lines.push(line);
                 line = [];
+                // Skip next byte if it is also a line delim (e.g. \r\n)
+                if (lineDelims.indexOf(next) >= 0 && next !== b) {
+                    i++;
+                }
             } else {
                 cell += b;
             }

src/core/config/Categories.json

@@ -54,7 +54,9 @@
             "From MessagePack",
             "To Braille",
             "From Braille",
-            "Parse TLV"
+            "Parse TLV",
+            "CSV to JSON",
+            "JSON to CSV"
         ]
     },
     {

src/core/lib/Binary.mjs

@@ -19,10 +19,10 @@ import Utils from "../Utils";
  *
  * @example
  * // returns "00010000 00100000 00110000"
- * toHex([10,20,30]);
+ * toBinary([10,20,30]);
  *
  * // returns "00010000 00100000 00110000"
- * toHex([10,20,30], ":");
+ * toBinary([10,20,30], ":");
  */
 export function toBinary(data, delim="Space", padding=8) {
     if (!data) return "";

src/core/operations/CSVToJSON.mjs

@@ -0,0 +1,80 @@
+/**
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import Utils from "../Utils";
+
+/**
+ * CSV to JSON operation
+ */
+class CSVToJSON extends Operation {
+
+    /**
+     * CSVToJSON constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "CSV to JSON";
+        this.module = "Default";
+        this.description = "Converts a CSV file to JSON format.";
+        this.infoURL = "https://wikipedia.org/wiki/Comma-separated_values";
+        this.inputType = "string";
+        this.outputType = "JSON";
+        this.args = [
+            {
+                name: "Cell delimiters",
+                type: "binaryShortString",
+                value: ","
+            },
+            {
+                name: "Row delimiters",
+                type: "binaryShortString",
+                value: "\\r\\n"
+            },
+            {
+                name: "Format",
+                type: "option",
+                value: ["Array of dictionaries", "Array of arrays"]
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {JSON}
+     */
+    run(input, args) {
+        const [cellDelims, rowDelims, format] = args;
+        let json, header;
+
+        try {
+            json = Utils.parseCSV(input, cellDelims.split(""), rowDelims.split(""));
+        } catch (err) {
+            throw new OperationError("Unable to parse CSV: " + err);
+        }
+
+        switch (format) {
+            case "Array of dictionaries":
+                header = json[0];
+                return json.slice(1).map(row => {
+                    const obj = {};
+                    header.forEach((h, i) => {
+                        obj[h] = row[i];
+                    });
+                    return obj;
+                });
+            case "Array of arrays":
+            default:
+                return json;
+        }
+    }
+
+}
+
+export default CSVToJSON;

src/core/operations/JSONToCSV.mjs

@@ -0,0 +1,111 @@
+/**
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+
+/**
+ * JSON to CSV operation
+ */
+class JSONToCSV extends Operation {
+
+    /**
+     * JSONToCSV constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "JSON to CSV";
+        this.module = "Default";
+        this.description = "Converts JSON data to a CSV based on the definition in RFC 4180.";
+        this.infoURL = "https://wikipedia.org/wiki/Comma-separated_values";
+        this.inputType = "JSON";
+        this.outputType = "string";
+        this.args = [
+            {
+                name: "Cell delimiter",
+                type: "binaryShortString",
+                value: ","
+            },
+            {
+                name: "Row delimiter",
+                type: "binaryShortString",
+                value: "\\r\\n"
+            }
+        ];
+    }
+
+    /**
+     * @param {JSON} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        const [cellDelim, rowDelim] = args;
+
+        // Record values so they don't have to be passed to other functions explicitly
+        this.cellDelim = cellDelim;
+        this.rowDelim = rowDelim;
+        const self = this;
+
+        try {
+            // If the JSON is an array of arrays, this is easy
+            if (input[0] instanceof Array) {
+                return input
+                    .map(row => row
+                        .map(self.escapeCellContents.bind(self))
+                        .join(cellDelim)
+                    )
+                    .join(rowDelim) +
+                    rowDelim;
+            }
+
+            // If it's an array of dictionaries...
+            const header = Object.keys(input[0]);
+            return header
+                .map(self.escapeCellContents.bind(self))
+                .join(cellDelim) +
+                rowDelim +
+                input
+                    .map(row => header
+                        .map(h => row[h])
+                        .map(self.escapeCellContents.bind(self))
+                        .join(cellDelim)
+                    )
+                    .join(rowDelim) +
+                rowDelim;
+        } catch (err) {
+            throw new OperationError("Unable to parse JSON to CSV: " + err.toString());
+        }
+    }
+
+    /**
+     * Correctly escapes a cell's contents based on the cell and row delimiters.
+     *
+     * @param {string} data
+     * @returns {string}
+     */
+    escapeCellContents(data) {
+        // Double quotes should be doubled up
+        data = data.replace(/"/g, '""');
+
+        // If the cell contains a cell or row delimiter or a double quote, it mut be enclosed in double quotes
+        if (
+            data.indexOf(this.cellDelim) >= 0 ||
+            data.indexOf(this.rowDelim) >= 0 ||
+            data.indexOf("\n") >= 0 ||
+            data.indexOf("\r") >= 0 ||
+            data.indexOf('"') >= 0
+        ) {
+            data = `"${data}"`;
+        }
+
+        return data;
+    }
+
+}
+
+export default JSONToCSV;

src/core/operations/ToTable.mjs

@@ -33,7 +33,7 @@ class ToTable extends Operation {
             {
                 "name": "Row delimiters",
                 "type": "binaryShortString",
-                "value": "\\n\\r"
+                "value": "\\r\\n"
             },
             {
                 "name": "Make first row header",

test/index.mjs

@@ -39,6 +39,7 @@ import "./tests/operations/Comment";
 import "./tests/operations/Compress";
 import "./tests/operations/ConditionalJump";
 import "./tests/operations/Crypt";
+import "./tests/operations/CSV";
 import "./tests/operations/DateTime";
 import "./tests/operations/ExtractEmailAddresses";
 import "./tests/operations/Fork";
@@ -126,12 +127,12 @@ function handleTestResult(testResult) {
 
 
 /**
- * Fail if the process takes longer than 10 seconds.
+ * Fail if the process takes longer than 60 seconds.
  */
 setTimeout(function() {
-    console.log("Tests took longer than 10 seconds to run, returning.");
+    console.log("Tests took longer than 60 seconds to run, returning.");
     process.exit(1);
-}, 10 * 1000);
+}, 60 * 1000);
 
 
 TestRegister.runTests()

test/tests/operations/CSV.mjs

@@ -0,0 +1,179 @@
+/**
+ * CSV tests.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ *
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+import TestRegister from "../../TestRegister";
+
+const EXAMPLE_CSV = `A,B,C,D,E,F\r
+1,2,3,4,5,6\r
+",",;,',"""",,\r
+"""hello""","a""1","multi\r
+line",,,end\r
+`;
+
+TestRegister.addTests([
+    {
+        name: "CSV to JSON: Array of dictionaries",
+        input: EXAMPLE_CSV,
+        expectedOutput: JSON.stringify([
+            {
+                "A": "1",
+                "B": "2",
+                "C": "3",
+                "D": "4",
+                "E": "5",
+                "F": "6"
+            },
+            {
+                "A": ",",
+                "B": ";",
+                "C": "'",
+                "D": "\"",
+                "E": "",
+                "F": ""
+            },
+            {
+                "A": "\"hello\"",
+                "B": "a\"1",
+                "C": "multi\r\nline",
+                "D": "",
+                "E": "",
+                "F": "end"
+            }
+        ], null, 4),
+        recipeConfig: [
+            {
+                op: "CSV to JSON",
+                args: [",", "\r\n", "Array of dictionaries"],
+            }
+        ],
+    },
+    {
+        name: "CSV to JSON: Array of arrays",
+        input: EXAMPLE_CSV,
+        expectedOutput: JSON.stringify([
+            [
+                "A",
+                "B",
+                "C",
+                "D",
+                "E",
+                "F"
+            ],
+            [
+                "1",
+                "2",
+                "3",
+                "4",
+                "5",
+                "6"
+            ],
+            [
+                ",",
+                ";",
+                "'",
+                "\"",
+                "",
+                ""
+            ],
+            [
+                "\"hello\"",
+                "a\"1",
+                "multi\r\nline",
+                "",
+                "",
+                "end"
+            ]
+        ], null, 4),
+        recipeConfig: [
+            {
+                op: "CSV to JSON",
+                args: [",", "\r\n", "Array of arrays"],
+            }
+        ],
+    },
+    {
+        name: "JSON to CSV: Array of dictionaries",
+        input: JSON.stringify([
+            {
+                "A": "1",
+                "B": "2",
+                "C": "3",
+                "D": "4",
+                "E": "5",
+                "F": "6"
+            },
+            {
+                "A": ",",
+                "B": ";",
+                "C": "'",
+                "D": "\"",
+                "E": "",
+                "F": ""
+            },
+            {
+                "A": "\"hello\"",
+                "B": "a\"1",
+                "C": "multi\r\nline",
+                "D": "",
+                "E": "",
+                "F": "end"
+            }
+        ]),
+        expectedOutput: EXAMPLE_CSV,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\r\n"],
+            }
+        ],
+    },
+    {
+        name: "JSON to CSV: Array of arrays",
+        input: JSON.stringify([
+            [
+                "A",
+                "B",
+                "C",
+                "D",
+                "E",
+                "F"
+            ],
+            [
+                "1",
+                "2",
+                "3",
+                "4",
+                "5",
+                "6"
+            ],
+            [
+                ",",
+                ";",
+                "'",
+                "\"",
+                "",
+                ""
+            ],
+            [
+                "\"hello\"",
+                "a\"1",
+                "multi\r\nline",
+                "",
+                "",
+                "end"
+            ]
+        ]),
+        expectedOutput: EXAMPLE_CSV,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\r\n"],
+            }
+        ],
+    },
+]);