diff --git a/docs/documentation.md b/docs/documentation.md index 333bd5f..b9fabb8 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -508,7 +508,7 @@ Compute the result. The pre-defined lazy operations will be executed in pipeline | ------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | | `dataframe` | clojask.classes.DataFrame.DataFrame / clojask.classes.JoinedDataFrame.JoinedDataFrame | The operated object | | | `num of workers` | int | The number of worker threads (besides the thread that reads the input and the calling thread, which writes the output) | An integer from 1 to 8, for every kind of compute. | -| `output path` | String / `nil` | The path of the output csv file | If the path already exists, will overwrite the file.
If `nil`, will store the output in memory as a vector of vectors, which represent each row, with the header row first unless `:header false`. See [example](https://github.com/clojure-finance/clojask-examples/blob/main/src/clojask_examples/in_memory.clj). | +| `output path` | String / `nil` | The path of the output file | The format follows the extension: csv, tsv, tab, txt and dat are written as text, `.xlsx` as an Excel workbook with one sheet, `Sheet1` (built in memory, so the output must fit in memory; `.xls` is rejected). If the path already exists, will overwrite the file.
If `nil`, will store the output in memory as a vector of vectors, which represent each row, with the header row first unless `:header false`. See [example](https://github.com/clojure-finance/clojask-examples/blob/main/src/clojask_examples/in_memory.clj). | | [`exception`] | Boolean | Whether an exception during calculation will cause termination | By default `false`. Is useful for debugging or detecting empty fields | | [`order`] | Boolean | Has no effect since 2.1 | Accepted so that existing calls keep working. Output rows are always in input order, except that group-by and outer joins write theirs in group order. | | [`output`] | Function | Specify how to output a row vector to the output file | Takes two arguments.
`writer` java.io.BufferedWriter
`rows` clojure.lang.PersistentVector (rows) of clojure.lang.PersistentVector (each row) | @@ -520,7 +520,7 @@ Compute the result. The pre-defined lazy operations will be executed in pipeline **Return** -A `clojask.classes.DataFrame.DataFrame` read from the output file (read without a header if `:header false`). / A vector of vectors, which represent each row (header row first unless `:header false`), if `output path` = `nil`. +A `clojask.classes.DataFrame.DataFrame` read from the output file (read without a header if `:header false`; an `.xlsx` file is read with `read-excel`). / A vector of vectors, which represent each row (header row first unless `:header false`), if `output path` = `nil`. **Example** diff --git a/src/main/clojure/clojask/dataframe.clj b/src/main/clojure/clojask/dataframe.clj index d66e4e7..d5b6d2b 100644 --- a/src/main/clojure/clojask/dataframe.clj +++ b/src/main/clojure/clojask/dataframe.clj @@ -1,6 +1,6 @@ (ns clojask.dataframe (:require clojask-io.core - [clojask-io.input :refer [read-file]] + [clojask-io.input :refer [read-file read-excel]] [clojask-io.output :as output] [clojask.classes.DataStat :refer [compute-stat]] [clojask.classes.ColInfo] @@ -339,7 +339,10 @@ header (if custom-header false header) ret (atom (transient [])) output-format (when output-dir (clojask-io.core/infer-format output-dir)) - output-func (if output-dir + ;; Excel output is collected in memory and written with write-excel + ;; at the end; a custom :output function still gets a file writer + excel-path (when (and output-dir (nil? output) (clojask-io.core/is-excel output-dir)) output-dir) + output-func (if (and output-dir (not excel-path)) ;; every dataframe has an initial output function (if (and (= (type this) clojask.classes.DataFrame.DataFrame) (.getPath this) (= output-format (clojask-io.core/infer-format (.getPath this)))) (or output (.getOutput this)) @@ -353,8 +356,10 @@ missing (vec (remove known (if (coll? select) select [select])))] (when (not= missing []) (throw (TypeException. (str "Selected columns do not exist: " (str/join ", " missing) ". get-col-names lists the valid names.")))))) + _ (when (and excel-path (not= output-format "xlsx")) + (throw (TypeException. (str "Excel output is written as .xlsx only, so " output-dir " cannot be written. Use a .xlsx path.")))) _ (u/init-file output-dir nil) - output-dir (or output-dir ".clojask/tmp.csv")] ;; fake one + output-dir (if excel-path ".clojask/tmp.csv" (or output-dir ".clojask/tmp.csv"))] ;; fake one (when custom-header (with-open [wrtr (io/writer output-dir)] (output-func wrtr [custom-header]))) @@ -373,8 +378,13 @@ (.compute this num-worker output-dir exception order select header output-func in-memory) ) (throw (TypeException. "Must compute on a clojask dataframe or joined dataframe")))) - (if (not= output-dir ".clojask/tmp.csv") + (cond + excel-path + (do (output/write-excel excel-path "Sheet1" (persistent! (deref ret))) + (dataframe (fn [] (read-excel excel-path "Sheet1" :stat true)) :if-header (boolean (or custom-header header)))) + (not= output-dir ".clojask/tmp.csv") (dataframe output-dir :if-header (boolean (or custom-header header))) + :else (persistent! (deref ret))))) (defn select-col diff --git a/test/clojask/regression_test.clj b/test/clojask/regression_test.clj index b17b229..07ca0a5 100644 --- a/test/clojask/regression_test.clj +++ b/test/clojask/regression_test.clj @@ -5,7 +5,8 @@ [clojure.string :as str] [clojask.dataframe :as ck] [clojask.api.aggregate :as agg] - [clojask.api.gb-aggregate :as gb]) + [clojask.api.gb-aggregate :as gb] + [clojask-io.input :refer [read-excel]]) (:import [com.clojask.exception TypeException OperationException])) (def input "test/clojask/Employees-example.csv") @@ -220,3 +221,33 @@ (ck/group-by df ["Department"]) (is (thrown? OperationException (ck/delete-col df ["Employee"]))) (is (thrown? OperationException (ck/reorder-col df ["Department"]))))) + +(defn- excel-rows [path] + (mapv vec (:data (read-excel path "Sheet1")))) + +(deftest compute-writes-xlsx + (testing "a simple compute" + (let [df (ck/dataframe input) + path (out "employees.xlsx") + result (ck/compute df 1 path)] + (is (= (ck/compute df 1 nil) (excel-rows path))) + (is (= (ck/get-col-names df) (ck/get-col-names result)) "returns a dataframe read from the file"))) + (testing "a group-by aggregate, without header" + (let [df (ck/dataframe input) + path (out "groups.xlsx")] + (ck/group-by df "Department") + (ck/aggregate df gb/max "Salary") + (ck/compute df 1 path :header false) + (is (= (set (rest (ck/compute df 1 nil))) (set (excel-rows path)))))) + (testing "an Excel input written back to xlsx, which used the input's csv writer" + (let [src (out "employees.xlsx") + path (out "employees-copy.xlsx")] + (ck/compute (ck/dataframe input) 1 src) + (ck/compute (ck/dataframe (fn [] (read-excel src "Sheet1" :stat true))) 1 path) + (is (= (excel-rows src) (excel-rows path)))))) + +(deftest compute-rejects-xls-output + (let [path (out "employees.xls")] + (spit path "keep me") + (is (thrown-with-msg? TypeException #"\.xlsx only" (ck/compute (ck/dataframe input) 1 path))) + (is (= "keep me" (slurp path)) "the existing file is not deleted")))