Improvements to notes parsing

package.json

@@ -21,7 +21,9 @@
     "commander": "^6.1.0",
     "fs-extra": "^4.0.3",
     "json2csv": "^3.11.5",
+    "pbf": "^3.2.1",
     "plist": "^2.1.0",
+    "protocol-buffers-schema": "^3.6.0",
     "sqlite3": "^4.0.0",
     "stat-mode": "^1.0.0",
     "strip-ansi": "^4.0.0"

tools/reports/notes/notes.js

@@ -1,3 +1,7 @@
+const fs = require('fs')
+const path = require('path')
+const zlib = require('zlib')
+const Pbf = require('pbf')
 const fileHash = require('../../util/backup_filehash')
 const log = require('../../util/log')
 const apple_timestamp = require('../../util/apple_timestamp')
@@ -20,22 +24,32 @@ module.exports = {
   output: {
     id: el => el.Z_PK,
     identifier: el => el.ZIDENTIFIER,
-    modified: el => (el.XFORMATTEDDATESTRING || el.XFORMATTEDDATESTRING1) + '',
+    created: el => el.X_FORMATTED_ZCREATIONDATE + '',
+    modified: el => el.X_FORMATTED_ZMODIFICATIONDATE + '',
     passwordProtected: el => !!el.ZISPASSWORDPROTECTED,
     title: el => (el.ZTITLE || el.ZTITLE1 || el.ZTITLE2 || '').trim() || null,
-    content: el => el.ZCONTENT || null
+    content: el => el.ZCONTENT || el.X_PBF_NOTE_TEXT || null
   }
 }
 
 function getAllNotes (backup) {
   return new Promise(async (resolve, reject) => {
     var newNotes
 
-    // Try iOS 10/11 query.
+    // Try iOS 14 query.
     try {
-      newNotes = await getNewNotesiOS10iOS11(backup)
+      newNotes = await getNewNotesiOS14(backup)
     } catch (e) {
-      log.verbose(`couldn't query notes as iOS10/11, trying iOS9`, e)
+      log.verbose(`couldn't query notes as iOS14, trying iOS10/11`, e)
+    }
+
+    // If iOS 14 query fails, try iOS 10/11.
+    if (newNotes == null) {
+      try {
+        newNotes = await getNewNotesiOS10iOS11(backup)
+      } catch (e) {
+        log.verbose(`couldn't query notes as iOS10/11 trying iOS9`, e)
+      }
     }
 
     // If iOS 10/11 query fails, try iOS 9.
@@ -68,43 +82,79 @@ function getAllNotes (backup) {
   })
 }
 
-function getNewNotesiOS9 (backup) {
+function getOldNotes (backup) {
   return new Promise((resolve, reject) => {
-    backup.openDatabase(NOTES2_DB)
+    backup.openDatabase(NOTES_DB)
       .then(db => {
-        db.all(`SELECT ZICCLOUDSYNCINGOBJECT.*, ZICNOTEDATA.ZDATA as X_CONTENT_DATA, ${apple_timestamp.parse('ZCREATIONDATE')} AS XFORMATTEDDATESTRING FROM ZICCLOUDSYNCINGOBJECT LEFT JOIN ZICNOTEDATA ON ZICCLOUDSYNCINGOBJECT.ZNOTE = ZICNOTEDATA.ZNOTE`, async function (err, rows) {
-          if (err) reject(err)
-
-          resolve(rows)
+        db.all(`SELECT *, ${apple_timestamp.parse('ZCREATIONDATE')} AS X_FORMATTED_ZCREATIONDATE, ${apple_timestamp.parse('ZMODIFICATIONDATE')} AS X_FORMATTED_ZMODIFICATIONDATE from ZNOTE LEFT JOIN ZNOTEBODY ON ZBODY = ZNOTEBODY.Z_PK`, function (err, rows) {
+          if (err) {
+            reject(err)
+          } else {
+            resolve(rows)
+          }
         })
       })
       .catch(reject)
   })
 }
 
-function getOldNotes (backup) {
+
+var NoteStoreProto = null;
+
+function decodeProtobufData (zdata) {
+  if (!NoteStoreProto) {
+    const compile = require('pbf/compile')
+    const schema = require('protocol-buffers-schema')
+    const proto = schema.parse(fs.readFileSync(path.resolve(__dirname, 'notestore.proto')))
+    NoteStoreProto = compile(proto).NoteStoreProto;
+  }
+
+  var note_text = null
+  if (zdata) {
+    const decompressed = zlib.gunzipSync(zdata)
+    if (decompressed) {
+      const noteData = NoteStoreProto.read(new Pbf(decompressed))
+      note_text = noteData.document.note.note_text
+    }
+  }
+  return note_text
+}
+
+
+function getNewNotes (backup, creationDateField, modificationDateField) {
   return new Promise((resolve, reject) => {
-    backup.openDatabase(NOTES_DB)
+    backup.openDatabase(NOTES2_DB)
       .then(db => {
-        db.all(`SELECT *, ${apple_timestamp.parse('ZCREATIONDATE')} AS XFORMATTEDDATESTRING from ZNOTE LEFT JOIN ZNOTEBODY ON ZBODY = ZNOTEBODY.Z_PK`, function (err, rows) {
-          if (err) reject(err)
-          resolve(rows)
+        db.all(`
+SELECT ZICCLOUDSYNCINGOBJECT.*,
+ZICNOTEDATA.ZDATA as X_CONTENT_DATA,
+${apple_timestamp.parse(creationDateField)} AS X_FORMATTED_ZCREATIONDATE,
+${apple_timestamp.parse(modificationDateField)} AS X_FORMATTED_ZMODIFICATIONDATE
+FROM ZICCLOUDSYNCINGOBJECT
+LEFT JOIN ZICNOTEDATA ON ZICCLOUDSYNCINGOBJECT.Z_PK = ZICNOTEDATA.ZNOTE
+`, async function (err, rows) {
+          if (err) {
+            reject(err)
+          } else {
+            rows.forEach(row => {
+              row.X_PBF_NOTE_TEXT = decodeProtobufData(row.X_CONTENT_DATA)
+            })
+            resolve(rows)
+          }
         })
       })
       .catch(reject)
   })
 }
 
+function getNewNotesiOS9 (backup) {
+  return getNewNotes(backup, 'ZCREATIONDATE', 'ZMODIFICATIONDATE')
+}
+
 function getNewNotesiOS10iOS11 (backup) {
-  return new Promise((resolve, reject) => {
-    backup.openDatabase(NOTES2_DB)
-      .then(db => {
-        db.all(`SELECT ZICCLOUDSYNCINGOBJECT.*, ZICNOTEDATA.ZDATA as X_CONTENT_DATA, ${apple_timestamp.parse('(ZCREATIONDATE')} AS XFORMATTEDDATESTRING, ${apple_timestamp.parse('ZCREATIONDATE1')} AS XFORMATTEDDATESTRING1 FROM ZICCLOUDSYNCINGOBJECT LEFT JOIN ZICNOTEDATA ON ZICCLOUDSYNCINGOBJECT.ZNOTE = ZICNOTEDATA.ZNOTE`, function (err, rows) {
-          if (err) reject(err)
+  return getNewNotes(backup, 'ZCREATIONDATE1', 'ZMODIFICATIONDATE1')
+}
 
-          resolve(rows)
-        })
-      })
-      .catch(reject)
-  })
+function getNewNotesiOS14 (backup) {
+  return getNewNotes(backup, 'ZCREATIONDATE3', 'ZMODIFICATIONDATE1')
 }

tools/reports/notes/notestore.proto

@@ -0,0 +1,213 @@
+syntax = "proto2";
+
+//
+// Common classes used across a few types
+//
+
+//Represents a color
+message Color {
+  required float red = 1;
+  required float green = 2;
+  required float blue = 3;
+  required float alpha = 4;
+}
+
+// Represents an attachment (embedded object)
+message AttachmentInfo {
+   optional string attachment_identifier = 1;
+   optional string type_uti = 2;
+}
+
+// Represents a font
+message Font {
+   optional string font_name = 1;
+   optional float point_size = 2;
+   optional int32 font_hints = 3;
+}
+
+// Styles a "Paragraph" (any run of characters in an AttributeRun)
+message ParagraphStyle {
+    optional int32 style_type = 1 [default = -1];
+    optional int32 alignment = 2;
+    optional int32 indent_amount = 4;
+    optional Checklist checklist = 5;
+}
+
+// Represents a checklist item
+message Checklist {
+  required bytes uuid = 1;
+  required int32 done = 2;
+}
+
+// Represents an object that has pointers to a key and a value, asserting 
+// somehow that the key object has to do with the value object.
+message DictionaryElement {
+  required ObjectID key = 1;
+  required ObjectID value = 2;
+}
+
+// A Dictionary holds many DictionaryElements
+message Dictionary {
+  repeated DictionaryElement element = 1;
+}
+
+// ObjectIDs are used to identify objects within the protobuf, offsets in an arry, or 
+// a simple String.
+message ObjectID {
+  required uint64 unsigned_integer_value = 2;
+  required string string_value = 4;
+  required int32 object_index = 6;
+}
+
+// Register Latest is used to identify the most recent version
+message RegisterLatest {
+  required ObjectID contents = 2;
+}
+
+// MapEntries have a key that maps to an array of key items and a value that points to an object.
+message MapEntry {
+  required int32 key = 1;
+  required ObjectID value = 2;
+}
+
+// Represents a "run" of characters that need to be styled/displayed/etc
+message AttributeRun {
+  required int32 length = 1;
+  optional ParagraphStyle paragraph_style = 2;
+  optional Font font = 3;
+  optional int32 font_weight = 5;
+  optional int32 underlined = 6;
+  optional int32 strikethrough = 7;
+  optional int32 superscript = 8; //Sign indicates super/sub
+  optional string link = 9;
+  optional Color color = 10;
+  optional AttachmentInfo attachment_info = 12;
+}
+
+//
+// Classes related to the overall Note protobufs
+//
+
+// Overarching object in a ZNOTEDATA.ZDATA blob
+message NoteStoreProto {
+  required Document document = 2;
+}
+
+// A Document has a Note within it.
+message Document {
+  required int32 version = 2;
+  required Note note = 3;
+}
+
+// A Note has both text, and then a lot of formatting entries.
+// Other fields are present and not yet included in this proto.
+message Note {
+  required string note_text = 2;
+  repeated AttributeRun attribute_run = 5;
+}
+
+//
+// Classes related to embedded objects
+//
+
+// Represents the top level object in a ZMERGEABLEDATA cell
+message MergableDataProto {
+  required MergableDataObject mergable_data_object = 2;
+}
+
+// Similar to Document for Notes, this is what holds the mergeable object
+message MergableDataObject {
+  required int32 version = 2; // Asserted to be version in https://github.com/dunhamsteve/notesutils
+  required MergeableDataObjectData mergeable_data_object_data = 3;
+}
+
+// This is the mergeable data object itself and has a lot of entries that are the parts of it 
+// along with arrays of key, type, and UUID items, depending on type.
+message MergeableDataObjectData {
+  repeated MergeableDataObjectEntry mergeable_data_object_entry = 3;
+  repeated string mergeable_data_object_key_item = 4;
+  repeated string mergeable_data_object_type_item = 5;
+  repeated bytes mergeable_data_object_uuid_item = 6;
+}
+
+// Each entry is part of the pbject. For example, one entry might be identifying which 
+// UUIDs are rows, and another might hold the text of a cell.
+message MergeableDataObjectEntry {
+  required RegisterLatest register_latest = 1;
+  optional List list = 5;
+  optional Dictionary dictionary = 6;
+  optional UnknownMergeableDataObjectEntryMessage unknown_message = 9;
+  optional Note note = 10;
+  optional MergeableDataObjectMap custom_map = 13;
+  optional OrderedSet ordered_set = 16;
+}
+
+// This is unknown, it first was noticed in folder order analysis.
+message UnknownMergeableDataObjectEntryMessage {
+  optional UnknownMergeableDataObjectEntryMessageEntry unknown_entry = 1;
+}
+
+// This is unknown, it first was noticed in folder order analysis.
+// "unknown_int2" is where the folder order is stored
+message UnknownMergeableDataObjectEntryMessageEntry {
+  optional int32 unknown_int1 = 1;
+  optional int64 unknown_int2 = 2;
+}
+
+
+// The Object Map uses its type to identify what you are looking at and 
+// then a map entry to do something with that value.
+message MergeableDataObjectMap {
+  required int32 type = 1;
+  repeated MapEntry map_entry = 3;
+}
+
+// An ordered set is used to hold structural information for embedded tables
+message OrderedSet {
+  required OrderedSetOrdering ordering = 1;
+  required Dictionary elements = 2;
+}
+
+
+// The ordered set ordering identifies rows and columns in embedded tables, with an array 
+// of the objects and contents that map lookup values to originals.
+message OrderedSetOrdering {
+  required OrderedSetOrderingArray array = 1;
+  required Dictionary contents = 2;
+}
+
+// This array holds both the text to replace and the array of UUIDs to tell what
+// embedded rows and columns are.
+message OrderedSetOrderingArray {
+  required Note contents = 1;
+  repeated OrderedSetOrderingArrayAttachment attachment = 2;
+}
+
+// This array identifies the UUIDs that are embedded table rows or columns
+message OrderedSetOrderingArrayAttachment {
+  required int32 index = 1;
+  required bytes uuid = 2;
+}
+
+// A List holds details about multiple objects
+message List {
+  repeated ListEntry list_entry = 1;
+}
+
+// A list Entry holds details about a specific object
+message ListEntry {
+  required ObjectID id = 2;
+  optional ListEntryDetails details = 3; // I dislike this naming, but don't have better information
+  required ListEntryDetails additional_details = 4;
+}
+
+// List Entry Details hold another object ID and unidentified mapping
+message ListEntryDetails {
+  optional ListEntryDetailsKey list_entry_details_key= 1;
+  optional ObjectID id = 2;
+}
+
+message ListEntryDetailsKey {
+  required int32 list_entry_details_type_index = 1;
+  required int32 list_entry_details_key = 2;
+}