From 6d02e90f26065c6be463374b0400410621c3484a Mon Sep 17 00:00:00 2001 From: "(none)" <(none)> Date: Wed, 8 Mar 2023 23:02:56 +0000 Subject: [PATCH] Fix note sql queries. Add protocol buffer parsing --- package.json | 2 + tools/reports/notes/notes.js | 102 +++++++++---- tools/reports/notes/notestore.proto | 213 ++++++++++++++++++++++++++++ 3 files changed, 291 insertions(+), 26 deletions(-) create mode 100644 tools/reports/notes/notestore.proto diff --git a/package.json b/package.json index b53ab9e..a4cb7f0 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,9 @@ "commander": "^6.1.0", "fs-extra": "^4.0.3", "json2csv": "^3.11.5", + "pbf": "^3.2.1", "plist": "^2.1.0", + "protocol-buffers-schema": "^3.6.0", "sqlite3": "^4.0.0", "stat-mode": "^1.0.0", "strip-ansi": "^4.0.0" diff --git a/tools/reports/notes/notes.js b/tools/reports/notes/notes.js index 7025a91..5ab696d 100644 --- a/tools/reports/notes/notes.js +++ b/tools/reports/notes/notes.js @@ -1,3 +1,7 @@ +const fs = require('fs') +const path = require('path') +const zlib = require('zlib') +const Pbf = require('pbf') const fileHash = require('../../util/backup_filehash') const log = require('../../util/log') const apple_timestamp = require('../../util/apple_timestamp') @@ -20,10 +24,11 @@ module.exports = { output: { id: el => el.Z_PK, identifier: el => el.ZIDENTIFIER, - modified: el => (el.XFORMATTEDDATESTRING || el.XFORMATTEDDATESTRING1) + '', + created: el => el.X_FORMATTED_ZCREATIONDATE + '', + modified: el => el.X_FORMATTED_ZMODIFICATIONDATE + '', passwordProtected: el => !!el.ZISPASSWORDPROTECTED, title: el => (el.ZTITLE || el.ZTITLE1 || el.ZTITLE2 || '').trim() || null, - content: el => el.ZCONTENT || null + content: el => el.ZCONTENT || el.X_PBF_NOTE_TEXT || null } } @@ -31,11 +36,20 @@ function getAllNotes (backup) { return new Promise(async (resolve, reject) => { var newNotes - // Try iOS 10/11 query. + // Try iOS 14 query. try { - newNotes = await getNewNotesiOS10iOS11(backup) + newNotes = await getNewNotesiOS14(backup) } catch (e) { - log.verbose(`couldn't query notes as iOS10/11, trying iOS9`, e) + log.verbose(`couldn't query notes as iOS14, trying iOS10/11`, e) + } + + // If iOS 14 query fails, try iOS 10/11. + if (newNotes == null) { + try { + newNotes = await getNewNotesiOS10iOS11(backup) + } catch (e) { + log.verbose(`couldn't query notes as iOS10/11 trying iOS9`, e) + } } // If iOS 10/11 query fails, try iOS 9. @@ -68,43 +82,79 @@ function getAllNotes (backup) { }) } -function getNewNotesiOS9 (backup) { +function getOldNotes (backup) { return new Promise((resolve, reject) => { - backup.openDatabase(NOTES2_DB) + backup.openDatabase(NOTES_DB) .then(db => { - db.all(`SELECT ZICCLOUDSYNCINGOBJECT.*, ZICNOTEDATA.ZDATA as X_CONTENT_DATA, ${apple_timestamp.parse('ZCREATIONDATE')} AS XFORMATTEDDATESTRING FROM ZICCLOUDSYNCINGOBJECT LEFT JOIN ZICNOTEDATA ON ZICCLOUDSYNCINGOBJECT.ZNOTE = ZICNOTEDATA.ZNOTE`, async function (err, rows) { - if (err) reject(err) - - resolve(rows) + db.all(`SELECT *, ${apple_timestamp.parse('ZCREATIONDATE')} AS X_FORMATTED_ZCREATIONDATE, ${apple_timestamp.parse('ZMODIFICATIONDATE')} AS X_FORMATTED_ZMODIFICATIONDATE from ZNOTE LEFT JOIN ZNOTEBODY ON ZBODY = ZNOTEBODY.Z_PK`, function (err, rows) { + if (err) { + reject(err) + } else { + resolve(rows) + } }) }) .catch(reject) }) } -function getOldNotes (backup) { + +var NoteStoreProto = null; + +function decodeProtobufData (zdata) { + if (!NoteStoreProto) { + const compile = require('pbf/compile') + const schema = require('protocol-buffers-schema') + const proto = schema.parse(fs.readFileSync(path.resolve(__dirname, 'notestore.proto'))) + NoteStoreProto = compile(proto).NoteStoreProto; + } + + var note_text = null + if (zdata) { + const decompressed = zlib.gunzipSync(zdata) + if (decompressed) { + const noteData = NoteStoreProto.read(new Pbf(decompressed)) + note_text = noteData.document.note.note_text + } + } + return note_text +} + + +function getNewNotes (backup, creationDateField, modificationDateField) { return new Promise((resolve, reject) => { - backup.openDatabase(NOTES_DB) + backup.openDatabase(NOTES2_DB) .then(db => { - db.all(`SELECT *, ${apple_timestamp.parse('ZCREATIONDATE')} AS XFORMATTEDDATESTRING from ZNOTE LEFT JOIN ZNOTEBODY ON ZBODY = ZNOTEBODY.Z_PK`, function (err, rows) { - if (err) reject(err) - resolve(rows) + db.all(` +SELECT ZICCLOUDSYNCINGOBJECT.*, +ZICNOTEDATA.ZDATA as X_CONTENT_DATA, +${apple_timestamp.parse(creationDateField)} AS X_FORMATTED_ZCREATIONDATE, +${apple_timestamp.parse(modificationDateField)} AS X_FORMATTED_ZMODIFICATIONDATE +FROM ZICCLOUDSYNCINGOBJECT +LEFT JOIN ZICNOTEDATA ON ZICCLOUDSYNCINGOBJECT.Z_PK = ZICNOTEDATA.ZNOTE +`, async function (err, rows) { + if (err) { + reject(err) + } else { + rows.forEach(row => { + row.X_PBF_NOTE_TEXT = decodeProtobufData(row.X_CONTENT_DATA) + }) + resolve(rows) + } }) }) .catch(reject) }) } +function getNewNotesiOS9 (backup) { + return getNewNotes(backup, 'ZCREATIONDATE', 'ZMODIFICATIONDATE') +} + function getNewNotesiOS10iOS11 (backup) { - return new Promise((resolve, reject) => { - backup.openDatabase(NOTES2_DB) - .then(db => { - db.all(`SELECT ZICCLOUDSYNCINGOBJECT.*, ZICNOTEDATA.ZDATA as X_CONTENT_DATA, ${apple_timestamp.parse('(ZCREATIONDATE')} AS XFORMATTEDDATESTRING, ${apple_timestamp.parse('ZCREATIONDATE1')} AS XFORMATTEDDATESTRING1 FROM ZICCLOUDSYNCINGOBJECT LEFT JOIN ZICNOTEDATA ON ZICCLOUDSYNCINGOBJECT.ZNOTE = ZICNOTEDATA.ZNOTE`, function (err, rows) { - if (err) reject(err) + return getNewNotes(backup, 'ZCREATIONDATE1', 'ZMODIFICATIONDATE1') +} - resolve(rows) - }) - }) - .catch(reject) - }) +function getNewNotesiOS14 (backup) { + return getNewNotes(backup, 'ZCREATIONDATE3', 'ZMODIFICATIONDATE1') } diff --git a/tools/reports/notes/notestore.proto b/tools/reports/notes/notestore.proto new file mode 100644 index 0000000..79861ff --- /dev/null +++ b/tools/reports/notes/notestore.proto @@ -0,0 +1,213 @@ +syntax = "proto2"; + +// +// Common classes used across a few types +// + +//Represents a color +message Color { + required float red = 1; + required float green = 2; + required float blue = 3; + required float alpha = 4; +} + +// Represents an attachment (embedded object) +message AttachmentInfo { + optional string attachment_identifier = 1; + optional string type_uti = 2; +} + +// Represents a font +message Font { + optional string font_name = 1; + optional float point_size = 2; + optional int32 font_hints = 3; +} + +// Styles a "Paragraph" (any run of characters in an AttributeRun) +message ParagraphStyle { + optional int32 style_type = 1 [default = -1]; + optional int32 alignment = 2; + optional int32 indent_amount = 4; + optional Checklist checklist = 5; +} + +// Represents a checklist item +message Checklist { + required bytes uuid = 1; + required int32 done = 2; +} + +// Represents an object that has pointers to a key and a value, asserting +// somehow that the key object has to do with the value object. +message DictionaryElement { + required ObjectID key = 1; + required ObjectID value = 2; +} + +// A Dictionary holds many DictionaryElements +message Dictionary { + repeated DictionaryElement element = 1; +} + +// ObjectIDs are used to identify objects within the protobuf, offsets in an arry, or +// a simple String. +message ObjectID { + required uint64 unsigned_integer_value = 2; + required string string_value = 4; + required int32 object_index = 6; +} + +// Register Latest is used to identify the most recent version +message RegisterLatest { + required ObjectID contents = 2; +} + +// MapEntries have a key that maps to an array of key items and a value that points to an object. +message MapEntry { + required int32 key = 1; + required ObjectID value = 2; +} + +// Represents a "run" of characters that need to be styled/displayed/etc +message AttributeRun { + required int32 length = 1; + optional ParagraphStyle paragraph_style = 2; + optional Font font = 3; + optional int32 font_weight = 5; + optional int32 underlined = 6; + optional int32 strikethrough = 7; + optional int32 superscript = 8; //Sign indicates super/sub + optional string link = 9; + optional Color color = 10; + optional AttachmentInfo attachment_info = 12; +} + +// +// Classes related to the overall Note protobufs +// + +// Overarching object in a ZNOTEDATA.ZDATA blob +message NoteStoreProto { + required Document document = 2; +} + +// A Document has a Note within it. +message Document { + required int32 version = 2; + required Note note = 3; +} + +// A Note has both text, and then a lot of formatting entries. +// Other fields are present and not yet included in this proto. +message Note { + required string note_text = 2; + repeated AttributeRun attribute_run = 5; +} + +// +// Classes related to embedded objects +// + +// Represents the top level object in a ZMERGEABLEDATA cell +message MergableDataProto { + required MergableDataObject mergable_data_object = 2; +} + +// Similar to Document for Notes, this is what holds the mergeable object +message MergableDataObject { + required int32 version = 2; // Asserted to be version in https://github.com/dunhamsteve/notesutils + required MergeableDataObjectData mergeable_data_object_data = 3; +} + +// This is the mergeable data object itself and has a lot of entries that are the parts of it +// along with arrays of key, type, and UUID items, depending on type. +message MergeableDataObjectData { + repeated MergeableDataObjectEntry mergeable_data_object_entry = 3; + repeated string mergeable_data_object_key_item = 4; + repeated string mergeable_data_object_type_item = 5; + repeated bytes mergeable_data_object_uuid_item = 6; +} + +// Each entry is part of the pbject. For example, one entry might be identifying which +// UUIDs are rows, and another might hold the text of a cell. +message MergeableDataObjectEntry { + required RegisterLatest register_latest = 1; + optional List list = 5; + optional Dictionary dictionary = 6; + optional UnknownMergeableDataObjectEntryMessage unknown_message = 9; + optional Note note = 10; + optional MergeableDataObjectMap custom_map = 13; + optional OrderedSet ordered_set = 16; +} + +// This is unknown, it first was noticed in folder order analysis. +message UnknownMergeableDataObjectEntryMessage { + optional UnknownMergeableDataObjectEntryMessageEntry unknown_entry = 1; +} + +// This is unknown, it first was noticed in folder order analysis. +// "unknown_int2" is where the folder order is stored +message UnknownMergeableDataObjectEntryMessageEntry { + optional int32 unknown_int1 = 1; + optional int64 unknown_int2 = 2; +} + + +// The Object Map uses its type to identify what you are looking at and +// then a map entry to do something with that value. +message MergeableDataObjectMap { + required int32 type = 1; + repeated MapEntry map_entry = 3; +} + +// An ordered set is used to hold structural information for embedded tables +message OrderedSet { + required OrderedSetOrdering ordering = 1; + required Dictionary elements = 2; +} + + +// The ordered set ordering identifies rows and columns in embedded tables, with an array +// of the objects and contents that map lookup values to originals. +message OrderedSetOrdering { + required OrderedSetOrderingArray array = 1; + required Dictionary contents = 2; +} + +// This array holds both the text to replace and the array of UUIDs to tell what +// embedded rows and columns are. +message OrderedSetOrderingArray { + required Note contents = 1; + repeated OrderedSetOrderingArrayAttachment attachment = 2; +} + +// This array identifies the UUIDs that are embedded table rows or columns +message OrderedSetOrderingArrayAttachment { + required int32 index = 1; + required bytes uuid = 2; +} + +// A List holds details about multiple objects +message List { + repeated ListEntry list_entry = 1; +} + +// A list Entry holds details about a specific object +message ListEntry { + required ObjectID id = 2; + optional ListEntryDetails details = 3; // I dislike this naming, but don't have better information + required ListEntryDetails additional_details = 4; +} + +// List Entry Details hold another object ID and unidentified mapping +message ListEntryDetails { + optional ListEntryDetailsKey list_entry_details_key= 1; + optional ObjectID id = 2; +} + +message ListEntryDetailsKey { + required int32 list_entry_details_type_index = 1; + required int32 list_entry_details_key = 2; +}