From 43d70d05545cd10efa18cb254fbd63017241f727 Mon Sep 17 00:00:00 2001 From: prismiwi2015 Date: Wed, 19 Aug 2026 13:27:26 +0200 Subject: [PATCH] Bring the stored submissions across, not just the forms Importing a form is the half of a migration somebody does immediately; the years of submissions sitting in Flamingo are the half they think of a week later. Forms now remember where they came from and how their fields map, entries import in chunks with their original dates, spam arrives as spam, and running it twice imports nothing twice. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0119U4sRRWGcTQdwYreTwpAp --- allterrain-forms.php | 29 ++ docs/hooks-reference.md | 75 +++- includes/importer-cf7.php | 259 +++++++++++- includes/importers.php | 508 ++++++++++++++++++++++- tests/phpunit/tests/importersEntries.php | 451 ++++++++++++++++++++ 5 files changed, 1308 insertions(+), 14 deletions(-) create mode 100644 tests/phpunit/tests/importersEntries.php diff --git a/allterrain-forms.php b/allterrain-forms.php index 40a01ad..2cda2f8 100644 --- a/allterrain-forms.php +++ b/allterrain-forms.php @@ -102,6 +102,35 @@ */ define( 'ATF_META_CONTEXT', '_atf_context' ); +/** + * Where an imported form came from: `{ importer, source }`. + * + * Recorded so the migration can be finished later. Importing the form is the + * urgent half and the half somebody does immediately; bringing the stored + * submissions across is the half they think of a week afterwards, by which time + * nothing else remembers which source form became which AllTerrain form. + */ +define( 'ATF_META_IMPORT_SOURCE', '_atf_import_source' ); + +/** + * Source field name => new field id, for an imported form. + * + * The importers already build this map to rewrite mail tags into merge tags; + * keeping it is what lets stored submissions be read afterwards, since every + * source keys its saved values by its own field names and nothing else can + * translate them. + */ +define( 'ATF_META_IMPORT_MAP', '_atf_import_map' ); + +/** + * The source record an imported entry came from: `{ importer, source }`. + * + * Makes importing entries repeatable: a second run skips what it already + * brought across instead of duplicating it, which matters because the natural + * response to a migration that looks incomplete is to run it again. + */ +define( 'ATF_META_ENTRY_SOURCE', '_atf_entry_source' ); + /** Design tokens for a user-created theme. */ define( 'ATF_META_TOKENS', '_atf_tokens' ); diff --git a/docs/hooks-reference.md b/docs/hooks-reference.md index a92393b..7407bdf 100644 --- a/docs/hooks-reference.md +++ b/docs/hooks-reference.md @@ -785,8 +785,8 @@ apply_filters( 'atf_importers', array $importers ); ``` The registry behind **Forms → Import**. Each entry is keyed by importer id and -holds four things: a `label`, and three callables — `available()` (does the -source plugin's data exist on this site), `forms()` (source id => title, for +holds four required things: a `label`, and three callables — `available()` (does +the source plugin's data exist on this site), `forms()` (source id => title, for the picker), and `import( $source_id )` (returns the new form id or a `WP_Error`). The built-in importers register through this same filter, exactly as a third-party source would. @@ -807,6 +807,66 @@ add_filter( 'atf_importers', function ( $importers ) { } ); ``` +#### Bringing the stored submissions too + +An importer may add two more callables. They are optional, and they come as a +pair: an importer that can count submissions but not import them would put a +number on screen with no button behind it, so offering only one of the two drops +both. + +```php +'entries' => fn( $source_id, $form_id = 0 ) => 412, +'import_entries' => 'my_forms_import_entries', // ( $source_id, $form_id, $limit ) +``` + +- **`entries( $source_id, $form_id = 0 )`** — how many stored submissions the + source holds for that form. Given a `$form_id`, report what is *still waiting* + rather than the total, so the offer disappears when the migration is finished. +- **`import_entries( $source_id, $form_id, $limit )`** — brings at most `$limit` + of them across and returns `{ imported, skipped, done, remaining }`, or a + `WP_Error`. Called repeatedly until `done`; a form that has been live for years + is the normal case, not the exceptional one. + +Store each one with **`atf_import_entry( $form_id, $args )`**, which handles the +parts every source gets wrong the same way: + +```php +atf_import_entry( $form_id, array( + 'values' => array( 'your-name' => 'Elena Ruiz' ), // keyed by SOURCE field name + 'importer' => 'my-forms', + 'record' => '4182', // the source's id for this submission + 'submitted_at' => 1741080900, // unix; the entry keeps this date, not today's + 'spam' => false, + 'ip' => '203.0.113.7', + 'user_agent' => 'Mozilla/5.0 …', +) ); +``` + +Three things it guarantees, each of which is a way migrations normally go wrong: + +- **Values are sanitised but deliberately not validated.** Validation enforces + *today's* rules — required fields, choice whitelists, bounds — against answers + given years ago under a form that may since have lost the option somebody + picked. Anything unsafe is stripped; anything merely unexpected is kept, because + a migration that dropped it would be deleting the history it was asked to move. +- **The entry keeps its original date.** A history that all arrives today is not + a history. +- **Running it twice imports nothing twice.** Each entry records + `{ importer, record }`, and a second pass skips what the first brought across — + which matters, because the natural response to a migration that looks incomplete + is to run it again. + +Reading the values requires knowing which source field became which new field. +`atf_create_imported_form()` takes that map as its fifth argument and keeps it on +the form; pass it when you import, or the submissions are unreadable afterwards. +`atf_form_import_source()` and `atf_form_import_map()` read both back. + +> **Watch the post statuses.** Both sides of this migration hide records behind +> `exclude_from_search` — Flamingo's spam status on the way in, every AllTerrain +> entry status on the way out. A query using `post_status => 'any'` means "every +> status *not* excluded from search", so it silently omits all of them. Name the +> statuses explicitly. + ### `atf_imported_schema` — Filter — *Experimental* ```php @@ -845,6 +905,17 @@ Fires after a form is imported. The source plugin's data is never modified, so importing is safe to repeat — hook here if you need to record the mapping or redirect shortcodes. +### `atf_entries_imported` — Action — *Experimental* + +```php +do_action( 'atf_entries_imported', int $form_id, array $result, array $source ); +``` + +Fires after each pass of importing stored submissions. `$result` is +`{ imported, skipped, done, remaining }`; `$source` is `{ importer, source }`. +Fires once per pass, not once per migration, so a form imported in chunks fires +it several times — check `$result['done']` for the last one. + --- ## Themes diff --git a/includes/importer-cf7.php b/includes/importer-cf7.php index b88de1c..fa50b75 100644 --- a/includes/importer-cf7.php +++ b/includes/importer-cf7.php @@ -29,10 +29,12 @@ */ function atf_register_cf7_importer( $importers ) { $importers['contact-form-7'] = array( - 'label' => __( 'Contact Form 7', 'allterrain-forms' ), - 'available' => 'atf_cf7_available', - 'forms' => 'atf_cf7_forms', - 'import' => 'atf_cf7_import', + 'label' => __( 'Contact Form 7', 'allterrain-forms' ), + 'available' => 'atf_cf7_available', + 'forms' => 'atf_cf7_forms', + 'import' => 'atf_cf7_import', + 'entries' => 'atf_cf7_entry_count', + 'import_entries' => 'atf_cf7_import_entries', ); return $importers; @@ -98,14 +100,259 @@ function atf_cf7_import( $source_id ) { return new WP_Error( 'atf_import_missing', __( 'That form no longer exists.', 'allterrain-forms' ) ); } + $template = (string) get_post_meta( $post->ID, '_form', true ); + $schema = atf_cf7_convert( - (string) get_post_meta( $post->ID, '_form', true ), + $template, (array) get_post_meta( $post->ID, '_mail', true ), (array) get_post_meta( $post->ID, '_mail_2', true ), (array) get_post_meta( $post->ID, '_messages', true ) ); - return atf_create_imported_form( $post->post_title, $schema, 'contact-form-7', (string) $post->ID ); + // Parsed a second time for its name => id map alone. The template is a few + // hundred bytes and this happens once per form ever imported; thread the map + // out of atf_cf7_convert() and every caller of it has to care about it. + $parsed = atf_cf7_parse_template( $template ); + + return atf_create_imported_form( + $post->post_title, + $schema, + 'contact-form-7', + (string) $post->ID, + $parsed['map'] + ); +} + +/** + * The Flamingo channel term id a CF7 form's messages are filed under. + * + * CF7 caches it in the form's own `_flamingo` meta on the first submission; when + * that is absent the term is looked up by slug, which is what CF7 named it after. + * + * @param WP_Post $form The CF7 form post. + * @return int Term id, or 0 when the form has no channel. + */ +function atf_cf7_channel_id( $form ) { + $meta = get_post_meta( $form->ID, '_flamingo', true ); + + if ( is_array( $meta ) && ! empty( $meta['channel'] ) ) { + return (int) $meta['channel']; + } + + $term = get_term_by( 'slug', $form->post_name, 'flamingo_inbound_channel' ); + + return $term && ! is_wp_error( $term ) ? (int) $term->term_id : 0; +} + +/** + * The post statuses a stored message worth importing can be in. + * + * Trash is excluded: a trashed message was thrown away on purpose, and a + * migration that resurrected it would undo a decision somebody made. + * + * Note that `'any'` cannot be used against these records. Flamingo registers its + * spam status with `exclude_from_search`, and `'any'` means "every status not + * excluded from search" — so `'any'` silently omits every spam message. + * + * @return string[] + */ +function atf_cf7_message_statuses() { + return array( 'publish', 'flamingo-spam' ); +} + +/** + * How many stored submissions a CF7 form has that have not been imported yet. + * + * Read straight from the posts table rather than through Flamingo, for the same + * reason the forms are: the moment somebody migrates is the moment the old + * plugins are being switched off, and a deactivated plugin registers no post + * type for `WP_Query` to find. + * + * @param string $source_id The CF7 post id. + * @param int $form_id The AllTerrain form the messages would land on. + * @return int + */ +function atf_cf7_entry_count( $source_id, $form_id = 0 ) { + global $wpdb; + + $form = get_post( absint( $source_id ) ); + + if ( ! $form || 'wpcf7_contact_form' !== $form->post_type ) { + return 0; + } + + $channel_id = atf_cf7_channel_id( $form ); + + if ( ! $channel_id ) { + return 0; + } + + $statuses = atf_cf7_message_statuses(); + $placeholders = implode( ', ', array_fill( 0, count( $statuses ), '%s' ) ); + + // phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- $placeholders is a generated list of %s. + $total = (int) $wpdb->get_var( + $wpdb->prepare( + "SELECT COUNT(*) FROM {$wpdb->posts} p + INNER JOIN {$wpdb->term_relationships} tr ON tr.object_id = p.ID + INNER JOIN {$wpdb->term_taxonomy} tt ON tt.term_taxonomy_id = tr.term_taxonomy_id + WHERE p.post_type = 'flamingo_inbound' + AND p.post_status IN ( {$placeholders} ) + AND tt.taxonomy = 'flamingo_inbound_channel' + AND tt.term_id = %d", + array_merge( $statuses, array( $channel_id ) ) + ) + ); + // phpcs:enable WordPress.DB.PreparedSQL.InterpolatedNotPrepared + + if ( ! $form_id ) { + return $total; + } + + return max( 0, $total - count( atf_imported_entry_keys( (int) $form_id ) ) ); +} + +/** + * Brings a slice of a CF7 form's stored submissions across as entries. + * + * Oldest first, so an interrupted migration leaves a contiguous history rather + * than a scatter, and so a second pass resumes where the first stopped. + * + * @param string $source_id The CF7 post id. + * @param int $form_id The AllTerrain form to import onto. + * @param int $limit How many to attempt in this pass. + * @return array|WP_Error { imported, skipped, done, remaining }. + */ +function atf_cf7_import_entries( $source_id, $form_id, $limit = 100 ) { + global $wpdb; + + $form = get_post( absint( $source_id ) ); + + if ( ! $form || 'wpcf7_contact_form' !== $form->post_type ) { + return new WP_Error( 'atf_import_missing', __( 'That form no longer exists.', 'allterrain-forms' ) ); + } + + $channel_id = atf_cf7_channel_id( $form ); + + if ( ! $channel_id ) { + return array( + 'imported' => 0, + 'skipped' => 0, + 'done' => true, + 'remaining' => 0, + ); + } + + $seen = atf_imported_entry_keys( (int) $form_id ); + $statuses = atf_cf7_message_statuses(); + $placeholders = implode( ', ', array_fill( 0, count( $statuses ), '%s' ) ); + + // Fetched a page at a time, and the page is larger than the limit because + // records already imported are skipped without costing a slot -- otherwise a + // second pass would spend its whole budget rediscovering the first pass. + // phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- $placeholders is a generated list of %s. + $rows = $wpdb->get_results( + $wpdb->prepare( + "SELECT p.ID, p.post_status, p.post_date_gmt FROM {$wpdb->posts} p + INNER JOIN {$wpdb->term_relationships} tr ON tr.object_id = p.ID + INNER JOIN {$wpdb->term_taxonomy} tt ON tt.term_taxonomy_id = tr.term_taxonomy_id + WHERE p.post_type = 'flamingo_inbound' + AND p.post_status IN ( {$placeholders} ) + AND tt.taxonomy = 'flamingo_inbound_channel' + AND tt.term_id = %d + ORDER BY p.post_date_gmt ASC, p.ID ASC + LIMIT %d", + array_merge( $statuses, array( $channel_id, count( $seen ) + (int) $limit ) ) + ) + ); + // phpcs:enable WordPress.DB.PreparedSQL.InterpolatedNotPrepared + + $imported = 0; + $skipped = 0; + + foreach ( (array) $rows as $row ) { + if ( $imported >= (int) $limit ) { + break; + } + + if ( isset( $seen[ atf_entry_source_key( 'contact-form-7', $row->ID ) ] ) ) { + ++$skipped; + continue; + } + + $result = atf_import_entry( + (int) $form_id, + array( + 'values' => atf_cf7_message_values( (int) $row->ID ), + 'importer' => 'contact-form-7', + 'record' => (string) $row->ID, + 'submitted_at' => (int) strtotime( $row->post_date_gmt . ' UTC' ), + 'spam' => 'flamingo-spam' === $row->post_status, + 'ip' => atf_cf7_message_meta( (int) $row->ID, 'remote_ip' ), + 'user_agent' => atf_cf7_message_meta( (int) $row->ID, 'user_agent' ), + ) + ); + + if ( is_wp_error( $result ) ) { + // A single unreadable record must not strand the rest of the + // migration, but an error that would repeat on every one of them -- + // a missing schema or field map -- is worth stopping for. + if ( in_array( $result->get_error_code(), array( 'atf_no_schema', 'atf_no_import_map' ), true ) ) { + return $result; + } + + ++$skipped; + continue; + } + + ++$imported; + } + + $remaining = atf_cf7_entry_count( $source_id, (int) $form_id ); + + return array( + 'imported' => $imported, + 'skipped' => $skipped, + 'done' => 0 === $remaining, + 'remaining' => $remaining, + ); +} + +/** + * The submitted values of one stored message, keyed by CF7 field name. + * + * Read from the per-field `_field_` rows rather than the `_fields` array, + * because Flamingo nulls every value inside `_fields` as it writes the per-field + * rows — the array survives as a list of names with nothing in it. + * + * @param int $message_id The `flamingo_inbound` post id. + * @return array Field name => value. + */ +function atf_cf7_message_values( $message_id ) { + $values = array(); + + foreach ( get_post_meta( $message_id ) as $key => $raw ) { + if ( 0 !== strpos( $key, '_field_' ) ) { + continue; + } + + $values[ substr( $key, 7 ) ] = maybe_unserialize( $raw[0] ); + } + + return $values; +} + +/** + * One of the special mail tags CF7 stores alongside a message. + * + * @param int $message_id The `flamingo_inbound` post id. + * @param string $key The tag name, e.g. `remote_ip`. + * @return string + */ +function atf_cf7_message_meta( $message_id, $key ) { + $meta = get_post_meta( $message_id, '_meta', true ); + + return is_array( $meta ) && isset( $meta[ $key ] ) ? (string) $meta[ $key ] : ''; } /** diff --git a/includes/importers.php b/includes/importers.php index cdb5595..3a62a83 100644 --- a/includes/importers.php +++ b/includes/importers.php @@ -28,11 +28,16 @@ * @since 0.2.0 * * @return array[] Importer id => { - * @type string $label Human name of the source plugin. - * @type callable $available Whether any of the source's data exists. - * @type callable $forms Source form id => title, for the picker. - * @type callable $import Imports one source form; returns the new - * form id or a WP_Error. + * @type string $label Human name of the source plugin. + * @type callable $available Whether any of the source's data exists. + * @type callable $forms Source form id => title, for the picker. + * @type callable $import Imports one source form; returns the new + * form id or a WP_Error. + * @type callable $entries Optional. How many stored submissions the + * source holds for one source form. + * @type callable $import_entries Optional. Imports a slice of them onto an + * AllTerrain form; returns { imported, + * skipped, done } or a WP_Error. * } */ function atf_importers() { @@ -61,6 +66,22 @@ function atf_importers() { continue; } + // Entry migration is optional, and half of it is useless: an importer + // that can count stored submissions but not import them would offer a + // number with no button behind it, and one that can import without + // counting cannot be asked how much work it has. Either both callbacks + // are present and callable, or the importer simply does forms. + foreach ( array( 'entries', 'import_entries' ) as $key ) { + if ( isset( $importer[ $key ] ) && ! is_callable( $importer[ $key ] ) ) { + unset( $importer['entries'], $importer['import_entries'] ); + break; + } + } + + if ( ! isset( $importer['entries'], $importer['import_entries'] ) ) { + unset( $importer['entries'], $importer['import_entries'] ); + } + $valid[ $id ] = $importer; } @@ -136,9 +157,13 @@ function atf_import_source_form( $importer_id, $source_id ) { * @param array $schema The converted schema, pre-normalisation. * @param string $importer_id Which importer produced it. * @param string $source_id The source plugin's id for the form. + * @param array $map Optional. Source field name => new field id. Kept + * on the form so its stored submissions can be read + * later; without it they are just values under names + * nothing here recognises. * @return int|WP_Error The new form's id. */ -function atf_create_imported_form( $title, $schema, $importer_id, $source_id ) { +function atf_create_imported_form( $title, $schema, $importer_id, $source_id, $map = array() ) { /** * Filters an imported schema before it is saved. * @@ -166,9 +191,330 @@ function atf_create_imported_form( $title, $schema, $importer_id, $source_id ) { atf_save_form_schema( $form_id, atf_normalize_schema( $schema ) ); + update_post_meta( + $form_id, + ATF_META_IMPORT_SOURCE, + array( + 'importer' => sanitize_key( $importer_id ), + 'source' => (string) $source_id, + ) + ); + + if ( $map ) { + update_post_meta( $form_id, ATF_META_IMPORT_MAP, array_map( 'strval', (array) $map ) ); + } + return $form_id; } +/** + * What an imported form was imported from. + * + * @param int $form_id The form. + * @return array|null { importer: string, source: string }, or null when the + * form was not imported. + */ +function atf_form_import_source( $form_id ) { + $source = get_post_meta( (int) $form_id, ATF_META_IMPORT_SOURCE, true ); + + if ( ! is_array( $source ) || empty( $source['importer'] ) || ! isset( $source['source'] ) ) { + return null; + } + + return array( + 'importer' => (string) $source['importer'], + 'source' => (string) $source['source'], + ); +} + +/** + * The source-field-name => field-id map recorded when a form was imported. + * + * @param int $form_id The form. + * @return array Source field name => field id. Empty when unknown. + */ +function atf_form_import_map( $form_id ) { + $map = get_post_meta( (int) $form_id, ATF_META_IMPORT_MAP, true ); + + return is_array( $map ) ? $map : array(); +} + +/** + * The forms that came from a source and could still have submissions to bring. + * + * Keyed by form id so the caller can offer each one by name. + * + * @return array[] Form id => { importer, source, label, count }. + */ +function atf_forms_with_importable_entries() { + $importers = atf_importers(); + $forms = get_posts( + array( + 'post_type' => ATF_FORM_TYPE, + 'post_status' => 'any', + 'posts_per_page' => -1, + 'fields' => 'ids', + 'meta_key' => ATF_META_IMPORT_SOURCE, + ) + ); + + $found = array(); + + foreach ( $forms as $form_id ) { + $source = atf_form_import_source( $form_id ); + + if ( ! $source || ! isset( $importers[ $source['importer'] ] ) ) { + continue; + } + + $importer = $importers[ $source['importer'] ]; + + if ( ! isset( $importer['entries'] ) || ! call_user_func( $importer['available'] ) ) { + continue; + } + + $count = (int) call_user_func( $importer['entries'], $source['source'], (int) $form_id ); + + if ( $count > 0 ) { + $found[ (int) $form_id ] = array( + 'importer' => $source['importer'], + 'source' => $source['source'], + 'label' => $importer['label'], + 'count' => $count, + ); + } + } + + return $found; +} + +/** + * The key that identifies one imported entry against its source record. + * + * A plain string rather than the `{ importer, source }` array used on forms, + * because this one is queried — `meta_value` matching an array means matching a + * serialised blob, which works until the day a value inside it changes shape. + * + * @param string $importer_id Which importer. + * @param string $record_id The source plugin's id for the stored submission. + * @return string + */ +function atf_entry_source_key( $importer_id, $record_id ) { + return sanitize_key( $importer_id ) . ':' . (string) $record_id; +} + +/** + * The source keys already imported onto a form. + * + * Read once per pass and held in memory: asking per record would be a query per + * record, and the answer cannot change underneath a single pass. + * + * @param int $form_id The form. + * @return array Source key => true. + */ +function atf_imported_entry_keys( $form_id ) { + global $wpdb; + + $keys = $wpdb->get_col( + $wpdb->prepare( + "SELECT meta_value FROM {$wpdb->postmeta} WHERE meta_key = %s AND post_id IN ( + SELECT post_id FROM {$wpdb->postmeta} WHERE meta_key = %s AND meta_value = %d + )", + ATF_META_ENTRY_SOURCE, + ATF_META_FORM, + (int) $form_id + ) + ); + + return array_fill_keys( (array) $keys, true ); +} + +/** + * Stores one migrated submission as an entry. + * + * # Sanitised, deliberately not validated + * + * Values go through `atf_sanitize_submission()`, which coerces each one to the + * shape its field type stores. They do **not** go through validation, and that + * is the important half: validation enforces today's rules — required fields, + * choice whitelists, bounds — against answers given years ago, under a form that + * may since have lost the option somebody picked. A migration that dropped those + * answers would be quietly deleting history it was asked to preserve. Anything + * unsafe is removed by sanitising; anything merely *unexpected* is kept. + * + * # The date is the submission's, not today's + * + * `atf_store_entry()` stamps the current time and the current request's IP and + * user agent, which is right for a real submission and wrong for every one of + * these. Both are corrected afterwards rather than by duplicating that function, + * so entry titles, value encoding and file parenting stay in one place. + * + * @param int $form_id The form to store against. + * @param array $args { + * The submission, as the source stored it. + * + * @type array $values Source field name => stored value. + * @type string $importer Importer id. + * @type string $record The source plugin's id for this submission. + * @type int $submitted_at Unix time of the original submission. + * @type bool $spam Whether the source marked it as spam. + * @type string $ip Original remote address, if the source kept one. + * @type string $user_agent Original user agent, if the source kept one. + * } + * @return int|WP_Error The entry id, or an error. + */ +function atf_import_entry( $form_id, array $args ) { + $args = wp_parse_args( + $args, + array( + 'values' => array(), + 'importer' => '', + 'record' => '', + 'submitted_at' => 0, + 'spam' => false, + 'ip' => '', + 'user_agent' => '', + ) + ); + + $form_id = (int) $form_id; + $schema = atf_get_form_schema( $form_id ); + + if ( ! $schema ) { + return new WP_Error( 'atf_no_schema', __( 'That form has no schema.', 'allterrain-forms' ) ); + } + + $map = atf_form_import_map( $form_id ); + + if ( ! $map ) { + return new WP_Error( + 'atf_no_import_map', + __( 'That form has no field map, so its stored submissions cannot be read. Import the form again to record one.', 'allterrain-forms' ) + ); + } + + // Source names to field ids. A source field the form no longer has — a tag + // deleted before the migration, or one the importer had no equivalent for — + // simply has nowhere to go. + $raw = array(); + + foreach ( (array) $args['values'] as $name => $value ) { + if ( isset( $map[ $name ] ) ) { + $raw[ $map[ $name ] ] = $value; + } + } + + $values = atf_sanitize_submission( $schema, $raw ); + + $entry_id = atf_store_entry( + $form_id, + $schema, + $values, + array( + 'spam' => (bool) $args['spam'], + 'reason' => $args['spam'] ? 'imported' : '', + ) + ); + + if ( is_wp_error( $entry_id ) ) { + return $entry_id; + } + + $submitted_at = (int) $args['submitted_at']; + + if ( $submitted_at > 0 ) { + $gmt = gmdate( 'Y-m-d H:i:s', $submitted_at ); + + wp_update_post( + array( + 'ID' => $entry_id, + 'post_date' => get_date_from_gmt( $gmt ), + 'post_date_gmt' => $gmt, + ) + ); + } + + $context = json_decode( (string) get_post_meta( $entry_id, ATF_META_CONTEXT, true ), true ); + $context = is_array( $context ) ? $context : array(); + + $context['ip'] = (string) $args['ip']; + $context['userAgent'] = (string) $args['user_agent']; + $context['referrer'] = ''; + $context['userId'] = 0; + $context['imported'] = sanitize_key( $args['importer'] ); + + if ( $submitted_at > 0 ) { + $context['submitted'] = gmdate( 'Y-m-d H:i:s', $submitted_at ); + } + + update_post_meta( $entry_id, ATF_META_CONTEXT, wp_slash( wp_json_encode( $context ) ) ); + update_post_meta( $entry_id, ATF_META_ENTRY_SOURCE, atf_entry_source_key( $args['importer'], $args['record'] ) ); + + return $entry_id; +} + +/** + * Brings a slice of one source form's stored submissions across. + * + * Sliced rather than done in one pass because a form that has been live for + * years is the normal case, not the exceptional one, and the request that + * imports it is an ordinary admin POST with an ordinary time limit. + * + * @param int $form_id The AllTerrain form to import onto. + * @param int $limit How many to attempt in this pass. + * @return array|WP_Error { imported, skipped, done, remaining }. + */ +function atf_import_form_entries( $form_id, $limit = 100 ) { + if ( ! atf_can_edit_forms() ) { + return new WP_Error( 'atf_forbidden', __( 'You cannot import submissions.', 'allterrain-forms' ), array( 'status' => 403 ) ); + } + + $form_id = (int) $form_id; + $source = atf_form_import_source( $form_id ); + + if ( ! $source ) { + return new WP_Error( 'atf_not_imported', __( 'That form was not imported, so it has no submissions to bring across.', 'allterrain-forms' ) ); + } + + $importers = atf_importers(); + + if ( ! isset( $importers[ $source['importer'] ]['import_entries'] ) ) { + return new WP_Error( 'atf_no_entry_import', __( 'That source cannot import stored submissions.', 'allterrain-forms' ) ); + } + + $result = call_user_func( + $importers[ $source['importer'] ]['import_entries'], + $source['source'], + $form_id, + max( 1, (int) $limit ) + ); + + if ( is_wp_error( $result ) ) { + return $result; + } + + $result = wp_parse_args( + (array) $result, + array( + 'imported' => 0, + 'skipped' => 0, + 'done' => true, + 'remaining' => 0, + ) + ); + + /** + * Fires after a pass of entry importing. + * + * @param int $form_id The form imported onto. + * @param array $result Counts: imported, skipped, done, remaining. + * @param array $source Where it came from: importer, source. + */ + do_action( 'atf_entries_imported', $form_id, $result, $source ); + + return $result; +} + /** * How many forms each available importer can see, keyed by importer id. * @@ -360,7 +706,46 @@ function atf_render_import_page() { ); } + // The outcome of an entry-import pass. Display only, same as above. + // phpcs:disable WordPress.Security.NonceVerification.Recommended + $entries = isset( $_GET['atf_entries'] ) ? absint( $_GET['atf_entries'] ) : 0; + $remaining = isset( $_GET['atf_remaining'] ) ? absint( $_GET['atf_remaining'] ) : 0; + $entry_error = isset( $_GET['atf_entry_error'] ) ? sanitize_key( wp_unslash( $_GET['atf_entry_error'] ) ) : ''; + $showed_entry = isset( $_GET['atf_entries'] ) || '' !== $entry_error; + // phpcs:enable WordPress.Security.NonceVerification.Recommended + + if ( '' !== $entry_error ) { + printf( + '

%s

', + esc_html__( 'Those submissions could not be brought across. Import the form again to record the field map, then try once more.', 'allterrain-forms' ) + ); + } elseif ( $showed_entry ) { + printf( + '

%2$s%3$s

', + $remaining ? 'info' : 'success', + esc_html( + sprintf( + /* translators: %d: number of submissions imported. */ + _n( '%d submission brought across.', '%d submissions brought across.', $entries, 'allterrain-forms' ), + $entries + ) + ), + esc_html( + $remaining ? ' ' . sprintf( + /* translators: %d: number still waiting. */ + _n( '%d still waiting — run it again.', '%d still waiting — run it again.', $remaining, 'allterrain-forms' ), + $remaining + ) : '' + ) + ); + } + if ( ! $available ) { + // Sources can vanish while their already-imported forms still have + // submissions waiting -- deleting CF7 leaves every Flamingo message + // exactly where it was -- so this section comes before the early return. + atf_render_entry_import_section(); + printf( '

%s

', esc_html__( 'No importable forms were found. Importers look for data from Contact Form 7, WPForms and Gravity Forms — the data is read even when the source plugin has been deactivated.', 'allterrain-forms' ) @@ -400,6 +785,12 @@ function atf_render_import_page() { ); } + // Forms already imported that still have submissions sitting in the old + // plugin. Offered above the pickers, because somebody who has just imported + // their forms is looking at an entries screen that says nothing yet, and + // this is the answer to why. + atf_render_entry_import_section(); + foreach ( $available as $id => $importer ) { $forms = call_user_func( $importer['forms'] ); @@ -458,6 +849,111 @@ function atf_render_import_page() { echo ''; } +/** + * Renders the "bring the submissions too" section of the Import page. + * + * Prints nothing at all when there is nothing to bring, so a site whose sources + * never stored anything — a plain CF7 install with no Flamingo — sees the page + * exactly as it was. + * + * @return void + */ +function atf_render_entry_import_section() { + $forms = atf_forms_with_importable_entries(); + + if ( ! $forms ) { + return; + } + + $total = 0; + + foreach ( $forms as $form ) { + $total += (int) $form['count']; + } + + printf( '

%s

', esc_html__( 'Stored submissions', 'allterrain-forms' ) ); + + printf( + '

%s

', + esc_html( + sprintf( + /* translators: %d: number of stored submissions found. */ + _n( + '%d stored submission is still held by the plugin it was submitted through. Bringing it across creates an entry with its original date; the original is left untouched.', + '%d stored submissions are still held by the plugins they were submitted through. Bringing them across creates entries with their original dates; the originals are left untouched.', + $total, + 'allterrain-forms' + ), + $total + ) + ) + ); + + echo ''; + + foreach ( $forms as $form_id => $form ) { + printf( + '', + esc_html( get_the_title( $form_id ) ), + esc_html( + sprintf( + /* translators: 1: source plugin name, 2: number of submissions. */ + __( 'from %1$s — %2$d waiting', 'allterrain-forms' ), + $form['label'], + (int) $form['count'] + ) + ), + esc_url( admin_url( 'admin-post.php' ) ), + (int) $form_id, + wp_nonce_field( 'atf-import', '_atf_nonce', true, false ), // phpcs:ignore WordPress.Security.EscapeOutput.OutputNotEscaped -- wp_nonce_field() builds escaped markup. + esc_html__( 'Bring submissions', 'allterrain-forms' ) + ); + } + + echo '
%1$s
%2$s		 +
+ + + %5$s + +
+
'; +} + +/** + * Handles the "bring the submissions too" POST. + * + * One pass per submit, and the page it returns to offers the next one. A form + * with tens of thousands of stored messages therefore takes several clicks + * rather than one request that times out halfway and leaves nobody sure how far + * it got. + * + * @return void + */ +function atf_handle_import_entries_post() { + if ( ! atf_can_edit_forms() ) { + wp_die( esc_html__( 'You cannot import submissions.', 'allterrain-forms' ) ); + } + + check_admin_referer( 'atf-import', '_atf_nonce' ); + + $form_id = isset( $_POST['form'] ) ? absint( wp_unslash( $_POST['form'] ) ) : 0; + $result = atf_import_form_entries( $form_id, 200 ); + + $args = array( 'page' => 'allterrain-forms-import' ); + + if ( is_wp_error( $result ) ) { + $args['atf_entry_error'] = $result->get_error_code(); + } else { + $args['atf_entries'] = (int) $result['imported']; + $args['atf_remaining'] = (int) $result['remaining']; + } + + wp_safe_redirect( add_query_arg( $args, admin_url( 'admin.php' ) ) ); + exit; +} +add_action( 'admin_post_atf_import_entries', 'atf_handle_import_entries_post' ); + /** * Handles the Import page's POST. * diff --git a/tests/phpunit/tests/importersEntries.php b/tests/phpunit/tests/importersEntries.php new file mode 100644 index 0000000..f22477c --- /dev/null +++ b/tests/phpunit/tests/importersEntries.php @@ -0,0 +1,451 @@ +` rows, a channel term, spam under Flamingo's own post status — + * because that storage is the only contract available once Contact Form 7 and + * Flamingo are switched off, which is precisely when somebody migrates. + * + * @package AllTerrain_Forms + * @group allterrain-forms + */ + +/** + * Entry migration: the registry contract, the CF7/Flamingo reader, and the rules + * about dates, spam and repeat runs. + * + * @group allterrain-forms + */ +class ATF_Test_Importers_Entries extends WP_UnitTestCase { + + /** + * A CF7 template with one field of each shape the messages below use. + * + * @var string + */ + const TEMPLATE = ' + + + + + + + +[submit "Send"]'; + + /** + * Every status an imported entry can land in. + * + * Spelled out because `'any'` would find none of them: entry statuses are + * registered `exclude_from_search`, and `'any'` means "every status not + * excluded from search". It is the same blind spot that hides Flamingo's + * spam status from a query on the other side of this migration. + * + * @var string[] + */ + const ENTRY_STATUSES = array( 'atf-unread', 'atf-read', 'atf-spam' ); + + /** + * The CF7 form post id. + * + * @var int + */ + protected $cf7_id = 0; + + /** + * The Flamingo channel term id. + * + * @var int + */ + protected $channel_id = 0; + + /** + * Registers what Flamingo and CF7 would have registered, then builds a form. + */ + public function set_up() { + parent::set_up(); + + // Both post types belong to plugins that are not loaded in the test + // suite. Registering them here is not a convenience: the importer is + // specifically built to read this data with those plugins gone, and the + // posts and terms are what survive them. + register_post_type( 'wpcf7_contact_form', array( 'public' => false ) ); + register_post_type( 'flamingo_inbound', array( 'public' => false ) ); + register_post_status( 'flamingo-spam', array( 'exclude_from_search' => true ) ); + register_taxonomy( 'flamingo_inbound_channel', 'flamingo_inbound' ); + + $this->cf7_id = self::factory()->post->create( + array( + 'post_type' => 'wpcf7_contact_form', + 'post_title' => 'Property Enquiry', + 'post_name' => 'property-enquiry', + ) + ); + + update_post_meta( $this->cf7_id, '_form', self::TEMPLATE ); + + $term = wp_insert_term( 'Property Enquiry', 'flamingo_inbound_channel', array( 'slug' => 'property-enquiry' ) ); + $this->channel_id = (int) $term['term_id']; + + update_post_meta( $this->cf7_id, '_flamingo', array( 'channel' => $this->channel_id ) ); + + // The plugin's capabilities are granted on activation, which does not + // run in the suite -- an administrator has none of them until this does. + atf_add_capabilities(); + wp_set_current_user( self::factory()->user->create( array( 'role' => 'administrator' ) ) ); + } + + /** + * Writes one stored message the way Flamingo does. + * + * @param array $fields Field name => value. + * @param string $date GMT date, `Y-m-d H:i:s`. + * @param bool $spam Whether it is spam. + * @return int The message post id. + */ + protected function add_message( $fields, $date, $spam = false ) { + $id = self::factory()->post->create( + array( + 'post_type' => 'flamingo_inbound', + 'post_status' => $spam ? 'flamingo-spam' : 'publish', + 'post_title' => 'Message', + 'post_date_gmt' => $date, + 'post_date' => $date, + ) + ); + + foreach ( $fields as $name => $value ) { + update_post_meta( $id, sanitize_key( '_field_' . $name ), $value ); + } + + // Flamingo nulls the values inside `_fields` as it writes the per-field + // rows. Reproduced because an importer that read this array instead + // would find names with nothing under them and import blank entries. + update_post_meta( $id, '_fields', array_map( '__return_null', $fields ) ); + update_post_meta( + $id, + '_meta', + array( + 'remote_ip' => '203.0.113.7', + 'user_agent' => 'Mozilla/5.0 (Test)', + ) + ); + + wp_set_object_terms( $id, array( $this->channel_id ), 'flamingo_inbound_channel' ); + + return $id; + } + + /** + * Imports the CF7 form and returns the new form's id. + * + * @return int + */ + protected function import_form() { + return atf_import_source_form( 'contact-form-7', (string) $this->cf7_id ); + } + + /** + * An importer offering only one half of the entry contract gets neither. + * + * A count with no import behind it would put a number on the screen that no + * button could act on. + * + * @covers ::atf_importers + */ + public function test_half_an_entry_contract_is_dropped() { + $filter = static function ( $importers ) { + $importers['halfway'] = array( + 'label' => 'Halfway', + 'available' => '__return_true', + 'forms' => '__return_empty_array', + 'import' => '__return_zero', + 'entries' => '__return_zero', + ); + + return $importers; + }; + + add_filter( 'atf_importers', $filter ); + $importers = atf_importers(); + remove_filter( 'atf_importers', $filter ); + + $this->assertArrayHasKey( 'halfway', $importers, 'The importer itself is still valid.' ); + $this->assertArrayNotHasKey( 'entries', $importers['halfway'] ); + $this->assertArrayNotHasKey( 'import_entries', $importers['halfway'] ); + } + + /** + * Importing a form records where it came from and how its fields map. + * + * Without both, its stored submissions are unreadable later. + * + * @covers ::atf_create_imported_form + * @covers ::atf_form_import_source + * @covers ::atf_form_import_map + */ + public function test_import_records_source_and_map() { + $form_id = $this->import_form(); + + $this->assertNotWPError( $form_id ); + + $source = atf_form_import_source( $form_id ); + + $this->assertSame( 'contact-form-7', $source['importer'] ); + $this->assertSame( (string) $this->cf7_id, $source['source'] ); + + $map = atf_form_import_map( $form_id ); + + $this->assertArrayHasKey( 'your-name', $map ); + $this->assertArrayHasKey( 'bid-funding', $map ); + } + + /** + * Stored messages become entries, with their values under the new field ids. + * + * @covers ::atf_cf7_import_entries + * @covers ::atf_import_entry + */ + public function test_messages_become_entries() { + $this->add_message( + array( + 'your-name' => 'Elena Ruiz', + 'your-email' => 'elena.ruiz@example.com', + 'bid-funding' => 'Cash buyer', + 'your-message' => 'Is the terrace south facing?', + ), + '2025-03-04 09:15:00' + ); + + $form_id = $this->import_form(); + $result = atf_import_form_entries( $form_id ); + + $this->assertSame( 1, $result['imported'] ); + $this->assertTrue( $result['done'] ); + + $entries = get_posts( + array( + 'post_type' => ATF_ENTRY_TYPE, + 'post_status' => self::ENTRY_STATUSES, + 'posts_per_page' => -1, + 'meta_key' => ATF_META_FORM, + 'meta_value' => $form_id, + ) + ); + + $this->assertCount( 1, $entries ); + + $map = atf_form_import_map( $form_id ); + $values = json_decode( get_post_meta( $entries[0]->ID, ATF_META_VALUES, true ), true ); + + $this->assertSame( 'Elena Ruiz', $values[ $map['your-name'] ] ); + $this->assertSame( 'Is the terrace south facing?', $values[ $map['your-message'] ] ); + } + + /** + * An entry keeps the date it was submitted, not the date it was migrated. + * + * A history that all arrives today is not a history. + * + * @covers ::atf_import_entry + */ + public function test_original_date_is_kept() { + $this->add_message( array( 'your-name' => 'Marcus Hale' ), '2024-11-19 14:02:00' ); + + $form_id = $this->import_form(); + atf_import_form_entries( $form_id ); + + $entries = get_posts( + array( + 'post_type' => ATF_ENTRY_TYPE, + 'post_status' => self::ENTRY_STATUSES, + 'posts_per_page' => -1, + 'meta_key' => ATF_META_FORM, + 'meta_value' => $form_id, + ) + ); + + $this->assertSame( '2024-11-19 14:02:00', $entries[0]->post_date_gmt ); + + $context = json_decode( get_post_meta( $entries[0]->ID, ATF_META_CONTEXT, true ), true ); + + $this->assertSame( '203.0.113.7', $context['ip'] ); + $this->assertSame( 'contact-form-7', $context['imported'] ); + } + + /** + * Spam arrives as spam rather than landing in the unread pile. + * + * Flamingo files spam under a status registered `exclude_from_search`, which + * is exactly the status a `post_status => 'any'` query silently omits — so + * this also pins that the reader does not use one. + * + * @covers ::atf_cf7_import_entries + */ + public function test_spam_is_imported_as_spam() { + $this->add_message( array( 'your-name' => 'SEO Growth Team' ), '2025-01-02 08:00:00', true ); + $this->add_message( array( 'your-name' => 'Priya Nandra' ), '2025-01-03 08:00:00' ); + + $form_id = $this->import_form(); + $result = atf_import_form_entries( $form_id ); + + $this->assertSame( 2, $result['imported'], 'The spam message was found as well as the published one.' ); + + $spam = get_posts( + array( + 'post_type' => ATF_ENTRY_TYPE, + 'post_status' => ATF_STATUS_SPAM, + 'posts_per_page' => -1, + 'meta_key' => ATF_META_FORM, + 'meta_value' => $form_id, + ) + ); + + $this->assertCount( 1, $spam ); + } + + /** + * Running it twice does not import anything twice. + * + * The natural response to a migration that looks incomplete is to run it + * again, so it has to be safe to. + * + * @covers ::atf_cf7_import_entries + */ + public function test_second_run_imports_nothing_new() { + $this->add_message( array( 'your-name' => 'Tomas Berg' ), '2025-02-01 10:00:00' ); + $this->add_message( array( 'your-name' => 'Sofia Marchetti' ), '2025-02-02 10:00:00' ); + + $form_id = $this->import_form(); + + $first = atf_import_form_entries( $form_id ); + $this->assertSame( 2, $first['imported'] ); + + $second = atf_import_form_entries( $form_id ); + $this->assertSame( 0, $second['imported'] ); + $this->assertSame( 2, $second['skipped'] ); + + $entries = get_posts( + array( + 'post_type' => ATF_ENTRY_TYPE, + 'post_status' => self::ENTRY_STATUSES, + 'posts_per_page' => -1, + 'meta_key' => ATF_META_FORM, + 'meta_value' => $form_id, + ) + ); + + $this->assertCount( 2, $entries ); + } + + /** + * A pass stops at the limit and says how much is left. + * + * @covers ::atf_cf7_import_entries + */ + public function test_import_is_chunked() { + for ( $i = 0; $i < 5; $i++ ) { + $this->add_message( array( 'your-name' => 'Person ' . $i ), sprintf( '2025-04-0%d 09:00:00', $i + 1 ) ); + } + + $form_id = $this->import_form(); + $result = atf_import_form_entries( $form_id, 2 ); + + $this->assertSame( 2, $result['imported'] ); + $this->assertFalse( $result['done'] ); + $this->assertSame( 3, $result['remaining'] ); + } + + /** + * The count reports what is still waiting, and reaches zero when done. + * + * @covers ::atf_cf7_entry_count + * @covers ::atf_forms_with_importable_entries + */ + public function test_count_falls_to_zero() { + $this->add_message( array( 'your-name' => 'Ruth Ellery' ), '2025-05-01 09:00:00' ); + + $form_id = $this->import_form(); + + $this->assertSame( 1, atf_cf7_entry_count( (string) $this->cf7_id, $form_id ) ); + $this->assertArrayHasKey( $form_id, atf_forms_with_importable_entries() ); + + atf_import_form_entries( $form_id ); + + $this->assertSame( 0, atf_cf7_entry_count( (string) $this->cf7_id, $form_id ) ); + $this->assertArrayNotHasKey( $form_id, atf_forms_with_importable_entries() ); + } + + /** + * An answer the form no longer offers is kept, not thrown away. + * + * Sanitising runs; validation deliberately does not. A choice retired since + * the message was sent is still what that person said. + * + * @covers ::atf_import_entry + */ + public function test_retired_choice_survives() { + $this->add_message( + array( + 'your-name' => 'Nils Andersen', + 'bid-funding' => 'Paying in gold bars', + ), + '2025-06-01 09:00:00' + ); + + $form_id = $this->import_form(); + atf_import_form_entries( $form_id ); + + $entries = get_posts( + array( + 'post_type' => ATF_ENTRY_TYPE, + 'post_status' => self::ENTRY_STATUSES, + 'posts_per_page' => -1, + 'meta_key' => ATF_META_FORM, + 'meta_value' => $form_id, + ) + ); + + $map = atf_form_import_map( $form_id ); + $values = json_decode( get_post_meta( $entries[0]->ID, ATF_META_VALUES, true ), true ); + + $this->assertSame( 'Paying in gold bars', $values[ $map['bid-funding'] ] ); + } + + /** + * A form that was never imported has nothing to bring across. + * + * @covers ::atf_import_form_entries + */ + public function test_native_form_is_rejected() { + $form_id = self::factory()->post->create( array( 'post_type' => ATF_FORM_TYPE ) ); + + $result = atf_import_form_entries( $form_id ); + + $this->assertWPError( $result ); + $this->assertSame( 'atf_not_imported', $result->get_error_code() ); + } + + /** + * The source's own records are never touched. + * + * @covers ::atf_cf7_import_entries + */ + public function test_source_is_left_alone() { + $message_id = $this->add_message( array( 'your-name' => 'Aoife Brennan' ), '2025-07-01 09:00:00' ); + + $form_id = $this->import_form(); + atf_import_form_entries( $form_id ); + + $message = get_post( $message_id ); + + $this->assertInstanceOf( 'WP_Post', $message ); + $this->assertSame( 'publish', $message->post_status ); + $this->assertSame( 'Aoife Brennan', get_post_meta( $message_id, '_field_your-name', true ) ); + } +}