diff --git a/ci/setup-remotedbs.sh b/ci/setup-remotedbs.sh index 97f8a42..65bfd6c 100755 --- a/ci/setup-remotedbs.sh +++ b/ci/setup-remotedbs.sh @@ -504,6 +504,28 @@ function setup_olr() fi } +function setup_postgres() +{ + echo "setting up postgres..." + if [ $INTERNAL -eq 1 ]; then + docker_compose -f testenv/postgres/synchdb-postgres-test-internal.yaml up -d + else + docker_compose -f testenv/postgres/synchdb-postgres-test.yaml up -d + fi + echo "sleep to give container time to startup..." + sleep 30 # Give containers time to fully start + + docker exec -i postgres psql -U postgres -d postgres -c "CREATE TABLE orders (order_number int primary key, order_date timestamp without time zone, purchaser int, quantity int , product_id int)" + docker exec -i postgres psql -U postgres -d postgres -c "INSERT INTO orders(order_number, order_date, purchaser, quantity, product_id) VALUES (10001, '2026-01-01', 1003, 2, 107)" + docker exec -i postgres psql -U postgres -d postgres -c "INSERT INTO orders(order_number, order_date, purchaser, quantity, product_id) VALUES (10002, '2026-01-01', 1003, 2, 107)" + docker exec -i postgres psql -U postgres -d postgres -c "INSERT INTO orders(order_number, order_date, purchaser, quantity, product_id) VALUES (10003, '2026-01-01', 1003, 2, 107)" + docker exec -i postgres psql -U postgres -d postgres -c "INSERT INTO orders(order_number, order_date, purchaser, quantity, product_id) VALUES (10004, '2026-01-01', 1003, 2, 107)" + + # install ddl trigger function + docker exec -i postgres psql -U postgres -d postgres < ./postgres-connector-src-ddl-setup.sql + exit 0 +} + function setup_remotedb() { dbtype="$1" @@ -530,6 +552,9 @@ function setup_remotedb() setup_ora19c "olr" setup_olr ;; + "postgres") + setup_postgres + ;; *) echo "$dbtype not supported" exit 1 diff --git a/ci/teardown-remotedbs.sh b/ci/teardown-remotedbs.sh index db05d51..6ed4b20 100755 --- a/ci/teardown-remotedbs.sh +++ b/ci/teardown-remotedbs.sh @@ -83,6 +83,13 @@ function teardown_synchdbnet() docker network rm synchdbnet >/dev/null 2>&1 } +function teardown_postgres() +{ + echo "tearing down postgres..." + docker stop postgres + docker rm postgres +} + function teardown_remotedb() { dbtype="$1" @@ -113,6 +120,9 @@ function teardown_remotedb() "oradata") teardown_oradata ;; + "postgres") + teardown_postgres + ;; *) echo "$dbtype not supported" exit 1 diff --git a/doc/docs/en/getting-started/quick_start.md b/doc/docs/en/getting-started/quick_start.md index f0150fe..1a82049 100644 --- a/doc/docs/en/getting-started/quick_start.md +++ b/doc/docs/en/getting-started/quick_start.md @@ -23,20 +23,21 @@ please select a quick deploy option: 4) synchdb + oracle23ai 5) synchdb + oracle19c 6) synchdb + olr(oracle19c) - 7) synchdb + all source databases - 8) custom deployment - 9) deploy monitoring - 10) teardown deployment + 7) synchdb + postgres18 + 8) synchdb + all source databases + 9) custom deployment + 10) deploy monitoring + 11) teardown deployment enter your selection: ``` * For synchdb deployment only, use option `1)`. * For synchdb + 1 source database, use option `2)` to `6)`. -* For synchdb + all source databases, use option `7)`. -* For synchdb + custom source databases, use option `8)`. -* For prometheus and grafana monitoring deployment, use option `9)`. -* to teardown all deployment, use option `10)`. +* For synchdb + all source databases, use option `8)`. +* For synchdb + custom source databases, use option `9)`. +* For prometheus and grafana monitoring deployment, use option `10)`. +* to teardown all deployment, use option `11)`. ## **Access Details of Source Databases for Testing** @@ -72,6 +73,13 @@ enter your selection: * service name: ORACLE +**Postgres:** + +* database: postgres +* schema: public +* user: postgres +* password: pass + ## **Access Synchdb with psql** Once deployed, synchdb can be accessed by: @@ -172,6 +180,21 @@ SELECT synchdb_add_olr_conninfo('olrconn', ``` +**Postgres:** +```sql +SELECT synchdb_add_conninfo('postgresconn', + 'postgres', + 5432, + 'postgres', + 'pass', + 'postgres', + 'public', + 'null', + 'null', + 'postgres'); + +``` + **View Created Connectors:** ```sql @@ -186,9 +209,9 @@ More details on creating a connector can be found [here](../../user-guide/create By default, source database names will be mapped to a schema name in destination. Object mappings can be used to change this schema name. Let's change the destination schema for `orders` table from oracle based connectors and leave the rest as default. ```sql -SELECT synchdb_add_objmap('oracleconn','table','free.c##dbzuser.orders','oracle23ai.orders'); -SELECT synchdb_add_objmap('ora19cconn','table','free.dbzuser.orders','oracle19c.orders'); -SELECT synchdb_add_objmap('olrconn','table','free.dbzuser.orders','olr.orders'); +SELECT synchdb_add_objmap('oracleconn','table','FREE.C##DBZUSER.ORDERS','oracle23ai.orders'); +SELECT synchdb_add_objmap('ora19cconn','table','FREE.DBZUSER.ORDERS','oracle19c.orders'); +SELECT synchdb_add_objmap('olrconn','table','FREE.DBZUSER.ORDERS','olr.orders'); ``` @@ -238,43 +261,125 @@ SELECT synchdb_add_jmx_exporter_conninfo( ``` +**Postgres:** +```sql +SELECT synchdb_add_jmx_exporter_conninfo( + 'postgresconn', + '/home/ubuntu/jmx_prometheus_javaagent-1.3.0.jar', + 9408, + '/home/ubuntu/jmxexport.conf'); + +``` + More details on creating a JMX Exporter can be found [here](../../monitoring/jmx_exporter/) -## **Start a Connector** +## **Start a Connector - Default Debezium Snapshot Engine** **MySQL:** ```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'debezium'; +SELECT pg_reload_conf(); SELECT synchdb_start_engine_bgw('mysqlconn'); ``` **Sqlserver:** ```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'debezium'; +SELECT pg_reload_conf(); SELECT synchdb_start_engine_bgw('sqlserverconn'); ``` **Oracle23ai:** ```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'debezium'; +SELECT pg_reload_conf(); SELECT synchdb_start_engine_bgw('oracleconn'); ``` **Oracle19c:** ```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'debezium'; +SELECT pg_reload_conf(); SELECT synchdb_start_engine_bgw('ora19cconn'); ``` **OLR(Oracle19c):** ```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'debezium'; +SELECT pg_reload_conf(); SELECT synchdb_start_engine_bgw('olrconn'); ``` +**Postgres:** +```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'debezium'; +SELECT pg_reload_conf(); +-- Requires user to pre-create schemas and tables: +CREATE SCHEMA postgres; +CREATE TABLE postgres.orders (order_number INT PRIMARY KEY, order_date TIMESTAMP WITHOUT TIME ZONE, purchaser INT, quantity INT , product_id INT); +SELECT synchdb_start_engine_bgw('postgresconn'); + +``` + More details on connector start can be found [here](../../user-guide/start_stop_connector/) -## Check Connector Running State +## **Start a Connector - FDW Snapshot Engine** + +**MySQL:** +```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'fdw'; +SELECT pg_reload_conf(); +SELECT synchdb_start_engine_bgw('mysqlconn'); + +``` + +**Sqlserver:** +```sql +-- not supported yet. Connector will default to debezium when started +ALTER SYSTEM SET synchdb.snapshot_engine = 'fdw'; +SELECT pg_reload_conf(); +SELECT synchdb_start_engine_bgw('sqlserverconn'); + +``` + +**Oracle23ai:** +```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'fdw'; +SELECT pg_reload_conf(); +SELECT synchdb_start_engine_bgw('oracleconn'); + +``` + +**Oracle19c:** +```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'fdw'; +SELECT pg_reload_conf(); +SELECT synchdb_start_engine_bgw('ora19cconn'); + +``` + +**OLR(Oracle19c):** +```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'fdw'; +SELECT pg_reload_conf(); +SELECT synchdb_start_engine_bgw('olrconn'); + +``` + +**Postgres:** +```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'fdw'; +SELECT pg_reload_conf(); +SELECT synchdb_start_engine_bgw('postgresconn'); + +``` + +## **Check Connector Running State** Use `synchdb_state_view()` to examine all connectors' running states. @@ -290,25 +395,20 @@ postgres=# SELECT * FROM synchdb_state_view; ---------------+----------------+--------+------------------+---------+----------+------------------------------------------------------------------------------------------------------ sqlserverconn | sqlserver | 579820 | initial snapshot | polling | no error | {"commit_lsn":"0000006a:00006608:0003","snapshot":true,"snapshot_completed":false} mysqlconn | mysql | 579845 | initial snapshot | polling | no error | {"ts_sec":1741301103,"file":"mysql-bin.000009","pos":574318212,"row":1,"server_id":223344,"event":2} - oracleconn | oracle | 580053 | initial snapshot | polling | no error | offset file not flushed yet - ora19cconn | oracle | 593421 | initial snapshot | polling | no error | offset file not flushed yet - olrconn | oracle | 601235 | initial snapshot | polling | no error | offset file not flushed yet -(5 rows) + oracleconn | oracle | 580053 | initial snapshot | polling | no error | {"commit_scn":"3118146:1:02001f00c0020000","snapshot_scn":"3081987","scn":"3118125"} + ora19cconn | oracle | 593421 | initial snapshot | polling | no error | {"commit_scn":"3118146:1:02001f00c0020000","snapshot_scn":"3081987","scn":"3118125"} + olrconn | oracle | 601235 | initial snapshot | polling | no error | {"scn":5031082, "c_scn":5031085, "c_idx":3} + postgresconn | postgres | 631565 | initial snapshot | polling | no error | {"lsn_proc":37396384,"messageType":"INSERT","lsn":37396384,"txId":1015,"ts_usec":1767740340957961} +(6 rows) ``` More on running states [here](../../monitoring/state_view/), and also running statistics [here](../../monitoring/stats_view/). - ## Check the Tables and Data from Initial Snapshot By default, the connector will perform a `initial` snapshot to capture both the table schema and initial data, convert and apply them to PostgreSQL under different `schema`. You should see something similar to the following: **MySQL:** -```sql -\dt inventory.* - -``` - ```sql \dt inventory.* List of relations @@ -324,10 +424,6 @@ By default, the connector will perform a `initial` snapshot to capture both the ``` **Sqlserver:** -```sql -\dt testdb.* - -``` ```sql \dt testdb.* @@ -343,11 +439,6 @@ By default, the connector will perform a `initial` snapshot to capture both the ``` **Oracle23ai** -```sql -\dt oracle23ai.* - -``` - ```sql \dt oracle23ai.* List of relations @@ -359,10 +450,6 @@ By default, the connector will perform a `initial` snapshot to capture both the ``` **Oracle19c** -```sql -\dt oracle19c.* - -``` ```sql \dt oracle19c.* @@ -374,10 +461,6 @@ By default, the connector will perform a `initial` snapshot to capture both the ``` **OLR** -```sql -\dt olr.* - -``` ```sql \dt olr.* @@ -388,7 +471,19 @@ By default, the connector will perform a `initial` snapshot to capture both the (1 row) ``` -## Similate an INSERT Event and Observe CDC +**Postgres** + +```sql +\dt postgres.* + List of relations + Schema | Name | Type | Owner +----------+--------+-------+-------- + postgres | orders | table | ubuntu +(1 row) + +``` + +## Simulate an INSERT Event and Observe CDC We can use `docker exec` to similate an INSERT for each connector type and observe the Change Data Capture (CDC). @@ -487,6 +582,25 @@ postgres=# SELECT * FROM olr.orders; ``` +**Postgres:** +```bash +docker exec -i postgres psql -U postgres -d postgres -c "INSERT INTO orders(order_number, order_date, purchaser, quantity, product_id) VALUES (10005, '2026-01-01', 1003, 10000, 102)" + +``` + +```sql +postgres=# SELECT * FROM postgres.orders; + order_number | order_date | purchaser | quantity | product_id +--------------+---------------------+-----------+----------+------------ + 10001 | 2024-01-01 00:00:00 | 1003 | 2 | 107 + 10002 | 2024-01-01 00:00:00 | 1003 | 2 | 107 + 10003 | 2024-01-01 00:00:00 | 1003 | 2 | 107 + 10004 | 2024-01-01 00:00:00 | 1003 | 2 | 107 + 10005 | 2026-01-01 00:00:00 | 1003 | 10000 | 102 +(5 rows) + +``` + ## Connector Metrics on Grafana - Optional ![img](../../images/grafana-home.jpg) @@ -558,4 +672,11 @@ SELECT synchdb_del_conninfo('ora19cconn'); SELECT synchdb_stop_engine_bgw('olrconn'); SELECT synchdb_del_conninfo('olrconn'); +``` + +**Postgres** +```sql +SELECT synchdb_stop_engine_bgw('postgresconn'); +SELECT synchdb_del_conninfo('postgresconn'); + ``` \ No newline at end of file diff --git a/doc/docs/zh/getting-started/quick_start.md b/doc/docs/zh/getting-started/quick_start.md index 0935c8d..648b583 100644 --- a/doc/docs/zh/getting-started/quick_start.md +++ b/doc/docs/zh/getting-started/quick_start.md @@ -23,20 +23,21 @@ please select a quick deploy option: 4) synchdb + oracle23ai 5) synchdb + oracle19c 6) synchdb + olr(oracle19c) - 7) synchdb + all source databases - 8) custom deployment - 9) deploy monitoring - 10) teardown deployment + 7) synchdb + postgres18 + 8) synchdb + all source databases + 9) custom deployment + 10) deploy monitoring + 11) teardown deployment enter your selection: ``` * 仅对于 synchdb 部署,请使用选项 `1)`。 * 对于 synchdb + 1 个源数据库,请使用选项 `2)` 至 `6)`。 -* 对于 synchdb + 所有源数据库,请使用选项 `7)`。 -* 对于 synchdb + 自定义源数据库,请使用选项 `8)`。 -* 对于 prometheus 和 grafana 监控部署,请使用选项 `9)`。 -* 要拆除所有部署,请使用选项 `10)`。 +* 对于 synchdb + 所有源数据库,请使用选项 `8)`。 +* 对于 synchdb + 自定义源数据库,请使用选项 `9)`。 +* 对于 prometheus 和 grafana 监控部署,请使用选项 `10)`。 +* 要拆除所有部署,请使用选项 `11)`。 ## **测试源数据库访问详情** @@ -72,6 +73,13 @@ enter your selection: * 服务名称:ORACLE +**Postgres:** + +* 数据库:postgres +* 模式:public +* 用户: postgres +* 密码: pass + ## **使用 psql 访问 Synchdb** 部署完成后,可以通过以下方式访问 synchdb: @@ -173,6 +181,20 @@ SELECT synchdb_add_olr_conninfo('olrconn', ``` +**Postgres:** +```sql +SELECT synchdb_add_conninfo('postgresconn', + 'postgres', + 5432, + 'postgres', + 'pass', + 'postgres', + 'public', + 'null', + 'null', + 'postgres'); + +``` **查看已创建的连接器:** @@ -188,9 +210,9 @@ SELECT * FROM synchdb_conninfo; 默认情况下,源数据库名称将映射到目标数据库中的架构名称。可以使用对象映射来更改此架构名称。让我们从基于 Oracle 的连接器中更改“orders”表的目标架构,其余部分保留默认值。 ```sql -SELECT synchdb_add_objmap('oracleconn','table','free.c##dbzuser.orders','oracle23ai.orders'); -SELECT synchdb_add_objmap('ora19cconn','table','free.dbzuser.orders','oracle19c.orders'); -SELECT synchdb_add_objmap('olrconn','table','free.dbzuser.orders','olr.orders'); +SELECT synchdb_add_objmap('oracleconn','table','FREE.C##DBZUSER.ORDERS','oracle23ai.orders'); +SELECT synchdb_add_objmap('ora19cconn','table','FREE.DBZUSER.ORDERS','oracle19c.orders'); +SELECT synchdb_add_objmap('olrconn','table','FREE.DBZUSER.ORDERS','olr.orders'); ``` @@ -240,41 +262,122 @@ SELECT synchdb_add_jmx_exporter_conninfo( ``` +**Postgres:** +```sql +SELECT synchdb_add_jmx_exporter_conninfo( + 'postgresconn', + '/home/ubuntu/jmx_prometheus_javaagent-1.3.0.jar', + 9408, + '/home/ubuntu/jmxexport.conf'); + +``` + 关于创建 JMX Exporter 的更多详细信息,请参见[此处](../../monitoring/jmx_exporter/) -## **启动连接器** +## **启动连接器 - 默认 debezium 快照引擎** **MySQL:** ```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'debezium'; +SELECT pg_reload_conf(); SELECT synchdb_start_engine_bgw('mysqlconn'); ``` **Sqlserver:** ```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'debezium'; +SELECT pg_reload_conf(); SELECT synchdb_start_engine_bgw('sqlserverconn'); ``` **Oracle23ai:** ```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'debezium'; +SELECT pg_reload_conf(); SELECT synchdb_start_engine_bgw('oracleconn'); ``` **Oracle19c:** ```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'debezium'; +SELECT pg_reload_conf(); SELECT synchdb_start_engine_bgw('ora19cconn'); + ``` **OLR(Oracle19c):** ```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'debezium'; +SELECT pg_reload_conf(); SELECT synchdb_start_engine_bgw('olrconn'); ``` +**Postgres:** +```sql +-- Requires user to pre-create schemas and tables: +CREATE SCHEMA postgres; +CREATE TABLE postgres.orders (order_number INT PRIMARY KEY, order_date TIMESTAMP WITHOUT TIME ZONE, purchaser INT, quantity INT , product_id INT); +SELECT synchdb_start_engine_bgw('postgresconn'); + +``` + 有关连接器启动的更多详细信息,请参见[此处](../../user-guide/start_stop_connector/) +## **启动连接器 - 使用 fdw 快照引擎** + +**MySQL:** +```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'fdw'; +SELECT pg_reload_conf(); +SELECT synchdb_start_engine_bgw('mysqlconn'); + +``` + +**Sqlserver:** +```sql +-- not supported yet. Connector will default to debezium when started +ALTER SYSTEM SET synchdb.snapshot_engine = 'fdw'; +SELECT pg_reload_conf(); +SELECT synchdb_start_engine_bgw('sqlserverconn'); + +``` + +**Oracle23ai:** +```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'fdw'; +SELECT pg_reload_conf(); +SELECT synchdb_start_engine_bgw('oracleconn'); + +``` + +**Oracle19c:** +```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'fdw'; +SELECT pg_reload_conf(); +SELECT synchdb_start_engine_bgw('ora19cconn'); + +``` + +**OLR(Oracle19c):** +```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'fdw'; +SELECT pg_reload_conf(); +SELECT synchdb_start_engine_bgw('olrconn'); + +``` + +**Postgres:** +```sql +ALTER SYSTEM SET synchdb.snapshot_engine = 'fdw'; +SELECT pg_reload_conf(); +SELECT synchdb_start_engine_bgw('postgresconn'); + +``` + ## **检查连接器运行状态** 使用“synchdb_state_view()”检查所有连接器的运行状态。 @@ -291,10 +394,11 @@ postgres=# SELECT * FROM synchdb_state_view; ---------------+----------------+--------+------------------+---------+----------+------------------------------------------------------------------------------------------------------ sqlserverconn | sqlserver | 579820 | initial snapshot | polling | no error | {"commit_lsn":"0000006a:00006608:0003","snapshot":true,"snapshot_completed":false} mysqlconn | mysql | 579845 | initial snapshot | polling | no error | {"ts_sec":1741301103,"file":"mysql-bin.000009","pos":574318212,"row":1,"server_id":223344,"event":2} - oracleconn | oracle | 580053 | initial snapshot | polling | no error | offset file not flushed yet - ora19cconn | oracle | 593421 | initial snapshot | polling | no error | offset file not flushed yet - olrconn | oracle | 601235 | initial snapshot | polling | no error | offset file not flushed yet -(5 rows) + oracleconn | oracle | 580053 | initial snapshot | polling | no error | {"commit_scn":"3118146:1:02001f00c0020000","snapshot_scn":"3081987","scn":"3118125"} + ora19cconn | oracle | 593421 | initial snapshot | polling | no error | {"commit_scn":"3118146:1:02001f00c0020000","snapshot_scn":"3081987","scn":"3118125"} + olrconn | oracle | 601235 | initial snapshot | polling | no error | {"scn":5031082, "c_scn":5031085, "c_idx":3} + postgresconn | postgres | 631565 | initial snapshot | polling | no error | {"lsn_proc":37396384,"messageType":"INSERT","lsn":37396384,"txId":1015,"ts_usec":1767740340957961} +(6 rows) ``` @@ -304,11 +408,6 @@ postgres=# SELECT * FROM synchdb_state_view; 默认情况下,连接器将执行“初始”快照,以捕获表模式和初始数据,然后转换并将它们应用到不同“模式”下的 PostgreSQL。您应该看到类似以下内容: **MySQL:** -```sql -\dt inventory.* - -``` - ```sql \dt inventory.* List of relations @@ -324,11 +423,6 @@ postgres=# SELECT * FROM synchdb_state_view; ``` **Sqlserver:** -```sql -\dt testdb.* - -``` - ```sql \dt testdb.* List of relations @@ -343,11 +437,6 @@ postgres=# SELECT * FROM synchdb_state_view; ``` **Oracle23ai** -```sql -\dt oracle23ai.* - -``` - ```sql \dt oracle23ai.* List of relations @@ -359,11 +448,6 @@ postgres=# SELECT * FROM synchdb_state_view; ``` **Oracle19c** -```sql -\dt oracle19c.* - -``` - ```sql \dt oracle19c.* List of relations @@ -374,11 +458,6 @@ postgres=# SELECT * FROM synchdb_state_view; ``` **OLR** -```sql -\dt olr.* - -``` - ```sql \dt olr.* List of relations @@ -388,6 +467,17 @@ postgres=# SELECT * FROM synchdb_state_view; (1 row) ``` +**Postgres** +```sql +\dt postgres.* + List of relations + Schema | Name | Type | Owner +----------+--------+-------+-------- + postgres | orders | table | ubuntu +(1 row) + +``` + ## 模拟 INSERT 事件并观察变更数据捕获 (CDC) 我们可以使用 `docker exec` 为每种连接器类型模拟一次 INSERT 操作,并观察变更数据捕获 (CDC)。 @@ -487,6 +577,25 @@ postgres=# SELECT * FROM olr.orders; ``` +**Postgres:** +```bash +docker exec -i postgres psql -U postgres -d postgres -c "INSERT INTO orders(order_number, order_date, purchaser, quantity, product_id) VALUES (10005, '2026-01-01', 1003, 10000, 102)" + +``` + +```sql +postgres=# SELECT * FROM postgres.orders; + order_number | order_date | purchaser | quantity | product_id +--------------+---------------------+-----------+----------+------------ + 10001 | 2024-01-01 00:00:00 | 1003 | 2 | 107 + 10002 | 2024-01-01 00:00:00 | 1003 | 2 | 107 + 10003 | 2024-01-01 00:00:00 | 1003 | 2 | 107 + 10004 | 2024-01-01 00:00:00 | 1003 | 2 | 107 + 10005 | 2026-01-01 00:00:00 | 1003 | 10000 | 102 +(5 rows) + +``` + ## Grafana 上的连接器指标 - 可选 ![img](../../images/grafana-home.jpg) @@ -558,4 +667,11 @@ SELECT synchdb_del_conninfo('ora19cconn'); SELECT synchdb_stop_engine_bgw('olrconn'); SELECT synchdb_del_conninfo('olrconn'); +``` + +**Postgres** +```sql +SELECT synchdb_stop_engine_bgw('postgresconn'); +SELECT synchdb_del_conninfo('postgresconn'); + ``` \ No newline at end of file diff --git a/doc/mkdocs.yml b/doc/mkdocs.yml index b592a40..82d3f54 100644 --- a/doc/mkdocs.yml +++ b/doc/mkdocs.yml @@ -84,7 +84,6 @@ plugins: JVM Memory Usage: Uso de memoria de JVM Attribute View: Vista de atributos JMX Exporter Settings: Configuración del exportador JMX - Openlog Replicator: Configurar Openlog Replicator Openlog Replicator Event Processor: Procesador de eventos del replicador Openlog Getting Started: Empezando Monitoring: escucha @@ -136,7 +135,6 @@ plugins: JVM Memory Usage: JVM 内存使用情况 Attribute View: 属性视图 JMX Exporter Settings: JMX exporter 设置 - Openlog Replicator: Openlog Replicator 设置 Openlog Replicator Event Processor: Openlog Replicator 事件处理器 Debezium Event Processor: Debezium 事件处理器 Getting Started: 快速入门 @@ -197,7 +195,6 @@ nav: - Object Transformation: user-guide/object_mapping_rules.md - Letter Casing Strategy: user-guide/letter_casing_strategy.md - Error Strategies: user-guide/configure_error_strategies.md - - Openlog Replicator: user-guide/configure_olr.md - Snapshot Engine: user-guide/configure_snapshot_engine.md - Changelog: changelog.md # - Homepage: https://www.synchdb.com diff --git a/ezdeploy.sh b/ezdeploy.sh index b668cf8..e88ab5f 100755 --- a/ezdeploy.sh +++ b/ezdeploy.sh @@ -91,7 +91,7 @@ function custom-deployment() { echo "" echo "please list source databases separate by comma" - echo "possible values: (mysql, sqlserver, oracle23ai, oracle19c, olr)" + echo "possible values: (mysql, sqlserver, oracle23ai, oracle19c, olr, postgres)" read -rp "your selection: " RAW_CHOICES IFS=', ' read -r -a _tokens <<< "$RAW_CHOICES" @@ -100,7 +100,7 @@ function custom-deployment() [[ -z "$t" ]] && continue key="${t,,}" case "$key" in - mysql|sqlserver|oracle23ai|oracle19c|olr) PICKED["$key"]=1 ;; + mysql|sqlserver|oracle23ai|oracle19c|olr|postgres) PICKED["$key"]=1 ;; *) echo "Ignoring unknown option: $t" ;; esac done @@ -175,10 +175,11 @@ echo -e "\t 3) synchdb + sqlserver" echo -e "\t 4) synchdb + oracle23ai" echo -e "\t 5) synchdb + oracle19c" echo -e "\t 6) synchdb + olr(oracle19c)" -echo -e "\t 7) synchdb + all source databases" -echo -e "\t 8) custom deployment" -echo -e "\t 9) deploy monitoring" -echo -e "\t10) teardown deployment" +echo -e "\t 7) synchdb + postgres18" +echo -e "\t 8) synchdb + all source databases" +echo -e "\t 9) custom deployment" +echo -e "\t10) deploy monitoring" +echo -e "\t11) teardown deployment" read -rp "enter your selection: " choice @@ -201,21 +202,26 @@ case "$choice" in deploy-sourcedb "olr" ;; 7) deploy-synchdb + deploy-sourcedb "postgres" + ;; + 8) deploy-synchdb deploy-sourcedb "mysql" deploy-sourcedb "sqlserver" deploy-sourcedb "oracle" deploy-sourcedb "olr" + deploy-sourcedb "postgres" ;; - 8) custom-deployment + 9) custom-deployment ;; - 9) deploy-monitoring + 10) deploy-monitoring ;; - 10) teardown-synchdb + 11) teardown-synchdb teardown-sourcedb "mysql" teardown-sourcedb "sqlserver" teardown-sourcedb "oracle" teardown-sourcedb "ora19c" teardown-sourcedb "olr" + teardown-sourcedb "postgres" teardown-monitoring clear-oradata teardown-sourcedb "synchdbnet" diff --git a/src/backend/converter/debezium_event_handler.c b/src/backend/converter/debezium_event_handler.c index ebff3ba..0e8ce7b 100644 --- a/src/backend/converter/debezium_event_handler.c +++ b/src/backend/converter/debezium_event_handler.c @@ -33,6 +33,7 @@ extern bool synchdb_log_event_on_error; extern char * g_eventStr; extern HTAB * dataCacheHash; extern int synchdb_letter_casing_strategy; +extern int synchdb_error_strategy; static DdlType name_to_ddltype(const char * name); static DbzType getDbzTypeFromString(const char * typestring); @@ -461,8 +462,6 @@ parseDBZDDL(Jsonb * jb, bool isfirst, bool islast, bool deriveMsg) /* free the data inside strinfo as we no longer needs it */ pfree(strinfo.data); -// fc_normalize_name(synchdb_letter_casing_strategy, ddlinfo->id, strlen(ddlinfo->id)); - if (ddlinfo->type == DDL_CREATE_TABLE || ddlinfo->type == DDL_ALTER_TABLE) { /* fetch payload.tableChanges.0.table.columns as jsonb */ @@ -597,9 +596,6 @@ parseDBZDDL(Jsonb * jb, bool isfirst, bool islast, bool deriveMsg) { elog(DEBUG1, "consuming %s = %s", key, value); ddlcol->name = pstrdup(value); - -// fc_normalize_name(synchdb_letter_casing_strategy, ddlcol->name, -// strlen(ddlcol->name)); } if (!strcmp(key, "length")) { @@ -789,8 +785,6 @@ parseDBZDML(Jsonb * jb, char op, ConnectorType type, Jsonb * source, bool isfirs dbzdml->dbz_ts_ms = strtoull(strinfo.data, NULL, 10); } -// fc_normalize_name(synchdb_letter_casing_strategy, objid.data, objid.len); - dbzdml->remoteObjectId = pstrdup(objid.data); dbzdml->mappedObjectId = transform_object_name(dbzdml->remoteObjectId, "table"); if (dbzdml->mappedObjectId) @@ -858,9 +852,6 @@ parseDBZDML(Jsonb * jb, char op, ConnectorType type, Jsonb * source, bool isfirs * convert db and table to all lower case letters. */ -// fc_normalize_name(synchdb_letter_casing_strategy, dbzdml->schema, strlen(dbzdml->schema)); -// fc_normalize_name(synchdb_letter_casing_strategy, dbzdml->table, strlen(dbzdml->table)); - /* prepare cache key */ strlcpy(cachekey.schema, dbzdml->schema, sizeof(cachekey.schema)); strlcpy(cachekey.table, dbzdml->table, sizeof(cachekey.table)); @@ -876,15 +867,25 @@ parseDBZDML(Jsonb * jb, char op, ConnectorType type, Jsonb * source, bool isfirs } else { - schemaoid = get_namespace_oid(dbzdml->schema, false); + schemaoid = get_namespace_oid(dbzdml->schema, true); if (!OidIsValid(schemaoid)) { char * msg = palloc0(SYNCHDB_ERRMSG_SIZE); snprintf(msg, SYNCHDB_ERRMSG_SIZE, "no valid OID found for schema '%s'", dbzdml->schema); set_shm_connector_errmsg(myConnectorId, msg); - /* trigger pg's error shutdown routine */ - elog(ERROR, "%s", msg); + if (synchdb_log_event_on_error && g_eventStr != NULL) + elog(LOG, "%s", g_eventStr); + + /* act based on error strategy */ + if (synchdb_error_strategy == STRAT_EXIT_ON_ERROR) + elog(ERROR, "%s", msg); + else + { + destroyDBZDML(dbzdml); + dbzdml = NULL; + goto end; + } } dbzdml->tableoid = get_relname_relid(dbzdml->table, schemaoid); @@ -894,8 +895,18 @@ parseDBZDML(Jsonb * jb, char op, ConnectorType type, Jsonb * source, bool isfirs snprintf(msg, SYNCHDB_ERRMSG_SIZE, "no valid OID found for table '%s'", dbzdml->table); set_shm_connector_errmsg(myConnectorId, msg); - /* trigger pg's error shutdown routine */ - elog(ERROR, "%s", msg); + if (synchdb_log_event_on_error && g_eventStr != NULL) + elog(LOG, "%s", g_eventStr); + + /* act based on error strategy */ + if (synchdb_error_strategy == STRAT_EXIT_ON_ERROR) + elog(ERROR, "%s", msg); + else + { + destroyDBZDML(dbzdml); + dbzdml = NULL; + goto end; + } } /* populate cached information */ diff --git a/src/backend/converter/format_converter.c b/src/backend/converter/format_converter.c index 0739ed3..28d3451 100644 --- a/src/backend/converter/format_converter.c +++ b/src/backend/converter/format_converter.c @@ -53,6 +53,9 @@ extern bool synchdb_dml_use_spi; extern int myConnectorId; extern int synchdb_letter_casing_strategy; +extern int synchdb_error_strategy; +extern bool synchdb_log_event_on_error; +extern char * g_eventStr; /* data transformation related hash tables */ HTAB * dataCacheHash = NULL; @@ -3803,15 +3806,21 @@ updateSynchdbAttribute(DBZ_DDL * dbzddl, PG_DDL * pgddl, ConnectorType conntype, return; } - schemaoid = get_namespace_oid(pgddl->schema, false); + schemaoid = get_namespace_oid(pgddl->schema, true); if (!OidIsValid(schemaoid)) { char * msg = palloc0(SYNCHDB_ERRMSG_SIZE); snprintf(msg, SYNCHDB_ERRMSG_SIZE, "no valid OID found for schema '%s'", pgddl->schema); set_shm_connector_errmsg(myConnectorId, msg); - /* trigger pg's error shutdown routine */ - elog(ERROR, "%s", msg); + if (synchdb_log_event_on_error && g_eventStr != NULL) + elog(LOG, "%s", g_eventStr); + + /* act based on error strategy */ + if (synchdb_error_strategy == STRAT_EXIT_ON_ERROR) + elog(ERROR, "%s", msg); + else + return; } tableoid = get_relname_relid(pgddl->tbname, schemaoid); @@ -3821,8 +3830,14 @@ updateSynchdbAttribute(DBZ_DDL * dbzddl, PG_DDL * pgddl, ConnectorType conntype, snprintf(msg, SYNCHDB_ERRMSG_SIZE, "no valid OID found for table '%s'", pgddl->tbname); set_shm_connector_errmsg(myConnectorId, msg); - /* trigger pg's error shutdown routine */ - elog(ERROR, "%s", msg); + if (synchdb_log_event_on_error && g_eventStr != NULL) + elog(LOG, "%s", g_eventStr); + + /* act based on error strategy */ + if (synchdb_error_strategy == STRAT_EXIT_ON_ERROR) + elog(ERROR, "%s", msg); + else + return; } appendStringInfo(&strinfo, "INSERT INTO %s (name, type, attrelid, attnum, " diff --git a/src/backend/converter/olr_event_handler.c b/src/backend/converter/olr_event_handler.c index 0887fe0..425a01b 100644 --- a/src/backend/converter/olr_event_handler.c +++ b/src/backend/converter/olr_event_handler.c @@ -38,6 +38,7 @@ extern char * dbz_skipped_operations; extern bool synchdb_log_event_on_error; extern char * g_eventStr; extern int synchdb_letter_casing_strategy; +extern int synchdb_error_strategy; /* Oracle raw parser function prototype */ typedef List * (*oracle_raw_parser_fn)(const char *str, RawParseMode mode); @@ -1544,7 +1545,7 @@ parseOLRDML(Jsonb * jb, char op, Jsonb * payload, orascn * scn, orascn * c_scn, } else { - schemaoid = get_namespace_oid(olrdml->schema, false); + schemaoid = get_namespace_oid(olrdml->schema, true); if (!OidIsValid(schemaoid)) { char * msg = palloc0(SYNCHDB_ERRMSG_SIZE); @@ -1554,8 +1555,15 @@ parseOLRDML(Jsonb * jb, char op, Jsonb * payload, orascn * scn, orascn * c_scn, if (synchdb_log_event_on_error && g_eventStr != NULL) elog(LOG, "%s", g_eventStr); - /* trigger pg's error shutdown routine */ - elog(ERROR, "%s", msg); + /* act based on error strategy */ + if (synchdb_error_strategy == STRAT_EXIT_ON_ERROR) + elog(ERROR, "%s", msg); + else + { + destroyOLRDML(olrdml); + olrdml = NULL; + goto end; + } } olrdml->tableoid = get_relname_relid(olrdml->table, schemaoid); @@ -1568,8 +1576,15 @@ parseOLRDML(Jsonb * jb, char op, Jsonb * payload, orascn * scn, orascn * c_scn, if (synchdb_log_event_on_error && g_eventStr != NULL) elog(LOG, "%s", g_eventStr); - /* trigger pg's error shutdown routine */ - elog(ERROR, "%s", msg); + /* act based on error strategy */ + if (synchdb_error_strategy == STRAT_EXIT_ON_ERROR) + elog(ERROR, "%s", msg); + else + { + destroyOLRDML(olrdml); + olrdml = NULL; + goto end; + } } /* populate cached information */ diff --git a/testenv/postgres/synchdb-postgres-test-internal.yaml b/testenv/postgres/synchdb-postgres-test-internal.yaml new file mode 100644 index 0000000..90b8c70 --- /dev/null +++ b/testenv/postgres/synchdb-postgres-test-internal.yaml @@ -0,0 +1,30 @@ +services: + db: + image: postgres:18.1 + container_name: postgres + shm_size: 256mb + ports: + - "5432:5432" + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: pass + POSTGRES_DB: postgres + command: + - postgres + - "-c" + - "wal_level=logical" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 5s + timeout: 3s + retries: 20 + restart: unless-stopped + networks: + - synchdbnet + +networks: + synchdbnet: + name: synchdbnet + driver: bridge + internal: true + diff --git a/testenv/postgres/synchdb-postgres-test.yaml b/testenv/postgres/synchdb-postgres-test.yaml new file mode 100644 index 0000000..c24e9d8 --- /dev/null +++ b/testenv/postgres/synchdb-postgres-test.yaml @@ -0,0 +1,22 @@ +services: + db: + image: postgres:18.1 + container_name: postgres + shm_size: 256mb + ports: + - "5432:5432" + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: pass + POSTGRES_DB: postgres + command: + - postgres + - "-c" + - "wal_level=logical" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 5s + timeout: 3s + retries: 20 + restart: unless-stopped + diff --git a/testenv/synchdb/Dockerfile b/testenv/synchdb/Dockerfile index 2661de6..d2283a0 100644 --- a/testenv/synchdb/Dockerfile +++ b/testenv/synchdb/Dockerfile @@ -5,15 +5,17 @@ FROM ubuntu:24.04 # We don't want a prompt during installation ENV DEBIAN_FRONTEND=noninteractive -ENV POSTGRES_VERSION=17 +ENV POSTGRES_VERSION=18 ENV JAVA_VERSION=17 ENV SYNCHDB_BRANCH=synchdb-devel ENV LIBPROTOBUF_C_VERSION=v1.5.2 -ENV POSTGRES_BRANCH=REL_17_6 +ENV POSTGRES_BRANCH=REL_18_1 +ENV ORACLE_FDW_VERSION=ORACLE_FDW_2_8_0 +ENV MYSQL_FDW_VERSION=REL-2_9_3 # Install necessary RUN apt-get update -RUN apt-get install -y vim wget gawk net-tools expect apt-utils openssh-client +RUN apt-get install -y vim wget gawk net-tools expect apt-utils openssh-client unzip RUN apt-get install -y zlib1g-dev libxml2-utils xsltproc ccache pkg-config RUN apt-get install -y build-essential git lcov bison flex RUN apt-get install -y openjdk-${JAVA_VERSION}-jdk maven @@ -26,6 +28,10 @@ RUN apt-get install -y cppcheck RUN apt-get install -y chrpath sudo RUN apt-get install -y autoconf automake libtool RUN apt-get install -y protobuf-compiler libprotobuf-dev libprotoc-dev +RUN apt-get install -y mysql-client-core-8.0 default-libmysqlclient-dev +RUN apt-get install -y libaio1t64 + +RUN ln -s /usr/lib/x86_64-linux-gnu/libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1 #RUN apt install -y postgresql-common #RUN echo -ne "\n" | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh @@ -63,6 +69,19 @@ RUN chown -R $USERNAME:$GROUPNAME $USER_HOME/.ssh RUN chmod 700 $USER_HOME/.ssh RUN chmod 600 $USER_HOME/.ssh/* +# OCI +WORKDIR $USER_HOME +RUN wget https://download.oracle.com/otn_software/linux/instantclient/2390000/instantclient-basic-linux.x64-23.9.0.25.07.zip +RUN wget https://download.oracle.com/otn_software/linux/instantclient/2390000/instantclient-sdk-linux.x64-23.9.0.25.07.zip +RUN mkdir -p /opt/oracle +RUN unzip -oq instantclient-basic-linux.x64-23.9.0.25.07.zip -d /opt/oracle +RUN unzip -oq instantclient-sdk-linux.x64-23.9.0.25.07.zip -d /opt/oracle +RUN ls -l /opt/oracle/instantclient_23_9 +RUN mkdir -p /usr/lib/oracle/23.9/client64 +RUN mkdir -p /usr/include/oracle/23.9 +RUN ln -sfn /opt/oracle/instantclient_23_9/sdk/include /usr/include/oracle/23.9/client64 +RUN ln -sfn /opt/oracle/instantclient_23_9 /usr/lib/oracle/23.9/client64/lib + # dependency for Openlog Replicator Connector WORKDIR $USER_HOME RUN git clone https://github.com/protobuf-c/protobuf-c.git --branch ${LIBPROTOBUF_C_VERSION} @@ -97,17 +116,27 @@ RUN cd postgres/contrib && \ RUN cd $USER_HOME/postgres/contrib && \ git clone https://github.com/Hornetlabs/synchdb.git --branch ${SYNCHDB_BRANCH} +# oracle_fdw +WORKDIR $USER_HOME/postgres/contrib +RUN git clone https://github.com/laurenz/oracle_fdw.git --branch ${ORACLE_FDW_VERSION} +RUN cd oracle_fdw && \ + make PG_CONFIG=/usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_config && \ + make install PG_CONFIG=/usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_config + +# mysql_fdw +WORKDIR $USER_HOME/postgres/contrib +RUN git clone https://github.com/EnterpriseDB/mysql_fdw.git --branch ${MYSQL_FDW_VERSION} +RUN cd mysql_fdw && \ + make PG_CONFIG=/usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_config && \ + make install PG_CONFIG=/usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_config + +# synchdb WORKDIR $USER_HOME/postgres/contrib/synchdb RUN make oracle_parser && make install_oracle_parser RUN make WITH_OLR=1 build_dbz RUN make WITH_OLR=1 RUN make WITH_OLR=1 install RUN make WITH_OLR=1 install_dbz -#RUN make oracle_parser && make install_oracle_parser -#RUN USE_PGXS=1 make WITH_OLR=1 build_dbz PG_CONFIG=/usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_config -#RUN USE_PGXS=1 make WITH_OLR=1 PG_CONFIG=/usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_config -#RUN USE_PGXS=1 make WITH_OLR=1 install PG_CONFIG=/usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_config -#RUN USE_PGXS=1 make WITH_OLR=1 install_dbz PG_CONFIG=/usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_config ENV PATH="/usr/lib/postgresql/${POSTGRES_VERSION}/bin:${PATH}" COPY init-synchdb.sh /usr/local/bin @@ -119,8 +148,9 @@ RUN mkdir -p /var/run/postgresql \ && chown $USERNAME:$USERNAME /var/run/postgresql \ && chmod 755 /var/run/postgresql -RUN echo "/usr/lib/jvm/java-${JAVA_VERSION}-openjdk-amd64/lib/server" >> /etc/ld.so.conf.d/x86_64-linux-gnu.conf \ - && ldconfig +RUN echo "/usr/lib/jvm/java-${JAVA_VERSION}-openjdk-amd64/lib/server" >> /etc/ld.so.conf.d/x86_64-linux-gnu.conf && \ + echo "/opt/oracle/instantclient_23_9" >> /etc/ld.so.conf.d/oracle-instantclient.conf && \ + ldconfig USER $USERNAME WORKDIR $USER_HOME diff --git a/testenv/synchdb/synchdb-test.yaml b/testenv/synchdb/synchdb-test.yaml index 4363c45..e7713b5 100644 --- a/testenv/synchdb/synchdb-test.yaml +++ b/testenv/synchdb/synchdb-test.yaml @@ -6,7 +6,7 @@ services: synchdb: container_name: synchdb - image: hgneon/synchdbtest:pg17 + image: hgneon/synchdbtest:pg18 networks: - synchdbnet