From e2c868fb401834b59524273519cd63889c9831ea Mon Sep 17 00:00:00 2001 From: mauro Date: Fri, 6 Oct 2017 11:39:57 +0200 Subject: [PATCH 1/2] Fixed `project_id` missing value issue. Now it is handled in the same way of Oracle procedures. --- .../postgresql/CREATE_TEMP_EID_TABLE.sql | 3 +- .../postgresql/CREATE_TEMP_VISIT_TABLE.sql | 1 - .../postgresql/INSERT_EID_MAP_FROMTEMP.sql | 2 +- .../INSERT_ENCOUNTERVISIT_FROMTEMP.sql | 3 +- .../postgresql/INSERT_PID_MAP_FROMTEMP.sql | 66 +++++++++---------- 5 files changed, 36 insertions(+), 39 deletions(-) diff --git a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/CREATE_TEMP_EID_TABLE.sql b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/CREATE_TEMP_EID_TABLE.sql index 48451184..c21f4cdf 100644 --- a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/CREATE_TEMP_EID_TABLE.sql +++ b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/CREATE_TEMP_EID_TABLE.sql @@ -4,8 +4,7 @@ BEGIN EXECUTE 'create table ' || tempPatientMappingTableName || ' ( ENCOUNTER_MAP_ID varchar(200) NOT NULL, ENCOUNTER_MAP_ID_SOURCE varchar(50) NOT NULL, - PROJECT_ID VARCHAR(50) NOT NULL, - PATIENT_MAP_ID varchar(200), + PATIENT_MAP_ID varchar(200), PATIENT_MAP_ID_SOURCE varchar(50), ENCOUNTER_ID varchar(200) NOT NULL, ENCOUNTER_ID_SOURCE varchar(50) , diff --git a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/CREATE_TEMP_VISIT_TABLE.sql b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/CREATE_TEMP_VISIT_TABLE.sql index 83b56dd1..00afbb90 100644 --- a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/CREATE_TEMP_VISIT_TABLE.sql +++ b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/CREATE_TEMP_VISIT_TABLE.sql @@ -5,7 +5,6 @@ BEGIN EXECUTE 'create table ' || tempTableName || ' ( encounter_id varchar(200) not null, encounter_id_source varchar(50) not null, - project_id varchar(50) not null, patient_id varchar(200) not null, patient_id_source varchar(50) not null, encounter_num numeric(38,0), diff --git a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/INSERT_EID_MAP_FROMTEMP.sql b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/INSERT_EID_MAP_FROMTEMP.sql index d2a8b25d..72b45bff 100644 --- a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/INSERT_EID_MAP_FROMTEMP.sql +++ b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/INSERT_EID_MAP_FROMTEMP.sql @@ -98,7 +98,7 @@ and coalesce(encounter_mapping.update_date,to_date(''01-JAN-1900'',''DD-MON-YYYY -- insert new mapping records i.e flagged P EXECUTE ' insert into encounter_mapping (encounter_ide,encounter_ide_source,encounter_ide_status,encounter_num,patient_ide,patient_ide_source,update_date,download_date,import_date,sourcesystem_cd,upload_id,project_id) - SELECT encounter_map_id,encounter_map_id_source,encounter_map_id_status,encounter_num,patient_map_id,patient_map_id_source,update_date,download_date,Now(),sourcesystem_cd,' || upload_id || ' , project_id + SELECT encounter_map_id,encounter_map_id_source,encounter_map_id_status,encounter_num,patient_map_id,patient_map_id_source,update_date,download_date,Now(),sourcesystem_cd,' || upload_id || ' , ''@'' project_id FROM ' || tempEidTableName || ' WHERE process_status_flag = ''P'' ' ; EXCEPTION diff --git a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/INSERT_ENCOUNTERVISIT_FROMTEMP.sql b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/INSERT_ENCOUNTERVISIT_FROMTEMP.sql index 233861e9..b5f1a59a 100644 --- a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/INSERT_ENCOUNTERVISIT_FROMTEMP.sql +++ b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/INSERT_ENCOUNTERVISIT_FROMTEMP.sql @@ -35,14 +35,13 @@ BEGIN , distinctTemp.patient_id_source , ''A'' , '|| upload_id ||' - , distinctTemp.project_id + , ''@'' project_id FROM (SELECT distinct encounter_id , encounter_id_source , patient_id , patient_id_source - , project_id FROM ' || tempTableName || ' temp WHERE NOT EXISTS (SELECT encounter_ide diff --git a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/INSERT_PID_MAP_FROMTEMP.sql b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/INSERT_PID_MAP_FROMTEMP.sql index 0cfd4fdb..b03324c6 100644 --- a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/INSERT_PID_MAP_FROMTEMP.sql +++ b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata/scripts/procedures/postgresql/INSERT_PID_MAP_FROMTEMP.sql @@ -1,67 +1,67 @@ -CREATE OR REPLACE FUNCTION insert_pid_map_fromtemp (tempPidTableName IN text, upload_id IN bigint, - errorMsg OUT text) +CREATE OR REPLACE FUNCTION insert_pid_map_fromtemp (tempPidTableName IN text, upload_id IN bigint, + errorMsg OUT text) RETURNS text AS $body$ DECLARE existingPatientNum varchar(32); maxPatientNum bigint; distinctPidCur REFCURSOR; -disPatientId varchar(100); +disPatientId varchar(100); disPatientIdSource varchar(100); BEGIN --delete the doublons - EXECUTE ' delete from ' || tempPidTableName || ' t1 where - oid > (select min(oid) from ' || tempPidTableName || ' t2 + EXECUTE ' delete from ' || tempPidTableName || ' t1 where + oid > (select min(oid) from ' || tempPidTableName || ' t2 where t1.patient_map_id = t2.patient_map_id and t1.patient_map_id_source = t2.patient_map_id_source) '; LOCK TABLE patient_mapping IN EXCLUSIVE MODE NOWAIT; - select max(patient_num) into STRICT maxPatientNum from patient_mapping ; + select max(patient_num) into STRICT maxPatientNum from patient_mapping ; -- set max patient num to zero of the value is null - if coalesce(maxPatientNum::text, '') = '' then + if coalesce(maxPatientNum::text, '') = '' then maxPatientNum := 0; end if; open distinctPidCur for EXECUTE 'SELECT distinct patient_id,patient_id_source from ' || tempPidTableName || '' ; loop FETCH distinctPidCur INTO disPatientId, disPatientIdSource; - IF NOT FOUND THEN EXIT; + IF NOT FOUND THEN EXIT; END IF; -- apply on distinctPidCur -- dbms_output.put_line(disPatientId); - if disPatientIdSource = 'HIVE' THEN + if disPatientIdSource = 'HIVE' THEN begin --check if hive number exist, if so assign that number to reset of map_id's within that pid select patient_num into existingPatientNum from patient_mapping where patient_num = CAST(disPatientId AS numeric) and patient_ide_source = 'HIVE'; EXCEPTION when NO_DATA_FOUND THEN existingPatientNum := null; end; - if (existingPatientNum IS NOT NULL AND existingPatientNum::text <> '') then + if (existingPatientNum IS NOT NULL AND existingPatientNum::text <> '') then EXECUTE ' update ' || tempPidTableName ||' set patient_num = CAST(patient_id AS numeric), process_status_flag = ''P'' where patient_id = $1 and not exists (select 1 from patient_mapping pm where pm.patient_ide = patient_map_id and pm.patient_ide_source = patient_map_id_source)' using disPatientId; - else - -- generate new patient_num i.e. take max(patient_num) + 1 - if maxPatientNum < CAST(disPatientId AS numeric) then + else + -- generate new patient_num i.e. take max(patient_num) + 1 + if maxPatientNum < CAST(disPatientId AS numeric) then maxPatientNum := disPatientId; end if ; - EXECUTE ' update ' || tempPidTableName ||' set patient_num = CAST(patient_id AS numeric), process_status_flag = ''P'' where + EXECUTE ' update ' || tempPidTableName ||' set patient_num = CAST(patient_id AS numeric), process_status_flag = ''P'' where patient_id = $1 and patient_id_source = ''HIVE'' and not exists (select 1 from patient_mapping pm where pm.patient_ide = patient_map_id and pm.patient_ide_source = patient_map_id_source)' using disPatientId; - end if; + end if; -- test if record fectched -- dbms_output.put_line(' HIVE '); - else + else begin - select patient_num into STRICT existingPatientNum from patient_mapping where patient_ide = disPatientId and - patient_ide_source = disPatientIdSource ; - -- test if record fetched. + select patient_num into STRICT existingPatientNum from patient_mapping where patient_ide = disPatientId and + patient_ide_source = disPatientIdSource ; + -- test if record fetched. EXCEPTION WHEN NO_DATA_FOUND THEN existingPatientNum := null; end; - if (existingPatientNum IS NOT NULL AND existingPatientNum::text <> '') then + if (existingPatientNum IS NOT NULL AND existingPatientNum::text <> '') then EXECUTE ' update ' || tempPidTableName ||' set patient_num = CAST($1 AS numeric) , process_status_flag = ''P'' where patient_id = $2 and not exists (select 1 from patient_mapping pm where pm.patient_ide = patient_map_id and pm.patient_ide_source = patient_map_id_source)' using existingPatientNum,disPatientId; - else - maxPatientNum := maxPatientNum + 1 ; + else + maxPatientNum := maxPatientNum + 1 ; EXECUTE 'insert into ' || tempPidTableName ||' ( patient_map_id ,patient_map_id_source @@ -74,7 +74,7 @@ BEGIN ,download_date ,import_date ,sourcesystem_cd - ,project_id) + ,project_id) values( $1 ,''HIVE'' @@ -88,19 +88,19 @@ BEGIN ,Now() ,''edu.harvard.i2b2.crc'' ,''HIVE'' - )' using maxPatientNum,maxPatientNum,maxPatientNum; - EXECUTE 'update ' || tempPidTableName ||' set patient_num = $1 , process_status_flag = ''P'' - where patient_id = $2 and not exists (select 1 from + )' using maxPatientNum,maxPatientNum,maxPatientNum; + EXECUTE 'update ' || tempPidTableName ||' set patient_num = $1 , process_status_flag = ''P'' + where patient_id = $2 and not exists (select 1 from patient_mapping pm where pm.patient_ide = patient_map_id and pm.patient_ide_source = patient_map_id_source)' using maxPatientNum, disPatientId ; end if ; -- dbms_output.put_line(' NOT HIVE '); - end if; + end if; END LOOP; close distinctPidCur ; -- do the mapping update if the update date is old EXECUTE ' UPDATE patient_mapping -SET +SET patient_num = CAST(temp.patient_id AS numeric) ,patient_ide_status = temp.patient_map_id_status ,update_date = temp.update_date @@ -109,17 +109,17 @@ patient_num = CAST(temp.patient_id AS numeric) ,sourcesystem_cd = temp.sourcesystem_cd ,upload_id = ' || upload_id ||' FROM '|| tempPidTableName || ' temp -WHERE -temp.patient_map_id = patient_mapping.patient_ide +WHERE +temp.patient_map_id = patient_mapping.patient_ide and temp.patient_map_id_source = patient_mapping.patient_ide_source and temp.patient_id_source = ''HIVE'' -and coalesce(temp.process_status_flag::text, '''') = '''' +and coalesce(temp.process_status_flag::text, '''') = '''' and coalesce(patient_mapping.update_date,to_date(''01-JAN-1900'',''DD-MON-YYYY'')) <= coalesce(temp.update_date,to_date(''01-JAN-1900'',''DD-MON-YYYY'')) '; -- insert new mapping records i.e flagged P EXECUTE ' insert into patient_mapping (patient_ide,patient_ide_source,patient_ide_status,patient_num,update_date,download_date,import_date,sourcesystem_cd,upload_id,project_id) - SELECT patient_map_id,patient_map_id_source,patient_map_id_status,patient_num,update_date,download_date,Now(),sourcesystem_cd,' || upload_id ||', project_id from '|| tempPidTableName || ' - where process_status_flag = ''P'' ' ; + SELECT patient_map_id,patient_map_id_source,patient_map_id_status,patient_num,update_date,download_date,Now(),sourcesystem_cd,' || upload_id ||', ''@'' project_id from '|| tempPidTableName || ' + where process_status_flag = ''P'' ' ; EXCEPTION WHEN OTHERS THEN RAISE EXCEPTION 'An error was encountered - % -ERROR- %',SQLSTATE,SQLERRM; END; From d9622829e451f4b3a467017800d01ed44bb77637 Mon Sep 17 00:00:00 2001 From: mauro Date: Fri, 6 Oct 2017 12:20:41 +0200 Subject: [PATCH 2/2] Fixed: "ERROR: relation "public.schemes" does not exist", " ERROR: relation "public.table_access" does not exist", " ERROR: relation "public.workplace_access" does not exist". Now PostgreSQL scripts have the same behaviour of Oracle ones. --- .../Hivedata/scripts/crc_db_lookup_postgresql_insert_data.sql | 2 +- .../Hivedata/scripts/im_db_lookup_postgresql_insert_data.sql | 2 +- .../Hivedata/scripts/ont_db_lookup_postgresql_insert_data.sql | 2 +- .../Hivedata/scripts/work_db_lookup_postgresql_insert_data.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Hivedata/scripts/crc_db_lookup_postgresql_insert_data.sql b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Hivedata/scripts/crc_db_lookup_postgresql_insert_data.sql index 55426bf8..1d5145d1 100644 --- a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Hivedata/scripts/crc_db_lookup_postgresql_insert_data.sql +++ b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Hivedata/scripts/crc_db_lookup_postgresql_insert_data.sql @@ -1,4 +1,4 @@ INSERT INTO CRC_DB_LOOKUP(c_domain_id, c_project_path, c_owner_id, c_db_fullschema, c_db_datasource, c_db_servertype, c_db_nicename, c_db_tooltip, c_comment, c_entry_date, c_change_date, c_status_cd) - VALUES('i2b2demo', '/Demo/', '@', 'public', 'java:/QueryToolDemoDS', 'POSTGRESQL', 'Demo', NULL, NULL, NULL, NULL, NULL); + VALUES('i2b2demo', '/Demo/', '@', 'i2b2demodata', 'java:/QueryToolDemoDS', 'POSTGRESQL', 'Demo', NULL, NULL, NULL, NULL, NULL); diff --git a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Hivedata/scripts/im_db_lookup_postgresql_insert_data.sql b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Hivedata/scripts/im_db_lookup_postgresql_insert_data.sql index f742ea58..a21ee041 100644 --- a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Hivedata/scripts/im_db_lookup_postgresql_insert_data.sql +++ b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Hivedata/scripts/im_db_lookup_postgresql_insert_data.sql @@ -1,2 +1,2 @@ INSERT INTO IM_DB_LOOKUP(c_domain_id, c_project_path, c_owner_id, c_db_fullschema, c_db_datasource, c_db_servertype, c_db_nicename, c_db_tooltip, c_comment, c_entry_date, c_change_date, c_status_cd) - VALUES('i2b2demo', 'Demo/', '@', 'public', 'java:/IMDemoDS', 'POSTGRESQL', 'IM', NULL, NULL, NULL, NULL, NULL); + VALUES('i2b2demo', 'Demo/', '@', 'i2b2imdata', 'java:/IMDemoDS', 'POSTGRESQL', 'IM', NULL, NULL, NULL, NULL, NULL); diff --git a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Hivedata/scripts/ont_db_lookup_postgresql_insert_data.sql b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Hivedata/scripts/ont_db_lookup_postgresql_insert_data.sql index 4e3ec980..b29bdf97 100644 --- a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Hivedata/scripts/ont_db_lookup_postgresql_insert_data.sql +++ b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Hivedata/scripts/ont_db_lookup_postgresql_insert_data.sql @@ -1,2 +1,2 @@ INSERT INTO ONT_DB_LOOKUP(c_domain_id, c_project_path, c_owner_id, c_db_fullschema, c_db_datasource, c_db_servertype, c_db_nicename, c_db_tooltip, c_comment, c_entry_date, c_change_date, c_status_cd) - VALUES('i2b2demo', 'Demo/', '@', 'public', 'java:/OntologyDemoDS', 'POSTGRESQL', 'Metadata', NULL, NULL, NULL, NULL, NULL); + VALUES('i2b2demo', 'Demo/', '@', 'i2b2metadata', 'java:/OntologyDemoDS', 'POSTGRESQL', 'Metadata', NULL, NULL, NULL, NULL, NULL); diff --git a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Hivedata/scripts/work_db_lookup_postgresql_insert_data.sql b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Hivedata/scripts/work_db_lookup_postgresql_insert_data.sql index c0aded62..9ddea05a 100644 --- a/edu.harvard.i2b2.data/Release_1-7/NewInstall/Hivedata/scripts/work_db_lookup_postgresql_insert_data.sql +++ b/edu.harvard.i2b2.data/Release_1-7/NewInstall/Hivedata/scripts/work_db_lookup_postgresql_insert_data.sql @@ -1,2 +1,2 @@ INSERT INTO WORK_DB_LOOKUP(c_domain_id, c_project_path, c_owner_id, c_db_fullschema, c_db_datasource, c_db_servertype, c_db_nicename, c_db_tooltip, c_comment, c_entry_date, c_change_date, c_status_cd) - VALUES('i2b2demo', 'Demo/', '@', 'public', 'java:/WorkplaceDemoDS', 'POSTGRESQL', 'Workplace', NULL, NULL, NULL, NULL, NULL); + VALUES('i2b2demo', 'Demo/', '@', 'i2b2workdata', 'java:/WorkplaceDemoDS', 'POSTGRESQL', 'Workplace', NULL, NULL, NULL, NULL, NULL);